RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(preview): handle non-UTF-8 encodings in CSV/TXT preview (#19222)
chanx020e079780fe
diff --git a/web/src/components/document-preview/csv-preview.tsx b/web/src/components/document-preview/csv-preview.tsxindex 66e27b1b17..db7a842253 100644--- a/web/src/components/document-preview/csv-preview.tsx+++ b/web/src/components/document-preview/csv-preview.tsx@@ -17,6 +17,7 @@import message from '@/components/ui/message';import { Spin } from '@/components/ui/spin';import request from '@/utils/request';+import { decodeBlobText } from '@/utils/file-util';import classNames from 'classnames';import Papa from 'papaparse';import React, { useEffect, useRef, useState } from 'react';@@ -62,14 +63,9 @@ const CSVFileViewer: React.FC<FileViewerProps> = ({ url }) => {},});- // parse CSV file- const reader = new FileReader();- reader.readAsText(res.data);- reader.onload = () => {- const parsedData = parseCSV(reader.result as string);- console.log('file loaded successfully', reader.result);- setData(parsedData);- };+ // Handles UTF-8/UTF-16 (BOM) as well as GB2312/GBK files+ const csvText = await decodeBlobText(res.data);+ setData(parseCSV(csvText));} catch (error) {message.error('CSV file parse failed');console.error('Error loading CSV file:', error);diff --git a/web/src/components/document-preview/txt-preview.tsx b/web/src/components/document-preview/txt-preview.tsxindex 8a509799f1..ae2bad1669 100644--- a/web/src/components/document-preview/txt-preview.tsx+++ b/web/src/components/document-preview/txt-preview.tsx@@ -17,6 +17,7 @@import message from '@/components/ui/message';import { Spin } from '@/components/ui/spin';import request from '@/utils/request';+import { decodeBlobText } from '@/utils/file-util';import classNames from 'classnames';import { useEffect, useState } from 'react';@@ -35,15 +36,10 @@ export const TxtPreviewer = ({ className, url }: TxtPreviewerProps) => {console.error('Error loading file:', err);},});- // blob to string- const reader = new FileReader();- reader.readAsText(res.data);- reader.onload = () => {- setData(reader.result as string);- setLoading(false);- console.log('file loaded successfully', reader.result);- };- console.log('file data:', res);+ // Handles UTF-8/UTF-16 (BOM) as well as GB2312/GBK files+ const text = await decodeBlobText(res.data);+ setData(text);+ setLoading(false);};useEffect(() => {if (url) {diff --git a/web/src/utils/file-util.ts b/web/src/utils/file-util.tsindex 7f8dbe9fbe..8f86c90ee7 100644--- a/web/src/utils/file-util.ts+++ b/web/src/utils/file-util.ts@@ -84,6 +84,31 @@ export const transformBase64ToFile = (return new File([u8arr], filename, { type: mimeType });};+// Decode a text file blob without assuming UTF-8. Detection order:+// 1. Unicode BOM (UTF-8 / UTF-16LE / UTF-16BE) — authoritative when present.+// 2. Strict UTF-8 decode (`fatal: true`) — throws on any byte sequence that+// is not valid UTF-8, which GB2312/GBK Chinese text almost always is.+// 3. GBK fallback — covers GB2312/GBK/GB18030 single- and double-byte text.+// Browsers map the 'gb2312' label to the same decoder as 'gbk'.+export const decodeBlobText = async (blob: Blob): Promise<string> => {+ const buffer = await blob.arrayBuffer();+ const bytes = new Uint8Array(buffer, 0, Math.min(3, buffer.byteLength));+ if (bytes[0] === 0xff && bytes[1] === 0xfe) {+ return new TextDecoder('utf-16le').decode(buffer);+ }+ if (bytes[0] === 0xfe && bytes[1] === 0xff) {+ return new TextDecoder('utf-16be').decode(buffer);+ }+ if (bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {+ return new TextDecoder('utf-8').decode(buffer);+ }+ try {+ return new TextDecoder('utf-8', { fatal: true }).decode(buffer);+ } catch {+ return new TextDecoder('gbk').decode(buffer);+ }+};+export const normFile = (e: any) => {if (Array.isArray(e)) {return e;