Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(preview): handle non-UTF-8 encodings in CSV/TXT preview (#19222)

chanx020e079780fe
diff --git a/web/src/components/document-preview/csv-preview.tsx b/web/src/components/document-preview/csv-preview.tsx
index 66e27b1b17..db7a842253 100644
--- a/web/src/components/document-preview/csv-preview.tsx
+++ b/web/src/components/document-preview/csv-preview.tsx
@@ -17,6 +17,7 @@
import message from '@/components/ui/message';
import { Spin } from '@/components/ui/spin';
import request from '@/utils/request';
+import { decodeBlobText } from '@/utils/file-util';
import classNames from 'classnames';
import Papa from 'papaparse';
import React, { useEffect, useRef, useState } from 'react';
@@ -62,14 +63,9 @@ const CSVFileViewer: React.FC<FileViewerProps> = ({ url }) => {
},
});
- // parse CSV file
- const reader = new FileReader();
- reader.readAsText(res.data);
- reader.onload = () => {
- const parsedData = parseCSV(reader.result as string);
- console.log('file loaded successfully', reader.result);
- setData(parsedData);
- };
+ // Handles UTF-8/UTF-16 (BOM) as well as GB2312/GBK files
+ const csvText = await decodeBlobText(res.data);
+ setData(parseCSV(csvText));
} catch (error) {
message.error('CSV file parse failed');
console.error('Error loading CSV file:', error);
diff --git a/web/src/components/document-preview/txt-preview.tsx b/web/src/components/document-preview/txt-preview.tsx
index 8a509799f1..ae2bad1669 100644
--- a/web/src/components/document-preview/txt-preview.tsx
+++ b/web/src/components/document-preview/txt-preview.tsx
@@ -17,6 +17,7 @@
import message from '@/components/ui/message';
import { Spin } from '@/components/ui/spin';
import request from '@/utils/request';
+import { decodeBlobText } from '@/utils/file-util';
import classNames from 'classnames';
import { useEffect, useState } from 'react';
@@ -35,15 +36,10 @@ export const TxtPreviewer = ({ className, url }: TxtPreviewerProps) => {
console.error('Error loading file:', err);
},
});
- // blob to string
- const reader = new FileReader();
- reader.readAsText(res.data);
- reader.onload = () => {
- setData(reader.result as string);
- setLoading(false);
- console.log('file loaded successfully', reader.result);
- };
- console.log('file data:', res);
+ // Handles UTF-8/UTF-16 (BOM) as well as GB2312/GBK files
+ const text = await decodeBlobText(res.data);
+ setData(text);
+ setLoading(false);
};
useEffect(() => {
if (url) {
diff --git a/web/src/utils/file-util.ts b/web/src/utils/file-util.ts
index 7f8dbe9fbe..8f86c90ee7 100644
--- a/web/src/utils/file-util.ts
+++ b/web/src/utils/file-util.ts
@@ -84,6 +84,31 @@ export const transformBase64ToFile = (
return new File([u8arr], filename, { type: mimeType });
};
+// Decode a text file blob without assuming UTF-8. Detection order:
+// 1. Unicode BOM (UTF-8 / UTF-16LE / UTF-16BE) — authoritative when present.
+// 2. Strict UTF-8 decode (`fatal: true`) — throws on any byte sequence that
+// is not valid UTF-8, which GB2312/GBK Chinese text almost always is.
+// 3. GBK fallback — covers GB2312/GBK/GB18030 single- and double-byte text.
+// Browsers map the 'gb2312' label to the same decoder as 'gbk'.
+export const decodeBlobText = async (blob: Blob): Promise<string> => {
+ const buffer = await blob.arrayBuffer();
+ const bytes = new Uint8Array(buffer, 0, Math.min(3, buffer.byteLength));
+ if (bytes[0] === 0xff && bytes[1] === 0xfe) {
+ return new TextDecoder('utf-16le').decode(buffer);
+ }
+ if (bytes[0] === 0xfe && bytes[1] === 0xff) {
+ return new TextDecoder('utf-16be').decode(buffer);
+ }
+ if (bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
+ return new TextDecoder('utf-8').decode(buffer);
+ }
+ try {
+ return new TextDecoder('utf-8', { fatal: true }).decode(buffer);
+ } catch {
+ return new TextDecoder('gbk').decode(buffer);
+ }
+};
+
export const normFile = (e: any) => {
if (Array.isArray(e)) {
return e;