Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(preview): reject invalid UTF-8 before decoding CSV (#4479)

XY8678c468b697
diff --git a/frontend/src/components/document-preview.encoding.test.ts b/frontend/src/components/document-preview.encoding.test.ts
new file mode 100644
index 000000000..de688e50b
--- /dev/null
+++ b/frontend/src/components/document-preview.encoding.test.ts
@@ -0,0 +1,88 @@
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import { createRequire } from 'node:module'
+import { fileURLToPath } from 'node:url'
+import { runInNewContext } from 'node:vm'
+import test from 'node:test'
+import { compileScript, parse } from '@vue/compiler-sfc'
+import { DOMParser } from '@xmldom/xmldom'
+import ts from 'typescript'
+import { createRenderer, h, ref } from 'vue'
+import * as filePreview from '../utils/filePreview'
+
+const require = createRequire(import.meta.url)
+const filename = fileURLToPath(new URL('./document-preview.vue', import.meta.url))
+const { descriptor } = parse(readFileSync(filename, 'utf8'), { filename })
+const script = compileScript(descriptor, { id: 'preview-encoding-test' }).content
+ .replace('__expose();', '')
+ .replace('return __returned__', '__expose(__returned__); return __returned__')
+const compiled = ts.transpileModule(script, {
+ compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 },
+}).outputText
+
+function fixture(bytes: Uint8Array, fileType: string) {
+ const exports: any = {}
+ runInNewContext(compiled, {
+ exports,
+ require(name: string) {
+ if (name === 'vue' || name === 'xlsx') return require(name)
+ if (name === '@/utils/filePreview') return filePreview
+ if (name === 'vue-i18n') return { useI18n: () => ({ t: (key: string) => key }) }
+ // Encoding and SheetJS rendering stay real. DOM sanitization and the
+ // unused preview formats/network adapters are outside this test's scope.
+ if (name === '@/utils/security') return { sanitizeHTML: (html: string) => html }
+ return {}
+ },
+ Blob, TextDecoder, URL, console,
+ document: { addEventListener() {}, removeEventListener() {}, activeElement: {} },
+ window: { addEventListener() {}, removeEventListener() {} },
+ })
+ const component = exports.default
+ component.render = () => null
+ const renderer = createRenderer<any, any>({
+ createElement: () => ({}), createText: () => ({}), createComment: () => ({}),
+ insert() {}, remove() {}, setElementText() {}, setText() {}, patchProp() {},
+ parentNode: () => null, nextSibling: () => null,
+ })
+ const instance = ref<any>()
+ const app = renderer.createApp({
+ render: () => h(component, {
+ ref: instance,
+ sourceBlob: new Blob([Uint8Array.from(bytes)]),
+ fileName: `encoding.${fileType}`, fileType,
+ // Invoke loadPreview explicitly so the test can await the whole pipeline.
+ active: false,
+ }),
+ })
+ app.mount({})
+ return { vm: instance.value, close: () => app.unmount() }
+}
+
+for (const fileType of ['csv', 'tsv', 'tab']) {
+ for (const encoding of ['gbk', 'utf8', 'utf8-bom']) {
+ test(`${fileType} preview preserves ${encoding} Chinese text and table columns`, async () => {
+ const separator = fileType === 'csv' ? ',' : '\t'
+ const text = `item${separator}count\r\n蜡${separator}1\r\n`
+ // C0 AF encodes 蜡 in GBK, but is an invalid overlong UTF-8 sequence.
+ const bytes = encoding === 'gbk'
+ ? Uint8Array.from([
+ ...new TextEncoder().encode(`item${separator}count\r\n`),
+ 0xC0, 0xAF,
+ ...new TextEncoder().encode(`${separator}1\r\n`),
+ ])
+ : new TextEncoder().encode((encoding === 'utf8-bom' ? '\uFEFF' : '') + text)
+ const f = fixture(bytes, fileType)
+ try {
+ await f.vm.loadPreview()
+ assert.equal(f.vm.error, '')
+ assert.equal(f.vm.loading, false)
+ assert.equal(f.vm.previewType, 'excel')
+ const doc = new DOMParser().parseFromString(f.vm.excelHtml, 'text/html')
+ const cells = Array.from(doc.getElementsByTagName('td'), cell => cell.textContent)
+ assert.deepEqual(cells, ['item', 'count', '蜡', '1'])
+ } finally {
+ f.close()
+ }
+ })
+ }
+}
diff --git a/frontend/src/utils/filePreview.test.ts b/frontend/src/utils/filePreview.test.ts
index 9d80b5773..0e684d7ec 100644
--- a/frontend/src/utils/filePreview.test.ts
+++ b/frontend/src/utils/filePreview.test.ts
@@ -5,6 +5,7 @@ import {
getHighlightLang,
getPreviewMimeType,
isKnownPreviewableExt,
+ isValidUTF8,
prettyPrintJson,
resolveFilePreviewExt,
resolvePreviewKind,
@@ -64,6 +65,39 @@ test('sniffPreview rejects NUL-containing binaries', () => {
assert.equal(sniffPreview(Uint8Array.from([0x00, 0x01, 0x02, 0x03])).kind, 'unsupported')
})
+test('isValidUTF8 rejects invalid scalar encodings so GBK CSV can fall back', () => {
+ // GBK for "蜡,1": accepting C0 AF as UTF-8 produces replacement characters.
+ const csv = Uint8Array.from([0xC0, 0xAF, 0x2C, 0x31])
+ assert.equal(isValidUTF8(csv), false)
+ assert.equal(new TextDecoder('gbk').decode(csv), '蜡,1')
+ for (const bytes of [
+ [0xC1, 0xBF], [0xE0, 0x80, 0xAF], [0xED, 0xA0, 0x80],
+ [0xF0, 0x80, 0x80, 0xAF], [0xF4, 0x90, 0x80, 0x80],
+ [0xF5, 0x80, 0x80, 0x80], [0x80],
+ ]) {
+ assert.equal(isValidUTF8(Uint8Array.from(bytes)), false, String(bytes))
+ assert.equal(sniffPreview(Uint8Array.from(bytes)).kind, 'unsupported', String(bytes))
+ }
+})
+
+test('isValidUTF8 only tolerates valid incomplete prefixes when requested', () => {
+ for (const bytes of [[0xC2], [0xE4, 0xB8], [0xF0, 0x9F, 0x98]]) {
+ assert.equal(isValidUTF8(Uint8Array.from(bytes)), false)
+ assert.equal(isValidUTF8(Uint8Array.from(bytes), true), true)
+ }
+ for (const bytes of [[0xC0], [0xF5], [0xE4, 0x41], [0xE0, 0x80], [0xED, 0xA0], [0xF4, 0x90]]) {
+ assert.equal(isValidUTF8(Uint8Array.from(bytes), true), false, String(bytes))
+ }
+})
+
+test('isValidUTF8 accepts valid Unicode including scalar boundaries', () => {
+ for (const text of ['', 'ASCII', '中文😀', '\u0080\u07FF\u0800\uD7FF\uE000\uFFFF', '\u{10000}\u{10FFFF}']) {
+ const bytes = new TextEncoder().encode(text)
+ assert.equal(isValidUTF8(bytes), true)
+ assert.equal(isValidUTF8(bytes, true), true)
+ }
+})
+
test('pretty-print helpers only rewrite canonical JSON documents', () => {
assert.equal(shouldPrettyPrintJson('json'), true)
assert.equal(shouldPrettyPrintJson('jsonl'), false)
diff --git a/frontend/src/utils/filePreview.ts b/frontend/src/utils/filePreview.ts
index 4f7c39191..d75945554 100644
--- a/frontend/src/utils/filePreview.ts
+++ b/frontend/src/utils/filePreview.ts
@@ -321,29 +321,14 @@ export function prettyPrintJson(text: string): string {
* reject valid text files.
*/
export function isValidUTF8(bytes: Uint8Array, allowTruncatedEnd = false): boolean {
- for (let i = 0; i < bytes.length;) {
- const b = bytes[i]
- let remaining = 0
- if (b <= 0x7F) {
- remaining = 0
- } else if ((b & 0xE0) === 0xC0) {
- remaining = 1
- } else if ((b & 0xF0) === 0xE0) {
- remaining = 2
- } else if ((b & 0xF8) === 0xF0) {
- remaining = 3
- } else {
- return false
- }
- if (i + remaining >= bytes.length) {
- return allowTruncatedEnd
- }
- for (let j = 1; j <= remaining; j++) {
- if ((bytes[i + j] & 0xC0) !== 0x80) return false
- }
- i += 1 + remaining
+ try {
+ // Streaming tolerates an incomplete final code point, but still rejects
+ // invalid bytes already present (including overlong and surrogate encodings).
+ new TextDecoder('utf-8', { fatal: true }).decode(bytes, { stream: allowTruncatedEnd })
+ return true
+ } catch {
+ return false
}
- return true
}
function asciiTag(bytes: Uint8Array, offset: number, length: number): string {