Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(preview): reject invalid UTF-8 before decoding CSV (#4479)
XY8678c468b697
diff --git a/frontend/src/components/document-preview.encoding.test.ts b/frontend/src/components/document-preview.encoding.test.tsnew file mode 100644index 000000000..de688e50b--- /dev/null+++ b/frontend/src/components/document-preview.encoding.test.ts@@ -0,0 +1,88 @@+import assert from 'node:assert/strict'+import { readFileSync } from 'node:fs'+import { createRequire } from 'node:module'+import { fileURLToPath } from 'node:url'+import { runInNewContext } from 'node:vm'+import test from 'node:test'+import { compileScript, parse } from '@vue/compiler-sfc'+import { DOMParser } from '@xmldom/xmldom'+import ts from 'typescript'+import { createRenderer, h, ref } from 'vue'+import * as filePreview from '../utils/filePreview'++const require = createRequire(import.meta.url)+const filename = fileURLToPath(new URL('./document-preview.vue', import.meta.url))+const { descriptor } = parse(readFileSync(filename, 'utf8'), { filename })+const script = compileScript(descriptor, { id: 'preview-encoding-test' }).content+ .replace('__expose();', '')+ .replace('return __returned__', '__expose(__returned__); return __returned__')+const compiled = ts.transpileModule(script, {+ compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 },+}).outputText++function fixture(bytes: Uint8Array, fileType: string) {+ const exports: any = {}+ runInNewContext(compiled, {+ exports,+ require(name: string) {+ if (name === 'vue' || name === 'xlsx') return require(name)+ if (name === '@/utils/filePreview') return filePreview+ if (name === 'vue-i18n') return { useI18n: () => ({ t: (key: string) => key }) }+ // Encoding and SheetJS rendering stay real. DOM sanitization and the+ // unused preview formats/network adapters are outside this test's scope.+ if (name === '@/utils/security') return { sanitizeHTML: (html: string) => html }+ return {}+ },+ Blob, TextDecoder, URL, console,+ document: { addEventListener() {}, removeEventListener() {}, activeElement: {} },+ window: { addEventListener() {}, removeEventListener() {} },+ })+ const component = exports.default+ component.render = () => null+ const renderer = createRenderer<any, any>({+ createElement: () => ({}), createText: () => ({}), createComment: () => ({}),+ insert() {}, remove() {}, setElementText() {}, setText() {}, patchProp() {},+ parentNode: () => null, nextSibling: () => null,+ })+ const instance = ref<any>()+ const app = renderer.createApp({+ render: () => h(component, {+ ref: instance,+ sourceBlob: new Blob([Uint8Array.from(bytes)]),+ fileName: `encoding.${fileType}`, fileType,+ // Invoke loadPreview explicitly so the test can await the whole pipeline.+ active: false,+ }),+ })+ app.mount({})+ return { vm: instance.value, close: () => app.unmount() }+}++for (const fileType of ['csv', 'tsv', 'tab']) {+ for (const encoding of ['gbk', 'utf8', 'utf8-bom']) {+ test(`${fileType} preview preserves ${encoding} Chinese text and table columns`, async () => {+ const separator = fileType === 'csv' ? ',' : '\t'+ const text = `item${separator}count\r\n蜡${separator}1\r\n`+ // C0 AF encodes 蜡 in GBK, but is an invalid overlong UTF-8 sequence.+ const bytes = encoding === 'gbk'+ ? Uint8Array.from([+ ...new TextEncoder().encode(`item${separator}count\r\n`),+ 0xC0, 0xAF,+ ...new TextEncoder().encode(`${separator}1\r\n`),+ ])+ : new TextEncoder().encode((encoding === 'utf8-bom' ? '\uFEFF' : '') + text)+ const f = fixture(bytes, fileType)+ try {+ await f.vm.loadPreview()+ assert.equal(f.vm.error, '')+ assert.equal(f.vm.loading, false)+ assert.equal(f.vm.previewType, 'excel')+ const doc = new DOMParser().parseFromString(f.vm.excelHtml, 'text/html')+ const cells = Array.from(doc.getElementsByTagName('td'), cell => cell.textContent)+ assert.deepEqual(cells, ['item', 'count', '蜡', '1'])+ } finally {+ f.close()+ }+ })+ }+}diff --git a/frontend/src/utils/filePreview.test.ts b/frontend/src/utils/filePreview.test.tsindex 9d80b5773..0e684d7ec 100644--- a/frontend/src/utils/filePreview.test.ts+++ b/frontend/src/utils/filePreview.test.ts@@ -5,6 +5,7 @@ import {getHighlightLang,getPreviewMimeType,isKnownPreviewableExt,+ isValidUTF8,prettyPrintJson,resolveFilePreviewExt,resolvePreviewKind,@@ -64,6 +65,39 @@ test('sniffPreview rejects NUL-containing binaries', () => {assert.equal(sniffPreview(Uint8Array.from([0x00, 0x01, 0x02, 0x03])).kind, 'unsupported')})+test('isValidUTF8 rejects invalid scalar encodings so GBK CSV can fall back', () => {+ // GBK for "蜡,1": accepting C0 AF as UTF-8 produces replacement characters.+ const csv = Uint8Array.from([0xC0, 0xAF, 0x2C, 0x31])+ assert.equal(isValidUTF8(csv), false)+ assert.equal(new TextDecoder('gbk').decode(csv), '蜡,1')+ for (const bytes of [+ [0xC1, 0xBF], [0xE0, 0x80, 0xAF], [0xED, 0xA0, 0x80],+ [0xF0, 0x80, 0x80, 0xAF], [0xF4, 0x90, 0x80, 0x80],+ [0xF5, 0x80, 0x80, 0x80], [0x80],+ ]) {+ assert.equal(isValidUTF8(Uint8Array.from(bytes)), false, String(bytes))+ assert.equal(sniffPreview(Uint8Array.from(bytes)).kind, 'unsupported', String(bytes))+ }+})++test('isValidUTF8 only tolerates valid incomplete prefixes when requested', () => {+ for (const bytes of [[0xC2], [0xE4, 0xB8], [0xF0, 0x9F, 0x98]]) {+ assert.equal(isValidUTF8(Uint8Array.from(bytes)), false)+ assert.equal(isValidUTF8(Uint8Array.from(bytes), true), true)+ }+ for (const bytes of [[0xC0], [0xF5], [0xE4, 0x41], [0xE0, 0x80], [0xED, 0xA0], [0xF4, 0x90]]) {+ assert.equal(isValidUTF8(Uint8Array.from(bytes), true), false, String(bytes))+ }+})++test('isValidUTF8 accepts valid Unicode including scalar boundaries', () => {+ for (const text of ['', 'ASCII', '中文😀', '\u0080\u07FF\u0800\uD7FF\uE000\uFFFF', '\u{10000}\u{10FFFF}']) {+ const bytes = new TextEncoder().encode(text)+ assert.equal(isValidUTF8(bytes), true)+ assert.equal(isValidUTF8(bytes, true), true)+ }+})+test('pretty-print helpers only rewrite canonical JSON documents', () => {assert.equal(shouldPrettyPrintJson('json'), true)assert.equal(shouldPrettyPrintJson('jsonl'), false)diff --git a/frontend/src/utils/filePreview.ts b/frontend/src/utils/filePreview.tsindex 4f7c39191..d75945554 100644--- a/frontend/src/utils/filePreview.ts+++ b/frontend/src/utils/filePreview.ts@@ -321,29 +321,14 @@ export function prettyPrintJson(text: string): string {* reject valid text files.*/export function isValidUTF8(bytes: Uint8Array, allowTruncatedEnd = false): boolean {- for (let i = 0; i < bytes.length;) {- const b = bytes[i]- let remaining = 0- if (b <= 0x7F) {- remaining = 0- } else if ((b & 0xE0) === 0xC0) {- remaining = 1- } else if ((b & 0xF0) === 0xE0) {- remaining = 2- } else if ((b & 0xF8) === 0xF0) {- remaining = 3- } else {- return false- }- if (i + remaining >= bytes.length) {- return allowTruncatedEnd- }- for (let j = 1; j <= remaining; j++) {- if ((bytes[i + j] & 0xC0) !== 0x80) return false- }- i += 1 + remaining+ try {+ // Streaming tolerates an incomplete final code point, but still rejects+ // invalid bytes already present (including overlong and surrogate encodings).+ new TextDecoder('utf-8', { fatal: true }).decode(bytes, { stream: allowTruncatedEnd })+ return true+ } catch {+ return false}- return true}function asciiTag(bytes: Uint8Array, offset: number, length: number): string {