RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): support non-UTF-8 decoding across text, csv, and epub parsers (#19243)
jay777215bcaa73cb74a
diff --git a/internal/parser/parser/charset.go b/internal/parser/parser/charset.goindex 014d7051fa..9b8a685511 100644--- a/internal/parser/parser/charset.go+++ b/internal/parser/parser/charset.go@@ -26,8 +26,12 @@ import ("bytes""fmt""io"+ "regexp""strings"+ "unicode/utf8"+ "github.com/saintfish/chardet"+ htmlcharset "golang.org/x/net/html/charset""golang.org/x/text/encoding""golang.org/x/text/encoding/charmap""golang.org/x/text/encoding/japanese"@@ -90,12 +94,26 @@ func decodeWithCharset(payload []byte, charset string) (string, error) {if enc, ok := charsetEncoding(charset); ok {return decodeTransform(payload, enc.NewDecoder())}- // Unknown charset: treat as latin-1.- runes := make([]rune, len(payload))- for i, b := range payload {- runes[i] = rune(b)+ if enc, _ := htmlcharset.Lookup(charset); enc != nil {+ return decodeTransform(payload, enc.NewDecoder())+ }+ return "", fmt.Errorf("unsupported charset: %s", charset)+}++// isRecognizedCharset reports whether charset maps to a known decoder+// either via our shared charsetEncoding resolver or htmlcharset.Lookup.+func isRecognizedCharset(charset string) bool {+ switch canonicalCharsetLabel(charset) {+ case "utf8", "":+ return true+ }+ if _, ok := charsetEncoding(charset); ok {+ return true+ }+ if enc, _ := htmlcharset.Lookup(charset); enc != nil {+ return true}- return string(runes), nil+ return false}// decodeTransform decodes payload through decoder, rejecting results that@@ -124,3 +142,169 @@ func decodeFirstCharsetMatch(data []byte, labels []string) (string, string, bool}return "", "", false}++// fallbackCharsetLabels is the ordered brute-force decode loop for bytes+// that are neither valid UTF-8 nor carrying a BOM / explicit charset declaration.+// GB18030 (the GBK superset) leads for CJK documents, followed by Big5, Shift-JIS,+// EUC-KR, and terminal ISO-8859-1.+var fallbackCharsetLabels = []string{"gb18030", "big5", "shift_jis", "euc-kr", "iso-8859-1"}++// charsetDetectConfidence is the minimum statistical-detection confidence+// (on chardet's 0-100 scale) allowed to override the fallback chain order.+const charsetDetectConfidence = 90++func detectedCharset(data []byte) string {+ sample := data+ if len(sample) > 1024 {+ sample = sample[:1024]+ }+ res, err := chardet.NewTextDetector().DetectBest(sample)+ if err != nil || res == nil || res.Charset == "" || res.Confidence < charsetDetectConfidence {+ return ""+ }+ detected := canonicalCharsetLabel(res.Charset)+ // The detector reports the GBK family as GB2312 / GBK; GB18030 is the+ // superset we ship, so both map onto the gb18030 candidate.+ if detected == "gb2312" || detected == "gbk" {+ detected = "gb18030"+ }+ for _, label := range fallbackCharsetLabels {+ if canonicalCharsetLabel(label) == detected {+ return label+ }+ }+ return ""+}++var (+ htmlMetaCharsetRe = regexp.MustCompile(`(?i)<meta[^>]*charset\s*=\s*["']?\s*([a-z0-9._+\-]+)`)+ xmlEncodingRe = regexp.MustCompile(`(?i)<\?xml[^>]*encoding\s*=\s*["']?\s*([a-z0-9._+\-]+)`)+)++func declaresWindows1252(data []byte) bool {+ if len(data) > 1024 {+ data = data[:1024]+ }+ for _, m := range htmlMetaCharsetRe.FindAllSubmatch(data, -1) {+ if _, name := htmlcharset.Lookup(string(m[1])); name == "windows-1252" {+ return true+ }+ }+ for _, m := range xmlEncodingRe.FindAllSubmatch(data, -1) {+ if _, name := htmlcharset.Lookup(string(m[1])); name == "windows-1252" {+ return true+ }+ }+ return false+}++func declaredXMLEncoding(data []byte) string {+ sample := data+ if len(sample) > 1024 {+ sample = sample[:1024]+ }+ m := xmlEncodingRe.FindSubmatch(sample)+ if len(m) > 1 {+ return string(m[1])+ }+ return ""+}++// extractCharsetHint extracts any explicit charset label from hint,+// whether hint is a bare charset ("gb2312") or a media type parameter ("text/plain; charset=gb2312").+func extractCharsetHint(hint string) string {+ hint = strings.TrimSpace(hint)+ if hint == "" {+ return ""+ }+ if !strings.Contains(hint, "/") {+ return hint+ }+ if idx := strings.Index(strings.ToLower(hint), "charset="); idx != -1 {+ cs := hint[idx+len("charset="):]+ cs = strings.Trim(cs, `"' `)+ if end := strings.IndexAny(cs, "; \t\r\n"); end != -1 {+ cs = cs[:end]+ }+ return cs+ }+ return ""+}++// DecodeToUTF8 converts arbitrary data bytes into UTF-8.+// If data is already valid UTF-8, it returns data untouched with encoding "utf-8".+// hint can be:+// - An explicit charset label (e.g. "gbk", "gb2312", "big5", from MIME headers)+// - A MIME Content-Type (e.g. "text/html", "application/xhtml+xml", "text/csv", "text/plain")+// - Empty string ""+//+// It returns the decoded UTF-8 bytes and the canonical encoding label used.+func DecodeToUTF8(data []byte, hint string) ([]byte, string) {+ if len(data) == 0 {+ return data, "utf-8"+ }++ // 1. Explicit charset declaration from hint (e.g. email MIME headers).+ // If the caller explicitly declared a non-UTF8 charset, prioritize trying it first.+ // This correctly handles 7-bit ASCII escape encodings like HZ-GB-2312, where bytes are+ // valid ASCII (thus valid UTF-8) but represent encoded Chinese escape sequences.+ // We verify the charset is recognized so unrecognized labels (e.g. "unknown-8bit")+ // do not corrupt UTF-8 text or bypass validation.+ if cs := extractCharsetHint(hint); cs != "" && canonicalCharsetLabel(cs) != "utf8" && isRecognizedCharset(cs) {+ if decoded, err := decodeWithCharset(data, cs); err == nil {+ return []byte(decoded), canonicalCharsetLabel(cs)+ }+ }++ // 2. BOM prescan (Byte Order Mark).+ // RFC 7303 §3.2 and XML specs define BOM as the authoritative physical encoding signature+ // that overrides conflicting in-band text declarations (e.g. UTF-8 BOM with <?xml encoding="ISO-8859-1"?>).+ // When contentType is "", DetermineEncoding reports certain=true strictly when a physical BOM is present.+ if enc, name, certain := htmlcharset.DetermineEncoding(data, ""); certain && enc != nil {+ if decoded, err := decodeTransform(data, enc.NewDecoder()); err == nil {+ return []byte(decoded), name+ }+ }++ // 3. XML declaration check (e.g. <?xml version="1.0" encoding="GB2312"?> in EPUB XHTML).+ // Evaluated before the UTF-8 fast-path so that 7-bit ASCII-compatible escape encodings+ // like HZ-GB-2312 declared in XML are decoded rather than treated as valid ASCII UTF-8.+ if xmlEnc := declaredXMLEncoding(data); xmlEnc != "" && canonicalCharsetLabel(xmlEnc) != "utf8" && isRecognizedCharset(xmlEnc) {+ if decoded, err := decodeWithCharset(data, xmlEnc); err == nil {+ return []byte(decoded), canonicalCharsetLabel(xmlEnc)+ }+ }++ // 4. Fast-path: already valid UTF-8.+ if utf8.Valid(data) {+ return data, "utf-8"+ }++ // 5. Document prescan via DetermineEncoding (handles HTML meta charset, and Content-Type)+ contentType := ""+ if strings.Contains(hint, "/") {+ contentType = hint+ }+ if enc, name, _ := htmlcharset.DetermineEncoding(data, contentType); enc != nil &&+ (name != "windows-1252" || declaresWindows1252(data)) {+ if decoded, err := decodeTransform(data, enc.NewDecoder()); err == nil {+ return []byte(decoded), name+ }+ }++ // 6. Statistical detection with chardet (confidence >= 90)+ if label := detectedCharset(data); label != "" {+ if decoded, err := decodeWithCharset(data, label); err == nil {+ return []byte(decoded), label+ }+ }++ // 7. Fallback chain: gb18030 -> big5 -> shift_jis -> euc-kr -> iso-8859-1+ if decoded, label, ok := decodeFirstCharsetMatch(data, fallbackCharsetLabels); ok {+ return []byte(decoded), label+ }++ // Defensive fallback: unreachable for non-empty input as terminal ISO-8859-1+ // accepts every byte sequence without producing replacement runes.+ return data, ""+}diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.goindex 83be10c042..cacd43c796 100644--- a/internal/parser/parser/charset_test.go+++ b/internal/parser/parser/charset_test.go@@ -1,6 +1,19 @@package parser-import "testing"+import (+ "archive/zip"+ "bytes"+ "context"+ "strings"+ "testing"+ "unicode/utf8"++ "golang.org/x/text/encoding/japanese"+ "golang.org/x/text/encoding/korean"+ "golang.org/x/text/encoding/simplifiedchinese"+ "golang.org/x/text/encoding/traditionalchinese"+ "golang.org/x/text/encoding/unicode"+)// TestCharsetEncodingSharedLabelContract locks the single-label contract of// charsetEncoding: every fallback label the HTML parser may walk (and the@@ -11,7 +24,7 @@ import "testing"// drift apart again.func TestCharsetEncodingSharedLabelContract(t *testing.T) {mailLabels := []string{"gb2312", "gbk", "gb18030", "hz-gb-2312", "big5", "shift_jis", "sjis", "euc-kr", "iso-8859-1", "latin1"}- for _, label := range append(mailLabels, htmlCharsetFallbackLabels...) {+ for _, label := range append(mailLabels, fallbackCharsetLabels...) {if _, ok := charsetEncoding(label); !ok {t.Errorf("charsetEncoding(%q) = (_, false); the shared resolver must cover every mail-chain and HTML-fallback label", label)}@@ -60,3 +73,221 @@ func TestDecodeFirstCharsetMatchAcceptance(t *testing.T) {t.Error("decodeFirstCharsetMatch over a rejecting candidate list must report failure")}}++func TestDecodeToUTF8(t *testing.T) {+ // 1. Empty input passes through+ out, label := DecodeToUTF8(nil, "")+ if len(out) != 0 || label != "utf-8" {+ t.Errorf("empty input: got (%d, %q), want (0, utf-8)", len(out), label)+ }++ // 2. Already UTF-8 passes through without alteration+ utf8Str := "你好,世界!This is valid UTF-8 text."+ out, label = DecodeToUTF8([]byte(utf8Str), "")+ if string(out) != utf8Str || label != "utf-8" {+ t.Errorf("UTF-8 input: got %q (%q), want %q (utf-8)", string(out), label, utf8Str)+ }++ // 3. GBK / GB2312 text+ chinese := "自然语言处理与知识图谱构建"+ gbkBytes, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(chinese))+ if err != nil {+ t.Fatalf("GBK encode: %v", err)+ }+ out, label = DecodeToUTF8(gbkBytes, "text/plain")+ if string(out) != chinese {+ t.Errorf("GBK decode: got %q, want %q", string(out), chinese)+ }+ if label != "gb18030" && label != "gbk" && label != "gb2312" {+ t.Errorf("GBK label: got %q, want gb18030/gbk/gb2312", label)+ }++ // 4. Big5 text+ traditional := "繁體中文測試資料"+ big5Bytes, err := traditionalchinese.Big5.NewEncoder().Bytes([]byte(traditional))+ if err != nil {+ t.Fatalf("Big5 encode: %v", err)+ }+ out, label = DecodeToUTF8(big5Bytes, "")+ if string(out) != traditional {+ t.Errorf("Big5 decode: got %q, want %q", string(out), traditional)+ }++ // 5. Shift-JIS Japanese text+ japaneseText := "日本語テキストのテストです"+ sjisBytes, err := japanese.ShiftJIS.NewEncoder().Bytes([]byte(japaneseText))+ if err != nil {+ t.Fatalf("Shift-JIS encode: %v", err)+ }+ out, _ = DecodeToUTF8(sjisBytes, "")+ if string(out) != japaneseText {+ t.Errorf("Shift-JIS decode: got %q, want %q", string(out), japaneseText)+ }++ // 6. EUC-KR Korean text+ koreanText := "한국어 시험: 이 문서는 EUC-KR로 인코딩되어 있습니다. 서울특별시."+ euckrBytes, err := korean.EUCKR.NewEncoder().Bytes([]byte(koreanText))+ if err != nil {+ t.Fatalf("EUC-KR encode: %v", err)+ }+ out, _ = DecodeToUTF8(euckrBytes, "")+ if string(out) != koreanText {+ t.Errorf("EUC-KR decode: got %q, want %q", string(out), koreanText)+ }++ // 7. XML declaration check+ xmlHeader := `<?xml version="1.0" encoding="GB2312"?><html><body><p>章节测试</p></body></html>`+ xmlGBK, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(xmlHeader))+ if err != nil {+ t.Fatalf("XML GBK encode: %v", err)+ }+ out, label = DecodeToUTF8(xmlGBK, "application/xhtml+xml")+ if !strings.Contains(string(out), "章节测试") {+ t.Errorf("XML declaration decode: missing expected text in %q", string(out))+ }+ if label != "gb2312" && label != "gb18030" && label != "gbk" {+ t.Errorf("XML declaration label: got %q", label)+ }+}++func TestCSVParser_GBK(t *testing.T) {+ csvText := "产品,分类,价格,备注\n笔记本电脑,电子,6999.00,\"特价销售,送鼠标\"\n无线耳机,配件,399.00,\"热销\"\n"+ gbkBytes, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(csvText))+ if err != nil {+ t.Fatalf("GBK encode: %v", err)+ }++ p := NewCSVParser()+ res := p.ParseWithResult(context.Background(), "products.csv", gbkBytes)+ if res.Err != nil {+ t.Fatalf("CSVParser.ParseWithResult failed: %v", res.Err)+ }+ if strings.ContainsRune(res.HTML, '\ufffd') {+ t.Errorf("CSV HTML output contains replacement runes: %s", res.HTML)+ }+ for _, want := range []string{"笔记本电脑", "电子", "特价销售,送鼠标", "无线耳机"} {+ if !strings.Contains(res.HTML, want) {+ t.Errorf("CSV HTML missing expected substring %q; got: %s", want, res.HTML)+ }+ }+ if enc, _ := res.File["encoding"].(string); enc != "gb18030" && enc != "gbk" && enc != "gb2312" {+ t.Errorf("CSV File.encoding = %q, want gbk/gb2312/gb18030", enc)+ }+}++func TestEPUBParser_GB2312(t *testing.T) {+ // Create an in-memory EPUB zip with GB2312-encoded chapter XHTML+ chapterContent := `<?xml version="1.0" encoding="GB2312"?>+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">+<html xmlns="http://www.w3.org/1999/xhtml">+<head><title>第一章</title></head>+<body><h1>第一章 破晓</h1><p>这是一段包含中文的EPUB正文内容。</p></body>+</html>`+ chapterGBK, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(chapterContent))+ if err != nil {+ t.Fatalf("GBK encode: %v", err)+ }++ containerXML := `<?xml version="1.0"?><container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container"><rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/></rootfiles></container>`+ contentOPF := `<?xml version="1.0" encoding="utf-8"?><package xmlns="http://www.idpf.org/2007/opf" unique-identifier="BookID" version="2.0"><metadata xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:title>测试</dc:title></metadata><manifest><item id="c1" href="c1.xhtml" media-type="application/xhtml+xml"/></manifest><spine><itemref idref="c1"/></spine></package>`++ buf := new(bytes.Buffer)+ zw := zip.NewWriter(buf)+ w, _ := zw.Create("META-INF/container.xml")+ w.Write([]byte(containerXML))+ w, _ = zw.Create("OEBPS/content.opf")+ w.Write([]byte(contentOPF))+ w, _ = zw.Create("OEBPS/c1.xhtml")+ w.Write(chapterGBK)+ zw.Close()++ p := NewEPUBParser()+ res := p.ParseWithResult(context.Background(), "book.epub", buf.Bytes())+ if res.Err != nil {+ t.Fatalf("EPUBParser.ParseWithResult failed: %v", res.Err)+ }+ if len(res.JSON) == 0 {+ t.Fatal("want non-empty JSON items from EPUB")+ }+ text, _ := res.JSON[0]["text"].(string)+ if strings.ContainsRune(text, '\ufffd') {+ t.Errorf("EPUB text contains replacement runes: %s", text)+ }+ for _, want := range []string{"第一章 破晓", "这是一段包含中文的EPUB正文内容。"} {+ if !strings.Contains(text, want) {+ t.Errorf("EPUB text missing %q; got: %s", want, text)+ }+ }+ if enc, _ := res.File["encoding"].(string); enc != "gb2312" && enc != "gbk" && enc != "gb18030" {+ t.Errorf("EPUB File.encoding = %q, want gb2312/gbk/gb18030", enc)+ }+}++func TestDecodeToUTF8_HZGB2312Hint(t *testing.T) {+ // HZ-GB-2312 escape sequences (~{...~}) are pure 7-bit ASCII, hence valid UTF-8.+ // An explicit hint must decode via HZ-GB-2312 rather than returning bytes untouched.+ hz := []byte("~{<4Ky~}")+ out, label := DecodeToUTF8(hz, "hz-gb-2312")+ if label != "hzgb2312" && label != "hz-gb-2312" {+ t.Errorf("got label %q, want hzgb2312 or hz-gb-2312", label)+ }+ if string(out) == string(hz) {+ t.Fatal("hint ignored; utf8 fast-path swallowed HZ escapes")+ }+ if !utf8.Valid(out) {+ t.Fatal("decoded HZ output must be valid UTF-8")+ }+}++func TestDecodeToUTF8_XMLEncoding_HZGB2312(t *testing.T) {+ // XML encoding declaration for HZ-GB-2312 must be evaluated before the UTF-8 fast path.+ xml := []byte("<?xml version=\"1.0\" encoding=\"HZ-GB-2312\"?><html><body>~{<4Ky~}</body></html>")+ out, label := DecodeToUTF8(xml, "application/xhtml+xml")+ if label != "hzgb2312" && label != "hz-gb-2312" {+ t.Errorf("got label %q, want hzgb2312", label)+ }+ if string(out) == string(xml) {+ t.Fatal("XML HZ-GB-2312 declaration ignored; utf8 fast-path swallowed HZ escapes")+ }+}++func TestDecodeToUTF8_UnrecognizedHint_UTF8Payload(t *testing.T) {+ // Unrecognized charset labels (e.g. unknown-8bit from MIME) must not trigger+ // a bogus latin-1 decode or corrupt valid UTF-8 input.+ utf8Text := []byte("这是一段标准的UTF-8中文内容")+ out, label := DecodeToUTF8(utf8Text, "unknown-8bit")+ if label != "utf-8" {+ t.Errorf("got label %q, want utf-8", label)+ }+ if string(out) != string(utf8Text) {+ t.Errorf("unrecognized hint corrupted UTF-8 text; got %q, want %q", string(out), string(utf8Text))+ }+}++func TestDecodeToUTF8_BOMOverridesConflictingXMLDeclaration(t *testing.T) {+ // RFC 7303: UTF-8 BOM (\xef\xbb\xbf) takes precedence over a conflicting XML declaration (e.g. ISO-8859-1).+ conflictXML := append([]byte("\xef\xbb\xbf"), []byte(`<?xml version="1.0" encoding="ISO-8859-1"?><html><body>中文测试内容</body></html>`)...)+ out, label := DecodeToUTF8(conflictXML, "application/xhtml+xml")+ if label != "utf-8" {+ t.Errorf("got label %q, want utf-8", label)+ }+ if !strings.Contains(string(out), "中文测试内容") {+ t.Fatalf("UTF-8 BOM was overridden by XML ISO-8859-1; got: %s", string(out))+ }+}++func TestDecodeToUTF8_UTF16LEBOM(t *testing.T) {+ // UTF-16LE with BOM (\xff\xfe) must decode to valid UTF-8.+ src := "UTF-16LE 中文测试"+ utf16Bytes, err := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder().Bytes([]byte(src))+ if err != nil {+ t.Fatalf("UTF-16 encode: %v", err)+ }+ out, label := DecodeToUTF8(utf16Bytes, "text/plain")+ if label != "utf-16le" {+ t.Errorf("got label %q, want utf-16le", label)+ }+ if !strings.Contains(string(out), src) {+ t.Errorf("got %q, want containing %q", string(out), src)+ }+}diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.goindex 11297a8661..57d6208731 100644--- a/internal/parser/parser/csv_parser.go+++ b/internal/parser/parser/csv_parser.go@@ -125,14 +125,15 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [// for CSV processing.}- text := string(data)+ decoded, encName := DecodeToUTF8(data, "text/csv")+ text := string(decoded)if strings.TrimSpace(text) == "" {return ParseResult{OutputFormat: "html",File: map[string]any{"name": filename,"size": len(data),- "encoding": "utf-8",+ "encoding": encName,},HTML: "<table><caption>" + csvSheetName + "</caption><tr><td></td></tr></table>",}@@ -161,7 +162,7 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [File: map[string]any{"name": filename,"size": len(data),- "encoding": "utf-8",+ "encoding": encName,},HTML: recordsToHTMLTableChunks(records, chunkRows, csvSheetName),}diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.goindex 67436fc0e7..70923ae4f9 100644--- a/internal/parser/parser/email_parser.go+++ b/internal/parser/parser/email_parser.go@@ -511,26 +511,13 @@ func walkHTMLBodyText(n *html.Node, w *leafWriter) {}}-// decodeMailPayload attempts multiple charset decodings.-// Mirrors Python's _decode_payload with fallback chain:-// utf-8 → gb2312 → gbk → gb18030 → latin1 → utf-8 (ignore).+// decodeMailPayload attempts charset decoding using the unified DecodeToUTF8 helper.func decodeMailPayload(payload []byte, charset string) string {if len(payload) == 0 {return ""}- if decoded, _, ok := decodeFirstCharsetMatch(payload, buildCharsetChain(charset)); ok {- return decoded- }- return string(payload)-}--func buildCharsetChain(declared string) []string {- chain := make([]string, 0, 7)- if declared != "" {- chain = append(chain, declared)- }- chain = append(chain, "utf-8", "gb2312", "gbk", "gb18030", "latin1")- return chain+ decoded, _ := DecodeToUTF8(payload, charset)+ return string(decoded)}// parseMSG parses an Outlook .msg (OLE2 compound document) file using thediff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.goindex 29d2c5eebe..59bd1d701b 100644--- a/internal/parser/parser/epub_parser.go+++ b/internal/parser/parser/epub_parser.go@@ -82,7 +82,7 @@ func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, datareturn ParseResult{Err: fmt.Errorf("epub: opf: %w", err)}}- items := extractEPUBTextItems(reader, opfPath, spineItems)+ items, detectedEnc := extractEPUBTextItems(reader, opfPath, spineItems)if items == nil {items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}}@@ -91,7 +91,7 @@ func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, dataFile: map[string]any{"name": filename,"size": len(data),- "encoding": "utf-8",+ "encoding": detectedEnc,},JSON: items,}@@ -194,10 +194,14 @@ var (epubWSRe = regexp.MustCompile(`\s+`))-func extractEPUBTextItems(r *zip.Reader, opfDir string, spineHrefs []string) []map[string]any {+func extractEPUBTextItems(r *zip.Reader, opfDir string, spineHrefs []string) ([]map[string]any, string) {var items []map[string]any+ detectedEncoding := "utf-8"for _, href := range spineHrefs {- text := readEPUBContentFile(r, opfDir, href)+ text, enc := readEPUBContentFile(r, opfDir, href)+ if detectedEncoding == "utf-8" && enc != "" && enc != "utf-8" {+ detectedEncoding = enc+ }if strings.TrimSpace(text) == "" {continue}@@ -206,13 +210,13 @@ func extractEPUBTextItems(r *zip.Reader, opfDir string, spineHrefs []string) []m"doc_type_kwd": "text",})}- return items+ return items, detectedEncoding}// readEPUBContentFile resolves a spine href (relative to the OPF// directory) inside the ZIP, reads the raw bytes, and strips HTML to-// return clean text.-func readEPUBContentFile(r *zip.Reader, opfDir, href string) string {+// return clean text and detected encoding.+func readEPUBContentFile(r *zip.Reader, opfDir, href string) (string, string) {// Resolve href relative to the directory containing the OPF.// Use path.Join/Dir (slash separator) because ZIP entry paths are always POSIX.resolved := path.Join(path.Dir(opfDir), href)@@ -230,15 +234,16 @@ func readEPUBContentFile(r *zip.Reader, opfDir, href string) string {}}if err != nil {- return ""+ return "", ""}defer f.Close()raw, err := io.ReadAll(f)if err != nil {- return ""+ return "", ""}- return stripHTMLTags(string(raw))+ decoded, enc := DecodeToUTF8(raw, "application/xhtml+xml")+ return stripHTMLTags(string(decoded)), enc}// stripHTMLTags removes HTML markup and returns normalized plain text.diff --git a/internal/parser/parser/html_charset_test.go b/internal/parser/parser/html_charset_test.goindex 6fcc981a5e..b42f6f58b6 100644--- a/internal/parser/parser/html_charset_test.go+++ b/internal/parser/parser/html_charset_test.go@@ -4,6 +4,7 @@ import ("context""strings""testing"+ "unicode/utf8""golang.org/x/text/encoding""golang.org/x/text/encoding/charmap"@@ -30,14 +31,14 @@ func gbkHTML(t *testing.T, withMeta bool) []byte {"<tr><td>应收账款</td><td>1200000000.00</td></tr></table>" +"<p>单位:元</p></body></html>"src := head + body- if !utf8Valid([]byte(src)) {+ if !utf8.Valid([]byte(src)) {t.Fatal("test source must be valid UTF-8 before re-encoding")}out, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(src))if err != nil {t.Fatalf("GBK encode: %v", err)}- if utf8Valid(out) {+ if utf8.Valid(out) {t.Fatal("GBK fixture unexpectedly decodes as UTF-8; fixture is ineffective")}return out@@ -52,7 +53,7 @@ func htmlEncodedFixture(t *testing.T, src string, enc encoding.Encoding) []byteif err != nil {t.Fatalf("fixture encode: %v", err)}- if utf8Valid(raw) {+ if utf8.Valid(raw) {t.Fatal("fixture unexpectedly decodes as UTF-8; fixture is ineffective")}return raw@@ -116,7 +117,7 @@ func TestHTMLParser_DecodesBig5MetaDeclared(t *testing.T) {if err != nil {t.Fatalf("Big5 encode: %v", err)}- if utf8Valid(raw) {+ if utf8.Valid(raw) {t.Fatal("Big5 fixture unexpectedly decodes as UTF-8; fixture is ineffective")}res := NewHTMLParser().ParseWithResult(context.Background(), "tw.html", raw)diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex c884078c9f..ae34b6e29f 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -20,13 +20,9 @@ import ("bytes""context""fmt"- "regexp""strings"- "github.com/saintfish/chardet"-"golang.org/x/net/html"- "golang.org/x/net/html/charset")type HTMLParser struct {@@ -110,136 +106,10 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data}}-// htmlCharsetFallbackLabels is the ordered brute-force decode loop for HTML-// bytes that are neither valid UTF-8 nor carrying a BOM / <meta charset>-// declaration, mirroring the pragmatic core of Python's find_codec-// (rag/nlp/__init__.py): legacy pages that omit a charset declaration are-// overwhelmingly GBK-family (saved-by-browser Chinese finance/gov pages), so-// GB18030 (the GBK superset) leads. Big5 / Shift-JIS / EUC-KR pages-// virtually always declare their charset and are caught by the meta prescan;-// when the declaration is missing, the statistical detector-// (htmlDetectedCharset) reorders the attempt so they are not silently-// absorbed by GB18030. ISO-8859-1 is intentionally terminal, the way latin1-// terminates Python's codec loop: it decodes ANY byte sequence without-// replacement runes, so once the chain reaches it, it always succeeds —-// Western pages without a declaration still yield readable text instead of-// U+FFFD soup. Labels resolve through the shared charsetEncoding in-// charset.go; only the HTML-specific ordering lives here.-var htmlCharsetFallbackLabels = []string{"gb18030", "big5", "shift_jis", "euc-kr", "iso-8859-1"}--// htmlCharsetDetectConfidence is the minimum statistical-detection-// confidence (on chardet's 0-100 scale) allowed to override the fallback-// chain order. Calibrated empirically: genuine CJK text scores 100, while-// the Go chardet port produces plausible-but-wrong guesses at low confidence-// (a sparse GBK balance-sheet page scores Big5 at 40; a short Big5 page-// scores ISO-8859-1 at 17). Only a confident pick may jump the queue;-// anything weaker keeps the GB18030-first domain heuristic.-const htmlCharsetDetectConfidence = 90--// htmlDetectedCharset mirrors Python find_codec's chardet.detect step for-// HTML bytes that are neither valid UTF-8 nor declared: a statistical-// detector picks which fallback-chain label to try FIRST, returned as one-// of htmlCharsetFallbackLabels. This matters because GB18030 maps nearly-// every byte sequence without replacement runes, so the ordered chain alone-// cannot recognize an undeclared Big5 / Shift-JIS / EUC-KR page — it would-// "decode" it as wrong Chinese text. Detection runs over the first 1024-// bytes, exactly like Python, but unlike Python it is gated on-// htmlCharsetDetectConfidence: the Go port's low-confidence guesses are-// demonstrably wrong often enough that they must not override the chain.-// A detection that fails, scores below the gate, or names an encoding the-// chain does not ship (e.g. windows-1252) returns "" and the chain order-// stands.-func htmlDetectedCharset(data []byte) string {- sample := data- if len(sample) > 1024 {- sample = sample[:1024]- }- res, err := chardet.NewTextDetector().DetectBest(sample)- if err != nil || res == nil || res.Charset == "" || res.Confidence < htmlCharsetDetectConfidence {- return ""- }- detected := canonicalCharsetLabel(res.Charset)- // The detector reports the GBK family as GB2312 / GBK; GB18030 is the- // superset we ship, so both map onto the gb18030 candidate.- if detected == "gb2312" || detected == "gbk" {- detected = "gb18030"- }- for _, label := range htmlCharsetFallbackLabels {- if canonicalCharsetLabel(label) == detected {- return label- }- }- return ""-}--// htmlMetaCharsetRe captures the charset label of a <meta> tag, matching both-// the <meta charset="..."> form and the http-equiv content-type form-// (<meta ... content="text/html; charset=...">), over the same first-1024-byte-// window x/net's prescan consumes.-var htmlMetaCharsetRe = regexp.MustCompile(`(?i)<meta[^>]*charset\s*=\s*["']?\s*([a-z0-9._+\-]+)`)--// htmlDeclaresWindows1252 reports whether data's <meta> tags declare-// windows-1252, either directly or via a WHATWG alias such as iso-8859-1 /-// us-ascii that charset.Lookup canonicalizes to windows-1252.-// charset.DetermineEncoding cannot tell a declared windows-1252 from its own-// "nothing declared" default: a meta prescan hit never sets certain (only-// BOMs do), so both come back as (windows-1252, certain=false). The-// declaration must therefore be confirmed explicitly, or a declared-// windows-1252 page is mis-decoded by the fallback chain: windows-1252's-// 0x80-0x9F range (smart quotes, €, ™) is C1 control characters in the-// chain's ISO-8859-1 terminal.-func htmlDeclaresWindows1252(data []byte) bool {- if len(data) > 1024 {- data = data[:1024]- }- for _, m := range htmlMetaCharsetRe.FindAllSubmatch(data, -1) {- if _, name := charset.Lookup(string(m[1])); name == "windows-1252" {- return true- }- }- return false-}-// decodeHTMLToUTF8 converts non-UTF-8 HTML bytes to UTF-8 and reports the-// encoding label used. Valid UTF-8 passes through untouched. Otherwise the-// document's own BOM or <meta charset> declaration wins — including a-// declared windows-1252: DetermineEncoding returns that name both for a real-// declaration and as its "nothing declared" default, with certain=false-// either way, so htmlDeclaresWindows1252 disambiguates the two. Undeclared-// documents first honor a high-confidence statistical pick-// (htmlDetectedCharset) and then walk the fallback chain-// (htmlCharsetFallbackLabels via the shared decodeFirstCharsetMatch), where-// each candidate must decode the whole blob without producing replacement-// runes. If everything fails the original bytes are returned so behavior-// never regresses below the previous UTF-8-only parse.+// encoding label used. It delegates to the shared DecodeToUTF8 helper.func decodeHTMLToUTF8(data []byte) ([]byte, string) {- if len(data) == 0 || utf8Valid(data) {- return data, "utf-8"- }- // The prescan declaration wins for every name except the undeclared- // windows-1252 sentinel: skipping the sentinel keeps undeclared pages on- // the detect-then-chain path, while a declared windows-1252 (or its- // iso-8859-1 / us-ascii aliases) is honored.- if enc, name, _ := charset.DetermineEncoding(data, ""); enc != nil &&- (name != "windows-1252" || htmlDeclaresWindows1252(data)) {- if decoded, err := decodeTransform(data, enc.NewDecoder()); err == nil {- return []byte(decoded), name- }- }- if label := htmlDetectedCharset(data); label != "" {- if decoded, err := decodeWithCharset(data, label); err == nil {- return []byte(decoded), label- }- }- if decoded, label, ok := decodeFirstCharsetMatch(data, htmlCharsetFallbackLabels); ok {- return []byte(decoded), label- }- // Unreachable for non-empty input: the terminal ISO-8859-1 candidate- // decodes every byte sequence without replacement runes, so the loop- // above always returns first (the label is therefore neutral rather- // than a bogus "utf-8" — the bytes are non-UTF-8 by construction).- // Kept as a never-worse-than-before safety net.- return data, ""+ return DecodeToUTF8(data, "text/html")}// walkHTMLBlocks emits one normalized item per block-leveldiff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.goindex a33d274a99..7fe65d9839 100644--- a/internal/parser/parser/parse_with_result_test.go+++ b/internal/parser/parser/parse_with_result_test.go@@ -38,6 +38,7 @@ import ("testing""golang.org/x/net/html"+ "golang.org/x/text/encoding/simplifiedchinese""ragflow/internal/utility")@@ -107,16 +108,28 @@ func TestTextParser_ParseWithResult_NoSizeCap(t *testing.T) {}}-// TestTextParser_ParseWithResult_InvalidUTF8 pins the UTF-8-// validation rule. Invalid bytes produce an error in the result-// (matching the python TxtParser's behaviour).-func TestTextParser_ParseWithResult_InvalidUTF8(t *testing.T) {+// TestTextParser_ParseWithResult_GBK pins that non-UTF-8 (e.g. GBK) text input+// is decoded to valid UTF-8 rather than returning an error.+func TestTextParser_ParseWithResult_GBK(t *testing.T) {ctx := t.Context()p := NewTextParser()- bad := []byte{0xff, 0xfe, 0xfd}- res := p.ParseWithResult(ctx, "bad.txt", bad)- if res.Err == nil {- t.Fatal("want error for invalid UTF-8, got nil")+ gbkBytes, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte("测试中文文本内容"))+ if err != nil {+ t.Fatalf("GBK encode: %v", err)+ }+ res := p.ParseWithResult(ctx, "chinese_gbk.txt", gbkBytes)+ if res.Err != nil {+ t.Fatalf("ParseWithResult(GBK): unexpected error %v", res.Err)+ }+ if len(res.JSON) == 0 {+ t.Fatal("want non-empty items")+ }+ got, _ := res.JSON[0]["text"].(string)+ if got != "测试中文文本内容" {+ t.Errorf("got %q, want %q", got, "测试中文文本内容")+ }+ if enc, _ := res.File["encoding"].(string); enc != "gb18030" && enc != "gbk" {+ t.Errorf("File.encoding = %q, want gb18030 or gbk", enc)}}diff --git a/internal/parser/parser/text_parser.go b/internal/parser/parser/text_parser.goindex 188f4a5bfb..4a88d9b856 100644--- a/internal/parser/parser/text_parser.go+++ b/internal/parser/parser/text_parser.go@@ -57,10 +57,8 @@ func NewTextParser() *TextParser {// non-empty JSON payload even for an empty input (mirrors the// MarkdownParser convention at markdown_parser.go:71-76).func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {- if !utf8Valid(data) {- return ParseResult{Err: errInvalidUTF8}- }- items := textParserItems(data)+ decoded, encName := DecodeToUTF8(data, "text/plain")+ items := textParserItems(decoded)if items == nil {items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}}@@ -69,7 +67,7 @@ func (p *TextParser) ParseWithResult(ctx context.Context, filename string, dataFile: map[string]any{"name": filename,"size": len(data),- "encoding": "utf-8",+ "encoding": encName,},JSON: items,}@@ -79,62 +77,6 @@ func (p *TextParser) String() string {return "TextParser"}-// errInvalidUTF8 is returned when the input bytes fail UTF-8-// validation. Matches the python TxtParser's behaviour of-// surfacing a clear error rather than emitting replacement bytes.-var errInvalidUTF8 = errInvalidUTF8Sentinel("parser: text input is not valid UTF-8")--type errInvalidUTF8Sentinel string--func (e errInvalidUTF8Sentinel) Error() string { return string(e) }--// utf8Valid is a tiny stdlib-free validator. We avoid-// unicode/utf8.Valid to keep this file dependency-light; the-// validation rule is the same (decode without rejecting bytes).-func utf8Valid(data []byte) bool {- for i := 0; i < len(data); {- r, size := decodeRune(data[i:])- if r == 0xFFFD && size == 1 {- return false- }- i += size- }- return true-}--// decodeRune is a minimal UTF-8 decoder that mirrors-// utf8.DecodeRune's signature: returns the rune and its byte-// width. Returns (RuneError, 1) on invalid sequences, matching-// the stdlib contract.-func decodeRune(p []byte) (rune, int) {- if len(p) == 0 {- return 0xFFFD, 0- }- c := p[0]- switch {- case c < 0x80:- return rune(c), 1- case c < 0xC2:- return 0xFFFD, 1- case c < 0xE0:- if len(p) < 2 || p[1]&0xC0 != 0x80 {- return 0xFFFD, 1- }- return rune(c&0x1F)<<6 | rune(p[1]&0x3F), 2- case c < 0xF0:- if len(p) < 3 || p[1]&0xC0 != 0x80 || p[2]&0xC0 != 0x80 {- return 0xFFFD, 1- }- return rune(c&0x0F)<<12 | rune(p[1]&0x3F)<<6 | rune(p[2]&0x3F), 3- case c < 0xF5:- if len(p) < 4 || p[1]&0xC0 != 0x80 || p[2]&0xC0 != 0x80 || p[3]&0xC0 != 0x80 {- return 0xFFFD, 1- }- return rune(c&0x07)<<18 | rune(p[1]&0x3F)<<12 | rune(p[2]&0x3F)<<6 | rune(p[3]&0x3F), 4- }- return 0xFFFD, 1-}-// defaultTextDelimiterPattern is the regexp alternation of the flow parser's// default delimiter set DefaultTextCodeDelimiter (rag/flow/parser/parser.py:_code// → deepdoc TxtParser default "\n!?;。;!?"), each rune re.escape'd to mirror