Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): support non-UTF-8 decoding across text, csv, and epub parsers (#19243)

jay777215bcaa73cb74a
diff --git a/internal/parser/parser/charset.go b/internal/parser/parser/charset.go
index 014d7051fa..9b8a685511 100644
--- a/internal/parser/parser/charset.go
+++ b/internal/parser/parser/charset.go
@@ -26,8 +26,12 @@ import (
"bytes"
"fmt"
"io"
+ "regexp"
"strings"
+ "unicode/utf8"
+ "github.com/saintfish/chardet"
+ htmlcharset "golang.org/x/net/html/charset"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/charmap"
"golang.org/x/text/encoding/japanese"
@@ -90,12 +94,26 @@ func decodeWithCharset(payload []byte, charset string) (string, error) {
if enc, ok := charsetEncoding(charset); ok {
return decodeTransform(payload, enc.NewDecoder())
}
- // Unknown charset: treat as latin-1.
- runes := make([]rune, len(payload))
- for i, b := range payload {
- runes[i] = rune(b)
+ if enc, _ := htmlcharset.Lookup(charset); enc != nil {
+ return decodeTransform(payload, enc.NewDecoder())
+ }
+ return "", fmt.Errorf("unsupported charset: %s", charset)
+}
+
+// isRecognizedCharset reports whether charset maps to a known decoder
+// either via our shared charsetEncoding resolver or htmlcharset.Lookup.
+func isRecognizedCharset(charset string) bool {
+ switch canonicalCharsetLabel(charset) {
+ case "utf8", "":
+ return true
+ }
+ if _, ok := charsetEncoding(charset); ok {
+ return true
+ }
+ if enc, _ := htmlcharset.Lookup(charset); enc != nil {
+ return true
}
- return string(runes), nil
+ return false
}
// decodeTransform decodes payload through decoder, rejecting results that
@@ -124,3 +142,169 @@ func decodeFirstCharsetMatch(data []byte, labels []string) (string, string, bool
}
return "", "", false
}
+
+// fallbackCharsetLabels is the ordered brute-force decode loop for bytes
+// that are neither valid UTF-8 nor carrying a BOM / explicit charset declaration.
+// GB18030 (the GBK superset) leads for CJK documents, followed by Big5, Shift-JIS,
+// EUC-KR, and terminal ISO-8859-1.
+var fallbackCharsetLabels = []string{"gb18030", "big5", "shift_jis", "euc-kr", "iso-8859-1"}
+
+// charsetDetectConfidence is the minimum statistical-detection confidence
+// (on chardet's 0-100 scale) allowed to override the fallback chain order.
+const charsetDetectConfidence = 90
+
+func detectedCharset(data []byte) string {
+ sample := data
+ if len(sample) > 1024 {
+ sample = sample[:1024]
+ }
+ res, err := chardet.NewTextDetector().DetectBest(sample)
+ if err != nil || res == nil || res.Charset == "" || res.Confidence < charsetDetectConfidence {
+ return ""
+ }
+ detected := canonicalCharsetLabel(res.Charset)
+ // The detector reports the GBK family as GB2312 / GBK; GB18030 is the
+ // superset we ship, so both map onto the gb18030 candidate.
+ if detected == "gb2312" || detected == "gbk" {
+ detected = "gb18030"
+ }
+ for _, label := range fallbackCharsetLabels {
+ if canonicalCharsetLabel(label) == detected {
+ return label
+ }
+ }
+ return ""
+}
+
+var (
+ htmlMetaCharsetRe = regexp.MustCompile(`(?i)<meta[^>]*charset\s*=\s*["']?\s*([a-z0-9._+\-]+)`)
+ xmlEncodingRe = regexp.MustCompile(`(?i)<\?xml[^>]*encoding\s*=\s*["']?\s*([a-z0-9._+\-]+)`)
+)
+
+func declaresWindows1252(data []byte) bool {
+ if len(data) > 1024 {
+ data = data[:1024]
+ }
+ for _, m := range htmlMetaCharsetRe.FindAllSubmatch(data, -1) {
+ if _, name := htmlcharset.Lookup(string(m[1])); name == "windows-1252" {
+ return true
+ }
+ }
+ for _, m := range xmlEncodingRe.FindAllSubmatch(data, -1) {
+ if _, name := htmlcharset.Lookup(string(m[1])); name == "windows-1252" {
+ return true
+ }
+ }
+ return false
+}
+
+func declaredXMLEncoding(data []byte) string {
+ sample := data
+ if len(sample) > 1024 {
+ sample = sample[:1024]
+ }
+ m := xmlEncodingRe.FindSubmatch(sample)
+ if len(m) > 1 {
+ return string(m[1])
+ }
+ return ""
+}
+
+// extractCharsetHint extracts any explicit charset label from hint,
+// whether hint is a bare charset ("gb2312") or a media type parameter ("text/plain; charset=gb2312").
+func extractCharsetHint(hint string) string {
+ hint = strings.TrimSpace(hint)
+ if hint == "" {
+ return ""
+ }
+ if !strings.Contains(hint, "/") {
+ return hint
+ }
+ if idx := strings.Index(strings.ToLower(hint), "charset="); idx != -1 {
+ cs := hint[idx+len("charset="):]
+ cs = strings.Trim(cs, `"' `)
+ if end := strings.IndexAny(cs, "; \t\r\n"); end != -1 {
+ cs = cs[:end]
+ }
+ return cs
+ }
+ return ""
+}
+
+// DecodeToUTF8 converts arbitrary data bytes into UTF-8.
+// If data is already valid UTF-8, it returns data untouched with encoding "utf-8".
+// hint can be:
+// - An explicit charset label (e.g. "gbk", "gb2312", "big5", from MIME headers)
+// - A MIME Content-Type (e.g. "text/html", "application/xhtml+xml", "text/csv", "text/plain")
+// - Empty string ""
+//
+// It returns the decoded UTF-8 bytes and the canonical encoding label used.
+func DecodeToUTF8(data []byte, hint string) ([]byte, string) {
+ if len(data) == 0 {
+ return data, "utf-8"
+ }
+
+ // 1. Explicit charset declaration from hint (e.g. email MIME headers).
+ // If the caller explicitly declared a non-UTF8 charset, prioritize trying it first.
+ // This correctly handles 7-bit ASCII escape encodings like HZ-GB-2312, where bytes are
+ // valid ASCII (thus valid UTF-8) but represent encoded Chinese escape sequences.
+ // We verify the charset is recognized so unrecognized labels (e.g. "unknown-8bit")
+ // do not corrupt UTF-8 text or bypass validation.
+ if cs := extractCharsetHint(hint); cs != "" && canonicalCharsetLabel(cs) != "utf8" && isRecognizedCharset(cs) {
+ if decoded, err := decodeWithCharset(data, cs); err == nil {
+ return []byte(decoded), canonicalCharsetLabel(cs)
+ }
+ }
+
+ // 2. BOM prescan (Byte Order Mark).
+ // RFC 7303 §3.2 and XML specs define BOM as the authoritative physical encoding signature
+ // that overrides conflicting in-band text declarations (e.g. UTF-8 BOM with <?xml encoding="ISO-8859-1"?>).
+ // When contentType is "", DetermineEncoding reports certain=true strictly when a physical BOM is present.
+ if enc, name, certain := htmlcharset.DetermineEncoding(data, ""); certain && enc != nil {
+ if decoded, err := decodeTransform(data, enc.NewDecoder()); err == nil {
+ return []byte(decoded), name
+ }
+ }
+
+ // 3. XML declaration check (e.g. <?xml version="1.0" encoding="GB2312"?> in EPUB XHTML).
+ // Evaluated before the UTF-8 fast-path so that 7-bit ASCII-compatible escape encodings
+ // like HZ-GB-2312 declared in XML are decoded rather than treated as valid ASCII UTF-8.
+ if xmlEnc := declaredXMLEncoding(data); xmlEnc != "" && canonicalCharsetLabel(xmlEnc) != "utf8" && isRecognizedCharset(xmlEnc) {
+ if decoded, err := decodeWithCharset(data, xmlEnc); err == nil {
+ return []byte(decoded), canonicalCharsetLabel(xmlEnc)
+ }
+ }
+
+ // 4. Fast-path: already valid UTF-8.
+ if utf8.Valid(data) {
+ return data, "utf-8"
+ }
+
+ // 5. Document prescan via DetermineEncoding (handles HTML meta charset, and Content-Type)
+ contentType := ""
+ if strings.Contains(hint, "/") {
+ contentType = hint
+ }
+ if enc, name, _ := htmlcharset.DetermineEncoding(data, contentType); enc != nil &&
+ (name != "windows-1252" || declaresWindows1252(data)) {
+ if decoded, err := decodeTransform(data, enc.NewDecoder()); err == nil {
+ return []byte(decoded), name
+ }
+ }
+
+ // 6. Statistical detection with chardet (confidence >= 90)
+ if label := detectedCharset(data); label != "" {
+ if decoded, err := decodeWithCharset(data, label); err == nil {
+ return []byte(decoded), label
+ }
+ }
+
+ // 7. Fallback chain: gb18030 -> big5 -> shift_jis -> euc-kr -> iso-8859-1
+ if decoded, label, ok := decodeFirstCharsetMatch(data, fallbackCharsetLabels); ok {
+ return []byte(decoded), label
+ }
+
+ // Defensive fallback: unreachable for non-empty input as terminal ISO-8859-1
+ // accepts every byte sequence without producing replacement runes.
+ return data, ""
+}
diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.go
index 83be10c042..cacd43c796 100644
--- a/internal/parser/parser/charset_test.go
+++ b/internal/parser/parser/charset_test.go
@@ -1,6 +1,19 @@
package parser
-import "testing"
+import (
+ "archive/zip"
+ "bytes"
+ "context"
+ "strings"
+ "testing"
+ "unicode/utf8"
+
+ "golang.org/x/text/encoding/japanese"
+ "golang.org/x/text/encoding/korean"
+ "golang.org/x/text/encoding/simplifiedchinese"
+ "golang.org/x/text/encoding/traditionalchinese"
+ "golang.org/x/text/encoding/unicode"
+)
// TestCharsetEncodingSharedLabelContract locks the single-label contract of
// charsetEncoding: every fallback label the HTML parser may walk (and the
@@ -11,7 +24,7 @@ import "testing"
// drift apart again.
func TestCharsetEncodingSharedLabelContract(t *testing.T) {
mailLabels := []string{"gb2312", "gbk", "gb18030", "hz-gb-2312", "big5", "shift_jis", "sjis", "euc-kr", "iso-8859-1", "latin1"}
- for _, label := range append(mailLabels, htmlCharsetFallbackLabels...) {
+ for _, label := range append(mailLabels, fallbackCharsetLabels...) {
if _, ok := charsetEncoding(label); !ok {
t.Errorf("charsetEncoding(%q) = (_, false); the shared resolver must cover every mail-chain and HTML-fallback label", label)
}
@@ -60,3 +73,221 @@ func TestDecodeFirstCharsetMatchAcceptance(t *testing.T) {
t.Error("decodeFirstCharsetMatch over a rejecting candidate list must report failure")
}
}
+
+func TestDecodeToUTF8(t *testing.T) {
+ // 1. Empty input passes through
+ out, label := DecodeToUTF8(nil, "")
+ if len(out) != 0 || label != "utf-8" {
+ t.Errorf("empty input: got (%d, %q), want (0, utf-8)", len(out), label)
+ }
+
+ // 2. Already UTF-8 passes through without alteration
+ utf8Str := "你好,世界!This is valid UTF-8 text."
+ out, label = DecodeToUTF8([]byte(utf8Str), "")
+ if string(out) != utf8Str || label != "utf-8" {
+ t.Errorf("UTF-8 input: got %q (%q), want %q (utf-8)", string(out), label, utf8Str)
+ }
+
+ // 3. GBK / GB2312 text
+ chinese := "自然语言处理与知识图谱构建"
+ gbkBytes, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(chinese))
+ if err != nil {
+ t.Fatalf("GBK encode: %v", err)
+ }
+ out, label = DecodeToUTF8(gbkBytes, "text/plain")
+ if string(out) != chinese {
+ t.Errorf("GBK decode: got %q, want %q", string(out), chinese)
+ }
+ if label != "gb18030" && label != "gbk" && label != "gb2312" {
+ t.Errorf("GBK label: got %q, want gb18030/gbk/gb2312", label)
+ }
+
+ // 4. Big5 text
+ traditional := "繁體中文測試資料"
+ big5Bytes, err := traditionalchinese.Big5.NewEncoder().Bytes([]byte(traditional))
+ if err != nil {
+ t.Fatalf("Big5 encode: %v", err)
+ }
+ out, label = DecodeToUTF8(big5Bytes, "")
+ if string(out) != traditional {
+ t.Errorf("Big5 decode: got %q, want %q", string(out), traditional)
+ }
+
+ // 5. Shift-JIS Japanese text
+ japaneseText := "日本語テキストのテストです"
+ sjisBytes, err := japanese.ShiftJIS.NewEncoder().Bytes([]byte(japaneseText))
+ if err != nil {
+ t.Fatalf("Shift-JIS encode: %v", err)
+ }
+ out, _ = DecodeToUTF8(sjisBytes, "")
+ if string(out) != japaneseText {
+ t.Errorf("Shift-JIS decode: got %q, want %q", string(out), japaneseText)
+ }
+
+ // 6. EUC-KR Korean text
+ koreanText := "한국어 시험: 이 문서는 EUC-KR로 인코딩되어 있습니다. 서울특별시."
+ euckrBytes, err := korean.EUCKR.NewEncoder().Bytes([]byte(koreanText))
+ if err != nil {
+ t.Fatalf("EUC-KR encode: %v", err)
+ }
+ out, _ = DecodeToUTF8(euckrBytes, "")
+ if string(out) != koreanText {
+ t.Errorf("EUC-KR decode: got %q, want %q", string(out), koreanText)
+ }
+
+ // 7. XML declaration check
+ xmlHeader := `<?xml version="1.0" encoding="GB2312"?><html><body><p>章节测试</p></body></html>`
+ xmlGBK, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(xmlHeader))
+ if err != nil {
+ t.Fatalf("XML GBK encode: %v", err)
+ }
+ out, label = DecodeToUTF8(xmlGBK, "application/xhtml+xml")
+ if !strings.Contains(string(out), "章节测试") {
+ t.Errorf("XML declaration decode: missing expected text in %q", string(out))
+ }
+ if label != "gb2312" && label != "gb18030" && label != "gbk" {
+ t.Errorf("XML declaration label: got %q", label)
+ }
+}
+
+func TestCSVParser_GBK(t *testing.T) {
+ csvText := "产品,分类,价格,备注\n笔记本电脑,电子,6999.00,\"特价销售,送鼠标\"\n无线耳机,配件,399.00,\"热销\"\n"
+ gbkBytes, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(csvText))
+ if err != nil {
+ t.Fatalf("GBK encode: %v", err)
+ }
+
+ p := NewCSVParser()
+ res := p.ParseWithResult(context.Background(), "products.csv", gbkBytes)
+ if res.Err != nil {
+ t.Fatalf("CSVParser.ParseWithResult failed: %v", res.Err)
+ }
+ if strings.ContainsRune(res.HTML, '\ufffd') {
+ t.Errorf("CSV HTML output contains replacement runes: %s", res.HTML)
+ }
+ for _, want := range []string{"笔记本电脑", "电子", "特价销售,送鼠标", "无线耳机"} {
+ if !strings.Contains(res.HTML, want) {
+ t.Errorf("CSV HTML missing expected substring %q; got: %s", want, res.HTML)
+ }
+ }
+ if enc, _ := res.File["encoding"].(string); enc != "gb18030" && enc != "gbk" && enc != "gb2312" {
+ t.Errorf("CSV File.encoding = %q, want gbk/gb2312/gb18030", enc)
+ }
+}
+
+func TestEPUBParser_GB2312(t *testing.T) {
+ // Create an in-memory EPUB zip with GB2312-encoded chapter XHTML
+ chapterContent := `<?xml version="1.0" encoding="GB2312"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head><title>第一章</title></head>
+<body><h1>第一章 破晓</h1><p>这是一段包含中文的EPUB正文内容。</p></body>
+</html>`
+ chapterGBK, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(chapterContent))
+ if err != nil {
+ t.Fatalf("GBK encode: %v", err)
+ }
+
+ containerXML := `<?xml version="1.0"?><container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container"><rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/></rootfiles></container>`
+ contentOPF := `<?xml version="1.0" encoding="utf-8"?><package xmlns="http://www.idpf.org/2007/opf" unique-identifier="BookID" version="2.0"><metadata xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:title>测试</dc:title></metadata><manifest><item id="c1" href="c1.xhtml" media-type="application/xhtml+xml"/></manifest><spine><itemref idref="c1"/></spine></package>`
+
+ buf := new(bytes.Buffer)
+ zw := zip.NewWriter(buf)
+ w, _ := zw.Create("META-INF/container.xml")
+ w.Write([]byte(containerXML))
+ w, _ = zw.Create("OEBPS/content.opf")
+ w.Write([]byte(contentOPF))
+ w, _ = zw.Create("OEBPS/c1.xhtml")
+ w.Write(chapterGBK)
+ zw.Close()
+
+ p := NewEPUBParser()
+ res := p.ParseWithResult(context.Background(), "book.epub", buf.Bytes())
+ if res.Err != nil {
+ t.Fatalf("EPUBParser.ParseWithResult failed: %v", res.Err)
+ }
+ if len(res.JSON) == 0 {
+ t.Fatal("want non-empty JSON items from EPUB")
+ }
+ text, _ := res.JSON[0]["text"].(string)
+ if strings.ContainsRune(text, '\ufffd') {
+ t.Errorf("EPUB text contains replacement runes: %s", text)
+ }
+ for _, want := range []string{"第一章 破晓", "这是一段包含中文的EPUB正文内容。"} {
+ if !strings.Contains(text, want) {
+ t.Errorf("EPUB text missing %q; got: %s", want, text)
+ }
+ }
+ if enc, _ := res.File["encoding"].(string); enc != "gb2312" && enc != "gbk" && enc != "gb18030" {
+ t.Errorf("EPUB File.encoding = %q, want gb2312/gbk/gb18030", enc)
+ }
+}
+
+func TestDecodeToUTF8_HZGB2312Hint(t *testing.T) {
+ // HZ-GB-2312 escape sequences (~{...~}) are pure 7-bit ASCII, hence valid UTF-8.
+ // An explicit hint must decode via HZ-GB-2312 rather than returning bytes untouched.
+ hz := []byte("~{<4Ky~}")
+ out, label := DecodeToUTF8(hz, "hz-gb-2312")
+ if label != "hzgb2312" && label != "hz-gb-2312" {
+ t.Errorf("got label %q, want hzgb2312 or hz-gb-2312", label)
+ }
+ if string(out) == string(hz) {
+ t.Fatal("hint ignored; utf8 fast-path swallowed HZ escapes")
+ }
+ if !utf8.Valid(out) {
+ t.Fatal("decoded HZ output must be valid UTF-8")
+ }
+}
+
+func TestDecodeToUTF8_XMLEncoding_HZGB2312(t *testing.T) {
+ // XML encoding declaration for HZ-GB-2312 must be evaluated before the UTF-8 fast path.
+ xml := []byte("<?xml version=\"1.0\" encoding=\"HZ-GB-2312\"?><html><body>~{<4Ky~}</body></html>")
+ out, label := DecodeToUTF8(xml, "application/xhtml+xml")
+ if label != "hzgb2312" && label != "hz-gb-2312" {
+ t.Errorf("got label %q, want hzgb2312", label)
+ }
+ if string(out) == string(xml) {
+ t.Fatal("XML HZ-GB-2312 declaration ignored; utf8 fast-path swallowed HZ escapes")
+ }
+}
+
+func TestDecodeToUTF8_UnrecognizedHint_UTF8Payload(t *testing.T) {
+ // Unrecognized charset labels (e.g. unknown-8bit from MIME) must not trigger
+ // a bogus latin-1 decode or corrupt valid UTF-8 input.
+ utf8Text := []byte("这是一段标准的UTF-8中文内容")
+ out, label := DecodeToUTF8(utf8Text, "unknown-8bit")
+ if label != "utf-8" {
+ t.Errorf("got label %q, want utf-8", label)
+ }
+ if string(out) != string(utf8Text) {
+ t.Errorf("unrecognized hint corrupted UTF-8 text; got %q, want %q", string(out), string(utf8Text))
+ }
+}
+
+func TestDecodeToUTF8_BOMOverridesConflictingXMLDeclaration(t *testing.T) {
+ // RFC 7303: UTF-8 BOM (\xef\xbb\xbf) takes precedence over a conflicting XML declaration (e.g. ISO-8859-1).
+ conflictXML := append([]byte("\xef\xbb\xbf"), []byte(`<?xml version="1.0" encoding="ISO-8859-1"?><html><body>中文测试内容</body></html>`)...)
+ out, label := DecodeToUTF8(conflictXML, "application/xhtml+xml")
+ if label != "utf-8" {
+ t.Errorf("got label %q, want utf-8", label)
+ }
+ if !strings.Contains(string(out), "中文测试内容") {
+ t.Fatalf("UTF-8 BOM was overridden by XML ISO-8859-1; got: %s", string(out))
+ }
+}
+
+func TestDecodeToUTF8_UTF16LEBOM(t *testing.T) {
+ // UTF-16LE with BOM (\xff\xfe) must decode to valid UTF-8.
+ src := "UTF-16LE 中文测试"
+ utf16Bytes, err := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder().Bytes([]byte(src))
+ if err != nil {
+ t.Fatalf("UTF-16 encode: %v", err)
+ }
+ out, label := DecodeToUTF8(utf16Bytes, "text/plain")
+ if label != "utf-16le" {
+ t.Errorf("got label %q, want utf-16le", label)
+ }
+ if !strings.Contains(string(out), src) {
+ t.Errorf("got %q, want containing %q", string(out), src)
+ }
+}
diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.go
index 11297a8661..57d6208731 100644
--- a/internal/parser/parser/csv_parser.go
+++ b/internal/parser/parser/csv_parser.go
@@ -125,14 +125,15 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
// for CSV processing.
}
- text := string(data)
+ decoded, encName := DecodeToUTF8(data, "text/csv")
+ text := string(decoded)
if strings.TrimSpace(text) == "" {
return ParseResult{
OutputFormat: "html",
File: map[string]any{
"name": filename,
"size": len(data),
- "encoding": "utf-8",
+ "encoding": encName,
},
HTML: "<table><caption>" + csvSheetName + "</caption><tr><td></td></tr></table>",
}
@@ -161,7 +162,7 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
File: map[string]any{
"name": filename,
"size": len(data),
- "encoding": "utf-8",
+ "encoding": encName,
},
HTML: recordsToHTMLTableChunks(records, chunkRows, csvSheetName),
}
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.go
index 67436fc0e7..70923ae4f9 100644
--- a/internal/parser/parser/email_parser.go
+++ b/internal/parser/parser/email_parser.go
@@ -511,26 +511,13 @@ func walkHTMLBodyText(n *html.Node, w *leafWriter) {
}
}
-// decodeMailPayload attempts multiple charset decodings.
-// Mirrors Python's _decode_payload with fallback chain:
-// utf-8 → gb2312 → gbk → gb18030 → latin1 → utf-8 (ignore).
+// decodeMailPayload attempts charset decoding using the unified DecodeToUTF8 helper.
func decodeMailPayload(payload []byte, charset string) string {
if len(payload) == 0 {
return ""
}
- if decoded, _, ok := decodeFirstCharsetMatch(payload, buildCharsetChain(charset)); ok {
- return decoded
- }
- return string(payload)
-}
-
-func buildCharsetChain(declared string) []string {
- chain := make([]string, 0, 7)
- if declared != "" {
- chain = append(chain, declared)
- }
- chain = append(chain, "utf-8", "gb2312", "gbk", "gb18030", "latin1")
- return chain
+ decoded, _ := DecodeToUTF8(payload, charset)
+ return string(decoded)
}
// parseMSG parses an Outlook .msg (OLE2 compound document) file using the
diff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.go
index 29d2c5eebe..59bd1d701b 100644
--- a/internal/parser/parser/epub_parser.go
+++ b/internal/parser/parser/epub_parser.go
@@ -82,7 +82,7 @@ func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, data
return ParseResult{Err: fmt.Errorf("epub: opf: %w", err)}
}
- items := extractEPUBTextItems(reader, opfPath, spineItems)
+ items, detectedEnc := extractEPUBTextItems(reader, opfPath, spineItems)
if items == nil {
items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
}
@@ -91,7 +91,7 @@ func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, data
File: map[string]any{
"name": filename,
"size": len(data),
- "encoding": "utf-8",
+ "encoding": detectedEnc,
},
JSON: items,
}
@@ -194,10 +194,14 @@ var (
epubWSRe = regexp.MustCompile(`\s+`)
)
-func extractEPUBTextItems(r *zip.Reader, opfDir string, spineHrefs []string) []map[string]any {
+func extractEPUBTextItems(r *zip.Reader, opfDir string, spineHrefs []string) ([]map[string]any, string) {
var items []map[string]any
+ detectedEncoding := "utf-8"
for _, href := range spineHrefs {
- text := readEPUBContentFile(r, opfDir, href)
+ text, enc := readEPUBContentFile(r, opfDir, href)
+ if detectedEncoding == "utf-8" && enc != "" && enc != "utf-8" {
+ detectedEncoding = enc
+ }
if strings.TrimSpace(text) == "" {
continue
}
@@ -206,13 +210,13 @@ func extractEPUBTextItems(r *zip.Reader, opfDir string, spineHrefs []string) []m
"doc_type_kwd": "text",
})
}
- return items
+ return items, detectedEncoding
}
// readEPUBContentFile resolves a spine href (relative to the OPF
// directory) inside the ZIP, reads the raw bytes, and strips HTML to
-// return clean text.
-func readEPUBContentFile(r *zip.Reader, opfDir, href string) string {
+// return clean text and detected encoding.
+func readEPUBContentFile(r *zip.Reader, opfDir, href string) (string, string) {
// Resolve href relative to the directory containing the OPF.
// Use path.Join/Dir (slash separator) because ZIP entry paths are always POSIX.
resolved := path.Join(path.Dir(opfDir), href)
@@ -230,15 +234,16 @@ func readEPUBContentFile(r *zip.Reader, opfDir, href string) string {
}
}
if err != nil {
- return ""
+ return "", ""
}
defer f.Close()
raw, err := io.ReadAll(f)
if err != nil {
- return ""
+ return "", ""
}
- return stripHTMLTags(string(raw))
+ decoded, enc := DecodeToUTF8(raw, "application/xhtml+xml")
+ return stripHTMLTags(string(decoded)), enc
}
// stripHTMLTags removes HTML markup and returns normalized plain text.
diff --git a/internal/parser/parser/html_charset_test.go b/internal/parser/parser/html_charset_test.go
index 6fcc981a5e..b42f6f58b6 100644
--- a/internal/parser/parser/html_charset_test.go
+++ b/internal/parser/parser/html_charset_test.go
@@ -4,6 +4,7 @@ import (
"context"
"strings"
"testing"
+ "unicode/utf8"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/charmap"
@@ -30,14 +31,14 @@ func gbkHTML(t *testing.T, withMeta bool) []byte {
"<tr><td>应收账款</td><td>1200000000.00</td></tr></table>" +
"<p>单位:元</p></body></html>"
src := head + body
- if !utf8Valid([]byte(src)) {
+ if !utf8.Valid([]byte(src)) {
t.Fatal("test source must be valid UTF-8 before re-encoding")
}
out, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(src))
if err != nil {
t.Fatalf("GBK encode: %v", err)
}
- if utf8Valid(out) {
+ if utf8.Valid(out) {
t.Fatal("GBK fixture unexpectedly decodes as UTF-8; fixture is ineffective")
}
return out
@@ -52,7 +53,7 @@ func htmlEncodedFixture(t *testing.T, src string, enc encoding.Encoding) []byte
if err != nil {
t.Fatalf("fixture encode: %v", err)
}
- if utf8Valid(raw) {
+ if utf8.Valid(raw) {
t.Fatal("fixture unexpectedly decodes as UTF-8; fixture is ineffective")
}
return raw
@@ -116,7 +117,7 @@ func TestHTMLParser_DecodesBig5MetaDeclared(t *testing.T) {
if err != nil {
t.Fatalf("Big5 encode: %v", err)
}
- if utf8Valid(raw) {
+ if utf8.Valid(raw) {
t.Fatal("Big5 fixture unexpectedly decodes as UTF-8; fixture is ineffective")
}
res := NewHTMLParser().ParseWithResult(context.Background(), "tw.html", raw)
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index c884078c9f..ae34b6e29f 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -20,13 +20,9 @@ import (
"bytes"
"context"
"fmt"
- "regexp"
"strings"
- "github.com/saintfish/chardet"
-
"golang.org/x/net/html"
- "golang.org/x/net/html/charset"
)
type HTMLParser struct {
@@ -110,136 +106,10 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
}
}
-// htmlCharsetFallbackLabels is the ordered brute-force decode loop for HTML
-// bytes that are neither valid UTF-8 nor carrying a BOM / <meta charset>
-// declaration, mirroring the pragmatic core of Python's find_codec
-// (rag/nlp/__init__.py): legacy pages that omit a charset declaration are
-// overwhelmingly GBK-family (saved-by-browser Chinese finance/gov pages), so
-// GB18030 (the GBK superset) leads. Big5 / Shift-JIS / EUC-KR pages
-// virtually always declare their charset and are caught by the meta prescan;
-// when the declaration is missing, the statistical detector
-// (htmlDetectedCharset) reorders the attempt so they are not silently
-// absorbed by GB18030. ISO-8859-1 is intentionally terminal, the way latin1
-// terminates Python's codec loop: it decodes ANY byte sequence without
-// replacement runes, so once the chain reaches it, it always succeeds —
-// Western pages without a declaration still yield readable text instead of
-// U+FFFD soup. Labels resolve through the shared charsetEncoding in
-// charset.go; only the HTML-specific ordering lives here.
-var htmlCharsetFallbackLabels = []string{"gb18030", "big5", "shift_jis", "euc-kr", "iso-8859-1"}
-
-// htmlCharsetDetectConfidence is the minimum statistical-detection
-// confidence (on chardet's 0-100 scale) allowed to override the fallback
-// chain order. Calibrated empirically: genuine CJK text scores 100, while
-// the Go chardet port produces plausible-but-wrong guesses at low confidence
-// (a sparse GBK balance-sheet page scores Big5 at 40; a short Big5 page
-// scores ISO-8859-1 at 17). Only a confident pick may jump the queue;
-// anything weaker keeps the GB18030-first domain heuristic.
-const htmlCharsetDetectConfidence = 90
-
-// htmlDetectedCharset mirrors Python find_codec's chardet.detect step for
-// HTML bytes that are neither valid UTF-8 nor declared: a statistical
-// detector picks which fallback-chain label to try FIRST, returned as one
-// of htmlCharsetFallbackLabels. This matters because GB18030 maps nearly
-// every byte sequence without replacement runes, so the ordered chain alone
-// cannot recognize an undeclared Big5 / Shift-JIS / EUC-KR page — it would
-// "decode" it as wrong Chinese text. Detection runs over the first 1024
-// bytes, exactly like Python, but unlike Python it is gated on
-// htmlCharsetDetectConfidence: the Go port's low-confidence guesses are
-// demonstrably wrong often enough that they must not override the chain.
-// A detection that fails, scores below the gate, or names an encoding the
-// chain does not ship (e.g. windows-1252) returns "" and the chain order
-// stands.
-func htmlDetectedCharset(data []byte) string {
- sample := data
- if len(sample) > 1024 {
- sample = sample[:1024]
- }
- res, err := chardet.NewTextDetector().DetectBest(sample)
- if err != nil || res == nil || res.Charset == "" || res.Confidence < htmlCharsetDetectConfidence {
- return ""
- }
- detected := canonicalCharsetLabel(res.Charset)
- // The detector reports the GBK family as GB2312 / GBK; GB18030 is the
- // superset we ship, so both map onto the gb18030 candidate.
- if detected == "gb2312" || detected == "gbk" {
- detected = "gb18030"
- }
- for _, label := range htmlCharsetFallbackLabels {
- if canonicalCharsetLabel(label) == detected {
- return label
- }
- }
- return ""
-}
-
-// htmlMetaCharsetRe captures the charset label of a <meta> tag, matching both
-// the <meta charset="..."> form and the http-equiv content-type form
-// (<meta ... content="text/html; charset=...">), over the same first-1024-byte
-// window x/net's prescan consumes.
-var htmlMetaCharsetRe = regexp.MustCompile(`(?i)<meta[^>]*charset\s*=\s*["']?\s*([a-z0-9._+\-]+)`)
-
-// htmlDeclaresWindows1252 reports whether data's <meta> tags declare
-// windows-1252, either directly or via a WHATWG alias such as iso-8859-1 /
-// us-ascii that charset.Lookup canonicalizes to windows-1252.
-// charset.DetermineEncoding cannot tell a declared windows-1252 from its own
-// "nothing declared" default: a meta prescan hit never sets certain (only
-// BOMs do), so both come back as (windows-1252, certain=false). The
-// declaration must therefore be confirmed explicitly, or a declared
-// windows-1252 page is mis-decoded by the fallback chain: windows-1252's
-// 0x80-0x9F range (smart quotes, €, ™) is C1 control characters in the
-// chain's ISO-8859-1 terminal.
-func htmlDeclaresWindows1252(data []byte) bool {
- if len(data) > 1024 {
- data = data[:1024]
- }
- for _, m := range htmlMetaCharsetRe.FindAllSubmatch(data, -1) {
- if _, name := charset.Lookup(string(m[1])); name == "windows-1252" {
- return true
- }
- }
- return false
-}
-
// decodeHTMLToUTF8 converts non-UTF-8 HTML bytes to UTF-8 and reports the
-// encoding label used. Valid UTF-8 passes through untouched. Otherwise the
-// document's own BOM or <meta charset> declaration wins — including a
-// declared windows-1252: DetermineEncoding returns that name both for a real
-// declaration and as its "nothing declared" default, with certain=false
-// either way, so htmlDeclaresWindows1252 disambiguates the two. Undeclared
-// documents first honor a high-confidence statistical pick
-// (htmlDetectedCharset) and then walk the fallback chain
-// (htmlCharsetFallbackLabels via the shared decodeFirstCharsetMatch), where
-// each candidate must decode the whole blob without producing replacement
-// runes. If everything fails the original bytes are returned so behavior
-// never regresses below the previous UTF-8-only parse.
+// encoding label used. It delegates to the shared DecodeToUTF8 helper.
func decodeHTMLToUTF8(data []byte) ([]byte, string) {
- if len(data) == 0 || utf8Valid(data) {
- return data, "utf-8"
- }
- // The prescan declaration wins for every name except the undeclared
- // windows-1252 sentinel: skipping the sentinel keeps undeclared pages on
- // the detect-then-chain path, while a declared windows-1252 (or its
- // iso-8859-1 / us-ascii aliases) is honored.
- if enc, name, _ := charset.DetermineEncoding(data, ""); enc != nil &&
- (name != "windows-1252" || htmlDeclaresWindows1252(data)) {
- if decoded, err := decodeTransform(data, enc.NewDecoder()); err == nil {
- return []byte(decoded), name
- }
- }
- if label := htmlDetectedCharset(data); label != "" {
- if decoded, err := decodeWithCharset(data, label); err == nil {
- return []byte(decoded), label
- }
- }
- if decoded, label, ok := decodeFirstCharsetMatch(data, htmlCharsetFallbackLabels); ok {
- return []byte(decoded), label
- }
- // Unreachable for non-empty input: the terminal ISO-8859-1 candidate
- // decodes every byte sequence without replacement runes, so the loop
- // above always returns first (the label is therefore neutral rather
- // than a bogus "utf-8" — the bytes are non-UTF-8 by construction).
- // Kept as a never-worse-than-before safety net.
- return data, ""
+ return DecodeToUTF8(data, "text/html")
}
// walkHTMLBlocks emits one normalized item per block-level
diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.go
index a33d274a99..7fe65d9839 100644
--- a/internal/parser/parser/parse_with_result_test.go
+++ b/internal/parser/parser/parse_with_result_test.go
@@ -38,6 +38,7 @@ import (
"testing"
"golang.org/x/net/html"
+ "golang.org/x/text/encoding/simplifiedchinese"
"ragflow/internal/utility"
)
@@ -107,16 +108,28 @@ func TestTextParser_ParseWithResult_NoSizeCap(t *testing.T) {
}
}
-// TestTextParser_ParseWithResult_InvalidUTF8 pins the UTF-8
-// validation rule. Invalid bytes produce an error in the result
-// (matching the python TxtParser's behaviour).
-func TestTextParser_ParseWithResult_InvalidUTF8(t *testing.T) {
+// TestTextParser_ParseWithResult_GBK pins that non-UTF-8 (e.g. GBK) text input
+// is decoded to valid UTF-8 rather than returning an error.
+func TestTextParser_ParseWithResult_GBK(t *testing.T) {
ctx := t.Context()
p := NewTextParser()
- bad := []byte{0xff, 0xfe, 0xfd}
- res := p.ParseWithResult(ctx, "bad.txt", bad)
- if res.Err == nil {
- t.Fatal("want error for invalid UTF-8, got nil")
+ gbkBytes, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte("测试中文文本内容"))
+ if err != nil {
+ t.Fatalf("GBK encode: %v", err)
+ }
+ res := p.ParseWithResult(ctx, "chinese_gbk.txt", gbkBytes)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult(GBK): unexpected error %v", res.Err)
+ }
+ if len(res.JSON) == 0 {
+ t.Fatal("want non-empty items")
+ }
+ got, _ := res.JSON[0]["text"].(string)
+ if got != "测试中文文本内容" {
+ t.Errorf("got %q, want %q", got, "测试中文文本内容")
+ }
+ if enc, _ := res.File["encoding"].(string); enc != "gb18030" && enc != "gbk" {
+ t.Errorf("File.encoding = %q, want gb18030 or gbk", enc)
}
}
diff --git a/internal/parser/parser/text_parser.go b/internal/parser/parser/text_parser.go
index 188f4a5bfb..4a88d9b856 100644
--- a/internal/parser/parser/text_parser.go
+++ b/internal/parser/parser/text_parser.go
@@ -57,10 +57,8 @@ func NewTextParser() *TextParser {
// non-empty JSON payload even for an empty input (mirrors the
// MarkdownParser convention at markdown_parser.go:71-76).
func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
- if !utf8Valid(data) {
- return ParseResult{Err: errInvalidUTF8}
- }
- items := textParserItems(data)
+ decoded, encName := DecodeToUTF8(data, "text/plain")
+ items := textParserItems(decoded)
if items == nil {
items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
}
@@ -69,7 +67,7 @@ func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data
File: map[string]any{
"name": filename,
"size": len(data),
- "encoding": "utf-8",
+ "encoding": encName,
},
JSON: items,
}
@@ -79,62 +77,6 @@ func (p *TextParser) String() string {
return "TextParser"
}
-// errInvalidUTF8 is returned when the input bytes fail UTF-8
-// validation. Matches the python TxtParser's behaviour of
-// surfacing a clear error rather than emitting replacement bytes.
-var errInvalidUTF8 = errInvalidUTF8Sentinel("parser: text input is not valid UTF-8")
-
-type errInvalidUTF8Sentinel string
-
-func (e errInvalidUTF8Sentinel) Error() string { return string(e) }
-
-// utf8Valid is a tiny stdlib-free validator. We avoid
-// unicode/utf8.Valid to keep this file dependency-light; the
-// validation rule is the same (decode without rejecting bytes).
-func utf8Valid(data []byte) bool {
- for i := 0; i < len(data); {
- r, size := decodeRune(data[i:])
- if r == 0xFFFD && size == 1 {
- return false
- }
- i += size
- }
- return true
-}
-
-// decodeRune is a minimal UTF-8 decoder that mirrors
-// utf8.DecodeRune's signature: returns the rune and its byte
-// width. Returns (RuneError, 1) on invalid sequences, matching
-// the stdlib contract.
-func decodeRune(p []byte) (rune, int) {
- if len(p) == 0 {
- return 0xFFFD, 0
- }
- c := p[0]
- switch {
- case c < 0x80:
- return rune(c), 1
- case c < 0xC2:
- return 0xFFFD, 1
- case c < 0xE0:
- if len(p) < 2 || p[1]&0xC0 != 0x80 {
- return 0xFFFD, 1
- }
- return rune(c&0x1F)<<6 | rune(p[1]&0x3F), 2
- case c < 0xF0:
- if len(p) < 3 || p[1]&0xC0 != 0x80 || p[2]&0xC0 != 0x80 {
- return 0xFFFD, 1
- }
- return rune(c&0x0F)<<12 | rune(p[1]&0x3F)<<6 | rune(p[2]&0x3F), 3
- case c < 0xF5:
- if len(p) < 4 || p[1]&0xC0 != 0x80 || p[2]&0xC0 != 0x80 || p[3]&0xC0 != 0x80 {
- return 0xFFFD, 1
- }
- return rune(c&0x07)<<18 | rune(p[1]&0x3F)<<12 | rune(p[2]&0x3F)<<6 | rune(p[3]&0x3F), 4
- }
- return 0xFFFD, 1
-}
-
// defaultTextDelimiterPattern is the regexp alternation of the flow parser's
// default delimiter set DefaultTextCodeDelimiter (rag/flow/parser/parser.py:_code
// → deepdoc TxtParser default "\n!?;。;!?"), each rune re.escape'd to mirror