Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(dataflow): decode non-UTF-8 HTML files before parsing in Go pipeline (#18801)

euvre9d9007745551
diff --git a/go.mod b/go.mod
index f582c20fa7..1dc791b0bd 100644
--- a/go.mod
+++ b/go.mod
@@ -54,6 +54,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/pkoukk/tiktoken-go v0.1.8
github.com/redis/go-redis/v9 v9.18.0
+ github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d
github.com/signintech/gopdf v0.36.1
github.com/siongui/gojianfan v0.0.0-20210926212422-2f175ac615de
github.com/spf13/viper v1.18.2
diff --git a/go.sum b/go.sum
index 523b03116c..ac4b75fe9c 100644
--- a/go.sum
+++ b/go.sum
@@ -480,6 +480,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
+github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=
github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
diff --git a/internal/parser/parser/charset.go b/internal/parser/parser/charset.go
new file mode 100644
index 0000000000..014d7051fa
--- /dev/null
+++ b/internal/parser/parser/charset.go
@@ -0,0 +1,126 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package parser
+
+// Shared charset-label resolution and decode-acceptance helpers. Both
+// email_parser.go (RFC 2047 headers, mail bodies) and html_parser.go
+// (decodeHTMLToUTF8) build on these, so a label resolves to the same
+// decoder everywhere and the "first candidate that decodes without
+// replacement runes" walk exists exactly once.
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "strings"
+
+ "golang.org/x/text/encoding"
+ "golang.org/x/text/encoding/charmap"
+ "golang.org/x/text/encoding/japanese"
+ "golang.org/x/text/encoding/korean"
+ "golang.org/x/text/encoding/simplifiedchinese"
+ "golang.org/x/text/encoding/traditionalchinese"
+ "golang.org/x/text/transform"
+)
+
+// canonicalCharsetLabel canonicalizes a charset label for comparison and
+// lookup: ASCII lower-casing with surrounding whitespace plus '-' and '_'
+// removed, so detector outputs and WHATWG spellings such as "GB-18030",
+// "Shift_JIS" or "EUC-KR" fold onto one canonical form.
+func canonicalCharsetLabel(label string) string {
+ return strings.NewReplacer("-", "", "_", "").Replace(strings.ToLower(strings.TrimSpace(label)))
+}
+
+// charsetEncoding maps a charset label to its decoder, covering the
+// charsets common in real-world mail and legacy HTML beyond utf-8: the
+// Chinese families (gb2312/gbk/gb18030, big5), shift_jis, euc-kr, and
+// latin-1. It is the single label resolver shared by the RFC 2047 header
+// decoder, decodeMailPayload (mail bodies), and decodeHTMLToUTF8 (HTML
+// bytes) so every caller resolves a charset label identically.
+//
+// The "gb2312" label maps to GBK, not HZGB2312: mail labeled gb2312 carries
+// plain 8-bit GB2312 bytes, while HZGB2312 only decodes the 7-bit HZ escape
+// form used under the distinct "hz-gb-2312" label.
+func charsetEncoding(charset string) (encoding.Encoding, bool) {
+ switch canonicalCharsetLabel(charset) {
+ case "gb2312", "gbk":
+ return simplifiedchinese.GBK, true
+ case "gb18030":
+ return simplifiedchinese.GB18030, true
+ case "hzgb2312":
+ return simplifiedchinese.HZGB2312, true
+ case "big5":
+ return traditionalchinese.Big5, true
+ case "shiftjis", "sjis":
+ return japanese.ShiftJIS, true
+ case "euckr":
+ return korean.EUCKR, true
+ case "iso88591", "latin1":
+ return charmap.ISO8859_1, true
+ }
+ return nil, false
+}
+
+// decodeWithCharset decodes payload under one charset label, rejecting the
+// result when it produces replacement runes (U+FFFD). An empty label is
+// treated as utf-8.
+func decodeWithCharset(payload []byte, charset string) (string, error) {
+ switch canonicalCharsetLabel(charset) {
+ case "utf8", "":
+ s := string(payload)
+ if !strings.ContainsRune(s, '\ufffd') {
+ return s, nil
+ }
+ return "", fmt.Errorf("invalid utf-8")
+ }
+ if enc, ok := charsetEncoding(charset); ok {
+ return decodeTransform(payload, enc.NewDecoder())
+ }
+ // Unknown charset: treat as latin-1.
+ runes := make([]rune, len(payload))
+ for i, b := range payload {
+ runes[i] = rune(b)
+ }
+ return string(runes), nil
+}
+
+// decodeTransform decodes payload through decoder, rejecting results that
+// contain replacement runes.
+func decodeTransform(payload []byte, decoder *encoding.Decoder) (string, error) {
+ reader := transform.NewReader(bytes.NewReader(payload), decoder)
+ decoded, err := io.ReadAll(reader)
+ if err != nil {
+ return "", err
+ }
+ if !strings.ContainsRune(string(decoded), '\ufffd') {
+ return string(decoded), nil
+ }
+ return "", fmt.Errorf("decode produced replacement characters")
+}
+
+// decodeFirstCharsetMatch walks the candidate labels in order and returns
+// the first decode that succeeds without replacement runes, along with the
+// label that produced it. It is the shared fallback-chain walker behind
+// both decodeMailPayload and decodeHTMLToUTF8.
+func decodeFirstCharsetMatch(data []byte, labels []string) (string, string, bool) {
+ for _, label := range labels {
+ if decoded, err := decodeWithCharset(data, label); err == nil {
+ return decoded, label, true
+ }
+ }
+ return "", "", false
+}
diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.go
new file mode 100644
index 0000000000..83be10c042
--- /dev/null
+++ b/internal/parser/parser/charset_test.go
@@ -0,0 +1,62 @@
+package parser
+
+import "testing"
+
+// TestCharsetEncodingSharedLabelContract locks the single-label contract of
+// charsetEncoding: every fallback label the HTML parser may walk (and the
+// mail chain ships) resolves through the shared resolver, detector/WHATWG
+// spellings fold onto the same encoder, and labels outside the shared
+// family (e.g. windows-1252, handled by the HTML prescan layer, or utf-8,
+// handled natively by decodeWithCharset) stay out so the two parsers cannot
+// drift apart again.
+func TestCharsetEncodingSharedLabelContract(t *testing.T) {
+ mailLabels := []string{"gb2312", "gbk", "gb18030", "hz-gb-2312", "big5", "shift_jis", "sjis", "euc-kr", "iso-8859-1", "latin1"}
+ for _, label := range append(mailLabels, htmlCharsetFallbackLabels...) {
+ if _, ok := charsetEncoding(label); !ok {
+ t.Errorf("charsetEncoding(%q) = (_, false); the shared resolver must cover every mail-chain and HTML-fallback label", label)
+ }
+ }
+
+ aliases := map[string]string{
+ "GB-18030": "gb18030",
+ "GB2312": "gb2312",
+ "Shift_JIS": "shift_jis",
+ "Shift-JIS": "shift_jis",
+ "EUC-KR": "euc-kr",
+ "ISO-8859-1": "iso-8859-1",
+ "ISO8859-1": "iso-8859-1",
+ "Latin1": "latin1",
+ " Big5 ": "big5",
+ }
+ for alias, label := range aliases {
+ aliasEnc, aliasOK := charsetEncoding(alias)
+ labelEnc, labelOK := charsetEncoding(label)
+ if !aliasOK || !labelOK || aliasEnc != labelEnc {
+ t.Errorf("charsetEncoding(%q) = (%v, %v), charsetEncoding(%q) = (%v, %v); alias spelling must fold onto the same encoder", alias, aliasEnc, aliasOK, label, labelEnc, labelOK)
+ }
+ }
+
+ for _, label := range []string{"windows-1252", "utf-8", "utf-16", "euc-jp", ""} {
+ if _, ok := charsetEncoding(label); ok {
+ t.Errorf("charsetEncoding(%q) = (_, true); labels handled elsewhere (prescan / native utf-8) must not enter the shared resolver", label)
+ }
+ }
+}
+
+// TestDecodeFirstCharsetMatchAcceptance pins the shared walker's contract:
+// the first candidate whose decode produces no replacement runes wins and
+// reports its own label; when every candidate is rejected the walker
+// reports failure so callers apply their own terminal fallback. (utf-8 is
+// the deterministic rejecting candidate: the Big5 bytes are invalid UTF-8.)
+func TestDecodeFirstCharsetMatchAcceptance(t *testing.T) {
+ big5Bytes := []byte{0xA4, 0xA4, 0xA4, 0xE5} // "中文" in Big5
+
+ decoded, label, ok := decodeFirstCharsetMatch(big5Bytes, []string{"utf-8", "big5"})
+ if !ok || label != "big5" || decoded != "中文" {
+ t.Errorf("decodeFirstCharsetMatch(big5) = (%q, %q, %v); want (中文, big5, true)", decoded, label, ok)
+ }
+
+ if _, _, ok := decodeFirstCharsetMatch(big5Bytes, []string{"utf-8"}); ok {
+ t.Error("decodeFirstCharsetMatch over a rejecting candidate list must report failure")
+ }
+}
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.go
index b5995e8b8b..67436fc0e7 100644
--- a/internal/parser/parser/email_parser.go
+++ b/internal/parser/parser/email_parser.go
@@ -33,12 +33,6 @@ import (
"github.com/AkmalOt/gomsg"
"golang.org/x/net/html"
- "golang.org/x/text/encoding"
- "golang.org/x/text/encoding/charmap"
- "golang.org/x/text/encoding/japanese"
- "golang.org/x/text/encoding/korean"
- "golang.org/x/text/encoding/simplifiedchinese"
- "golang.org/x/text/encoding/traditionalchinese"
"golang.org/x/text/transform"
"ragflow/internal/utility"
@@ -218,35 +212,6 @@ func targetFieldsSet(fields []string) map[string]bool {
// -- header decoding (RFC 2047) --
-// charsetEncoding maps a MIME charset label to its decoder, covering the
-// charsets common in real-world mail beyond utf-8: the Chinese families
-// (gb2312/gbk/gb18030, big5), shift_jis, euc-kr, and latin-1. It is shared by
-// the RFC 2047 header decoder and decodeMailPayload so headers and bodies
-// resolve a charset label identically.
-//
-// The "gb2312" label maps to GBK, not HZGB2312: mail labeled gb2312 carries
-// plain 8-bit GB2312 bytes, while HZGB2312 only decodes the 7-bit HZ escape
-// form used under the distinct "hz-gb-2312" label.
-func charsetEncoding(charset string) (encoding.Encoding, bool) {
- switch strings.ToLower(strings.TrimSpace(charset)) {
- case "gb2312", "gbk":
- return simplifiedchinese.GBK, true
- case "gb18030":
- return simplifiedchinese.GB18030, true
- case "hz-gb-2312":
- return simplifiedchinese.HZGB2312, true
- case "big5":
- return traditionalchinese.Big5, true
- case "shift_jis", "shift-jis", "sjis":
- return japanese.ShiftJIS, true
- case "euc-kr":
- return korean.EUCKR, true
- case "iso-8859-1", "iso8859-1", "latin1":
- return charmap.ISO8859_1, true
- }
- return nil, false
-}
-
// rfc2047Decoder decodes RFC 2047 encoded-words (e.g. "=?utf-8?B?...?=") in
// header values. utf-8 is handled natively by mime; the CharsetReader covers
// the non-UTF-8 charsets via charsetEncoding.
@@ -553,16 +518,8 @@ func decodeMailPayload(payload []byte, charset string) string {
if len(payload) == 0 {
return ""
}
- charsets := buildCharsetChain(charset)
- for _, enc := range charsets {
- if enc == "" {
- // raw bytes → already fallthrough
- return string(payload)
- }
- decoded, err := decodeWithCharset(payload, enc)
- if err == nil {
- return decoded
- }
+ if decoded, _, ok := decodeFirstCharsetMatch(payload, buildCharsetChain(charset)); ok {
+ return decoded
}
return string(payload)
}
@@ -576,39 +533,6 @@ func buildCharsetChain(declared string) []string {
return chain
}
-func decodeWithCharset(payload []byte, charset string) (string, error) {
- charset = strings.ToLower(strings.TrimSpace(charset))
- switch charset {
- case "utf-8", "utf8", "":
- s := string(payload)
- if !strings.ContainsRune(s, '\ufffd') {
- return s, nil
- }
- return "", fmt.Errorf("invalid utf-8")
- }
- if enc, ok := charsetEncoding(charset); ok {
- return decodeTransform(payload, enc.NewDecoder())
- }
- // Unknown charset: treat as latin-1.
- runes := make([]rune, len(payload))
- for i, b := range payload {
- runes[i] = rune(b)
- }
- return string(runes), nil
-}
-
-func decodeTransform(payload []byte, decoder *encoding.Decoder) (string, error) {
- reader := transform.NewReader(bytes.NewReader(payload), decoder)
- decoded, err := io.ReadAll(reader)
- if err != nil {
- return "", err
- }
- if !strings.ContainsRune(string(decoded), '\ufffd') {
- return string(decoded), nil
- }
- return "", fmt.Errorf("decode produced replacement characters")
-}
-
// parseMSG parses an Outlook .msg (OLE2 compound document) file using the
// gomsg library, mirroring the Python flow parser's _email() .msg branch
// (rag/flow/parser/parser.py), which parses via extract_msg. The output map
diff --git a/internal/parser/parser/html_charset_test.go b/internal/parser/parser/html_charset_test.go
new file mode 100644
index 0000000000..6fcc981a5e
--- /dev/null
+++ b/internal/parser/parser/html_charset_test.go
@@ -0,0 +1,337 @@
+package parser
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "golang.org/x/text/encoding"
+ "golang.org/x/text/encoding/charmap"
+ "golang.org/x/text/encoding/japanese"
+ "golang.org/x/text/encoding/korean"
+ "golang.org/x/text/encoding/simplifiedchinese"
+ "golang.org/x/text/encoding/traditionalchinese"
+ "golang.org/x/text/encoding/unicode"
+)
+
+// gbkHTML is a Sina-finance-style balance-sheet page encoded in GBK with a
+// GB2312 meta declaration, the shape reported by the "agent data pipeline
+// parses HTML into mojibake" issue. GBK bytes double as GB18030 input.
+func gbkHTML(t *testing.T, withMeta bool) []byte {
+ t.Helper()
+ head := "<html><head><title>贵州茅台(600519)资产负债表</title></head>"
+ if withMeta {
+ head = `<html><head><meta http-equiv="Content-Type" content="text/html; charset=GB2312">` +
+ "<title>贵州茅台(600519)资产负债表</title></head>"
+ }
+ body := "<body><h1>贵州茅台(600519)资产负债表</h1>" +
+ "<table><tr><th>项目</th><th>20251231</th></tr>" +
+ "<tr><td>货币资金</td><td>56000000000.00</td></tr>" +
+ "<tr><td>应收账款</td><td>1200000000.00</td></tr></table>" +
+ "<p>单位:元</p></body></html>"
+ src := head + body
+ if !utf8Valid([]byte(src)) {
+ t.Fatal("test source must be valid UTF-8 before re-encoding")
+ }
+ out, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(src))
+ if err != nil {
+ t.Fatalf("GBK encode: %v", err)
+ }
+ if utf8Valid(out) {
+ t.Fatal("GBK fixture unexpectedly decodes as UTF-8; fixture is ineffective")
+ }
+ return out
+}
+
+// htmlEncodedFixture re-encodes src with enc and guards that the fixture is
+// effective: it must not itself be valid UTF-8, otherwise a decode test would
+// trivially pass through the fast path and prove nothing.
+func htmlEncodedFixture(t *testing.T, src string, enc encoding.Encoding) []byte {
+ t.Helper()
+ raw, err := enc.NewEncoder().Bytes([]byte(src))
+ if err != nil {
+ t.Fatalf("fixture encode: %v", err)
+ }
+ if utf8Valid(raw) {
+ t.Fatal("fixture unexpectedly decodes as UTF-8; fixture is ineffective")
+ }
+ return raw
+}
+
+// itemsText concatenates the text of every item in res, newline separated.
+func itemsText(res ParseResult) string {
+ var sb strings.Builder
+ for _, it := range res.JSON {
+ if s, ok := it["text"].(string); ok {
+ sb.WriteString(s)
+ sb.WriteString("\n")
+ }
+ }
+ return sb.String()
+}
+
+// assertDecodedChinese fails res on a parse error, any U+FFFD replacement
+// rune, or missing expected Chinese substrings.
+func assertDecodedChinese(t *testing.T, res ParseResult) {
+ t.Helper()
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ got := itemsText(res)
+ if strings.ContainsRune(got, '\ufffd') {
+ t.Errorf("output contains U+FFFD replacement runes: %q", got)
+ }
+ for _, want := range []string{"\u8d35\u5dde\u8305\u53f0", "\u8d44\u4ea7\u8d1f\u503a\u8868", "\u8d27\u5e01\u8d44\u91d1", "\u5e94\u6536\u8d26\u6b3e"} {
+ if !strings.Contains(got, want) {
+ t.Errorf("decoded text missing %q; got: %q", want, got)
+ }
+ }
+}
+
+// TestHTMLParser_DecodesGBKMetaDeclared reproduces the reported issue: a
+// GBK-encoded HTML file parsed by the data pipeline surfaced as mojibake
+// because x/net/html assumes UTF-8. The meta charset declaration must drive
+// the decode, mirroring Python's find_codec path.
+func TestHTMLParser_DecodesGBKMetaDeclared(t *testing.T) {
+ res := NewHTMLParser().ParseWithResult(context.Background(), "balance.html", gbkHTML(t, true))
+ assertDecodedChinese(t, res)
+ if enc, _ := res.File["encoding"].(string); enc == "" || enc == "utf-8" {
+ t.Errorf("File.encoding = %q, want the detected non-UTF-8 label", enc)
+ }
+}
+
+// TestHTMLParser_DecodesGBKWithoutDeclaration covers GBK bytes with no meta
+// declaration: the GB18030 fallback chain candidate must decode them.
+func TestHTMLParser_DecodesGBKWithoutDeclaration(t *testing.T) {
+ res := NewHTMLParser().ParseWithResult(context.Background(), "balance.html", gbkHTML(t, false))
+ assertDecodedChinese(t, res)
+}
+
+// TestHTMLParser_DecodesBig5MetaDeclared covers a Big5 page with its meta
+// declaration.
+func TestHTMLParser_DecodesBig5MetaDeclared(t *testing.T) {
+ src := `<html><head><meta charset="big5"></head><body><h1>` +
+ "\u53f0\u7063\u5927\u5b78" + `</h1><p>` + "\u8cc7\u6599\u5eab" + `</p></body></html>`
+ raw, err := traditionalchinese.Big5.NewEncoder().Bytes([]byte(src))
+ if err != nil {
+ t.Fatalf("Big5 encode: %v", err)
+ }
+ if utf8Valid(raw) {
+ t.Fatal("Big5 fixture unexpectedly decodes as UTF-8; fixture is ineffective")
+ }
+ res := NewHTMLParser().ParseWithResult(context.Background(), "tw.html", raw)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ got := itemsText(res)
+ if strings.ContainsRune(got, '\ufffd') {
+ t.Errorf("output contains U+FFFD replacement runes: %q", got)
+ }
+ for _, want := range []string{"\u53f0\u7063\u5927\u5b78", "\u8cc7\u6599\u5eab"} {
+ if !strings.Contains(got, want) {
+ t.Errorf("decoded text missing %q; got: %q", want, got)
+ }
+ }
+}
+
+// TestHTMLParser_UTF8Passthrough ensures valid UTF-8 input is untouched and
+// still reported as utf-8.
+func TestHTMLParser_UTF8Passthrough(t *testing.T) {
+ src := "<html><body><h1>\u8d35\u5dde\u8305\u53f0</h1><p>plain utf-8</p></body></html>"
+ res := NewHTMLParser().ParseWithResult(context.Background(), "u.html", []byte(src))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ got := itemsText(res)
+ if !strings.Contains(got, "plain utf-8") || !strings.Contains(got, "\u8d35\u5dde\u8305\u53f0") {
+ t.Errorf("utf-8 passthrough lost content; got: %q", got)
+ }
+ if enc, _ := res.File["encoding"].(string); enc != "utf-8" {
+ t.Errorf("File.encoding = %q, want utf-8", enc)
+ }
+}
+
+// TestHTMLParser_DecodeTableSurvivesGBK verifies the structured table item
+// (doc_type_kwd:"table") is emitted from the decoded document with its
+// Chinese cell text intact.
+func TestHTMLParser_DecodeTableSurvivesGBK(t *testing.T) {
+ res := NewHTMLParser().ParseWithResult(context.Background(), "balance.html", gbkHTML(t, true))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ var tableText string
+ for _, it := range res.JSON {
+ if it["doc_type_kwd"] == "table" {
+ tableText, _ = it["text"].(string)
+ }
+ }
+ if tableText == "" {
+ t.Fatalf("no structured table item emitted; items: %#v", res.JSON)
+ }
+ if !strings.Contains(tableText, "\u8d27\u5e01\u8d44\u91d1") {
+ t.Errorf("table item missing decoded cell text; got: %q", tableText)
+ }
+}
+
+// assertUndeclaredDecode parses an undeclared (no <meta charset>) document
+// encoded with enc and asserts the original text round-trips: no U+FFFD and
+// every want substring survives, proving the statistical detector steered the
+// decode instead of the GB18030-first chain order hijacking the bytes.
+func assertUndeclaredDecode(t *testing.T, raw []byte, wantLabel string, wants []string) {
+ t.Helper()
+ res := NewHTMLParser().ParseWithResult(context.Background(), "doc.html", raw)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ got := itemsText(res)
+ if strings.ContainsRune(got, '\ufffd') {
+ t.Errorf("output contains U+FFFD replacement runes: %q", got)
+ }
+ for _, want := range wants {
+ if !strings.Contains(got, want) {
+ t.Errorf("decoded text missing %q; got: %q", want, got)
+ }
+ }
+ if enc, _ := res.File["encoding"].(string); enc != wantLabel {
+ t.Errorf("File.encoding = %q, want %q", enc, wantLabel)
+ }
+}
+
+// TestHTMLParser_DecodesBig5WithoutDeclaration pins the undeclared-Big5 risk
+// the review flagged: every Big5 double-byte pair is also a valid GB18030
+// pair, so the ordered chain alone silently decoded Big5 pages as wrong
+// Chinese text. The statistical detector must pick Big5 first (mirroring
+// Python find_codec's chardet step) and round-trip the original text.
+func TestHTMLParser_DecodesBig5WithoutDeclaration(t *testing.T) {
+ src := "<html><body><h1>\u53f0\u7063\u5927\u5b78\u8cc7\u6599\u5eab</h1>" +
+ "<p>\u9019\u662f\u4e00\u4efd\u7e41\u9ad4\u4e2d\u6587\u6e2c\u8a66\u6587\u4ef6\uff0c\u7528\u65bc\u9a57\u8b49\u5b57\u5143\u7de8\u78bc\u5075\u6e2c\u3002</p></body></html>"
+ assertUndeclaredDecode(t, htmlEncodedFixture(t, src, traditionalchinese.Big5), "big5",
+ []string{"\u53f0\u7063\u5927\u5b78\u8cc7\u6599\u5eab", "\u7e41\u9ad4\u4e2d\u6587"})
+}
+
+// TestHTMLParser_DecodesShiftJISWithoutDeclaration covers an undeclared
+// Shift-JIS page: the detector must steer the decode to Shift-JIS rather
+// than letting GB18030 absorb the bytes as mojibake.
+func TestHTMLParser_DecodesShiftJISWithoutDeclaration(t *testing.T) {
+ src := "<html><body><h1>\u65e5\u672c\u8a9e\u306e\u30c6\u30b9\u30c8</h1>" +
+ "<p>\u3053\u308c\u306f\u30b7\u30d5\u30c8JIS\u3067\u30a8\u30f3\u30b3\u30fc\u30c9\u3055\u308c\u305f\u6587\u66f8\u3067\u3059\u3002\u6771\u4eac\u90fd\u5e81\u306e\u8cc7\u6599\u3002</p></body></html>"
+ assertUndeclaredDecode(t, htmlEncodedFixture(t, src, japanese.ShiftJIS), "shift_jis",
+ []string{"\u65e5\u672c\u8a9e\u306e\u30c6\u30b9\u30c8", "\u6771\u4eac\u90fd\u5e81"})
+}
+
+// TestHTMLParser_DecodesEUCKRWithoutDeclaration covers an undeclared EUC-KR
+// page: detection picks the EUC-KR candidate before the chain order runs.
+func TestHTMLParser_DecodesEUCKRWithoutDeclaration(t *testing.T) {
+ src := "<html><body><h1>\ud55c\uad6d\uc5b4 \uc2dc\ud5d8</h1>" +
+ "<p>\uc774 \ubb38\uc11c\ub294 EUC-KR\ub85c \uc778\ucf54\ub529\ub418\uc5b4 \uc788\uc2b5\ub2c8\ub2e4. \uc11c\uc6b8\ud2b9\ubcc4\uc2dc.</p></body></html>"
+ assertUndeclaredDecode(t, htmlEncodedFixture(t, src, korean.EUCKR), "euc-kr",
+ []string{"\ud55c\uad6d\uc5b4 \uc2dc\ud5d8", "\uc11c\uc6b8\ud2b9\ubcc4\uc2dc"})
+}
+
+// TestHTMLParser_Latin1TerminalFallback asserts the ISO-8859-1 terminal
+// guarantee: an undeclared Western page round-trips through ISO-8859-1
+// (either by detection or as the chain's any-byte-decodes terminal), never
+// as U+FFFD soup.
+func TestHTMLParser_Latin1TerminalFallback(t *testing.T) {
+ src := "<html><body><h1>Caf\u00e9 r\u00e9sum\u00e9</h1><p>Na\u00efve fa\u00e7ade \u00e0 gogo.</p></body></html>"
+ assertUndeclaredDecode(t, htmlEncodedFixture(t, src, charmap.ISO8859_1), "iso-8859-1",
+ []string{"Caf\u00e9 r\u00e9sum\u00e9", "Na\u00efve fa\u00e7ade"})
+}
+
+// TestHTMLParser_DecodesUTF16BOM covers the BOM path of the HTML5 prescan
+// (charset.DetermineEncoding): a UTF-16LE document with its BOM decodes even
+// though no <meta charset> is declared — a net-new improvement over Python's
+// find_codec, which has no BOM handling.
+func TestHTMLParser_DecodesUTF16BOM(t *testing.T) {
+ src := "<html><body><h1>\u8cb4\u5dde\u8305\u81fa</h1><p>utf16 bom doc</p></body></html>"
+ raw, err := unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM).NewEncoder().Bytes([]byte(src))
+ if err != nil {
+ t.Fatalf("UTF-16LE encode: %v", err)
+ }
+ raw = append([]byte{0xff, 0xfe}, raw...)
+ res := NewHTMLParser().ParseWithResult(context.Background(), "u16.html", raw)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ got := itemsText(res)
+ if !strings.Contains(got, "\u8cb4\u5dde\u8305\u81fa") || !strings.Contains(got, "utf16 bom doc") {
+ t.Errorf("UTF-16 BOM decode lost content; got: %q", got)
+ }
+}
+
+// TestHTMLParser_EmptyInputNoPanic proves the never-worse-than-before claim
+// at the degenerate boundary: empty input parses without error.
+func TestHTMLParser_EmptyInputNoPanic(t *testing.T) {
+ res := NewHTMLParser().ParseWithResult(context.Background(), "empty.html", nil)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult(empty): %v", res.Err)
+ }
+}
+
+// TestDecodeHTMLToUTF8_NeverWorseThanBefore feeds hostile bytes that no
+// multibyte candidate decodes cleanly: whatever accepts them (in practice
+// the ISO-8859-1 terminal) must produce replacement-free output with a
+// non-empty label, and empty input passes through untouched.
+func TestDecodeHTMLToUTF8_NeverWorseThanBefore(t *testing.T) {
+ out, label := decodeHTMLToUTF8(nil)
+ if len(out) != 0 || label != "utf-8" {
+ t.Errorf("empty input: got (%d bytes, %q), want (0, utf-8)", len(out), label)
+ }
+ raw := []byte{0x81, 0x20, 0xfe, 0x21, 0x8f, 0x39, 0x41, 0x7e}
+ out, label = decodeHTMLToUTF8(raw)
+ if len(out) == 0 || label == "" {
+ t.Errorf("hostile bytes: got (%d bytes, %q), want non-empty decode and label", len(out), label)
+ }
+ if strings.ContainsRune(string(out), '\ufffd') {
+ t.Errorf("hostile bytes decoded to replacement runes: %q", out)
+ }
+}
+
+// TestHTMLParser_DecodesDeclaredWindows1252 pins the declared-windows-1252
+// path: DetermineEncoding returns windows-1252 (certain=false) both for a
+// real declaration and as its "nothing declared" sentinel, so the declaration
+// must be confirmed explicitly. The 0x80-0x9F range (smart quotes, €, ™, —)
+// differs from the ISO-8859-1 terminal, which decodes those bytes as C1
+// control characters instead of punctuation.
+func TestHTMLParser_DecodesDeclaredWindows1252(t *testing.T) {
+ src := `<html><head><meta charset="windows-1252"></head><body>` +
+ "<h1>\u201cSmart\u201d pricing \u2014 20% off</h1>" +
+ "<p>Paste\u2122 caf\u00e9 \u20ac3.50</p></body></html>"
+ raw := htmlEncodedFixture(t, src, charmap.Windows1252)
+ res := NewHTMLParser().ParseWithResult(context.Background(), "win1252.html", raw)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if enc, _ := res.File["encoding"].(string); enc != "windows-1252" {
+ t.Errorf("File.encoding = %q, want windows-1252", enc)
+ }
+ got := itemsText(res)
+ if strings.ContainsRune(got, '\ufffd') {
+ t.Errorf("output contains U+FFFD replacement runes: %q", got)
+ }
+ for _, want := range []string{"\u201cSmart\u201d", "\u2014", "\u2122", "\u20ac"} {
+ if !strings.Contains(got, want) {
+ t.Errorf("decoded text missing %q; got: %q", want, got)
+ }
+ }
+}
+
+// TestHTMLParser_DecodesDeclaredLatin1AliasAsWindows1252 covers the
+// http-equiv pragma form declaring iso-8859-1: WHATWG folds that label (and
+// us-ascii) onto windows-1252, so the page must decode with windows-1252
+// semantics — smart quotes survive instead of becoming C1 controls.
+func TestHTMLParser_DecodesDeclaredLatin1AliasAsWindows1252(t *testing.T) {
+ src := `<html><head><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"></head><body>` +
+ "<p>He said \u201cna\u00efve\u201d \u2014 caf\u00e9 \u2122</p></body></html>"
+ raw := htmlEncodedFixture(t, src, charmap.Windows1252)
+ res := NewHTMLParser().ParseWithResult(context.Background(), "latin1.html", raw)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if enc, _ := res.File["encoding"].(string); enc != "windows-1252" {
+ t.Errorf("File.encoding = %q, want windows-1252", enc)
+ }
+ if got := itemsText(res); !strings.Contains(got, "\u201cna\u00efve\u201d") {
+ t.Errorf("decoded text lost windows-1252 punctuation; got: %q", got)
+ }
+}
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index e6e73c5bf4..c884078c9f 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -20,9 +20,13 @@ import (
"bytes"
"context"
"fmt"
+ "regexp"
"strings"
+ "github.com/saintfish/chardet"
+
"golang.org/x/net/html"
+ "golang.org/x/net/html/charset"
)
type HTMLParser struct {
@@ -67,6 +71,11 @@ func (p *HTMLParser) ConfigureFromSetup(setup map[string]any) {
// separate ck_type — the python HtmlParser collapses inline
// formatting into the parent block's text.
func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
+ // x/net/html assumes UTF-8 input, so a GBK/Big5/Shift-JIS page would
+ // otherwise surface as U+FFFD mojibake. Decode first, mirroring the
+ // Python dataflow path (RAGFlowHtmlParser decodes the blob via
+ // rag.nlp.find_codec before parsing). See decodeHTMLToUTF8.
+ data, encName := decodeHTMLToUTF8(data)
// remove_header_footer: pre-parse strip of <header>/<footer> tags
// and ARIA role=banner/contentinfo elements (mirrors Python
// parser.py:1083-1084 remove_header_footer_html_blob).
@@ -95,12 +104,144 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
OutputFormat: "json",
File: map[string]any{
"name": filename,
- "encoding": "utf-8",
+ "encoding": encName,
},
JSON: items,
}
}
+// htmlCharsetFallbackLabels is the ordered brute-force decode loop for HTML
+// bytes that are neither valid UTF-8 nor carrying a BOM / <meta charset>
+// declaration, mirroring the pragmatic core of Python's find_codec
+// (rag/nlp/__init__.py): legacy pages that omit a charset declaration are
+// overwhelmingly GBK-family (saved-by-browser Chinese finance/gov pages), so
+// GB18030 (the GBK superset) leads. Big5 / Shift-JIS / EUC-KR pages
+// virtually always declare their charset and are caught by the meta prescan;
+// when the declaration is missing, the statistical detector
+// (htmlDetectedCharset) reorders the attempt so they are not silently
+// absorbed by GB18030. ISO-8859-1 is intentionally terminal, the way latin1
+// terminates Python's codec loop: it decodes ANY byte sequence without
+// replacement runes, so once the chain reaches it, it always succeeds —
+// Western pages without a declaration still yield readable text instead of
+// U+FFFD soup. Labels resolve through the shared charsetEncoding in
+// charset.go; only the HTML-specific ordering lives here.
+var htmlCharsetFallbackLabels = []string{"gb18030", "big5", "shift_jis", "euc-kr", "iso-8859-1"}
+
+// htmlCharsetDetectConfidence is the minimum statistical-detection
+// confidence (on chardet's 0-100 scale) allowed to override the fallback
+// chain order. Calibrated empirically: genuine CJK text scores 100, while
+// the Go chardet port produces plausible-but-wrong guesses at low confidence
+// (a sparse GBK balance-sheet page scores Big5 at 40; a short Big5 page
+// scores ISO-8859-1 at 17). Only a confident pick may jump the queue;
+// anything weaker keeps the GB18030-first domain heuristic.
+const htmlCharsetDetectConfidence = 90
+
+// htmlDetectedCharset mirrors Python find_codec's chardet.detect step for
+// HTML bytes that are neither valid UTF-8 nor declared: a statistical
+// detector picks which fallback-chain label to try FIRST, returned as one
+// of htmlCharsetFallbackLabels. This matters because GB18030 maps nearly
+// every byte sequence without replacement runes, so the ordered chain alone
+// cannot recognize an undeclared Big5 / Shift-JIS / EUC-KR page — it would
+// "decode" it as wrong Chinese text. Detection runs over the first 1024
+// bytes, exactly like Python, but unlike Python it is gated on
+// htmlCharsetDetectConfidence: the Go port's low-confidence guesses are
+// demonstrably wrong often enough that they must not override the chain.
+// A detection that fails, scores below the gate, or names an encoding the
+// chain does not ship (e.g. windows-1252) returns "" and the chain order
+// stands.
+func htmlDetectedCharset(data []byte) string {
+ sample := data
+ if len(sample) > 1024 {
+ sample = sample[:1024]
+ }
+ res, err := chardet.NewTextDetector().DetectBest(sample)
+ if err != nil || res == nil || res.Charset == "" || res.Confidence < htmlCharsetDetectConfidence {
+ return ""
+ }
+ detected := canonicalCharsetLabel(res.Charset)
+ // The detector reports the GBK family as GB2312 / GBK; GB18030 is the
+ // superset we ship, so both map onto the gb18030 candidate.
+ if detected == "gb2312" || detected == "gbk" {
+ detected = "gb18030"
+ }
+ for _, label := range htmlCharsetFallbackLabels {
+ if canonicalCharsetLabel(label) == detected {
+ return label
+ }
+ }
+ return ""
+}
+
+// htmlMetaCharsetRe captures the charset label of a <meta> tag, matching both
+// the <meta charset="..."> form and the http-equiv content-type form
+// (<meta ... content="text/html; charset=...">), over the same first-1024-byte
+// window x/net's prescan consumes.
+var htmlMetaCharsetRe = regexp.MustCompile(`(?i)<meta[^>]*charset\s*=\s*["']?\s*([a-z0-9._+\-]+)`)
+
+// htmlDeclaresWindows1252 reports whether data's <meta> tags declare
+// windows-1252, either directly or via a WHATWG alias such as iso-8859-1 /
+// us-ascii that charset.Lookup canonicalizes to windows-1252.
+// charset.DetermineEncoding cannot tell a declared windows-1252 from its own
+// "nothing declared" default: a meta prescan hit never sets certain (only
+// BOMs do), so both come back as (windows-1252, certain=false). The
+// declaration must therefore be confirmed explicitly, or a declared
+// windows-1252 page is mis-decoded by the fallback chain: windows-1252's
+// 0x80-0x9F range (smart quotes, €, ™) is C1 control characters in the
+// chain's ISO-8859-1 terminal.
+func htmlDeclaresWindows1252(data []byte) bool {
+ if len(data) > 1024 {
+ data = data[:1024]
+ }
+ for _, m := range htmlMetaCharsetRe.FindAllSubmatch(data, -1) {
+ if _, name := charset.Lookup(string(m[1])); name == "windows-1252" {
+ return true
+ }
+ }
+ return false
+}
+
+// decodeHTMLToUTF8 converts non-UTF-8 HTML bytes to UTF-8 and reports the
+// encoding label used. Valid UTF-8 passes through untouched. Otherwise the
+// document's own BOM or <meta charset> declaration wins — including a
+// declared windows-1252: DetermineEncoding returns that name both for a real
+// declaration and as its "nothing declared" default, with certain=false
+// either way, so htmlDeclaresWindows1252 disambiguates the two. Undeclared
+// documents first honor a high-confidence statistical pick
+// (htmlDetectedCharset) and then walk the fallback chain
+// (htmlCharsetFallbackLabels via the shared decodeFirstCharsetMatch), where
+// each candidate must decode the whole blob without producing replacement
+// runes. If everything fails the original bytes are returned so behavior
+// never regresses below the previous UTF-8-only parse.
+func decodeHTMLToUTF8(data []byte) ([]byte, string) {
+ if len(data) == 0 || utf8Valid(data) {
+ return data, "utf-8"
+ }
+ // The prescan declaration wins for every name except the undeclared
+ // windows-1252 sentinel: skipping the sentinel keeps undeclared pages on
+ // the detect-then-chain path, while a declared windows-1252 (or its
+ // iso-8859-1 / us-ascii aliases) is honored.
+ if enc, name, _ := charset.DetermineEncoding(data, ""); enc != nil &&
+ (name != "windows-1252" || htmlDeclaresWindows1252(data)) {
+ if decoded, err := decodeTransform(data, enc.NewDecoder()); err == nil {
+ return []byte(decoded), name
+ }
+ }
+ if label := htmlDetectedCharset(data); label != "" {
+ if decoded, err := decodeWithCharset(data, label); err == nil {
+ return []byte(decoded), label
+ }
+ }
+ if decoded, label, ok := decodeFirstCharsetMatch(data, htmlCharsetFallbackLabels); ok {
+ return []byte(decoded), label
+ }
+ // Unreachable for non-empty input: the terminal ISO-8859-1 candidate
+ // decodes every byte sequence without replacement runes, so the loop
+ // above always returns first (the label is therefore neutral rather
+ // than a bogus "utf-8" — the bytes are non-UTF-8 by construction).
+ // Kept as a never-worse-than-before safety net.
+ return data, ""
+}
+
// walkHTMLBlocks emits one normalized item per block-level
// descendant of root. Inline elements (b, i, a, span, …) are
// collapsed into the parent's text via leafText. <script>,