RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(dataflow): decode non-UTF-8 HTML files before parsing in Go pipeline (#18801)
euvre9d9007745551
diff --git a/go.mod b/go.modindex f582c20fa7..1dc791b0bd 100644--- a/go.mod+++ b/go.mod@@ -54,6 +54,7 @@ require (github.com/pkg/errors v0.9.1github.com/pkoukk/tiktoken-go v0.1.8github.com/redis/go-redis/v9 v9.18.0+ github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456dgithub.com/signintech/gopdf v0.36.1github.com/siongui/gojianfan v0.0.0-20210926212422-2f175ac615degithub.com/spf13/viper v1.18.2diff --git a/go.sum b/go.sumindex 523b03116c..ac4b75fe9c 100644--- a/go.sum+++ b/go.sum@@ -480,6 +480,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6kegithub.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=+github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=+github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=diff --git a/internal/parser/parser/charset.go b/internal/parser/parser/charset.gonew file mode 100644index 0000000000..014d7051fa--- /dev/null+++ b/internal/parser/parser/charset.go@@ -0,0 +1,126 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package parser++// Shared charset-label resolution and decode-acceptance helpers. Both+// email_parser.go (RFC 2047 headers, mail bodies) and html_parser.go+// (decodeHTMLToUTF8) build on these, so a label resolves to the same+// decoder everywhere and the "first candidate that decodes without+// replacement runes" walk exists exactly once.++import (+ "bytes"+ "fmt"+ "io"+ "strings"++ "golang.org/x/text/encoding"+ "golang.org/x/text/encoding/charmap"+ "golang.org/x/text/encoding/japanese"+ "golang.org/x/text/encoding/korean"+ "golang.org/x/text/encoding/simplifiedchinese"+ "golang.org/x/text/encoding/traditionalchinese"+ "golang.org/x/text/transform"+)++// canonicalCharsetLabel canonicalizes a charset label for comparison and+// lookup: ASCII lower-casing with surrounding whitespace plus '-' and '_'+// removed, so detector outputs and WHATWG spellings such as "GB-18030",+// "Shift_JIS" or "EUC-KR" fold onto one canonical form.+func canonicalCharsetLabel(label string) string {+ return strings.NewReplacer("-", "", "_", "").Replace(strings.ToLower(strings.TrimSpace(label)))+}++// charsetEncoding maps a charset label to its decoder, covering the+// charsets common in real-world mail and legacy HTML beyond utf-8: the+// Chinese families (gb2312/gbk/gb18030, big5), shift_jis, euc-kr, and+// latin-1. It is the single label resolver shared by the RFC 2047 header+// decoder, decodeMailPayload (mail bodies), and decodeHTMLToUTF8 (HTML+// bytes) so every caller resolves a charset label identically.+//+// The "gb2312" label maps to GBK, not HZGB2312: mail labeled gb2312 carries+// plain 8-bit GB2312 bytes, while HZGB2312 only decodes the 7-bit HZ escape+// form used under the distinct "hz-gb-2312" label.+func charsetEncoding(charset string) (encoding.Encoding, bool) {+ switch canonicalCharsetLabel(charset) {+ case "gb2312", "gbk":+ return simplifiedchinese.GBK, true+ case "gb18030":+ return simplifiedchinese.GB18030, true+ case "hzgb2312":+ return simplifiedchinese.HZGB2312, true+ case "big5":+ return traditionalchinese.Big5, true+ case "shiftjis", "sjis":+ return japanese.ShiftJIS, true+ case "euckr":+ return korean.EUCKR, true+ case "iso88591", "latin1":+ return charmap.ISO8859_1, true+ }+ return nil, false+}++// decodeWithCharset decodes payload under one charset label, rejecting the+// result when it produces replacement runes (U+FFFD). An empty label is+// treated as utf-8.+func decodeWithCharset(payload []byte, charset string) (string, error) {+ switch canonicalCharsetLabel(charset) {+ case "utf8", "":+ s := string(payload)+ if !strings.ContainsRune(s, '\ufffd') {+ return s, nil+ }+ return "", fmt.Errorf("invalid utf-8")+ }+ if enc, ok := charsetEncoding(charset); ok {+ return decodeTransform(payload, enc.NewDecoder())+ }+ // Unknown charset: treat as latin-1.+ runes := make([]rune, len(payload))+ for i, b := range payload {+ runes[i] = rune(b)+ }+ return string(runes), nil+}++// decodeTransform decodes payload through decoder, rejecting results that+// contain replacement runes.+func decodeTransform(payload []byte, decoder *encoding.Decoder) (string, error) {+ reader := transform.NewReader(bytes.NewReader(payload), decoder)+ decoded, err := io.ReadAll(reader)+ if err != nil {+ return "", err+ }+ if !strings.ContainsRune(string(decoded), '\ufffd') {+ return string(decoded), nil+ }+ return "", fmt.Errorf("decode produced replacement characters")+}++// decodeFirstCharsetMatch walks the candidate labels in order and returns+// the first decode that succeeds without replacement runes, along with the+// label that produced it. It is the shared fallback-chain walker behind+// both decodeMailPayload and decodeHTMLToUTF8.+func decodeFirstCharsetMatch(data []byte, labels []string) (string, string, bool) {+ for _, label := range labels {+ if decoded, err := decodeWithCharset(data, label); err == nil {+ return decoded, label, true+ }+ }+ return "", "", false+}diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.gonew file mode 100644index 0000000000..83be10c042--- /dev/null+++ b/internal/parser/parser/charset_test.go@@ -0,0 +1,62 @@+package parser++import "testing"++// TestCharsetEncodingSharedLabelContract locks the single-label contract of+// charsetEncoding: every fallback label the HTML parser may walk (and the+// mail chain ships) resolves through the shared resolver, detector/WHATWG+// spellings fold onto the same encoder, and labels outside the shared+// family (e.g. windows-1252, handled by the HTML prescan layer, or utf-8,+// handled natively by decodeWithCharset) stay out so the two parsers cannot+// drift apart again.+func TestCharsetEncodingSharedLabelContract(t *testing.T) {+ mailLabels := []string{"gb2312", "gbk", "gb18030", "hz-gb-2312", "big5", "shift_jis", "sjis", "euc-kr", "iso-8859-1", "latin1"}+ for _, label := range append(mailLabels, htmlCharsetFallbackLabels...) {+ if _, ok := charsetEncoding(label); !ok {+ t.Errorf("charsetEncoding(%q) = (_, false); the shared resolver must cover every mail-chain and HTML-fallback label", label)+ }+ }++ aliases := map[string]string{+ "GB-18030": "gb18030",+ "GB2312": "gb2312",+ "Shift_JIS": "shift_jis",+ "Shift-JIS": "shift_jis",+ "EUC-KR": "euc-kr",+ "ISO-8859-1": "iso-8859-1",+ "ISO8859-1": "iso-8859-1",+ "Latin1": "latin1",+ " Big5 ": "big5",+ }+ for alias, label := range aliases {+ aliasEnc, aliasOK := charsetEncoding(alias)+ labelEnc, labelOK := charsetEncoding(label)+ if !aliasOK || !labelOK || aliasEnc != labelEnc {+ t.Errorf("charsetEncoding(%q) = (%v, %v), charsetEncoding(%q) = (%v, %v); alias spelling must fold onto the same encoder", alias, aliasEnc, aliasOK, label, labelEnc, labelOK)+ }+ }++ for _, label := range []string{"windows-1252", "utf-8", "utf-16", "euc-jp", ""} {+ if _, ok := charsetEncoding(label); ok {+ t.Errorf("charsetEncoding(%q) = (_, true); labels handled elsewhere (prescan / native utf-8) must not enter the shared resolver", label)+ }+ }+}++// TestDecodeFirstCharsetMatchAcceptance pins the shared walker's contract:+// the first candidate whose decode produces no replacement runes wins and+// reports its own label; when every candidate is rejected the walker+// reports failure so callers apply their own terminal fallback. (utf-8 is+// the deterministic rejecting candidate: the Big5 bytes are invalid UTF-8.)+func TestDecodeFirstCharsetMatchAcceptance(t *testing.T) {+ big5Bytes := []byte{0xA4, 0xA4, 0xA4, 0xE5} // "中文" in Big5++ decoded, label, ok := decodeFirstCharsetMatch(big5Bytes, []string{"utf-8", "big5"})+ if !ok || label != "big5" || decoded != "中文" {+ t.Errorf("decodeFirstCharsetMatch(big5) = (%q, %q, %v); want (中文, big5, true)", decoded, label, ok)+ }++ if _, _, ok := decodeFirstCharsetMatch(big5Bytes, []string{"utf-8"}); ok {+ t.Error("decodeFirstCharsetMatch over a rejecting candidate list must report failure")+ }+}diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.goindex b5995e8b8b..67436fc0e7 100644--- a/internal/parser/parser/email_parser.go+++ b/internal/parser/parser/email_parser.go@@ -33,12 +33,6 @@ import ("github.com/AkmalOt/gomsg""golang.org/x/net/html"- "golang.org/x/text/encoding"- "golang.org/x/text/encoding/charmap"- "golang.org/x/text/encoding/japanese"- "golang.org/x/text/encoding/korean"- "golang.org/x/text/encoding/simplifiedchinese"- "golang.org/x/text/encoding/traditionalchinese""golang.org/x/text/transform""ragflow/internal/utility"@@ -218,35 +212,6 @@ func targetFieldsSet(fields []string) map[string]bool {// -- header decoding (RFC 2047) ---// charsetEncoding maps a MIME charset label to its decoder, covering the-// charsets common in real-world mail beyond utf-8: the Chinese families-// (gb2312/gbk/gb18030, big5), shift_jis, euc-kr, and latin-1. It is shared by-// the RFC 2047 header decoder and decodeMailPayload so headers and bodies-// resolve a charset label identically.-//-// The "gb2312" label maps to GBK, not HZGB2312: mail labeled gb2312 carries-// plain 8-bit GB2312 bytes, while HZGB2312 only decodes the 7-bit HZ escape-// form used under the distinct "hz-gb-2312" label.-func charsetEncoding(charset string) (encoding.Encoding, bool) {- switch strings.ToLower(strings.TrimSpace(charset)) {- case "gb2312", "gbk":- return simplifiedchinese.GBK, true- case "gb18030":- return simplifiedchinese.GB18030, true- case "hz-gb-2312":- return simplifiedchinese.HZGB2312, true- case "big5":- return traditionalchinese.Big5, true- case "shift_jis", "shift-jis", "sjis":- return japanese.ShiftJIS, true- case "euc-kr":- return korean.EUCKR, true- case "iso-8859-1", "iso8859-1", "latin1":- return charmap.ISO8859_1, true- }- return nil, false-}-// rfc2047Decoder decodes RFC 2047 encoded-words (e.g. "=?utf-8?B?...?=") in// header values. utf-8 is handled natively by mime; the CharsetReader covers// the non-UTF-8 charsets via charsetEncoding.@@ -553,16 +518,8 @@ func decodeMailPayload(payload []byte, charset string) string {if len(payload) == 0 {return ""}- charsets := buildCharsetChain(charset)- for _, enc := range charsets {- if enc == "" {- // raw bytes → already fallthrough- return string(payload)- }- decoded, err := decodeWithCharset(payload, enc)- if err == nil {- return decoded- }+ if decoded, _, ok := decodeFirstCharsetMatch(payload, buildCharsetChain(charset)); ok {+ return decoded}return string(payload)}@@ -576,39 +533,6 @@ func buildCharsetChain(declared string) []string {return chain}-func decodeWithCharset(payload []byte, charset string) (string, error) {- charset = strings.ToLower(strings.TrimSpace(charset))- switch charset {- case "utf-8", "utf8", "":- s := string(payload)- if !strings.ContainsRune(s, '\ufffd') {- return s, nil- }- return "", fmt.Errorf("invalid utf-8")- }- if enc, ok := charsetEncoding(charset); ok {- return decodeTransform(payload, enc.NewDecoder())- }- // Unknown charset: treat as latin-1.- runes := make([]rune, len(payload))- for i, b := range payload {- runes[i] = rune(b)- }- return string(runes), nil-}--func decodeTransform(payload []byte, decoder *encoding.Decoder) (string, error) {- reader := transform.NewReader(bytes.NewReader(payload), decoder)- decoded, err := io.ReadAll(reader)- if err != nil {- return "", err- }- if !strings.ContainsRune(string(decoded), '\ufffd') {- return string(decoded), nil- }- return "", fmt.Errorf("decode produced replacement characters")-}-// parseMSG parses an Outlook .msg (OLE2 compound document) file using the// gomsg library, mirroring the Python flow parser's _email() .msg branch// (rag/flow/parser/parser.py), which parses via extract_msg. The output mapdiff --git a/internal/parser/parser/html_charset_test.go b/internal/parser/parser/html_charset_test.gonew file mode 100644index 0000000000..6fcc981a5e--- /dev/null+++ b/internal/parser/parser/html_charset_test.go@@ -0,0 +1,337 @@+package parser++import (+ "context"+ "strings"+ "testing"++ "golang.org/x/text/encoding"+ "golang.org/x/text/encoding/charmap"+ "golang.org/x/text/encoding/japanese"+ "golang.org/x/text/encoding/korean"+ "golang.org/x/text/encoding/simplifiedchinese"+ "golang.org/x/text/encoding/traditionalchinese"+ "golang.org/x/text/encoding/unicode"+)++// gbkHTML is a Sina-finance-style balance-sheet page encoded in GBK with a+// GB2312 meta declaration, the shape reported by the "agent data pipeline+// parses HTML into mojibake" issue. GBK bytes double as GB18030 input.+func gbkHTML(t *testing.T, withMeta bool) []byte {+ t.Helper()+ head := "<html><head><title>贵州茅台(600519)资产负债表</title></head>"+ if withMeta {+ head = `<html><head><meta http-equiv="Content-Type" content="text/html; charset=GB2312">` ++ "<title>贵州茅台(600519)资产负债表</title></head>"+ }+ body := "<body><h1>贵州茅台(600519)资产负债表</h1>" ++ "<table><tr><th>项目</th><th>20251231</th></tr>" ++ "<tr><td>货币资金</td><td>56000000000.00</td></tr>" ++ "<tr><td>应收账款</td><td>1200000000.00</td></tr></table>" ++ "<p>单位:元</p></body></html>"+ src := head + body+ if !utf8Valid([]byte(src)) {+ t.Fatal("test source must be valid UTF-8 before re-encoding")+ }+ out, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(src))+ if err != nil {+ t.Fatalf("GBK encode: %v", err)+ }+ if utf8Valid(out) {+ t.Fatal("GBK fixture unexpectedly decodes as UTF-8; fixture is ineffective")+ }+ return out+}++// htmlEncodedFixture re-encodes src with enc and guards that the fixture is+// effective: it must not itself be valid UTF-8, otherwise a decode test would+// trivially pass through the fast path and prove nothing.+func htmlEncodedFixture(t *testing.T, src string, enc encoding.Encoding) []byte {+ t.Helper()+ raw, err := enc.NewEncoder().Bytes([]byte(src))+ if err != nil {+ t.Fatalf("fixture encode: %v", err)+ }+ if utf8Valid(raw) {+ t.Fatal("fixture unexpectedly decodes as UTF-8; fixture is ineffective")+ }+ return raw+}++// itemsText concatenates the text of every item in res, newline separated.+func itemsText(res ParseResult) string {+ var sb strings.Builder+ for _, it := range res.JSON {+ if s, ok := it["text"].(string); ok {+ sb.WriteString(s)+ sb.WriteString("\n")+ }+ }+ return sb.String()+}++// assertDecodedChinese fails res on a parse error, any U+FFFD replacement+// rune, or missing expected Chinese substrings.+func assertDecodedChinese(t *testing.T, res ParseResult) {+ t.Helper()+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ got := itemsText(res)+ if strings.ContainsRune(got, '\ufffd') {+ t.Errorf("output contains U+FFFD replacement runes: %q", got)+ }+ for _, want := range []string{"\u8d35\u5dde\u8305\u53f0", "\u8d44\u4ea7\u8d1f\u503a\u8868", "\u8d27\u5e01\u8d44\u91d1", "\u5e94\u6536\u8d26\u6b3e"} {+ if !strings.Contains(got, want) {+ t.Errorf("decoded text missing %q; got: %q", want, got)+ }+ }+}++// TestHTMLParser_DecodesGBKMetaDeclared reproduces the reported issue: a+// GBK-encoded HTML file parsed by the data pipeline surfaced as mojibake+// because x/net/html assumes UTF-8. The meta charset declaration must drive+// the decode, mirroring Python's find_codec path.+func TestHTMLParser_DecodesGBKMetaDeclared(t *testing.T) {+ res := NewHTMLParser().ParseWithResult(context.Background(), "balance.html", gbkHTML(t, true))+ assertDecodedChinese(t, res)+ if enc, _ := res.File["encoding"].(string); enc == "" || enc == "utf-8" {+ t.Errorf("File.encoding = %q, want the detected non-UTF-8 label", enc)+ }+}++// TestHTMLParser_DecodesGBKWithoutDeclaration covers GBK bytes with no meta+// declaration: the GB18030 fallback chain candidate must decode them.+func TestHTMLParser_DecodesGBKWithoutDeclaration(t *testing.T) {+ res := NewHTMLParser().ParseWithResult(context.Background(), "balance.html", gbkHTML(t, false))+ assertDecodedChinese(t, res)+}++// TestHTMLParser_DecodesBig5MetaDeclared covers a Big5 page with its meta+// declaration.+func TestHTMLParser_DecodesBig5MetaDeclared(t *testing.T) {+ src := `<html><head><meta charset="big5"></head><body><h1>` ++ "\u53f0\u7063\u5927\u5b78" + `</h1><p>` + "\u8cc7\u6599\u5eab" + `</p></body></html>`+ raw, err := traditionalchinese.Big5.NewEncoder().Bytes([]byte(src))+ if err != nil {+ t.Fatalf("Big5 encode: %v", err)+ }+ if utf8Valid(raw) {+ t.Fatal("Big5 fixture unexpectedly decodes as UTF-8; fixture is ineffective")+ }+ res := NewHTMLParser().ParseWithResult(context.Background(), "tw.html", raw)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ got := itemsText(res)+ if strings.ContainsRune(got, '\ufffd') {+ t.Errorf("output contains U+FFFD replacement runes: %q", got)+ }+ for _, want := range []string{"\u53f0\u7063\u5927\u5b78", "\u8cc7\u6599\u5eab"} {+ if !strings.Contains(got, want) {+ t.Errorf("decoded text missing %q; got: %q", want, got)+ }+ }+}++// TestHTMLParser_UTF8Passthrough ensures valid UTF-8 input is untouched and+// still reported as utf-8.+func TestHTMLParser_UTF8Passthrough(t *testing.T) {+ src := "<html><body><h1>\u8d35\u5dde\u8305\u53f0</h1><p>plain utf-8</p></body></html>"+ res := NewHTMLParser().ParseWithResult(context.Background(), "u.html", []byte(src))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ got := itemsText(res)+ if !strings.Contains(got, "plain utf-8") || !strings.Contains(got, "\u8d35\u5dde\u8305\u53f0") {+ t.Errorf("utf-8 passthrough lost content; got: %q", got)+ }+ if enc, _ := res.File["encoding"].(string); enc != "utf-8" {+ t.Errorf("File.encoding = %q, want utf-8", enc)+ }+}++// TestHTMLParser_DecodeTableSurvivesGBK verifies the structured table item+// (doc_type_kwd:"table") is emitted from the decoded document with its+// Chinese cell text intact.+func TestHTMLParser_DecodeTableSurvivesGBK(t *testing.T) {+ res := NewHTMLParser().ParseWithResult(context.Background(), "balance.html", gbkHTML(t, true))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ var tableText string+ for _, it := range res.JSON {+ if it["doc_type_kwd"] == "table" {+ tableText, _ = it["text"].(string)+ }+ }+ if tableText == "" {+ t.Fatalf("no structured table item emitted; items: %#v", res.JSON)+ }+ if !strings.Contains(tableText, "\u8d27\u5e01\u8d44\u91d1") {+ t.Errorf("table item missing decoded cell text; got: %q", tableText)+ }+}++// assertUndeclaredDecode parses an undeclared (no <meta charset>) document+// encoded with enc and asserts the original text round-trips: no U+FFFD and+// every want substring survives, proving the statistical detector steered the+// decode instead of the GB18030-first chain order hijacking the bytes.+func assertUndeclaredDecode(t *testing.T, raw []byte, wantLabel string, wants []string) {+ t.Helper()+ res := NewHTMLParser().ParseWithResult(context.Background(), "doc.html", raw)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ got := itemsText(res)+ if strings.ContainsRune(got, '\ufffd') {+ t.Errorf("output contains U+FFFD replacement runes: %q", got)+ }+ for _, want := range wants {+ if !strings.Contains(got, want) {+ t.Errorf("decoded text missing %q; got: %q", want, got)+ }+ }+ if enc, _ := res.File["encoding"].(string); enc != wantLabel {+ t.Errorf("File.encoding = %q, want %q", enc, wantLabel)+ }+}++// TestHTMLParser_DecodesBig5WithoutDeclaration pins the undeclared-Big5 risk+// the review flagged: every Big5 double-byte pair is also a valid GB18030+// pair, so the ordered chain alone silently decoded Big5 pages as wrong+// Chinese text. The statistical detector must pick Big5 first (mirroring+// Python find_codec's chardet step) and round-trip the original text.+func TestHTMLParser_DecodesBig5WithoutDeclaration(t *testing.T) {+ src := "<html><body><h1>\u53f0\u7063\u5927\u5b78\u8cc7\u6599\u5eab</h1>" ++ "<p>\u9019\u662f\u4e00\u4efd\u7e41\u9ad4\u4e2d\u6587\u6e2c\u8a66\u6587\u4ef6\uff0c\u7528\u65bc\u9a57\u8b49\u5b57\u5143\u7de8\u78bc\u5075\u6e2c\u3002</p></body></html>"+ assertUndeclaredDecode(t, htmlEncodedFixture(t, src, traditionalchinese.Big5), "big5",+ []string{"\u53f0\u7063\u5927\u5b78\u8cc7\u6599\u5eab", "\u7e41\u9ad4\u4e2d\u6587"})+}++// TestHTMLParser_DecodesShiftJISWithoutDeclaration covers an undeclared+// Shift-JIS page: the detector must steer the decode to Shift-JIS rather+// than letting GB18030 absorb the bytes as mojibake.+func TestHTMLParser_DecodesShiftJISWithoutDeclaration(t *testing.T) {+ src := "<html><body><h1>\u65e5\u672c\u8a9e\u306e\u30c6\u30b9\u30c8</h1>" ++ "<p>\u3053\u308c\u306f\u30b7\u30d5\u30c8JIS\u3067\u30a8\u30f3\u30b3\u30fc\u30c9\u3055\u308c\u305f\u6587\u66f8\u3067\u3059\u3002\u6771\u4eac\u90fd\u5e81\u306e\u8cc7\u6599\u3002</p></body></html>"+ assertUndeclaredDecode(t, htmlEncodedFixture(t, src, japanese.ShiftJIS), "shift_jis",+ []string{"\u65e5\u672c\u8a9e\u306e\u30c6\u30b9\u30c8", "\u6771\u4eac\u90fd\u5e81"})+}++// TestHTMLParser_DecodesEUCKRWithoutDeclaration covers an undeclared EUC-KR+// page: detection picks the EUC-KR candidate before the chain order runs.+func TestHTMLParser_DecodesEUCKRWithoutDeclaration(t *testing.T) {+ src := "<html><body><h1>\ud55c\uad6d\uc5b4 \uc2dc\ud5d8</h1>" ++ "<p>\uc774 \ubb38\uc11c\ub294 EUC-KR\ub85c \uc778\ucf54\ub529\ub418\uc5b4 \uc788\uc2b5\ub2c8\ub2e4. \uc11c\uc6b8\ud2b9\ubcc4\uc2dc.</p></body></html>"+ assertUndeclaredDecode(t, htmlEncodedFixture(t, src, korean.EUCKR), "euc-kr",+ []string{"\ud55c\uad6d\uc5b4 \uc2dc\ud5d8", "\uc11c\uc6b8\ud2b9\ubcc4\uc2dc"})+}++// TestHTMLParser_Latin1TerminalFallback asserts the ISO-8859-1 terminal+// guarantee: an undeclared Western page round-trips through ISO-8859-1+// (either by detection or as the chain's any-byte-decodes terminal), never+// as U+FFFD soup.+func TestHTMLParser_Latin1TerminalFallback(t *testing.T) {+ src := "<html><body><h1>Caf\u00e9 r\u00e9sum\u00e9</h1><p>Na\u00efve fa\u00e7ade \u00e0 gogo.</p></body></html>"+ assertUndeclaredDecode(t, htmlEncodedFixture(t, src, charmap.ISO8859_1), "iso-8859-1",+ []string{"Caf\u00e9 r\u00e9sum\u00e9", "Na\u00efve fa\u00e7ade"})+}++// TestHTMLParser_DecodesUTF16BOM covers the BOM path of the HTML5 prescan+// (charset.DetermineEncoding): a UTF-16LE document with its BOM decodes even+// though no <meta charset> is declared — a net-new improvement over Python's+// find_codec, which has no BOM handling.+func TestHTMLParser_DecodesUTF16BOM(t *testing.T) {+ src := "<html><body><h1>\u8cb4\u5dde\u8305\u81fa</h1><p>utf16 bom doc</p></body></html>"+ raw, err := unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM).NewEncoder().Bytes([]byte(src))+ if err != nil {+ t.Fatalf("UTF-16LE encode: %v", err)+ }+ raw = append([]byte{0xff, 0xfe}, raw...)+ res := NewHTMLParser().ParseWithResult(context.Background(), "u16.html", raw)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ got := itemsText(res)+ if !strings.Contains(got, "\u8cb4\u5dde\u8305\u81fa") || !strings.Contains(got, "utf16 bom doc") {+ t.Errorf("UTF-16 BOM decode lost content; got: %q", got)+ }+}++// TestHTMLParser_EmptyInputNoPanic proves the never-worse-than-before claim+// at the degenerate boundary: empty input parses without error.+func TestHTMLParser_EmptyInputNoPanic(t *testing.T) {+ res := NewHTMLParser().ParseWithResult(context.Background(), "empty.html", nil)+ if res.Err != nil {+ t.Fatalf("ParseWithResult(empty): %v", res.Err)+ }+}++// TestDecodeHTMLToUTF8_NeverWorseThanBefore feeds hostile bytes that no+// multibyte candidate decodes cleanly: whatever accepts them (in practice+// the ISO-8859-1 terminal) must produce replacement-free output with a+// non-empty label, and empty input passes through untouched.+func TestDecodeHTMLToUTF8_NeverWorseThanBefore(t *testing.T) {+ out, label := decodeHTMLToUTF8(nil)+ if len(out) != 0 || label != "utf-8" {+ t.Errorf("empty input: got (%d bytes, %q), want (0, utf-8)", len(out), label)+ }+ raw := []byte{0x81, 0x20, 0xfe, 0x21, 0x8f, 0x39, 0x41, 0x7e}+ out, label = decodeHTMLToUTF8(raw)+ if len(out) == 0 || label == "" {+ t.Errorf("hostile bytes: got (%d bytes, %q), want non-empty decode and label", len(out), label)+ }+ if strings.ContainsRune(string(out), '\ufffd') {+ t.Errorf("hostile bytes decoded to replacement runes: %q", out)+ }+}++// TestHTMLParser_DecodesDeclaredWindows1252 pins the declared-windows-1252+// path: DetermineEncoding returns windows-1252 (certain=false) both for a+// real declaration and as its "nothing declared" sentinel, so the declaration+// must be confirmed explicitly. The 0x80-0x9F range (smart quotes, €, ™, —)+// differs from the ISO-8859-1 terminal, which decodes those bytes as C1+// control characters instead of punctuation.+func TestHTMLParser_DecodesDeclaredWindows1252(t *testing.T) {+ src := `<html><head><meta charset="windows-1252"></head><body>` ++ "<h1>\u201cSmart\u201d pricing \u2014 20% off</h1>" ++ "<p>Paste\u2122 caf\u00e9 \u20ac3.50</p></body></html>"+ raw := htmlEncodedFixture(t, src, charmap.Windows1252)+ res := NewHTMLParser().ParseWithResult(context.Background(), "win1252.html", raw)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if enc, _ := res.File["encoding"].(string); enc != "windows-1252" {+ t.Errorf("File.encoding = %q, want windows-1252", enc)+ }+ got := itemsText(res)+ if strings.ContainsRune(got, '\ufffd') {+ t.Errorf("output contains U+FFFD replacement runes: %q", got)+ }+ for _, want := range []string{"\u201cSmart\u201d", "\u2014", "\u2122", "\u20ac"} {+ if !strings.Contains(got, want) {+ t.Errorf("decoded text missing %q; got: %q", want, got)+ }+ }+}++// TestHTMLParser_DecodesDeclaredLatin1AliasAsWindows1252 covers the+// http-equiv pragma form declaring iso-8859-1: WHATWG folds that label (and+// us-ascii) onto windows-1252, so the page must decode with windows-1252+// semantics — smart quotes survive instead of becoming C1 controls.+func TestHTMLParser_DecodesDeclaredLatin1AliasAsWindows1252(t *testing.T) {+ src := `<html><head><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"></head><body>` ++ "<p>He said \u201cna\u00efve\u201d \u2014 caf\u00e9 \u2122</p></body></html>"+ raw := htmlEncodedFixture(t, src, charmap.Windows1252)+ res := NewHTMLParser().ParseWithResult(context.Background(), "latin1.html", raw)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if enc, _ := res.File["encoding"].(string); enc != "windows-1252" {+ t.Errorf("File.encoding = %q, want windows-1252", enc)+ }+ if got := itemsText(res); !strings.Contains(got, "\u201cna\u00efve\u201d") {+ t.Errorf("decoded text lost windows-1252 punctuation; got: %q", got)+ }+}diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex e6e73c5bf4..c884078c9f 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -20,9 +20,13 @@ import ("bytes""context""fmt"+ "regexp""strings"+ "github.com/saintfish/chardet"+"golang.org/x/net/html"+ "golang.org/x/net/html/charset")type HTMLParser struct {@@ -67,6 +71,11 @@ func (p *HTMLParser) ConfigureFromSetup(setup map[string]any) {// separate ck_type — the python HtmlParser collapses inline// formatting into the parent block's text.func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {+ // x/net/html assumes UTF-8 input, so a GBK/Big5/Shift-JIS page would+ // otherwise surface as U+FFFD mojibake. Decode first, mirroring the+ // Python dataflow path (RAGFlowHtmlParser decodes the blob via+ // rag.nlp.find_codec before parsing). See decodeHTMLToUTF8.+ data, encName := decodeHTMLToUTF8(data)// remove_header_footer: pre-parse strip of <header>/<footer> tags// and ARIA role=banner/contentinfo elements (mirrors Python// parser.py:1083-1084 remove_header_footer_html_blob).@@ -95,12 +104,144 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, dataOutputFormat: "json",File: map[string]any{"name": filename,- "encoding": "utf-8",+ "encoding": encName,},JSON: items,}}+// htmlCharsetFallbackLabels is the ordered brute-force decode loop for HTML+// bytes that are neither valid UTF-8 nor carrying a BOM / <meta charset>+// declaration, mirroring the pragmatic core of Python's find_codec+// (rag/nlp/__init__.py): legacy pages that omit a charset declaration are+// overwhelmingly GBK-family (saved-by-browser Chinese finance/gov pages), so+// GB18030 (the GBK superset) leads. Big5 / Shift-JIS / EUC-KR pages+// virtually always declare their charset and are caught by the meta prescan;+// when the declaration is missing, the statistical detector+// (htmlDetectedCharset) reorders the attempt so they are not silently+// absorbed by GB18030. ISO-8859-1 is intentionally terminal, the way latin1+// terminates Python's codec loop: it decodes ANY byte sequence without+// replacement runes, so once the chain reaches it, it always succeeds —+// Western pages without a declaration still yield readable text instead of+// U+FFFD soup. Labels resolve through the shared charsetEncoding in+// charset.go; only the HTML-specific ordering lives here.+var htmlCharsetFallbackLabels = []string{"gb18030", "big5", "shift_jis", "euc-kr", "iso-8859-1"}++// htmlCharsetDetectConfidence is the minimum statistical-detection+// confidence (on chardet's 0-100 scale) allowed to override the fallback+// chain order. Calibrated empirically: genuine CJK text scores 100, while+// the Go chardet port produces plausible-but-wrong guesses at low confidence+// (a sparse GBK balance-sheet page scores Big5 at 40; a short Big5 page+// scores ISO-8859-1 at 17). Only a confident pick may jump the queue;+// anything weaker keeps the GB18030-first domain heuristic.+const htmlCharsetDetectConfidence = 90++// htmlDetectedCharset mirrors Python find_codec's chardet.detect step for+// HTML bytes that are neither valid UTF-8 nor declared: a statistical+// detector picks which fallback-chain label to try FIRST, returned as one+// of htmlCharsetFallbackLabels. This matters because GB18030 maps nearly+// every byte sequence without replacement runes, so the ordered chain alone+// cannot recognize an undeclared Big5 / Shift-JIS / EUC-KR page — it would+// "decode" it as wrong Chinese text. Detection runs over the first 1024+// bytes, exactly like Python, but unlike Python it is gated on+// htmlCharsetDetectConfidence: the Go port's low-confidence guesses are+// demonstrably wrong often enough that they must not override the chain.+// A detection that fails, scores below the gate, or names an encoding the+// chain does not ship (e.g. windows-1252) returns "" and the chain order+// stands.+func htmlDetectedCharset(data []byte) string {+ sample := data+ if len(sample) > 1024 {+ sample = sample[:1024]+ }+ res, err := chardet.NewTextDetector().DetectBest(sample)+ if err != nil || res == nil || res.Charset == "" || res.Confidence < htmlCharsetDetectConfidence {+ return ""+ }+ detected := canonicalCharsetLabel(res.Charset)+ // The detector reports the GBK family as GB2312 / GBK; GB18030 is the+ // superset we ship, so both map onto the gb18030 candidate.+ if detected == "gb2312" || detected == "gbk" {+ detected = "gb18030"+ }+ for _, label := range htmlCharsetFallbackLabels {+ if canonicalCharsetLabel(label) == detected {+ return label+ }+ }+ return ""+}++// htmlMetaCharsetRe captures the charset label of a <meta> tag, matching both+// the <meta charset="..."> form and the http-equiv content-type form+// (<meta ... content="text/html; charset=...">), over the same first-1024-byte+// window x/net's prescan consumes.+var htmlMetaCharsetRe = regexp.MustCompile(`(?i)<meta[^>]*charset\s*=\s*["']?\s*([a-z0-9._+\-]+)`)++// htmlDeclaresWindows1252 reports whether data's <meta> tags declare+// windows-1252, either directly or via a WHATWG alias such as iso-8859-1 /+// us-ascii that charset.Lookup canonicalizes to windows-1252.+// charset.DetermineEncoding cannot tell a declared windows-1252 from its own+// "nothing declared" default: a meta prescan hit never sets certain (only+// BOMs do), so both come back as (windows-1252, certain=false). The+// declaration must therefore be confirmed explicitly, or a declared+// windows-1252 page is mis-decoded by the fallback chain: windows-1252's+// 0x80-0x9F range (smart quotes, €, ™) is C1 control characters in the+// chain's ISO-8859-1 terminal.+func htmlDeclaresWindows1252(data []byte) bool {+ if len(data) > 1024 {+ data = data[:1024]+ }+ for _, m := range htmlMetaCharsetRe.FindAllSubmatch(data, -1) {+ if _, name := charset.Lookup(string(m[1])); name == "windows-1252" {+ return true+ }+ }+ return false+}++// decodeHTMLToUTF8 converts non-UTF-8 HTML bytes to UTF-8 and reports the+// encoding label used. Valid UTF-8 passes through untouched. Otherwise the+// document's own BOM or <meta charset> declaration wins — including a+// declared windows-1252: DetermineEncoding returns that name both for a real+// declaration and as its "nothing declared" default, with certain=false+// either way, so htmlDeclaresWindows1252 disambiguates the two. Undeclared+// documents first honor a high-confidence statistical pick+// (htmlDetectedCharset) and then walk the fallback chain+// (htmlCharsetFallbackLabels via the shared decodeFirstCharsetMatch), where+// each candidate must decode the whole blob without producing replacement+// runes. If everything fails the original bytes are returned so behavior+// never regresses below the previous UTF-8-only parse.+func decodeHTMLToUTF8(data []byte) ([]byte, string) {+ if len(data) == 0 || utf8Valid(data) {+ return data, "utf-8"+ }+ // The prescan declaration wins for every name except the undeclared+ // windows-1252 sentinel: skipping the sentinel keeps undeclared pages on+ // the detect-then-chain path, while a declared windows-1252 (or its+ // iso-8859-1 / us-ascii aliases) is honored.+ if enc, name, _ := charset.DetermineEncoding(data, ""); enc != nil &&+ (name != "windows-1252" || htmlDeclaresWindows1252(data)) {+ if decoded, err := decodeTransform(data, enc.NewDecoder()); err == nil {+ return []byte(decoded), name+ }+ }+ if label := htmlDetectedCharset(data); label != "" {+ if decoded, err := decodeWithCharset(data, label); err == nil {+ return []byte(decoded), label+ }+ }+ if decoded, label, ok := decodeFirstCharsetMatch(data, htmlCharsetFallbackLabels); ok {+ return []byte(decoded), label+ }+ // Unreachable for non-empty input: the terminal ISO-8859-1 candidate+ // decodes every byte sequence without replacement runes, so the loop+ // above always returns first (the label is therefore neutral rather+ // than a bogus "utf-8" — the bytes are non-UTF-8 by construction).+ // Kept as a never-worse-than-before safety net.+ return data, ""+}+// walkHTMLBlocks emits one normalized item per block-level// descendant of root. Inline elements (b, i, a, span, …) are// collapsed into the parent's text via leafText. <script>,