RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): drop the BOM and decode JSON and Markdown input in Go parsers (#20225)
Jinebb71552a265
diff --git a/internal/parser/parser/charset.go b/internal/parser/parser/charset.goindex 9b8a685511..6db1208a92 100644--- a/internal/parser/parser/charset.go+++ b/internal/parser/parser/charset.go@@ -231,8 +231,13 @@ func extractCharsetHint(hint string) string {return ""}+// utf8BOM is the byte-order mark as it appears once the payload is UTF-8.+var utf8BOM = []byte("\ufeff")+// DecodeToUTF8 converts arbitrary data bytes into UTF-8.// If data is already valid UTF-8, it returns data untouched with encoding "utf-8".+// A leading byte-order mark is dropped on every path, as Python's decode_text+// does, so it never leaks into the first heading, CSV header, or JSON token.// hint can be:// - An explicit charset label (e.g. "gbk", "gb2312", "big5", from MIME headers)// - A MIME Content-Type (e.g. "text/html", "application/xhtml+xml", "text/csv", "text/plain")@@ -240,6 +245,11 @@ func extractCharsetHint(hint string) string {//// It returns the decoded UTF-8 bytes and the canonical encoding label used.func DecodeToUTF8(data []byte, hint string) ([]byte, string) {+ decoded, label := decodeToUTF8(data, hint)+ return bytes.TrimPrefix(decoded, utf8BOM), label+}++func decodeToUTF8(data []byte, hint string) ([]byte, string) {if len(data) == 0 {return data, "utf-8"}diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.goindex 26383ccaf0..7eccf252f8 100644--- a/internal/parser/parser/charset_test.go+++ b/internal/parser/parser/charset_test.go@@ -299,3 +299,45 @@ func TestDecodeToUTF8_UTF16LEBOM(t *testing.T) {t.Errorf("got %q, want containing %q", string(out), src)}}++// A byte-order mark marks the encoding; it is not content. Python's+// decode_text drops it, so a leading U+FEFF must not reach the parsers.+func TestDecodeToUTF8_DropsByteOrderMark(t *testing.T) {+ src := "name,备注\n"+ utf16le, err := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder().Bytes([]byte(src))+ if err != nil {+ t.Fatalf("UTF-16LE encode: %v", err)+ }+ utf16be, err := unicode.UTF16(unicode.BigEndian, unicode.UseBOM).NewEncoder().Bytes([]byte(src))+ if err != nil {+ t.Fatalf("UTF-16BE encode: %v", err)+ }+ cases := []struct {+ name string+ data []byte+ label string+ }{+ {"utf-8", append([]byte("\xef\xbb\xbf"), src...), "utf-8"},+ {"utf-16le", utf16le, "utf-16le"},+ {"utf-16be", utf16be, "utf-16be"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ out, label := DecodeToUTF8(tc.data, "text/plain")+ if string(out) != src {+ t.Errorf("got %q, want %q", out, src)+ }+ if label != tc.label {+ t.Errorf("got label %q, want %q", label, tc.label)+ }+ })+ }+}++func TestDecodeToUTF8_KeepsNonLeadingZeroWidthNoBreakSpace(t *testing.T) {+ src := "a\ufeffb"+ out, _ := DecodeToUTF8([]byte(src), "text/plain")+ if string(out) != src {+ t.Errorf("got %q, want %q", out, src)+ }+}diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.goindex 4e45e23515..f50c38cec9 100644--- a/internal/parser/parser/csv_parser_test.go+++ b/internal/parser/parser/csv_parser_test.go@@ -59,6 +59,22 @@ func TestCSVParser_EmitsSpreadsheetRows(t *testing.T) {}}+// Excel's "CSV UTF-8" export starts with a BOM; it must not become part of+// the first column name.+func TestCSVParser_UTF8BOM(t *testing.T) {+ res := NewCSVParser().ParseWithResult(context.Background(), "bom.csv", []byte("\xef\xbb\xbfName,Age\nAlice,30\n"))+ if res.Err != nil {+ t.Fatalf("ParseWithResult failed: %v", res.Err)+ }+ if len(res.JSON) != 1 {+ t.Fatalf("items = %#v, want one table item", res.JSON)+ }+ text, _ := res.JSON[0]["text"].(string)+ if !strings.Contains(text, "<tr><th>Name</th><th>Age</th></tr>") {+ t.Errorf("header row = %q", text)+ }+}+// TestCSVParserHTML4ExcelIsIgnored pins the retirement: html4excel is still// accepted at the entry (with a deprecation warning) but selects nothing —// the wire is identical to the default path.diff --git a/internal/parser/parser/json_parser.go b/internal/parser/parser/json_parser.goindex 9f0481aa9c..22523895e0 100644--- a/internal/parser/parser/json_parser.go+++ b/internal/parser/parser/json_parser.go@@ -53,14 +53,15 @@ func (p *JSONParser) String() string {// shape (array, single object, or line-delimited) and emits one JSON// item per logical record, with {text, doc_type_kwd:"text"}.func (p *JSONParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {- text := string(bytes.TrimSpace(data))+ decoded, encName := DecodeToUTF8(data, "application/json")+ text := string(bytes.TrimSpace(decoded))if text == "" {return ParseResult{OutputFormat: "json",File: map[string]any{"name": filename,"size": len(data),- "encoding": "utf-8",+ "encoding": encName,},JSON: []map[string]any{{"text": "", "doc_type_kwd": "text"}},}@@ -75,7 +76,7 @@ func (p *JSONParser) ParseWithResult(ctx context.Context, filename string, dataFile: map[string]any{"name": filename,"size": len(data),- "encoding": "utf-8",+ "encoding": encName,},JSON: items,}diff --git a/internal/parser/parser/json_parser_test.go b/internal/parser/parser/json_parser_test.gonew file mode 100644index 0000000000..6abb6b5ae3--- /dev/null+++ b/internal/parser/parser/json_parser_test.go@@ -0,0 +1,85 @@+package parser++import (+ "context"+ "testing"++ "golang.org/x/text/encoding/simplifiedchinese"+ "golang.org/x/text/encoding/unicode"+)++func jsonTexts(res ParseResult) []string {+ texts := make([]string, 0, len(res.JSON))+ for _, item := range res.JSON {+ text, _ := item["text"].(string)+ texts = append(texts, text)+ }+ return texts+}++func assertJSONTexts(t *testing.T, res ParseResult, want ...string) {+ t.Helper()+ got := jsonTexts(res)+ if len(got) != len(want) {+ t.Fatalf("got %d items %q, want %q", len(got), got, want)+ }+ for i := range want {+ if got[i] != want[i] {+ t.Errorf("item %d = %q, want %q", i, got[i], want[i])+ }+ }+}++func TestJSONParser_SplitsArrayAndLines(t *testing.T) {+ p := NewJSONParser()+ res := p.ParseWithResult(context.Background(), "a.json", []byte(`[{"q":"a"},{"q":"b"}]`))+ assertJSONTexts(t, res, `{"q":"a"}`, `{"q":"b"}`)++ res = p.ParseWithResult(context.Background(), "a.jsonl", []byte("{\"q\":\"a\"}\n{\"q\":\"b\"}\n"))+ assertJSONTexts(t, res, `{"q":"a"}`, `{"q":"b"}`)+}++// Files saved by Windows editors and Excel start with a UTF-8 BOM. The shape+// detection looks at the first byte, so an undecoded BOM turned the whole+// file into a single raw text item.+func TestJSONParser_UTF8BOM(t *testing.T) {+ p := NewJSONParser()+ bom := "\xef\xbb\xbf"++ res := p.ParseWithResult(context.Background(), "a.json", []byte(bom+`[{"q":"a"},{"q":"b"}]`))+ assertJSONTexts(t, res, `{"q":"a"}`, `{"q":"b"}`)++ res = p.ParseWithResult(context.Background(), "a.jsonl", []byte(bom+"{\"q\":\"a\"}\n{\"q\":\"b\"}\n"))+ assertJSONTexts(t, res, `{"q":"a"}`, `{"q":"b"}`)+}++// Non-UTF-8 files are decoded before json.Unmarshal, which would otherwise+// replace every invalid byte with U+FFFD.+func TestJSONParser_DecodesNonUTF8(t *testing.T) {+ src := `[{"q":"中文问题"}]`+ gbk, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(src))+ if err != nil {+ t.Fatalf("GBK encode: %v", err)+ }+ utf16le, err := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder().Bytes([]byte(src))+ if err != nil {+ t.Fatalf("UTF-16LE encode: %v", err)+ }+ cases := []struct {+ name string+ data []byte+ encoding string+ }{+ {"gbk", gbk, "gb18030"},+ {"utf-16le", utf16le, "utf-16le"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ res := NewJSONParser().ParseWithResult(context.Background(), "a.json", tc.data)+ assertJSONTexts(t, res, `{"q":"中文问题"}`)+ if res.File["encoding"] != tc.encoding {+ t.Errorf("encoding = %v, want %q", res.File["encoding"], tc.encoding)+ }+ })+ }+}diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.goindex 49e4431993..606501a266 100644--- a/internal/parser/parser/markdown_parser.go+++ b/internal/parser/parser/markdown_parser.go@@ -96,7 +96,8 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {// the base64-encoded image payload. The legacy debug-print path has// been removed; callers consume ParseResult directly.func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {- rawText := string(data)+ decoded, _ := DecodeToUTF8(data, "text/markdown")+ rawText := string(decoded)// Render any GFM/HTML table inline as an HTML block before parsing. This// keeps the document as one item per top-level block (the table becomes a// normal text item) instead of collapsing the whole document into a singlediff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.goindex 07e475c720..d91241b1e2 100644--- a/internal/parser/parser/markdown_parser_test.go+++ b/internal/parser/parser/markdown_parser_test.go@@ -37,6 +37,24 @@ func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) {}}+// A UTF-8 BOM in front of the first heading must not demote it to a paragraph.+func TestMarkdownParser_ParseWithResult_UTF8BOM(t *testing.T) {+ p, err := NewMarkdownParser(GoMarkdown)+ if err != nil {+ t.Fatalf("NewMarkdownParser: %v", err)+ }+ res := p.ParseWithResult(t.Context(), "bom.md", []byte("\xef\xbb\xbf# Hello\n\nBody.\n"))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if got, _ := res.JSON[0]["text"].(string); got != "# Hello" {+ t.Fatalf("first item text = %q, want %q", got, "# Hello")+ }+ if got, _ := res.JSON[0]["ck_type"].(string); got != "heading" {+ t.Fatalf("first item ck_type = %q, want %q", got, "heading")+ }+}+// TestMarkdownParser_ListItemsKeepWordBoundaries pins the shape of a list in the parsed output:// one item per entry, marker included, and no gluing between entries.//