Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): drop the BOM and decode JSON and Markdown input in Go parsers (#20225)

Jinebb71552a265
diff --git a/internal/parser/parser/charset.go b/internal/parser/parser/charset.go
index 9b8a685511..6db1208a92 100644
--- a/internal/parser/parser/charset.go
+++ b/internal/parser/parser/charset.go
@@ -231,8 +231,13 @@ func extractCharsetHint(hint string) string {
return ""
}
+// utf8BOM is the byte-order mark as it appears once the payload is UTF-8.
+var utf8BOM = []byte("\ufeff")
+
// DecodeToUTF8 converts arbitrary data bytes into UTF-8.
// If data is already valid UTF-8, it returns data untouched with encoding "utf-8".
+// A leading byte-order mark is dropped on every path, as Python's decode_text
+// does, so it never leaks into the first heading, CSV header, or JSON token.
// hint can be:
// - An explicit charset label (e.g. "gbk", "gb2312", "big5", from MIME headers)
// - A MIME Content-Type (e.g. "text/html", "application/xhtml+xml", "text/csv", "text/plain")
@@ -240,6 +245,11 @@ func extractCharsetHint(hint string) string {
//
// It returns the decoded UTF-8 bytes and the canonical encoding label used.
func DecodeToUTF8(data []byte, hint string) ([]byte, string) {
+ decoded, label := decodeToUTF8(data, hint)
+ return bytes.TrimPrefix(decoded, utf8BOM), label
+}
+
+func decodeToUTF8(data []byte, hint string) ([]byte, string) {
if len(data) == 0 {
return data, "utf-8"
}
diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.go
index 26383ccaf0..7eccf252f8 100644
--- a/internal/parser/parser/charset_test.go
+++ b/internal/parser/parser/charset_test.go
@@ -299,3 +299,45 @@ func TestDecodeToUTF8_UTF16LEBOM(t *testing.T) {
t.Errorf("got %q, want containing %q", string(out), src)
}
}
+
+// A byte-order mark marks the encoding; it is not content. Python's
+// decode_text drops it, so a leading U+FEFF must not reach the parsers.
+func TestDecodeToUTF8_DropsByteOrderMark(t *testing.T) {
+ src := "name,备注\n"
+ utf16le, err := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder().Bytes([]byte(src))
+ if err != nil {
+ t.Fatalf("UTF-16LE encode: %v", err)
+ }
+ utf16be, err := unicode.UTF16(unicode.BigEndian, unicode.UseBOM).NewEncoder().Bytes([]byte(src))
+ if err != nil {
+ t.Fatalf("UTF-16BE encode: %v", err)
+ }
+ cases := []struct {
+ name string
+ data []byte
+ label string
+ }{
+ {"utf-8", append([]byte("\xef\xbb\xbf"), src...), "utf-8"},
+ {"utf-16le", utf16le, "utf-16le"},
+ {"utf-16be", utf16be, "utf-16be"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ out, label := DecodeToUTF8(tc.data, "text/plain")
+ if string(out) != src {
+ t.Errorf("got %q, want %q", out, src)
+ }
+ if label != tc.label {
+ t.Errorf("got label %q, want %q", label, tc.label)
+ }
+ })
+ }
+}
+
+func TestDecodeToUTF8_KeepsNonLeadingZeroWidthNoBreakSpace(t *testing.T) {
+ src := "a\ufeffb"
+ out, _ := DecodeToUTF8([]byte(src), "text/plain")
+ if string(out) != src {
+ t.Errorf("got %q, want %q", out, src)
+ }
+}
diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.go
index 4e45e23515..f50c38cec9 100644
--- a/internal/parser/parser/csv_parser_test.go
+++ b/internal/parser/parser/csv_parser_test.go
@@ -59,6 +59,22 @@ func TestCSVParser_EmitsSpreadsheetRows(t *testing.T) {
}
}
+// Excel's "CSV UTF-8" export starts with a BOM; it must not become part of
+// the first column name.
+func TestCSVParser_UTF8BOM(t *testing.T) {
+ res := NewCSVParser().ParseWithResult(context.Background(), "bom.csv", []byte("\xef\xbb\xbfName,Age\nAlice,30\n"))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult failed: %v", res.Err)
+ }
+ if len(res.JSON) != 1 {
+ t.Fatalf("items = %#v, want one table item", res.JSON)
+ }
+ text, _ := res.JSON[0]["text"].(string)
+ if !strings.Contains(text, "<tr><th>Name</th><th>Age</th></tr>") {
+ t.Errorf("header row = %q", text)
+ }
+}
+
// TestCSVParserHTML4ExcelIsIgnored pins the retirement: html4excel is still
// accepted at the entry (with a deprecation warning) but selects nothing —
// the wire is identical to the default path.
diff --git a/internal/parser/parser/json_parser.go b/internal/parser/parser/json_parser.go
index 9f0481aa9c..22523895e0 100644
--- a/internal/parser/parser/json_parser.go
+++ b/internal/parser/parser/json_parser.go
@@ -53,14 +53,15 @@ func (p *JSONParser) String() string {
// shape (array, single object, or line-delimited) and emits one JSON
// item per logical record, with {text, doc_type_kwd:"text"}.
func (p *JSONParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
- text := string(bytes.TrimSpace(data))
+ decoded, encName := DecodeToUTF8(data, "application/json")
+ text := string(bytes.TrimSpace(decoded))
if text == "" {
return ParseResult{
OutputFormat: "json",
File: map[string]any{
"name": filename,
"size": len(data),
- "encoding": "utf-8",
+ "encoding": encName,
},
JSON: []map[string]any{{"text": "", "doc_type_kwd": "text"}},
}
@@ -75,7 +76,7 @@ func (p *JSONParser) ParseWithResult(ctx context.Context, filename string, data
File: map[string]any{
"name": filename,
"size": len(data),
- "encoding": "utf-8",
+ "encoding": encName,
},
JSON: items,
}
diff --git a/internal/parser/parser/json_parser_test.go b/internal/parser/parser/json_parser_test.go
new file mode 100644
index 0000000000..6abb6b5ae3
--- /dev/null
+++ b/internal/parser/parser/json_parser_test.go
@@ -0,0 +1,85 @@
+package parser
+
+import (
+ "context"
+ "testing"
+
+ "golang.org/x/text/encoding/simplifiedchinese"
+ "golang.org/x/text/encoding/unicode"
+)
+
+func jsonTexts(res ParseResult) []string {
+ texts := make([]string, 0, len(res.JSON))
+ for _, item := range res.JSON {
+ text, _ := item["text"].(string)
+ texts = append(texts, text)
+ }
+ return texts
+}
+
+func assertJSONTexts(t *testing.T, res ParseResult, want ...string) {
+ t.Helper()
+ got := jsonTexts(res)
+ if len(got) != len(want) {
+ t.Fatalf("got %d items %q, want %q", len(got), got, want)
+ }
+ for i := range want {
+ if got[i] != want[i] {
+ t.Errorf("item %d = %q, want %q", i, got[i], want[i])
+ }
+ }
+}
+
+func TestJSONParser_SplitsArrayAndLines(t *testing.T) {
+ p := NewJSONParser()
+ res := p.ParseWithResult(context.Background(), "a.json", []byte(`[{"q":"a"},{"q":"b"}]`))
+ assertJSONTexts(t, res, `{"q":"a"}`, `{"q":"b"}`)
+
+ res = p.ParseWithResult(context.Background(), "a.jsonl", []byte("{\"q\":\"a\"}\n{\"q\":\"b\"}\n"))
+ assertJSONTexts(t, res, `{"q":"a"}`, `{"q":"b"}`)
+}
+
+// Files saved by Windows editors and Excel start with a UTF-8 BOM. The shape
+// detection looks at the first byte, so an undecoded BOM turned the whole
+// file into a single raw text item.
+func TestJSONParser_UTF8BOM(t *testing.T) {
+ p := NewJSONParser()
+ bom := "\xef\xbb\xbf"
+
+ res := p.ParseWithResult(context.Background(), "a.json", []byte(bom+`[{"q":"a"},{"q":"b"}]`))
+ assertJSONTexts(t, res, `{"q":"a"}`, `{"q":"b"}`)
+
+ res = p.ParseWithResult(context.Background(), "a.jsonl", []byte(bom+"{\"q\":\"a\"}\n{\"q\":\"b\"}\n"))
+ assertJSONTexts(t, res, `{"q":"a"}`, `{"q":"b"}`)
+}
+
+// Non-UTF-8 files are decoded before json.Unmarshal, which would otherwise
+// replace every invalid byte with U+FFFD.
+func TestJSONParser_DecodesNonUTF8(t *testing.T) {
+ src := `[{"q":"中文问题"}]`
+ gbk, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(src))
+ if err != nil {
+ t.Fatalf("GBK encode: %v", err)
+ }
+ utf16le, err := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder().Bytes([]byte(src))
+ if err != nil {
+ t.Fatalf("UTF-16LE encode: %v", err)
+ }
+ cases := []struct {
+ name string
+ data []byte
+ encoding string
+ }{
+ {"gbk", gbk, "gb18030"},
+ {"utf-16le", utf16le, "utf-16le"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ res := NewJSONParser().ParseWithResult(context.Background(), "a.json", tc.data)
+ assertJSONTexts(t, res, `{"q":"中文问题"}`)
+ if res.File["encoding"] != tc.encoding {
+ t.Errorf("encoding = %v, want %q", res.File["encoding"], tc.encoding)
+ }
+ })
+ }
+}
diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go
index 49e4431993..606501a266 100644
--- a/internal/parser/parser/markdown_parser.go
+++ b/internal/parser/parser/markdown_parser.go
@@ -96,7 +96,8 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {
// the base64-encoded image payload. The legacy debug-print path has
// been removed; callers consume ParseResult directly.
func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
- rawText := string(data)
+ decoded, _ := DecodeToUTF8(data, "text/markdown")
+ rawText := string(decoded)
// Render any GFM/HTML table inline as an HTML block before parsing. This
// keeps the document as one item per top-level block (the table becomes a
// normal text item) instead of collapsing the whole document into a single
diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go
index 07e475c720..d91241b1e2 100644
--- a/internal/parser/parser/markdown_parser_test.go
+++ b/internal/parser/parser/markdown_parser_test.go
@@ -37,6 +37,24 @@ func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) {
}
}
+// A UTF-8 BOM in front of the first heading must not demote it to a paragraph.
+func TestMarkdownParser_ParseWithResult_UTF8BOM(t *testing.T) {
+ p, err := NewMarkdownParser(GoMarkdown)
+ if err != nil {
+ t.Fatalf("NewMarkdownParser: %v", err)
+ }
+ res := p.ParseWithResult(t.Context(), "bom.md", []byte("\xef\xbb\xbf# Hello\n\nBody.\n"))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if got, _ := res.JSON[0]["text"].(string); got != "# Hello" {
+ t.Fatalf("first item text = %q, want %q", got, "# Hello")
+ }
+ if got, _ := res.JSON[0]["ck_type"].(string); got != "heading" {
+ t.Fatalf("first item ck_type = %q, want %q", got, "heading")
+ }
+}
+
// TestMarkdownParser_ListItemsKeepWordBoundaries pins the shape of a list in the parsed output:
// one item per entry, marker included, and no gluing between entries.
//