Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): keep JSON numbers and <, >, & as written in JSON items (#21101)

Lukas4c2b098c9393
diff --git a/internal/parser/parser/json_parser.go b/internal/parser/parser/json_parser.go
index 491e03bc2b..e49862aa80 100644
--- a/internal/parser/parser/json_parser.go
+++ b/internal/parser/parser/json_parser.go
@@ -34,7 +34,9 @@ import (
"bytes"
"context"
"encoding/json"
+ "errors"
"fmt"
+ "io"
"strings"
)
@@ -136,7 +138,7 @@ func isJSONL(text string) bool {
// parseJSONArray unmarshals a JSON array and emits one item per element.
func parseJSONArray(text string) []map[string]any {
var arr []any
- if err := json.Unmarshal([]byte(text), &arr); err != nil {
+ if err := unmarshalJSON([]byte(text), &arr); err != nil {
// Fallback: emit as plain text.
return []map[string]any{{"text": text, "doc_type_kwd": "text"}}
}
@@ -152,7 +154,7 @@ func parseJSONArray(text string) []map[string]any {
})
} else {
// Re-marshal non-string elements.
- b, err := json.Marshal(elem)
+ b, err := marshalJSON(elem)
if err != nil {
items = append(items, map[string]any{
"text": fmt.Sprintf("%v", elem),
@@ -172,10 +174,10 @@ func parseJSONArray(text string) []map[string]any {
// parseSingleJSONObject unmarshals a single JSON object and emits one item.
func parseSingleJSONObject(text string) []map[string]any {
var obj map[string]any
- if err := json.Unmarshal([]byte(text), &obj); err != nil {
+ if err := unmarshalJSON([]byte(text), &obj); err != nil {
return []map[string]any{{"text": text, "doc_type_kwd": "text"}}
}
- b, err := json.Marshal(obj)
+ b, err := marshalJSON(obj)
if err != nil {
return []map[string]any{{"text": fmt.Sprintf("%v", obj), "doc_type_kwd": "text"}}
}
@@ -195,10 +197,10 @@ func parseJSONLines(text string) []map[string]any {
}
// Each line should be a JSON object.
var obj map[string]any
- if err := json.Unmarshal([]byte(line), &obj); err != nil {
+ if err := unmarshalJSON([]byte(line), &obj); err != nil {
// Also try as a string or other standalone value.
var val any
- if err2 := json.Unmarshal([]byte(line), &val); err2 != nil {
+ if err2 := unmarshalJSON([]byte(line), &val); err2 != nil {
// Unparseable line — skip, matching Python behaviour.
continue
} else {
@@ -208,7 +210,7 @@ func parseJSONLines(text string) []map[string]any {
"doc_type_kwd": "text",
})
} else {
- b, _ := json.Marshal(val)
+ b, _ := marshalJSON(val)
items = append(items, map[string]any{
"text": string(b),
"doc_type_kwd": "text",
@@ -217,7 +219,7 @@ func parseJSONLines(text string) []map[string]any {
}
continue
}
- b, err := json.Marshal(obj)
+ b, err := marshalJSON(obj)
if err != nil {
continue
}
@@ -228,3 +230,32 @@ func parseJSONLines(text string) []map[string]any {
}
return items
}
+
+// unmarshalJSON is json.Unmarshal with each number kept as written, as a
+// json.Number. Decoded into a float64, an id above 2^53 changed digits
+// (12345678901234567890 became 12345678901234567000) and 1.50 became 1.5.
+// Data after the value is an error, as it is for json.Unmarshal.
+func unmarshalJSON(data []byte, v any) error {
+ decoder := json.NewDecoder(bytes.NewReader(data))
+ decoder.UseNumber()
+ if err := decoder.Decode(v); err != nil {
+ return err
+ }
+ if _, err := decoder.Token(); err != io.EOF {
+ return errors.New("json: data after the top-level value")
+ }
+ return nil
+}
+
+// marshalJSON is json.Marshal without HTML escaping. json.Marshal writes
+// <, > and & as \u003c, \u003e and \u0026, which is meant for embedding in
+// HTML and garbles the text a chunk is searched and read as.
+func marshalJSON(v any) ([]byte, error) {
+ var buf bytes.Buffer
+ encoder := json.NewEncoder(&buf)
+ encoder.SetEscapeHTML(false)
+ if err := encoder.Encode(v); err != nil {
+ return nil, err
+ }
+ return bytes.TrimSuffix(buf.Bytes(), []byte("\n")), nil
+}
diff --git a/internal/parser/parser/json_parser_test.go b/internal/parser/parser/json_parser_test.go
index 08f12d3636..2f7e8c67f9 100644
--- a/internal/parser/parser/json_parser_test.go
+++ b/internal/parser/parser/json_parser_test.go
@@ -39,6 +39,41 @@ func TestJSONParser_SplitsArrayAndLines(t *testing.T) {
assertJSONTexts(t, res, `{"q":"a"}`, `{"q":"b"}`)
}
+// Each number keeps the digits it was written with, and <, > and & stay
+// readable. Decoded into a float64 and re-encoded with json.Marshal, an id
+// above 2^53 changed digits and "a < b" became "a \u003c b".
+func TestJSONParser_KeepsValuesAsWritten(t *testing.T) {
+ p := NewJSONParser()
+ cases := []struct {
+ name, file, data string
+ want []string
+ }{
+ {
+ name: "object",
+ file: "a.json",
+ data: `{"id": 12345678901234567890, "price": 1.50, "note": "a < b && c > d"}`,
+ want: []string{`{"id":12345678901234567890,"note":"a < b && c > d","price":1.50}`},
+ },
+ {
+ name: "array",
+ file: "a.json",
+ data: `[{"id": 9007199254740993}, 1e5]`,
+ want: []string{`{"id":9007199254740993}`, `1e5`},
+ },
+ {
+ name: "lines",
+ file: "a.jsonl",
+ data: "{\"id\": 12345678901234567890}\n{\"html\": \"<b>x</b>\"}\n",
+ want: []string{`{"id":12345678901234567890}`, `{"html":"<b>x</b>"}`},
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ assertJSONTexts(t, p.ParseWithResult(context.Background(), tc.file, []byte(tc.data)), tc.want...)
+ })
+ }
+}
+
// Files saved by Windows editors and Excel start with a UTF-8 BOM. The shape
// detection looks at the first byte, so an undecoded BOM turned the whole
// file into a single raw text item.