RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): keep JSON numbers and <, >, & as written in JSON items (#21101)
Lukas4c2b098c9393
diff --git a/internal/parser/parser/json_parser.go b/internal/parser/parser/json_parser.goindex 491e03bc2b..e49862aa80 100644--- a/internal/parser/parser/json_parser.go+++ b/internal/parser/parser/json_parser.go@@ -34,7 +34,9 @@ import ("bytes""context""encoding/json"+ "errors""fmt"+ "io""strings")@@ -136,7 +138,7 @@ func isJSONL(text string) bool {// parseJSONArray unmarshals a JSON array and emits one item per element.func parseJSONArray(text string) []map[string]any {var arr []any- if err := json.Unmarshal([]byte(text), &arr); err != nil {+ if err := unmarshalJSON([]byte(text), &arr); err != nil {// Fallback: emit as plain text.return []map[string]any{{"text": text, "doc_type_kwd": "text"}}}@@ -152,7 +154,7 @@ func parseJSONArray(text string) []map[string]any {})} else {// Re-marshal non-string elements.- b, err := json.Marshal(elem)+ b, err := marshalJSON(elem)if err != nil {items = append(items, map[string]any{"text": fmt.Sprintf("%v", elem),@@ -172,10 +174,10 @@ func parseJSONArray(text string) []map[string]any {// parseSingleJSONObject unmarshals a single JSON object and emits one item.func parseSingleJSONObject(text string) []map[string]any {var obj map[string]any- if err := json.Unmarshal([]byte(text), &obj); err != nil {+ if err := unmarshalJSON([]byte(text), &obj); err != nil {return []map[string]any{{"text": text, "doc_type_kwd": "text"}}}- b, err := json.Marshal(obj)+ b, err := marshalJSON(obj)if err != nil {return []map[string]any{{"text": fmt.Sprintf("%v", obj), "doc_type_kwd": "text"}}}@@ -195,10 +197,10 @@ func parseJSONLines(text string) []map[string]any {}// Each line should be a JSON object.var obj map[string]any- if err := json.Unmarshal([]byte(line), &obj); err != nil {+ if err := unmarshalJSON([]byte(line), &obj); err != nil {// Also try as a string or other standalone value.var val any- if err2 := json.Unmarshal([]byte(line), &val); err2 != nil {+ if err2 := unmarshalJSON([]byte(line), &val); err2 != nil {// Unparseable line — skip, matching Python behaviour.continue} else {@@ -208,7 +210,7 @@ func parseJSONLines(text string) []map[string]any {"doc_type_kwd": "text",})} else {- b, _ := json.Marshal(val)+ b, _ := marshalJSON(val)items = append(items, map[string]any{"text": string(b),"doc_type_kwd": "text",@@ -217,7 +219,7 @@ func parseJSONLines(text string) []map[string]any {}continue}- b, err := json.Marshal(obj)+ b, err := marshalJSON(obj)if err != nil {continue}@@ -228,3 +230,32 @@ func parseJSONLines(text string) []map[string]any {}return items}++// unmarshalJSON is json.Unmarshal with each number kept as written, as a+// json.Number. Decoded into a float64, an id above 2^53 changed digits+// (12345678901234567890 became 12345678901234567000) and 1.50 became 1.5.+// Data after the value is an error, as it is for json.Unmarshal.+func unmarshalJSON(data []byte, v any) error {+ decoder := json.NewDecoder(bytes.NewReader(data))+ decoder.UseNumber()+ if err := decoder.Decode(v); err != nil {+ return err+ }+ if _, err := decoder.Token(); err != io.EOF {+ return errors.New("json: data after the top-level value")+ }+ return nil+}++// marshalJSON is json.Marshal without HTML escaping. json.Marshal writes+// <, > and & as \u003c, \u003e and \u0026, which is meant for embedding in+// HTML and garbles the text a chunk is searched and read as.+func marshalJSON(v any) ([]byte, error) {+ var buf bytes.Buffer+ encoder := json.NewEncoder(&buf)+ encoder.SetEscapeHTML(false)+ if err := encoder.Encode(v); err != nil {+ return nil, err+ }+ return bytes.TrimSuffix(buf.Bytes(), []byte("\n")), nil+}diff --git a/internal/parser/parser/json_parser_test.go b/internal/parser/parser/json_parser_test.goindex 08f12d3636..2f7e8c67f9 100644--- a/internal/parser/parser/json_parser_test.go+++ b/internal/parser/parser/json_parser_test.go@@ -39,6 +39,41 @@ func TestJSONParser_SplitsArrayAndLines(t *testing.T) {assertJSONTexts(t, res, `{"q":"a"}`, `{"q":"b"}`)}+// Each number keeps the digits it was written with, and <, > and & stay+// readable. Decoded into a float64 and re-encoded with json.Marshal, an id+// above 2^53 changed digits and "a < b" became "a \u003c b".+func TestJSONParser_KeepsValuesAsWritten(t *testing.T) {+ p := NewJSONParser()+ cases := []struct {+ name, file, data string+ want []string+ }{+ {+ name: "object",+ file: "a.json",+ data: `{"id": 12345678901234567890, "price": 1.50, "note": "a < b && c > d"}`,+ want: []string{`{"id":12345678901234567890,"note":"a < b && c > d","price":1.50}`},+ },+ {+ name: "array",+ file: "a.json",+ data: `[{"id": 9007199254740993}, 1e5]`,+ want: []string{`{"id":9007199254740993}`, `1e5`},+ },+ {+ name: "lines",+ file: "a.jsonl",+ data: "{\"id\": 12345678901234567890}\n{\"html\": \"<b>x</b>\"}\n",+ want: []string{`{"id":12345678901234567890}`, `{"html":"<b>x</b>"}`},+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ assertJSONTexts(t, p.ParseWithResult(context.Background(), tc.file, []byte(tc.data)), tc.want...)+ })+ }+}+// Files saved by Windows editors and Excel start with a UTF-8 BOM. The shape// detection looks at the first byte, so an undecoded BOM turned the whole// file into a single raw text item.