Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docparser): keep JSON numbers and <, >, & as written in JSON uploads (#4436)
Lukase2a58ea3cbfa
diff --git a/internal/infrastructure/docparser/json_converter.go b/internal/infrastructure/docparser/json_converter.goindex a637ab81b..78fdd3ba3 100644--- a/internal/infrastructure/docparser/json_converter.go+++ b/internal/infrastructure/docparser/json_converter.go@@ -1,6 +1,7 @@package docparserimport (+ "bytes""encoding/json""errors""fmt"@@ -52,8 +53,14 @@ func jsonToMarkdown(data []byte) (string, error) {}data = trimBOM(data)+ // UseNumber keeps each number as written. Decoded into a float64, an id+ // above 2^53 changed digits: 12345678901234567890 came out as+ // 12345678901234567000. ValidateJSONContent above has already rejected+ // anything after the value.+ decoder := json.NewDecoder(bytes.NewReader(data))+ decoder.UseNumber()var parsed interface{}- if err := json.Unmarshal(data, &parsed); err != nil {+ if err := decoder.Decode(&parsed); err != nil {return "", fmt.Errorf("failed to parse JSON: %w", err)}@@ -206,7 +213,7 @@ func listToDictPreprocess(data interface{}) interface{} {// jsonSize returns the serialized JSON byte length of a value.func jsonSize(v interface{}) int {- b, err := json.Marshal(v)+ b, err := marshalJSON(v, "")if err != nil {return 0}@@ -215,13 +222,28 @@ func jsonSize(v interface{}) int {// formatValue formats a JSON value with indentation.func formatValue(v interface{}) string {- b, err := json.MarshalIndent(v, "", " ")+ b, err := marshalJSON(v, " ")if err != nil {- b, _ = json.Marshal(v)+ b, _ = marshalJSON(v, "")}return string(b)}+// marshalJSON encodes v like json.Marshal, indented when indent is set, but+// leaves <, > and & as they are. json.Marshal writes them as \u003c, \u003e+// and \u0026, which is meant for embedding in HTML and only garbles the text+// a chunk is read and searched as.+func marshalJSON(v interface{}, indent string) ([]byte, error) {+ var buf bytes.Buffer+ encoder := json.NewEncoder(&buf)+ encoder.SetEscapeHTML(false)+ encoder.SetIndent("", indent)+ if err := encoder.Encode(v); err != nil {+ return nil, err+ }+ return bytes.TrimSuffix(buf.Bytes(), []byte("\n")), nil+}+// wrapCodeBlock wraps content in a fenced JSON code block.func wrapCodeBlock(content string) string {return "```json\n" + content + "\n```"diff --git a/internal/infrastructure/docparser/json_converter_test.go b/internal/infrastructure/docparser/json_converter_test.goindex 15c36e750..9d5c1575d 100644--- a/internal/infrastructure/docparser/json_converter_test.go+++ b/internal/infrastructure/docparser/json_converter_test.go@@ -386,6 +386,50 @@ func TestJsonToMarkdown_SpecialCharsInStrings(t *testing.T) {})}+func TestJsonToMarkdown_KeepsNumbersAsWritten(t *testing.T) {+ // Decoded into a float64, an id above 2^53 changed digits.+ input := `{"order_id": 12345678901234567890, "big": 9007199254740993, "price": 1.50, "ratio": 1e5}`+ result, err := jsonToMarkdown([]byte(input))+ visualReport(t, "Numbers keep the digits they were written with", input, result, err, []checkResult{+ check("no error", err == nil),+ check("all blocks are valid JSON", allBlocksValidJSON(result)),+ check("order_id keeps all 20 digits", strings.Contains(result, `"order_id": 12345678901234567890`)),+ check("2^53+1 is not rounded to 2^53", strings.Contains(result, `"big": 9007199254740993`)),+ check("1.50 keeps its trailing zero", strings.Contains(result, `"price": 1.50`)),+ check("1e5 keeps its exponent", strings.Contains(result, `"ratio": 1e5`)),+ })+}++func TestJsonToMarkdown_KeepsNumbersAsWrittenAcrossChunks(t *testing.T) {+ // Large enough to take the recursive split path.+ items := make([]string, 0, 60)+ for i := 0; i < 60; i++ {+ item := fmt.Sprintf(`"item_%02d": {"id": 1234567890123456789%d, "label": "entry %d"}`, i, i%10, i)+ items = append(items, item)+ }+ input := "{" + strings.Join(items, ", ") + "}"+ result, err := jsonToMarkdown([]byte(input))+ visualReport(t, "Numbers keep their digits in a split document", input, result, err, []checkResult{+ check("no error", err == nil),+ check("split into several blocks", len(splitBlocks(result)) > 1),+ check("first id kept", strings.Contains(result, `"id": 12345678901234567890`)),+ check("last id kept", strings.Contains(result, `"id": 12345678901234567899`)),+ })+}++func TestJsonToMarkdown_KeepsAngleBracketsAndAmpersands(t *testing.T) {+ // json.Marshal writes <, > and & as \u003c, \u003e and \u0026.+ input := `{"note": "a < b && c > d", "html": "<b>bold</b>"}`+ result, err := jsonToMarkdown([]byte(input))+ visualReport(t, "<, > and & stay readable", input, result, err, []checkResult{+ check("no error", err == nil),+ check("all blocks are valid JSON", allBlocksValidJSON(result)),+ check("comparison text kept", strings.Contains(result, `"note": "a < b && c > d"`)),+ check("markup kept", strings.Contains(result, `"html": "<b>bold</b>"`)),+ check("no \\u003c escapes", !strings.Contains(result, `\u003c`)),+ })+}+func TestJsonToMarkdown_NullValues(t *testing.T) {input := `{"name": "test", "value": null, "list": [null, 1, null]}`result, err := jsonToMarkdown([]byte(input))