Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docparser): keep JSON numbers and <, >, & as written in JSON uploads (#4436)

Lukase2a58ea3cbfa
diff --git a/internal/infrastructure/docparser/json_converter.go b/internal/infrastructure/docparser/json_converter.go
index a637ab81b..78fdd3ba3 100644
--- a/internal/infrastructure/docparser/json_converter.go
+++ b/internal/infrastructure/docparser/json_converter.go
@@ -1,6 +1,7 @@
package docparser
import (
+ "bytes"
"encoding/json"
"errors"
"fmt"
@@ -52,8 +53,14 @@ func jsonToMarkdown(data []byte) (string, error) {
}
data = trimBOM(data)
+ // UseNumber keeps each number as written. Decoded into a float64, an id
+ // above 2^53 changed digits: 12345678901234567890 came out as
+ // 12345678901234567000. ValidateJSONContent above has already rejected
+ // anything after the value.
+ decoder := json.NewDecoder(bytes.NewReader(data))
+ decoder.UseNumber()
var parsed interface{}
- if err := json.Unmarshal(data, &parsed); err != nil {
+ if err := decoder.Decode(&parsed); err != nil {
return "", fmt.Errorf("failed to parse JSON: %w", err)
}
@@ -206,7 +213,7 @@ func listToDictPreprocess(data interface{}) interface{} {
// jsonSize returns the serialized JSON byte length of a value.
func jsonSize(v interface{}) int {
- b, err := json.Marshal(v)
+ b, err := marshalJSON(v, "")
if err != nil {
return 0
}
@@ -215,13 +222,28 @@ func jsonSize(v interface{}) int {
// formatValue formats a JSON value with indentation.
func formatValue(v interface{}) string {
- b, err := json.MarshalIndent(v, "", " ")
+ b, err := marshalJSON(v, " ")
if err != nil {
- b, _ = json.Marshal(v)
+ b, _ = marshalJSON(v, "")
}
return string(b)
}
+// marshalJSON encodes v like json.Marshal, indented when indent is set, but
+// leaves <, > and & as they are. json.Marshal writes them as \u003c, \u003e
+// and \u0026, which is meant for embedding in HTML and only garbles the text
+// a chunk is read and searched as.
+func marshalJSON(v interface{}, indent string) ([]byte, error) {
+ var buf bytes.Buffer
+ encoder := json.NewEncoder(&buf)
+ encoder.SetEscapeHTML(false)
+ encoder.SetIndent("", indent)
+ if err := encoder.Encode(v); err != nil {
+ return nil, err
+ }
+ return bytes.TrimSuffix(buf.Bytes(), []byte("\n")), nil
+}
+
// wrapCodeBlock wraps content in a fenced JSON code block.
func wrapCodeBlock(content string) string {
return "```json\n" + content + "\n```"
diff --git a/internal/infrastructure/docparser/json_converter_test.go b/internal/infrastructure/docparser/json_converter_test.go
index 15c36e750..9d5c1575d 100644
--- a/internal/infrastructure/docparser/json_converter_test.go
+++ b/internal/infrastructure/docparser/json_converter_test.go
@@ -386,6 +386,50 @@ func TestJsonToMarkdown_SpecialCharsInStrings(t *testing.T) {
})
}
+func TestJsonToMarkdown_KeepsNumbersAsWritten(t *testing.T) {
+ // Decoded into a float64, an id above 2^53 changed digits.
+ input := `{"order_id": 12345678901234567890, "big": 9007199254740993, "price": 1.50, "ratio": 1e5}`
+ result, err := jsonToMarkdown([]byte(input))
+ visualReport(t, "Numbers keep the digits they were written with", input, result, err, []checkResult{
+ check("no error", err == nil),
+ check("all blocks are valid JSON", allBlocksValidJSON(result)),
+ check("order_id keeps all 20 digits", strings.Contains(result, `"order_id": 12345678901234567890`)),
+ check("2^53+1 is not rounded to 2^53", strings.Contains(result, `"big": 9007199254740993`)),
+ check("1.50 keeps its trailing zero", strings.Contains(result, `"price": 1.50`)),
+ check("1e5 keeps its exponent", strings.Contains(result, `"ratio": 1e5`)),
+ })
+}
+
+func TestJsonToMarkdown_KeepsNumbersAsWrittenAcrossChunks(t *testing.T) {
+ // Large enough to take the recursive split path.
+ items := make([]string, 0, 60)
+ for i := 0; i < 60; i++ {
+ item := fmt.Sprintf(`"item_%02d": {"id": 1234567890123456789%d, "label": "entry %d"}`, i, i%10, i)
+ items = append(items, item)
+ }
+ input := "{" + strings.Join(items, ", ") + "}"
+ result, err := jsonToMarkdown([]byte(input))
+ visualReport(t, "Numbers keep their digits in a split document", input, result, err, []checkResult{
+ check("no error", err == nil),
+ check("split into several blocks", len(splitBlocks(result)) > 1),
+ check("first id kept", strings.Contains(result, `"id": 12345678901234567890`)),
+ check("last id kept", strings.Contains(result, `"id": 12345678901234567899`)),
+ })
+}
+
+func TestJsonToMarkdown_KeepsAngleBracketsAndAmpersands(t *testing.T) {
+ // json.Marshal writes <, > and & as \u003c, \u003e and \u0026.
+ input := `{"note": "a < b && c > d", "html": "<b>bold</b>"}`
+ result, err := jsonToMarkdown([]byte(input))
+ visualReport(t, "<, > and & stay readable", input, result, err, []checkResult{
+ check("no error", err == nil),
+ check("all blocks are valid JSON", allBlocksValidJSON(result)),
+ check("comparison text kept", strings.Contains(result, `"note": "a < b && c > d"`)),
+ check("markup kept", strings.Contains(result, `"html": "<b>bold</b>"`)),
+ check("no \\u003c escapes", !strings.Contains(result, `\u003c`)),
+ })
+}
+
func TestJsonToMarkdown_NullValues(t *testing.T) {
input := `{"name": "test", "value": null, "list": [null, 1, null]}`
result, err := jsonToMarkdown([]byte(input))