Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(models): preserve string fields in streamed tool arguments (#4253)

Rudy Mizrahi Celeklicdd70b35d522
diff --git a/internal/models/api/json_field_extractor.go b/internal/models/api/json_field_extractor.go
index 08f050794..c9bb47c94 100644
--- a/internal/models/api/json_field_extractor.go
+++ b/internal/models/api/json_field_extractor.go
@@ -2,6 +2,7 @@ package api
import (
"strings"
+ "unicode/utf16"
"unicode/utf8"
)
@@ -79,36 +80,35 @@ func (e *JSONFieldExtractor) IsDone() bool {
// findFieldValueStart finds the byte offset where the field's string value content begins
// (after the opening quote of the value). Returns -1 if not found.
func findFieldValueStart(buf string, fieldName string) int {
- // Look for "fieldName" key followed by colon and opening quote
key := `"` + fieldName + `"`
- idx := strings.Index(buf, key)
- if idx < 0 {
- return -1
- }
-
- // Skip past the key
- pos := idx + len(key)
-
- // Skip whitespace and colon
- for pos < len(buf) {
- ch := buf[pos]
- if ch == ':' {
+ for offset := 0; offset < len(buf); {
+ idx := strings.Index(buf[offset:], key)
+ if idx < 0 {
+ return -1
+ }
+ pos := offset + idx + len(key)
+ offset = pos
+ for pos < len(buf) && strings.ContainsRune(" \t\n\r", rune(buf[pos])) {
pos++
+ }
+ if pos >= len(buf) {
+ return -1
+ }
+ if buf[pos] != ':' {
continue
}
- if ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' {
+ pos++
+ for pos < len(buf) && strings.ContainsRune(" \t\n\r", rune(buf[pos])) {
pos++
- continue
}
- if ch == '"' {
- // Found the opening quote of the value
+ if pos >= len(buf) {
+ return -1
+ }
+ if buf[pos] == '"' {
return pos + 1
}
- // Unexpected character
- return -1
}
-
- return -1 // Haven't seen the opening quote yet
+ return -1
}
// findSafeEnd finds the safe end position for emission within the value content.
@@ -130,6 +130,17 @@ func findSafeEnd(value string, from int) (int, bool) {
if i+5 >= len(value) {
return i, false
}
+ if isHighSurrogate(unicodeEscape(value[i+2 : i+6])) {
+ rest := value[i+6:]
+ // A high surrogate may pair with the next streamed escape.
+ if rest == "" || rest == `\` || (strings.HasPrefix(rest, `\u`) && len(rest) < 6) {
+ return i, false
+ }
+ if len(rest) >= 6 && strings.HasPrefix(rest, `\u`) && isLowSurrogate(unicodeEscape(rest[2:6])) {
+ i += 12
+ continue
+ }
+ }
i += 6
} else {
// Simple escape: \", \\, \n, \t, \r, \/, \b, \f
@@ -190,21 +201,16 @@ func unescapeJSONString(s string) string {
case 'u':
// Unicode escape \uXXXX
if i+5 < len(s) {
- // Parse hex digits
- hexStr := s[i+2 : i+6]
- var codepoint int
- for _, h := range hexStr {
- codepoint <<= 4
- switch {
- case h >= '0' && h <= '9':
- codepoint += int(h - '0')
- case h >= 'a' && h <= 'f':
- codepoint += int(h-'a') + 10
- case h >= 'A' && h <= 'F':
- codepoint += int(h-'A') + 10
+ codepoint := unicodeEscape(s[i+2 : i+6])
+ if isHighSurrogate(codepoint) && i+11 < len(s) && s[i+6:i+8] == `\u` {
+ low := unicodeEscape(s[i+8 : i+12])
+ if isLowSurrogate(low) {
+ b.WriteRune(utf16.DecodeRune(codepoint, low))
+ i += 12
+ continue
}
}
- b.WriteRune(rune(codepoint))
+ b.WriteRune(codepoint)
i += 6
} else {
b.WriteByte(s[i])
@@ -222,3 +228,22 @@ func unescapeJSONString(s string) string {
return b.String()
}
+
+func unicodeEscape(hex string) rune {
+ var codepoint rune
+ for _, h := range hex {
+ codepoint <<= 4
+ switch {
+ case h >= '0' && h <= '9':
+ codepoint += h - '0'
+ case h >= 'a' && h <= 'f':
+ codepoint += h - 'a' + 10
+ case h >= 'A' && h <= 'F':
+ codepoint += h - 'A' + 10
+ }
+ }
+ return codepoint
+}
+
+func isHighSurrogate(r rune) bool { return r >= 0xd800 && r <= 0xdbff }
+func isLowSurrogate(r rune) bool { return r >= 0xdc00 && r <= 0xdfff }
diff --git a/internal/models/api/json_field_extractor_semantics_test.go b/internal/models/api/json_field_extractor_semantics_test.go
new file mode 100644
index 000000000..ae1f6f40a
--- /dev/null
+++ b/internal/models/api/json_field_extractor_semantics_test.go
@@ -0,0 +1,110 @@
+package api
+
+import (
+ "encoding/json"
+ "testing"
+ "unicode/utf8"
+)
+
+func TestJSONFieldExtractorSupplementaryUnicodeMatchesJSON(t *testing.T) {
+ tests := []struct {
+ name string
+ field string
+ arguments string
+ }{
+ {"content pair", "content", `{"path":"report.txt","content":"Launch \ud83d\ude80\n"}`},
+ {"path pair", "path", `{"path":"\ud83d\udcdd.txt","content":"done"}`},
+ {"two pairs", "content", `{"path":"report.txt","content":"\uD83D\uDC69\u200d\uD83D\uDD2C"}`},
+ {"thinking pair", "thought", `{"thought":"Delivery \ud83d\ude80","next_thought_needed":false}`},
+ {"BMP control", "content", `{"path":"report.txt","content":"\u4e16\u754c"}`},
+ {"literal control", "content", `{"path":"report.txt","content":"Launch 🚀"}`},
+ {"unpaired control", "content", `{"path":"report.txt","content":"\ud83d x \udc00"}`},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ checkFieldAtEverySplit(t, tt.arguments, tt.field)
+ })
+ }
+}
+
+func TestJSONFieldExtractorFindsKeyAfterSameNamedValue(t *testing.T) {
+ tests := []struct {
+ field string
+ arguments string
+ }{
+ {"content", `{"path":"content","content":"real file contents\n"}`},
+ {"path", `{"content":"path","path":"notes/report.txt"}`},
+ {"content", `{"path":"normal.txt","content":"ordinary text"}`},
+ {"path", `{"path":"normal.txt","content":"ordinary text"}`},
+ }
+ for _, tt := range tests {
+ t.Run(tt.field+"/"+tt.arguments, func(t *testing.T) {
+ checkFieldAtEverySplit(t, tt.arguments, tt.field)
+ })
+ }
+}
+
+func checkFieldAtEverySplit(t *testing.T, arguments, field string) {
+ t.Helper()
+ var expected map[string]json.RawMessage
+ if err := json.Unmarshal([]byte(arguments), &expected); err != nil {
+ t.Fatal(err)
+ }
+ var want string
+ if err := json.Unmarshal(expected[field], &want); err != nil {
+ t.Fatal(err)
+ }
+ for split := 0; split <= len(arguments); split++ {
+ // Outer provider events decode their argument deltas as UTF-8 strings.
+ if !utf8.ValidString(arguments[:split]) || !utf8.ValidString(arguments[split:]) {
+ continue
+ }
+ e := NewJSONFieldExtractor(field)
+ first := e.Feed(arguments[:split])
+ second := e.Feed(arguments[split:])
+ if !utf8.ValidString(first) || !utf8.ValidString(second) {
+ t.Fatalf("split %d emitted invalid UTF-8: %q / %q", split, first, second)
+ }
+ if got := first + second; got != want || !e.IsDone() {
+ t.Fatalf("split %d: got %q, done=%t; JSON decoded %q", split, got, e.IsDone(), want)
+ }
+ }
+}
+
+func TestJSONFieldExtractorHoldsPotentialSurrogatePair(t *testing.T) {
+ e := NewJSONFieldExtractor("content")
+ if got := e.Feed(`{"path":"report.txt","content":"Launch \ud83d`); got != "Launch " {
+ t.Fatalf("high-surrogate prefix = %q, want only complete text", got)
+ }
+ if got := e.Feed(`\ud`); got != "" {
+ t.Fatalf("partial low surrogate = %q", got)
+ }
+ if got := e.Feed(`e80"}`); got != "🚀" || !e.IsDone() {
+ t.Fatalf("completed pair = %q, done=%t", got, e.IsDone())
+ }
+}
+
+func TestSandboxWriteProgressRetainsJSONFieldValues(t *testing.T) {
+ tests := []string{
+ `{"path":"content","content":"Launch \ud83d\ude80\n"}`,
+ `{"content":"path","path":"notes/report.txt"}`,
+ `{"path":"normal.txt","content":"ordinary text\n"}`,
+ }
+ for _, arguments := range tests {
+ t.Run(arguments, func(t *testing.T) {
+ var want struct {
+ Path string `json:"path"`
+ Content string `json:"content"`
+ }
+ if err := json.Unmarshal([]byte(arguments), &want); err != nil {
+ t.Fatal(err)
+ }
+ progress := NewSandboxFileProgress("write_sandbox_file")
+ payload, ok := progress.Feed(arguments)
+ if !ok || payload["path"] != want.Path || payload["bytes"] != len(want.Content) {
+ t.Fatalf("progress = %#v, emitted=%t; want path %q and %d bytes",
+ payload, ok, want.Path, len(want.Content))
+ }
+ })
+ }
+}