Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(models): preserve string fields in streamed tool arguments (#4253)
Rudy Mizrahi Celeklicdd70b35d522
diff --git a/internal/models/api/json_field_extractor.go b/internal/models/api/json_field_extractor.goindex 08f050794..c9bb47c94 100644--- a/internal/models/api/json_field_extractor.go+++ b/internal/models/api/json_field_extractor.go@@ -2,6 +2,7 @@ package apiimport ("strings"+ "unicode/utf16""unicode/utf8")@@ -79,36 +80,35 @@ func (e *JSONFieldExtractor) IsDone() bool {// findFieldValueStart finds the byte offset where the field's string value content begins// (after the opening quote of the value). Returns -1 if not found.func findFieldValueStart(buf string, fieldName string) int {- // Look for "fieldName" key followed by colon and opening quotekey := `"` + fieldName + `"`- idx := strings.Index(buf, key)- if idx < 0 {- return -1- }-- // Skip past the key- pos := idx + len(key)-- // Skip whitespace and colon- for pos < len(buf) {- ch := buf[pos]- if ch == ':' {+ for offset := 0; offset < len(buf); {+ idx := strings.Index(buf[offset:], key)+ if idx < 0 {+ return -1+ }+ pos := offset + idx + len(key)+ offset = pos+ for pos < len(buf) && strings.ContainsRune(" \t\n\r", rune(buf[pos])) {pos+++ }+ if pos >= len(buf) {+ return -1+ }+ if buf[pos] != ':' {continue}- if ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' {+ pos+++ for pos < len(buf) && strings.ContainsRune(" \t\n\r", rune(buf[pos])) {pos++- continue}- if ch == '"' {- // Found the opening quote of the value+ if pos >= len(buf) {+ return -1+ }+ if buf[pos] == '"' {return pos + 1}- // Unexpected character- return -1}-- return -1 // Haven't seen the opening quote yet+ return -1}// findSafeEnd finds the safe end position for emission within the value content.@@ -130,6 +130,17 @@ func findSafeEnd(value string, from int) (int, bool) {if i+5 >= len(value) {return i, false}+ if isHighSurrogate(unicodeEscape(value[i+2 : i+6])) {+ rest := value[i+6:]+ // A high surrogate may pair with the next streamed escape.+ if rest == "" || rest == `\` || (strings.HasPrefix(rest, `\u`) && len(rest) < 6) {+ return i, false+ }+ if len(rest) >= 6 && strings.HasPrefix(rest, `\u`) && isLowSurrogate(unicodeEscape(rest[2:6])) {+ i += 12+ continue+ }+ }i += 6} else {// Simple escape: \", \\, \n, \t, \r, \/, \b, \f@@ -190,21 +201,16 @@ func unescapeJSONString(s string) string {case 'u':// Unicode escape \uXXXXif i+5 < len(s) {- // Parse hex digits- hexStr := s[i+2 : i+6]- var codepoint int- for _, h := range hexStr {- codepoint <<= 4- switch {- case h >= '0' && h <= '9':- codepoint += int(h - '0')- case h >= 'a' && h <= 'f':- codepoint += int(h-'a') + 10- case h >= 'A' && h <= 'F':- codepoint += int(h-'A') + 10+ codepoint := unicodeEscape(s[i+2 : i+6])+ if isHighSurrogate(codepoint) && i+11 < len(s) && s[i+6:i+8] == `\u` {+ low := unicodeEscape(s[i+8 : i+12])+ if isLowSurrogate(low) {+ b.WriteRune(utf16.DecodeRune(codepoint, low))+ i += 12+ continue}}- b.WriteRune(rune(codepoint))+ b.WriteRune(codepoint)i += 6} else {b.WriteByte(s[i])@@ -222,3 +228,22 @@ func unescapeJSONString(s string) string {return b.String()}++func unicodeEscape(hex string) rune {+ var codepoint rune+ for _, h := range hex {+ codepoint <<= 4+ switch {+ case h >= '0' && h <= '9':+ codepoint += h - '0'+ case h >= 'a' && h <= 'f':+ codepoint += h - 'a' + 10+ case h >= 'A' && h <= 'F':+ codepoint += h - 'A' + 10+ }+ }+ return codepoint+}++func isHighSurrogate(r rune) bool { return r >= 0xd800 && r <= 0xdbff }+func isLowSurrogate(r rune) bool { return r >= 0xdc00 && r <= 0xdfff }diff --git a/internal/models/api/json_field_extractor_semantics_test.go b/internal/models/api/json_field_extractor_semantics_test.gonew file mode 100644index 000000000..ae1f6f40a--- /dev/null+++ b/internal/models/api/json_field_extractor_semantics_test.go@@ -0,0 +1,110 @@+package api++import (+ "encoding/json"+ "testing"+ "unicode/utf8"+)++func TestJSONFieldExtractorSupplementaryUnicodeMatchesJSON(t *testing.T) {+ tests := []struct {+ name string+ field string+ arguments string+ }{+ {"content pair", "content", `{"path":"report.txt","content":"Launch \ud83d\ude80\n"}`},+ {"path pair", "path", `{"path":"\ud83d\udcdd.txt","content":"done"}`},+ {"two pairs", "content", `{"path":"report.txt","content":"\uD83D\uDC69\u200d\uD83D\uDD2C"}`},+ {"thinking pair", "thought", `{"thought":"Delivery \ud83d\ude80","next_thought_needed":false}`},+ {"BMP control", "content", `{"path":"report.txt","content":"\u4e16\u754c"}`},+ {"literal control", "content", `{"path":"report.txt","content":"Launch 🚀"}`},+ {"unpaired control", "content", `{"path":"report.txt","content":"\ud83d x \udc00"}`},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ checkFieldAtEverySplit(t, tt.arguments, tt.field)+ })+ }+}++func TestJSONFieldExtractorFindsKeyAfterSameNamedValue(t *testing.T) {+ tests := []struct {+ field string+ arguments string+ }{+ {"content", `{"path":"content","content":"real file contents\n"}`},+ {"path", `{"content":"path","path":"notes/report.txt"}`},+ {"content", `{"path":"normal.txt","content":"ordinary text"}`},+ {"path", `{"path":"normal.txt","content":"ordinary text"}`},+ }+ for _, tt := range tests {+ t.Run(tt.field+"/"+tt.arguments, func(t *testing.T) {+ checkFieldAtEverySplit(t, tt.arguments, tt.field)+ })+ }+}++func checkFieldAtEverySplit(t *testing.T, arguments, field string) {+ t.Helper()+ var expected map[string]json.RawMessage+ if err := json.Unmarshal([]byte(arguments), &expected); err != nil {+ t.Fatal(err)+ }+ var want string+ if err := json.Unmarshal(expected[field], &want); err != nil {+ t.Fatal(err)+ }+ for split := 0; split <= len(arguments); split++ {+ // Outer provider events decode their argument deltas as UTF-8 strings.+ if !utf8.ValidString(arguments[:split]) || !utf8.ValidString(arguments[split:]) {+ continue+ }+ e := NewJSONFieldExtractor(field)+ first := e.Feed(arguments[:split])+ second := e.Feed(arguments[split:])+ if !utf8.ValidString(first) || !utf8.ValidString(second) {+ t.Fatalf("split %d emitted invalid UTF-8: %q / %q", split, first, second)+ }+ if got := first + second; got != want || !e.IsDone() {+ t.Fatalf("split %d: got %q, done=%t; JSON decoded %q", split, got, e.IsDone(), want)+ }+ }+}++func TestJSONFieldExtractorHoldsPotentialSurrogatePair(t *testing.T) {+ e := NewJSONFieldExtractor("content")+ if got := e.Feed(`{"path":"report.txt","content":"Launch \ud83d`); got != "Launch " {+ t.Fatalf("high-surrogate prefix = %q, want only complete text", got)+ }+ if got := e.Feed(`\ud`); got != "" {+ t.Fatalf("partial low surrogate = %q", got)+ }+ if got := e.Feed(`e80"}`); got != "🚀" || !e.IsDone() {+ t.Fatalf("completed pair = %q, done=%t", got, e.IsDone())+ }+}++func TestSandboxWriteProgressRetainsJSONFieldValues(t *testing.T) {+ tests := []string{+ `{"path":"content","content":"Launch \ud83d\ude80\n"}`,+ `{"content":"path","path":"notes/report.txt"}`,+ `{"path":"normal.txt","content":"ordinary text\n"}`,+ }+ for _, arguments := range tests {+ t.Run(arguments, func(t *testing.T) {+ var want struct {+ Path string `json:"path"`+ Content string `json:"content"`+ }+ if err := json.Unmarshal([]byte(arguments), &want); err != nil {+ t.Fatal(err)+ }+ progress := NewSandboxFileProgress("write_sandbox_file")+ payload, ok := progress.Feed(arguments)+ if !ok || payload["path"] != want.Path || payload["bytes"] != len(want.Content) {+ t.Fatalf("progress = %#v, emitted=%t; want path %q and %d bytes",+ payload, ok, want.Path, len(want.Content))+ }+ })+ }+}