Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a long body's top level is split at its members once, not by each decode of it

yetoneec6fba472050
diff --git a/internal/gateway/fastjson.go b/internal/gateway/fastjson.go
index f6b18fa3f..b9f4acf0a 100644
--- a/internal/gateway/fastjson.go
+++ b/internal/gateway/fastjson.go
@@ -38,7 +38,16 @@ import (
// of its own (a slice's old elements are decoded over, not replaced).
func decodeJSON(data []byte, dst any) error {
v := reflect.ValueOf(dst)
- if v.Kind() != reflect.Pointer || v.IsNil() || !v.Elem().IsZero() || !jsonValid(data) {
+ if v.Kind() != reflect.Pointer || v.IsNil() || !v.Elem().IsZero() {
+ return json.Unmarshal(data, dst)
+ }
+ var members []memberSpan
+ if len(data) >= validMemoMin {
+ var valid bool
+ if valid, members = longBody(data); !valid {
+ return json.Unmarshal(data, dst)
+ }
+ } else if !jsonCheck(data) {
return json.Unmarshal(data, dst)
}
start := skipJSONSpace(data, 0)
@@ -49,7 +58,7 @@ func decodeJSON(data []byte, dst any) error {
// of the value's opening and its close.
return json.Unmarshal(append(data[:start+1:start+1], close), dst)
}
- if err := fillJSON(trimJSON(data), v.Elem()); err != nil {
+ if err := fillJSONIn(trimJSON(data), v.Elem(), members); err != nil {
v.Elem().SetZero()
return json.Unmarshal(data, dst)
}
@@ -170,7 +179,11 @@ func (f *fastFields) field(key []byte) int {
// fillJSON decodes raw, a whole valid JSON value, into v; errSlowJSON or
// another error leaves it to encoding/json.
-func fillJSON(raw []byte, v reflect.Value) error {
+func fillJSON(raw []byte, v reflect.Value) error { return fillJSONIn(raw, v, nil) }
+
+// fillJSONIn is fillJSON with where raw's members are, when known
+// (longBody): a struct or map at raw's top is filled from them.
+func fillJSONIn(raw []byte, v reflect.Value, members []memberSpan) error {
t := v.Type()
if t == rawMessageType {
v.SetBytes(bytes.Clone(raw)) // a copy, as encoding/json makes
@@ -193,7 +206,7 @@ func fillJSON(raw []byte, v reflect.Value) error {
if v.IsNil() {
v.Set(reflect.New(t.Elem()))
}
- return fillJSON(raw, v.Elem())
+ return fillJSONIn(raw, v.Elem(), members)
case reflect.String:
if null {
return nil
@@ -231,7 +244,7 @@ func fillJSON(raw []byte, v reflect.Value) error {
}
var seen []bool
var err error
- eachMember(raw, func(key, val []byte) bool {
+ eachMemberIn(raw, members, func(key, val []byte) bool {
name, ok := unquoteJSON(key)
if !ok {
err = errSlowJSON
@@ -292,7 +305,7 @@ func fillJSON(raw []byte, v reflect.Value) error {
v.Set(reflect.MakeMap(t))
}
var err error
- eachMember(raw, func(key, val []byte) bool {
+ eachMemberIn(raw, members, func(key, val []byte) bool {
name, ok := unquoteJSON(key)
if !ok {
err = errSlowJSON
@@ -353,6 +366,29 @@ func trimJSON(data []byte) []byte {
// eachMember calls f with each member of obj, a valid JSON object, its key
// still quoted, until f returns false.
func eachMember(obj []byte, f func(key, val []byte) bool) {
+ eachMemberAt(obj, func(k0, k1, v0, v1 int) bool { return f(obj[k0:k1], obj[v0:v1:v1]) })
+}
+
+// eachMemberIn is eachMember, from where its members are when known
+// (longBody), else by splitting obj.
+func eachMemberIn(obj []byte, members []memberSpan, f func(key, val []byte) bool) {
+ if members == nil {
+ if h := topSplit.Load(); h != nil && len(obj) >= validMemoMin {
+ (*h)(obj)
+ }
+ eachMember(obj, f)
+ return
+ }
+ for _, m := range members {
+ if !f(obj[m.k0:m.k1], obj[m.v0:m.v1:m.v1]) {
+ return
+ }
+ }
+}
+
+// eachMemberAt is eachMember by offsets: the key at [k0, k1), the value
+// at [v0, v1).
+func eachMemberAt(obj []byte, f func(k0, k1, v0, v1 int) bool) {
i := skipJSONSpace(obj, 1)
if obj[i] == '}' {
return
@@ -360,12 +396,12 @@ func eachMember(obj []byte, f func(key, val []byte) bool) {
for {
k := i
i = skipJSONString(obj, i)
- key := obj[k:i]
+ k1 := i
i = skipJSONSpace(obj, i)
i = skipJSONSpace(obj, i+1) // ':'
s := i
i = skipJSONValue(obj, i)
- if !f(key, obj[s:i:i]) {
+ if !f(k, k1, s, i) {
return
}
i = skipJSONSpace(obj, i)
@@ -575,7 +611,7 @@ const validMemoMin = 32 << 10
// seed of this process's own.
var validMemo struct {
sync.Mutex
- sums [8]validSum
+ sums [8]validEntry
next int
}
@@ -584,6 +620,67 @@ type validSum struct {
sum uint64
}
+// validEntry is a body remembered valid, and where its top level's
+// members are, once a decode has split it (nil before, and for a body not
+// an object).
+type validEntry struct {
+ validSum
+ members []memberSpan
+}
+
+// memberSpan is a member of a body's top-level object: its key, quoted,
+// at [k0, k1) and its value at [v0, v1), of the body trimmed (trimJSON).
+type memberSpan struct{ k0, k1, v0, v1 int }
+
+// topSplit, when set (tests), is told of each object of validMemoMin or
+// more that decodeJSON splits at its members.
+var topSplit atomic.Pointer[func(data []byte)]
+
+// longBody is what decodeJSON knows of a body of validMemoMin or more:
+// valid or not, and where its top level's members are. The members are
+// remembered with the body, as its validity is: a request's body was split
+// at its top level again by each of the dozen decodes it went through,
+// each a walk over every byte of the conversation.
+func longBody(data []byte) (valid bool, members []memberSpan) {
+ s := validSum{len(data), maphash.Bytes(validSeed, data)}
+ validMemo.Lock()
+ i := slices.IndexFunc(validMemo.sums[:], func(e validEntry) bool { return e.validSum == s })
+ if i >= 0 && validMemo.sums[i].members != nil {
+ members = validMemo.sums[i].members
+ validMemo.Unlock()
+ return true, members
+ }
+ validMemo.Unlock()
+ if i < 0 {
+ if f := validScanned.Load(); f != nil {
+ (*f)(data)
+ }
+ if !jsonCheck(data) {
+ return false, nil
+ }
+ }
+ raw := trimJSON(data)
+ if raw[0] == '{' {
+ if f := topSplit.Load(); f != nil {
+ (*f)(data)
+ }
+ members = []memberSpan{}
+ eachMemberAt(raw, func(k0, k1, v0, v1 int) bool {
+ members = append(members, memberSpan{k0, k1, v0, v1})
+ return true
+ })
+ }
+ validMemo.Lock()
+ if i = slices.IndexFunc(validMemo.sums[:], func(e validEntry) bool { return e.validSum == s }); i >= 0 {
+ validMemo.sums[i].members = members
+ } else {
+ validMemo.sums[validMemo.next] = validEntry{s, members}
+ validMemo.next = (validMemo.next + 1) % len(validMemo.sums)
+ }
+ validMemo.Unlock()
+ return true, members
+}
+
var validSeed = maphash.MakeSeed()
// validScanned, when set (tests), is told of each body of validMemoMin or
@@ -599,7 +696,7 @@ func jsonValid(data []byte) bool {
}
s := validSum{len(data), maphash.Bytes(validSeed, data)}
validMemo.Lock()
- known := slices.Contains(validMemo.sums[:], s)
+ known := slices.ContainsFunc(validMemo.sums[:], func(e validEntry) bool { return e.validSum == s })
validMemo.Unlock()
if known {
return true
@@ -611,7 +708,7 @@ func jsonValid(data []byte) bool {
return false
}
validMemo.Lock()
- validMemo.sums[validMemo.next] = s
+ validMemo.sums[validMemo.next] = validEntry{validSum: s}
validMemo.next = (validMemo.next + 1) % len(validMemo.sums)
validMemo.Unlock()
return true
diff --git a/internal/gateway/fastjson_memo_test.go b/internal/gateway/fastjson_memo_test.go
index 52e9189b0..4e1f8d70d 100644
--- a/internal/gateway/fastjson_memo_test.go
+++ b/internal/gateway/fastjson_memo_test.go
@@ -2,6 +2,7 @@ package gateway
import (
"bytes"
+ "encoding/json"
"strings"
"sync/atomic"
"testing"
@@ -45,6 +46,63 @@ func TestDecodeJSONChecksALongBodyOnce(t *testing.T) {
}
}
+// A long body's top level is split at its members once, not by each decode
+// of it: each reader of a request (the model, the stream flag, the
+// messages, the tools…) walked every byte of the conversation again to
+// find its member. What each is given is what encoding/json gives it.
+func TestDecodeJSONSplitsALongBodyOnce(t *testing.T) {
+ body := []byte(`{"model":"` + t.Name() + `","stream":true,"max_tokens":9,"messages":[{"role":"user","content":"` +
+ strings.Repeat("a long prompt ", 6000) + `"}],"metadata":{"user_id":"u"}}`)
+ var n atomic.Int32
+ f := func(data []byte) {
+ if bytes.Equal(data, body) {
+ n.Add(1)
+ }
+ }
+ topSplit.Store(&f)
+ t.Cleanup(func() { topSplit.Store(nil) })
+ type msgs struct {
+ Messages []struct {
+ Role string `json:"role"`
+ Content string `json:"content"`
+ } `json:"messages"`
+ }
+ for range 4 {
+ var a struct {
+ Model string `json:"model"`
+ Stream bool `json:"stream"`
+ }
+ var b msgs
+ var c *msgs
+ var d map[string]json.RawMessage
+ var e struct {
+ Metadata struct {
+ UserID string `json:"user_id"`
+ } `json:"metadata"`
+ Max int `json:"max_tokens"`
+ }
+ for _, dst := range []any{&a, &b, &c, &d, &e} {
+ if err := decodeJSON(body, dst); err != nil {
+ t.Fatal(err)
+ }
+ }
+ wa, wb, we := a, msgs{}, e
+ wa.Model, wa.Stream, we.Metadata.UserID, we.Max = "", false, "", 0
+ var wd map[string]json.RawMessage
+ json.Unmarshal(body, &wa)
+ json.Unmarshal(body, &wb)
+ json.Unmarshal(body, &wd)
+ json.Unmarshal(body, &we)
+ if a != wa || len(b.Messages) != 1 || b.Messages[0] != wb.Messages[0] || c == nil || len(c.Messages) != 1 ||
+ len(d) != len(wd) || string(d["messages"]) != string(wd["messages"]) || e != we {
+ t.Fatalf("decoded otherwise than encoding/json: %+v %+v %d %v", a, e, len(d), c == nil)
+ }
+ }
+ if got := n.Load(); got != 1 {
+ t.Fatalf("a %d-byte body's top level was split %d times over 20 decodes, want 1", len(body), got)
+ }
+}
+
// What is remembered is a valid body: an invalid one, or a valid one
// changed by a byte at the same length, is checked again and refused.
func TestDecodeJSONMemoKeepsRefusing(t *testing.T) {
diff --git a/internal/gateway/fastjson_split_bench_test.go b/internal/gateway/fastjson_split_bench_test.go
new file mode 100644
index 000000000..9f3a5ed99
--- /dev/null
+++ b/internal/gateway/fastjson_split_bench_test.go
@@ -0,0 +1,35 @@
+package gateway
+
+import (
+ "strings"
+ "testing"
+)
+
+// BenchmarkDecodeReaders is a request's readers decoding one 200KB body,
+// each into the few members it wants: the model and stream flag, the
+// metadata, the max tokens.
+func BenchmarkDecodeReaders(b *testing.B) {
+ body := []byte(`{"model":"claude-x","stream":true,"max_tokens":9,"messages":[{"role":"user","content":"` +
+ strings.Repeat("a long prompt with \\\"quotes\\\" and words ", 5000) + `"}],"metadata":{"user_id":"u"}}`)
+ b.SetBytes(int64(len(body)))
+ b.ReportAllocs()
+ for b.Loop() {
+ var a struct {
+ Model string `json:"model"`
+ Stream bool `json:"stream"`
+ }
+ var e struct {
+ Metadata struct {
+ UserID string `json:"user_id"`
+ } `json:"metadata"`
+ }
+ var m struct {
+ Max int `json:"max_tokens"`
+ }
+ for _, dst := range []any{&a, &e, &m} {
+ if err := decodeJSON(body, dst); err != nil {
+ b.Fatal(err)
+ }
+ }
+ }
+}