Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a long body's top level is split at its members once, not by each decode of it
yetoneec6fba472050
diff --git a/internal/gateway/fastjson.go b/internal/gateway/fastjson.goindex f6b18fa3f..b9f4acf0a 100644--- a/internal/gateway/fastjson.go+++ b/internal/gateway/fastjson.go@@ -38,7 +38,16 @@ import (// of its own (a slice's old elements are decoded over, not replaced).func decodeJSON(data []byte, dst any) error {v := reflect.ValueOf(dst)- if v.Kind() != reflect.Pointer || v.IsNil() || !v.Elem().IsZero() || !jsonValid(data) {+ if v.Kind() != reflect.Pointer || v.IsNil() || !v.Elem().IsZero() {+ return json.Unmarshal(data, dst)+ }+ var members []memberSpan+ if len(data) >= validMemoMin {+ var valid bool+ if valid, members = longBody(data); !valid {+ return json.Unmarshal(data, dst)+ }+ } else if !jsonCheck(data) {return json.Unmarshal(data, dst)}start := skipJSONSpace(data, 0)@@ -49,7 +58,7 @@ func decodeJSON(data []byte, dst any) error {// of the value's opening and its close.return json.Unmarshal(append(data[:start+1:start+1], close), dst)}- if err := fillJSON(trimJSON(data), v.Elem()); err != nil {+ if err := fillJSONIn(trimJSON(data), v.Elem(), members); err != nil {v.Elem().SetZero()return json.Unmarshal(data, dst)}@@ -170,7 +179,11 @@ func (f *fastFields) field(key []byte) int {// fillJSON decodes raw, a whole valid JSON value, into v; errSlowJSON or// another error leaves it to encoding/json.-func fillJSON(raw []byte, v reflect.Value) error {+func fillJSON(raw []byte, v reflect.Value) error { return fillJSONIn(raw, v, nil) }++// fillJSONIn is fillJSON with where raw's members are, when known+// (longBody): a struct or map at raw's top is filled from them.+func fillJSONIn(raw []byte, v reflect.Value, members []memberSpan) error {t := v.Type()if t == rawMessageType {v.SetBytes(bytes.Clone(raw)) // a copy, as encoding/json makes@@ -193,7 +206,7 @@ func fillJSON(raw []byte, v reflect.Value) error {if v.IsNil() {v.Set(reflect.New(t.Elem()))}- return fillJSON(raw, v.Elem())+ return fillJSONIn(raw, v.Elem(), members)case reflect.String:if null {return nil@@ -231,7 +244,7 @@ func fillJSON(raw []byte, v reflect.Value) error {}var seen []boolvar err error- eachMember(raw, func(key, val []byte) bool {+ eachMemberIn(raw, members, func(key, val []byte) bool {name, ok := unquoteJSON(key)if !ok {err = errSlowJSON@@ -292,7 +305,7 @@ func fillJSON(raw []byte, v reflect.Value) error {v.Set(reflect.MakeMap(t))}var err error- eachMember(raw, func(key, val []byte) bool {+ eachMemberIn(raw, members, func(key, val []byte) bool {name, ok := unquoteJSON(key)if !ok {err = errSlowJSON@@ -353,6 +366,29 @@ func trimJSON(data []byte) []byte {// eachMember calls f with each member of obj, a valid JSON object, its key// still quoted, until f returns false.func eachMember(obj []byte, f func(key, val []byte) bool) {+ eachMemberAt(obj, func(k0, k1, v0, v1 int) bool { return f(obj[k0:k1], obj[v0:v1:v1]) })+}++// eachMemberIn is eachMember, from where its members are when known+// (longBody), else by splitting obj.+func eachMemberIn(obj []byte, members []memberSpan, f func(key, val []byte) bool) {+ if members == nil {+ if h := topSplit.Load(); h != nil && len(obj) >= validMemoMin {+ (*h)(obj)+ }+ eachMember(obj, f)+ return+ }+ for _, m := range members {+ if !f(obj[m.k0:m.k1], obj[m.v0:m.v1:m.v1]) {+ return+ }+ }+}++// eachMemberAt is eachMember by offsets: the key at [k0, k1), the value+// at [v0, v1).+func eachMemberAt(obj []byte, f func(k0, k1, v0, v1 int) bool) {i := skipJSONSpace(obj, 1)if obj[i] == '}' {return@@ -360,12 +396,12 @@ func eachMember(obj []byte, f func(key, val []byte) bool) {for {k := ii = skipJSONString(obj, i)- key := obj[k:i]+ k1 := ii = skipJSONSpace(obj, i)i = skipJSONSpace(obj, i+1) // ':'s := ii = skipJSONValue(obj, i)- if !f(key, obj[s:i:i]) {+ if !f(k, k1, s, i) {return}i = skipJSONSpace(obj, i)@@ -575,7 +611,7 @@ const validMemoMin = 32 << 10// seed of this process's own.var validMemo struct {sync.Mutex- sums [8]validSum+ sums [8]validEntrynext int}@@ -584,6 +620,67 @@ type validSum struct {sum uint64}+// validEntry is a body remembered valid, and where its top level's+// members are, once a decode has split it (nil before, and for a body not+// an object).+type validEntry struct {+ validSum+ members []memberSpan+}++// memberSpan is a member of a body's top-level object: its key, quoted,+// at [k0, k1) and its value at [v0, v1), of the body trimmed (trimJSON).+type memberSpan struct{ k0, k1, v0, v1 int }++// topSplit, when set (tests), is told of each object of validMemoMin or+// more that decodeJSON splits at its members.+var topSplit atomic.Pointer[func(data []byte)]++// longBody is what decodeJSON knows of a body of validMemoMin or more:+// valid or not, and where its top level's members are. The members are+// remembered with the body, as its validity is: a request's body was split+// at its top level again by each of the dozen decodes it went through,+// each a walk over every byte of the conversation.+func longBody(data []byte) (valid bool, members []memberSpan) {+ s := validSum{len(data), maphash.Bytes(validSeed, data)}+ validMemo.Lock()+ i := slices.IndexFunc(validMemo.sums[:], func(e validEntry) bool { return e.validSum == s })+ if i >= 0 && validMemo.sums[i].members != nil {+ members = validMemo.sums[i].members+ validMemo.Unlock()+ return true, members+ }+ validMemo.Unlock()+ if i < 0 {+ if f := validScanned.Load(); f != nil {+ (*f)(data)+ }+ if !jsonCheck(data) {+ return false, nil+ }+ }+ raw := trimJSON(data)+ if raw[0] == '{' {+ if f := topSplit.Load(); f != nil {+ (*f)(data)+ }+ members = []memberSpan{}+ eachMemberAt(raw, func(k0, k1, v0, v1 int) bool {+ members = append(members, memberSpan{k0, k1, v0, v1})+ return true+ })+ }+ validMemo.Lock()+ if i = slices.IndexFunc(validMemo.sums[:], func(e validEntry) bool { return e.validSum == s }); i >= 0 {+ validMemo.sums[i].members = members+ } else {+ validMemo.sums[validMemo.next] = validEntry{s, members}+ validMemo.next = (validMemo.next + 1) % len(validMemo.sums)+ }+ validMemo.Unlock()+ return true, members+}+var validSeed = maphash.MakeSeed()// validScanned, when set (tests), is told of each body of validMemoMin or@@ -599,7 +696,7 @@ func jsonValid(data []byte) bool {}s := validSum{len(data), maphash.Bytes(validSeed, data)}validMemo.Lock()- known := slices.Contains(validMemo.sums[:], s)+ known := slices.ContainsFunc(validMemo.sums[:], func(e validEntry) bool { return e.validSum == s })validMemo.Unlock()if known {return true@@ -611,7 +708,7 @@ func jsonValid(data []byte) bool {return false}validMemo.Lock()- validMemo.sums[validMemo.next] = s+ validMemo.sums[validMemo.next] = validEntry{validSum: s}validMemo.next = (validMemo.next + 1) % len(validMemo.sums)validMemo.Unlock()return truediff --git a/internal/gateway/fastjson_memo_test.go b/internal/gateway/fastjson_memo_test.goindex 52e9189b0..4e1f8d70d 100644--- a/internal/gateway/fastjson_memo_test.go+++ b/internal/gateway/fastjson_memo_test.go@@ -2,6 +2,7 @@ package gatewayimport ("bytes"+ "encoding/json""strings""sync/atomic""testing"@@ -45,6 +46,63 @@ func TestDecodeJSONChecksALongBodyOnce(t *testing.T) {}}+// A long body's top level is split at its members once, not by each decode+// of it: each reader of a request (the model, the stream flag, the+// messages, the tools…) walked every byte of the conversation again to+// find its member. What each is given is what encoding/json gives it.+func TestDecodeJSONSplitsALongBodyOnce(t *testing.T) {+ body := []byte(`{"model":"` + t.Name() + `","stream":true,"max_tokens":9,"messages":[{"role":"user","content":"` ++ strings.Repeat("a long prompt ", 6000) + `"}],"metadata":{"user_id":"u"}}`)+ var n atomic.Int32+ f := func(data []byte) {+ if bytes.Equal(data, body) {+ n.Add(1)+ }+ }+ topSplit.Store(&f)+ t.Cleanup(func() { topSplit.Store(nil) })+ type msgs struct {+ Messages []struct {+ Role string `json:"role"`+ Content string `json:"content"`+ } `json:"messages"`+ }+ for range 4 {+ var a struct {+ Model string `json:"model"`+ Stream bool `json:"stream"`+ }+ var b msgs+ var c *msgs+ var d map[string]json.RawMessage+ var e struct {+ Metadata struct {+ UserID string `json:"user_id"`+ } `json:"metadata"`+ Max int `json:"max_tokens"`+ }+ for _, dst := range []any{&a, &b, &c, &d, &e} {+ if err := decodeJSON(body, dst); err != nil {+ t.Fatal(err)+ }+ }+ wa, wb, we := a, msgs{}, e+ wa.Model, wa.Stream, we.Metadata.UserID, we.Max = "", false, "", 0+ var wd map[string]json.RawMessage+ json.Unmarshal(body, &wa)+ json.Unmarshal(body, &wb)+ json.Unmarshal(body, &wd)+ json.Unmarshal(body, &we)+ if a != wa || len(b.Messages) != 1 || b.Messages[0] != wb.Messages[0] || c == nil || len(c.Messages) != 1 ||+ len(d) != len(wd) || string(d["messages"]) != string(wd["messages"]) || e != we {+ t.Fatalf("decoded otherwise than encoding/json: %+v %+v %d %v", a, e, len(d), c == nil)+ }+ }+ if got := n.Load(); got != 1 {+ t.Fatalf("a %d-byte body's top level was split %d times over 20 decodes, want 1", len(body), got)+ }+}+// What is remembered is a valid body: an invalid one, or a valid one// changed by a byte at the same length, is checked again and refused.func TestDecodeJSONMemoKeepsRefusing(t *testing.T) {diff --git a/internal/gateway/fastjson_split_bench_test.go b/internal/gateway/fastjson_split_bench_test.gonew file mode 100644index 000000000..9f3a5ed99--- /dev/null+++ b/internal/gateway/fastjson_split_bench_test.go@@ -0,0 +1,35 @@+package gateway++import (+ "strings"+ "testing"+)++// BenchmarkDecodeReaders is a request's readers decoding one 200KB body,+// each into the few members it wants: the model and stream flag, the+// metadata, the max tokens.+func BenchmarkDecodeReaders(b *testing.B) {+ body := []byte(`{"model":"claude-x","stream":true,"max_tokens":9,"messages":[{"role":"user","content":"` ++ strings.Repeat("a long prompt with \\\"quotes\\\" and words ", 5000) + `"}],"metadata":{"user_id":"u"}}`)+ b.SetBytes(int64(len(body)))+ b.ReportAllocs()+ for b.Loop() {+ var a struct {+ Model string `json:"model"`+ Stream bool `json:"stream"`+ }+ var e struct {+ Metadata struct {+ UserID string `json:"user_id"`+ } `json:"metadata"`+ }+ var m struct {+ Max int `json:"max_tokens"`+ }+ for _, dst := range []any{&a, &e, &m} {+ if err := decodeJSON(body, dst); err != nil {+ b.Fatal(err)+ }+ }+ }+}