Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a long request body is checked for valid JSON once, not by every decodeJSON
yetone938c31424ed9
diff --git a/internal/gateway/fastjson.go b/internal/gateway/fastjson.goindex 9eda5f1f0..f6b18fa3f 100644--- a/internal/gateway/fastjson.go+++ b/internal/gateway/fastjson.go@@ -6,9 +6,11 @@ import ("encoding/binary""encoding/json""errors"+ "hash/maphash""reflect""slices""sync"+ "sync/atomic""unicode""unicode/utf16""unicode/utf8"@@ -562,9 +564,61 @@ func hex4(s []byte) rune {// jsonMaxDepth is encoding/json's maxNestingDepth.const jsonMaxDepth = 10000+// validMemoMin is the size from which jsonValid remembers what it found+// valid. A request body is decoded a dozen times or more on its way+// (modelOf, parse, the prompt, the conversation, the effort, the images…),+// and each decodeJSON checked all of it again: ~2.5ms a time at 2 MB, a+// fifth of the CPU of serving it.+const validMemoMin = 32 << 10++// validMemo is the last bodies found valid, by length and a hash under a+// seed of this process's own.+var validMemo struct {+ sync.Mutex+ sums [8]validSum+ next int+}++type validSum struct {+ n int+ sum uint64+}++var validSeed = maphash.MakeSeed()++// validScanned, when set (tests), is told of each body of validMemoMin or+// more that jsonValid checks byte by byte.+var validScanned atomic.Pointer[func(data []byte)]+// jsonValid is json.Valid, without stepping through each byte by a call:// the grammar, and the depth past which encoding/json gives up, are its.+// A long body found valid is remembered, and not checked again.func jsonValid(data []byte) bool {+ if len(data) < validMemoMin {+ return jsonCheck(data)+ }+ s := validSum{len(data), maphash.Bytes(validSeed, data)}+ validMemo.Lock()+ known := slices.Contains(validMemo.sums[:], s)+ validMemo.Unlock()+ if known {+ return true+ }+ if f := validScanned.Load(); f != nil {+ (*f)(data)+ }+ if !jsonCheck(data) {+ return false+ }+ validMemo.Lock()+ validMemo.sums[validMemo.next] = s+ validMemo.next = (validMemo.next + 1) % len(validMemo.sums)+ validMemo.Unlock()+ return true+}++// jsonCheck is jsonValid's check itself.+func jsonCheck(data []byte) bool {i := skipJSONSpace(data, 0)i, ok := validJSONValue(data, i, 0)return ok && skipJSONSpace(data, i) == len(data)diff --git a/internal/gateway/fastjson_memo_test.go b/internal/gateway/fastjson_memo_test.gonew file mode 100644index 000000000..52e9189b0--- /dev/null+++ b/internal/gateway/fastjson_memo_test.go@@ -0,0 +1,72 @@+package gateway++import (+ "bytes"+ "strings"+ "sync/atomic"+ "testing"+)++// memoBody is a request body past validMemoMin, unique to the test.+func memoBody(tag string) []byte {+ return []byte(`{"model":"` + tag + `","messages":[{"role":"user","content":"` ++ strings.Repeat("a long prompt ", 6000) + `"}]}`)+}++// countScans counts jsonValid's byte-by-byte checks of bodies equal to b.+func countScans(t *testing.T, b []byte) *atomic.Int32 {+ var n atomic.Int32+ f := func(data []byte) {+ if bytes.Equal(data, b) {+ n.Add(1)+ }+ }+ validScanned.Store(&f)+ t.Cleanup(func() { validScanned.Store(nil) })+ return &n+}++// A long body decoded by each reader of a request is checked for valid+// JSON once, not by every decodeJSON: at 2 MB each check cost ~2.5ms, and+// a request's body went through a dozen of them.+func TestDecodeJSONChecksALongBodyOnce(t *testing.T) {+ body := memoBody(t.Name())+ scans := countScans(t, body)+ for range 6 {+ var v struct {+ Model string `json:"model"`+ }+ if err := decodeJSON(body, &v); err != nil || v.Model != t.Name() {+ t.Fatalf("decode: %v %q", err, v.Model)+ }+ }+ if n := scans.Load(); n != 1 {+ t.Fatalf("a %d-byte body was checked %d times over six decodes, want 1", len(body), n)+ }+}++// What is remembered is a valid body: an invalid one, or a valid one+// changed by a byte at the same length, is checked again and refused.+func TestDecodeJSONMemoKeepsRefusing(t *testing.T) {+ body := memoBody(t.Name())+ var v map[string]any+ if err := decodeJSON(body, &v); err != nil {+ t.Fatal(err)+ }+ bad := bytes.Clone(body)+ bad[len(bad)-1] = '!'+ for range 3 {+ var w map[string]any+ if err := decodeJSON(bad, &w); err == nil {+ t.Fatal("a body with a broken end decoded")+ }+ }+ trunc := body[:len(body)-1]+ var w map[string]any+ if err := decodeJSON(trunc, &w); err == nil {+ t.Fatal("a truncated body decoded")+ }+ if !jsonValid(body) || jsonValid(bad) {+ t.Fatal("jsonValid's answer changed")+ }+}