Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a long request body is checked for valid JSON once, not by every decodeJSON

yetone938c31424ed9
diff --git a/internal/gateway/fastjson.go b/internal/gateway/fastjson.go
index 9eda5f1f0..f6b18fa3f 100644
--- a/internal/gateway/fastjson.go
+++ b/internal/gateway/fastjson.go
@@ -6,9 +6,11 @@ import (
"encoding/binary"
"encoding/json"
"errors"
+ "hash/maphash"
"reflect"
"slices"
"sync"
+ "sync/atomic"
"unicode"
"unicode/utf16"
"unicode/utf8"
@@ -562,9 +564,61 @@ func hex4(s []byte) rune {
// jsonMaxDepth is encoding/json's maxNestingDepth.
const jsonMaxDepth = 10000
+// validMemoMin is the size from which jsonValid remembers what it found
+// valid. A request body is decoded a dozen times or more on its way
+// (modelOf, parse, the prompt, the conversation, the effort, the images…),
+// and each decodeJSON checked all of it again: ~2.5ms a time at 2 MB, a
+// fifth of the CPU of serving it.
+const validMemoMin = 32 << 10
+
+// validMemo is the last bodies found valid, by length and a hash under a
+// seed of this process's own.
+var validMemo struct {
+ sync.Mutex
+ sums [8]validSum
+ next int
+}
+
+type validSum struct {
+ n int
+ sum uint64
+}
+
+var validSeed = maphash.MakeSeed()
+
+// validScanned, when set (tests), is told of each body of validMemoMin or
+// more that jsonValid checks byte by byte.
+var validScanned atomic.Pointer[func(data []byte)]
+
// jsonValid is json.Valid, without stepping through each byte by a call:
// the grammar, and the depth past which encoding/json gives up, are its.
+// A long body found valid is remembered, and not checked again.
func jsonValid(data []byte) bool {
+ if len(data) < validMemoMin {
+ return jsonCheck(data)
+ }
+ s := validSum{len(data), maphash.Bytes(validSeed, data)}
+ validMemo.Lock()
+ known := slices.Contains(validMemo.sums[:], s)
+ validMemo.Unlock()
+ if known {
+ return true
+ }
+ if f := validScanned.Load(); f != nil {
+ (*f)(data)
+ }
+ if !jsonCheck(data) {
+ return false
+ }
+ validMemo.Lock()
+ validMemo.sums[validMemo.next] = s
+ validMemo.next = (validMemo.next + 1) % len(validMemo.sums)
+ validMemo.Unlock()
+ return true
+}
+
+// jsonCheck is jsonValid's check itself.
+func jsonCheck(data []byte) bool {
i := skipJSONSpace(data, 0)
i, ok := validJSONValue(data, i, 0)
return ok && skipJSONSpace(data, i) == len(data)
diff --git a/internal/gateway/fastjson_memo_test.go b/internal/gateway/fastjson_memo_test.go
new file mode 100644
index 000000000..52e9189b0
--- /dev/null
+++ b/internal/gateway/fastjson_memo_test.go
@@ -0,0 +1,72 @@
+package gateway
+
+import (
+ "bytes"
+ "strings"
+ "sync/atomic"
+ "testing"
+)
+
+// memoBody is a request body past validMemoMin, unique to the test.
+func memoBody(tag string) []byte {
+ return []byte(`{"model":"` + tag + `","messages":[{"role":"user","content":"` +
+ strings.Repeat("a long prompt ", 6000) + `"}]}`)
+}
+
+// countScans counts jsonValid's byte-by-byte checks of bodies equal to b.
+func countScans(t *testing.T, b []byte) *atomic.Int32 {
+ var n atomic.Int32
+ f := func(data []byte) {
+ if bytes.Equal(data, b) {
+ n.Add(1)
+ }
+ }
+ validScanned.Store(&f)
+ t.Cleanup(func() { validScanned.Store(nil) })
+ return &n
+}
+
+// A long body decoded by each reader of a request is checked for valid
+// JSON once, not by every decodeJSON: at 2 MB each check cost ~2.5ms, and
+// a request's body went through a dozen of them.
+func TestDecodeJSONChecksALongBodyOnce(t *testing.T) {
+ body := memoBody(t.Name())
+ scans := countScans(t, body)
+ for range 6 {
+ var v struct {
+ Model string `json:"model"`
+ }
+ if err := decodeJSON(body, &v); err != nil || v.Model != t.Name() {
+ t.Fatalf("decode: %v %q", err, v.Model)
+ }
+ }
+ if n := scans.Load(); n != 1 {
+ t.Fatalf("a %d-byte body was checked %d times over six decodes, want 1", len(body), n)
+ }
+}
+
+// What is remembered is a valid body: an invalid one, or a valid one
+// changed by a byte at the same length, is checked again and refused.
+func TestDecodeJSONMemoKeepsRefusing(t *testing.T) {
+ body := memoBody(t.Name())
+ var v map[string]any
+ if err := decodeJSON(body, &v); err != nil {
+ t.Fatal(err)
+ }
+ bad := bytes.Clone(body)
+ bad[len(bad)-1] = '!'
+ for range 3 {
+ var w map[string]any
+ if err := decodeJSON(bad, &w); err == nil {
+ t.Fatal("a body with a broken end decoded")
+ }
+ }
+ trunc := body[:len(body)-1]
+ var w map[string]any
+ if err := decodeJSON(trunc, &w); err == nil {
+ t.Fatal("a truncated body decoded")
+ }
+ if !jsonValid(body) || jsonValid(bad) {
+ t.Fatal("jsonValid's answer changed")
+ }
+}