Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a Chat request is decoded for silent assistant turns only when it has one (yg6888 on X)

yetonebdf4d7a9cc88
diff --git a/internal/gateway/chat_empty_assistant_test.go b/internal/gateway/chat_empty_assistant_test.go
index 44f198c5a..6eefc2f69 100644
--- a/internal/gateway/chat_empty_assistant_test.go
+++ b/internal/gateway/chat_empty_assistant_test.go
@@ -2,7 +2,9 @@ package gateway
import (
"bytes"
+ "context"
"encoding/json"
+ "runtime"
"strings"
"testing"
@@ -245,6 +247,54 @@ func TestChatRelaySilentAssistantsOnly(t *testing.T) {
}
}
+// silentAssistantsOut runs on every attempt of a relayed Chat request, and
+// a long conversation with nothing to leave out costs it a look at the
+// assistant turns, not a decode of every message: it decoded all of a
+// 1.5 MB conversation into maps on each attempt, ~4 MB allocated a request
+// (yg6888 on X). A silent turn at the very end, or written with escapes,
+// is still found.
+func TestSilentAssistantsOutDecodesOnlyAssistants(t *testing.T) {
+ var turns []string
+ for i := range 200 {
+ turns = append(turns, `{"role":"user","content":"`+strings.Repeat("read this file ", 300)+`"}`,
+ `{"role":"assistant","content":"ok `+strings.Repeat("x", i)+`"}`)
+ }
+ body := []byte(`{"model":"m","messages":[` + strings.Join(turns, ",") + `]}`)
+ ctx, done := context.WithCancel(context.Background())
+ defer done()
+ holdBody(ctx, body) // as readRequestBody holds a request's
+ if got := silentAssistantsOut(body); !bytes.Equal(got, body) {
+ t.Fatal("a clean body was rewritten")
+ }
+ var before, after runtime.MemStats
+ runtime.ReadMemStats(&before)
+ const n = 5
+ for range n {
+ silentAssistantsOut(body)
+ }
+ runtime.ReadMemStats(&after)
+ if per := (after.TotalAlloc - before.TotalAlloc) / n; per > uint64(len(body))/8 {
+ t.Fatalf("looking at a %d-byte clean body allocated %d bytes", len(body), per)
+ }
+
+ for _, last := range []string{
+ `{"role":"assistant","content":"","reasoning_content":"cut"}`,
+ `{"role":"assistant","content":null}`,
+ `{"role":"%u0061ssistant","content":null}`,
+ `{"role":"assistant","content":"\n \t "}`,
+ `{"role":"assistant","content":"\n%u3000\t "}`,
+ `{"content":[{"type":"text","text":" "}],"role":"assistant"}`,
+ } {
+ b := []byte(`{"model":"m","messages":[` + strings.Join(turns, ",") + `,` + strings.ReplaceAll(last, "%u", "\x5cu") + `,{"role":"user","content":"go"}]}`)
+ var q struct {
+ Messages []json.RawMessage `json:"messages"`
+ }
+ if err := json.Unmarshal(silentAssistantsOut(b), &q); err != nil || len(q.Messages) != len(turns)+1 {
+ t.Fatalf("%s: %d messages sent, want %d (%v)", last, len(q.Messages), len(turns)+1, err)
+ }
+ }
+}
+
// A Resume's prefill, magpie's own ask to go on from a reply cut short,
// stays however little of the reply there was: without it the upstream
// would answer from the start instead of going on.
diff --git a/internal/gateway/chattidy.go b/internal/gateway/chattidy.go
index 9832d2739..c872e7b31 100644
--- a/internal/gateway/chattidy.go
+++ b/internal/gateway/chattidy.go
@@ -434,7 +434,7 @@ func silentAssistantsOut(body []byte) []byte {
return body
}
var q map[string]json.RawMessage
- if decodeJSON(body, &q) != nil {
+ if decodeJSON(body, &q) != nil || !anySilentAssistant(q["messages"]) {
return body
}
dec := json.NewDecoder(bytes.NewReader(q["messages"]))
@@ -455,6 +455,80 @@ func silentAssistantsOut(body []byte) []byte {
return nb
}
+// anySilentAssistant says whether a Chat request's messages hold a turn
+// silentAssistant leaves out, decoding only the assistant turns to see.
+// silentAssistantsOut runs on every attempt of every Chat request relayed
+// as sent, and almost none has such a turn: decoding every message to
+// look made it the gateway's largest allocation after the body itself
+// (~4 MB a request for a 1.5 MB conversation, yg6888 on X).
+func anySilentAssistant(msgs json.RawMessage) bool {
+ if len(msgs) == 0 || msgs[0] != '[' { // valid: decodeJSON checked the body it is in
+ return false
+ }
+ found := false
+ eachElement(msgs, func(el []byte) bool {
+ if el[0] != '{' {
+ return true
+ }
+ assistant, said := false, false
+ eachMember(el, func(key, val []byte) bool {
+ switch {
+ case jsonStringIs(key, "role"):
+ assistant = jsonStringIs(val, "assistant")
+ case jsonStringIs(key, "content"):
+ said = textSaysSomething(val)
+ }
+ return true
+ })
+ if !assistant || said {
+ return true // a turn with text in it is never left out
+ }
+ dec := json.NewDecoder(bytes.NewReader(el))
+ dec.UseNumber()
+ var m map[string]any
+ found = dec.Decode(&m) == nil && silentAssistant(m)
+ return !found
+ })
+ return found
+}
+
+// textSaysSomething says whether q, a valid JSON value, is a string with
+// an ASCII character in it that isn't a space: text strings.TrimSpace
+// leaves. Escapes are passed over, so "\n" and an escaped U+3000 say
+// nothing here, and what it can't tell is false, for silentAssistant to
+// look at.
+func textSaysSomething(q []byte) bool {
+ if len(q) < 2 || q[0] != '"' {
+ return false
+ }
+ for i := 1; i < len(q)-1; i++ {
+ switch c := q[i]; {
+ case c == '\\':
+ if q[i+1] == 'u' {
+ i += 5
+ } else {
+ i++
+ }
+ case c > ' ' && c < 0x7f:
+ return true
+ }
+ }
+ return false
+}
+
+// jsonStringIs says whether q, a JSON value, is the string s, unquoting
+// only a string with an escape in it.
+func jsonStringIs(q []byte, s string) bool {
+ if len(q) == len(s)+2 && q[0] == '"' && string(q[1:len(q)-1]) == s {
+ return true
+ }
+ if len(q) == 0 || q[0] != '"' || bytes.IndexByte(q, '\\') < 0 {
+ return false
+ }
+ u, ok := unquoteJSON(q)
+ return ok && u == s
+}
+
// reasoningDetailsAsContent leaves reasoning_details out of every message
// of a Chat request, its thinking kept in reasoning_content when the
// message has none: Kimi's API turns reasoning_details away in whatever