Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
package gateway
import (
"strings"
"testing"
"github.com/yetone/magpie/internal/provider"
)
// The error object ASD-max's Codex got (#1944), as the ChatGPT backend
// puts it in a response.failed: a conversation past the model's window.
const overflowError = `{"message":"Your input exceeds the context window of this model. Please adjust your input and try again.","type":"invalid_request_error","param":"input","code":"context_length_exceeded"}`
const overflowFailed = `event: response.failed` + "\n" + `data: {"type":"response.failed","response":{"id":"r0","status":"failed","error":` + overflowError + `}}`
// overflowStream is what the ChatGPT backend answered gpt-5.5 asked with
// about 2 MB of input on 2026-10-13, as it came (its safety_identifier
// left out): a 200, its frames, an error event and response.failed, each
// saying context_length_exceeded.
var overflowStream = []string{
`event: response.created` + "\n" + `data: {"type":"response.created","response":{"id":"resp_0aef60959b6fc063016acdd9f0ac6c87d09ebb9c76c496f0c9","object":"response","created_at":1791875568,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":"ffbebcaa-3636-42d4-a4ae-ca41c6dcec8e","prompt_cache_retention":"24h","reasoning":{"context":"current_turn","effort":"medium","mode":"standard","summary":null},"safety_identifier":"user-redacted","service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0}`,
`event: response.in_progress` + "\n" + `data: {"type":"response.in_progress","response":{"id":"resp_0aef60959b6fc063016acdd9f0ac6c87d09ebb9c76c496f0c9","object":"response","created_at":1791875568,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":"ffbebcaa-3636-42d4-a4ae-ca41c6dcec8e","prompt_cache_retention":"24h","reasoning":{"context":"current_turn","effort":"medium","mode":"standard","summary":null},"safety_identifier":"user-redacted","service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1}`,
`event: error` + "\n" + `data: {"type":"error","error":{"type":"invalid_request_error","code":"context_length_exceeded","message":"Your input exceeds the context window of this model. Please adjust your input and try again.","param":"input"},"sequence_number":2}`,
`event: response.failed` + "\n" + `data: {"type":"response.failed","response":{"id":"resp_0aef60959b6fc063016acdd9f0ac6c87d09ebb9c76c496f0c9","object":"response","created_at":1791875568,"status":"failed","access_programs":null,"background":false,"completed_at":null,"error":{"code":"context_length_exceeded","message":"Your input exceeds the context window of this model. Please adjust your input and try again."},"frequency_penalty":0.0,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.5","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":"ffbebcaa-3636-42d4-a4ae-ca41c6dcec8e","prompt_cache_retention":"24h","reasoning":{"context":"current_turn","effort":"medium","mode":"standard","summary":null},"safety_identifier":"user-redacted","service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":3}`,
}
// An error in a stream that says the conversation is too long for the
// model stands for a 400, not a rate limit: its "context_length_exceeded"
// held quotaWords' "exceeded", and the same request was asked again 7 or 8
// times (#1944). A real rate limit, an overload and a spent quota read as
// before.
func TestStreamOverflowNotRateLimit(t *testing.T) {
for _, c := range []struct {
ev string
want int
}{
{overflowFailed, 400},
{`event: error` + "\n" + `data: {"type":"error","error":` + overflowError + `}`, 400},
{`event: response.failed` + "\n" + `data: {"type":"response.failed","response":{"status":"failed","error":{"message":"Rate limit reached for gpt-5.5 in organization org-x on tokens per min (TPM): Limit 30000, Used 25000, Requested 9000. Please try again in 8s.","type":"tokens","param":null,"code":"rate_limit_exceeded"}}}`, 429},
{overloadedFailed, 529},
{overflowStream[2], 400},
{overflowStream[3], 400},
{`event: error` + "\n" + `data: {"type":"error","error":{"message":"The usage limit has been reached","type":"usage_limit_reached"}}`, 429},
} {
kind, status, msg := streamEvent([]byte(c.ev + "\n\n"))
if kind != eventError || status != c.want {
t.Errorf("%s: kind %d status %d (want %d) %q", c.ev, kind, status, c.want, msg)
}
}
if quotaSaid(overflowError) || !quotaSaid("You exceeded your current quota, please check your plan and billing details.") {
t.Error("quotaSaid")
}
}
// What #1944 saw, end to end: Codex through a group of one member whose
// stream, the backend's own, fails as too long. The agent is told
// context_length_exceeded at once, to compact, and the member isn't asked
// again; before, the failure was a rate limit, the group's patience asked
// the same member again, and the second reply here, not the agent's
// error, was what it got.
func TestCodexOverflowInGroupNotAskedAgain(t *testing.T) {
fresh(t)
failed := reply{200, "text/event-stream", sse(overflowStream...)}
done := reply{200, "text/event-stream", sse(
`event: response.output_text.delta`+"\n"+`data: {"type":"response.output_text.delta","delta":"hello"}`,
`event: response.completed`+"\n"+`data: {"type":"response.completed","response":{"id":"r1","status":"completed","output":[]}}`)}
a := &scripted{replies: []reply{failed, done}}
responsesOn(t, "a", a)
if err := provider.SaveGroup(provider.Group{Name: "G", Members: []string{"a/m"}, Routing: provider.Ordered}); err != nil {
t.Fatal(err)
}
s := New()
code, body := sendTo(s, "/v1/responses", `{"model":"group/g","stream":true,"input":"hi"}`)
if !strings.Contains(body, "context_length_exceeded") || strings.Contains(body, "hello") || a.n != 1 {
t.Fatalf("%d %q (asked %d times)", code, body, a.n)
}
r := s.trace.routes[len(s.trace.routes)-1]
for _, try := range r.Tries {
if try.Fail == failRate || try.Rest != nil {
t.Errorf("an overflow read as a rate limit: %+v", try)
}
}
}