Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
package gateway
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/yetone/magpie/internal/provider"
)
func init() { retryPause = time.Millisecond }
// scripted answers each request with the next of its replies, the last
// one over and over.
type scripted struct {
replies []reply
n int
hang chan struct{} // when set, the first request waits on it
hanging chan struct{} // when set, closed as the first request begins to wait on hang
limited http.Header // headers each 429 goes with
}
type reply struct {
code int
ctype string
body string
}
func (s *scripted) ServeHTTP(w http.ResponseWriter, r *http.Request) {
io.ReadAll(r.Body)
i := min(s.n, len(s.replies)-1)
s.n++
if s.hang != nil && i == 0 {
if s.hanging != nil {
close(s.hanging)
}
<-s.hang
}
x := s.replies[i]
if x.ctype == "" {
x.ctype = "application/json"
}
w.Header().Set("Content-Type", x.ctype)
if x.code == http.StatusTooManyRequests {
for k, vs := range s.limited {
w.Header()[k] = vs
}
}
if x.code != 0 {
w.WriteHeader(x.code)
}
io.WriteString(w, x.body)
}
const chatOK = `{"id":"ok","choices":[{"index":0,"message":{"role":"assistant","content":"hello"},"finish_reason":"stop"}]}`
func scriptedOn(t *testing.T, id string, proto provider.Protocol, s *scripted) {
t.Helper()
up := httptest.NewServer(s)
t.Cleanup(up.Close)
p := provider.Provider{ID: id, Name: strings.ToUpper(id), Key: "k", Models: []string{"m"}}
switch proto {
case provider.Anthropic:
p.Anthropic = up.URL
default:
p.Chat = up.URL + "/v1"
}
if err := provider.Save(p); err != nil {
t.Fatal(err)
}
}
// A provider that can't serve the model — turned away, not the request at
// fault — hands the request to the next member rather than to the agent.
func TestGroupFailsOverAnUnservedModel(t *testing.T) {
fresh(t)
a := &scripted{replies: []reply{{400, "", `{"error":{"message":"model \"gpt-6-sol\" is not accessible via the /chat/completions endpoint","code":"unsupported_api_for_model"}}`}}}
b := &scripted{replies: []reply{{200, "", chatOK}}}
scriptedOn(t, "a", provider.Chat, a)
scriptedOn(t, "b", provider.Chat, b)
if err := provider.SaveGroup(provider.Group{Name: "G", Members: []string{"a/m", "b/m"}, Routing: provider.Ordered}); err != nil {
t.Fatal(err)
}
s := New()
code, body := postAs(t, s, "", `{"model":"group/g","messages":[{"role":"user","content":"hi"}]}`)
if code != 200 || !strings.Contains(body, "hello") || a.n != 1 || b.n != 1 {
t.Fatalf("%d %s (a %d, b %d)", code, body, a.n, b.n)
}
for _, x := range []struct {
status int
body string
want bool
}{
{401, `{"error":{"message":"invalid api key"}}`, true},
{404, `{"error":{"message":"not found"}}`, true},
{400, `{"error":{"message":"The model gpt-x does not exist"}}`, true},
{400, `{"error":{"message":"messages: field required"}}`, false},
{400, `{"error":{"message":"This model's maximum context length is 128000 tokens"}}`, false},
{400, `{"error":{"message":"Illegal API invocation from an unapproved channel"}}`, true},
} {
if got := retryable(x.status, []byte(x.body)); got != x.want {
t.Errorf("retryable(%d, %s) = %v", x.status, x.body, got)
}
}
}
// An error a stream begins with, before any of its content, is a failure
// the next one answers, as an error status is; one that comes after
// content has begun reaches the agent as it came.
func TestStreamErrorBeforeContentFailsOver(t *testing.T) {
fresh(t)
overloaded := sse(`event: message_start`+"\n"+`data: {"type":"message_start","message":{"id":"m1","role":"assistant","content":[],"usage":{"input_tokens":3}}}`,
`event: ping`+"\n"+`data: {"type":"ping"}`,
`event: error`+"\n"+`data: {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}`)
good := sse(`event: message_start`+"\n"+`data: {"type":"message_start","message":{"id":"m2","role":"assistant","content":[],"usage":{"input_tokens":3}}}`,
`event: content_block_start`+"\n"+`data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`,
`event: content_block_delta`+"\n"+`data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"from b"}}`,
`event: message_stop`+"\n"+`data: {"type":"message_stop"}`)
a := &scripted{replies: []reply{{200, "text/event-stream", overloaded}}}
b := &scripted{replies: []reply{{200, "text/event-stream", good}}}
scriptedOn(t, "a", provider.Anthropic, a)
scriptedOn(t, "b", provider.Anthropic, b)
if err := provider.SaveGroup(provider.Group{Name: "G", Members: []string{"a/m", "b/m"}, Routing: provider.Ordered}); err != nil {
t.Fatal(err)
}
s := New()
send := func() (int, string) {
rec := httptest.NewRecorder()
s.Handler().ServeHTTP(rec, httptest.NewRequest("POST", "/v1/messages",
strings.NewReader(`{"model":"group/g","max_tokens":10,"stream":true,"messages":[{"role":"user","content":"hi"}]}`)))
return rec.Code, rec.Body.String()
}
code, body := send()
if code != 200 || !strings.Contains(body, "from b") || strings.Contains(body, "Overloaded") || strings.Contains(body, `"m1"`) {
t.Fatalf("%d %s %+v", code, body, s.trace.routes[len(s.trace.routes)-1])
}
r := s.trace.routes[len(s.trace.routes)-1]
if len(r.Tries) != 2 || r.Tries[0].Status != 529 || r.Tries[0].Rest == nil {
t.Fatalf("tries: %+v", r.Tries)
}
// after content, the error is the agent's to see
fresh(t)
late := sse(`event: message_start`+"\n"+`data: {"type":"message_start","message":{"id":"m1","role":"assistant","content":[]}}`,
`event: content_block_delta`+"\n"+`data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"half"}}`,
`event: error`+"\n"+`data: {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}`)
a = &scripted{replies: []reply{{200, "text/event-stream", late}}}
b = &scripted{replies: []reply{{200, "text/event-stream", good}}}
scriptedOn(t, "a", provider.Anthropic, a)
scriptedOn(t, "b", provider.Anthropic, b)
provider.SaveGroup(provider.Group{Name: "G", Members: []string{"a/m", "b/m"}, Routing: provider.Ordered})
s = New()
if code, body := send(); code != 200 || !strings.Contains(body, "half") || !strings.Contains(body, "Overloaded") || b.n != 0 {
t.Fatalf("late: %d %s (b %d)", code, body, b.n)
}
for _, x := range []struct {
ev string
kind int
}{
{`: keep-alive`, eventLead},
{`data: {"id":"c","choices":[{"index":0,"delta":{"role":"assistant","content":""}}]}`, eventLead},
{`data: {"id":"c","choices":[{"index":0,"delta":{"content":"hi"}}]}`, eventContent},
{`data: {"error":{"message":"upstream failed"}}`, eventError},
{`event: response.created` + "\n" + `data: {"type":"response.created","response":{"id":"r"}}`, eventLead},
{`event: codex.rate_limits` + "\n" + `data: {"type":"codex.rate_limits","plan_type":"plus","rate_limits":{"allowed":true}}`, eventLead},
{`event: response.failed` + "\n" + `data: {"type":"response.failed","response":{"error":{"code":"server_error","message":"boom"}}}`, eventError},
{`event: response.output_text.delta` + "\n" + `data: {"type":"response.output_text.delta","delta":"hi"}`, eventContent},
{`data: [DONE]`, eventContent},
} {
if k, _, _ := streamEvent([]byte(x.ev)); k != x.kind {
t.Errorf("streamEvent(%s) = %d, want %d", x.ev, k, x.kind)
}
}
}
// The last one left — or the only one — is tried again after a failure
// that may pass, a moment later; one that won't is the agent's at once.
func TestLastOneLeftIsTriedAgain(t *testing.T) {
fresh(t)
a := &scripted{replies: []reply{{503, "", `{"error":{"message":"upstream connect error"}}`}, {502, "", `{"error":{"message":"bad gateway"}}`}, {200, "", chatOK}}}
scriptedOn(t, "a", provider.Chat, a)
s := New()
code, body := postAs(t, s, "", `{"model":"a/m","messages":[{"role":"user","content":"hi"}]}`)
if code != 200 || !strings.Contains(body, "hello") || a.n != 3 {
t.Fatalf("%d %s (tried %d times)", code, body, a.n)
}
r := s.trace.routes[len(s.trace.routes)-1]
if len(r.Tries) != 3 || r.Tries[0].Again == 0 || r.Tries[1].Again == 0 || r.Tries[2].Status != 200 {
t.Fatalf("tries: %+v", r.Tries)
}
fresh(t)
a = &scripted{replies: []reply{{503, "", `{"error":{"message":"down"}}`}}}
scriptedOn(t, "a", provider.Chat, a)
// down for good: the same-one retries, then the rounds of its
// patience (singlePatience; with retryPause at 1ms the rounds'
// backstop comes first), then the agent has the 503
if code, _ := postAs(t, New(), "", `{"model":"a/m","messages":[{"role":"user","content":"hi"}]}`); code != 503 || a.n != 1+lastRetries+replanRounds {
t.Fatalf("gave up with %d after %d tries", code, a.n)
}
fresh(t)
a = &scripted{replies: []reply{{400, "", `{"error":{"message":"messages: field required"}}`}}}
scriptedOn(t, "a", provider.Chat, a)
if code, _ := postAs(t, New(), "", `{"model":"a/m","messages":[{"role":"user","content":"hi"}]}`); code != 400 || a.n != 1 {
t.Fatalf("a request at fault: %d after %d tries", code, a.n)
}
}
// A rate limit on the last one left is waited out, a moment, whether the
// vendor says how long or not (#503: a relay's 429 with no Retry-After went
// straight to the agent); one that won't clear in seconds — a plan used up,
// no money left, a Retry-After an hour away — is the agent's at once.
func TestRateLimitIsWaitedOut(t *testing.T) {
limited := `{"error":{"message":"Rate limit exceeded, please try again later"}}`
for _, x := range []struct {
name string
said string
header http.Header
times int // how many times it is limited first
code, n int
}{
{"retry-after", limited, http.Header{"Retry-After": {"1"}}, 1, 200, 2},
{"no retry-after", limited, nil, 1, 200, 2},
{"relay's own words", `{"error":{"message":"当前分组上游负载已饱和,请稍后再试"}}`, nil, 2, 200, 3},
{"thrice", limited, nil, 3, 200, 4},
// past the same-one retries, it is waited out within the plain
// provider's patience (zoecats on Discord), and given up on once
// that is spent (the rounds' backstop, with retryPause at 1ms)
{"past the same-one retries", limited, nil, 9, 200, 10},
{"never clears", limited, nil, 99, 429, 1 + rateRetries + replanRounds},
{"usage limit", `{"error":{"message":"You've hit your usage limit. Your limit resets in 5 hours."}}`, nil, 9, 429, 1},
{"no balance", `{"error":{"message":"余额不足"}}`, nil, 9, 429, 1},
{"an hour away", limited, http.Header{"Retry-After": {"3600"}}, 9, 429, 1},
} {
fresh(t)
a := &scripted{limited: x.header}
for range x.times {
a.replies = append(a.replies, reply{429, "", x.said})
}
a.replies = append(a.replies, reply{200, "", chatOK})
scriptedOn(t, "a", provider.Chat, a)
start := time.Now()
code, body := postAs(t, New(), "", `{"model":"a/m","messages":[{"role":"user","content":"hi"}]}`)
if code != x.code || a.n != x.n {
t.Errorf("%s: %d after %d tries: %s", x.name, code, a.n, body)
}
if x.n == 1 && time.Since(start) > time.Second {
t.Errorf("%s: passed on after %v", x.name, time.Since(start))
}
}
}
// An agent that goes away mid-request isn't a provider failing: nobody
// rests for it and nobody else is asked.
func TestCanceledRequestRestsNobody(t *testing.T) {
fresh(t)
a := &scripted{replies: []reply{{200, "", chatOK}}, hang: make(chan struct{}), hanging: make(chan struct{})}
b := &scripted{replies: []reply{{200, "", chatOK}}}
scriptedOn(t, "a", provider.Chat, a)
scriptedOn(t, "b", provider.Chat, b)
provider.SaveGroup(provider.Group{Name: "G", Members: []string{"a/m", "b/m"}, Routing: provider.Ordered})
s := New()
ctx, cancel := context.WithCancel(context.Background())
served := make(chan struct{})
// the agent goes away while a is asked (not a moment after the request
// began: on a busy machine a may not be asked yet, #2175), and a never
// answers until the gateway is done with the request
go func() {
select {
case <-a.hanging:
cancel()
case <-served:
}
select {
case <-served:
case <-time.After(10 * time.Second):
}
close(a.hang)
}()
rec := httptest.NewRecorder()
s.Handler().ServeHTTP(rec, httptest.NewRequest("POST", "/v1/chat/completions",
strings.NewReader(`{"model":"group/g","messages":[{"role":"user","content":"hi"}]}`)).WithContext(ctx))
close(served)
r := s.trace.routes[len(s.trace.routes)-1]
if b.n != 0 || len(r.Tries) != 1 || r.Tries[0].Fail != failCanceled || r.Tries[0].Rest != nil || s.resting("a") {
t.Fatalf("b tried %d, tries %+v", b.n, r.Tries)
}
}