Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

11.3 KB
Raw Blame
package gateway

import (
	"encoding/json"
	"io"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/yetone/magpie/internal/provider"
	"github.com/yetone/magpie/internal/usage"
)

// servedBy is a vendor that answers in its protocol, streamed when asked,
// naming model as the one that answered.
func servedBy(proto provider.Protocol, model string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		b, _ := io.ReadAll(r.Body)
		var req struct {
			Stream bool `json:"stream"`
		}
		json.Unmarshal(b, &req)
		m, _ := json.Marshal(model)
		M := string(m)
		if !req.Stream {
			w.Header().Set("Content-Type", "application/json")
			switch proto {
			case provider.Chat:
				io.WriteString(w, `{"id":"c1","object":"chat.completion","model":`+M+`,"choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":2}}`)
			case provider.Responses:
				io.WriteString(w, `{"id":"r1","object":"response","model":`+M+`,"status":"completed","output":[{"type":"message","id":"msg_1","role":"assistant","content":[{"type":"output_text","text":"hi"}]}],"usage":{"input_tokens":5,"output_tokens":2}}`)
			default:
				io.WriteString(w, `{"id":"msg_1","type":"message","role":"assistant","model":`+M+`,"content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn","usage":{"input_tokens":5,"output_tokens":2}}`)
			}
			return
		}
		var events []string
		switch proto {
		case provider.Chat:
			events = []string{
				`data: {"id":"c1","model":` + M + `,"choices":[{"index":0,"delta":{"role":"assistant","content":"hi"}}]}`,
				`data: {"id":"c1","model":` + M + `,"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":2}}`,
				`data: [DONE]`,
			}
		case provider.Responses:
			events = []string{
				"event: response.created\ndata: " + `{"type":"response.created","response":{"id":"r1","model":` + M + `,"status":"in_progress","output":[]}}`,
				"event: response.output_item.added\ndata: " + `{"type":"response.output_item.added","output_index":0,"item":{"type":"message","id":"msg_1","role":"assistant","content":[]}}`,
				"event: response.output_text.delta\ndata: " + `{"type":"response.output_text.delta","item_id":"msg_1","output_index":0,"content_index":0,"delta":"hi"}`,
				"event: response.output_item.done\ndata: " + `{"type":"response.output_item.done","output_index":0,"item":{"type":"message","id":"msg_1","role":"assistant","content":[{"type":"output_text","text":"hi"}]}}`,
				"event: response.completed\ndata: " + `{"type":"response.completed","response":{"id":"r1","model":` + M + `,"status":"completed","output":[],"usage":{"input_tokens":5,"output_tokens":2}}}`,
			}
		default:
			events = []string{
				"event: message_start\ndata: " + `{"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","model":` + M + `,"content":[],"usage":{"input_tokens":5,"output_tokens":0}}}`,
				"event: content_block_start\ndata: " + `{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}`,
				"event: content_block_delta\ndata: " + `{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hi"}}`,
				"event: content_block_stop\ndata: " + `{"type":"content_block_stop","index":0}`,
				"event: message_delta\ndata: " + `{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":2}}`,
				"event: message_stop\ndata: " + `{"type":"message_stop"}`,
			}
		}
		w.Header().Set("Content-Type", "text/event-stream")
		for _, ev := range events {
			io.WriteString(w, ev+"\n\n")
		}
	}
}

// The model a vendor's reply says answered is kept on the request's
// route, its try and its log entry, whatever the vendor speaks, streamed
// or not, relayed or translated; the route says it was swapped when it is
// another model than the one asked for, and not when it is that model's
// dated name.
func TestServedModelRecorded(t *testing.T) {
	bodies := map[string]string{
		"/v1/messages":         `{"model":"fake/sol","max_tokens":100,"messages":[{"role":"user","content":"hi"}]`,
		"/v1/chat/completions": `{"model":"fake/sol","messages":[{"role":"user","content":"hi"}]`,
		"/v1/responses":        `{"model":"fake/sol","input":"hi"`,
	}
	type path struct {
		name  string
		proto provider.Protocol // the vendor's
		agent string            // the agent's endpoint
	}
	paths := []path{
		{"anthropic", provider.Anthropic, "/v1/messages"},
		{"chat", provider.Chat, "/v1/chat/completions"},
		{"responses", provider.Responses, "/v1/responses"},
		{"chat from anthropic", provider.Anthropic, "/v1/chat/completions"},
		{"anthropic from chat", provider.Chat, "/v1/messages"},
		{"anthropic from responses", provider.Responses, "/v1/messages"},
	}
	for _, pa := range paths {
		for _, stream := range []bool{true, false} {
			for _, c := range []struct {
				served  string
				swapped bool
			}{
				{"luna", true},
				{"sol-2026-01-01", false},
				{"models/SOL", false},
			} {
				name := pa.name + "/" + c.served
				if stream {
					name += "/stream"
				}
				t.Run(name, func(t *testing.T) {
					fresh(t)
					up := httptest.NewServer(servedBy(pa.proto, c.served))
					t.Cleanup(up.Close)
					p := provider.Provider{ID: "fake", Name: "Fake", Key: "k", Models: []string{"sol"}}
					switch pa.proto {
					case provider.Chat:
						p.Chat = up.URL + "/v1"
					case provider.Responses:
						p.Responses = up.URL + "/v1"
					case provider.Anthropic:
						p.Anthropic = up.URL
					}
					if err := provider.Save(p); err != nil {
						t.Fatal(err)
					}
					body := bodies[pa.agent]
					if stream {
						body += `,"stream":true`
					}
					s := New()
					rec := httptest.NewRecorder()
					s.Handler().ServeHTTP(rec, httptest.NewRequest("POST", pa.agent, strings.NewReader(body+"}")))
					if rec.Code != 200 || !strings.Contains(rec.Body.String(), "hi") {
						t.Fatalf("%d %s", rec.Code, rec.Body)
					}
					r := lastRoute(s)
					if r.Served != c.served || r.Swapped != c.swapped {
						t.Errorf("route served %q swapped %v; want %q %v", r.Served, r.Swapped, c.served, c.swapped)
					}
					if len(r.Tries) != 1 || r.Tries[0].Model != "sol" || r.Tries[0].Served != c.served || r.Tries[0].Swapped != c.swapped {
						t.Errorf("tries %+v", r.Tries)
					}
					if calls := s.Recent(); len(calls) == 0 || calls[0].Usage.Served != c.served {
						t.Errorf("request log: %+v", calls)
					}
					// the usage ledger has the model asked for, the one sent
					// and the one that answered
					if u := usage.Load(time.Time{}); len(u) != 1 || u[0].Requested != "fake/sol" || u[0].Model != "sol" || u[0].Served != c.served {
						t.Errorf("usage %+v", u)
					}
				})
			}
		}
	}
}

// A reply that names no model marks nothing.
func TestServedModelUnnamed(t *testing.T) {
	fresh(t)
	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		io.WriteString(w, `{"id":"c1","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}]}`)
	}))
	t.Cleanup(up.Close)
	if err := provider.Save(provider.Provider{ID: "fake", Name: "Fake", Key: "k", Models: []string{"sol"}, Chat: up.URL + "/v1"}); err != nil {
		t.Fatal(err)
	}
	s := New()
	rec := httptest.NewRecorder()
	s.Handler().ServeHTTP(rec, httptest.NewRequest("POST", "/v1/chat/completions", strings.NewReader(`{"model":"fake/sol","messages":[{"role":"user","content":"hi"}]}`)))
	if r := lastRoute(s); rec.Code != 200 || r.Served != "" || r.Swapped || r.Tries[0].Swapped {
		t.Fatalf("%d, route %+v", rec.Code, r)
	}
	if u := usage.Load(time.Time{}); len(u) != 1 || u[0].Requested != "fake/sol" || u[0].Served != "" {
		t.Errorf("usage %+v", u)
	}
}

// swapped tells another model from the one asked for, and not the same
// one under its dated, pinned or prefixed name.
func TestSwappedModel(t *testing.T) {
	for _, c := range []struct {
		sent, served string
		want         bool
	}{
		{"sol", "luna", true},
		{"gpt-5", "gpt-5-mini", true},
		{"gpt-5", "gpt-5-mini-2025-08-07", true},
		{"gpt-4o", "gpt-4o-mini", true},
		{"claude-opus-4", "claude-opus-4-1-20250805", true},
		{"deepseek-v3", "deepseek-v2", true},
		{"gpt-5", "gpt-5.1", true},
		{"gpt-5", "gpt-5-2025-08-07", false},
		{"gpt-5", "GPT-5", false},
		{"gpt-4", "gpt-4-0613", false},
		{"claude-sonnet-4-5", "claude-sonnet-4-5-20250929", false},
		{"claude-3-5-sonnet-latest", "claude-3-5-sonnet-20241022", false},
		{"claude-sonnet-4-5-20250929", "claude-sonnet-4-5", false},
		{"gemini-2.5-pro", "models/gemini-2.5-pro", false},
		{"gemini-1.5-pro", "gemini-1.5-pro-002", false},
		{"gemini-2.5-pro", "gemini-2.5-pro-preview-06-05", false},
		{"openai/gpt-5", "gpt-5-2025-08-07", false},
		{"us.anthropic.claude-sonnet-4-20250514-v1:0", "claude-sonnet-4-20250514", false},
		{"auto", "gpt-5-mini", false}, // Copilot's Auto, Cursor's: the vendor was asked to pick
		{"copilot/auto", "claude-sonnet-5", false},
		// a remote magpie's routing group, answered by the member it routed to
		{"group/auto-deepseek-v4-1-flash", "deepseek/deepseek-v4.1-flash", false},
		{"group/fast", "luna", false},
		{"sol", "", false},
		{"", "luna", false},
	} {
		if got := swapped(c.sent, c.served); got != c.want {
			t.Errorf("swapped(%q, %q) = %v, want %v", c.sent, c.served, got, c.want)
		}
	}
}

// The sniffer reads the model a relayed reply names as it goes by.
func TestSniffServedModel(t *testing.T) {
	for _, c := range []struct {
		proto provider.Protocol
		ct    string
		body  string
	}{
		{provider.Chat, "text/event-stream", "data: {\"model\":\"luna\",\"choices\":[]}\n\ndata: [DONE]\n\n"},
		{provider.Chat, "application/json", `{"model":"luna","choices":[]}`},
		{provider.Responses, "text/event-stream", "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"model\":\"luna\"}}\n\n"},
		{provider.Responses, "application/json", `{"object":"response","model":"luna"}`},
		{provider.Anthropic, "text/event-stream", "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"luna\"}}\n\n"},
		{provider.Anthropic, "application/json", `{"type":"message","model":"luna"}`},
	} {
		s := newSniffer(c.proto, c.ct)
		s.write([]byte(c.body))
		if got := s.usage().Served; got != "luna" {
			t.Errorf("%s %s: served %q", c.proto, c.ct, got)
		}
	}
}

// A turn on one of Codex's own models, relayed to the ChatGPT backend,
// keeps the model its reply names as the others do.
func TestCodexOwnModelServedTraced(t *testing.T) {
	for _, c := range []struct {
		served  string
		swapped bool
	}{{"gpt-5.4-mini", true}, {"gpt-5.5-2026-04-01", false}} {
		setup(t, provider.Chat, &fake{t: t})
		chatgpt(t, func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Content-Type", "text/event-stream")
			io.WriteString(w, sse(`data: {"type":"response.created","response":{"id":"r1","model":"`+c.served+`"}}`,
				`data: {"type":"response.completed","response":{"id":"r1","model":"`+c.served+`","usage":{"input_tokens":9,"output_tokens":2}}}`))
		})
		s := New()
		rec := httptest.NewRecorder()
		req := httptest.NewRequest("POST", CodexPath+"/responses", strings.NewReader(`{"model":"gpt-5.5","stream":true,"input":"hi"}`))
		req.Header.Set("Authorization", "Bearer chatgpt-token")
		s.Handler().ServeHTTP(rec, req)
		r := lastRoute(s)
		if r.Served != c.served || r.Swapped != c.swapped || r.Tries[0].Served != c.served || r.Tries[0].Swapped != c.swapped {
			t.Errorf("%s: route %+v", c.served, r)
		}
	}
}