Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

6.9 KB
Raw Blame
package gateway

import (
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/yetone/magpie/internal/provider"
)

// #2274 (huangzuomin): Codex compacting through magpie on
// antigravity/gemini-3.8-flash asked for 1169 summaries in 6 hours. Each
// summary's stream was let through at its first words, so Antigravity
// cutting it short ("the reply ended before it was complete"), or a 429
// "Individual quota reached" halfway, was the only answer: the next
// account was never asked, nobody rested, and Codex got a 502 it asked
// again and again on.

// compactPair: "ag" (the model Codex compacts on) falls back to "spare",
// each a fake on proto's wire.
func compactPair(t *testing.T, proto provider.Protocol, ag, spare *fake) {
	t.Helper()
	noRests()
	t.Cleanup(noRests)
	t.Setenv("XDG_CONFIG_HOME", t.TempDir())
	t.Setenv("XDG_CACHE_HOME", t.TempDir())
	for _, x := range []struct {
		p provider.Provider
		f *fake
	}{
		{provider.Provider{ID: "ag", Name: "AG", Key: "k", Models: []string{"m1"}, Fallback: []string{"spare/m1"}}, ag},
		{provider.Provider{ID: "spare", Name: "Spare", Key: "k", Models: []string{"m1"}}, spare},
	} {
		up := httptest.NewServer(x.f)
		t.Cleanup(up.Close)
		switch proto {
		case provider.Chat:
			x.p.Chat = up.URL + "/v1"
		default:
			x.p.Responses = up.URL + "/v1"
		}
		if err := provider.Save(x.p); err != nil {
			t.Fatal(err)
		}
	}
}

// codexCompactOf is compact404Request on model, in conversation key.
func codexCompactOf(model, key string) string {
	b := strings.Replace(compact404Request, `"model":"fake/m1"`, `"model":"`+model+`","prompt_cache_key":"`+key+`"`, 1)
	return b
}

// the summary's stream, cut or failed after its first words, on each wire
func cutReplies(proto provider.Protocol) map[string]string {
	if proto == provider.Chat {
		// (a Chat stream may end without [DONE] and be whole, so one that
		// just stops isn't cut: errEndedShort)
		return map[string]string{
			"quota halfway": sse(`data: {"choices":[{"delta":{"content":"PARTIAL summary of"}}]}`,
				`data: {"error":{"message":"Individual quota reached","code":429}}`),
		}
	}
	return map[string]string{
		"cut": sse(`data: {"type":"response.created","response":{"id":"r1"}}`,
			`data: {"type":"response.output_text.delta","delta":"PARTIAL summary of"}`,
			`data: {"type":"response.output_item.done","output_index":0,"item":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"PARTIAL summary of"}]}}`),
		"quota halfway": sse(`data: {"type":"response.created","response":{"id":"r1"}}`,
			`data: {"type":"response.output_text.delta","delta":"PARTIAL summary of"}`,
			`data: {"type":"response.failed","response":{"id":"r1","status":"failed","error":{"code":"rate_limit_exceeded","message":"Individual quota reached"}}}`),
		"incomplete": sse(`data: {"type":"response.created","response":{"id":"r1"}}`,
			`data: {"type":"response.output_item.done","output_index":0,"item":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"PARTIAL summary of"}]}}`,
			`data: {"type":"response.incomplete","response":{"id":"r1","status":"incomplete","incomplete_details":{"reason":"max_output_tokens"},"output":[]}}`),
	}
}

func wholeSummary(proto provider.Protocol) string {
	if proto == provider.Chat {
		return sse(`data: {"choices":[{"delta":{"content":"SUMMARY"}}]}`,
			`data: {"choices":[{"delta":{},"finish_reason":"stop"}]}`, `data: [DONE]`)
	}
	return sse(`data: {"type":"response.output_item.done","output_index":0,"item":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"SUMMARY"}]}}`,
		`data: {"type":"response.completed","response":{"id":"r2","status":"completed","output":[]}}`)
}

// A summary cut short, or failed halfway, goes to the next candidate, and
// the whole one it writes is the compaction; nothing of the cut one is.
func TestCodexCompactCutSummaryFailsOver(t *testing.T) {
	for _, proto := range []provider.Protocol{provider.Responses, provider.Chat} {
		for name, cut := range cutReplies(proto) {
			t.Run(string(proto)+"/"+name, func(t *testing.T) {
				ag := &fake{t: t, reply: cut}
				spare := &fake{t: t, reply: wholeSummary(proto)}
				compactPair(t, proto, ag, spare)
				code, body := codexPost(t, codexCompactOf("ag/m1", "thread-1"))
				if code != 200 {
					t.Fatalf("%d %s", code, body)
				}
				if got := compactedText(t, body); got != "SUMMARY" {
					t.Errorf("summary %q, want the spare's whole one", got)
				}
				if spare.calls != 1 {
					t.Errorf("spare asked %d times", spare.calls)
				}
				_, rested := restOf("ag")
				if want := name != "incomplete"; rested != want {
					// a vendor that broke off rests; a model that reached
					// its output limit is no fault of the account
					t.Errorf("ag rested %v, want %v", rested, want)
				}
			})
		}
	}
}

// With nobody left, a cut summary is no summary: Codex is told why, and
// not handed the words before the cut as the conversation's summary.
func TestCodexCompactCutSummaryIsNoSummary(t *testing.T) {
	for name, cut := range cutReplies(provider.Responses) {
		t.Run(name, func(t *testing.T) {
			f := &fake{t: t, reply: cut}
			setup(t, provider.Responses, f)
			code, body := codexPost(t, compact404Request)
			if code < 400 || strings.Contains(body, magpieCompaction) {
				t.Fatalf("%d %s", code, body)
			}
		})
	}
}

// One conversation's compaction that fails compactFailsMost times in a
// row is answered at once after, with a 400 Codex doesn't ask again on,
// no model asked; another conversation's still is, and one that worked
// clears the count.
func TestCodexCompactKeepsFailingIsHeld(t *testing.T) {
	f := &fake{t: t, reply: cutReplies(provider.Responses)["cut"]}
	setup(t, provider.Responses, f)
	s := New()
	for i := range compactFailsMost {
		if rec := codexPostTo(s, codexCompactOf("fake/m1", "thread-1")); rec.Code < 500 {
			t.Fatalf("try %d: %d %s", i, rec.Code, rec.Body)
		}
	}
	asked := f.calls
	rec := codexPostTo(s, codexCompactOf("fake/m1", "thread-1"))
	if rec.Code != 400 || !strings.Contains(rec.Body.String(), "3 times in a row") {
		t.Fatalf("held: %d %s", rec.Code, rec.Body)
	}
	if f.calls != asked {
		t.Errorf("the model was asked again (%d calls, %d before)", f.calls, asked)
	}
	if rec := codexPostTo(s, codexCompactOf("fake/m1", "thread-2")); rec.Code == 400 || f.calls == asked {
		t.Errorf("another conversation: %d, calls %d", rec.Code, f.calls)
	}

	// a summary that came clears it
	s2 := New()
	f.reply = cutReplies(provider.Responses)["cut"]
	for range compactFailsMost - 1 {
		codexPostTo(s2, codexCompactOf("fake/m1", "thread-3"))
	}
	f.reply = wholeSummary(provider.Responses)
	if rec := codexPostTo(s2, codexCompactOf("fake/m1", "thread-3")); rec.Code != 200 {
		t.Fatalf("whole: %d %s", rec.Code, rec.Body)
	}
	f.reply = cutReplies(provider.Responses)["cut"]
	codexPostTo(s2, codexCompactOf("fake/m1", "thread-3"))
	if rec := codexPostTo(s2, codexCompactOf("fake/m1", "thread-3")); rec.Code == 400 {
		t.Errorf("held after a summary came: %s", rec.Body)
	}
}