Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

8.6 KB
Raw Blame
package gateway

import (
	"cmp"
	"encoding/base64"
	"fmt"
	"math/rand/v2"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/yetone/magpie/internal/catalog"
	"github.com/yetone/magpie/internal/provider"
	"github.com/yetone/magpie/internal/settings"
)

// manyEyes is a key of a router listing n models that see, the cheapest a
// small model of a big vendor; all of them enabled, or three. (All reads
// the providers' file each time, which is as long as what is enabled.)
func manyEyes(t *testing.T, n int, all bool) {
	t.Helper()
	fresh(t)
	var ms []catalog.Model
	var every []string
	for i := range n {
		id := fmt.Sprintf("vendor%d/model-%d", i, i)
		if i == n-1 {
			id = "google/gemini-3-flash"
		}
		ms = append(ms, catalog.Model{ID: id, Images: true, ImageInput: imageInputBool(true), Price: &catalog.Price{Input: float64(n - i), Output: 1}})
		every = append(every, id)
	}
	ids := []string{ms[0].ID, ms[1].ID, "google/gemini-3-flash"}
	if all {
		ids = every
	}
	if err := provider.Save(provider.Provider{ID: "eyes", Name: "Eyes", Chat: "http://eyes.invalid/v1", Key: "k", Models: ids}); err != nil {
		t.Fatal(err)
	}
	if err := catalog.SaveLive("eyes", "http://eyes.invalid/v1", ms); err != nil {
		t.Fatal(err)
	}
}

// AutoVision is asked for every agent's list of models (provider.Described,
// dsh's every 30s, the window's state), and built every provider's list
// again each time, twice, then asked whether each of its models was out of
// its allowance: 5.3 GB of a day's 36 GB of allocations. Asked again with
// nothing changed, it costs the same whatever the lists' length.
func TestAutoVisionDoesNotBuildListsAgain(t *testing.T) {
	allocs := func(n int) float64 {
		manyEyes(t, n, false)
		if got := AutoVision(); got != "eyes/google/gemini-3-flash" {
			t.Fatalf("AutoVision = %q of %d", got, n)
		}
		return testing.AllocsPerRun(20, func() { AutoVision() })
	}
	few, many := allocs(4), allocs(400)
	t.Logf("allocations an AutoVision: %.0f with 4 models, %.0f with 400", few, many)
	if many > few+50 {
		t.Fatalf("AutoVision made %.0f allocations with 400 models, %.0f with 4: the lists are built again", many, few)
	}
	// and the searcher's pick, built the same way
	p, err := provider.Find("eyes")
	if err != nil {
		t.Fatal(err)
	}
	searcherModel(*p)
	if a := testing.AllocsPerRun(20, func() { searcherModel(*p) }); a > 50 {
		t.Fatalf("searcherModel made %.0f allocations with 400 models", a)
	}
}

// What AutoVision keeps is built again when what it was built of changes:
// a provider saved (its picks), a vendor's list fetched, an account signed
// in, the settings and the providers' file written by another magpie.
func TestAutoVisionSeesEachChange(t *testing.T) {
	routerUp(t, "text")
	if m := AutoVision(); m != "" {
		t.Fatalf("AutoVision = %q with no model that sees enabled", m)
	}

	// a provider saved: Haiku enabled
	p, err := provider.Find("router")
	if err != nil {
		t.Fatal(err)
	}
	p.Models = []string{"text", "anthropic/claude-haiku-5.5", "google/gemini-3-flash"}
	if err := provider.Save(*p); err != nil {
		t.Fatal(err)
	}
	if m := AutoVision(); m != "router/google/gemini-3-flash" {
		t.Fatalf("AutoVision = %q after Haiku and Gemini were enabled", m)
	}

	// the vendor's list fetched anew: Haiku the cheaper now
	if err := catalog.SaveLive("router", p.Chat, []catalog.Model{
		{ID: "text", ImageInput: imageInputBool(false), Price: &catalog.Price{Input: 0.1, Output: 0.1}},
		{ID: "anthropic/claude-haiku-5.5", Images: true, ImageInput: imageInputBool(true), Price: &catalog.Price{Input: 0.1, Output: 0.2}},
		{ID: "google/gemini-3-flash", Images: true, ImageInput: imageInputBool(true), Price: &catalog.Price{Input: 0.3, Output: 2.5}},
	}); err != nil {
		t.Fatal(err)
	}
	if m := AutoVision(); m != "router/anthropic/claude-haiku-5.5" {
		t.Fatalf("AutoVision = %q after the list priced Haiku the cheaper", m)
	}

	// the providers' file written by another magpie (magpie provider
	// edit in a terminal): Haiku taken out
	b, err := os.ReadFile(provider.Path())
	if err != nil {
		t.Fatal(err)
	}
	other := strings.Replace(string(b), `"anthropic/claude-haiku-5.5",`, "", 1)
	if other == string(b) {
		t.Fatalf("no Haiku pick in %s", b)
	}
	if err := os.WriteFile(provider.Path(), []byte(other), 0o600); err != nil {
		t.Fatal(err)
	}
	if m := AutoVision(); m != "router/google/gemini-3-flash" {
		t.Fatalf("AutoVision = %q after another magpie took Haiku out", m)
	}

	// a setting saved moves what it is kept by
	before := provider.StampNow()
	st := settings.Load()
	st.Searcher = "off"
	if err := settings.Save(st); err != nil {
		t.Fatal(err)
	}
	if provider.StampNow() == before {
		t.Fatal("a setting saved left the stamp where it was")
	}

	// an account signed in, in this same home: Codex's tier goes first
	if err := catalog.SaveLive("codex", provider.CodexBase, []catalog.Model{{ID: "gpt-6.1-sol", Images: true, ImageInput: imageInputBool(true)}}); err != nil {
		t.Fatal(err)
	}
	home, _ := os.UserHomeDir()
	os.MkdirAll(filepath.Join(home, ".codex"), 0o755)
	claims := func(m map[string]any) string {
		return "h." + base64.RawURLEncoding.EncodeToString(mustJSON(m)) + ".s"
	}
	auth := map[string]any{"auth_mode": "chatgpt", "tokens": map[string]any{
		"id_token":      claims(map[string]any{"email": "[email protected]"}),
		"access_token":  claims(map[string]any{"exp": time.Now().Add(time.Hour).Unix(), "who": "acct-1"}),
		"refresh_token": "r-acct-1", "account_id": "acct-1"}}
	if err := os.WriteFile(filepath.Join(home, ".codex", "auth.json"), mustJSON(auth), 0o600); err != nil {
		t.Fatal(err)
	}
	provider.ForgetAccounts() // the 30s accounts are looked for again in
	t.Cleanup(provider.ForgetAccounts)
	if m := AutoVision(); m != "codex/gpt-6.1-sol" {
		t.Fatalf("AutoVision = %q after Codex was signed in", m)
	}
}

// A model out of its allowance is passed over as soon as it is, though
// the lists are kept: what is out is asked each time.
func TestAutoVisionKeptListsAskAllowanceEachTime(t *testing.T) {
	codexSignedIn(t)
	used := map[string]float64{"[email protected]": 20}
	capUsage(t, used)
	if err := catalog.SaveLive("codex", provider.CodexBase, []catalog.Model{{ID: "gpt-6.1-sol", Images: true, ImageInput: imageInputBool(true)}}); err != nil {
		t.Fatal(err)
	}
	if got, ready := autoVision(); got != "codex/gpt-6.1-sol" || !ready {
		t.Fatalf("autoVision = %q %v", got, ready)
	}
	used["[email protected]"] = 100
	if got, ready := autoVision(); got != "codex/gpt-6.1-sol" || ready {
		t.Fatalf("autoVision = %q %v with the only account used up", got, ready)
	}
}

// rankSmall's order is pickSmall's pick of every subset, as pickSmall
// was before the ranking: the first of a subset is its pick.
func TestRankSmallIsPickSmall(t *testing.T) {
	pickSmall := func(ms []catalog.Model) string {
		if big := slices.DeleteFunc(slices.Clone(ms), func(m catalog.Model) bool {
			v, _, ok := strings.Cut(m.ID, "/")
			return ok && !slices.Contains([]string{"anthropic", "openai", "google", "x-ai"}, v)
		}); len(big) > 0 {
			ms = big
		}
		cost := func(m catalog.Model) float64 {
			if m.Price == nil {
				return 1e9
			}
			return m.Price.Input + m.Price.Output
		}
		cheapest := func(ms []catalog.Model) string {
			if len(ms) == 0 {
				return ""
			}
			return slices.MinFunc(ms, func(a, b catalog.Model) int { return cmp.Compare(cost(a), cost(b)) }).ID
		}
		small := slices.DeleteFunc(slices.Clone(ms), func(m catalog.Model) bool {
			l := strings.ReplaceAll(strings.ToLower(m.ID), "gemini", "")
			return !strings.Contains(l, "haiku") && !strings.Contains(l, "mini") && !strings.Contains(l, "flash")
		})
		if id := cheapest(small); id != "" {
			return id
		}
		return cheapest(ms)
	}
	r := rand.New(rand.NewPCG(1, 2))
	vendors := []string{"", "anthropic/", "openai/", "google/", "x-ai/", "meta/", "qwen/"}
	names := []string{"claude-haiku-5", "gpt-6-mini", "gemini-3-flash", "gemini-3-pro", "big", "o9", "llama"}
	for range 2000 {
		var ms []catalog.Model
		for i := range r.IntN(8) + 1 {
			m := catalog.Model{ID: fmt.Sprintf("%s%s-%d", vendors[r.IntN(len(vendors))], names[r.IntN(len(names))], i)}
			if r.IntN(4) > 0 {
				m.Price = &catalog.Price{Input: float64(r.IntN(3)), Output: float64(r.IntN(3))}
			}
			ms = append(ms, m)
		}
		cs := rankSmall(ms)
		if cs[0].id != pickSmall(ms) {
			t.Fatalf("rankSmall's first %q, pickSmall %q of %v", cs[0].id, pickSmall(ms), ms)
		}
		// a subset: those not spent
		keep := map[string]bool{}
		var sub []catalog.Model
		for _, m := range ms {
			if r.IntN(2) == 0 {
				keep[m.ID] = true
				sub = append(sub, m)
			}
		}
		first := ""
		for _, c := range cs {
			if keep[c.id] {
				first = c.id
				break
			}
		}
		if first != pickSmall(sub) {
			t.Fatalf("rankSmall's first of a subset %q, pickSmall %q of %v", first, pickSmall(sub), sub)
		}
	}
}