Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
package gateway
// Affinity: a conversation stays with the key or account that answered it,
// so that what the vendor cached of it — the whole conversation so far, on
// every request — is read again rather than sent afresh to someone else
// and paid for in full. Routing still decides who goes first when nobody
// has answered yet, and whenever the one that did is resting or all but
// used up.
//
// How long it stays is the provider's or group's affinity: for the whole
// session; within a turn only — while the agent sends tool results back,
// until the user speaks again; never; or, by default, worked out from the
// conversation itself: within a turn always, and across turns while what
// the vendor said it read from its cache the last time, or wrote to it, is
// worth keeping and not yet gone cold. Written counts as read: a first
// turn reads nothing and writes the whole prompt, which the next turn
// reads if it stays (vilicvane on Discord: the second turn moved, and the
// cache just built was thrown away). A vendor that keeps a prompt cached
// without reporting it — OpenAI's automatic prefix cache, Gemini — says
// neither of a turn that read nothing, and what its cache holds then is
// the prompt that turn was sent: that counts as written too, so its next
// turn stays rather than building the cache afresh elsewhere.
//
// Who answered is kept on disk as well (affinity.json, next to the
// providers), so a restart — an update's included — doesn't hand a
// conversation to whoever routing puts first while the vendor still has
// it cached at the one before (vincentzhang on Discord).
import (
"crypto/sha256"
"encoding/json"
"net/http"
"os"
"path/filepath"
"regexp"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/yetone/magpie/internal/edit"
"github.com/yetone/magpie/internal/provider"
"github.com/yetone/magpie/internal/steady"
)
const (
// cacheWorth is how many tokens read from the vendor's cache, or
// written to it — or, from a vendor that reports neither, sent — make
// a conversation worth keeping where it is across turns.
cacheWorth = 1024
// cacheCold is how long a vendor keeps a prompt cached without it being
// read: five minutes at Anthropic and at OpenAI, the shortest there is.
cacheCold = 5 * time.Minute
// chatGPTCold is how long the ChatGPT backend keeps a Codex
// conversation's prompt cached without it being read. Measured on a
// real account (10-27, gpt-5.6-terra), a Codex thread left 7, 12, 25,
// 45 and 60 minutes read back from cache all it had sent before (79–99%
// of each request), so it is
// held for the hour measured, not cacheCold's five minutes.
// Moving a conversation between two ChatGPT accounts after five
// minutes sent its whole context to the other uncached, and the
// reasoning the first had sealed was refused there (leelantao on X:
// two Plus accounts' allowance spent 90% in a day).
chatGPTCold = time.Hour
// cacheHour is how long Anthropic keeps what was written with
// cache_control's ttl "1h" (cache_creation's ephemeral_1h_input_tokens)
// without it being read, as Claude Code asks of a subscription (#2302)
cacheHour = time.Hour
// stickKeep is how long a conversation's last answerer is remembered.
stickKeep = 24 * time.Hour
// sticksKept is how many conversations, the latest, are kept on disk.
sticksKept = 512
)
// coldAfter is how long p's vendor keeps a prompt cached without it being
// read: cacheCold, or chatGPTCold for a ChatGPT account.
func coldAfter(p provider.Provider) time.Duration {
if p.Account != nil && p.Account.Agent == "codex" {
return chatGPTCold
}
return cacheCold
}
// Affinity is what the trace tells of a request's conversation and whether
// it stayed with who answered it last.
type Affinity struct {
Mode string `json:"mode"` // the provider's or group's: "", session, turn, off
Turn int `json:"turn"` // the user's turns in the conversation so far
Within bool `json:"within,omitempty"` // the agent sending tool results back: mid-turn
Last string `json:"last,omitempty"` // who answered the conversation last
LastTurn int `json:"lastTurn,omitempty"`
At time.Time `json:"at,omitempty"` // when
CacheRead int `json:"cacheRead,omitempty"` // tokens that answer read from the vendor's cache
CacheWrite int `json:"cacheWrite,omitempty"` // and wrote to it, or was sent where none is
Kept bool `json:"kept,omitempty"` // Last was put first for it
// Why it was kept — "session", "turn", "cache" — or not: "off", "first"
// (nobody has answered it yet), "new-turn", "no-cache" (the vendor
// read and wrote too little of its cache to keep, or was sent too
// little for a cache it keeps without saying so), "cold" (too long
// ago), "resting", "spent", "gone" (no longer one to route to),
// "main" (a provider's fallback answered it, and the provider's own
// model is ready again: backToMain).
Why string `json:"why"`
}
type stick struct {
rest string
who string // the key or account, however many were on
model string // the model it answered as: a group may have several on one account
effort string // the effort the group's member it answered as is fixed at
turn int
at time.Time
cacheRead int
cacheWrite int
hour hourCache
}
// hourCache is the part of a conversation's prompt the vendor said it keeps
// for an hour: Anthropic's ephemeral_1h_input_tokens, with what was read
// before them, since its 1-hour breakpoints come before any 5-minute one.
// It stays warm past cacheCold, until an hour after it was last written or
// read (#2302: Stays: Auto moved a conversation as "cold" 5 minutes after
// a 1-hour write). Vendors that report no 1-hour write never have one.
type hourCache struct {
tokens int
at time.Time
}
// after is what is kept for an hour once a reply with usage u came at now.
// A reply that wrote for an hour keeps all it read and that; one that read
// the whole of what was kept refreshes it; one that read less no longer
// starts with it (moved, compacted), and it is gone.
func (h hourCache) after(u Usage, now time.Time) hourCache {
switch {
case u.CacheWrite1h > 0:
return hourCache{tokens: u.CacheRead + u.CacheWrite1h, at: now}
case h.tokens > 0 && now.Sub(h.at) <= cacheHour && u.CacheRead >= h.tokens:
return hourCache{tokens: h.tokens, at: now}
}
return hourCache{}
}
// warm is whether enough is still kept at now to stay for.
func (h hourCache) warm(now time.Time) bool {
return h.tokens >= cacheWorth && now.Sub(h.at) <= cacheHour
}
var sticks = struct {
sync.Mutex
m map[string]stick // scope|conversation → who answered it last
// opening: scope|conversation → who a conversation that nobody has
// answered yet was sent to, while its first request is out. In turn,
// leastHeld counts them with those answered, so a burst of new
// conversations spreads rather than all going to the one holding the
// fewest a moment before (#1342)
opening map[string]stick
// the file as last read or written: another magpie's writes (the one
// handing over to this one) are read again when a conversation is missed
from string
mod time.Time
}{m: map[string]stick{}, opening: map[string]stick{}}
// sticksSaving keeps one write of the file at a time, the latest last.
var sticksSaving sync.Mutex
// sticksFlushAfter is how long a conversation answered waits before the
// file is written: what is answered meanwhile goes in the same write, so a
// gateway answering many requests a second writes it once in a while, not
// once a request (up to 512 conversations marshalled, written and renamed
// on every answer, each after the last). A crash loses the last moment's;
// flushSticks, as the gateway ends, loses nothing. In a test binary it is
// 0, the file written at once, as it was: a write left waiting would
// outlive the test and write into its temp folder as it is removed.
var sticksFlushAfter = func() time.Duration {
if testing.Testing() {
return 0
}
return 2 * time.Second
}()
// sticksFlush is the write noteSticks has waiting, for the file at path.
var sticksFlush struct {
sync.Mutex
pending bool
path string
}
// noteSticks has the file written within sticksFlushAfter: what this
// magpie answered is in memory at once.
func noteSticks() {
if sticksFlushAfter <= 0 {
saveSticks()
return
}
path := sticksPath()
sticksFlush.Lock()
if sticksFlush.pending && sticksFlush.path == path {
sticksFlush.Unlock()
return
}
sticksFlush.pending, sticksFlush.path = true, path
sticksFlush.Unlock()
time.AfterFunc(sticksFlushAfter, func() { flushSticksAt(path) })
}
// flushSticks writes what noteSticks has waiting: the gateway calls it as
// it ends, so the magpie it hands over to, or the next start, reads it.
func flushSticks() { flushSticksAt(sticksPath()) }
// FlushSticks is flushSticks, for magpie ending on a signal before the
// gateway's own end.
func FlushSticks() { flushSticks() }
// flushSticksAt writes the file at path when a write for it is waiting:
// one waiting for another (a test's HOME before this one's) is dropped.
func flushSticksAt(path string) {
sticksFlush.Lock()
if !sticksFlush.pending || sticksFlush.path != path {
sticksFlush.Unlock()
return
}
sticksFlush.pending = false
sticksFlush.Unlock()
if path == sticksPath() {
saveSticks()
}
}
// savedStick is a stick as the file has it.
type savedStick struct {
Rest string `json:"rest"`
Who string `json:"who"`
Model string `json:"model,omitempty"`
Effort string `json:"effort,omitempty"`
Turn int `json:"turn,omitempty"`
At time.Time `json:"at"`
CacheRead int `json:"cacheRead,omitempty"`
// CacheWrite is absent from a file written before it was kept, and
// reads as 0: such a stick is judged by what was read, as it was then
CacheWrite int `json:"cacheWrite,omitempty"`
// CacheHour and CacheHourAt are its hourCache, absent where none was
// written for an hour
CacheHour int `json:"cacheHour,omitempty"`
CacheHourAt time.Time `json:"cacheHourAt,omitzero"`
}
func sticksPath() string { return filepath.Join(filepath.Dir(provider.Path()), "affinity.json") }
// stickOf is who answered key last, read from the file again when it isn't
// known here and the file changed since: after a restart, or written by
// the magpie that handed over. Called with sticks held.
func stickOf(key string) (stick, bool) {
if st, ok := sticks.m[key]; ok {
return st, true
}
path := sticksPath()
fi, err := os.Stat(path)
if err != nil || path == sticks.from && fi.ModTime().Equal(sticks.mod) {
return stick{}, false
}
sticks.from, sticks.mod = path, fi.ModTime()
b, err := os.ReadFile(path)
if err != nil {
return stick{}, false
}
var saved map[string]savedStick
if decodeJSON(b, &saved) != nil {
return stick{}, false
}
for k, v := range saved {
if time.Since(v.At) > stickKeep || v.Who == "" {
continue
}
if st, ok := sticks.m[k]; ok && !st.at.Before(v.At) {
continue
}
sticks.m[k] = stick{rest: v.Rest, who: v.Who, model: v.Model, effort: v.Effort, turn: v.Turn, at: v.At, cacheRead: v.CacheRead, cacheWrite: v.CacheWrite,
hour: hourCache{tokens: v.CacheHour, at: v.CacheHourAt}}
}
st, ok := sticks.m[key]
return st, ok
}
// saveSticks writes the latest conversations' answerers to the file, only
// for this user to read.
func saveSticks() {
sticksSaving.Lock()
defer sticksSaving.Unlock()
sticksFlush.Lock()
if sticksFlush.path == sticksPath() {
sticksFlush.pending = false // this write is the one waiting
}
sticksFlush.Unlock()
now := time.Now()
type kept struct {
k string
v savedStick
}
var ks []kept
sticks.Lock()
for k, st := range sticks.m {
if now.Sub(st.at) <= stickKeep {
ks = append(ks, kept{k, savedStick{st.rest, st.who, st.model, st.effort, st.turn, st.at, st.cacheRead, st.cacheWrite, st.hour.tokens, st.hour.at}})
}
}
sticks.Unlock()
slices.SortFunc(ks, func(a, b kept) int { return b.v.At.Compare(a.v.At) })
ks = ks[:min(len(ks), sticksKept)]
out := make(map[string]savedStick, len(ks))
for _, k := range ks {
out[k.k] = k.v
}
b, err := json.Marshal(out)
if err != nil {
return
}
path := sticksPath()
if os.MkdirAll(filepath.Dir(path), 0o700) != nil {
return
}
path, err = edit.Target(path) // a symlink stays, its target written
if err != nil {
return
}
tmp := path + ".magpie-tmp"
if os.WriteFile(tmp, b, 0o600) != nil {
return
}
if steady.Rename(tmp, path) != nil {
os.Remove(tmp)
return
}
if fi, err := os.Stat(path); err == nil {
sticks.Lock()
// what this magpie wrote isn't read back as another's
sticks.from, sticks.mod = sticksPath(), fi.ModTime()
sticks.Unlock()
}
}
// turnOf counts the user's turns in a request's conversation, and says
// whether it is the agent handing tool results back within one.
func turnOf(from provider.Protocol, body []byte) (turn int, within bool) {
f := factsOf(from, body)
return f.turn, f.within
}
// turnIn is turnOf for a request already parsed. The last of the user's
// messages tells whether the turn goes on: an agent may put its own notes
// after it (Claude Code a system message after the tool results).
//
// A turn begins with what the user said (userSaid) or an image or file of
// theirs. A user message the agent wrote alone — a task notification, a
// todo nudge, an agent list, Codex's context — begins none: after an
// answer it carries the turn on, and right after the user's own message
// it leaves that as it was.
func turnIn(req *Request) (turn int, within bool) {
answered := false
for _, m := range req.Messages {
if m.Role == "assistant" {
answered = true
}
if m.Role != "user" {
continue
}
said, result := userSaid(m) != "", false
for _, p := range m.Parts {
switch p.Kind {
case Image, File:
said = true
case ToolResult:
result = true
}
}
switch {
case result:
within = true
case said:
turn++
within = false
case answered:
within = turn > 0
}
answered = false
}
return turn, within
}
// strictKeep is, for a conversation that stays strictly where it is
// (provider.AffinityStrict, #1797), the account that answered it brought
// back when the plan left it out at a usage cap: the cap set on it keeps
// new conversations off it, and one already on it goes on there, up to
// the vendor's own limit. One left out only as it won't spend its credits
// isn't brought back, the user's own setting, and the conversation doesn't
// move either: held is it, for the request to be refused as held. Neither
// for another mode, nor for a conversation nobody has answered yet.
func strictKeep(scope, mode string, in http.Header, body []byte, cs []candidate, pl planned) ([]candidate, planned, []Weighed) {
if mode != provider.AffinityStrict {
return cs, pl, nil
}
sticks.Lock()
st, had := stickOf(stickKey(scope, in, body))
sticks.Unlock()
if !had || time.Since(st.at) > stickKeep || slices.ContainsFunc(cs, func(c candidate) bool { return c.sameVendor(st.who, st.model) }) {
return cs, pl, nil
}
// the left's line for c, as it was left out
leftAt := func(c candidate) int {
return slices.IndexFunc(pl.left, func(w Weighed) bool {
return w.ID == c.rest && w.Model == c.model && w.Fixed == c.effort && w.Capped > 0
})
}
var back []candidate
var order []Weighed
for _, c := range pl.capped {
if !c.sameVendor(st.who, st.model) {
continue
}
i := leftAt(c)
if i < 0 {
continue
}
w := pl.left[i]
w.Unlisted, w.Capped, w.CapBack = false, 0, nil
pl.left = slices.Delete(slices.Clone(pl.left), i, i+1)
back, order = append(back, c), append(order, w)
}
if len(back) > 0 {
pl.capped = slices.DeleteFunc(slices.Clone(pl.capped), func(c candidate) bool { return c.sameVendor(st.who, st.model) })
return append(back, cs...), planned{order: append(order, pl.order...), left: pl.left, held: pl.held, capped: pl.capped}, nil
}
var held []Weighed
for _, c := range pl.held {
if !c.sameVendor(st.who, st.model) {
continue
}
if i := slices.IndexFunc(pl.left, func(w Weighed) bool { return w.ID == c.rest && w.NoCredits }); i >= 0 {
held = append(held, pl.left[i])
}
}
if len(held) > 0 {
// only its own reset, should it spend one by itself, brings it back
pl.held = slices.DeleteFunc(slices.Clone(pl.held), func(c candidate) bool { return !c.sameVendor(st.who, st.model) })
return nil, planned{order: nil, left: pl.left, held: pl.held, capped: pl.capped}, held[:1]
}
return cs, pl, nil
}
// strictOnly leaves of cs, for a conversation kept strictly, the key or
// account cs[at] alone: first as it answered, then its other models a
// group has on it, which a rule may still pick between. Nobody else is
// asked, nor a reset spent elsewhere: its error is the agent's.
func strictOnly(cs []candidate, pl planned, at int) ([]candidate, planned) {
who, model := cs[at].who(), cs[at].model
keep := []int{at}
for i, c := range cs {
if i != at && c.sameVendor(who, model) {
keep = append(keep, i)
}
}
out := make([]candidate, 0, len(keep))
order := make([]Weighed, 0, len(keep))
for _, i := range keep {
out = append(out, cs[i])
if i < len(pl.order) {
w := pl.order[i]
w.Turn = false
order = append(order, w)
}
}
pl.order = order
pl.held = slices.DeleteFunc(slices.Clone(pl.held), func(c candidate) bool { return !c.sameVendor(who, model) })
return out, pl
}
// stickKey is the conversation a stick is kept for: scope's, by the
// conversation's id, and a Claude Code subagent's apart from its lead's.
// Claude Code sends its session's id with every request of the session,
// its subagents' too, which say only by x-claude-code-agent-id that they
// are one (subagentOf). Under the session's key, a subagent's answer took
// the lead's stick over: the lead's next turn was judged by the cache of
// the subagent's conversation, moved to another account as "no-cache" or
// "cold" or to wherever the subagent had gone, and the lead's whole
// conversation written to that account's cache again (slkassinthehood on
// X: two Pro accounts' 5 hours spent in ten minutes). The lead's key is
// the session's as before, so what is kept of it, and Codex's leadFirst,
// which reads a lead's stick by its thread id, are as they were.
func stickKey(scope string, in http.Header, body []byte) string {
key := scope + "|" + conversationID(in, body)
if sub := subOf(in); sub != "" {
key += "|agent:" + sub
}
return key
}
// affine puts first whoever answered the conversation last, while its
// affinity says to keep it there. Its turn is counted in written, the body
// as the agent wrote it, which the turn's folder is read from too
// (dirOf): one parse of a held body serves both (factsOf), where counting
// it in the masked body parsed the conversation a second time, ~27 MB
// more for each 4 MB Codex request in flight.
func affine(scope, mode string, rotate bool, in http.Header, from provider.Protocol, body, written []byte, cs []candidate, pl planned) ([]candidate, planned, *Affinity, string) {
key := stickKey(scope, in, body)
a := &Affinity{Mode: mode}
a.Turn, a.Within = turnOf(from, written)
sticks.Lock()
st, had := stickOf(key)
sticks.Unlock()
if had && time.Since(st.at) > stickKeep {
had = false
}
if had {
a.Last, a.LastTurn, a.At, a.CacheRead, a.CacheWrite = st.rest, st.turn, st.at, st.cacheRead, st.cacheWrite
}
// the account and the model that answered: a group with Opus and
// Sonnet both on one Claude account, a rule sending the turn to Opus,
// had the turn's next request kept to the account and so to Sonnet,
// its first member there; the account alone when the model is gone,
// and behind a remote magpie the provider there alone (sameVendor)
at := -1
for _, same := range []func(candidate) bool{
func(c candidate) bool { return c.who() == st.who && c.model == st.model && c.effort == st.effort },
func(c candidate) bool { return c.who() == st.who && c.model == st.model },
func(c candidate) bool { return c.sameVendor(st.who, st.model) },
} {
for i, c := range cs {
if had && at < 0 && same(c) {
at, a.Last = i, c.rest // as it goes by now
}
}
}
switch {
case mode == provider.AffinityOff:
a.Why = "off"
case !had:
a.Why = "first"
case at < 0:
a.Why = "gone"
case mode == provider.AffinityStrict:
// whatever it is — resting, all but used up, past its cap
// (strictKeep) — it is the one asked (#1797)
a.Why = "strict"
case pl.order[at].Rest != nil:
a.Why = "resting"
case !a.Within && backToMain(pl.order, at):
// a provider's fallback is its emergency path, not a place a
// conversation stays: once one of the provider's own keys or
// accounts is ready again (its rest over, its quota back) the
// user's next turn goes to it, whatever the fallback has cached
// (01huadalang on Discord). Within a turn it stays where it is:
// the turn's tool loop runs on the reasoning the fallback sealed
// for it, which the provider can't read, and the whole prompt is
// sent afresh to the provider at the next turn all the same
a.Why = "main"
case at > 0 && pl.order[at].Known && pl.order[at].Used >= provider.SpentShareOf(cs[at].p.Routing):
// all but used up, and put behind another: first still — the
// one key or account it has (#1016), or every one used up — it
// doesn't move
a.Why = "spent"
case mode == provider.AffinitySession:
a.Why = "session"
case a.Within:
a.Why = "turn"
case mode == provider.AffinityTurn, rotate:
a.Why = "new-turn"
case st.cacheRead+st.cacheWrite < cacheWorth:
// what it wrote counts with what it read: the cache a first turn
// built is there for the next to read
a.Why = "no-cache"
case time.Since(st.at) > coldAfter(cs[at].p) && !st.hour.warm(time.Now()):
// past what the vendor keeps unread (five minutes, longer at
// the ChatGPT backend), kept still where the vendor said it
// keeps enough for an hour (#2302)
a.Why = "cold"
default:
a.Why = "cache"
}
switch a.Why {
case "session", "turn", "cache":
a.Kept = true
if at > 0 {
weightedKept(cs[0], cs[at])
cs = append(append([]candidate{cs[at]}, cs[:at]...), cs[at+1:]...)
order := append(append([]Weighed{pl.order[at]}, pl.order[:at]...), pl.order[at+1:]...)
pl.order = order
}
for j := range pl.order {
pl.order[j].Turn = false // kept, whoever's turn it was
}
case "strict":
a.Kept = true
if at > 0 {
weightedKept(cs[0], cs[at])
}
cs, pl = strictOnly(cs, pl, at)
case "new-turn":
if rotate {
cs, pl = after(cs, pl, at)
}
case "first":
if rotate {
cs, pl = leastHeld(scope, key, cs, pl)
}
}
return cs, pl, a, key
}
// backToMain says whether the one that answered last, order[at], is a
// provider's fallback while the provider's own model stands first, ready:
// restLast puts a resting one behind every ready one, so the first is the
// provider's own only when it can take the request. A routing group's
// members are never fallbacks, so its own affinity is as it was.
func backToMain(order []Weighed, at int) bool {
return at > 0 && at < len(order) && order[at].Fallback && !order[0].Fallback && order[0].Rest == nil
}
// heldFor is how long after its last answer a conversation still counts as
// holding the one that answered it: a session whose user is reading or
// typing is still going.
const heldFor = 30 * time.Minute
// leastHeld puts first, for a conversation nobody has answered yet, the
// one in turn that the fewest other conversations in scope are held on
// (#946): in turn counts requests, not conversations, so a second session
// started after the first had sent an even number of them went to the
// model the first was kept on. Of those held on as few, routing's order
// stands. A conversation whose first request is still out counts where it
// was sent (sticks.opening, under key until opened ends it): many started
// at once all went to the one holding the fewest before any of them was
// answered (#1342: 24 requests at once over 5 accounts).
func leastHeld(scope, key string, cs []candidate, pl planned) ([]candidate, planned) {
if len(cs) < 2 || len(pl.order) < len(cs) {
return cs, pl
}
if strings.HasPrefix(scope, provider.GroupPrefix) {
if g, ms, ok := provider.FindGroup(scope); ok {
for _, m := range ms {
// a group in the group, or a provider routed its own way:
// its own routing orders its models or keys (#576)
if len(m.Path) > 1 || ownRouting(m.Provider, g) {
return cs, pl
}
}
}
}
now := time.Now()
held := map[string]int{}
sticks.Lock()
defer sticks.Unlock()
for _, m := range []map[string]stick{sticks.m, sticks.opening} {
for k, st := range m {
if strings.HasPrefix(k, scope+"|") && now.Sub(st.at) <= heldFor {
held[st.who+"/"+provider.WithMemberEffort(st.model, st.effort)]++
}
}
}
defer func() {
// counted from now until it is answered or ends
sticks.opening[key] = stick{rest: cs[0].rest, who: cs[0].who(), model: cs[0].model, effort: cs[0].effort, at: now}
}()
if len(held) == 0 {
return cs, pl
}
seat := func(c candidate) string { return c.who() + "/" + provider.WithMemberEffort(c.model, c.effort) }
best := -1
for i, c := range cs {
if pl.order[i].Rest != nil || pl.order[i].Aside {
continue
}
if best < 0 || held[seat(c)] < held[seat(cs[best])] {
best = i
}
}
if best <= 0 {
return cs, pl
}
cs = append(append([]candidate{cs[best]}, cs[:best]...), cs[best+1:]...)
pl.order = append(append([]Weighed{pl.order[best]}, pl.order[:best]...), pl.order[best+1:]...)
for j := range pl.order {
pl.order[j].Turn = j == 0
}
return cs, pl
}
// after puts first the one after cs[at], round from the end, that isn't
// resting.
func after(cs []candidate, pl planned, at int) ([]candidate, planned) {
for d := 1; d < len(cs); d++ {
i := (at + d) % len(cs)
if pl.order[i].Rest != nil || pl.order[i].Aside {
continue // a turn goes round the keys routed over only
}
if i > 0 {
cs = append(append([]candidate{cs[i]}, cs[:i]...), cs[i+1:]...)
pl.order = append(append([]Weighed{pl.order[i]}, pl.order[:i]...), pl.order[i+1:]...)
}
for j := range pl.order {
pl.order[j].Turn = j == 0 // its turn after the last's, not by the count
}
break
}
return cs, pl
}
// answered remembers who answered a conversation, and what it read from
// the vendor's cache and wrote to it doing so — u. A vendor that keeps a
// prompt cached without reporting it — OpenAI's automatic prefix cache,
// Gemini — says neither of a turn that read none: what its cache holds
// then is the prompt that turn was sent, and that stands for the write, so
// the next turn stays to read it rather than moving as no-cache and
// building it afresh elsewhere. A reply that reports no usage at all
// (Input 0) counts as no cache, as it did.
func answered(key string, c candidate, turn int, u Usage) {
cacheRead, cacheWrite := u.CacheRead, u.CacheWrite
if cacheRead == 0 && cacheWrite == 0 {
cacheWrite = u.Input
}
now := time.Now()
sticks.Lock()
var hour hourCache
if st, ok := sticks.m[key]; ok && st.who == c.who() && st.model == c.model {
hour = st.hour // a cache is the account's, of the model
}
hour = hour.after(u, now)
sticks.m[key] = stick{rest: c.rest, who: c.who(), model: c.model, effort: c.effort, turn: turn, at: now, cacheRead: cacheRead, cacheWrite: cacheWrite, hour: hour}
delete(sticks.opening, key)
if len(sticks.m) > 4096 {
for k, st := range sticks.m {
if now.Sub(st.at) > stickKeep {
delete(sticks.m, k)
}
}
}
sticks.Unlock()
noteSticks()
}
// unanswered forgets that c answered a conversation, when its reply to it
// broke off (#733): the agent's retry goes by routing again, not back to
// the one that just failed it. The account alone is enough: affine's
// widest match keeps a conversation on an account whose answerer's model
// has since left the group, so a stick that only matched by its model
// would leave the next request kept on the account that just broke off.
// opened ends a request's part in leastHeld's count of conversations being
// opened, however it ended.
func opened(key string) {
sticks.Lock()
delete(sticks.opening, key)
sticks.Unlock()
}
func unanswered(key string, c candidate) {
sticks.Lock()
st, ok := stickOf(key)
ok = ok && c.sameVendor(st.who, st.model)
if ok {
delete(sticks.m, key)
}
sticks.Unlock()
if ok {
noteSticks()
}
}
// foreignReasoning is how a vendor refuses reasoning another account (or
// organization, or vendor) sealed: OpenAI's "The encrypted content for
// item rs_… could not be verified", invalid_encrypted_content; xAI's
// (Grok's API and a SuperGrok account alike) "Could not decrypt the
// provided encrypted_content. Ensure the value is the unmodified
// encrypted_content from a previous response."
var foreignReasoning = regexp.MustCompile(`(?i)invalid_encrypted_content|encrypted[ _]content.{0,80}could not be (verified|decrypted)|could not (decrypt|verify).{0,40}encrypted[ _]content`)
// refusedSeals remembers, per conversation and the key or account that
// refused it, the sealed reasoning another one wrote. Codex hands every
// earlier turn's reasoning back on each request, so once a conversation
// has moved, what was sealed before it moved is left out of its later
// requests there up front, rather than refused and asked again each turn.
// Reasoning the one it moved to sealed itself still goes along, for its
// cache and its train of thought.
var refusedSeals = struct {
sync.Mutex
m map[string]sealsRefused // scope|conversation|who → what it refused
}{m: map[string]sealsRefused{}}
type sealsRefused struct {
at time.Time
seals map[[sha256.Size]byte]bool
}
// sealedItem is what's read of an input item to tell sealed reasoning.
type sealedItem struct {
Type string `json:"type"`
Enc string `json:"encrypted_content"`
}
// sealedKinds are the input items a vendor seals for itself: its
// reasoning, and the compaction a conversation's earlier turns were
// folded into (OpenAI's, when Codex compacts on its own models; magpie's
// own is text by the time a request is sent).
var sealedKinds = map[string]bool{"reasoning": true, "compaction": true, "compaction_summary": true}
var compactionKinds = []string{"compaction", "compaction_summary"}
// seals are the sealed reasoning and compactions (encrypted_content) in a
// Responses request's input, of these types.
func seals(body []byte, kinds ...string) [][sha256.Size]byte {
var q struct {
Input []sealedItem `json:"input"`
}
if decodeJSON(body, &q) != nil {
return nil
}
var out [][sha256.Size]byte
for _, it := range q.Input {
if slices.Contains(kinds, it.Type) && it.Enc != "" {
out = append(out, sha256.Sum256([]byte(it.Enc)))
}
}
return out
}
// refused notes that who turned away the sealed items of these types in
// body, in the conversation key names.
func refused(key, who string, body []byte, kinds ...string) {
ss := seals(body, kinds...)
if len(ss) == 0 {
return
}
now := time.Now()
refusedSeals.Lock()
defer refusedSeals.Unlock()
k := key + "|" + who
r := refusedSeals.m[k]
if r.seals == nil {
r.seals = map[[sha256.Size]byte]bool{}
}
for _, s := range ss {
r.seals[s] = true
}
r.at = now
refusedSeals.m[k] = r
if len(refusedSeals.m) > 4096 {
for k, r := range refusedSeals.m {
if now.Sub(r.at) > stickKeep {
delete(refusedSeals.m, k)
}
}
}
}
// withoutRefused takes out of a Responses request the sealed reasoning
// and compactions who already refused in this conversation; the rest
// stays as it is.
func withoutRefused(key, who string, body []byte) ([]byte, bool) {
k := key + "|" + who
refusedSeals.Lock()
r, ok := refusedSeals.m[k]
if ok && time.Since(r.at) > stickKeep {
delete(refusedSeals.m, k)
ok = false
}
refusedSeals.Unlock()
if !ok {
return nil, false
}
var q map[string]json.RawMessage
if decodeJSONView(body, &q) != nil {
return nil, false
}
items, isList := inputList(q["input"])
if !isList {
return nil, false
}
kept := items[:0:0]
for _, it := range items {
var t sealedItem
if decodeJSONView(it, &t) == nil && sealedKinds[t.Type] && t.Enc != "" && r.seals[sha256.Sum256([]byte(t.Enc))] {
continue
}
kept = append(kept, it)
}
if len(kept) == len(items) {
return nil, false
}
q["input"], _ = json.Marshal(kept)
b, err := marshalJSON(q)
return b, err == nil
}
// withoutReasoning takes the sealed reasoning out of a Responses request's
// input — what another account wrote and this one can't read. What was
// said and done stays; only the model's private notes to itself go.
func withoutReasoning(body []byte) ([]byte, bool) {
return withoutKinds(body, "reasoning")
}
// withoutCompaction takes out of a Responses request's input the sealed
// compaction of its earlier turns — OpenAI's, which no other vendor or
// account can read (waroy: Grok's "Could not decrypt the provided
// encrypted_content" with the reasoning already gone). What the turns
// since said and did stays; the summary of those before goes.
func withoutCompaction(body []byte) ([]byte, bool) {
var q struct {
Input []sealedItem `json:"input"`
}
if decodeJSON(body, &q) != nil {
return nil, false
}
for _, it := range q.Input {
if slices.Contains(compactionKinds, it.Type) && it.Enc != "" {
return withoutKinds(body, compactionKinds...)
}
}
return nil, false
}
// withoutKinds is a Responses request without its input items of these
// types.
func withoutKinds(body []byte, kinds ...string) ([]byte, bool) {
var q map[string]json.RawMessage
if decodeJSONView(body, &q) != nil {
return nil, false
}
items, isList := inputList(q["input"])
if !isList {
return nil, false
}
kept := items[:0:0]
for _, it := range items {
var t struct {
Type string `json:"type"`
}
if decodeJSONView(it, &t) == nil && slices.Contains(kinds, t.Type) {
continue
}
kept = append(kept, it)
}
if len(kept) == len(items) {
return nil, false
}
q["input"], _ = json.Marshal(kept)
b, err := marshalJSON(q)
return b, err == nil
}