Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
usage: a call through magpie adds up the heatmap only for the chunks it changed (#2278)
yetoneb69f8e96af15
diff --git a/internal/usage/heatmap.go b/internal/usage/heatmap.goindex d666f28c9..6c2b0d80e 100644--- a/internal/usage/heatmap.go+++ b/internal/usage/heatmap.go@@ -10,7 +10,10 @@ package usageimport ("slices"+ "strings""time"++ "github.com/yetone/magpie/internal/sessions")// HeatmapWeeks is how many weeks the heatmap shows, this one the last.@@ -59,52 +62,127 @@ func HeatmapOf(f Filter) Heatmap {// heatmapOf sums the rows visit gives it by their day in now's zone (the// reader's), from since to now's day; a row outside those days is left out.func heatmapOf(since, now time.Time, visit func(add func(Row))) *Heatmap {- h := &Heatmap{From: since.Format(time.DateOnly), To: now.Format(time.DateOnly), Days: []Day{}}- at := map[string]*Day{}+ days := heatDays{}+ visit(func(r Row) { days.add(&r, since, now.Location()) })+ return heatmapFrom(since, now, days.list())+}++// heatDays adds rows up a day each, by the day's name.+type heatDays struct {+ at map[string]*Day// a day's name is made once for each day, not for each of its calls- type ymd struct{ y, yd int }- names := map[ymd]string{}- visit(func(r Row) {- if r.IsRejected() || r.IsProbe() || r.Time.Before(since) {- return- }- t := r.Time.In(now.Location())- k := ymd{t.Year(), t.YearDay()}- d, ok := names[k]- if !ok {- d = t.Format(time.DateOnly)- names[k] = d+ names map[[2]int]string+}++// add adds r to its day in loc, unless the heatmap doesn't count it.+func (h *heatDays) add(r *Row, since time.Time, loc *time.Location) {+ if r.IsRejected() || r.IsProbe() || r.Time.Before(since) {+ return+ }+ if h.at == nil {+ h.at, h.names = map[string]*Day{}, map[[2]int]string{}+ }+ t := r.Time.In(loc)+ k := [2]int{t.Year(), t.YearDay()}+ d, ok := h.names[k]+ if !ok {+ d = t.Format(time.DateOnly)+ h.names[k] = d+ }+ x := h.at[d]+ if x == nil {+ x = &Day{Date: d}+ h.at[d] = x+ }+ if !r.HandedOn {+ x.Calls++ // a request, not each try at it+ }+ x.Tokens += r.Input + r.Output + r.CacheRead + r.CacheWrite+ switch {+ case r.Priced:+ x.Cost += r.Cost+ case r.Input+r.Output > 0:+ x.Unpriced+++ }+}++// addDay adds a day's sums to its day.+func (h *heatDays) addDay(d Day) {+ if h.at == nil {+ h.at = map[string]*Day{}+ }+ x := h.at[d.Date]+ if x == nil {+ x = &Day{Date: d.Date}+ h.at[d.Date] = x+ }+ x.Calls += d.Calls+ x.Tokens += d.Tokens+ x.Cost += d.Cost+ x.Unpriced += d.Unpriced+}++// list is the days, oldest first.+func (h *heatDays) list() []Day {+ out := make([]Day, 0, len(h.at))+ for _, x := range h.at {+ out = append(out, *x)+ }+ slices.SortFunc(out, func(a, b Day) int { return strings.Compare(a.Date, b.Date) })+ return out+}++// heatmapFrom is days from since to now's day, a day after it left out.+func heatmapFrom(since, now time.Time, days []Day) *Heatmap {+ out := &Heatmap{From: since.Format(time.DateOnly), To: now.Format(time.DateOnly), Days: []Day{}}+ for _, d := range days {+ if d.Date <= out.To {+ out.Days = append(out.Days, d)}- if d > h.To {- return+ }+ return out+}++// heatOf is heatmapOf of ps's rows f keeps, added up a chunk at a time:+// cache's sums of a chunk paired as before are taken as they are. A call+// through magpie changes the gateway's log, and the Requests tab's refresh+// read every row of the year again for it, and paired them: ~4 GB a+// minute and seconds of CPU a refresh while an agent worked (#2278).+func (ps *pairedRows) heatOf(since, now time.Time, f Filter, cache *partCache, identities map[string]sessions.CodexIdentity) *Heatmap {+ sigs := ps.sigs(identities)+ var rows chunkRows+ defer rows.release()+ var r Row+ all := heatDays{}+ for _, c := range ps.all {+ sig := sigs[c]+ var part *chunkPart+ if cache != nil && c.serial != 0 {+ if old := cache.old[c.serial]; old != nil && old.sig == sig {+ part = old+ }}- x := at[d]- if x == nil {- x = &Day{Date: d}- at[d] = x+ if part == nil {+ if heatReading != nil {+ heatReading(c)+ }+ one := heatDays{}+ ps.visitChunk(&rows, c, &r, func(_ timedRef, r *Row) {+ if f.keepsRow(r) {+ one.add(r, since, now.Location())+ }+ })+ part = &chunkPart{sig: sig, days: one.list()}}- if !r.HandedOn {- x.Calls++ // a request, not each try at it+ if cache != nil && c.serial != 0 {+ cache.next[c.serial] = part}- x.Tokens += r.Input + r.Output + r.CacheRead + r.CacheWrite- switch {- case r.Priced:- x.Cost += r.Cost- case r.Input+r.Output > 0:- x.Unpriced+++ for _, d := range part.days {+ all.addDay(d)}- })- for _, x := range at {- h.Days = append(h.Days, *x)}- slices.SortFunc(h.Days, func(a, b Day) int {- if a.Date < b.Date {- return -1- }- if a.Date > b.Date {- return 1- }- return 0- })- return h+ return heatmapFrom(since, now, all.list())}++// heatReading, in tests, is told each chunk heatOf adds up again.+var heatReading func(*rowChunk)diff --git a/internal/usage/heatmap_test.go b/internal/usage/heatmap_test.goindex 966d226c9..89888083c 100644--- a/internal/usage/heatmap_test.go+++ b/internal/usage/heatmap_test.go@@ -141,3 +141,48 @@ func TestHeatmapKeptWhileOnlySessionsChange(t *testing.T) {t.Fatalf("heatFresh up, %d calls", n)}}++// A call through magpie changes the gateway's log, and the heatmap is+// worked out again at once: from the sums of the chunks it had, adding up+// again only those the call changed, not every row of the year (#2278).+// What it shows is what a cold build shows.+func TestHeatmapAfterACallAddsUpOnlyWhatChanged(t *testing.T) {+ pageHome(t)+ now := holdClock(t, time.Date(2026, 9, 30, 12, 0, 0, 0, time.Local))+ const files = 8+ for i := range files {+ at := now.AddDate(0, 0, -3*i).Add(-time.Hour)+ id := string(rune('a' + i))+ path := filepath.Join(sessions.CodexDir(), "sessions", at.Format("2006/01/02"), "rollout-"+at.Format("2006-01-02T15-04-05")+"-"+id+".jsonl")+ writeCodexCall(t, path, id, "s"+id, `"vscode"`, "resp-"+id, "m", "low", at, 100+i, 10)+ }+ Append(Record{Time: now.AddDate(0, 0, -40), Agent: "claude-code", Provider: "anthropic", Model: "x", Input: 3, Status: 200})+ read := 0+ heatReading = func(*rowChunk) { read++ }+ t.Cleanup(func() { heatReading = nil })+ calls := func(h Heatmap) int {+ n := 0+ for _, d := range h.Days {+ n += d.Calls+ }+ return n+ }+ if h := HeatmapOf(Filter{}); calls(h) != files+1 || read < files {+ t.Fatalf("cold: %d calls, %d chunks added up", calls(h), read)+ }+ read = 0+ Append(Record{Time: now.Add(-time.Minute), Agent: "claude-code", Provider: "anthropic", Model: "x", Input: 5, Status: 200})+ warm := HeatmapOf(Filter{})+ if calls(warm) != files+2 {+ t.Fatalf("a call through magpie isn't counted at once: %d calls", calls(warm))+ }+ if read > 1 {+ t.Fatalf("a call added up %d chunks again, want only the one it changed", read)+ }+ requestCache.Lock()+ requestCache.heatParts, requestCache.heats, requestCache.pages = nil, nil, map[pageKey]RequestPage{}+ requestCache.Unlock()+ if cold := HeatmapOf(Filter{}); !reflect.DeepEqual(cold, warm) {+ t.Fatalf("kept sums %+v, cold %+v", warm, cold)+ }+}diff --git a/internal/usage/request_cells.go b/internal/usage/request_cells.goindex 2d7e726c3..1dc9a6d3f 100644--- a/internal/usage/request_cells.go+++ b/internal/usage/request_cells.go@@ -85,6 +85,8 @@ type chunkPart struct {cells []pageCellkept int // rows the filter keepsfirst time.Time // the oldest charted row's time+ // days are the heatmap's sums of the chunk, oldest first (heatOf)+ days []Day}// partKey is what a chunk's cells are of besides the chunk: the page'sdiff --git a/internal/usage/request_page.go b/internal/usage/request_page.goindex 441fc31b9..8aa28cc30 100644--- a/internal/usage/request_page.go+++ b/internal/usage/request_page.go@@ -535,6 +535,9 @@ type requestIndex struct {// names a call changes (meta), and read without the lock, since a// part is never written once madeparts map[partKey]map[uint64]*chunkPart+ // heatParts are the heatmap's sums of each chunk (heatOf), kept as+ // parts are but apart, so that the heatmap's never push a page's out+ heatParts map[partKey]map[uint64]*chunkPart// pairs are the last pairing of each of a few periods (pairMemo),// kept as parts arepairs map[Period]*pairMemo@@ -767,10 +770,12 @@ func queryPageOf(p Period, f Filter, offset, limit int, readSource func(sessionsclose(fl.done)}()// a rolling window's start moves with now, so its cells are never the- // same twice; the heatmap is added up on its own+ // same twice; the heatmap's sums are kept apart (heatParts)var cache *partCachepk := partKey{p: p, f: f, since: p.Since(now), until: p.Until(now), loc: now.Location()}- if p.Last() == 0 && p != heatmapPeriod {+ if p == heatmapPeriod {+ cache = &partCache{old: idx.heatParts[pk], next: map[uint64]*chunkPart{}}+ } else if p.Last() == 0 {cache = &partCache{old: idx.parts[pk], next: map[uint64]*chunkPart{}, pairs: idx.pairs[p]}}shared := idx@@ -794,7 +799,12 @@ func queryPageOf(p Period, f Filter, offset, limit int, readSource func(sessionsreturn page}shared.noteBuilt(b)- if cache != nil {+ if p == heatmapPeriod {+ if shared.heatParts == nil || len(shared.heatParts) >= partSets && shared.heatParts[pk] == nil {+ shared.heatParts = map[partKey]map[uint64]*chunkPart{}+ }+ shared.heatParts[pk] = cache.next+ } else if cache != nil {if shared.parts == nil || len(shared.parts) >= partSets && shared.parts[pk] == nil {shared.parts = map[partKey]map[uint64]*chunkPart{}}@@ -1061,6 +1071,7 @@ func (idx *requestIndex) adopt(st requestState) {idx.pages = nilidx.heats = nilidx.parts = nil+ idx.heatParts = nilidx.pairs = nilidx.key = ""}@@ -1593,19 +1604,6 @@ func pairRows(p Period, now time.Time, gateways, chunks, others []*rowChunk, idereturn &pairedRows{all: all, skip: skip, matched: matched, pairs: pairs, compactions: compactions, since: since, until: until}}-// visit calls fn with each row of the period that no other one stands for.-func (ps *pairedRows) visit(fn func(timedRef, *Row)) {- var rows chunkRows- defer rows.release()- // one row for the visit, written over for each: fn is not known- // here, so a row of each would be an allocation of its own, and fn- // keeps none of them- var r Row- for _, c := range ps.all {- ps.visitChunk(&rows, c, &r, fn)- }-}-// visitActors is visit of the rows of the chunks a Codex thread is known// for some row of (codexActors), the only rows Codex's effort adds up: it// decoded every session file's chunk, ~90 MB of what eight Sessions polls@@ -1651,7 +1649,7 @@ func buildRequestBlocksOf(p Period, f Filter, offset, limit int, now time.Time,func buildRequestBlocksIn(p Period, f Filter, offset, limit int, now time.Time, gateways, chunks, others []*rowChunk, names map[string]string, identities map[string]sessions.CodexIdentity, cache *partCache) RequestPage {since := p.Since(now)var ps *pairedRows- if cache != nil {+ if cache != nil && p != heatmapPeriod {ps, cache.pairsNext = pairRowsMemo(p, now, gateways, chunks, others, identities, cache.pairs)} else {ps = pairRows(p, now, gateways, chunks, others, identities)@@ -1662,13 +1660,7 @@ func buildRequestBlocksIn(p Period, f Filter, offset, limit int, now time.Time,// The heatmap's page is only its days, all HeatmapOf reads: over its// year the groups, shares and series below took seconds a call (#3493).if p == heatmapPeriod {- out.Heat = heatmapOf(since, now, func(add func(Row)) {- ps.visit(func(_ timedRef, r *Row) {- if chartFilter.keepsRow(r) {- add(*r)- }- })- })+ out.Heat = ps.heatOf(since, now, chartFilter, cache, identities)return out}agents, providers := map[string]bool{}, map[string]bool{}