Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
sessions: a poll's file lists, merges and copy groups are made once (#2278)
yetone68b31ce236db
diff --git a/internal/sessions/access.go b/internal/sessions/access.goindex 317e5bbcd..fdcbcd518 100644--- a/internal/sessions/access.go+++ b/internal/sessions/access.go@@ -4,6 +4,7 @@ import ("errors""os""path/filepath"+ "slices""sync""testing"@@ -149,8 +150,14 @@ func SetReads(all *bool, agents map[string]bool) error {// allowed is fs without the files of agents magpie may not read.func allowed(fs []file) []file {r := currentReads()- out := fs[:0:0]- for _, f := range fs {+ // most polls read every agent: fs as it is then, not a copy of each+ // file listed (#2278)+ k := slices.IndexFunc(fs, func(f file) bool { return !r.Reads(f.agent) })+ if k < 0 {+ return fs+ }+ out := slices.Clone(fs[:k])+ for _, f := range fs[k+1:] {if r.Reads(f.agent) {out = append(out, f)}diff --git a/internal/sessions/calls.go b/internal/sessions/calls.goindex a4fcb111c..aabd1f25a 100644--- a/internal/sessions/calls.go+++ b/internal/sessions/calls.go@@ -301,7 +301,12 @@ func callsFor(since time.Time, session, agent string) []Call {func callFiles() []file { return callFilesWith(codexFiles()) }// callFilesWith is callFiles with codex the rollouts codexFiles found.-func callFilesWith(codex []file) []file {+func callFilesWith(codex []file) []file { return distinctOf(callFileLists(codex)...) }++// callFileLists are callFilesWith's lists, each path not yet kept once:+// the callers that list more add theirs and keep each path once, over+// all of them (distinctOf), instead of copying callFilesWith's again.+func callFileLists(codex []file) [][]file {lists := [][]file{ccFiles("claude", ClaudeDir())}for _, d := range desktopReadDirs() {// Cowork keeps a Claude Code folder of its own for each session@@ -310,24 +315,7 @@ func callFilesWith(codex []file) []file {lists = append(lists, ccFiles("claude-desktop", h))}}- lists = append(lists, allowed(codex), wslFiles("claude", "codex"))- // sized once: a file is ~200 bytes, and growing a list of ~20k of them- // on every 15 s poll was ~36 MB a read- n := 0- for _, fs := range lists {- n += len(fs)- }- out := make([]file, 0, n)- seen := make(map[string]bool, n)- for _, fs := range lists {- for _, f := range fs {- if !seen[f.path] {- seen[f.path] = true- out = append(out, f)- }- }- }- return out+ return append(lists, allowed(codex), wslFiles("claude", "codex"))}// callSources are callFiles and the other agents' session files. Most are@@ -344,17 +332,11 @@ func callSources() []file { return callSourcesWith(codexFiles()) }// callSourcesWith is callSources with codex the rollouts codexFiles found.func callSourcesWith(codex []file) []file {- out := callFilesWith(codex)- out = append(out, openCodeCallFiles()...)- out = append(out, zcodeCallFiles()...)- out = append(out, dshCallFiles()...)- out = append(out, minimaxCallFiles()...)- out = append(out, kimiCallFiles()...)- out = append(out, wslFiles("kimi")...)- out = append(out, piCallFiles()...)- out = append(out, codebuddyCallFiles()...)- out = append(out, agyCallFiles()...) // #2097- return allowed(distinctPaths(append(out, workbuddyCallFiles()...)))+ lists := append(callFileLists(codex), openCodeCallFiles(), zcodeCallFiles(), dshCallFiles(),+ minimaxCallFiles(), kimiCallFiles(), wslFiles("kimi"), piCallFiles(), codebuddyCallFiles(),+ agyCallFiles(), // #2097+ workbuddyCallFiles())+ return allowed(distinctOf(lists...))}// desktopDataDirs are Claude Desktop's folders on this computer thatdiff --git a/internal/sessions/cc_copies.go b/internal/sessions/cc_copies.goindex 06e23c606..ad7cc4b10 100644--- a/internal/sessions/cc_copies.go+++ b/internal/sessions/cc_copies.go@@ -53,6 +53,10 @@ func ccFamily(files []file) (copies, branches map[string]string) {// files start alike when they start with the same message, or one// starts with a compaction's summary the other holds: a copy made// after /compact holds only what follows the summary+ // a file is held by its place in files, not copied: ~200 bytes for+ // each Claude Code file on each poll, and a group made for each of+ // them though only one of two or more is judged, were ~140 MB in 80s+ // of polls (#2278)var all []ccOneup := []int{} // each file's group, by its index in all (union-find)find := func(i int) int {@@ -63,7 +67,8 @@ func ccFamily(files []file) (copies, branches map[string]string) {return i}at := map[string]int{} // a message a file starts with or holds a summary as- for _, f := range files {+ for k := range files {+ f := &files[k]if !f.main || !parserFor(f.agent).claude {continue}@@ -83,10 +88,18 @@ func ccFamily(files []file) (copies, branches map[string]string) {}}}+ size := make([]int, len(all))+ for i := range all {+ size[find(i)]+++ }groups := map[int][]ccOne{}for i, o := range all {- r := find(i)- groups[r] = append(groups[r], o)+ if r := find(i); size[r] > 1 {+ if groups[r] == nil {+ groups[r] = make([]ccOne, 0, size[r])+ }+ groups[r] = append(groups[r], o)+ }}var out map[string]stringused := map[string]bool{}@@ -96,7 +109,7 @@ func ccFamily(files []file) (copies, branches map[string]string) {}holds := func(b ccOne, uuid string) bool {used[b.f.path] = true- return b.st.Leaf == uuid || ccUUIDsOf(b.f)[uuid]+ return b.st.Leaf == uuid || ccUUIDsOf(*b.f)[uuid]}// before reports whether b is kept over a: the one that holds// the other's first message too (a copy made after a compaction@@ -166,11 +179,9 @@ func ccFamily(files []file) (copies, branches map[string]string) {}// a file judged here that no longer starts like another lets its// uuids go; another caller's files are left as they are- for _, g := range groups {- for _, o := range g {- if !used[o.f.path] {- delete(ccUUIDMemo, o.f.path)- }+ for _, o := range all {+ if !used[o.f.path] {+ delete(ccUUIDMemo, o.f.path)}}return out, branches@@ -178,7 +189,7 @@ func ccFamily(files []file) (copies, branches map[string]string) {// ccOne is a Claude-format main file and its parse, as ccFamily judges it.type ccOne struct {- f file+ f *file // in the files ccFamily was givenst *state}@@ -189,7 +200,7 @@ func headFirst(a, b ccOne, holds func(ccOne, string) bool) bool {return ab}ida, idb := strings.TrimPrefix(a.f.key, a.f.agent+":"), strings.TrimPrefix(b.f.key, b.f.agent+":")- if fa, fb := ccForkedFrom(a.f), ccForkedFrom(b.f); (fb == ida) != (fa == idb) {+ if fa, fb := ccForkedFrom(*a.f), ccForkedFrom(*b.f); (fb == ida) != (fa == idb) {return fb == ida}if !a.st.Start.Equal(b.st.Start) && !a.st.Start.IsZero() && !b.st.Start.IsZero() {diff --git a/internal/sessions/codex.go b/internal/sessions/codex.goindex 1d89c7d51..03a509921 100644--- a/internal/sessions/codex.go+++ b/internal/sessions/codex.go@@ -394,12 +394,17 @@ func codexSubagentTaskTitles(files []file) map[codexTaskTitle]string {if f.agent != "codex" || cache[f.path] == nil {continue}+ st := cache[f.path]+ // most threads spawn no agent: four maps for each of them on every+ // poll were ~140 MB in 80s of polls over ~34k sessions (#2278)+ if len(st.CodexSpawnConflicts) == 0 && len(st.CodexAgentConflicts) == 0 && len(st.CodexSpawnNames) == 0 && len(st.CodexAgentIDs) == 0 {+ continue+ }calls := parents[f.key]if calls == nil {calls = &parentCalls{names: map[string]string{}, children: map[string]string{}, badNames: map[string]bool{}, badChildren: map[string]bool{}}parents[f.key] = calls}- st := cache[f.path]for id := range st.CodexSpawnConflicts {calls.badNames[id] = true}diff --git a/internal/sessions/external.go b/internal/sessions/external.goindex f5d0ad7ac..04650f846 100644--- a/internal/sessions/external.go+++ b/internal/sessions/external.go@@ -34,18 +34,35 @@ type ExternalSession struct {// reader; magpie's own record of one is deleted by DeleteGateway.func MergeExternal(native []Session, external []ExternalSession) []Session {names := desktopExternalTitles(external)- seen := map[string]bool{}+ // keyed by the pair, not "agent|id": a string made for each of ~5000+ // sessions on each of a poll's listings, and the set and the list grown+ // an append at a time, were ~200 MB in 80s of polls (#2278)+ type agentID struct{ agent, id string }+ seen := make(map[agentID]struct{}, len(native))for _, s := range native {- for _, k := range GatewayKeys(s) {- seen[k] = true+ // GatewayKeys, as pairs+ seen[agentID{s.Agent, s.ID}] = struct{}{}+ if s.Agent == "raft" {+ seen[agentID{"pi", s.ID}] = struct{}{}}for _, id := range s.Copies {- seen[s.Agent+"|"+id] = true+ seen[agentID{s.Agent, id}] = struct{}{}}}- out := append([]Session(nil), native...)+ added := func(e ExternalSession) bool {+ _, dup := seen[agentID{e.Agent, e.ID}]+ return e.Agent != "" && e.ID != "" && !dup+ }+ n := len(native)+ for _, e := range external {+ if added(e) {+ n+++ }+ }+ out := make([]Session, len(native), n)+ copy(out, native)for _, e := range external {- if e.Agent == "" || e.ID == "" || seen[e.Agent+"|"+e.ID] {+ if !added(e) {continue}s := Session{ReadOnly: true, Gateway: true, Transcript: true, Agent: e.Agent, ID: e.ID, Title: cmp.Or(names[e.Agent+"|"+e.ID], e.Title, e.ID), Start: e.Start, Last: e.Last, Models: []Model{}, Unpriced: e.Unpriced}diff --git a/internal/sessions/raft.go b/internal/sessions/raft.goindex 59b191cf2..5748a715b 100644--- a/internal/sessions/raft.go+++ b/internal/sessions/raft.go@@ -48,7 +48,7 @@ func raftFilesIn(dir string) []file {// GatewayKeys are the "agent|id" keys a session's gateway record can// carry: its own, and for Raft's, Pi's, as Raft's Pi tells the gateway-// it is Pi.+// it is Pi. MergeExternal keeps the same keys as pairs.func GatewayKeys(s Session) []string {k := []string{s.Agent + "|" + s.ID}if s.Agent == "raft" {diff --git a/internal/sessions/sessions.go b/internal/sessions/sessions.goindex 28cd7485d..906527336 100644--- a/internal/sessions/sessions.go+++ b/internal/sessions/sessions.go@@ -627,15 +627,35 @@ func allFiles() []file {if filesHook != nil {filesHook()}- var out []file- for _, fs := range [][]file{callFiles(), openCodeFiles(), piFiles(), raftFiles(),- wslFiles("pi", "raft", "omp", "kimi"), zcodeFiles(), dshFiles(), clineFiles(), ccFiles("qoder", QoderDir("qoder")), ccFiles("qoder-cn", QoderDir("qoder-cn")),- grokFiles(), workbuddyFiles(), codebuddyFiles(), droidFiles(), ompFiles(), cursorFiles(), hermesFiles(), almaFiles(), kimiFiles(), agyFiles()} {- out = append(out, fs...)- }// each lister asked Reads already; a file listed under another// agent's name is held to that agent too- return allowed(distinctPaths(out))+ return allowed(distinctOf(append(callFileLists(codexFiles()), openCodeFiles(), piFiles(), raftFiles(),+ wslFiles("pi", "raft", "omp", "kimi"), zcodeFiles(), dshFiles(), clineFiles(), ccFiles("qoder", QoderDir("qoder")), ccFiles("qoder-cn", QoderDir("qoder-cn")),+ grokFiles(), workbuddyFiles(), codebuddyFiles(), droidFiles(), ompFiles(), cursorFiles(), hermesFiles(), almaFiles(), kimiFiles(), agyFiles())...))+}++// distinctOf is lists one after another with each path kept once, the+// first time it is listed (distinctPaths), in one slice and one set made+// at their size: a file is ~200 bytes, and the call files were copied and+// a set of them made once, then copied and a set made again with the+// other agents' (allFiles, callSources), ~10 MB on each of a poll's+// several passes over ~34k sessions (#2278).+func distinctOf(lists ...[]file) []file {+ n := 0+ for _, fs := range lists {+ n += len(fs)+ }+ out := make([]file, 0, n)+ seen := make(map[string]struct{}, n)+ for _, fs := range lists {+ for _, f := range fs {+ if _, ok := seen[f.path]; !ok {+ seen[f.path] = struct{}{}+ out = append(out, f)+ }+ }+ }+ return out}// distinctPaths is fs with each path kept once, the first time it is@@ -1435,27 +1455,21 @@ func listSessions(limit int) []Session {defer useDBs()()files := allFiles()groups := byKey(files, nil)- // each session's latest file, beside its key: a map of them was- // ~72 MB of eight Sessions polls over 34k sessions (#2278)- type keyed struct {- key string- latest time.Time- }- order := make([]keyed, 0, len(groups))- for k, fs := range groups {- o := keyed{key: k}- for _, f := range fs {- if f.mod.After(o.latest) {- o.latest = f.mod- }+ latest := map[string]time.Time{}+ for _, f := range files {+ if f.mod.After(latest[f.key]) {+ latest[f.key] = f.mod}- order = append(order, o)}- slices.SortFunc(order, func(a, b keyed) int {- if c := b.latest.Compare(a.latest); c != 0 {- return c+ keys := make([]string, 0, len(groups))+ for k := range groups {+ keys = append(keys, k)+ }+ sort.Slice(keys, func(i, j int) bool {+ if !latest[keys[i]].Equal(latest[keys[j]]) {+ return latest[keys[i]].After(latest[keys[j]])}- return strings.Compare(a.key, b.key)+ return keys[i] < keys[j]})// the keys are not truncated yet: assemble below drops an empty// session, and a nonempty one must take its place rather than the@@ -1471,11 +1485,8 @@ func listSessions(limit int) []Session {copyIDs := copiesByKeeper(copies, groups)price, titles := pricer(), codexTitleOf()- // made to size: grown by append, a list of every session was copied- // through each size it grew past, ~130 MB of eight polls (#2278)- out := make([]Session, 0, min(limit, len(order)))- for _, o := range order {- k := o.key+ out := []Session{}+ for _, k := range keys {if copies[k] != "" {continue}