Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

sessions: a poll's file lists, merges and copy groups are made once (#2278)

yetone68b31ce236db
diff --git a/internal/sessions/access.go b/internal/sessions/access.go
index 317e5bbcd..fdcbcd518 100644
--- a/internal/sessions/access.go
+++ b/internal/sessions/access.go
@@ -4,6 +4,7 @@ import (
"errors"
"os"
"path/filepath"
+ "slices"
"sync"
"testing"
@@ -149,8 +150,14 @@ func SetReads(all *bool, agents map[string]bool) error {
// allowed is fs without the files of agents magpie may not read.
func allowed(fs []file) []file {
r := currentReads()
- out := fs[:0:0]
- for _, f := range fs {
+ // most polls read every agent: fs as it is then, not a copy of each
+ // file listed (#2278)
+ k := slices.IndexFunc(fs, func(f file) bool { return !r.Reads(f.agent) })
+ if k < 0 {
+ return fs
+ }
+ out := slices.Clone(fs[:k])
+ for _, f := range fs[k+1:] {
if r.Reads(f.agent) {
out = append(out, f)
}
diff --git a/internal/sessions/calls.go b/internal/sessions/calls.go
index a4fcb111c..aabd1f25a 100644
--- a/internal/sessions/calls.go
+++ b/internal/sessions/calls.go
@@ -301,7 +301,12 @@ func callsFor(since time.Time, session, agent string) []Call {
func callFiles() []file { return callFilesWith(codexFiles()) }
// callFilesWith is callFiles with codex the rollouts codexFiles found.
-func callFilesWith(codex []file) []file {
+func callFilesWith(codex []file) []file { return distinctOf(callFileLists(codex)...) }
+
+// callFileLists are callFilesWith's lists, each path not yet kept once:
+// the callers that list more add theirs and keep each path once, over
+// all of them (distinctOf), instead of copying callFilesWith's again.
+func callFileLists(codex []file) [][]file {
lists := [][]file{ccFiles("claude", ClaudeDir())}
for _, d := range desktopReadDirs() {
// Cowork keeps a Claude Code folder of its own for each session
@@ -310,24 +315,7 @@ func callFilesWith(codex []file) []file {
lists = append(lists, ccFiles("claude-desktop", h))
}
}
- lists = append(lists, allowed(codex), wslFiles("claude", "codex"))
- // sized once: a file is ~200 bytes, and growing a list of ~20k of them
- // on every 15 s poll was ~36 MB a read
- n := 0
- for _, fs := range lists {
- n += len(fs)
- }
- out := make([]file, 0, n)
- seen := make(map[string]bool, n)
- for _, fs := range lists {
- for _, f := range fs {
- if !seen[f.path] {
- seen[f.path] = true
- out = append(out, f)
- }
- }
- }
- return out
+ return append(lists, allowed(codex), wslFiles("claude", "codex"))
}
// callSources are callFiles and the other agents' session files. Most are
@@ -344,17 +332,11 @@ func callSources() []file { return callSourcesWith(codexFiles()) }
// callSourcesWith is callSources with codex the rollouts codexFiles found.
func callSourcesWith(codex []file) []file {
- out := callFilesWith(codex)
- out = append(out, openCodeCallFiles()...)
- out = append(out, zcodeCallFiles()...)
- out = append(out, dshCallFiles()...)
- out = append(out, minimaxCallFiles()...)
- out = append(out, kimiCallFiles()...)
- out = append(out, wslFiles("kimi")...)
- out = append(out, piCallFiles()...)
- out = append(out, codebuddyCallFiles()...)
- out = append(out, agyCallFiles()...) // #2097
- return allowed(distinctPaths(append(out, workbuddyCallFiles()...)))
+ lists := append(callFileLists(codex), openCodeCallFiles(), zcodeCallFiles(), dshCallFiles(),
+ minimaxCallFiles(), kimiCallFiles(), wslFiles("kimi"), piCallFiles(), codebuddyCallFiles(),
+ agyCallFiles(), // #2097
+ workbuddyCallFiles())
+ return allowed(distinctOf(lists...))
}
// desktopDataDirs are Claude Desktop's folders on this computer that
diff --git a/internal/sessions/cc_copies.go b/internal/sessions/cc_copies.go
index 06e23c606..ad7cc4b10 100644
--- a/internal/sessions/cc_copies.go
+++ b/internal/sessions/cc_copies.go
@@ -53,6 +53,10 @@ func ccFamily(files []file) (copies, branches map[string]string) {
// files start alike when they start with the same message, or one
// starts with a compaction's summary the other holds: a copy made
// after /compact holds only what follows the summary
+ // a file is held by its place in files, not copied: ~200 bytes for
+ // each Claude Code file on each poll, and a group made for each of
+ // them though only one of two or more is judged, were ~140 MB in 80s
+ // of polls (#2278)
var all []ccOne
up := []int{} // each file's group, by its index in all (union-find)
find := func(i int) int {
@@ -63,7 +67,8 @@ func ccFamily(files []file) (copies, branches map[string]string) {
return i
}
at := map[string]int{} // a message a file starts with or holds a summary as
- for _, f := range files {
+ for k := range files {
+ f := &files[k]
if !f.main || !parserFor(f.agent).claude {
continue
}
@@ -83,10 +88,18 @@ func ccFamily(files []file) (copies, branches map[string]string) {
}
}
}
+ size := make([]int, len(all))
+ for i := range all {
+ size[find(i)]++
+ }
groups := map[int][]ccOne{}
for i, o := range all {
- r := find(i)
- groups[r] = append(groups[r], o)
+ if r := find(i); size[r] > 1 {
+ if groups[r] == nil {
+ groups[r] = make([]ccOne, 0, size[r])
+ }
+ groups[r] = append(groups[r], o)
+ }
}
var out map[string]string
used := map[string]bool{}
@@ -96,7 +109,7 @@ func ccFamily(files []file) (copies, branches map[string]string) {
}
holds := func(b ccOne, uuid string) bool {
used[b.f.path] = true
- return b.st.Leaf == uuid || ccUUIDsOf(b.f)[uuid]
+ return b.st.Leaf == uuid || ccUUIDsOf(*b.f)[uuid]
}
// before reports whether b is kept over a: the one that holds
// the other's first message too (a copy made after a compaction
@@ -166,11 +179,9 @@ func ccFamily(files []file) (copies, branches map[string]string) {
}
// a file judged here that no longer starts like another lets its
// uuids go; another caller's files are left as they are
- for _, g := range groups {
- for _, o := range g {
- if !used[o.f.path] {
- delete(ccUUIDMemo, o.f.path)
- }
+ for _, o := range all {
+ if !used[o.f.path] {
+ delete(ccUUIDMemo, o.f.path)
}
}
return out, branches
@@ -178,7 +189,7 @@ func ccFamily(files []file) (copies, branches map[string]string) {
// ccOne is a Claude-format main file and its parse, as ccFamily judges it.
type ccOne struct {
- f file
+ f *file // in the files ccFamily was given
st *state
}
@@ -189,7 +200,7 @@ func headFirst(a, b ccOne, holds func(ccOne, string) bool) bool {
return ab
}
ida, idb := strings.TrimPrefix(a.f.key, a.f.agent+":"), strings.TrimPrefix(b.f.key, b.f.agent+":")
- if fa, fb := ccForkedFrom(a.f), ccForkedFrom(b.f); (fb == ida) != (fa == idb) {
+ if fa, fb := ccForkedFrom(*a.f), ccForkedFrom(*b.f); (fb == ida) != (fa == idb) {
return fb == ida
}
if !a.st.Start.Equal(b.st.Start) && !a.st.Start.IsZero() && !b.st.Start.IsZero() {
diff --git a/internal/sessions/codex.go b/internal/sessions/codex.go
index 1d89c7d51..03a509921 100644
--- a/internal/sessions/codex.go
+++ b/internal/sessions/codex.go
@@ -394,12 +394,17 @@ func codexSubagentTaskTitles(files []file) map[codexTaskTitle]string {
if f.agent != "codex" || cache[f.path] == nil {
continue
}
+ st := cache[f.path]
+ // most threads spawn no agent: four maps for each of them on every
+ // poll were ~140 MB in 80s of polls over ~34k sessions (#2278)
+ if len(st.CodexSpawnConflicts) == 0 && len(st.CodexAgentConflicts) == 0 && len(st.CodexSpawnNames) == 0 && len(st.CodexAgentIDs) == 0 {
+ continue
+ }
calls := parents[f.key]
if calls == nil {
calls = &parentCalls{names: map[string]string{}, children: map[string]string{}, badNames: map[string]bool{}, badChildren: map[string]bool{}}
parents[f.key] = calls
}
- st := cache[f.path]
for id := range st.CodexSpawnConflicts {
calls.badNames[id] = true
}
diff --git a/internal/sessions/external.go b/internal/sessions/external.go
index f5d0ad7ac..04650f846 100644
--- a/internal/sessions/external.go
+++ b/internal/sessions/external.go
@@ -34,18 +34,35 @@ type ExternalSession struct {
// reader; magpie's own record of one is deleted by DeleteGateway.
func MergeExternal(native []Session, external []ExternalSession) []Session {
names := desktopExternalTitles(external)
- seen := map[string]bool{}
+ // keyed by the pair, not "agent|id": a string made for each of ~5000
+ // sessions on each of a poll's listings, and the set and the list grown
+ // an append at a time, were ~200 MB in 80s of polls (#2278)
+ type agentID struct{ agent, id string }
+ seen := make(map[agentID]struct{}, len(native))
for _, s := range native {
- for _, k := range GatewayKeys(s) {
- seen[k] = true
+ // GatewayKeys, as pairs
+ seen[agentID{s.Agent, s.ID}] = struct{}{}
+ if s.Agent == "raft" {
+ seen[agentID{"pi", s.ID}] = struct{}{}
}
for _, id := range s.Copies {
- seen[s.Agent+"|"+id] = true
+ seen[agentID{s.Agent, id}] = struct{}{}
}
}
- out := append([]Session(nil), native...)
+ added := func(e ExternalSession) bool {
+ _, dup := seen[agentID{e.Agent, e.ID}]
+ return e.Agent != "" && e.ID != "" && !dup
+ }
+ n := len(native)
+ for _, e := range external {
+ if added(e) {
+ n++
+ }
+ }
+ out := make([]Session, len(native), n)
+ copy(out, native)
for _, e := range external {
- if e.Agent == "" || e.ID == "" || seen[e.Agent+"|"+e.ID] {
+ if !added(e) {
continue
}
s := Session{ReadOnly: true, Gateway: true, Transcript: true, Agent: e.Agent, ID: e.ID, Title: cmp.Or(names[e.Agent+"|"+e.ID], e.Title, e.ID), Start: e.Start, Last: e.Last, Models: []Model{}, Unpriced: e.Unpriced}
diff --git a/internal/sessions/raft.go b/internal/sessions/raft.go
index 59b191cf2..5748a715b 100644
--- a/internal/sessions/raft.go
+++ b/internal/sessions/raft.go
@@ -48,7 +48,7 @@ func raftFilesIn(dir string) []file {
// GatewayKeys are the "agent|id" keys a session's gateway record can
// carry: its own, and for Raft's, Pi's, as Raft's Pi tells the gateway
-// it is Pi.
+// it is Pi. MergeExternal keeps the same keys as pairs.
func GatewayKeys(s Session) []string {
k := []string{s.Agent + "|" + s.ID}
if s.Agent == "raft" {
diff --git a/internal/sessions/sessions.go b/internal/sessions/sessions.go
index 28cd7485d..906527336 100644
--- a/internal/sessions/sessions.go
+++ b/internal/sessions/sessions.go
@@ -627,15 +627,35 @@ func allFiles() []file {
if filesHook != nil {
filesHook()
}
- var out []file
- for _, fs := range [][]file{callFiles(), openCodeFiles(), piFiles(), raftFiles(),
- wslFiles("pi", "raft", "omp", "kimi"), zcodeFiles(), dshFiles(), clineFiles(), ccFiles("qoder", QoderDir("qoder")), ccFiles("qoder-cn", QoderDir("qoder-cn")),
- grokFiles(), workbuddyFiles(), codebuddyFiles(), droidFiles(), ompFiles(), cursorFiles(), hermesFiles(), almaFiles(), kimiFiles(), agyFiles()} {
- out = append(out, fs...)
- }
// each lister asked Reads already; a file listed under another
// agent's name is held to that agent too
- return allowed(distinctPaths(out))
+ return allowed(distinctOf(append(callFileLists(codexFiles()), openCodeFiles(), piFiles(), raftFiles(),
+ wslFiles("pi", "raft", "omp", "kimi"), zcodeFiles(), dshFiles(), clineFiles(), ccFiles("qoder", QoderDir("qoder")), ccFiles("qoder-cn", QoderDir("qoder-cn")),
+ grokFiles(), workbuddyFiles(), codebuddyFiles(), droidFiles(), ompFiles(), cursorFiles(), hermesFiles(), almaFiles(), kimiFiles(), agyFiles())...))
+}
+
+// distinctOf is lists one after another with each path kept once, the
+// first time it is listed (distinctPaths), in one slice and one set made
+// at their size: a file is ~200 bytes, and the call files were copied and
+// a set of them made once, then copied and a set made again with the
+// other agents' (allFiles, callSources), ~10 MB on each of a poll's
+// several passes over ~34k sessions (#2278).
+func distinctOf(lists ...[]file) []file {
+ n := 0
+ for _, fs := range lists {
+ n += len(fs)
+ }
+ out := make([]file, 0, n)
+ seen := make(map[string]struct{}, n)
+ for _, fs := range lists {
+ for _, f := range fs {
+ if _, ok := seen[f.path]; !ok {
+ seen[f.path] = struct{}{}
+ out = append(out, f)
+ }
+ }
+ }
+ return out
}
// distinctPaths is fs with each path kept once, the first time it is
@@ -1435,27 +1455,21 @@ func listSessions(limit int) []Session {
defer useDBs()()
files := allFiles()
groups := byKey(files, nil)
- // each session's latest file, beside its key: a map of them was
- // ~72 MB of eight Sessions polls over 34k sessions (#2278)
- type keyed struct {
- key string
- latest time.Time
- }
- order := make([]keyed, 0, len(groups))
- for k, fs := range groups {
- o := keyed{key: k}
- for _, f := range fs {
- if f.mod.After(o.latest) {
- o.latest = f.mod
- }
+ latest := map[string]time.Time{}
+ for _, f := range files {
+ if f.mod.After(latest[f.key]) {
+ latest[f.key] = f.mod
}
- order = append(order, o)
}
- slices.SortFunc(order, func(a, b keyed) int {
- if c := b.latest.Compare(a.latest); c != 0 {
- return c
+ keys := make([]string, 0, len(groups))
+ for k := range groups {
+ keys = append(keys, k)
+ }
+ sort.Slice(keys, func(i, j int) bool {
+ if !latest[keys[i]].Equal(latest[keys[j]]) {
+ return latest[keys[i]].After(latest[keys[j]])
}
- return strings.Compare(a.key, b.key)
+ return keys[i] < keys[j]
})
// the keys are not truncated yet: assemble below drops an empty
// session, and a nonempty one must take its place rather than the
@@ -1471,11 +1485,8 @@ func listSessions(limit int) []Session {
copyIDs := copiesByKeeper(copies, groups)
price, titles := pricer(), codexTitleOf()
- // made to size: grown by append, a list of every session was copied
- // through each size it grew past, ~130 MB of eight polls (#2278)
- out := make([]Session, 0, min(limit, len(order)))
- for _, o := range order {
- k := o.key
+ out := []Session{}
+ for _, k := range keys {
if copies[k] != "" {
continue
}