Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
usage: a cold build's chunks are of the file's pages once it is written (#3493)
yetonecbbf3163cb3c
diff --git a/internal/usage/request_disk.go b/internal/usage/request_disk.goindex a73db352a..1d8fe0dc2 100644--- a/internal/usage/request_disk.go+++ b/internal/usage/request_disk.go@@ -2,6 +2,7 @@ package usageimport ("bufio"+ "bytes""crypto/sha256""embed""encoding/binary"@@ -16,6 +17,7 @@ import ("runtime/debug""strings""sync"+ "sync/atomic""time""github.com/yetone/magpie/internal/sessions"@@ -232,7 +234,7 @@ type diskMapping struct {}// diskUnmapped, in tests, is told each time a mapping is let go of.-var diskUnmapped func()+var diskUnmapped atomic.Pointer[func()]// readMapped runs fn, which reads m's pages (none, with m nil). A fault// reading them, the file cut short under the mapping by another program,@@ -405,6 +407,38 @@ func toDisk(root, meta string, chunks map[string]*rowChunk, on map[string]bool)return os.Rename(out.Name(), path)}+// onDisk is a copy of each chunk of chunks whose Archive is of the heap,+// with the same Archive of the pages of the file toDisk just wrote; save+// swaps them in for the chunks of the same serial. Made+// in this run, they held a cold build's archives on the heap until the+// next start: 32MB of 15.7k sessions, and a resting heap 40MB over a warm+// start's (#3493). Nothing on a system that maps no file.+func onDisk(root, meta string, chunks map[string]*rowChunk) map[string]*rowChunk {+ kept := map[string]*rowChunk{}+ readDiskEntries(requestDiskPath(), requestDiskBinary(), root, meta, func(raw []byte, m *diskMapping) bool {+ if m == nil {+ return false+ }+ path, ok := diskEntryPath(raw)+ if !ok {+ return false+ }+ c := chunks[path]+ if c == nil || c.serial == 0 || c.mapped != nil || len(c.Archive) == 0 {+ return true+ }+ e, err := readDiskEntry(raw)+ if err != nil || e.Source.Size != c.Source.Size || !e.Source.Modified.Equal(c.Source.Modified) || !bytes.Equal(e.Archive, c.Archive) {+ return true+ }+ next := *c+ next.Archive, next.mapped = e.Archive, m+ kept[path] = &next+ return true+ })+ return kept+}+// diskEntryPath is the path of the source an entry is of.func diskEntryPath(b []byte) (string, bool) {n, k := binary.Uvarint(b)diff --git a/internal/usage/request_disk_map_unix.go b/internal/usage/request_disk_map_unix.goindex 8aeff620a..80d2c78cb 100644--- a/internal/usage/request_disk_map_unix.go+++ b/internal/usage/request_disk_map_unix.go@@ -29,8 +29,8 @@ func mapDiskFile(path string) (*diskMapping, bool) {m := &diskMapping{data: data}runtime.AddCleanup(m, func(data []byte) {unix.Munmap(data)- if diskUnmapped != nil {- diskUnmapped()+ if f := diskUnmapped.Load(); f != nil {+ (*f)()}}, data)return m, truediff --git a/internal/usage/request_disk_map_unix_test.go b/internal/usage/request_disk_map_unix_test.goindex b83383a8e..b1d9f4173 100644--- a/internal/usage/request_disk_map_unix_test.go+++ b/internal/usage/request_disk_map_unix_test.go@@ -7,6 +7,7 @@ import ("os""path/filepath""runtime"+ "sync""sync/atomic""testing""time"@@ -24,8 +25,9 @@ func TestRequestDiskChunksMapped(t *testing.T) {requestDiskLeast = 2t.Cleanup(func() { requestDiskLeast = least })var unmapped atomic.Int32- diskUnmapped = func() { unmapped.Add(1) }- t.Cleanup(func() { diskUnmapped = nil })+ count := func() { unmapped.Add(1) }+ diskUnmapped.Store(&count)+ t.Cleanup(func() { diskUnmapped.Store(nil) })dir := filepath.Join(sessions.CodexDir(), "sessions", "2026", "09", "30")for i := range 4 {writeCodexCall(t, filepath.Join(dir, fmt.Sprintf("rollout-2026-09-30T10-00-0%d-s%d.jsonl", i, i)), fmt.Sprintf("s%d", i), fmt.Sprintf("native-%d", i), `"cli"`, fmt.Sprintf("resp-%d", i), "m", "high", now.Add(-time.Duration(i+1)*time.Hour), 10*(i+1), i)@@ -75,6 +77,68 @@ func TestRequestDiskChunksMapped(t *testing.T) {}}+// The chunks a cold build made are of the file's pages once it is written,+// as a warm start's are, not of the heap until the next start (#3493).+func TestRequestDiskBuiltChunksMapped(t *testing.T) {+ pageHome(t)+ now := holdClock(t, time.Date(2026, 9, 30, 12, 0, 0, 0, time.Local))+ least := requestDiskLeast+ requestDiskLeast = 2+ t.Cleanup(func() { requestDiskLeast = least })+ dir := filepath.Join(sessions.CodexDir(), "sessions", "2026", "09", "30")+ for i := range 4 {+ writeCodexCall(t, filepath.Join(dir, fmt.Sprintf("rollout-2026-09-30T10-00-0%d-s%d.jsonl", i, i)), fmt.Sprintf("s%d", i), fmt.Sprintf("native-%d", i), `"cli"`, fmt.Sprintf("resp-%d", i), "m", "high", now.Add(-time.Duration(i+1)*time.Hour), 10*(i+1), i)+ }+ // the write waits while another page copies the chunks with a use of+ // its own, and a third has copied them before the write and keeps+ // them after it+ release, hold, held := make(chan struct{}), make(chan struct{}), make(chan struct{})+ savingDisk = func() { <-release }+ t.Cleanup(func() { savingDisk = nil })+ first := queryPage(All, Filter{}, 0, 100, sessions.ReadCallSource)+ queryPage(All, Filter{}, 1, 100, sessions.ReadCallSource)+ var once sync.Once+ building = func() { once.Do(func() { close(held); <-hold }) }+ t.Cleanup(func() { building = nil })+ done := make(chan struct{})+ go func() {+ defer close(done)+ queryPage(All, Filter{}, 2, 100, sessions.ReadCallSource)+ }()+ <-held+ close(release)+ requestDiskSaves.Wait()+ close(hold)+ <-done++ requestCache.Lock()+ mapped, archived := 0, 0+ for _, c := range requestCache.chunks {+ if len(c.Archive) == 0 {+ continue+ }+ archived+++ if c.mapped == nil {+ t.Errorf("%s: still of the heap after the file was written", filepath.Base(c.Source.Path))+ continue+ }+ at, start := uintptr(unsafe.Pointer(unsafe.SliceData(c.Archive))), uintptr(unsafe.Pointer(unsafe.SliceData(c.mapped.data)))+ if at < start || at+uintptr(len(c.Archive)) > start+uintptr(len(c.mapped.data)) {+ t.Errorf("%s: archive outside the mapping", filepath.Base(c.Source.Path))+ }+ mapped+++ }+ requestCache.Unlock()+ if archived != 4 || mapped != 4 {+ t.Fatalf("%d of %d archived chunks of the file's pages, want 4 of 4", mapped, archived)+ }+ requestCache.Lock()+ requestCache.pages, requestCache.parts, requestCache.pairs, requestCache.heats = nil, nil, nil, nil+ requestCache.key = ""+ requestCache.Unlock()+ equalPage(t, queryPage(All, Filter{}, 0, 100, sessions.ReadCallSource), first)+}+// A file cut short under its mapping by another program is a read that// failed, not a crash.func TestRequestDiskMappingCutShort(t *testing.T) {diff --git a/internal/usage/request_page.go b/internal/usage/request_page.goindex 289fdd2fa..1f3a9e97e 100644--- a/internal/usage/request_page.go+++ b/internal/usage/request_page.go@@ -1028,9 +1028,18 @@ func (idx *requestIndex) keep(read *requestIndex, on map[string]bool) bool {return false}for path, c := range read.chunks {- if prev := idx.chunks[path]; prev == nil || prev.Used <= c.Used {- idx.chunks[path] = c+ prev := idx.chunks[path]+ if prev != nil && prev.Used > c.Used {+ continue+ }+ if prev != nil && prev.serial == c.serial && prev.mapped != nil && c.mapped == nil {+ // copied before the file of it was written (save): its archive+ // stays of the file's pages+ next := *c+ next.Archive, next.mapped = prev.Archive, prev.mapped+ c = &next}+ idx.chunks[path] = c}for path := range idx.chunks {if !on[path] {@@ -1068,6 +1077,9 @@ var requestDiskLeastRows = 20000// requestDiskSaves are the writes under way, for the tests to wait on.var requestDiskSaves sync.WaitGroup+// savingDisk, in tests, is called by a write before it writes the file.+var savingDisk func()+// save writes the file of idx's chunks behind the lock: chunks are never// changed once made, so a copy of the map is enough. idx is locked.func (idx *requestIndex) save() {@@ -1080,10 +1092,25 @@ func (idx *requestIndex) save() {requestDiskSaves.Add(1)go func() {defer requestDiskSaves.Done()- _ = toDisk(root, meta, chunks, on)+ if savingDisk != nil {+ savingDisk()+ }+ var kept map[string]*rowChunk+ if toDisk(root, meta, chunks, on) == nil {+ kept = onDisk(root, meta, chunks)+ }idx.Lock()defer idx.Unlock()idx.saving = false+ // a read since has copied the chunk with a use of its own: the same+ // chunk while its serial is+ for path, c := range kept {+ if cur := idx.chunks[path]; cur != nil && cur.serial == c.serial && cur.mapped == nil {+ next := *cur+ next.Archive, next.mapped = c.Archive, c.mapped+ idx.chunks[path] = &next+ }+ }}()}