Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

usage: a cold build's chunks are of the file's pages once it is written (#3493)

yetonecbbf3163cb3c
diff --git a/internal/usage/request_disk.go b/internal/usage/request_disk.go
index a73db352a..1d8fe0dc2 100644
--- a/internal/usage/request_disk.go
+++ b/internal/usage/request_disk.go
@@ -2,6 +2,7 @@ package usage
import (
"bufio"
+ "bytes"
"crypto/sha256"
"embed"
"encoding/binary"
@@ -16,6 +17,7 @@ import (
"runtime/debug"
"strings"
"sync"
+ "sync/atomic"
"time"
"github.com/yetone/magpie/internal/sessions"
@@ -232,7 +234,7 @@ type diskMapping struct {
}
// diskUnmapped, in tests, is told each time a mapping is let go of.
-var diskUnmapped func()
+var diskUnmapped atomic.Pointer[func()]
// readMapped runs fn, which reads m's pages (none, with m nil). A fault
// reading them, the file cut short under the mapping by another program,
@@ -405,6 +407,38 @@ func toDisk(root, meta string, chunks map[string]*rowChunk, on map[string]bool)
return os.Rename(out.Name(), path)
}
+// onDisk is a copy of each chunk of chunks whose Archive is of the heap,
+// with the same Archive of the pages of the file toDisk just wrote; save
+// swaps them in for the chunks of the same serial. Made
+// in this run, they held a cold build's archives on the heap until the
+// next start: 32MB of 15.7k sessions, and a resting heap 40MB over a warm
+// start's (#3493). Nothing on a system that maps no file.
+func onDisk(root, meta string, chunks map[string]*rowChunk) map[string]*rowChunk {
+ kept := map[string]*rowChunk{}
+ readDiskEntries(requestDiskPath(), requestDiskBinary(), root, meta, func(raw []byte, m *diskMapping) bool {
+ if m == nil {
+ return false
+ }
+ path, ok := diskEntryPath(raw)
+ if !ok {
+ return false
+ }
+ c := chunks[path]
+ if c == nil || c.serial == 0 || c.mapped != nil || len(c.Archive) == 0 {
+ return true
+ }
+ e, err := readDiskEntry(raw)
+ if err != nil || e.Source.Size != c.Source.Size || !e.Source.Modified.Equal(c.Source.Modified) || !bytes.Equal(e.Archive, c.Archive) {
+ return true
+ }
+ next := *c
+ next.Archive, next.mapped = e.Archive, m
+ kept[path] = &next
+ return true
+ })
+ return kept
+}
+
// diskEntryPath is the path of the source an entry is of.
func diskEntryPath(b []byte) (string, bool) {
n, k := binary.Uvarint(b)
diff --git a/internal/usage/request_disk_map_unix.go b/internal/usage/request_disk_map_unix.go
index 8aeff620a..80d2c78cb 100644
--- a/internal/usage/request_disk_map_unix.go
+++ b/internal/usage/request_disk_map_unix.go
@@ -29,8 +29,8 @@ func mapDiskFile(path string) (*diskMapping, bool) {
m := &diskMapping{data: data}
runtime.AddCleanup(m, func(data []byte) {
unix.Munmap(data)
- if diskUnmapped != nil {
- diskUnmapped()
+ if f := diskUnmapped.Load(); f != nil {
+ (*f)()
}
}, data)
return m, true
diff --git a/internal/usage/request_disk_map_unix_test.go b/internal/usage/request_disk_map_unix_test.go
index b83383a8e..b1d9f4173 100644
--- a/internal/usage/request_disk_map_unix_test.go
+++ b/internal/usage/request_disk_map_unix_test.go
@@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"runtime"
+ "sync"
"sync/atomic"
"testing"
"time"
@@ -24,8 +25,9 @@ func TestRequestDiskChunksMapped(t *testing.T) {
requestDiskLeast = 2
t.Cleanup(func() { requestDiskLeast = least })
var unmapped atomic.Int32
- diskUnmapped = func() { unmapped.Add(1) }
- t.Cleanup(func() { diskUnmapped = nil })
+ count := func() { unmapped.Add(1) }
+ diskUnmapped.Store(&count)
+ t.Cleanup(func() { diskUnmapped.Store(nil) })
dir := filepath.Join(sessions.CodexDir(), "sessions", "2026", "09", "30")
for i := range 4 {
writeCodexCall(t, filepath.Join(dir, fmt.Sprintf("rollout-2026-09-30T10-00-0%d-s%d.jsonl", i, i)), fmt.Sprintf("s%d", i), fmt.Sprintf("native-%d", i), `"cli"`, fmt.Sprintf("resp-%d", i), "m", "high", now.Add(-time.Duration(i+1)*time.Hour), 10*(i+1), i)
@@ -75,6 +77,68 @@ func TestRequestDiskChunksMapped(t *testing.T) {
}
}
+// The chunks a cold build made are of the file's pages once it is written,
+// as a warm start's are, not of the heap until the next start (#3493).
+func TestRequestDiskBuiltChunksMapped(t *testing.T) {
+ pageHome(t)
+ now := holdClock(t, time.Date(2026, 9, 30, 12, 0, 0, 0, time.Local))
+ least := requestDiskLeast
+ requestDiskLeast = 2
+ t.Cleanup(func() { requestDiskLeast = least })
+ dir := filepath.Join(sessions.CodexDir(), "sessions", "2026", "09", "30")
+ for i := range 4 {
+ writeCodexCall(t, filepath.Join(dir, fmt.Sprintf("rollout-2026-09-30T10-00-0%d-s%d.jsonl", i, i)), fmt.Sprintf("s%d", i), fmt.Sprintf("native-%d", i), `"cli"`, fmt.Sprintf("resp-%d", i), "m", "high", now.Add(-time.Duration(i+1)*time.Hour), 10*(i+1), i)
+ }
+ // the write waits while another page copies the chunks with a use of
+ // its own, and a third has copied them before the write and keeps
+ // them after it
+ release, hold, held := make(chan struct{}), make(chan struct{}), make(chan struct{})
+ savingDisk = func() { <-release }
+ t.Cleanup(func() { savingDisk = nil })
+ first := queryPage(All, Filter{}, 0, 100, sessions.ReadCallSource)
+ queryPage(All, Filter{}, 1, 100, sessions.ReadCallSource)
+ var once sync.Once
+ building = func() { once.Do(func() { close(held); <-hold }) }
+ t.Cleanup(func() { building = nil })
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ queryPage(All, Filter{}, 2, 100, sessions.ReadCallSource)
+ }()
+ <-held
+ close(release)
+ requestDiskSaves.Wait()
+ close(hold)
+ <-done
+
+ requestCache.Lock()
+ mapped, archived := 0, 0
+ for _, c := range requestCache.chunks {
+ if len(c.Archive) == 0 {
+ continue
+ }
+ archived++
+ if c.mapped == nil {
+ t.Errorf("%s: still of the heap after the file was written", filepath.Base(c.Source.Path))
+ continue
+ }
+ at, start := uintptr(unsafe.Pointer(unsafe.SliceData(c.Archive))), uintptr(unsafe.Pointer(unsafe.SliceData(c.mapped.data)))
+ if at < start || at+uintptr(len(c.Archive)) > start+uintptr(len(c.mapped.data)) {
+ t.Errorf("%s: archive outside the mapping", filepath.Base(c.Source.Path))
+ }
+ mapped++
+ }
+ requestCache.Unlock()
+ if archived != 4 || mapped != 4 {
+ t.Fatalf("%d of %d archived chunks of the file's pages, want 4 of 4", mapped, archived)
+ }
+ requestCache.Lock()
+ requestCache.pages, requestCache.parts, requestCache.pairs, requestCache.heats = nil, nil, nil, nil
+ requestCache.key = ""
+ requestCache.Unlock()
+ equalPage(t, queryPage(All, Filter{}, 0, 100, sessions.ReadCallSource), first)
+}
+
// A file cut short under its mapping by another program is a read that
// failed, not a crash.
func TestRequestDiskMappingCutShort(t *testing.T) {
diff --git a/internal/usage/request_page.go b/internal/usage/request_page.go
index 289fdd2fa..1f3a9e97e 100644
--- a/internal/usage/request_page.go
+++ b/internal/usage/request_page.go
@@ -1028,9 +1028,18 @@ func (idx *requestIndex) keep(read *requestIndex, on map[string]bool) bool {
return false
}
for path, c := range read.chunks {
- if prev := idx.chunks[path]; prev == nil || prev.Used <= c.Used {
- idx.chunks[path] = c
+ prev := idx.chunks[path]
+ if prev != nil && prev.Used > c.Used {
+ continue
+ }
+ if prev != nil && prev.serial == c.serial && prev.mapped != nil && c.mapped == nil {
+ // copied before the file of it was written (save): its archive
+ // stays of the file's pages
+ next := *c
+ next.Archive, next.mapped = prev.Archive, prev.mapped
+ c = &next
}
+ idx.chunks[path] = c
}
for path := range idx.chunks {
if !on[path] {
@@ -1068,6 +1077,9 @@ var requestDiskLeastRows = 20000
// requestDiskSaves are the writes under way, for the tests to wait on.
var requestDiskSaves sync.WaitGroup
+// savingDisk, in tests, is called by a write before it writes the file.
+var savingDisk func()
+
// save writes the file of idx's chunks behind the lock: chunks are never
// changed once made, so a copy of the map is enough. idx is locked.
func (idx *requestIndex) save() {
@@ -1080,10 +1092,25 @@ func (idx *requestIndex) save() {
requestDiskSaves.Add(1)
go func() {
defer requestDiskSaves.Done()
- _ = toDisk(root, meta, chunks, on)
+ if savingDisk != nil {
+ savingDisk()
+ }
+ var kept map[string]*rowChunk
+ if toDisk(root, meta, chunks, on) == nil {
+ kept = onDisk(root, meta, chunks)
+ }
idx.Lock()
defer idx.Unlock()
idx.saving = false
+ // a read since has copied the chunk with a use of its own: the same
+ // chunk while its serial is
+ for path, c := range kept {
+ if cur := idx.chunks[path]; cur != nil && cur.serial == c.serial && cur.mapped == nil {
+ next := *cur
+ next.Archive, next.mapped = c.Archive, c.mapped
+ idx.chunks[path] = &next
+ }
+ }
}()
}