Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
sessions: a dsh session read for calls is read on, not whole
yetoneec396be60c8f
diff --git a/internal/sessions/call_cache.go b/internal/sessions/call_cache.goindex 4127f1c86..04e9f759c 100644--- a/internal/sessions/call_cache.go+++ b/internal/sessions/call_cache.go@@ -8,6 +8,7 @@ import ("bytes""encoding/binary""encoding/gob"+ "encoding/json""hash/crc32""hash/fnv""io"@@ -176,12 +177,11 @@ func keepCalls(path string, st *callFile) {// wholeCallReaders read an agent's calls from one session file at a time,// rather than a line at a time: its calls are not lines, or its file cannot// be read on from the middle of one. Every agent in callSources that is not-// in callFiles needs an entry here or in lineCallReaders, or its calls are-// never read.+// in callFiles needs an entry here, in lineCallReaders or in+// resumeCallReaders, or its calls are never read.var wholeCallReaders = map[string]func(file) (*callFile, bool){"opencode": readOpenCodeCalls,"zcode": readZCodeCalls,- "dsh": readDshCalls,"workbuddy": readWorkBuddyCalls,"codebuddy": readWorkBuddyCalls, // #1779// #1605@@ -189,6 +189,17 @@ var wholeCallReaders = map[string]func(file) (*callFile, bool){"agy": readAgyCalls, // #2097}+// resumeCallReaders read an agent's calls from a file only appended to that+// the line reader can't read on through, as dsh's packed file of zstd+// frames: each is handed what the last read came to and reads on from+// where it left off, or whole when it can't. It gives how many of the+// calls it hands back were in old, kept as they were (0 when it read+// whole, or what it had counted no longer stands), so only the rest are+// appended to the disk shard.+var resumeCallReaders = map[string]func(f file, old *callFile) (st *callFile, kept int, ok bool){+ "dsh": readDshCalls,+}+// lineCallReader reads an agent's calls a line at a time, as Claude Code's// and Codex's are: a file that only grew is read on from where the last read// left off (prepareCalls), and only the calls it adds are written to the@@ -245,6 +256,22 @@ func readCalls(f file) *callFile {// that changed is read again, and one that did not is kept. OpenCode's// and ZCode's are rows of a database or JSON documents; dsh's file is// packed in frames, so it cannot be read on from the middle of one.+ if read, ok := resumeCallReaders[f.agent]; ok {+ st, kept, readOK := read(f, old)+ if !readOK && old != nil {+ return old+ }+ if readOK {+ st.Size, st.Mod = f.size, f.mod.UnixNano()+ writeCalls(st, kept, kept > 0)+ }+ callsMu.Lock()+ if readOK && generation == callGeneration {+ keepCalls(f.path, st)+ }+ callsMu.Unlock()+ return st+ }if read, ok := wholeCallReaders[f.agent]; ok {st, readOK := read(f)if !readOK && old != nil {@@ -332,6 +359,8 @@ type callDiskState struct {At, End, AsstEnd int64Session, Requested stringLastUser, LastAsst time.Time+ // dsh's carry (dshCarry), as JSON: nil when there is none+ Dsh []byte// Pi's and omp's carry (piCarry), flat: PiOn when there is onePiOn, PiHeader boolPiCwd string@@ -343,6 +372,9 @@ func diskCallState(s *callFile) callDiskState {d := callDiskState{ContentHash: s.ContentHash, Size: s.Size, Mod: s.Mod, Off: s.Off, Head: s.Head, HeadSize: s.HeadSize,Path: s.Path, Agent: s.Agent, At: s.At, End: s.End, AsstEnd: s.AsstEnd, Session: s.Session, Requested: s.Requested,LastUser: s.LastUser, LastAsst: s.LastAsst}+ if s.Dsh != nil {+ d.Dsh, _ = json.Marshal(s.Dsh)+ }if p := s.Pi; p != nil {d.PiOn, d.PiHeader, d.PiCwd, d.PiSince, d.PiProvider, d.PiModel = true, p.Header, p.Cwd, p.Since, p.Provider, p.Model}@@ -352,6 +384,12 @@ func (s callDiskState) restore() *callFile {st := &callFile{ContentHash: s.ContentHash, Size: s.Size, Mod: s.Mod, Off: s.Off,Head: s.Head, HeadSize: s.HeadSize, Path: s.Path, Agent: s.Agent, At: s.At, End: s.End,AsstEnd: s.AsstEnd, Session: s.Session, Requested: s.Requested, LastUser: s.LastUser, LastAsst: s.LastAsst}+ if len(s.Dsh) > 0 {+ var c dshCarry+ if json.Unmarshal(s.Dsh, &c) == nil {+ st.Dsh = &c+ }+ }if s.PiOn {st.Pi = &piCarry{s.PiHeader, s.PiCwd, s.PiSince, s.PiProvider, s.PiModel}}diff --git a/internal/sessions/calls.go b/internal/sessions/calls.goindex 87ecfbdaa..ab2863655 100644--- a/internal/sessions/calls.go+++ b/internal/sessions/calls.go@@ -96,6 +96,8 @@ type callFile struct {CX *cxRun// Pi and ompPi *piCarry+ // dsh+ Dsh *dshCarry}// cxRun is where a rollout's read stands: what its lines have said so far.@@ -310,14 +312,14 @@ func callFiles() []file {// callSources are callFiles and the other agents' session files. Most are// read whole rather than line by line: OpenCode's (#680) and ZCode's, whose-// calls are rows of their database or their JSON files; DeepSeek Harness's,-// whose session file is packed in frames and so cannot be read on from the-// middle of one; and WorkBuddy's, whose usage lines repeat a reply's id as it+// calls are rows of their database or their JSON files; and WorkBuddy's, whose usage lines repeat a reply's id as it// goes on, as CodeBuddy Code's do (#1779); and MiniMax Code's, whose messages// file a later generation of its history may replace (#1605). Kimi Code's// (#1607), Pi's and omp's (#1951) are lines of a file only appended to,-// read on as callFiles' are.-// Every agent here needs an entry in wholeCallReaders or lineCallReaders.+// read on as callFiles' are; DeepSeek Harness's, a file only appended to+// that may be packed in zstd frames, is read on from the frame it was+// left in (readDshCalls). Every agent here needs an entry in+// wholeCallReaders, lineCallReaders or resumeCallReaders.func callSources() []file {out := callFiles()out = append(out, openCodeCallFiles()...)@@ -404,6 +406,14 @@ func (st *callFile) clone() *callFile {p := *st.Pic.Pi = &p}+ if st.Dsh != nil {+ d := *st.Dsh+ if d.Tail != nil {+ t := *d.Tail+ d.Tail = &t+ }+ c.Dsh = &d+ }return &c}diff --git a/internal/sessions/calls_dsh.go b/internal/sessions/calls_dsh.goindex cc8f72563..e93cf2997 100644--- a/internal/sessions/calls_dsh.go+++ b/internal/sessions/calls_dsh.go@@ -1,11 +1,8 @@package sessionsimport (- "bufio""bytes""encoding/json"- "errors"- "io""strings")@@ -37,82 +34,89 @@ func dshCallFiles() []file {return dshFiles()}-// readDshCalls reads a dsh session whole: packed in frames, it can't be read-// on from the middle of one (as parseDsh says).-func readDshCalls(f file) (*callFile, bool) {- r, err := dshOpen(f.path)- if err != nil {- return &callFile{Path: f.path, Agent: "dsh"}, false- }- defer r.Close()- st := &callFile{Agent: "dsh", Path: f.path, Session: strings.TrimPrefix(f.key, "dsh:")}- br := bufio.NewReaderSize(r, 1<<20)- var h dshHead- // the events a fork was seeded with are the session it came from, not- // this one: parseDsh skips them, and counting them here would spend the- // same request once per fork (dsh-a's first reply is also dsh-c's and- // dsh-d's first lines)- var evs []dshEvent- var seeds []bool- cut := -1- for first := true; ; first = false {- b, err := br.ReadBytes('\n')- if err != nil {- // a line still being written, or a frame cut short- if !errors.Is(err, io.EOF) || len(bytes.TrimSpace(b)) == 0 {- break- }- }+// readDshCalls reads a dsh session's calls. dsh only appends to a session+// file, a line at a time or, packed, a frame of lines at a time, so a file+// read before is read on from where that read left off, a packed one from+// the frame it left off in (frames.go), as parseDsh reads it. A file cut+// short or replaced is read whole again, and a line or frame not yet+// written whole waits for the next read. kept is how many of st's calls+// were old's.+func readDshCalls(f file, old *callFile) (st *callFile, kept int, ok bool) {+ var c dshCarry+ var off, skip int64+ if old != nil && old.Dsh != nil && old.Dsh.Tail.holds(f.path) {+ st = old.clone()+ c, off, skip = *st.Dsh, st.Dsh.Tail.Off, st.Dsh.Tail.Skip+ kept = len(st.Calls)+ } else {+ st = &callFile{Agent: "dsh", Path: f.path, Session: strings.TrimPrefix(f.key, "dsh:")}+ }+ headed := c.Head.Type == "session"+ ok = true+ each := func(b []byte) bool {b = bytes.TrimSpace(b)if len(b) == 0 || len(b) > maxLine {- if err != nil {- break- }- continue+ return true}- if first {- if json.Unmarshal(b, &h) != nil || h.Type != "session" {- return st, false+ if !headed {+ headed = true+ if json.Unmarshal(b, &c.Head) != nil || c.Head.Type != "session" {+ ok = false+ return false}if st.Session == "" {- st.Session = h.ID+ st.Session = c.Head.ID}- continue+ return true}// only an assistant/message carries a call, and only the ones a// seed ends: every other line is passed over undecoded- if bytes.Contains(b, []byte(`"assistant/message"`)) || bytes.Contains(b, []byte(`"session/end-seed"`)) {- var e dshEvent- if json.Unmarshal(b, &e) == nil && e.Type != "" {- evs = append(evs, e)- seeds = append(seeds, e.Type == "session/end-seed" && e.Data.Inherited)- }+ if !bytes.Contains(b, []byte(`"assistant/message"`)) && !bytes.Contains(b, []byte(`"session/end-seed"`)) {+ return true}- if err != nil {- break+ var e dshEvent+ if json.Unmarshal(b, &e) != nil {+ return true}- }- if h.Version >= 2 && h.IsSeeded {- for i, seed := range seeds {- if seed {- cut = i+ switch e.Type {+ case "session/end-seed":+ // the events a fork was seeded with are the session it came+ // from, not this one: parseDsh skips them, and counting them+ // here would spend the same request once per fork (dsh-a's+ // first reply is also dsh-c's and dsh-d's first lines). From+ // format 2 the seed ends with this marker: what came before+ // it is not this session's.+ if e.Data.Inherited && c.Head.Version >= 2 && c.Head.IsSeeded {+ st.Calls, kept = nil, 0+ }+ case "assistant/message":+ // before format 2 the seed is counted in events+ if c.Head.Version < 2 && e.Seq != nil && *e.Seq < c.Head.SeedLength {+ return true+ }+ if call, ok := dshCall(st, e); ok {+ st.Calls = append(st.Calls, call)}}+ return true}- for i, e := range evs {- if e.Type != "assistant/message" {- continue- }- // before format 2 the seed is counted in events, after it by the- // marker it ends with- if i <= cut || h.Version < 2 && e.Seq != nil && *e.Seq < h.SeedLength {- continue- }- if c, ok := dshCall(st, e); ok {- st.Calls = append(st.Calls, c)- }+ var next, nextSkip, end int64+ var err error+ if packed(f.path) {+ next, nextSkip, end, err = scanFrames(f.path, off, skip, each)+ } else {+ next, err = scanAt(f.path, off, nil, func(b []byte, _, _ int64) bool { return each(b) })+ end = next+ }+ if !ok || err != nil && !headed {+ return &callFile{Path: f.path, Agent: "dsh"}, 0, false+ }+ c.Tail = nil+ if err == nil {+ c.Tail = markTail(f.path, next, nextSkip, end)}- return st, true+ st.Dsh = &c+ return st, kept, true}// dshCall is an assistant/message as a call: its model, its tokens and thediff --git a/internal/sessions/calls_dsh_test.go b/internal/sessions/calls_dsh_test.goindex ca5a26773..36f16cf83 100644--- a/internal/sessions/calls_dsh_test.go+++ b/internal/sessions/calls_dsh_test.go@@ -1,8 +1,16 @@package sessionsimport (+ "bytes"+ "fmt"+ "os"+ "path/filepath"+ "slices"+ "sync""testing""time"++ "github.com/klauspost/compress/zstd")// dshCalls are the DeepSeek Harness calls Calls gives.@@ -129,3 +137,157 @@ func TestDshGatewayProviderTakesEveryRouteOfMagpies(t *testing.T) {}}}++// dshCallReader reads the calls of the dsh session file at path as the+// ledger does, and says where each scan of it started.+func dshCallReader(t *testing.T, path string) func() ([]int, []int64) {+ var mu sync.Mutex+ var starts []int64+ scanHook = func(p string, off int64) {+ if p == path {+ mu.Lock()+ starts = append(starts, off)+ mu.Unlock()+ }+ }+ t.Cleanup(func() { scanHook = nil })+ return func() ([]int, []int64) {+ t.Helper()+ mu.Lock()+ starts = nil+ mu.Unlock()+ var in []int+ found := false+ for _, s := range CallSources() {+ if s.Path == path {+ found = true+ for _, c := range ReadCallSource(s) {+ in = append(in, c.Input)+ }+ }+ }+ if !found {+ t.Fatalf("%s is no call source", path)+ }+ slices.Sort(in)+ mu.Lock()+ defer mu.Unlock()+ return in, append([]int64(nil), starts...)+ }+}++// A dsh session read for calls before is read on from where it was left,+// packed or not: a frame appended is read alone, a frame or line not yet+// written whole waits, the read goes on from the disk shard after the+// in-memory one is let go, and a file cut short is read whole again.+func TestDshCallsReadOnlyWhatWasAppended(t *testing.T) {+ inZone(t, 0)+ home := setupAgents(t)+ plain := filepath.Join(home, ".dsh", "sessions", "work-dsh-3f2a", "dsh-a", "session.v3.jsonl")+ b, err := os.ReadFile(plain)+ if err != nil {+ t.Fatal(err)+ }+ os.Remove(plain)+ enc, _ := zstd.NewWriter(nil)+ defer enc.Close()+ frame := func(s string) string { return string(enc.EncodeAll([]byte(s), nil)) }+ p := plain + ".zstd"+ half := bytes.IndexByte(b[len(b)/2:], '\n') + len(b)/2 + 1+ orig := frame(string(b[:half])) + frame(string(b[half:]))+ if err := os.WriteFile(p, []byte(orig), 0o644); err != nil {+ t.Fatal(err)+ }+ reply := func(n int) string {+ return fmt.Sprintf(`{"type":"assistant/message","seq":%d,"time":%d,"data":{"message":{"source":{"model":"deepseek-v4-flash"}},"usage":{"inputTokens":%d,"outputTokens":1,"cacheReadTokens":0}}}`+"\n", n, 1790503290000+int64(n)*1000, n)+ }+ size := func() int64 {+ t.Helper()+ fi, err := os.Stat(p)+ if err != nil {+ t.Fatal(err)+ }+ return fi.Size()+ }+ read := dshCallReader(t, p)+ check := func(what string, from int64, want ...int) {+ t.Helper()+ in, starts := read()+ if len(starts) == 0 {+ t.Fatalf("%s: not read", what)+ }+ for _, at := range starts {+ if at != from {+ t.Fatalf("%s: read from %d, want %d (%v)", what, at, from, starts)+ }+ }+ if !slices.Equal(in, want) {+ t.Fatalf("%s: calls %v, want %v", what, in, want)+ }+ }+ check("first read", 0, 10, 100)++ was := size()+ appendText(t, p, frame(reply(6)))+ check("a frame appended", was, 6, 10, 100)++ was = size()+ f := frame(reply(7))+ appendText(t, p, f[:len(f)/2])+ check("half a frame", was, 6, 10, 100)+ appendText(t, p, f[len(f)/2:])+ check("the half frame finished", was, 6, 7, 10, 100)++ // a frame ending inside a line: the line is read once whole, from+ // the frame it starts in+ was = size()+ r9 := reply(9)+ appendText(t, p, frame(reply(8)+r9[:20]))+ check("a frame ending in a line", was, 6, 7, 8, 10, 100)+ appendText(t, p, frame(r9[20:]))+ check("the line finished in the next frame", was, 6, 7, 8, 9, 10, 100)++ resetCalls()+ was = size()+ appendText(t, p, frame(reply(11)))+ check("from the disk shard", was, 6, 7, 8, 9, 10, 11, 100)++ if err := os.WriteFile(p, []byte(orig), 0o644); err != nil {+ t.Fatal(err)+ }+ check("cut short", 0, 10, 100)+}++// A fork's end-seed that comes in a later read still ends what it was+// seeded with: what was counted before it is its source's, in memory and+// in the disk shard.+func TestDshCallsEndSeedAcrossReads(t *testing.T) {+ inZone(t, 0)+ home := setupAgents(t)+ p := filepath.Join(home, ".dsh", "sessions", "work-dsh-3f2a", "dsh-c", "session.v3.jsonl")+ b, err := os.ReadFile(p)+ if err != nil {+ t.Fatal(err)+ }+ cut := bytes.Index(b, []byte(`{"type":"session/end-seed"`))+ rest := bytes.IndexByte(b[cut:], '\n') + cut + 1+ if err := os.WriteFile(p, b[:cut], 0o644); err != nil {+ t.Fatal(err)+ }+ read := dshCallReader(t, p)+ if in, _ := read(); !slices.Equal(in, []int{100}) {+ t.Fatalf("before its end-seed: %v", in)+ }+ appendText(t, p, string(b[cut:rest]))+ if in, starts := read(); len(in) != 0 || !slices.Equal(starts, []int64{int64(cut)}) {+ t.Fatalf("its end-seed read on from %v: %v, want none", starts, in)+ }+ appendText(t, p, string(b[rest:]))+ if in, starts := read(); !slices.Equal(in, []int{50}) || !slices.Equal(starts, []int64{int64(rest)}) {+ t.Fatalf("the fork's own, read on from %v: %v", starts, in)+ }+ resetCalls()+ if in, starts := read(); !slices.Equal(in, []int{50}) || len(starts) != 0 {+ t.Fatalf("from the disk shard (read from %v): %v", starts, in)+ }+}