Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
sessions: a dsh session read before is read on from where it was left, not whole
yetoneb3642e727700
diff --git a/internal/sessions/dsh.go b/internal/sessions/dsh.goindex 55c0865db..baf62c6c2 100644--- a/internal/sessions/dsh.go+++ b/internal/sessions/dsh.go@@ -4,7 +4,6 @@ import ("bufio""bytes""encoding/json"- "errors""fmt""io""os"@@ -242,86 +241,116 @@ type dshEvent struct {var dshWanted = [][]byte{[]byte(`"user/message"`), []byte(`"session/title"`), []byte(`"assistant/message"`), []byte(`"session/end-seed"`)}-// parseDsh reads a session file whole: compressed, it can't be read on from-// the middle of a frame.-func parseDsh(f file) *state {- s := &state{Size: f.size, Mod: f.mod.UnixNano()}- r, err := dshOpen(f.path)- if err != nil {- return s+// dshCarry is what a read of a dsh session file carries on with: where it+// read to and the header it read first.+type dshCarry struct {+ Tail *tail `json:"tail,omitempty"`+ Head dshHead `json:"head"`+}++// parseDsh reads a session file. dsh only appends to one, a line at a time+// or, packed, a frame of lines at a time, so a file read before is read on+// from where that read left off: a packed one from the frame it left off+// in. A file cut short or replaced is read whole again, and a line or frame+// not yet written whole waits for the next read.+func parseDsh(f file, old *state) *state {+ var s *state+ var c dshCarry+ var off, skip int64+ if old != nil && old.Dsh != nil && old.Dsh.Tail.holds(f.path) {+ s, c = old.clone(), *old.Dsh+ off, skip = c.Tail.Off, c.Tail.Skip+ } else {+ s = &state{}}- defer r.Close()- br := bufio.NewReaderSize(r, 1<<20)- var h dshHead- var evs []dshEvent- for first := true; ; first = false {- b, err := br.ReadBytes('\n')- if err != nil {- // a line still being written, or a frame cut short- if !errors.Is(err, io.EOF) || len(bytes.TrimSpace(b)) == 0 {- break- }+ s.Size, s.Mod = f.size, f.mod.UnixNano()+ headed := c.Tail != nil+ ok := true+ each := func(b []byte) bool {+ b = bytes.TrimSpace(b)+ if len(b) == 0 || len(b) > maxLine {+ return true}- if b = bytes.TrimSpace(b); len(b) > 0 && len(b) <= maxLine {- if first {- if json.Unmarshal(b, &h) != nil || h.Type != "session" {- return s- }- } else if dshWant(b) {- var e dshEvent- if json.Unmarshal(b, &e) == nil && e.Type != "" {- evs = append(evs, e)- }- } else if t := dshNum(b, "time"); t > 0 {- // only its time and place looked at- e := dshEvent{Time: t}- if n := dshNum(b, "seq"); n >= 0 {- e.Seq = &n- }- evs = append(evs, e)+ if !headed {+ headed = true+ if json.Unmarshal(b, &c.Head) != nil || c.Head.Type != "session" {+ ok = false+ return false}+ s.ID, s.Cwd = c.Head.ID, c.Head.Cwd+ s.saw(ms(c.Head.CreatedAt), f.main)+ return true}- if err != nil {- break- }+ dshCount(s, &c.Head, b, f.main)+ return true}- s.ID, s.Cwd = h.ID, h.Cwd- // the events a fork was seeded with- cut := -1- if h.Version >= 2 && h.IsSeeded {- for i, e := range evs {- if e.Type == "session/end-seed" && e.Data.Inherited {- cut = i- }+ var next, nextSkip, end int64+ var err error+ if packed(f.path) {+ next, nextSkip, end, err = scanFrames(f.path, off, skip, each)+ } else {+ next, err = scanAt(f.path, off, nil, func(b []byte, _, _ int64) bool { return each(b) })+ end = next+ }+ if !ok {+ // not a session's file: only its size and time+ return &state{Size: f.size, Mod: f.mod.UnixNano()}+ }+ c.Tail = nil+ if err == nil {+ c.Tail = markTail(f.path, next, nextSkip, end)+ }+ s.Dsh = &c+ return s+}++// dshCount counts one event of a session file after its header.+func dshCount(s *state, h *dshHead, b []byte, main bool) {+ var e dshEvent+ if dshWant(b) {+ if json.Unmarshal(b, &e) != nil || e.Type == "" {+ return}+ } else if t := dshNum(b, "time"); t > 0 {+ // only its time and place looked at+ e.Time = t+ if n := dshNum(b, "seq"); n >= 0 {+ e.Seq = &n+ }+ } else {+ return}- s.saw(ms(h.CreatedAt), f.main)- for i, e := range evs {- inherited := i <= cut || h.Version < 2 && e.Seq != nil && *e.Seq < h.SeedLength- switch e.Type {- case "user/message":- if f.main && s.Title == "" && s.First == "" && e.Data.Source.Kind == "user" {- if t := ccText(e.Data.Content); strings.HasPrefix(t, "<") {- s.First = untagged(t)- } else {- s.Title = title(t)- }- }- case "session/title":- if t := title(e.Data.Title); t != "" {- s.Named = t+ switch e.Type {+ case "user/message":+ if main && s.Title == "" && s.First == "" && e.Data.Source.Kind == "user" {+ if t := ccText(e.Data.Content); strings.HasPrefix(t, "<") {+ s.First = untagged(t)+ } else {+ s.Title = title(t)}}- if inherited {- continue+ case "session/title":+ if t := title(e.Data.Title); t != "" {+ s.Named = t}- at := ms(e.Time)- s.saw(at, f.main)- if u := e.Data.Usage; e.Type == "assistant/message" && u != nil && e.Data.Message.Source.Model != "" {- s.use(dateOf(at), e.Data.Message.Source.Model, Tokens{Input: u.Input, Output: u.Output, CacheRead: u.CacheRead, CacheWrite: u.CacheWrite})+ case "session/end-seed":+ if h.Version >= 2 && h.IsSeeded && e.Data.Inherited {+ // the events a fork was seeded with end here: what was+ // counted of them is its source's+ n := &state{ID: s.ID, Cwd: s.Cwd, Title: s.Title, First: s.First, Named: s.Named, Size: s.Size, Mod: s.Mod}+ n.saw(ms(h.CreatedAt), main)+ *s = *n+ return}}- return s+ if h.Version < 2 && e.Seq != nil && *e.Seq < h.SeedLength {+ return+ }+ at := ms(e.Time)+ s.saw(at, main)+ if u := e.Data.Usage; e.Type == "assistant/message" && u != nil && e.Data.Message.Source.Model != "" {+ s.use(dateOf(at), e.Data.Message.Source.Model, Tokens{Input: u.Input, Output: u.Output, CacheRead: u.CacheRead, CacheWrite: u.CacheWrite})+ }}func dshWant(b []byte) bool {diff --git a/internal/sessions/frames.go b/internal/sessions/frames.gonew file mode 100644index 000000000..520a12674--- /dev/null+++ b/internal/sessions/frames.go@@ -0,0 +1,203 @@+package sessions++import (+ "bufio"+ "bytes"+ "encoding/binary"+ "errors"+ "io"+ "os"++ "github.com/klauspost/compress/zstd"+)++// Reading on through a file of zstd frames, as dsh writes its sessions: a+// batch of whole lines a frame, appended. A frame can't be read from its+// middle, but it can be found without decoding it (RFC 8878: its header,+// then blocks that each say how long they are), so a read goes on from the+// frame it left off in, passing over what that frame gave it before. A+// last frame not yet written whole waits for the next read.++var errFrame = errors.New("not a zstd frame")++// readFrame reads the frame at r into buf: a zstd frame whole, nil for a+// skippable one. io.ErrUnexpectedEOF when the file ends inside it, io.EOF+// when it ends before it.+func readFrame(r *bufio.Reader, buf []byte) ([]byte, int64, error) {+ buf = buf[:0]+ var n int64+ take := func(k int) ([]byte, error) {+ start := len(buf)+ buf = append(buf, make([]byte, k)...)+ if _, err := io.ReadFull(r, buf[start:]); err != nil {+ if errors.Is(err, io.EOF) {+ err = io.ErrUnexpectedEOF+ }+ return nil, err+ }+ n += int64(k)+ return buf[start:], nil+ }+ if _, err := r.Peek(1); err != nil {+ return nil, 0, io.EOF+ }+ magic, err := take(4)+ if err != nil {+ return nil, 0, err+ }+ m := binary.LittleEndian.Uint32(magic)+ if m&0xFFFFFFF0 == 0x184D2A50 {+ size, err := take(4)+ if err != nil {+ return nil, 0, err+ }+ k := int64(binary.LittleEndian.Uint32(size))+ if d, err := r.Discard(int(k)); err != nil || int64(d) < k {+ return nil, 0, io.ErrUnexpectedEOF+ }+ return nil, n + k, nil+ }+ if m != 0xFD2FB528 {+ return nil, 0, errFrame+ }+ fhd, err := take(1)+ if err != nil {+ return nil, 0, err+ }+ d := fhd[0]+ single := d>>5&1 == 1+ extra := 0+ if !single {+ extra++ // window descriptor+ }+ extra += []int{0, 1, 2, 4}[d&3] // dictionary id+ switch d >> 6 {+ case 0:+ if single {+ extra+++ }+ case 1:+ extra += 2+ case 2:+ extra += 4+ case 3:+ extra += 8+ }+ if _, err := take(extra); err != nil {+ return nil, 0, err+ }+ for {+ h, err := take(3)+ if err != nil {+ return nil, 0, err+ }+ v := uint32(h[0]) | uint32(h[1])<<8 | uint32(h[2])<<16+ size := int(v >> 3)+ switch v >> 1 & 3 {+ case 1:+ size = 1 // RLE: one byte, repeated+ case 3:+ return nil, 0, errFrame+ }+ if _, err := take(size); err != nil {+ return nil, 0, err+ }+ if v&1 == 1 {+ break+ }+ }+ if d>>2&1 == 1 {+ if _, err := take(4); err != nil {+ return nil, 0, err+ }+ }+ return buf, n, nil+}++// scanFrames hands fn each whole line of a file of zstd frames, from the+// frame at off, skip bytes into what that frame holds. It returns where to+// go on from (the frame the first line not yet whole starts in, and how far+// into it) and where the last whole frame read ends.+func scanFrames(path string, off, skip int64, fn func([]byte) bool) (next, nextSkip, end int64, err error) {+ if scanHook != nil {+ scanHook(path, off)+ }+ next, nextSkip, end = off, skip, off+ f, err := os.Open(path)+ if err != nil {+ return+ }+ defer f.Close()+ if _, err = f.Seek(off, io.SeekStart); err != nil {+ return+ }+ dec, err := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecoderLowmem(true))+ if err != nil {+ return+ }+ defer dec.Close()+ r := bufio.NewReaderSize(f, 1<<20)+ var frame, out, pending []byte+ at := off+ first := true+ for {+ var n int64+ frame, n, err = readFrame(r, frame)+ if err != nil {+ // the end, a frame still being written, or not one at all:+ // what was read stands, the rest waits for the next read+ err = nil+ return+ }+ start := at+ at += n+ end = at+ if frame == nil {+ if len(pending) == 0 {+ next, nextSkip = at, 0+ }+ continue+ }+ if out, err = dec.DecodeAll(frame, out[:0]); err != nil {+ err = nil+ end = start+ return+ }+ from := int64(0)+ if first {+ first = false+ if skip > int64(len(out)) {+ err = errFrame+ return+ }+ from = skip+ }+ data := out[from:]+ for {+ i := bytes.IndexByte(data, '\n')+ if i < 0 {+ break+ }+ line := data[:i]+ if len(pending) > 0 {+ line = append(pending, line...)+ pending = pending[:0]+ }+ if !fn(line) {+ return+ }+ from += int64(i + 1)+ data = data[i+1:]+ }+ if len(data) == 0 {+ if len(pending) == 0 {+ next, nextSkip = at, 0+ }+ continue+ }+ if len(pending) == 0 {+ next, nextSkip = start, from+ }+ pending = append(pending, data...)+ }+}diff --git a/internal/sessions/grow_test.go b/internal/sessions/grow_test.goindex 1324eef26..3c7aecccb 100644--- a/internal/sessions/grow_test.go+++ b/internal/sessions/grow_test.go@@ -7,6 +7,8 @@ import ("os""path/filepath""testing"++ "github.com/klauspost/compress/zstd")// grower reads one session file as refresh does, each parse on from the@@ -287,3 +289,114 @@ func TestKimiForkMarkerAcrossReads(t *testing.T) {t.Fatalf("the fork's own: %+v", g.old.Models)}}++const dshReply = `{"type":"assistant/message","seq":6,"time":1790503290000,"data":{"message":{"source":{"model":"deepseek-v4-flash"}},"usage":{"inputTokens":1,"outputTokens":1,"cacheReadTokens":0}}}` + "\n"++// dsh appends to a session file a line at a time: a reply is read alone.+func TestDshReadsOnlyWhatWasAppended(t *testing.T) {+ inZone(t, 0)+ home := setupAgents(t)+ p := filepath.Join(home, ".dsh", "sessions", "work-dsh-3f2a", "dsh-a", "session.v3.jsonl")+ checkGrows(t, &grower{t: t, files: dshFiles, path: p}, dshReply)+}++// Packed, dsh appends a frame of lines at a time: a frame appended is read+// alone, from where it starts; a frame not yet written whole waits, and so+// does a line a frame ends inside of, read on from the frame it starts in.+func TestDshPackedReadsOnlyWhatWasAppended(t *testing.T) {+ inZone(t, 0)+ home := setupAgents(t)+ plain := filepath.Join(home, ".dsh", "sessions", "work-dsh-3f2a", "dsh-a", "session.v3.jsonl")+ b, err := os.ReadFile(plain)+ if err != nil {+ t.Fatal(err)+ }+ os.Remove(plain)+ enc, _ := zstd.NewWriter(nil)+ defer enc.Close()+ frame := func(s string) []byte { return enc.EncodeAll([]byte(s), nil) }+ p := plain + ".zstd"+ half := bytes.IndexByte(b[len(b)/2:], '\n') + len(b)/2 + 1+ orig := append(frame(string(b[:half])), frame(string(b[half:]))...)+ os.WriteFile(p, orig, 0o644)+ g := &grower{t: t, files: dshFiles, path: p}+ if at := g.read(); at != 0 {+ t.Fatalf("first read started at %d", at)+ }+ reply := func(seq int) string {+ return fmt.Sprintf(`{"type":"assistant/message","seq":%d,"time":%d,"data":{"message":{"source":{"model":"deepseek-v4-flash"}},"usage":{"inputTokens":%d,"outputTokens":1,"cacheReadTokens":0}}}`+"\n", seq, 1790503290000+int64(seq)*1000, seq)+ }+ // a frame appended: only it is read+ was := g.size()+ g.append(string(frame(reply(6))))+ if at := g.read(); at != was {+ t.Fatalf("after a frame appended to %d bytes, read from %d", was, at)+ }+ // half a frame, still being written: it waits for the rest+ was = g.size()+ f := frame(reply(7))+ g.append(string(f[:len(f)/2]))+ if at := g.read(); at != was {+ t.Fatalf("with half a frame at %d, read from %d", was, at)+ }+ g.append(string(f[len(f)/2:]))+ if at := g.read(); at != was {+ t.Fatalf("the half frame finished at %d, read from %d", was, at)+ }+ // a frame that ends inside a line: the line goes on in the next+ // frame, and the next read starts at the frame it starts in, past+ // what of that frame it counted+ was = g.size()+ r9 := reply(9)+ g.append(string(frame(reply(8) + r9[:20])))+ if at := g.read(); at != was {+ t.Fatalf("a frame ending in a line at %d, read from %d", was, at)+ }+ g.append(string(frame(r9[20:])))+ if at := g.read(); at != was {+ t.Fatalf("the line finished in the next frame, read from %d, not its frame's %d", at, was)+ }+ if m := g.old.Models["deepseek-v4-flash"]; m.Input != 10+6+7+8+9 {+ t.Fatalf("flash, each reply once: %+v", m)+ }+ // cut short: read whole again+ os.WriteFile(p, orig, 0o644)+ if at := g.read(); at != 0 {+ t.Fatalf("cut short, read from %d", at)+ }+ // replaced by one longer, its first frame changed: read whole+ g.append(string(frame(reply(6))))+ g.read()+ other := append(frame(string(bytes.Replace(b[:half], []byte("house rules"), []byte("house_rules"), 1))), frame(string(b[half:]))...)+ for len(other) <= int(g.size()) {+ other = append(other, frame(reply(6))...)+ }+ os.WriteFile(p, other, 0o644)+ if at := g.read(); at != 0 {+ t.Fatalf("replaced, read from %d", at)+ }+}++// A fork's end-seed that comes in a later read still ends what it was+// seeded with: what was counted before it is its source's.+func TestDshEndSeedAcrossReads(t *testing.T) {+ inZone(t, 0)+ home := setupAgents(t)+ p := filepath.Join(home, ".dsh", "sessions", "work-dsh-3f2a", "dsh-c", "session.v3.jsonl")+ b, err := os.ReadFile(p)+ if err != nil {+ t.Fatal(err)+ }+ cut := bytes.Index(b, []byte(`{"type":"session/end-seed"`))+ rest := bytes.IndexByte(b[cut:], '\n') + cut + 1+ os.WriteFile(p, b[:cut], 0o644)+ g := &grower{t: t, files: dshFiles, path: p}+ g.read()+ g.append(string(b[cut:rest]))+ g.read()+ g.append(string(b[rest:]))+ g.read()+ if m := g.old.Models["deepseek-v4-pro"]; m.Input != 50 || len(g.old.Models) != 1 || g.old.Title != "Tidy the build script" {+ t.Fatalf("the fork's own: %+v, title %q", g.old.Models, g.old.Title)+ }+}diff --git a/internal/sessions/sessions.go b/internal/sessions/sessions.goindex 71c81ff0d..44e88442d 100644--- a/internal/sessions/sessions.go+++ b/internal/sessions/sessions.go@@ -219,6 +219,8 @@ type state struct {// Kimi Code: where its wire file was read to, and what the read// carries on withKimi *kimiCarry `json:"kimi,omitempty"`+ // dsh: where its file was read to, and its header+ Dsh *dshCarry `json:"dsh,omitempty"`}// day is one local date's share of a file: tokens by model, and the time@@ -1196,7 +1198,7 @@ func parserFor(agent string) sessionParser {case "opencode", "zcode":return sessionParser{whole: parseOpenCode}case "dsh":- return sessionParser{whole: parseDsh}+ return sessionParser{grow: parseDsh}case "cline":return sessionParser{whole: parseCline}case "grok":diff --git a/internal/sessions/tail.go b/internal/sessions/tail.goindex 69544e494..3ceb8553c 100644--- a/internal/sessions/tail.go+++ b/internal/sessions/tail.go@@ -5,14 +5,14 @@ import "os"// tail is where a reader of a file another agent appends to left off: the// offset after the last whole line it handled, and enough of the file as it// was to know it again. A summary kept with one reads on from Off when the-// file has only grown, instead of reading it whole again (Grok Build's,-// Droid's and Kimi Code's sessions are kept so). A file that shrank, or whose-// start or sampled prefix is no longer what was read, is read whole again:-// it was rewritten or replaced, not appended to.+// file has only grown, instead of reading it whole again (dsh's, Grok+// Build's, Droid's and Kimi Code's sessions are kept so). A file that+// shrank, or whose start or sampled prefix is no longer what was read, is+// read whole again: it was rewritten or replaced, not appended to.//-// A packed file (frames of zstd) is read on from the frame at Off, Skip-// bytes into what that frame holds, and Size there is the end of that frame:-// the bytes hashed always take in what is read again.+// A packed file (frames of zstd, frames.go) is read on from the frame at+// Off, Skip bytes into what that frame holds, and Size there is the end of+// that frame: the bytes hashed always take in what is read again.type tail struct {Off int64 `json:"off"`Skip int64 `json:"skip,omitempty"`