Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

sessions: a dsh session read before is read on from where it was left, not whole

yetoneb3642e727700
diff --git a/internal/sessions/dsh.go b/internal/sessions/dsh.go
index 55c0865db..baf62c6c2 100644
--- a/internal/sessions/dsh.go
+++ b/internal/sessions/dsh.go
@@ -4,7 +4,6 @@ import (
"bufio"
"bytes"
"encoding/json"
- "errors"
"fmt"
"io"
"os"
@@ -242,86 +241,116 @@ type dshEvent struct {
var dshWanted = [][]byte{[]byte(`"user/message"`), []byte(`"session/title"`), []byte(`"assistant/message"`), []byte(`"session/end-seed"`)}
-// parseDsh reads a session file whole: compressed, it can't be read on from
-// the middle of a frame.
-func parseDsh(f file) *state {
- s := &state{Size: f.size, Mod: f.mod.UnixNano()}
- r, err := dshOpen(f.path)
- if err != nil {
- return s
+// dshCarry is what a read of a dsh session file carries on with: where it
+// read to and the header it read first.
+type dshCarry struct {
+ Tail *tail `json:"tail,omitempty"`
+ Head dshHead `json:"head"`
+}
+
+// parseDsh reads a session file. dsh only appends to one, a line at a time
+// or, packed, a frame of lines at a time, so a file read before is read on
+// from where that read left off: a packed one from the frame it left off
+// in. A file cut short or replaced is read whole again, and a line or frame
+// not yet written whole waits for the next read.
+func parseDsh(f file, old *state) *state {
+ var s *state
+ var c dshCarry
+ var off, skip int64
+ if old != nil && old.Dsh != nil && old.Dsh.Tail.holds(f.path) {
+ s, c = old.clone(), *old.Dsh
+ off, skip = c.Tail.Off, c.Tail.Skip
+ } else {
+ s = &state{}
}
- defer r.Close()
- br := bufio.NewReaderSize(r, 1<<20)
- var h dshHead
- var evs []dshEvent
- for first := true; ; first = false {
- b, err := br.ReadBytes('\n')
- if err != nil {
- // a line still being written, or a frame cut short
- if !errors.Is(err, io.EOF) || len(bytes.TrimSpace(b)) == 0 {
- break
- }
+ s.Size, s.Mod = f.size, f.mod.UnixNano()
+ headed := c.Tail != nil
+ ok := true
+ each := func(b []byte) bool {
+ b = bytes.TrimSpace(b)
+ if len(b) == 0 || len(b) > maxLine {
+ return true
}
- if b = bytes.TrimSpace(b); len(b) > 0 && len(b) <= maxLine {
- if first {
- if json.Unmarshal(b, &h) != nil || h.Type != "session" {
- return s
- }
- } else if dshWant(b) {
- var e dshEvent
- if json.Unmarshal(b, &e) == nil && e.Type != "" {
- evs = append(evs, e)
- }
- } else if t := dshNum(b, "time"); t > 0 {
- // only its time and place looked at
- e := dshEvent{Time: t}
- if n := dshNum(b, "seq"); n >= 0 {
- e.Seq = &n
- }
- evs = append(evs, e)
+ if !headed {
+ headed = true
+ if json.Unmarshal(b, &c.Head) != nil || c.Head.Type != "session" {
+ ok = false
+ return false
}
+ s.ID, s.Cwd = c.Head.ID, c.Head.Cwd
+ s.saw(ms(c.Head.CreatedAt), f.main)
+ return true
}
- if err != nil {
- break
- }
+ dshCount(s, &c.Head, b, f.main)
+ return true
}
- s.ID, s.Cwd = h.ID, h.Cwd
- // the events a fork was seeded with
- cut := -1
- if h.Version >= 2 && h.IsSeeded {
- for i, e := range evs {
- if e.Type == "session/end-seed" && e.Data.Inherited {
- cut = i
- }
+ var next, nextSkip, end int64
+ var err error
+ if packed(f.path) {
+ next, nextSkip, end, err = scanFrames(f.path, off, skip, each)
+ } else {
+ next, err = scanAt(f.path, off, nil, func(b []byte, _, _ int64) bool { return each(b) })
+ end = next
+ }
+ if !ok {
+ // not a session's file: only its size and time
+ return &state{Size: f.size, Mod: f.mod.UnixNano()}
+ }
+ c.Tail = nil
+ if err == nil {
+ c.Tail = markTail(f.path, next, nextSkip, end)
+ }
+ s.Dsh = &c
+ return s
+}
+
+// dshCount counts one event of a session file after its header.
+func dshCount(s *state, h *dshHead, b []byte, main bool) {
+ var e dshEvent
+ if dshWant(b) {
+ if json.Unmarshal(b, &e) != nil || e.Type == "" {
+ return
}
+ } else if t := dshNum(b, "time"); t > 0 {
+ // only its time and place looked at
+ e.Time = t
+ if n := dshNum(b, "seq"); n >= 0 {
+ e.Seq = &n
+ }
+ } else {
+ return
}
- s.saw(ms(h.CreatedAt), f.main)
- for i, e := range evs {
- inherited := i <= cut || h.Version < 2 && e.Seq != nil && *e.Seq < h.SeedLength
- switch e.Type {
- case "user/message":
- if f.main && s.Title == "" && s.First == "" && e.Data.Source.Kind == "user" {
- if t := ccText(e.Data.Content); strings.HasPrefix(t, "<") {
- s.First = untagged(t)
- } else {
- s.Title = title(t)
- }
- }
- case "session/title":
- if t := title(e.Data.Title); t != "" {
- s.Named = t
+ switch e.Type {
+ case "user/message":
+ if main && s.Title == "" && s.First == "" && e.Data.Source.Kind == "user" {
+ if t := ccText(e.Data.Content); strings.HasPrefix(t, "<") {
+ s.First = untagged(t)
+ } else {
+ s.Title = title(t)
}
}
- if inherited {
- continue
+ case "session/title":
+ if t := title(e.Data.Title); t != "" {
+ s.Named = t
}
- at := ms(e.Time)
- s.saw(at, f.main)
- if u := e.Data.Usage; e.Type == "assistant/message" && u != nil && e.Data.Message.Source.Model != "" {
- s.use(dateOf(at), e.Data.Message.Source.Model, Tokens{Input: u.Input, Output: u.Output, CacheRead: u.CacheRead, CacheWrite: u.CacheWrite})
+ case "session/end-seed":
+ if h.Version >= 2 && h.IsSeeded && e.Data.Inherited {
+ // the events a fork was seeded with end here: what was
+ // counted of them is its source's
+ n := &state{ID: s.ID, Cwd: s.Cwd, Title: s.Title, First: s.First, Named: s.Named, Size: s.Size, Mod: s.Mod}
+ n.saw(ms(h.CreatedAt), main)
+ *s = *n
+ return
}
}
- return s
+ if h.Version < 2 && e.Seq != nil && *e.Seq < h.SeedLength {
+ return
+ }
+ at := ms(e.Time)
+ s.saw(at, main)
+ if u := e.Data.Usage; e.Type == "assistant/message" && u != nil && e.Data.Message.Source.Model != "" {
+ s.use(dateOf(at), e.Data.Message.Source.Model, Tokens{Input: u.Input, Output: u.Output, CacheRead: u.CacheRead, CacheWrite: u.CacheWrite})
+ }
}
func dshWant(b []byte) bool {
diff --git a/internal/sessions/frames.go b/internal/sessions/frames.go
new file mode 100644
index 000000000..520a12674
--- /dev/null
+++ b/internal/sessions/frames.go
@@ -0,0 +1,203 @@
+package sessions
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/binary"
+ "errors"
+ "io"
+ "os"
+
+ "github.com/klauspost/compress/zstd"
+)
+
+// Reading on through a file of zstd frames, as dsh writes its sessions: a
+// batch of whole lines a frame, appended. A frame can't be read from its
+// middle, but it can be found without decoding it (RFC 8878: its header,
+// then blocks that each say how long they are), so a read goes on from the
+// frame it left off in, passing over what that frame gave it before. A
+// last frame not yet written whole waits for the next read.
+
+var errFrame = errors.New("not a zstd frame")
+
+// readFrame reads the frame at r into buf: a zstd frame whole, nil for a
+// skippable one. io.ErrUnexpectedEOF when the file ends inside it, io.EOF
+// when it ends before it.
+func readFrame(r *bufio.Reader, buf []byte) ([]byte, int64, error) {
+ buf = buf[:0]
+ var n int64
+ take := func(k int) ([]byte, error) {
+ start := len(buf)
+ buf = append(buf, make([]byte, k)...)
+ if _, err := io.ReadFull(r, buf[start:]); err != nil {
+ if errors.Is(err, io.EOF) {
+ err = io.ErrUnexpectedEOF
+ }
+ return nil, err
+ }
+ n += int64(k)
+ return buf[start:], nil
+ }
+ if _, err := r.Peek(1); err != nil {
+ return nil, 0, io.EOF
+ }
+ magic, err := take(4)
+ if err != nil {
+ return nil, 0, err
+ }
+ m := binary.LittleEndian.Uint32(magic)
+ if m&0xFFFFFFF0 == 0x184D2A50 {
+ size, err := take(4)
+ if err != nil {
+ return nil, 0, err
+ }
+ k := int64(binary.LittleEndian.Uint32(size))
+ if d, err := r.Discard(int(k)); err != nil || int64(d) < k {
+ return nil, 0, io.ErrUnexpectedEOF
+ }
+ return nil, n + k, nil
+ }
+ if m != 0xFD2FB528 {
+ return nil, 0, errFrame
+ }
+ fhd, err := take(1)
+ if err != nil {
+ return nil, 0, err
+ }
+ d := fhd[0]
+ single := d>>5&1 == 1
+ extra := 0
+ if !single {
+ extra++ // window descriptor
+ }
+ extra += []int{0, 1, 2, 4}[d&3] // dictionary id
+ switch d >> 6 {
+ case 0:
+ if single {
+ extra++
+ }
+ case 1:
+ extra += 2
+ case 2:
+ extra += 4
+ case 3:
+ extra += 8
+ }
+ if _, err := take(extra); err != nil {
+ return nil, 0, err
+ }
+ for {
+ h, err := take(3)
+ if err != nil {
+ return nil, 0, err
+ }
+ v := uint32(h[0]) | uint32(h[1])<<8 | uint32(h[2])<<16
+ size := int(v >> 3)
+ switch v >> 1 & 3 {
+ case 1:
+ size = 1 // RLE: one byte, repeated
+ case 3:
+ return nil, 0, errFrame
+ }
+ if _, err := take(size); err != nil {
+ return nil, 0, err
+ }
+ if v&1 == 1 {
+ break
+ }
+ }
+ if d>>2&1 == 1 {
+ if _, err := take(4); err != nil {
+ return nil, 0, err
+ }
+ }
+ return buf, n, nil
+}
+
+// scanFrames hands fn each whole line of a file of zstd frames, from the
+// frame at off, skip bytes into what that frame holds. It returns where to
+// go on from (the frame the first line not yet whole starts in, and how far
+// into it) and where the last whole frame read ends.
+func scanFrames(path string, off, skip int64, fn func([]byte) bool) (next, nextSkip, end int64, err error) {
+ if scanHook != nil {
+ scanHook(path, off)
+ }
+ next, nextSkip, end = off, skip, off
+ f, err := os.Open(path)
+ if err != nil {
+ return
+ }
+ defer f.Close()
+ if _, err = f.Seek(off, io.SeekStart); err != nil {
+ return
+ }
+ dec, err := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecoderLowmem(true))
+ if err != nil {
+ return
+ }
+ defer dec.Close()
+ r := bufio.NewReaderSize(f, 1<<20)
+ var frame, out, pending []byte
+ at := off
+ first := true
+ for {
+ var n int64
+ frame, n, err = readFrame(r, frame)
+ if err != nil {
+ // the end, a frame still being written, or not one at all:
+ // what was read stands, the rest waits for the next read
+ err = nil
+ return
+ }
+ start := at
+ at += n
+ end = at
+ if frame == nil {
+ if len(pending) == 0 {
+ next, nextSkip = at, 0
+ }
+ continue
+ }
+ if out, err = dec.DecodeAll(frame, out[:0]); err != nil {
+ err = nil
+ end = start
+ return
+ }
+ from := int64(0)
+ if first {
+ first = false
+ if skip > int64(len(out)) {
+ err = errFrame
+ return
+ }
+ from = skip
+ }
+ data := out[from:]
+ for {
+ i := bytes.IndexByte(data, '\n')
+ if i < 0 {
+ break
+ }
+ line := data[:i]
+ if len(pending) > 0 {
+ line = append(pending, line...)
+ pending = pending[:0]
+ }
+ if !fn(line) {
+ return
+ }
+ from += int64(i + 1)
+ data = data[i+1:]
+ }
+ if len(data) == 0 {
+ if len(pending) == 0 {
+ next, nextSkip = at, 0
+ }
+ continue
+ }
+ if len(pending) == 0 {
+ next, nextSkip = start, from
+ }
+ pending = append(pending, data...)
+ }
+}
diff --git a/internal/sessions/grow_test.go b/internal/sessions/grow_test.go
index 1324eef26..3c7aecccb 100644
--- a/internal/sessions/grow_test.go
+++ b/internal/sessions/grow_test.go
@@ -7,6 +7,8 @@ import (
"os"
"path/filepath"
"testing"
+
+ "github.com/klauspost/compress/zstd"
)
// grower reads one session file as refresh does, each parse on from the
@@ -287,3 +289,114 @@ func TestKimiForkMarkerAcrossReads(t *testing.T) {
t.Fatalf("the fork's own: %+v", g.old.Models)
}
}
+
+const dshReply = `{"type":"assistant/message","seq":6,"time":1790503290000,"data":{"message":{"source":{"model":"deepseek-v4-flash"}},"usage":{"inputTokens":1,"outputTokens":1,"cacheReadTokens":0}}}` + "\n"
+
+// dsh appends to a session file a line at a time: a reply is read alone.
+func TestDshReadsOnlyWhatWasAppended(t *testing.T) {
+ inZone(t, 0)
+ home := setupAgents(t)
+ p := filepath.Join(home, ".dsh", "sessions", "work-dsh-3f2a", "dsh-a", "session.v3.jsonl")
+ checkGrows(t, &grower{t: t, files: dshFiles, path: p}, dshReply)
+}
+
+// Packed, dsh appends a frame of lines at a time: a frame appended is read
+// alone, from where it starts; a frame not yet written whole waits, and so
+// does a line a frame ends inside of, read on from the frame it starts in.
+func TestDshPackedReadsOnlyWhatWasAppended(t *testing.T) {
+ inZone(t, 0)
+ home := setupAgents(t)
+ plain := filepath.Join(home, ".dsh", "sessions", "work-dsh-3f2a", "dsh-a", "session.v3.jsonl")
+ b, err := os.ReadFile(plain)
+ if err != nil {
+ t.Fatal(err)
+ }
+ os.Remove(plain)
+ enc, _ := zstd.NewWriter(nil)
+ defer enc.Close()
+ frame := func(s string) []byte { return enc.EncodeAll([]byte(s), nil) }
+ p := plain + ".zstd"
+ half := bytes.IndexByte(b[len(b)/2:], '\n') + len(b)/2 + 1
+ orig := append(frame(string(b[:half])), frame(string(b[half:]))...)
+ os.WriteFile(p, orig, 0o644)
+ g := &grower{t: t, files: dshFiles, path: p}
+ if at := g.read(); at != 0 {
+ t.Fatalf("first read started at %d", at)
+ }
+ reply := func(seq int) string {
+ return fmt.Sprintf(`{"type":"assistant/message","seq":%d,"time":%d,"data":{"message":{"source":{"model":"deepseek-v4-flash"}},"usage":{"inputTokens":%d,"outputTokens":1,"cacheReadTokens":0}}}`+"\n", seq, 1790503290000+int64(seq)*1000, seq)
+ }
+ // a frame appended: only it is read
+ was := g.size()
+ g.append(string(frame(reply(6))))
+ if at := g.read(); at != was {
+ t.Fatalf("after a frame appended to %d bytes, read from %d", was, at)
+ }
+ // half a frame, still being written: it waits for the rest
+ was = g.size()
+ f := frame(reply(7))
+ g.append(string(f[:len(f)/2]))
+ if at := g.read(); at != was {
+ t.Fatalf("with half a frame at %d, read from %d", was, at)
+ }
+ g.append(string(f[len(f)/2:]))
+ if at := g.read(); at != was {
+ t.Fatalf("the half frame finished at %d, read from %d", was, at)
+ }
+ // a frame that ends inside a line: the line goes on in the next
+ // frame, and the next read starts at the frame it starts in, past
+ // what of that frame it counted
+ was = g.size()
+ r9 := reply(9)
+ g.append(string(frame(reply(8) + r9[:20])))
+ if at := g.read(); at != was {
+ t.Fatalf("a frame ending in a line at %d, read from %d", was, at)
+ }
+ g.append(string(frame(r9[20:])))
+ if at := g.read(); at != was {
+ t.Fatalf("the line finished in the next frame, read from %d, not its frame's %d", at, was)
+ }
+ if m := g.old.Models["deepseek-v4-flash"]; m.Input != 10+6+7+8+9 {
+ t.Fatalf("flash, each reply once: %+v", m)
+ }
+ // cut short: read whole again
+ os.WriteFile(p, orig, 0o644)
+ if at := g.read(); at != 0 {
+ t.Fatalf("cut short, read from %d", at)
+ }
+ // replaced by one longer, its first frame changed: read whole
+ g.append(string(frame(reply(6))))
+ g.read()
+ other := append(frame(string(bytes.Replace(b[:half], []byte("house rules"), []byte("house_rules"), 1))), frame(string(b[half:]))...)
+ for len(other) <= int(g.size()) {
+ other = append(other, frame(reply(6))...)
+ }
+ os.WriteFile(p, other, 0o644)
+ if at := g.read(); at != 0 {
+ t.Fatalf("replaced, read from %d", at)
+ }
+}
+
+// A fork's end-seed that comes in a later read still ends what it was
+// seeded with: what was counted before it is its source's.
+func TestDshEndSeedAcrossReads(t *testing.T) {
+ inZone(t, 0)
+ home := setupAgents(t)
+ p := filepath.Join(home, ".dsh", "sessions", "work-dsh-3f2a", "dsh-c", "session.v3.jsonl")
+ b, err := os.ReadFile(p)
+ if err != nil {
+ t.Fatal(err)
+ }
+ cut := bytes.Index(b, []byte(`{"type":"session/end-seed"`))
+ rest := bytes.IndexByte(b[cut:], '\n') + cut + 1
+ os.WriteFile(p, b[:cut], 0o644)
+ g := &grower{t: t, files: dshFiles, path: p}
+ g.read()
+ g.append(string(b[cut:rest]))
+ g.read()
+ g.append(string(b[rest:]))
+ g.read()
+ if m := g.old.Models["deepseek-v4-pro"]; m.Input != 50 || len(g.old.Models) != 1 || g.old.Title != "Tidy the build script" {
+ t.Fatalf("the fork's own: %+v, title %q", g.old.Models, g.old.Title)
+ }
+}
diff --git a/internal/sessions/sessions.go b/internal/sessions/sessions.go
index 71c81ff0d..44e88442d 100644
--- a/internal/sessions/sessions.go
+++ b/internal/sessions/sessions.go
@@ -219,6 +219,8 @@ type state struct {
// Kimi Code: where its wire file was read to, and what the read
// carries on with
Kimi *kimiCarry `json:"kimi,omitempty"`
+ // dsh: where its file was read to, and its header
+ Dsh *dshCarry `json:"dsh,omitempty"`
}
// day is one local date's share of a file: tokens by model, and the time
@@ -1196,7 +1198,7 @@ func parserFor(agent string) sessionParser {
case "opencode", "zcode":
return sessionParser{whole: parseOpenCode}
case "dsh":
- return sessionParser{whole: parseDsh}
+ return sessionParser{grow: parseDsh}
case "cline":
return sessionParser{whole: parseCline}
case "grok":
diff --git a/internal/sessions/tail.go b/internal/sessions/tail.go
index 69544e494..3ceb8553c 100644
--- a/internal/sessions/tail.go
+++ b/internal/sessions/tail.go
@@ -5,14 +5,14 @@ import "os"
// tail is where a reader of a file another agent appends to left off: the
// offset after the last whole line it handled, and enough of the file as it
// was to know it again. A summary kept with one reads on from Off when the
-// file has only grown, instead of reading it whole again (Grok Build's,
-// Droid's and Kimi Code's sessions are kept so). A file that shrank, or whose
-// start or sampled prefix is no longer what was read, is read whole again:
-// it was rewritten or replaced, not appended to.
+// file has only grown, instead of reading it whole again (dsh's, Grok
+// Build's, Droid's and Kimi Code's sessions are kept so). A file that
+// shrank, or whose start or sampled prefix is no longer what was read, is
+// read whole again: it was rewritten or replaced, not appended to.
//
-// A packed file (frames of zstd) is read on from the frame at Off, Skip
-// bytes into what that frame holds, and Size there is the end of that frame:
-// the bytes hashed always take in what is read again.
+// A packed file (frames of zstd, frames.go) is read on from the frame at
+// Off, Skip bytes into what that frame holds, and Size there is the end of
+// that frame: the bytes hashed always take in what is read again.
type tail struct {
Off int64 `json:"off"`
Skip int64 `json:"skip,omitempty"`