Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix(cli): read the whole Claude Code transcript; price 1h cache writes and Opus/Sonnet 5.5 (#1880)
Aman Agarwal4f095b5abf4c
diff --git a/cli/internal/coding/hook/claudecode/handle.go b/cli/internal/coding/hook/claudecode/handle.goindex 709ebfe5..ae976f0e 100644--- a/cli/internal/coding/hook/claudecode/handle.go+++ b/cli/internal/coding/hook/claudecode/handle.go@@ -913,7 +913,7 @@ func stringFieldFromInput(raw json.RawMessage, field string) string {return s}-// tailTranscript scans the end of a Claude Code transcript JSONL file+// tailTranscript scans a Claude Code transcript JSONL file// for the final usage line. The transcript format is one JSON object// per line, with assistant turns carrying a `usage` object that totals// input/output/cache tokens for that turn. We sum the assistant turns@@ -931,21 +931,17 @@ func tailTranscript(path string) (model string, cost float64, inTokens, outTokenpath = filepath.Join(home, strings.TrimPrefix(path, "~/"))}}- // F3: bound the read to the last `DefaultCap` bytes so the hook- // never blocks on a multi-GB transcript. Marathon sessions whose- // usage rolls up across the boundary will undercount tokens —- // we accept that in exchange for predictable hook latency. The- // last-N-bytes window comfortably holds a multi-hour session at- // realistic Claude turn sizes.- lines := tailfile.Tail(path, tailfile.DefaultCap)- if len(lines) == 0 {- return "", 0, 0, 0, 0- }+ // Stream the whole transcript, decoding only the usage-related fields.+ // A tail read undercounts: tool results push even short sessions past+ // tailfile.DefaultCap, and the session total must cover every request.type usage struct {InputTokens int64 `json:"input_tokens"`OutputTokens int64 `json:"output_tokens"`CacheCreationInputTokens int64 `json:"cache_creation_input_tokens"`CacheReadInputTokens int64 `json:"cache_read_input_tokens"`+ CacheCreation struct {+ Ephemeral1hInputTokens int64 `json:"ephemeral_1h_input_tokens"`+ } `json:"cache_creation"`}type turn struct {Type string `json:"type"`@@ -953,54 +949,73 @@ func tailTranscript(path string) (model string, cost float64, inTokens, outTokenMessage struct {ID string `json:"id"`Model string `json:"model"`- Usage usage `json:"usage"`+ Usage *usage `json:"usage"`} `json:"message"`}// Claude Code writes one line per content block (thinking / text /// tool_use) of an API response, each repeating the response's usage.// Count every response once — the last line of a message wins, as- // in coalesceAssistants.- byMessage := map[string]int{}- var turns []turn- for _, line := range lines {+ // in coalesceAssistants. Keep running totals plus a small usage record+ // per response (to swap in a later line), not every decoded response.+ type tally struct{ in, out, read, creation, creation1h int64 }+ add := func(a, b tally, sign int64) tally {+ return tally{a.in + sign*b.in, a.out + sign*b.out, a.read + sign*b.read,+ a.creation + sign*b.creation, a.creation1h + sign*b.creation1h}+ }+ byMessage := map[string]tally{}+ var sum tally+ var lastModel string+ var seen bool+ // Only these fields are kept from each line, so a line carrying a huge+ // tool input or result costs no extra memory.+ keep := []string{"type", "requestId", "message.id", "message.model", "message.usage"}+ err := tailfile.EachJSONLFields(path, keep, func(obj []byte) {var t turn- if err := json.Unmarshal(line, &t); err != nil {- continue+ if err := json.Unmarshal(obj, &t); err != nil || t.Message.Usage == nil {+ return+ }+ u := *t.Message.Usage+ // Anthropic transcript reports cache creation + cache read as+ // separate counters; "fresh" input is the standard input_tokens.+ // The OTel field gen_ai.usage.input_tokens should equal total+ // distinct input the model saw, so we sum them.+ r := tally{+ in: u.InputTokens + u.CacheCreationInputTokens + u.CacheReadInputTokens,+ out: u.OutputTokens,+ read: u.CacheReadInputTokens,+ creation: u.CacheCreationInputTokens,+ creation1h: u.CacheCreation.Ephemeral1hInputTokens,}key := t.Message.IDif key == "" {key = t.RequestID}if key != "" {- if i, seen := byMessage[key]; seen {- turns[i] = t- continue+ if prev, ok := byMessage[key]; ok {+ sum = add(sum, prev, -1)}- byMessage[key] = len(turns)+ byMessage[key] = r}- turns = append(turns, t)- }-- var ti, to, cacheRead, cacheCreation int64- var lastModel string- for _, t := range turns {- // Anthropic transcript reports cache creation + cache read as- // separate counters; "fresh" input is the standard input_tokens.- // The OTel field gen_ai.usage.input_tokens should equal total- // distinct input the model saw, so we sum them.- fresh := t.Message.Usage.InputTokens- creation := t.Message.Usage.CacheCreationInputTokens- read := t.Message.Usage.CacheReadInputTokens- ti += fresh + creation + read- cacheRead += read- cacheCreation += creation- to += t.Message.Usage.OutputTokens+ sum = add(sum, r, 1)if t.Message.Model != "" {lastModel = t.Message.Model}+ seen = true+ })+ // A partial read (I/O error, or a usage field too large to keep) must not+ // be published as the authoritative session total. Returning zeros lets+ // the session span fall back to the per-turn counters.+ if err != nil || !seen {+ return "", 0, 0, 0, 0}+ ti, to, cacheRead, cacheCreation, cacheCreation1h := sum.in, sum.out, sum.read, sum.creation, sum.creation1hrate := pricing.Lookup(lastModel)cost = rate.Cost(ti, to, cacheRead, cacheCreation)+ if cacheCreation1h > 0 && rate.CacheCreationPer1M > 0 {+ // 1-hour cache writes bill at 2x base input, not the 5-minute+ // write rate in the table; Claude Code uses them for long sessions.+ cost += float64(cacheCreation1h) * (2*rate.InputPer1M - rate.CacheCreationPer1M) / 1_000_000.0+ }return lastModel, cost, ti, to, ti + to}diff --git a/cli/internal/coding/hook/claudecode/transcript_usage_test.go b/cli/internal/coding/hook/claudecode/transcript_usage_test.goindex 24bdf644..d2a16299 100644--- a/cli/internal/coding/hook/claudecode/transcript_usage_test.go+++ b/cli/internal/coding/hook/claudecode/transcript_usage_test.go@@ -8,6 +8,7 @@ import ("testing""github.com/openlit/openlit/cli/internal/coding/pricing"+ "github.com/openlit/openlit/cli/internal/coding/tailfile")// TestTailTranscriptCountsEachMessageOnce covers the SessionEnd token@@ -61,3 +62,55 @@ func TestTailTranscriptCountsEachMessageOnce(t *testing.T) {t.Errorf("tailTranscript cost = %.8f, want %.8f", cost, want)}}++// Tool results make transcripts exceed tailfile.DefaultCap quickly; usage+// before the last DefaultCap bytes (and around oversized lines) must count.+func TestTailTranscriptReadsPastTailCap(t *testing.T) {+ first := `{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-opus-5-5","usage":{"input_tokens":100,"output_tokens":10}}}`+ filler := `{"type":"user","message":{"content":"` + strings.Repeat("x", int(tailfile.DefaultCap)+1024) + `"}}`+ last := `{"type":"assistant","requestId":"req_2","message":{"id":"msg_2","model":"claude-opus-5-5","usage":{"input_tokens":200,"output_tokens":20}}}`+ path := filepath.Join(t.TempDir(), "session.jsonl")+ if err := os.WriteFile(path, []byte(first+"\n"+filler+"\n"+last+"\n"), 0o600); err != nil {+ t.Fatal(err)+ }+ if _, _, in, out, _ := tailTranscript(path); in != 300 || out != 30 {+ t.Fatalf("in=%d out=%d, want 300/30", in, out)+ }+}++// Claude Code's 1-hour prompt-cache writes bill at 2x base input, not the+// 5-minute write rate.+func TestTailTranscriptPricesOneHourCacheWrites(t *testing.T) {+ line := `{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-opus-5-5","usage":{"input_tokens":0,"cache_creation_input_tokens":1000000,"cache_read_input_tokens":0,"output_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":1000000}}}}`+ path := filepath.Join(t.TempDir(), "session.jsonl")+ if err := os.WriteFile(path, []byte(line+"\n"), 0o600); err != nil {+ t.Fatal(err)+ }+ if _, cost, _, _, _ := tailTranscript(path); cost < 7.999 || cost > 8.001 {+ t.Fatalf("cost = %v, want 8.00 (1M 1h cache-write tokens at 2x $4 input)", cost)+ }+}++// A transcript that can't be read completely must not produce a session+// total; zeros make the session span fall back to the per-turn counters.+func TestTailTranscriptPublishesNothingOnReadError(t *testing.T) {+ dir := t.TempDir() // opening a directory succeeds, reading it fails+ if model, cost, in, out, total := tailTranscript(dir); model != "" || cost != 0 || in != 0 || out != 0 || total != 0 {+ t.Fatalf("got model=%q cost=%v in=%d out=%d total=%d, want zeros", model, cost, in, out, total)+ }+}++// A usage-bearing line larger than any line buffer (a big Write tool input)+// still counts, rather than making the rollup fall back.+func TestTailTranscriptCountsHugeUsageLines(t *testing.T) {+ huge := `{"type":"assistant","requestId":"req_1","message":{"id":"msg_1","model":"claude-opus-5-5","content":[{"type":"tool_use","input":{"content":"` ++ strings.Repeat("x", 20*1024*1024) + `"}}],"usage":{"input_tokens":100,"output_tokens":10}}}`+ small := `{"type":"assistant","requestId":"req_2","message":{"id":"msg_2","model":"claude-opus-5-5","usage":{"input_tokens":200,"output_tokens":20}}}`+ path := filepath.Join(t.TempDir(), "session.jsonl")+ if err := os.WriteFile(path, []byte(huge+"\n"+small+"\n"), 0o600); err != nil {+ t.Fatal(err)+ }+ if _, _, in, out, _ := tailTranscript(path); in != 300 || out != 30 {+ t.Fatalf("in=%d out=%d, want 300/30", in, out)+ }+}diff --git a/cli/internal/coding/pricing/pricing.go b/cli/internal/coding/pricing/pricing.goindex 80d0a1bc..004aa432 100644--- a/cli/internal/coding/pricing/pricing.go+++ b/cli/internal/coding/pricing/pricing.go@@ -186,6 +186,17 @@ var table = []entry{match: []string{"claude-opus-4-7-fast", "claude-opus-4-6-fast", "claude-opus-4-5-fast"},rate: Rate{InputPer1M: 30.00, OutputPer1M: 150.00, CachedReadPer1M: 3.00, CacheCreationPer1M: 37.50},},+ {+ // Claude Opus 5.5: $4/$20, cache read $0.20.+ match: []string{"claude-opus-5-5", "claude-opus-5.5"},+ rate: Rate{InputPer1M: 4.00, OutputPer1M: 20.00, CachedReadPer1M: 0.20, CacheCreationPer1M: 5.00},+ },+ {+ // Claude Sonnet 5.5: $2/$10, cache read $0.20. Must precede the+ // "claude-sonnet-5" substring entry, which would otherwise match.+ match: []string{"claude-sonnet-5-5", "claude-sonnet-5.5"},+ rate: Rate{InputPer1M: 2.00, OutputPer1M: 10.00, CachedReadPer1M: 0.20, CacheCreationPer1M: 2.50},+ },{// Claude Fable 5 (~2x Opus 4.8).match: []string{"claude-fable-5", "claude-fable", "fable-5"},diff --git a/cli/internal/coding/pricing/pricing_test.go b/cli/internal/coding/pricing/pricing_test.goindex 90c7d305..e5f7ff0f 100644--- a/cli/internal/coding/pricing/pricing_test.go+++ b/cli/internal/coding/pricing/pricing_test.go@@ -29,6 +29,9 @@ func TestLookupAnthropic(t *testing.T) {// Claude Fable 5 / Sonnet 5 / Sonnet 1M from Cursor pricing table.{"claude-fable-5", Rate{InputPer1M: 10.00, OutputPer1M: 50.00, CachedReadPer1M: 1.00, CacheCreationPer1M: 12.50}},{"claude-sonnet-5", Rate{InputPer1M: 3.00, OutputPer1M: 15.00, CachedReadPer1M: 0.30, CacheCreationPer1M: 3.75}},+ // Opus 5.5 / Sonnet 5.5; sonnet-5-5 must not fall into the Sonnet 5 bucket.+ {"claude-opus-5-5", Rate{InputPer1M: 4.00, OutputPer1M: 20.00, CachedReadPer1M: 0.20, CacheCreationPer1M: 5.00}},+ {"claude-sonnet-5-5", Rate{InputPer1M: 2.00, OutputPer1M: 10.00, CachedReadPer1M: 0.20, CacheCreationPer1M: 2.50}},{"claude-4-sonnet-1m", Rate{InputPer1M: 6.00, OutputPer1M: 22.50, CachedReadPer1M: 0.60, CacheCreationPer1M: 7.50}},// Legacy Opus 4.0 / 4.1 are still on the original $15/$75 list.{"claude-opus-4-0", Rate{InputPer1M: 15.00, OutputPer1M: 75.00, CachedReadPer1M: 1.50, CacheCreationPer1M: 18.75}},diff --git a/cli/internal/coding/tailfile/tailfile.go b/cli/internal/coding/tailfile/tailfile.goindex 80030c28..8bf170fa 100644--- a/cli/internal/coding/tailfile/tailfile.go+++ b/cli/internal/coding/tailfile/tailfile.go@@ -16,8 +16,11 @@ package tailfileimport ("bufio""bytes"+ "encoding/json"+ "errors""io""os"+ "strings")// DefaultCap is the byte budget for a tail-read. 4 MiB lets a@@ -92,3 +95,332 @@ func Tail(path string, cap int64) [][]byte {}return out}++// MaxFieldValue caps one captured field value in EachJSONLFields. Kept+// fields are small (ids, model names, usage objects); a larger value means+// the line isn't what the caller expects and is reported, not truncated.+const MaxFieldValue = 64 * 1024++const maxKeyLen = 256++// ErrFieldTooLarge reports that a kept field exceeded MaxFieldValue, so a+// rollup built from fn is incomplete.+var ErrFieldTooLarge = errors.New("tailfile: kept JSONL field exceeds MaxFieldValue")++// EachJSONLFields streams a JSONL file and, for every line that has at least+// one of the dotted `keep` paths (e.g. "message.usage"), calls fn with a+// compact JSON object holding only those fields. Everything else on the line+// is scanned and discarded, so memory stays bounded however long a line is:+// a transcript line carrying a multi-megabyte tool input costs no more than+// a short one. Paths only match object keys ("message.usage"), not array+// elements; malformed lines are skipped.+//+// Used by end-of-session rollups that must see the whole transcript (tool+// results make transcripts grow past DefaultCap within minutes, so a tail+// read undercounts). It returns any non-EOF read error, and ErrFieldTooLarge+// if a kept value had to be dropped; in both cases the caller must not treat+// what fn saw as complete. fn's slice is only valid during the call.+func EachJSONLFields(path string, keep []string, fn func(obj []byte)) error {+ f, err := os.Open(path) //nolint:gosec // path comes from the agent's own payload+ if err != nil {+ return err+ }+ defer f.Close()+ return eachJSONLFields(f, keep, fn)+}++func eachJSONLFields(src io.Reader, keep []string, fn func(obj []byte)) error {+ s := newFieldScanner(keep)+ r := bufio.NewReaderSize(src, 64*1024)+ tooLarge := false+ for {+ chunk, err := r.ReadSlice('\n')+ for _, c := range chunk {+ if c == '\n' { // valid JSONL has no raw newline inside a string+ if obj, ok := s.endLine(); ok {+ fn(obj)+ }+ tooLarge = tooLarge || s.overflow+ s.reset()+ continue+ }+ s.feed(c)+ }+ if err == bufio.ErrBufferFull {+ continue // the line continues in the next chunk+ }+ if err == io.EOF {+ if obj, ok := s.endLine(); ok { // final line without a trailing newline+ fn(obj)+ }+ tooLarge = tooLarge || s.overflow+ break+ }+ if err != nil {+ return err+ }+ }+ if tooLarge {+ return ErrFieldTooLarge+ }+ return nil+}++// fieldScanner is a byte-at-a-time JSON scanner for one line. It tracks only+// container nesting, the current key per object level, and string/escape+// state, and copies out the raw bytes of values at kept paths.+type fieldScanner struct {+ keep map[string]bool+ maxDepth int // deepest kept path; values below it are never captured++ stack []frame+ inStr bool+ esc bool+ strKey bool // the current string is an object key+ bad bool // line is not a JSON object (or is malformed)+ opened bool // saw the line's opening '{'++ capturing bool+ capPath string+ capDepth int // stack depth at which the captured value started+ capKind byte // '"' string, '{' / '[' container, 'l' literal+ capBuf []byte+ overflow bool+ found map[string][]byte+}++type frame struct {+ obj bool+ key []byte+ keyLong bool+ expectKey bool+ valueSeen bool+}++func newFieldScanner(keep []string) *fieldScanner {+ s := &fieldScanner{keep: map[string]bool{}, found: map[string][]byte{}}+ for _, p := range keep {+ s.keep[p] = true+ if d := strings.Count(p, ".") + 1; d > s.maxDepth {+ s.maxDepth = d+ }+ }+ return s+}++func (s *fieldScanner) reset() {+ s.stack, s.inStr, s.esc, s.strKey, s.bad, s.opened = s.stack[:0], false, false, false, false, false+ s.capturing, s.capBuf, s.overflow = false, s.capBuf[:0], false+ clear(s.found)+}++// path of the value about to start at the current position ("" if not keepable).+func (s *fieldScanner) valuePath() string {+ if len(s.stack) == 0 || len(s.stack) > s.maxDepth {+ return ""+ }+ parts := make([]string, 0, len(s.stack))+ for _, f := range s.stack {+ if !f.obj || f.keyLong {+ return "" // array elements and over-long keys never match+ }+ parts = append(parts, string(f.key))+ }+ p := strings.Join(parts, ".")+ if !s.keep[p] {+ return ""+ }+ return p+}++// startValue is called when a value begins at the top frame.+func (s *fieldScanner) startValue(c byte, kind byte) {+ top := &s.stack[len(s.stack)-1]+ if top.valueSeen {+ return+ }+ top.valueSeen = true+ if s.capturing {+ return+ }+ if p := s.valuePath(); p != "" {+ s.capturing, s.capPath, s.capDepth, s.capKind = true, p, len(s.stack), kind+ s.capBuf = append(s.capBuf[:0], c)+ }+}++func (s *fieldScanner) appendCap(c byte) {+ if len(s.capBuf) >= MaxFieldValue {+ s.overflow = true+ return+ }+ s.capBuf = append(s.capBuf, c)+}++func (s *fieldScanner) finishCap() {+ if !s.overflow {+ s.found[s.capPath] = append([]byte(nil), s.capBuf...)+ }+ s.capturing = false+}++func (s *fieldScanner) feed(c byte) {+ if s.bad {+ return+ }+ if s.inStr {+ if s.capturing {+ s.appendCap(c)+ }+ switch {+ case s.esc:+ s.esc = false+ case c == '\\':+ s.esc = true+ case c == '"':+ s.inStr = false+ if s.strKey {+ return+ }+ if s.capturing && s.capKind == '"' && len(s.stack) == s.capDepth {+ s.finishCap()+ }+ return+ }+ if s.strKey {+ top := &s.stack[len(s.stack)-1]+ if len(top.key) < maxKeyLen {+ top.key = append(top.key, c)+ } else {+ top.keyLong = true+ }+ }+ return+ }++ // A literal (number / true / false / null) ends at the next delimiter.+ if s.capturing && s.capKind == 'l' && len(s.stack) == s.capDepth {+ switch c {+ case ',', '}', ']', ' ', '\t', '\r':+ s.finishCap()+ default:+ s.appendCap(c)+ return+ }+ }++ switch c {+ case ' ', '\t', '\r':+ if s.capturing {+ s.appendCap(c)+ }+ case '{', '[':+ if !s.opened {+ if c != '{' {+ s.bad = true+ return+ }+ s.opened = true+ } else if len(s.stack) == 0 {+ s.bad = true // content after the top-level object+ return+ } else {+ if s.capturing {+ s.appendCap(c)+ }+ s.startValue(c, c)+ }+ s.stack = append(s.stack, frame{obj: c == '{', expectKey: c == '{'})+ case '}', ']':+ if len(s.stack) == 0 {+ s.bad = true+ return+ }+ if s.capturing {+ s.appendCap(c)+ }+ s.stack = s.stack[:len(s.stack)-1]+ if s.capturing && s.capKind != 'l' && len(s.stack) == s.capDepth {+ s.finishCap()+ }+ case ':':+ if len(s.stack) == 0 || !s.stack[len(s.stack)-1].obj {+ s.bad = true+ return+ }+ if s.capturing {+ s.appendCap(c)+ }+ top := &s.stack[len(s.stack)-1]+ top.expectKey, top.valueSeen = false, false+ case ',':+ if len(s.stack) == 0 {+ s.bad = true+ return+ }+ if s.capturing {+ s.appendCap(c)+ }+ top := &s.stack[len(s.stack)-1]+ top.valueSeen = false+ if top.obj {+ top.expectKey, top.key, top.keyLong = true, top.key[:0], false+ }+ case '"':+ if len(s.stack) == 0 {+ s.bad = true+ return+ }+ s.inStr = true+ if s.capturing {+ s.appendCap(c) // opening quote of a key or a string inside the captured value+ }+ top := &s.stack[len(s.stack)-1]+ if top.obj && top.expectKey {+ s.strKey = true+ top.key, top.keyLong = top.key[:0], false+ return+ }+ s.strKey = false+ s.startValue(c, '"')+ default: // start (or continuation) of a literal+ if len(s.stack) == 0 {+ s.bad = true+ return+ }+ if s.capturing {+ s.appendCap(c)+ }+ s.startValue(c, 'l')+ }+}++// endLine returns the kept fields of a well-formed line as one compact JSON+// object, nested by path ("message.usage" -> {"message":{"usage":...}}).+func (s *fieldScanner) endLine() ([]byte, bool) {+ if s.capturing && s.capKind == 'l' && len(s.stack) == s.capDepth {+ s.finishCap()+ }+ if s.bad || !s.opened || len(s.stack) != 0 || s.inStr || len(s.found) == 0 {+ return nil, false+ }+ root := map[string]any{}+ for p, raw := range s.found {+ node := root+ parts := strings.Split(p, ".")+ for _, k := range parts[:len(parts)-1] {+ child, ok := node[k].(map[string]any)+ if !ok {+ child = map[string]any{}+ node[k] = child+ }+ node = child+ }+ node[parts[len(parts)-1]] = json.RawMessage(raw)+ }+ obj, err := json.Marshal(root)+ if err != nil {+ return nil, false+ }+ return obj, true+}diff --git a/cli/internal/coding/tailfile/tailfile_test.go b/cli/internal/coding/tailfile/tailfile_test.gonew file mode 100644index 00000000..f8c6ae1e--- /dev/null+++ b/cli/internal/coding/tailfile/tailfile_test.go@@ -0,0 +1,153 @@+package tailfile++import (+ "encoding/json"+ "errors"+ "io"+ "runtime"+ "strings"+ "testing"+ "testing/iotest"+)++var keep = []string{"type", "requestId", "message.id", "message.model", "message.usage"}++func collect(t *testing.T, src io.Reader) ([]map[string]any, error) {+ t.Helper()+ var got []map[string]any+ err := eachJSONLFields(src, keep, func(obj []byte) {+ var m map[string]any+ if err := json.Unmarshal(obj, &m); err != nil {+ t.Fatalf("fn got invalid JSON %q: %v", obj, err)+ }+ got = append(got, m)+ })+ return got, err+}++func usageOf(t *testing.T, m map[string]any) map[string]any {+ t.Helper()+ msg, _ := m["message"].(map[string]any)+ u, _ := msg["usage"].(map[string]any)+ return u+}++func TestEachJSONLFieldsKeepsOnlyRequestedFields(t *testing.T) {+ src := strings.NewReader(+ `{"type":"assistant","requestId":"req_1","uuid":"x","message":{"id":"msg_1","model":"claude-opus-5-5","content":[{"type":"text","text":"hi"}],"usage":{"input_tokens":3,"output_tokens":4}}}` + "\n" ++ `{"type":"user","message":{"role":"user","content":"no usage here"}}` + "\n")+ got, err := collect(t, src)+ if err != nil {+ t.Fatal(err)+ }+ if len(got) != 2 {+ t.Fatalf("got %d objects, want 2", len(got))+ }+ if got[0]["requestId"] != "req_1" || got[0]["uuid"] != nil {+ t.Fatalf("first object = %v", got[0])+ }+ msg := got[0]["message"].(map[string]any)+ if msg["id"] != "msg_1" || msg["model"] != "claude-opus-5-5" || msg["content"] != nil {+ t.Fatalf("message = %v", msg)+ }+ if u := usageOf(t, got[0]); u["input_tokens"] != 3.0 || u["output_tokens"] != 4.0 {+ t.Fatalf("usage = %v", u)+ }+ if usageOf(t, got[1]) != nil {+ t.Fatalf("user line should carry no usage: %v", got[1])+ }+}++// A line far longer than any buffer (a big Write tool input) is scanned, not+// held, and its usage — which comes after the content — is still found.+func TestEachJSONLFieldsHandlesHugeLines(t *testing.T) {+ big := strings.Repeat("x", 20*1024*1024)+ line := `{"type":"assistant","message":{"id":"msg_1","content":[{"type":"tool_use","input":{"content":"` + big ++ `"}}],"usage":{"input_tokens":7,"output_tokens":8}}}`+ got, err := collect(t, strings.NewReader(line+"\n"))+ if err != nil {+ t.Fatal(err)+ }+ if len(got) != 1 || usageOf(t, got[0])["output_tokens"] != 8.0 {+ t.Fatalf("got %v", got)+ }+}++// "usage" inside a string value (e.g. code being written) is not the field.+func TestEachJSONLFieldsIgnoresUsageInsideStrings(t *testing.T) {+ line := `{"type":"assistant","message":{"id":"m","content":"print(\"usage\": {\"input_tokens\": 99})","usage":{"input_tokens":1}}}`+ got, err := collect(t, strings.NewReader(line))+ if err != nil {+ t.Fatal(err)+ }+ if len(got) != 1 || usageOf(t, got[0])["input_tokens"] != 1.0 {+ t.Fatalf("got %v", got)+ }+}++func TestEachJSONLFieldsHandlesEscapesAndNesting(t *testing.T) {+ line := `{"type":"assistant","message":{"id":"a\"b\\","content":[{"usage":{"input_tokens":5}}],"usage":{"input_tokens":2,"cache_creation":{"ephemeral_1h_input_tokens":6}}}}`+ got, err := collect(t, strings.NewReader(line))+ if err != nil {+ t.Fatal(err)+ }+ msg := got[0]["message"].(map[string]any)+ if msg["id"] != `a"b\` {+ t.Fatalf("id = %q", msg["id"])+ }+ u := usageOf(t, got[0])+ if u["input_tokens"] != 2.0 || u["cache_creation"].(map[string]any)["ephemeral_1h_input_tokens"] != 6.0 {+ t.Fatalf("usage = %v (the array element's usage must not win)", u)+ }+}++func TestEachJSONLFieldsSkipsMalformedLines(t *testing.T) {+ src := strings.NewReader("not json\n" + `{"type":"assistant","message":{"usage":{"input_tokens":1}` + "\n" + // truncated+ `[1,2]` + "\n" + `{"type":"assistant","message":{"usage":{"input_tokens":2}}}` + "\n")+ got, err := collect(t, src)+ if err != nil {+ t.Fatal(err)+ }+ if len(got) != 1 || usageOf(t, got[0])["input_tokens"] != 2.0 {+ t.Fatalf("got %v", got)+ }+}++func TestEachJSONLFieldsReportsOversizedKeptField(t *testing.T) {+ line := `{"type":"assistant","message":{"usage":{"note":"` + strings.Repeat("x", MaxFieldValue) + `"}}}`+ _, err := collect(t, strings.NewReader(line+"\n"+`{"type":"assistant","message":{"usage":{"input_tokens":1}}}`))+ if !errors.Is(err, ErrFieldTooLarge) {+ t.Fatalf("err = %v, want ErrFieldTooLarge", err)+ }+}++func TestEachJSONLFieldsPropagatesReadErrors(t *testing.T) {+ boom := errors.New("disk went away")+ src := io.MultiReader(strings.NewReader(`{"type":"assistant","message":{"usage":{"input_tokens":1}}}`+"\n"), iotest.ErrReader(boom))+ got, err := collect(t, src)+ if !errors.Is(err, boom) {+ t.Fatalf("err = %v, want the read error", err)+ }+ if len(got) != 1 {+ t.Fatalf("got %d objects before the error, want 1", len(got))+ }+}++// Scanning a huge line must not allocate in proportion to it.+func TestEachJSONLFieldsMemoryIsBounded(t *testing.T) {+ line := `{"type":"assistant","message":{"content":"` + strings.Repeat("x", 20*1024*1024) + `","usage":{"input_tokens":1}}}` + "\n"+ var before, after runtime.MemStats+ runtime.GC()+ runtime.ReadMemStats(&before)+ n := 0+ if err := eachJSONLFields(strings.NewReader(line), keep, func([]byte) { n++ }); err != nil {+ t.Fatal(err)+ }+ runtime.ReadMemStats(&after)+ if n != 1 {+ t.Fatalf("got %d objects, want 1", n)+ }+ if alloc := after.TotalAlloc - before.TotalAlloc; alloc > 2<<20 {+ t.Fatalf("allocated %d bytes scanning a 20 MiB line, want < 2 MiB", alloc)+ }+}