Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Merge main into token-cost-estimates

Claudec9072c060bb3
diff --cc cli/beacon/internal/factorysession/usage_test.go
index 00000000,660d0e82..e09871cc
mode 000000,100644..100644
--- a/cli/beacon/internal/factorysession/usage_test.go
+++ b/cli/beacon/internal/factorysession/usage_test.go
@@@ -1,0 -1,282 +1,293 @@@
+ package factorysession
+
+ import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/tokens"
+ )
+
+ // usageFixture is one Factory session on disk whose settings file a test rewrites between sweeps,
+ // the way Droid rewrites <session-id>.settings.json with new cumulative totals after each turn.
+ type usageFixture struct {
+ t *testing.T
+ root string
+ sessionPath string
+ settingsPath string
+ statePath string
+ out bytes.Buffer
+ settingsTick time.Time
+ }
+
+ func newUsageFixture(t *testing.T) *usageFixture {
+ t.Helper()
+ root := t.TempDir()
+ project := filepath.Join(root, "-repo")
+ if err := os.MkdirAll(project, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ f := &usageFixture{
+ t: t,
+ root: root,
+ sessionPath: filepath.Join(project, "session-1.jsonl"),
+ settingsPath: filepath.Join(project, "session-1.settings.json"),
+ statePath: filepath.Join(t.TempDir(), "factory-state.json"),
+ settingsTick: time.Unix(1770000000, 0),
+ }
+ writeFile(t, f.sessionPath,
+ `{"type":"session_start","id":"session-1","title":"hi","cwd":"/repo"}`+"\n"+
+ `{"type":"message","id":"u1","timestamp":"2026-05-16T22:11:04.611Z","message":{"role":"user","content":[{"type":"text","text":"hello"}]}}`+"\n")
+ return f
+ }
+
+ // writeSettings replaces the settings file and moves its mtime forward a second, so every
+ // rewrite is visible to the collector even on filesystems with coarse timestamps.
+ func (f *usageFixture) writeSettings(body string) {
+ f.t.Helper()
+ writeFile(f.t, f.settingsPath, body)
+ f.settingsTick = f.settingsTick.Add(time.Second)
+ if err := os.Chtimes(f.settingsPath, f.settingsTick, f.settingsTick); err != nil {
+ f.t.Fatal(err)
+ }
+ }
+
+ func (f *usageFixture) sweep() Summary {
+ f.t.Helper()
+ summary, err := CollectOnce(CollectOptions{SessionsDir: f.root, StatePath: f.statePath, Print: true, Out: &f.out})
+ if err != nil {
+ f.t.Fatal(err)
+ }
+ return summary
+ }
+
+ func (f *usageFixture) events() []schema.Event {
+ f.t.Helper()
+ var events []schema.Event
+ scanner := bufio.NewScanner(bytes.NewReader(f.out.Bytes()))
+ scanner.Buffer(make([]byte, 0, 64*1024), 4<<20)
+ for scanner.Scan() {
+ var ev schema.Event
+ if err := json.Unmarshal(scanner.Bytes(), &ev); err != nil {
+ f.t.Fatal(err)
+ }
+ events = append(events, ev)
+ }
+ return events
+ }
+
+ func (f *usageFixture) usageEvents() []schema.Event {
+ var out []schema.Event
+ for _, ev := range f.events() {
+ if ev.Event.Action == "token.usage" {
+ out = append(out, ev)
+ }
+ }
+ return out
+ }
+
+ // Factory keeps tokenUsage as the session's running total, so the sum of what Beacon emits across
+ // several settings rewrites must equal the final total, not the sum of every total it saw.
+ func TestCollectOnceEmitsFactorySettingsUsageAsDeltas(t *testing.T) {
+ f := newUsageFixture(t)
+ f.writeSettings(`{"model":"claude-opus-4-7","tokenUsage":{"inputTokens":100,"outputTokens":10,"cacheCreationTokens":1000,"cacheReadTokens":5000,"thinkingTokens":4,"factoryCredits":1.5}}`)
+ f.sweep()
+ f.writeSettings(`{"model":"claude-opus-4-7","tokenUsage":{"inputTokens":250,"outputTokens":30,"cacheCreationTokens":1200,"cacheReadTokens":12000,"thinkingTokens":9,"factoryCredits":3.25}}`)
+ f.sweep()
+ // A rewrite that leaves the totals alone (Droid also stores the model, active time and
+ // other settings there) is not new spend.
+ f.writeSettings(`{"model":"claude-opus-4-7","assistantActiveTimeMs":900,"tokenUsage":{"inputTokens":250,"outputTokens":30,"cacheCreationTokens":1200,"cacheReadTokens":12000,"thinkingTokens":9,"factoryCredits":3.25}}`)
+ if summary := f.sweep(); summary.EventsEmitted != 0 {
+ t.Fatalf("unchanged totals emitted %d events, want 0", summary.EventsEmitted)
+ }
+ f.writeSettings(`{"model":"claude-opus-4-7","tokenUsage":{"inputTokens":400,"outputTokens":75,"cacheCreationTokens":1500,"cacheReadTokens":20000,"thinkingTokens":20,"factoryCredits":6}}`)
+ f.sweep()
+ // A re-run over a store that has not changed emits nothing.
+ if summary := f.sweep(); summary.EventsEmitted != 0 {
+ t.Fatalf("idempotent re-run emitted %d events, want 0", summary.EventsEmitted)
+ }
+
+ report := tokens.Aggregate(f.events(), tokens.Options{})
+ want := tokens.Usage{
+ InputTokens: 400,
+ OutputTokens: 75,
+ CacheCreationInputTokens: 1500,
+ CacheReadInputTokens: 20000,
+ ReasoningOutputTokens: 20,
+ Events: 3,
+ }
- if report.Totals != want {
- t.Fatalf("aggregated totals = %+v, want the final cumulative value %+v", report.Totals, want)
++ // Compare the counts only: the report also carries a list-price estimate, which this test
++ // does not pin.
++ got := tokens.Usage{
++ InputTokens: report.Totals.InputTokens,
++ OutputTokens: report.Totals.OutputTokens,
++ CacheCreationInputTokens: report.Totals.CacheCreationInputTokens,
++ CacheReadInputTokens: report.Totals.CacheReadInputTokens,
++ ReasoningOutputTokens: report.Totals.ReasoningOutputTokens,
++ CostUSD: report.Totals.CostUSD,
++ Events: report.Totals.Events,
++ }
++ if got != want {
++ t.Fatalf("aggregated totals = %+v, want the final cumulative value %+v", got, want)
+ }
+
+ usage := f.usageEvents()
+ if len(usage) != 3 {
+ t.Fatalf("token.usage events = %d, want 3", len(usage))
+ }
+ ids := map[string]bool{}
+ var credits float64
+ for _, ev := range usage {
+ if ids[ev.Event.ID] {
+ t.Fatalf("duplicate event id %s", ev.Event.ID)
+ }
+ ids[ev.Event.ID] = true
+ if ev.GenAI.Usage.CostUSD != nil {
+ t.Fatalf("Factory credits are not USD; cost_usd = %v", *ev.GenAI.Usage.CostUSD)
+ }
+ credits += ev.Raw["factory_credits_delta"].(float64)
+ }
+ if credits != 6 {
+ t.Fatalf("summed factory_credits_delta = %v, want the final total 6", credits)
+ }
+ last := usage[len(usage)-1]
+ if got := last.Raw["factory_credits_total"]; got != 6.0 {
+ t.Fatalf("factory_credits_total = %v, want 6", got)
+ }
+ if got := *last.GenAI.Usage.InputTokens; got != 150 {
+ t.Fatalf("last delta input = %d, want 150", got)
+ }
+ }
+
+ // A field that goes down is a rewritten settings file, not negative spend. It emits nothing for
+ // that field and the lower value becomes the baseline, so growth after the rewrite still counts.
+ func TestCollectOnceFactoryUsageDecreaseRebases(t *testing.T) {
+ f := newUsageFixture(t)
+ f.writeSettings(`{"tokenUsage":{"inputTokens":100,"outputTokens":50}}`)
+ f.sweep()
+ f.writeSettings(`{"tokenUsage":{"inputTokens":40,"outputTokens":60}}`)
+ f.sweep()
+ f.writeSettings(`{"tokenUsage":{"inputTokens":70,"outputTokens":60}}`)
+ f.sweep()
+
+ report := tokens.Aggregate(f.events(), tokens.Options{})
+ if report.Totals.InputTokens != 130 || report.Totals.OutputTokens != 60 {
+ t.Fatalf("totals = %+v, want input 100+0+30=130 and output 50+10=60", report.Totals)
+ }
+ for _, ev := range f.usageEvents() {
+ u := ev.GenAI.Usage
+ if (u.InputTokens != nil && *u.InputTokens < 0) || (u.OutputTokens != nil && *u.OutputTokens < 0) {
+ t.Fatalf("negative delta emitted: %+v", u)
+ }
+ }
+ }
+
+ // inclusiveTokenUsage adds the usage of the session's child (subagent) sessions, which Factory
+ // stores as sessions of their own and Beacon collects separately. Counting it on the parent would
+ // count every child twice, so it is used only when it cannot include a child.
+ func TestCollectOnceFactoryInclusiveUsageNeverDoubleCountsChildren(t *testing.T) {
+ f := newUsageFixture(t)
+ // Older settings without tokenUsage and no children: inclusive is the session's own usage.
+ f.writeSettings(`{"inclusiveTokenUsage":{"inputTokens":100,"outputTokens":10}}`)
+ f.sweep()
+ // tokenUsage appears alongside an inclusive total that now counts a child session.
+ f.writeSettings(`{"tokenUsage":{"inputTokens":150,"outputTokens":20},"inclusiveTokenUsage":{"inputTokens":900,"outputTokens":90},"childInclusiveTokenUsageBySessionId":{"child-1":{"inputTokens":750,"outputTokens":70}}}`)
+ f.sweep()
+ // Only the inclusive total is present while a child is: nothing of the parent's own usage
+ // can be separated out, so nothing is emitted and the baseline stays.
+ f.writeSettings(`{"inclusiveTokenUsage":{"inputTokens":1000,"outputTokens":100},"childInclusiveTokenUsageBySessionId":{"child-1":{"inputTokens":800,"outputTokens":75}}}`)
+ if summary := f.sweep(); summary.EventsEmitted != 0 {
+ t.Fatalf("inclusive-with-children sweep emitted %d events, want 0", summary.EventsEmitted)
+ }
+ f.writeSettings(`{"tokenUsage":{"inputTokens":200,"outputTokens":25},"inclusiveTokenUsage":{"inputTokens":1000,"outputTokens":100},"childInclusiveTokenUsageBySessionId":{"child-1":{"inputTokens":800,"outputTokens":75}}}`)
+ f.sweep()
+
+ report := tokens.Aggregate(f.events(), tokens.Options{})
+ if report.Totals.InputTokens != 200 || report.Totals.OutputTokens != 25 {
+ t.Fatalf("totals = %+v, want the session's own final usage 200/25", report.Totals)
+ }
+ }
+
+ // State written before Beacon tracked the totals already holds a token.usage event for the
+ // settings it last saw. Upgrading must not count that total a second time.
+ func TestCollectOnceFactoryUsageUpgradeSeedsBaselineFromUnchangedSettings(t *testing.T) {
+ f := newUsageFixture(t)
+ f.writeSettings(`{"tokenUsage":{"inputTokens":100,"outputTokens":10}}`)
+ f.sweep()
+
+ state, err := LoadState(f.statePath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // What the previous release stored: the settings mtime, no totals.
+ state.Sessions["session-1"].Usage = nil
+ if err := state.Save(f.statePath); err != nil {
+ t.Fatal(err)
+ }
+ if summary := f.sweep(); summary.EventsEmitted != 0 {
+ t.Fatalf("upgrade sweep emitted %d events, want 0", summary.EventsEmitted)
+ }
+ f.writeSettings(`{"tokenUsage":{"inputTokens":160,"outputTokens":12}}`)
+ f.sweep()
+
+ report := tokens.Aggregate(f.events(), tokens.Options{})
+ if report.Totals.InputTokens != 160 || report.Totals.OutputTokens != 12 {
+ t.Fatalf("totals = %+v, want 160/12", report.Totals)
+ }
+ }
+
+ // The collector re-reads a session whose file moved (a renamed working directory) from the top,
+ // but its settings totals are the same running figure, so they are not counted again.
+ func TestCollectOnceFactoryUsageSurvivesAMovedSession(t *testing.T) {
+ f := newUsageFixture(t)
+ f.writeSettings(`{"tokenUsage":{"inputTokens":100,"outputTokens":10}}`)
+ f.sweep()
+
+ moved := filepath.Join(f.root, "-repo2")
+ if err := os.Rename(filepath.Dir(f.sessionPath), moved); err != nil {
+ t.Fatal(err)
+ }
+ f.sessionPath = filepath.Join(moved, "session-1.jsonl")
+ f.settingsPath = filepath.Join(moved, "session-1.settings.json")
+ f.sweep()
+ f.writeSettings(`{"tokenUsage":{"inputTokens":130,"outputTokens":10}}`)
+ f.sweep()
+
+ report := tokens.Aggregate(f.events(), tokens.Options{})
+ if report.Totals.InputTokens != 130 || report.Totals.OutputTokens != 10 {
+ t.Fatalf("totals = %+v, want 130/10", report.Totals)
+ }
+ }
+
+ // A write that fails leaves the totals uncommitted, so the retry emits the same delta again
+ // rather than losing it.
+ func TestCollectOnceFactoryUsageRetriesAfterFailedWrite(t *testing.T) {
+ f := newUsageFixture(t)
+ f.writeSettings(`{"tokenUsage":{"inputTokens":100,"outputTokens":10}}`)
+ f.sweep()
+ f.writeSettings(`{"tokenUsage":{"inputTokens":180,"outputTokens":15}}`)
+
+ original := emitEvent
+ emitEvent = func(event schema.Event, opts CollectOptions) error {
+ if event.Event.Action == "token.usage" {
+ return os.ErrPermission
+ }
+ return original(event, opts)
+ }
+ _, err := CollectOnce(CollectOptions{SessionsDir: f.root, StatePath: f.statePath, Print: true, Out: &f.out})
+ emitEvent = original
+ if err == nil {
+ t.Fatal("expected the failed write to surface")
+ }
+ f.sweep()
+
+ report := tokens.Aggregate(f.events(), tokens.Options{})
+ if report.Totals.InputTokens != 180 || report.Totals.OutputTokens != 15 {
+ t.Fatalf("totals = %+v, want 180/15", report.Totals)
+ }
+ }
diff --cc cli/beacon/internal/tokens/tokens.go
index 26dd5517,d6660837..42200fc6
--- a/cli/beacon/internal/tokens/tokens.go
+++ b/cli/beacon/internal/tokens/tokens.go
@@@ -147,9 -111,11 +147,14 @@@ type Report struct
SessionDetail *SessionDetail `json:"session_detail,omitempty"`
EventsWithUsage int `json:"events_with_usage"`
TotalEvents int `json:"total_events"`
+ // Pricing says where every estimate in the report came from: the catalog, the rates each
+ // model was priced at, and the models it could not price.
+ Pricing *PricingSummary `json:"pricing,omitempty"`
+ // SuppressedPollEvents counts poll-backfill usage events (collection_method=poll) dropped
+ // because a live channel had already reported the same session's usage from that point on.
+ // They are left out of every total and of EventsWithUsage; the count is here so a reader can
+ // see the overlap was found rather than wonder where a backfill's rows went.
+ SuppressedPollEvents int `json:"suppressed_poll_events,omitempty"`
}
// usageEvent is one usage-bearing event with its usage normalized to a delta
@@@ -179,14 -145,13 +184,17 @@@ type usageEvent struct
cumulative bool
metricName string
rawSource string
- // collectionMethod, codexSessionSource and tokenSource are what requestScoped (cost.go)
- // reads to tell one model request from a sum of several.
+ sourceStart time.Time
+ seriesField string
+ seriesValue float64
+
+ // collectionMethod is harness.collection_method, lowercased: whether this usage came from
- // live capture or from a poll of the runtime's own session store.
- collectionMethod string
++ // live capture or from a poll of the runtime's own session store. requestScoped (cost.go)
++ // also reads it, with codexSessionSource and tokenSource, to tell one model request from a
++ // sum of several.
+ collectionMethod string
+ codexSessionSource string
+ tokenSource string
- sourceStart time.Time
- seriesField string
- seriesValue float64
}
// Aggregate builds a token usage report from endpoint events. Events from
@@@ -209,14 -177,8 +220,12 @@@ func aggregate(events []schema.Event, o
opts.NearLimitRatio = defaultNearLimitRatio
}
report := Report{TotalEvents: len(events)}
- usageEvents := collectUsageEvents(events, sessionUsers)
- usageEvents = preferCodexTurnSpans(usageEvents)
- usageEvents = dedupeOverlappingChannels(usageEvents)
- resolveCumulativeSeries(usageEvents)
+ usageEvents, suppressed := resolveUsageEvents(events, sessionUserContexts(contextEvents), liveCaptureStarts(contextEvents, events))
+ report.SuppressedPollEvents = suppressed
+ // Last, on the final per-event deltas: an estimate made before the passes above would price
+ // tokens they are about to remove as duplicates, and price a cumulative counter's running
+ // total instead of its interval delta.
+ report.Pricing = priceUsageEvents(usageEvents, priceCatalog())
byModel := map[string]*Usage{}
bySession := map[string]*Usage{}
diff --cc docs/cli/token-usage.mdx
index 1b7e2410,9e10662d..25683439
--- a/docs/cli/token-usage.mdx
+++ b/docs/cli/token-usage.mdx
@@@ -23,10 -23,10 +23,10 @@@ Beacon normalizes token telemetry from
A few rules shape how the numbers read:
- - **Counts are kept disjoint.** `input_tokens`, `output_tokens`, `cache_read`, and `cache_creation` do not overlap, so a total is `input + output + cache_read + cache_creation` without double-counting. Codex reports input inclusive of both cache reads and cache writes, so Beacon subtracts both before storing uncached input.
+ - **Counts are kept disjoint.** `input_tokens`, `output_tokens`, `cache_read`, and `cache_creation` do not overlap, so a total is `input + output + cache_read + cache_creation` without double-counting. Codex reports input inclusive of both cache reads and cache writes, so Beacon subtracts both before storing uncached input. The same applies to any source using the OpenTelemetry GenAI names (`gen_ai.usage.input_tokens`, or the older `prompt_tokens` spellings), whose input includes cached tokens by definition: Beacon subtracts the cache read and cache creation counts reported alongside it, unless the input is smaller than they are and so is already uncached. Claude Code's own counts are already disjoint and are stored as reported.
-- **Cost is runtime-reported only.** `cost_usd` reflects what the runtime emits. Beacon never derives cost from local pricing tables, so cost appears only for runtimes that report it.
+- **Reported and estimated cost are kept apart.** `cost_usd` is what the runtime reported and nothing else. Beside it, every report carries a list-price estimate (`estimated_cost_usd`) for every runtime, including the ones that report no cost, and an effective cost that picks one of the two. See [Cost: reported, estimated, effective](#cost-reported-estimated-effective). Events never carry an estimate; it is computed when the report is built.
- **Attribution depends on the metadata a runtime attaches.** Model, harness, repository, and run grouping work whenever those fields are present. Per-session grouping (`--session`, the session step tree) requires a session or conversation id on the usage signal. Per-user grouping uses the event user directly or a matching session context event from a local hook.
- - **Duplicate channels are de-duplicated.** When a runtime emits the same usage on more than one signal (Claude Code reports it on both logs and metrics), Beacon counts it once.
+ - **Duplicate channels are de-duplicated.** When a runtime emits the same usage on more than one signal (Claude Code reports it on both logs and metrics), Beacon counts it once. The same holds when one session is read both live (OTLP, a hook, or a Beacon-managed plugin) and by a `beacon endpoint <runtime> sync` backfill of the runtime's own session store (`harness.collection_method=poll`), as Claude Code, Codex CLI, Cline, OpenCode, OpenClaw Gateway and Pi sessions can be. Within one session on one endpoint, live capture is authoritative from the moment its channel started capturing that session: backfilled usage from then on gives up the fields the live channel reports and is dropped once nothing is left, while backfilled usage from before it is kept, since it is the only record of turns from before Beacon's live capture started. Usage without a session id, or without a usable timestamp, is never matched this way, and a live channel that reports only context occupancy does not suppress anything. `--json` reports how many backfilled events were dropped as `suppressed_poll_events`.
### Per-runtime coverage
diff --cc docs/telemetry-schema/normalization.mdx
index 86842de6,02f8db02..8c0136bb
--- a/docs/telemetry-schema/normalization.mdx
+++ b/docs/telemetry-schema/normalization.mdx
@@@ -185,7 -185,9 +185,9 @@@ Beacon preserves OpenTelemetry GenAI se
| `gen_ai.usage.reasoning.output_tokens` | `gen_ai.usage.reasoning.output_tokens` |
| `gen_ai.usage.cost`, runtime `*.cost.usage` metric datapoints | `gen_ai.usage.cost_usd` |
+ Beacon's `gen_ai.usage.input_tokens` is uncached input only, disjoint from cache reads and cache creation. The OTel GenAI semantic conventions define the source attribute the other way: `gen_ai.usage.input_tokens` "SHOULD include all types of input tokens, including cached tokens", and each cache count "SHOULD be included in `gen_ai.usage.input_tokens`". OpenAI-style `prompt_tokens` likewise counts cached prompt tokens. So for those three names Beacon stores the input count minus the cache read and cache creation counts on the same span or log record. If the input is smaller than those cache counts it cannot contain them, the source is already reporting uncached input, and it is stored unchanged. Claude Code's bare `input_tokens` attribute is Anthropic's uncached count and is never reduced, and `raw` keeps the value as reported. The semconv `gen_ai.client.token.usage` metric has only `input` and `output` token types, so its input datapoint has no cache count to reduce and is stored as reported.
+
-Usage also arrives as metric datapoints, not just span and log attributes. Claude Code's `claude_code.token.usage` and `claude_code.cost.usage` are expanded per datapoint. Codex's live source is its completed turn span, whose input, cache-read, cache-write, output, and reasoning attributes map into the same `gen_ai.usage` fields with `thread.id` promoted to `session.id`. Legacy `codex.turn.token_usage` metric rows remain supported for older logs. `gen_ai.usage.cost_usd` carries runtime-reported cost only and is never derived from a local pricing table. See [How token attribution works](/cli/token-usage#how-token-attribution-works) for how these normalized fields aggregate into local reports.
+Usage also arrives as metric datapoints, not just span and log attributes. Claude Code's `claude_code.token.usage` and `claude_code.cost.usage` are expanded per datapoint. Codex's live source is its completed turn span, whose input, cache-read, cache-write, output, and reasoning attributes map into the same `gen_ai.usage` fields with `thread.id` promoted to `session.id`. Legacy `codex.turn.token_usage` metric rows remain supported for older logs. `gen_ai.usage.cost_usd` carries runtime-reported cost only and is never derived from a local pricing table. Token reports add a separately labeled list-price estimate at report time; it is never written into an event (see [Cost: reported, estimated, effective](/cli/token-usage#cost-reported-estimated-effective)). See [How token attribution works](/cli/token-usage#how-token-attribution-works) for how these normalized fields aggregate into local reports.
## Prompt and content handling