The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
package tokens
import (
"sort"
"strings"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)
// Token spend coverage: which runtimes on this machine are contributing token telemetry, and
// which are not.
//
// A token report answers "what did I spend". It cannot answer "is this all of it", and that is
// the question that matters before anyone treats a number as a bill. A runtime whose usage never
// arrives is indistinguishable, in every rollup Beacon prints, from a runtime nobody used: both
// are simply absent. So a total can quietly omit an entire agent and still look complete.
//
// Coverage answers the second question by joining what is installed against what actually
// reported, and it exists because the alternative is a reader inferring coverage from a docs
// table. The table describes what Beacon is built to read; only the log knows what this machine
// produced.
//
// The expectation table below is what keeps the report from crying wolf. Cursor's hooks carry no
// token counts at all, so Cursor contributing nothing is correct behavior, not a fault -- reported
// as NotInstrumented rather than Silent. Silent is reserved for the case worth investigating: a
// runtime that ran, that Beacon is built to read tokens from, and that reported none anyway.
// Coverage status values.
const (
// CoverageCovered: the runtime reported usage in the window. Nothing to do.
CoverageCovered = "covered"
// CoverageSilent: the runtime produced events but no usage, and Beacon is built to read
// usage from it. This is the actionable one -- a broken hook, an old runtime build, or a
// capture path that regressed.
CoverageSilent = "silent"
// CoverageInactive: the runtime is installed but produced no events at all in the window.
// Usually means it was not used; can also mean its hooks are not installed.
CoverageInactive = "inactive"
// CoverageNotInstrumented: the runtime produced events, and Beacon cannot read token usage
// from it. Absence of tokens here is expected and is not a fault to chase.
CoverageNotInstrumented = "not_instrumented"
)
// Expectation values: what Beacon can read from a runtime, independent of what it did read.
const (
// ExpectReported: usage is collected from this runtime and verified against captured
// payloads.
ExpectReported = "reported"
// ExpectGenericOTLP: usage is collected only if the runtime emits the OpenTelemetry GenAI
// semconv names. The generic path is tested; these runtimes' own payloads are not pinned,
// so silence here may mean the runtime spells its attributes differently rather than that
// anything is broken.
ExpectGenericOTLP = "generic_otlp"
// ExpectNone: the runtime exposes no token counts on the surface Beacon collects from.
ExpectNone = "none"
)
// usageExpectation records, per canonical harness name, whether Beacon can read token usage from
// that runtime and why. Keys are canonical harness names, so callers must normalize before
// looking up.
//
// This duplicates the per-runtime table in docs/cli/token-usage.mdx by necessity: the docs are for
// a reader deciding whether to expect numbers, this is for the report deciding whether silence is
// a fault. They must be kept in step, and the reason strings are written to be printed.
var usageExpectation = map[string]struct {
expect string
reason string
}{
"claude_code": {ExpectReported, "OTLP token and cost telemetry"},
"codex_cli": {ExpectReported, "per-turn usage trace; Codex emits no cost"},
"codex_desktop": {ExpectReported, "per-turn usage trace; Codex emits no cost"},
"claude_cowork": {ExpectReported, "OTLP token and cost telemetry"},
"cline": {ExpectReported, "plugin reports usage once per task"},
"opencode": {ExpectReported, "plugin reports usage per assistant message"},
"pi_cli": {ExpectReported, "extension reports usage and cost"},
"omp": {ExpectReported, "extension reports usage and cost"},
"prime_agent": {ExpectReported, "extension reports usage and cost"},
// Senpi's extension forwards the same finalized-message usage object as the rest of the Pi
// family, through the same mapper (cli/beacon-hooks/cmd/pi_family.go). Without its own key it
// fell to the unrecognized default and was described as a runtime nobody had looked at.
"omo_senpi": {ExpectReported, "extension reports usage and cost"},
// Reported live by the managed plugin's `llm_output` hook, the only OpenClaw hook that carries
// usage. It is a conversation hook, so OpenClaw withholds it from a non-bundled plugin until
// `plugins.entries.beacon-endpoint.hooks.allowConversationAccess` is set -- which is why an
// OpenClaw endpoint can be collecting everything else live and reporting no tokens at all.
// `beacon endpoint hooks status --harness openclaw` is where that gap is surfaced. The
// session-file sync reads the usage OpenClaw commits on each assistant message
// (openclawsession/records.go), and needs no grant.
"openclaw_gateway": {ExpectReported, "plugin reports usage per model response when conversation access is granted; beacon endpoint integrations openclaw sessions sync reads it from session files"},
"vercel_fx": {ExpectReported, "session store carries cumulative usage and cost"},
"deepseek_harness": {ExpectReported, "native session backfill reports usage when DeepSeek persists it"},
// Read from the executor's own stream: OpenCode's per-step usage and cost, and Codex's per-turn
// thread totals, differenced. Neither stream names the model, so these tokens are unpriced
// unless OpenCode reported a cost.
"harness2": {ExpectReported, "rollout streams carry per-step (OpenCode) or per-turn (Codex) usage; OpenCode reports cost"},
"copilot_cli": {ExpectReported, "session store carries output tokens, cumulative model usage, and AI-credit cost"},
"asymptote_observe": {ExpectReported, "SDK spans carry semconv usage"},
// Read from Gemini CLI's own gemini_cli.api_response log, which it emits for every model
// response whenever its telemetry is on -- not from the semconv names -- so a Gemini session
// with events and no usage is a fault worth reporting. The record shape is pinned from Gemini
// CLI's telemetry source rather than from a recorded session.
"gemini_cli": {ExpectReported, "api_response log reports input, cache, thought and tool tokens; Gemini CLI emits no cost"},
// Runtimes whose usage reaches Beacon only through a session-store sync. Their hooks carry no
// token counts, so a window of hook activity with no usage is spend missing from the total --
// silent, which is right -- but the fix is running the sync rather than repairing a hook, so
// the note names the command.
//
// Hermes keeps running per-session totals, cost included, in its state database, and the sync
// differences them into token.usage deltas (hermessession/collect.go, mapUsage).
"hermes": {ExpectReported, "beacon endpoint hermes sync reads session totals and cost; hook payloads carry none"},
// Factory keeps running totals in each session's settings file, and the sync differences them
// (factorysession/mapper.go, emitSettingsUsage). Its OTLP export still contributes when it
// carries the semconv names; the sync is the path Beacon reads Factory's own record from.
"factory": {ExpectReported, "beacon endpoint factory sync reads session settings totals; OTLP only if it emits GenAI semconv usage"},
// The factory_droid spelling never comes from the sync, which writes harness "factory" and is
// not normalized onto this name. It is what a Factory OTLP export can name itself, so it keeps
// the generic-OTLP expectation rather than pointing at a command that cannot clear it.
"factory_droid": {ExpectGenericOTLP, "only if it emits OTel GenAI semconv usage"},
"vscode_copilot": {ExpectGenericOTLP, "only if it emits OTel GenAI semconv usage"},
// goose's chat spans carry the full semconv usage set, but they reach Beacon only over OTLP
// configured by hand, and its hook adapter carries none. Without its own key it fell to the
// unrecognized default.
"goose": {ExpectGenericOTLP, "chat spans carry GenAI semconv usage, but only over manually configured OTLP"},
// Grok Bot runs on a Cursor-hosted cloud computer and reaches Beacon only through Cursor's
// server-side OpenTelemetry export, so whether usage arrives depends on whether that export
// carries the semconv names -- the generic-OTLP case exactly, not a runtime Beacon reads.
"grok_bot": {ExpectGenericOTLP, "only if Cursor's server-side OTel export carries GenAI semconv usage"},
"cursor": {ExpectNone, "no usage on any hook payload; preCompact reports context only"},
// Antigravity CLI's hooks carry no token counts, but every planner response it commits to its
// transcript does, and `beacon endpoint antigravity sync` reads them -- same shape as Hermes
// above and Devin CLI below.
"antigravity_cli": {ExpectReported, "beacon endpoint antigravity sync reads per-response token counts; hook payloads carry none"},
"grok": {ExpectNone, "hook payloads carry no token counts"},
// Devin reaches the log under three names, and all three need an entry. The hook installer
// writes --platform devin-cli and devin-desktop, an older install still writes plain devin,
// and NormalizeHarnessName passes all three through unchanged. Keying only "devin" left the
// other two unrecognized, which classified a Devin session that correctly reported no tokens
// as silent -- the one status this table exists to keep clean.
"devin": {ExpectNone, "hook payloads carry no token counts"},
// Devin CLI's committed session store carries per-message metrics and the per-turn model,
// which the hook payload never sees -- same shape as DeepSeek Harness above.
"devin-cli": {ExpectReported, "beacon endpoint devin sync reads per-message token metrics; hook payloads carry none"},
"devin-desktop": {ExpectNone, "Cascade/Windsurf hook payloads carry no token counts"},
"muse_code": {ExpectNone, "usage arrives on PostLLMCall, which Beacon does not subscribe to"},
"qwen_code": {ExpectNone, "Stop reports context occupancy, not per-call usage"},
"claude_web": {ExpectNone, "no recorded claude.ai stream has carried a usage object"},
"chatgpt_web": {ExpectNone, "the chat stream reports no token counts"},
"openhands": {ExpectNone, "hook payloads carry no token counts"},
"kiro": {ExpectNone, "hook payloads carry no token counts"},
// Kimi Code assembles each hook payload from a fixed envelope plus per-event fields, and no
// usage count is among them on any of the twenty events it exposes. The runtime does count
// tokens -- the TUI shows context occupancy, and PreCompact carries a `token_count` for the
// history it is about to summarize -- but neither is per-call spend, so promoting either into
// gen_ai.usage would make a level look like an additive total. A Kimi Code session reporting
// no tokens is therefore correct rather than silent.
//
// Unlike DeepSeek Harness above, which moved to ExpectReported once it gained a session
// backfill, there is no poll path here to read usage from: Beacon collects Kimi Code through
// hooks only, and the runtime's own session store is not read.
"kimi_code": {ExpectNone, "hook payloads carry no token counts; PreCompact reports context only"},
}
// InstalledConfig is the part of a config-scanner row that install detection reads.
type InstalledConfig struct {
Runtime string
Path string
Kind string
BeaconManaged bool
Exists bool
}
// InstalledRuntimes selects the scanner rows that are evidence a runtime is wired up to report.
//
// Only rows Beacon itself configured count. Everything else -- a config file that merely exists --
// is not evidence of anything this report can use, and three rounds of review found three separate
// ways it lies: a shell profile exists on every machine with a shell, the workspace .mcp.json is
// read by several runtimes but filed under one, and VS Code's settings.json exists wherever VS Code
// does, Copilot Chat or not. Each produced a permanently inactive row for a product nobody had.
// Excluding them one at a time was losing; the shape of the mistake was using the scanner to answer
// a question it does not answer.
//
// The scanner reports which config files exist that some runtime might read. Coverage needs to know
// which runtimes Beacon set up to send telemetry, because that is the only population where silence
// is informative: Beacon wired it, so it should be reporting, so nothing arriving is worth a look.
// A runtime present on the machine but never configured by Beacon would not be reporting either
// way, and saying so every week is noise rather than coverage.
//
// beacon_managed is exactly that signal. It is set when a file carries Beacon's hook command, its
// managed-plugin marker, or an OTLP endpoint pointing at the local collector -- in each case
// because Beacon put it there.
//
// A runtime configured by hand, outside Beacon, is not counted. If it emits events it still appears
// in the report through the log; if it emits nothing, Beacon has no grounds to claim it should have.
func InstalledRuntimes(configs []InstalledConfig) []string {
seen := map[string]bool{}
var out []string
for _, config := range configs {
if !config.Exists || !config.BeaconManaged {
continue
}
name := strings.TrimSpace(config.Runtime)
if name == "" || seen[name] {
continue
}
seen[name] = true
out = append(out, name)
}
sort.Strings(out)
return out
}
// RuntimeCoverage is one runtime's line in the coverage report.
type RuntimeCoverage struct {
Harness string `json:"harness"`
Status string `json:"status"`
Expectation string `json:"expectation"`
Reason string `json:"reason,omitempty"`
Installed bool `json:"installed"`
Events int `json:"events"`
UsageEvents int `json:"usage_events"`
Tokens int64 `json:"tokens"`
}
// CoverageReport is the whole join: one line per runtime that is either installed or seen in the
// log, plus the counts a reader needs to judge the window itself.
type CoverageReport struct {
Runtimes []RuntimeCoverage `json:"runtimes"`
// Silent counts runtimes worth investigating. Zero is the good state.
Silent int `json:"silent"`
// Covered counts runtimes that contributed usage.
Covered int `json:"covered"`
// TotalEvents is the size of the window the join was computed over. A coverage report over
// an empty window says nothing, and a reader needs to be able to see that.
TotalEvents int `json:"total_events"`
}
// Coverage joins installed runtimes against the runtimes that actually reported token usage.
//
// installed carries runtime identifiers as the inventory scanner spells them; they are normalized
// here, so callers pass them through unchanged. A runtime seen in the log but not installed is
// still reported -- telemetry can arrive from a runtime whose config the scanner does not know,
// and dropping it would hide exactly the coverage a reader is looking for.
func Coverage(events []schema.Event, installed []string) CoverageReport {
type counters struct {
events int
usageEvents int
tokens int64
installed bool
}
byHarness := map[string]*counters{}
get := func(name string) *counters {
if byHarness[name] == nil {
byHarness[name] = &counters{}
}
return byHarness[name]
}
for _, name := range installed {
normalized := asymptoteobserve.NormalizeHarnessName(strings.TrimSpace(name))
if normalized == "" {
continue
}
get(normalized).installed = true
}
// Usage is counted through the same collector the report uses, so a runtime counts as
// covered on exactly the events that would contribute to its totals -- including the
// dedupe and cumulative-delta handling. Counting raw gen_ai.usage blocks here instead
// would report a runtime as covered whose usage the report then discards.
usageEvents, _ := resolveUsageEvents(events, sessionUserContexts(events), liveCaptureStarts(events))
for _, event := range events {
name := asymptoteobserve.NormalizeHarnessName(strings.TrimSpace(event.Harness.Name))
if name == "" {
continue
}
get(name).events++
}
for _, ue := range usageEvents {
if ue.contextOnly {
// The collector also yields events that reported only context occupancy. Those are
// not spend, so counting one here would mark a runtime covered on the strength of a
// number that never enters a total -- the precise false reassurance this report
// exists to prevent. Qwen Code is the live case: it reports how full its window was
// and never what a turn cost.
continue
}
name := asymptoteobserve.NormalizeHarnessName(strings.TrimSpace(ue.harness))
if name == "" {
continue
}
c := get(name)
c.usageEvents++
c.tokens += ue.usage.TotalTokens()
}
report := CoverageReport{TotalEvents: len(events)}
for name, c := range byHarness {
expectation, reason := expectationFor(name)
line := RuntimeCoverage{
Harness: name,
Expectation: expectation,
Reason: reason,
Installed: c.installed,
Events: c.events,
UsageEvents: c.usageEvents,
Tokens: c.tokens,
}
switch {
case c.usageEvents > 0:
line.Status = CoverageCovered
report.Covered++
case c.events == 0:
line.Status = CoverageInactive
case expectation == ExpectNone:
line.Status = CoverageNotInstrumented
default:
line.Status = CoverageSilent
report.Silent++
}
report.Runtimes = append(report.Runtimes, line)
}
// Silent first, because that is the only status anyone has to act on; then by spend, so the
// runtimes that matter most sit at the top of each group.
sort.SliceStable(report.Runtimes, func(i, j int) bool {
left, right := report.Runtimes[i], report.Runtimes[j]
if rank := statusRank(left.Status) - statusRank(right.Status); rank != 0 {
return rank < 0
}
if left.Tokens != right.Tokens {
return left.Tokens > right.Tokens
}
return left.Harness < right.Harness
})
return report
}
func statusRank(status string) int {
switch status {
case CoverageSilent:
return 0
case CoverageCovered:
return 1
case CoverageInactive:
return 2
default:
return 3
}
}
// expectationFor reports what Beacon can read from a harness. An unrecognized harness is treated
// as generic OTLP rather than as "none": a runtime Beacon has never heard of reaching the log at
// all means something is exporting semconv telemetry, and reporting it as silent when it carries
// no usage is the honest answer -- calling it not_instrumented would assert knowledge about a
// runtime nobody has looked at.
func expectationFor(harness string) (string, string) {
if entry, ok := usageExpectation[harness]; ok {
return entry.expect, entry.reason
}
return ExpectGenericOTLP, "unrecognized runtime; usage is read only from OTel GenAI semconv names"
}