Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

18.6 KB
Raw Blame
package tokens

import (
	"sort"
	"strings"

	"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
	"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)

// Token spend coverage: which runtimes on this machine are contributing token telemetry, and
// which are not.
//
// A token report answers "what did I spend". It cannot answer "is this all of it", and that is
// the question that matters before anyone treats a number as a bill. A runtime whose usage never
// arrives is indistinguishable, in every rollup Beacon prints, from a runtime nobody used: both
// are simply absent. So a total can quietly omit an entire agent and still look complete.
//
// Coverage answers the second question by joining what is installed against what actually
// reported, and it exists because the alternative is a reader inferring coverage from a docs
// table. The table describes what Beacon is built to read; only the log knows what this machine
// produced.
//
// The expectation table below is what keeps the report from crying wolf. Cursor's hooks carry no
// token counts at all, so Cursor contributing nothing is correct behavior, not a fault -- reported
// as NotInstrumented rather than Silent. Silent is reserved for the case worth investigating: a
// runtime that ran, that Beacon is built to read tokens from, and that reported none anyway.

// Coverage status values.
const (
	// CoverageCovered: the runtime reported usage in the window. Nothing to do.
	CoverageCovered = "covered"
	// CoverageSilent: the runtime produced events but no usage, and Beacon is built to read
	// usage from it. This is the actionable one -- a broken hook, an old runtime build, or a
	// capture path that regressed.
	CoverageSilent = "silent"
	// CoverageInactive: the runtime is installed but produced no events at all in the window.
	// Usually means it was not used; can also mean its hooks are not installed.
	CoverageInactive = "inactive"
	// CoverageNotInstrumented: the runtime produced events, and Beacon cannot read token usage
	// from it. Absence of tokens here is expected and is not a fault to chase.
	CoverageNotInstrumented = "not_instrumented"
)

// Expectation values: what Beacon can read from a runtime, independent of what it did read.
const (
	// ExpectReported: usage is collected from this runtime and verified against captured
	// payloads.
	ExpectReported = "reported"
	// ExpectGenericOTLP: usage is collected only if the runtime emits the OpenTelemetry GenAI
	// semconv names. The generic path is tested; these runtimes' own payloads are not pinned,
	// so silence here may mean the runtime spells its attributes differently rather than that
	// anything is broken.
	ExpectGenericOTLP = "generic_otlp"
	// ExpectNone: the runtime exposes no token counts on the surface Beacon collects from.
	ExpectNone = "none"
)

// usageExpectation records, per canonical harness name, whether Beacon can read token usage from
// that runtime and why. Keys are canonical harness names, so callers must normalize before
// looking up.
//
// This duplicates the per-runtime table in docs/cli/token-usage.mdx by necessity: the docs are for
// a reader deciding whether to expect numbers, this is for the report deciding whether silence is
// a fault. They must be kept in step, and the reason strings are written to be printed.
var usageExpectation = map[string]struct {
	expect string
	reason string
}{
	"claude_code":   {ExpectReported, "OTLP token and cost telemetry"},
	"codex_cli":     {ExpectReported, "per-turn usage trace; Codex emits no cost"},
	"codex_desktop": {ExpectReported, "per-turn usage trace; Codex emits no cost"},
	"claude_cowork": {ExpectReported, "OTLP token and cost telemetry"},
	"cline":         {ExpectReported, "plugin reports usage once per task"},
	"opencode":      {ExpectReported, "plugin reports usage per assistant message"},
	"pi_cli":        {ExpectReported, "extension reports usage and cost"},
	"omp":           {ExpectReported, "extension reports usage and cost"},
	"prime_agent":   {ExpectReported, "extension reports usage and cost"},
	// Senpi's extension forwards the same finalized-message usage object as the rest of the Pi
	// family, through the same mapper (cli/beacon-hooks/cmd/pi_family.go). Without its own key it
	// fell to the unrecognized default and was described as a runtime nobody had looked at.
	"omo_senpi": {ExpectReported, "extension reports usage and cost"},
	// Reported live by the managed plugin's `llm_output` hook, the only OpenClaw hook that carries
	// usage. It is a conversation hook, so OpenClaw withholds it from a non-bundled plugin until
	// `plugins.entries.beacon-endpoint.hooks.allowConversationAccess` is set -- which is why an
	// OpenClaw endpoint can be collecting everything else live and reporting no tokens at all.
	// `beacon endpoint hooks status --harness openclaw` is where that gap is surfaced. The
	// session-file sync reads the usage OpenClaw commits on each assistant message
	// (openclawsession/records.go), and needs no grant.
	"openclaw_gateway": {ExpectReported, "plugin reports usage per model response when conversation access is granted; beacon endpoint integrations openclaw sessions sync reads it from session files"},
	"vercel_fx":        {ExpectReported, "session store carries cumulative usage and cost"},
	"deepseek_harness": {ExpectReported, "native session backfill reports usage when DeepSeek persists it"},
	// Read from the executor's own stream: OpenCode's per-step usage and cost, and Codex's per-turn
	// thread totals, differenced. Neither stream names the model, so these tokens are unpriced
	// unless OpenCode reported a cost.
	"harness2":          {ExpectReported, "rollout streams carry per-step (OpenCode) or per-turn (Codex) usage; OpenCode reports cost"},
	"copilot_cli":       {ExpectReported, "session store carries output tokens, cumulative model usage, and AI-credit cost"},
	"asymptote_observe": {ExpectReported, "SDK spans carry semconv usage"},
	// Read from Gemini CLI's own gemini_cli.api_response log, which it emits for every model
	// response whenever its telemetry is on -- not from the semconv names -- so a Gemini session
	// with events and no usage is a fault worth reporting. The record shape is pinned from Gemini
	// CLI's telemetry source rather than from a recorded session.
	"gemini_cli": {ExpectReported, "api_response log reports input, cache, thought and tool tokens; Gemini CLI emits no cost"},

	// Runtimes whose usage reaches Beacon only through a session-store sync. Their hooks carry no
	// token counts, so a window of hook activity with no usage is spend missing from the total --
	// silent, which is right -- but the fix is running the sync rather than repairing a hook, so
	// the note names the command.
	//
	// Hermes keeps running per-session totals, cost included, in its state database, and the sync
	// differences them into token.usage deltas (hermessession/collect.go, mapUsage).
	"hermes": {ExpectReported, "beacon endpoint hermes sync reads session totals and cost; hook payloads carry none"},
	// Factory keeps running totals in each session's settings file, and the sync differences them
	// (factorysession/mapper.go, emitSettingsUsage). Its OTLP export still contributes when it
	// carries the semconv names; the sync is the path Beacon reads Factory's own record from.
	"factory": {ExpectReported, "beacon endpoint factory sync reads session settings totals; OTLP only if it emits GenAI semconv usage"},
	// The factory_droid spelling never comes from the sync, which writes harness "factory" and is
	// not normalized onto this name. It is what a Factory OTLP export can name itself, so it keeps
	// the generic-OTLP expectation rather than pointing at a command that cannot clear it.
	"factory_droid": {ExpectGenericOTLP, "only if it emits OTel GenAI semconv usage"},

	"vscode_copilot": {ExpectGenericOTLP, "only if it emits OTel GenAI semconv usage"},
	// goose's chat spans carry the full semconv usage set, but they reach Beacon only over OTLP
	// configured by hand, and its hook adapter carries none. Without its own key it fell to the
	// unrecognized default.
	"goose": {ExpectGenericOTLP, "chat spans carry GenAI semconv usage, but only over manually configured OTLP"},
	// Grok Bot runs on a Cursor-hosted cloud computer and reaches Beacon only through Cursor's
	// server-side OpenTelemetry export, so whether usage arrives depends on whether that export
	// carries the semconv names -- the generic-OTLP case exactly, not a runtime Beacon reads.
	"grok_bot": {ExpectGenericOTLP, "only if Cursor's server-side OTel export carries GenAI semconv usage"},

	"cursor": {ExpectNone, "no usage on any hook payload; preCompact reports context only"},
	// Antigravity CLI's hooks carry no token counts, but every planner response it commits to its
	// transcript does, and `beacon endpoint antigravity sync` reads them -- same shape as Hermes
	// above and Devin CLI below.
	"antigravity_cli": {ExpectReported, "beacon endpoint antigravity sync reads per-response token counts; hook payloads carry none"},
	"grok":            {ExpectNone, "hook payloads carry no token counts"},
	// Devin reaches the log under three names, and all three need an entry. The hook installer
	// writes --platform devin-cli and devin-desktop, an older install still writes plain devin,
	// and NormalizeHarnessName passes all three through unchanged. Keying only "devin" left the
	// other two unrecognized, which classified a Devin session that correctly reported no tokens
	// as silent -- the one status this table exists to keep clean.
	"devin": {ExpectNone, "hook payloads carry no token counts"},
	// Devin CLI's committed session store carries per-message metrics and the per-turn model,
	// which the hook payload never sees -- same shape as DeepSeek Harness above.
	"devin-cli":     {ExpectReported, "beacon endpoint devin sync reads per-message token metrics; hook payloads carry none"},
	"devin-desktop": {ExpectNone, "Cascade/Windsurf hook payloads carry no token counts"},
	"muse_code":     {ExpectNone, "usage arrives on PostLLMCall, which Beacon does not subscribe to"},
	"qwen_code":     {ExpectNone, "Stop reports context occupancy, not per-call usage"},
	"claude_web":    {ExpectNone, "no recorded claude.ai stream has carried a usage object"},
	"chatgpt_web":   {ExpectNone, "the chat stream reports no token counts"},
	"openhands":     {ExpectNone, "hook payloads carry no token counts"},
	"kiro":          {ExpectNone, "hook payloads carry no token counts"},
	// Kimi Code assembles each hook payload from a fixed envelope plus per-event fields, and no
	// usage count is among them on any of the twenty events it exposes. The runtime does count
	// tokens -- the TUI shows context occupancy, and PreCompact carries a `token_count` for the
	// history it is about to summarize -- but neither is per-call spend, so promoting either into
	// gen_ai.usage would make a level look like an additive total. A Kimi Code session reporting
	// no tokens is therefore correct rather than silent.
	//
	// Unlike DeepSeek Harness above, which moved to ExpectReported once it gained a session
	// backfill, there is no poll path here to read usage from: Beacon collects Kimi Code through
	// hooks only, and the runtime's own session store is not read.
	"kimi_code": {ExpectNone, "hook payloads carry no token counts; PreCompact reports context only"},
}

// InstalledConfig is the part of a config-scanner row that install detection reads.
type InstalledConfig struct {
	Runtime       string
	Path          string
	Kind          string
	BeaconManaged bool
	Exists        bool
}

// InstalledRuntimes selects the scanner rows that are evidence a runtime is wired up to report.
//
// Only rows Beacon itself configured count. Everything else -- a config file that merely exists --
// is not evidence of anything this report can use, and three rounds of review found three separate
// ways it lies: a shell profile exists on every machine with a shell, the workspace .mcp.json is
// read by several runtimes but filed under one, and VS Code's settings.json exists wherever VS Code
// does, Copilot Chat or not. Each produced a permanently inactive row for a product nobody had.
// Excluding them one at a time was losing; the shape of the mistake was using the scanner to answer
// a question it does not answer.
//
// The scanner reports which config files exist that some runtime might read. Coverage needs to know
// which runtimes Beacon set up to send telemetry, because that is the only population where silence
// is informative: Beacon wired it, so it should be reporting, so nothing arriving is worth a look.
// A runtime present on the machine but never configured by Beacon would not be reporting either
// way, and saying so every week is noise rather than coverage.
//
// beacon_managed is exactly that signal. It is set when a file carries Beacon's hook command, its
// managed-plugin marker, or an OTLP endpoint pointing at the local collector -- in each case
// because Beacon put it there.
//
// A runtime configured by hand, outside Beacon, is not counted. If it emits events it still appears
// in the report through the log; if it emits nothing, Beacon has no grounds to claim it should have.
func InstalledRuntimes(configs []InstalledConfig) []string {
	seen := map[string]bool{}
	var out []string
	for _, config := range configs {
		if !config.Exists || !config.BeaconManaged {
			continue
		}
		name := strings.TrimSpace(config.Runtime)
		if name == "" || seen[name] {
			continue
		}
		seen[name] = true
		out = append(out, name)
	}
	sort.Strings(out)
	return out
}

// RuntimeCoverage is one runtime's line in the coverage report.
type RuntimeCoverage struct {
	Harness     string `json:"harness"`
	Status      string `json:"status"`
	Expectation string `json:"expectation"`
	Reason      string `json:"reason,omitempty"`
	Installed   bool   `json:"installed"`
	Events      int    `json:"events"`
	UsageEvents int    `json:"usage_events"`
	Tokens      int64  `json:"tokens"`
}

// CoverageReport is the whole join: one line per runtime that is either installed or seen in the
// log, plus the counts a reader needs to judge the window itself.
type CoverageReport struct {
	Runtimes []RuntimeCoverage `json:"runtimes"`
	// Silent counts runtimes worth investigating. Zero is the good state.
	Silent int `json:"silent"`
	// Covered counts runtimes that contributed usage.
	Covered int `json:"covered"`
	// TotalEvents is the size of the window the join was computed over. A coverage report over
	// an empty window says nothing, and a reader needs to be able to see that.
	TotalEvents int `json:"total_events"`
}

// Coverage joins installed runtimes against the runtimes that actually reported token usage.
//
// installed carries runtime identifiers as the inventory scanner spells them; they are normalized
// here, so callers pass them through unchanged. A runtime seen in the log but not installed is
// still reported -- telemetry can arrive from a runtime whose config the scanner does not know,
// and dropping it would hide exactly the coverage a reader is looking for.
func Coverage(events []schema.Event, installed []string) CoverageReport {
	type counters struct {
		events      int
		usageEvents int
		tokens      int64
		installed   bool
	}
	byHarness := map[string]*counters{}
	get := func(name string) *counters {
		if byHarness[name] == nil {
			byHarness[name] = &counters{}
		}
		return byHarness[name]
	}

	for _, name := range installed {
		normalized := asymptoteobserve.NormalizeHarnessName(strings.TrimSpace(name))
		if normalized == "" {
			continue
		}
		get(normalized).installed = true
	}

	// Usage is counted through the same collector the report uses, so a runtime counts as
	// covered on exactly the events that would contribute to its totals -- including the
	// dedupe and cumulative-delta handling. Counting raw gen_ai.usage blocks here instead
	// would report a runtime as covered whose usage the report then discards.
	usageEvents, _ := resolveUsageEvents(events, sessionUserContexts(events), liveCaptureStarts(events))

	for _, event := range events {
		name := asymptoteobserve.NormalizeHarnessName(strings.TrimSpace(event.Harness.Name))
		if name == "" {
			continue
		}
		get(name).events++
	}
	for _, ue := range usageEvents {
		if ue.contextOnly {
			// The collector also yields events that reported only context occupancy. Those are
			// not spend, so counting one here would mark a runtime covered on the strength of a
			// number that never enters a total -- the precise false reassurance this report
			// exists to prevent. Qwen Code is the live case: it reports how full its window was
			// and never what a turn cost.
			continue
		}
		name := asymptoteobserve.NormalizeHarnessName(strings.TrimSpace(ue.harness))
		if name == "" {
			continue
		}
		c := get(name)
		c.usageEvents++
		c.tokens += ue.usage.TotalTokens()
	}

	report := CoverageReport{TotalEvents: len(events)}
	for name, c := range byHarness {
		expectation, reason := expectationFor(name)
		line := RuntimeCoverage{
			Harness:     name,
			Expectation: expectation,
			Reason:      reason,
			Installed:   c.installed,
			Events:      c.events,
			UsageEvents: c.usageEvents,
			Tokens:      c.tokens,
		}
		switch {
		case c.usageEvents > 0:
			line.Status = CoverageCovered
			report.Covered++
		case c.events == 0:
			line.Status = CoverageInactive
		case expectation == ExpectNone:
			line.Status = CoverageNotInstrumented
		default:
			line.Status = CoverageSilent
			report.Silent++
		}
		report.Runtimes = append(report.Runtimes, line)
	}

	// Silent first, because that is the only status anyone has to act on; then by spend, so the
	// runtimes that matter most sit at the top of each group.
	sort.SliceStable(report.Runtimes, func(i, j int) bool {
		left, right := report.Runtimes[i], report.Runtimes[j]
		if rank := statusRank(left.Status) - statusRank(right.Status); rank != 0 {
			return rank < 0
		}
		if left.Tokens != right.Tokens {
			return left.Tokens > right.Tokens
		}
		return left.Harness < right.Harness
	})
	return report
}

func statusRank(status string) int {
	switch status {
	case CoverageSilent:
		return 0
	case CoverageCovered:
		return 1
	case CoverageInactive:
		return 2
	default:
		return 3
	}
}

// expectationFor reports what Beacon can read from a harness. An unrecognized harness is treated
// as generic OTLP rather than as "none": a runtime Beacon has never heard of reaching the log at
// all means something is exporting semconv telemetry, and reporting it as silent when it carries
// no usage is the honest answer -- calling it not_instrumented would assert knowledge about a
// runtime nobody has looked at.
func expectationFor(harness string) (string, string) {
	if entry, ok := usageExpectation[harness]; ok {
		return entry.expect, entry.reason
	}
	return ExpectGenericOTLP, "unrecognized runtime; usage is read only from OTel GenAI semconv names"
}