Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Fix two coverage classification bugs found in review

Claude54cd55afdfac
diff --git a/cli/beacon/cmd/token_usage.go b/cli/beacon/cmd/token_usage.go
index 11075c9a..811d4f06 100644
--- a/cli/beacon/cmd/token_usage.go
+++ b/cli/beacon/cmd/token_usage.go
@@ -115,9 +115,16 @@ func runTokenCoverage(cmd *cobra.Command, logPath string, query dashboard.EventQ
}
// Installed runtimes come from the config scanner rather than from the log, because the
// whole question is which installed runtime is missing from the log.
+ //
+ // Shell-profile rows are skipped. The scanner lists a runtime's shell profile as one way to
+ // detect it -- it is where Copilot CLI's and Factory Droid's launch environment is
+ // configured -- but the file it looks at is the user's ~/.zshrc, which exists on nearly
+ // every machine whether or not the product does. Counting it as evidence of an install put
+ // a permanent inactive row for Copilot CLI on every endpoint that has a shell. A profile is
+ // evidence of a shell, not of a runtime.
installed := []string{}
for _, config := range endpointinventory.Scan(endpointinventory.Options{}).Configs {
- if config.Exists {
+ if config.Exists && config.ConfigKind != endpointinventory.KindProfile {
installed = append(installed, config.Runtime)
}
}
diff --git a/cli/beacon/cmd/token_usage_test.go b/cli/beacon/cmd/token_usage_test.go
index 7218c501..a86f045e 100644
--- a/cli/beacon/cmd/token_usage_test.go
+++ b/cli/beacon/cmd/token_usage_test.go
@@ -191,3 +191,37 @@ func TestTokenUsageCoverageIgnoresFiltersThatWouldHideSilentRuntimes(t *testing.
t.Fatalf("--model changed the coverage report:\nwithout:\n%s\nwith:\n%s", unfiltered, filtered)
}
}
+
+// A shell profile is evidence of a shell, not of a runtime.
+//
+// The config scanner lists ~/.zshrc as one way to detect Copilot CLI and Factory, because that is
+// where their launch environment is configured -- but that file exists on nearly every machine
+// whether or not the product does. Counting it as an install put a permanent `inactive` row for
+// Copilot CLI on every endpoint with a shell, which is noise in the one report whose value
+// depends on every row meaning something.
+func TestTokenUsageCoverageDoesNotTreatAShellProfileAsAnInstall(t *testing.T) {
+ home := t.TempDir()
+ t.Setenv("HOME", home)
+ t.Setenv("SHELL", "/bin/zsh")
+ // The only runtime evidence on this machine: a shell profile. No product configs at all.
+ if err := os.WriteFile(filepath.Join(home, ".zshrc"), []byte("export PATH=$PATH\n"), 0o600); err != nil {
+ t.Fatalf("write shell profile: %v", err)
+ }
+
+ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")
+ line := `{"timestamp":"2026-06-11T10:00:00Z","vendor":"beacon","product":"endpoint-agent","schema_version":"1.0","event":{"kind":"agent_runtime","action":"token.usage","category":"metric"},"severity":"info","endpoint":{"hostname":"h"},"harness":{"name":"claude_code"},"model":"claude-opus-5","session":{"id":"s1"},"gen_ai":{"usage":{"input_tokens":10}},"message":"usage"}`
+ if err := os.WriteFile(logPath, []byte(line+"\n"), 0o600); err != nil {
+ t.Fatalf("write log: %v", err)
+ }
+
+ output := runTokenUsageCommand(t, "--log-path", logPath, "--coverage", "--json")
+ var report tokens.CoverageReport
+ if err := json.Unmarshal([]byte(output), &report); err != nil {
+ t.Fatalf("unmarshal coverage report: %v\n%s", err, output)
+ }
+ for _, runtime := range report.Runtimes {
+ if runtime.Installed && (runtime.Harness == "copilot_cli" || runtime.Harness == "factory") {
+ t.Errorf("%s reported as installed on the strength of a shell profile alone", runtime.Harness)
+ }
+ }
+}
diff --git a/cli/beacon/internal/tokens/coverage.go b/cli/beacon/internal/tokens/coverage.go
index 3a7f292f..98ee1552 100644
--- a/cli/beacon/internal/tokens/coverage.go
+++ b/cli/beacon/internal/tokens/coverage.go
@@ -88,13 +88,20 @@ var usageExpectation = map[string]struct {
"antigravity_cli": {ExpectNone, "hook payloads carry no token counts"},
"grok": {ExpectNone, "hook payloads carry no token counts"},
"hermes": {ExpectNone, "hook payloads carry no token counts"},
- "devin": {ExpectNone, "hook payloads carry no token counts"},
- "muse_code": {ExpectNone, "usage arrives on PostLLMCall, which Beacon does not subscribe to"},
- "qwen_code": {ExpectNone, "Stop carries session context counters, not per-call usage"},
- "prime_agent": {ExpectNone, "no managed extension ships yet"},
- "claude_web": {ExpectNone, "no recorded claude.ai stream has carried a usage object"},
- "chatgpt_web": {ExpectNone, "the chat stream reports no token counts"},
- "openhands": {ExpectNone, "hook payloads carry no token counts"},
+ // Devin reaches the log under three names, and all three need an entry. The hook installer
+ // writes --platform devin-cli and devin-desktop, an older install still writes plain devin,
+ // and NormalizeHarnessName passes all three through unchanged. Keying only "devin" left the
+ // other two unrecognized, which classified a Devin session that correctly reported no tokens
+ // as silent -- the one status this table exists to keep clean.
+ "devin": {ExpectNone, "hook payloads carry no token counts"},
+ "devin-cli": {ExpectNone, "hook payloads carry no token counts"},
+ "devin-desktop": {ExpectNone, "Cascade/Windsurf hook payloads carry no token counts"},
+ "muse_code": {ExpectNone, "usage arrives on PostLLMCall, which Beacon does not subscribe to"},
+ "qwen_code": {ExpectNone, "Stop carries session context counters, not per-call usage"},
+ "prime_agent": {ExpectNone, "no managed extension ships yet"},
+ "claude_web": {ExpectNone, "no recorded claude.ai stream has carried a usage object"},
+ "chatgpt_web": {ExpectNone, "the chat stream reports no token counts"},
+ "openhands": {ExpectNone, "hook payloads carry no token counts"},
}
// RuntimeCoverage is one runtime's line in the coverage report.
diff --git a/cli/beacon/internal/tokens/coverage_test.go b/cli/beacon/internal/tokens/coverage_test.go
index 4d750879..6c1bddfb 100644
--- a/cli/beacon/internal/tokens/coverage_test.go
+++ b/cli/beacon/internal/tokens/coverage_test.go
@@ -196,3 +196,43 @@ func TestUsageExpectationKeysAreCanonicalHarnessNames(t *testing.T) {
}
}
}
+
+// Every runtime the config scanner can report must resolve to an expectation entry.
+//
+// This is the invariant the Devin bug broke: the scanner reports "devin-cli" and
+// "devin-desktop", NormalizeHarnessName passes both through unchanged, and the table keyed only
+// "devin" -- so a Devin session that correctly reported no tokens fell to the unrecognized
+// default and was classified silent, which is the one status this table exists to keep clean.
+//
+// The list is duplicated from the inventory scanner rather than imported, deliberately: importing
+// it would make the test track a rename automatically and prove nothing, while a literal list
+// fails and makes someone look. Add a runtime to the scanner, add it here and to usageExpectation.
+func TestEveryScannedRuntimeHasAnExpectation(t *testing.T) {
+ // Mirrors the runtime identifiers in internal/endpoint/inventory/inventory.go.
+ scanned := []string{
+ "antigravity_cli", "claude_code", "cline", "codex_cli", "copilot_cli", "cursor",
+ "devin-cli", "devin-desktop", "factory", "gemini_cli", "grok", "hermes",
+ "muse_code", "omp", "opencode", "pi_cli", "qwen_code", "vscode",
+ }
+ for _, runtime := range scanned {
+ harness := normalizedHarnessForTest(runtime)
+ if _, ok := usageExpectation[harness]; !ok {
+ t.Errorf("scanner runtime %q normalizes to %q, which has no usageExpectation entry; "+
+ "a session from it that reports no tokens would be classified silent", runtime, harness)
+ }
+ }
+}
+
+// The Devin regression in report form: a Devin CLI or Devin Desktop session that reports no
+// tokens must not be alerted on.
+func TestCoverageDoesNotFlagAnyDevinSpelling(t *testing.T) {
+ for _, harness := range []string{"devin", "devin-cli", "devin-desktop"} {
+ report := Coverage([]schema.Event{plainEvent(harness)}, []string{harness})
+ if got := lineFor(t, report, harness).Status; got != CoverageNotInstrumented {
+ t.Errorf("%s status = %q, want not_instrumented -- Devin reports no token counts", harness, got)
+ }
+ if report.Silent != 0 {
+ t.Errorf("%s produced silent=%d, want 0", harness, report.Silent)
+ }
+ }
+}