The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Fix two coverage classification bugs found in review
Claude54cd55afdfac
diff --git a/cli/beacon/cmd/token_usage.go b/cli/beacon/cmd/token_usage.goindex 11075c9a..811d4f06 100644--- a/cli/beacon/cmd/token_usage.go+++ b/cli/beacon/cmd/token_usage.go@@ -115,9 +115,16 @@ func runTokenCoverage(cmd *cobra.Command, logPath string, query dashboard.EventQ}// Installed runtimes come from the config scanner rather than from the log, because the// whole question is which installed runtime is missing from the log.+ //+ // Shell-profile rows are skipped. The scanner lists a runtime's shell profile as one way to+ // detect it -- it is where Copilot CLI's and Factory Droid's launch environment is+ // configured -- but the file it looks at is the user's ~/.zshrc, which exists on nearly+ // every machine whether or not the product does. Counting it as evidence of an install put+ // a permanent inactive row for Copilot CLI on every endpoint that has a shell. A profile is+ // evidence of a shell, not of a runtime.installed := []string{}for _, config := range endpointinventory.Scan(endpointinventory.Options{}).Configs {- if config.Exists {+ if config.Exists && config.ConfigKind != endpointinventory.KindProfile {installed = append(installed, config.Runtime)}}diff --git a/cli/beacon/cmd/token_usage_test.go b/cli/beacon/cmd/token_usage_test.goindex 7218c501..a86f045e 100644--- a/cli/beacon/cmd/token_usage_test.go+++ b/cli/beacon/cmd/token_usage_test.go@@ -191,3 +191,37 @@ func TestTokenUsageCoverageIgnoresFiltersThatWouldHideSilentRuntimes(t *testing.t.Fatalf("--model changed the coverage report:\nwithout:\n%s\nwith:\n%s", unfiltered, filtered)}}++// A shell profile is evidence of a shell, not of a runtime.+//+// The config scanner lists ~/.zshrc as one way to detect Copilot CLI and Factory, because that is+// where their launch environment is configured -- but that file exists on nearly every machine+// whether or not the product does. Counting it as an install put a permanent `inactive` row for+// Copilot CLI on every endpoint with a shell, which is noise in the one report whose value+// depends on every row meaning something.+func TestTokenUsageCoverageDoesNotTreatAShellProfileAsAnInstall(t *testing.T) {+ home := t.TempDir()+ t.Setenv("HOME", home)+ t.Setenv("SHELL", "/bin/zsh")+ // The only runtime evidence on this machine: a shell profile. No product configs at all.+ if err := os.WriteFile(filepath.Join(home, ".zshrc"), []byte("export PATH=$PATH\n"), 0o600); err != nil {+ t.Fatalf("write shell profile: %v", err)+ }++ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")+ line := `{"timestamp":"2026-06-11T10:00:00Z","vendor":"beacon","product":"endpoint-agent","schema_version":"1.0","event":{"kind":"agent_runtime","action":"token.usage","category":"metric"},"severity":"info","endpoint":{"hostname":"h"},"harness":{"name":"claude_code"},"model":"claude-opus-5","session":{"id":"s1"},"gen_ai":{"usage":{"input_tokens":10}},"message":"usage"}`+ if err := os.WriteFile(logPath, []byte(line+"\n"), 0o600); err != nil {+ t.Fatalf("write log: %v", err)+ }++ output := runTokenUsageCommand(t, "--log-path", logPath, "--coverage", "--json")+ var report tokens.CoverageReport+ if err := json.Unmarshal([]byte(output), &report); err != nil {+ t.Fatalf("unmarshal coverage report: %v\n%s", err, output)+ }+ for _, runtime := range report.Runtimes {+ if runtime.Installed && (runtime.Harness == "copilot_cli" || runtime.Harness == "factory") {+ t.Errorf("%s reported as installed on the strength of a shell profile alone", runtime.Harness)+ }+ }+}diff --git a/cli/beacon/internal/tokens/coverage.go b/cli/beacon/internal/tokens/coverage.goindex 3a7f292f..98ee1552 100644--- a/cli/beacon/internal/tokens/coverage.go+++ b/cli/beacon/internal/tokens/coverage.go@@ -88,13 +88,20 @@ var usageExpectation = map[string]struct {"antigravity_cli": {ExpectNone, "hook payloads carry no token counts"},"grok": {ExpectNone, "hook payloads carry no token counts"},"hermes": {ExpectNone, "hook payloads carry no token counts"},- "devin": {ExpectNone, "hook payloads carry no token counts"},- "muse_code": {ExpectNone, "usage arrives on PostLLMCall, which Beacon does not subscribe to"},- "qwen_code": {ExpectNone, "Stop carries session context counters, not per-call usage"},- "prime_agent": {ExpectNone, "no managed extension ships yet"},- "claude_web": {ExpectNone, "no recorded claude.ai stream has carried a usage object"},- "chatgpt_web": {ExpectNone, "the chat stream reports no token counts"},- "openhands": {ExpectNone, "hook payloads carry no token counts"},+ // Devin reaches the log under three names, and all three need an entry. The hook installer+ // writes --platform devin-cli and devin-desktop, an older install still writes plain devin,+ // and NormalizeHarnessName passes all three through unchanged. Keying only "devin" left the+ // other two unrecognized, which classified a Devin session that correctly reported no tokens+ // as silent -- the one status this table exists to keep clean.+ "devin": {ExpectNone, "hook payloads carry no token counts"},+ "devin-cli": {ExpectNone, "hook payloads carry no token counts"},+ "devin-desktop": {ExpectNone, "Cascade/Windsurf hook payloads carry no token counts"},+ "muse_code": {ExpectNone, "usage arrives on PostLLMCall, which Beacon does not subscribe to"},+ "qwen_code": {ExpectNone, "Stop carries session context counters, not per-call usage"},+ "prime_agent": {ExpectNone, "no managed extension ships yet"},+ "claude_web": {ExpectNone, "no recorded claude.ai stream has carried a usage object"},+ "chatgpt_web": {ExpectNone, "the chat stream reports no token counts"},+ "openhands": {ExpectNone, "hook payloads carry no token counts"},}// RuntimeCoverage is one runtime's line in the coverage report.diff --git a/cli/beacon/internal/tokens/coverage_test.go b/cli/beacon/internal/tokens/coverage_test.goindex 4d750879..6c1bddfb 100644--- a/cli/beacon/internal/tokens/coverage_test.go+++ b/cli/beacon/internal/tokens/coverage_test.go@@ -196,3 +196,43 @@ func TestUsageExpectationKeysAreCanonicalHarnessNames(t *testing.T) {}}}++// Every runtime the config scanner can report must resolve to an expectation entry.+//+// This is the invariant the Devin bug broke: the scanner reports "devin-cli" and+// "devin-desktop", NormalizeHarnessName passes both through unchanged, and the table keyed only+// "devin" -- so a Devin session that correctly reported no tokens fell to the unrecognized+// default and was classified silent, which is the one status this table exists to keep clean.+//+// The list is duplicated from the inventory scanner rather than imported, deliberately: importing+// it would make the test track a rename automatically and prove nothing, while a literal list+// fails and makes someone look. Add a runtime to the scanner, add it here and to usageExpectation.+func TestEveryScannedRuntimeHasAnExpectation(t *testing.T) {+ // Mirrors the runtime identifiers in internal/endpoint/inventory/inventory.go.+ scanned := []string{+ "antigravity_cli", "claude_code", "cline", "codex_cli", "copilot_cli", "cursor",+ "devin-cli", "devin-desktop", "factory", "gemini_cli", "grok", "hermes",+ "muse_code", "omp", "opencode", "pi_cli", "qwen_code", "vscode",+ }+ for _, runtime := range scanned {+ harness := normalizedHarnessForTest(runtime)+ if _, ok := usageExpectation[harness]; !ok {+ t.Errorf("scanner runtime %q normalizes to %q, which has no usageExpectation entry; "++ "a session from it that reports no tokens would be classified silent", runtime, harness)+ }+ }+}++// The Devin regression in report form: a Devin CLI or Devin Desktop session that reports no+// tokens must not be alerted on.+func TestCoverageDoesNotFlagAnyDevinSpelling(t *testing.T) {+ for _, harness := range []string{"devin", "devin-cli", "devin-desktop"} {+ report := Coverage([]schema.Event{plainEvent(harness)}, []string{harness})+ if got := lineFor(t, report, harness).Status; got != CoverageNotInstrumented {+ t.Errorf("%s status = %q, want not_instrumented -- Devin reports no token counts", harness, got)+ }+ if report.Silent != 0 {+ t.Errorf("%s produced silent=%d, want 0", harness, report.Silent)+ }+ }+}