The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Make install detection a rule instead of a list of special cases
Claudeb1bf24ec40b8
diff --git a/cli/beacon/cmd/token_usage.go b/cli/beacon/cmd/token_usage.goindex fdc87e2c..4a7d36bf 100644--- a/cli/beacon/cmd/token_usage.go+++ b/cli/beacon/cmd/token_usage.go@@ -11,6 +11,7 @@ import (endpointinventory "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/inventory""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/lifecycle""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/tokens"+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve")type tokenUsageOptions struct {@@ -114,25 +115,33 @@ func runTokenCoverage(cmd *cobra.Command, logPath string, query dashboard.EventQreturn err}// Installed runtimes come from the config scanner rather than from the log, because the- // whole question is which installed runtime is missing from the log.- //- // Shell-profile rows are skipped. The scanner lists a runtime's shell profile as one way to- // detect it -- it is where Copilot CLI's and Factory Droid's launch environment is- // configured -- but the file it looks at is the user's ~/.zshrc, which exists on nearly- // every machine whether or not the product does. Counting it as evidence of an install put- // a permanent inactive row for Copilot CLI on every endpoint that has a shell. A profile is- // evidence of a shell, not of a runtime.- //- // Unless Beacon put the runtime there: a profile row is only BeaconManaged when the- // runtime's own OTLP export is configured in it -- OTEL_TELEMETRY_ENDPOINT for Factory- // Droid, COPILOT_OTEL_ENABLED for Copilot CLI -- which is real evidence the runtime is set- // up and expected to report. Skipping those too would hide a configured Copilot CLI that- // stopped reporting, which is exactly the case this report exists to surface.- installed := []string{}+ // whole question is which installed runtime is missing from the log. tokens.InstalledRuntimes+ // decides which scanner rows are real evidence of an install; the scanner reports files a+ // runtime might read, which is not the same thing.+ configs := make([]tokens.InstalledConfig, 0)for _, config := range endpointinventory.Scan(endpointinventory.Options{}).Configs {- if config.Exists && (config.ConfigKind != endpointinventory.KindProfile || config.BeaconManaged) {- installed = append(installed, config.Runtime)+ configs = append(configs, tokens.InstalledConfig{+ Runtime: config.Runtime,+ Path: config.Path,+ Kind: config.ConfigKind,+ BeaconManaged: config.BeaconManaged,+ Exists: config.Exists,+ })+ }+ installed := tokens.InstalledRuntimes(configs)+ // A harness scope narrows the events, so it has to narrow the installed list with them.+ // Otherwise every other installed runtime has no events in the filtered set and is reported+ // inactive -- which reads as "installed but unused this window" when the truth is only that+ // the reader asked about a different runtime.+ if harness := strings.TrimSpace(scoped.Harness); harness != "" {+ want := asymptoteobserve.NormalizeHarnessName(harness)+ kept := installed[:0]+ for _, name := range installed {+ if asymptoteobserve.NormalizeHarnessName(name) == want {+ kept = append(kept, name)+ }}+ installed = kept}report := tokens.Coverage(events, installed)out := cmd.OutOrStdout()diff --git a/cli/beacon/cmd/token_usage_test.go b/cli/beacon/cmd/token_usage_test.goindex a86f045e..799aac38 100644--- a/cli/beacon/cmd/token_usage_test.go+++ b/cli/beacon/cmd/token_usage_test.go@@ -8,6 +8,7 @@ import ("strings""testing"+ endpointinventory "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/inventory""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/tokens")@@ -225,3 +226,56 @@ func TestTokenUsageCoverageDoesNotTreatAShellProfileAsAnInstall(t *testing.T) {}}}++// tokens.ConfigKindProfile duplicates inventory.KindProfile so the tokens package stays+// independent of the scanner. Pinned here, where both are importable, because a silent drift+// would stop excluding shell profiles and quietly refill the report with false installs.+func TestTokenCoverageProfileKindConstantMatchesInventory(t *testing.T) {+ if tokens.ConfigKindProfile != endpointinventory.KindProfile {+ t.Fatalf("tokens.ConfigKindProfile = %q, inventory.KindProfile = %q; they must agree",+ tokens.ConfigKindProfile, endpointinventory.KindProfile)+ }+}++// A harness scope narrows the events, so it has to narrow the installed list with them. Without+// that, every other installed runtime has no events in the filtered set and is reported inactive+// -- which reads as "installed but unused this window" when the truth is only that the reader+// asked about a different runtime.+func TestTokenUsageCoverageHarnessScopeDoesNotStrandOtherRuntimes(t *testing.T) {+ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")+ lines := []string{+ `{"timestamp":"2026-06-11T10:00:00Z","vendor":"beacon","product":"endpoint-agent","schema_version":"1.0","event":{"kind":"agent_runtime","action":"token.usage","category":"metric"},"severity":"info","endpoint":{"hostname":"h"},"harness":{"name":"claude_code"},"model":"claude-opus-5","session":{"id":"s1"},"gen_ai":{"usage":{"input_tokens":10}},"message":"usage"}`,+ `{"timestamp":"2026-06-11T10:01:00Z","vendor":"beacon","product":"endpoint-agent","schema_version":"1.0","event":{"kind":"agent_runtime","action":"command.executed","category":"command"},"severity":"info","endpoint":{"hostname":"h"},"harness":{"name":"codex_cli"},"session":{"id":"s2"},"message":"ls"}`,+ }+ if err := os.WriteFile(logPath, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil {+ t.Fatalf("write log: %v", err)+ }++ // Hermetic: two runtimes installed, so scoping to one has something to strand. Reading the+ // real HOME here would make the test pass or fail on what the machine happens to have.+ home := t.TempDir()+ t.Setenv("HOME", home)+ for _, rel := range []string{".claude/settings.json", ".codex/config.toml"} {+ path := filepath.Join(home, rel)+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {+ t.Fatalf("mkdir: %v", err)+ }+ if err := os.WriteFile(path, []byte("{}\n"), 0o600); err != nil {+ t.Fatalf("write %s: %v", rel, err)+ }+ }++ output := runTokenUsageCommand(t, "--log-path", logPath, "--coverage", "--json", "--harness", "claude_code")+ var report tokens.CoverageReport+ if err := json.Unmarshal([]byte(output), &report); err != nil {+ t.Fatalf("unmarshal coverage report: %v\n%s", err, output)+ }+ for _, runtime := range report.Runtimes {+ if runtime.Harness != "claude_code" {+ t.Errorf("scoping to claude_code still reported %q as %q", runtime.Harness, runtime.Status)+ }+ }+ if len(report.Runtimes) != 1 || report.Runtimes[0].Status != tokens.CoverageCovered {+ t.Fatalf("want exactly the scoped runtime, covered; got %+v", report.Runtimes)+ }+}diff --git a/cli/beacon/internal/tokens/coverage.go b/cli/beacon/internal/tokens/coverage.goindex 261e0ac4..8aac5983 100644--- a/cli/beacon/internal/tokens/coverage.go+++ b/cli/beacon/internal/tokens/coverage.go@@ -1,6 +1,7 @@package tokensimport (+ "path/filepath""sort""strings"@@ -108,6 +109,70 @@ var usageExpectation = map[string]struct {"openhands": {ExpectNone, "hook payloads carry no token counts"},}+// ConfigKindProfile mirrors inventory.KindProfile. It is duplicated rather than imported to keep+// this package independent of the scanner; cmd pins the two equal in a test.+const ConfigKindProfile = "profile"++// sharedConfigBasenames are config files that more than one runtime reads, which the scanner+// nonetheless attributes to a single runtime.+//+// `.mcp.json` is the case: a workspace MCP server list that Claude Code, fx and others all read,+// which the scanner files under fx alone -- deliberately, because attributing one file to several+// runtimes would report one server several times. That is right for an inventory of MCP servers+// and wrong as evidence of an install: a repository whose .mcp.json exists for Claude Code would+// otherwise report fx as installed, and fx would sit in every coverage report as a permanently+// inactive runtime nobody has.+var sharedConfigBasenames = map[string]bool{".mcp.json": true}++// InstalledConfig is the part of a config-scanner row that install detection reads.+type InstalledConfig struct {+ Runtime string+ Path string+ Kind string+ BeaconManaged bool+ Exists bool+}++// InstalledRuntimes selects the scanner rows that are real evidence a runtime is installed.+//+// The scanner answers "which config files exist that some runtime might read", which is a+// different question from "is this runtime installed", and the gap between them is all+// false positives -- each one a permanently inactive row for a product nobody has. A status that+// is wrong on most machines is one people learn to skip, which costs more than the rows are worth.+//+// A row counts when Beacon itself wired the runtime up, since that is unambiguous whatever file it+// lives in. Otherwise the file has to be one only that runtime reads: not a shell profile, which+// exists on nearly every machine, and not a config shared between runtimes.+//+// Two exclusions today, both found in review rather than by design, which is why the rule is+// written as a rule with a named list instead of a condition per file. A third shared path is+// likelier than not.+func InstalledRuntimes(configs []InstalledConfig) []string {+ seen := map[string]bool{}+ var out []string+ for _, config := range configs {+ if !config.Exists {+ continue+ }+ if !config.BeaconManaged {+ if config.Kind == ConfigKindProfile {+ continue+ }+ if sharedConfigBasenames[strings.ToLower(filepath.Base(config.Path))] {+ continue+ }+ }+ name := strings.TrimSpace(config.Runtime)+ if name == "" || seen[name] {+ continue+ }+ seen[name] = true+ out = append(out, name)+ }+ sort.Strings(out)+ return out+}+// RuntimeCoverage is one runtime's line in the coverage report.type RuntimeCoverage struct {Harness string `json:"harness"`diff --git a/cli/beacon/internal/tokens/coverage_test.go b/cli/beacon/internal/tokens/coverage_test.goindex 34c8787e..76ee0afb 100644--- a/cli/beacon/internal/tokens/coverage_test.go+++ b/cli/beacon/internal/tokens/coverage_test.go@@ -262,3 +262,87 @@ func TestNewHarnessesAreClassifiedNotAlerted(t *testing.T) {t.Fatalf("got %d rows, want grok and grok_bot kept apart: %+v", len(both.Runtimes), both.Runtimes)}}++// Install detection reads the config scanner, which answers "which config files exist that some+// runtime might read" -- a different question from "is this runtime installed". Every row where+// those two diverge is a permanently inactive line for a product nobody has, and a status that is+// wrong on most machines is one people stop reading.+func TestInstalledRuntimesRequiresEvidenceThatIsActuallyTheRuntimes(t *testing.T) {+ for _, tc := range []struct {+ name string+ config InstalledConfig+ want bool+ }{+ {+ name: "a runtime-exclusive config counts",+ config: InstalledConfig{Runtime: "claude_code", Path: "/home/u/.claude/settings.json", Kind: "native_config", Exists: true},+ want: true,+ },+ {+ // ~/.zshrc exists on nearly every machine whether or not the product does.+ name: "a shell profile alone does not",+ config: InstalledConfig{Runtime: "copilot_cli", Path: "/home/u/.zshrc", Kind: ConfigKindProfile, Exists: true},+ want: false,+ },+ {+ // Beacon writing the runtime's OTLP export into the profile is unambiguous.+ name: "a shell profile Beacon wired up does",+ config: InstalledConfig{Runtime: "copilot_cli", Path: "/home/u/.zshrc", Kind: ConfigKindProfile, BeaconManaged: true, Exists: true},+ want: true,+ },+ {+ // The scanner files the workspace .mcp.json under fx alone, though Claude Code and+ // others read it too. Right for an MCP inventory, wrong as proof fx is installed.+ name: "a config shared between runtimes does not",+ config: InstalledConfig{Runtime: "vercel_fx", Path: "/repo/.mcp.json", Kind: "native_config", Exists: true},+ want: false,+ },+ {+ name: "fx's own MCP file does",+ config: InstalledConfig{Runtime: "vercel_fx", Path: "/home/u/.fx/mcp.json", Kind: "native_config", Exists: true},+ want: true,+ },+ {+ name: "a config that does not exist does not",+ config: InstalledConfig{Runtime: "cline", Path: "/home/u/.cline/config.json", Kind: "native_config", Exists: false},+ want: false,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ got := len(InstalledRuntimes([]InstalledConfig{tc.config})) == 1+ if got != tc.want {+ t.Fatalf("InstalledRuntimes(%+v) counted=%v, want %v", tc.config, got, tc.want)+ }+ })+ }+}++// A runtime with several scanner rows is one runtime.+func TestInstalledRuntimesDeduplicates(t *testing.T) {+ got := InstalledRuntimes([]InstalledConfig{+ {Runtime: "claude_code", Path: "/home/u/.claude/settings.json", Kind: "native_config", Exists: true},+ {Runtime: "claude_code", Path: "/home/u/.claude.json", Kind: "native_config", Exists: true},+ })+ if len(got) != 1 || got[0] != "claude_code" {+ t.Fatalf("InstalledRuntimes = %v, want [claude_code]", got)+ }+}++// One weak row must not be rescued by another weak row for the same runtime, and a strong row+// must still count when a weak one is present.+func TestInstalledRuntimesMixesWeakAndStrongRowsCorrectly(t *testing.T) {+ onlyWeak := InstalledRuntimes([]InstalledConfig{+ {Runtime: "vercel_fx", Path: "/repo/.mcp.json", Kind: "native_config", Exists: true},+ {Runtime: "copilot_cli", Path: "/home/u/.zshrc", Kind: ConfigKindProfile, Exists: true},+ })+ if len(onlyWeak) != 0 {+ t.Fatalf("InstalledRuntimes = %v, want none -- both rows are shared files", onlyWeak)+ }+ withStrong := InstalledRuntimes([]InstalledConfig{+ {Runtime: "vercel_fx", Path: "/repo/.mcp.json", Kind: "native_config", Exists: true},+ {Runtime: "vercel_fx", Path: "/home/u/.fx/settings.json", Kind: "native_config", Exists: true},+ })+ if len(withStrong) != 1 {+ t.Fatalf("InstalledRuntimes = %v, want vercel_fx counted on its own settings file", withStrong)+ }+}