Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Make install detection a rule instead of a list of special cases

Claudeb1bf24ec40b8
diff --git a/cli/beacon/cmd/token_usage.go b/cli/beacon/cmd/token_usage.go
index fdc87e2c..4a7d36bf 100644
--- a/cli/beacon/cmd/token_usage.go
+++ b/cli/beacon/cmd/token_usage.go
@@ -11,6 +11,7 @@ import (
endpointinventory "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/inventory"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/lifecycle"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/tokens"
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)
type tokenUsageOptions struct {
@@ -114,25 +115,33 @@ func runTokenCoverage(cmd *cobra.Command, logPath string, query dashboard.EventQ
return err
}
// Installed runtimes come from the config scanner rather than from the log, because the
- // whole question is which installed runtime is missing from the log.
- //
- // Shell-profile rows are skipped. The scanner lists a runtime's shell profile as one way to
- // detect it -- it is where Copilot CLI's and Factory Droid's launch environment is
- // configured -- but the file it looks at is the user's ~/.zshrc, which exists on nearly
- // every machine whether or not the product does. Counting it as evidence of an install put
- // a permanent inactive row for Copilot CLI on every endpoint that has a shell. A profile is
- // evidence of a shell, not of a runtime.
- //
- // Unless Beacon put the runtime there: a profile row is only BeaconManaged when the
- // runtime's own OTLP export is configured in it -- OTEL_TELEMETRY_ENDPOINT for Factory
- // Droid, COPILOT_OTEL_ENABLED for Copilot CLI -- which is real evidence the runtime is set
- // up and expected to report. Skipping those too would hide a configured Copilot CLI that
- // stopped reporting, which is exactly the case this report exists to surface.
- installed := []string{}
+ // whole question is which installed runtime is missing from the log. tokens.InstalledRuntimes
+ // decides which scanner rows are real evidence of an install; the scanner reports files a
+ // runtime might read, which is not the same thing.
+ configs := make([]tokens.InstalledConfig, 0)
for _, config := range endpointinventory.Scan(endpointinventory.Options{}).Configs {
- if config.Exists && (config.ConfigKind != endpointinventory.KindProfile || config.BeaconManaged) {
- installed = append(installed, config.Runtime)
+ configs = append(configs, tokens.InstalledConfig{
+ Runtime: config.Runtime,
+ Path: config.Path,
+ Kind: config.ConfigKind,
+ BeaconManaged: config.BeaconManaged,
+ Exists: config.Exists,
+ })
+ }
+ installed := tokens.InstalledRuntimes(configs)
+ // A harness scope narrows the events, so it has to narrow the installed list with them.
+ // Otherwise every other installed runtime has no events in the filtered set and is reported
+ // inactive -- which reads as "installed but unused this window" when the truth is only that
+ // the reader asked about a different runtime.
+ if harness := strings.TrimSpace(scoped.Harness); harness != "" {
+ want := asymptoteobserve.NormalizeHarnessName(harness)
+ kept := installed[:0]
+ for _, name := range installed {
+ if asymptoteobserve.NormalizeHarnessName(name) == want {
+ kept = append(kept, name)
+ }
}
+ installed = kept
}
report := tokens.Coverage(events, installed)
out := cmd.OutOrStdout()
diff --git a/cli/beacon/cmd/token_usage_test.go b/cli/beacon/cmd/token_usage_test.go
index a86f045e..799aac38 100644
--- a/cli/beacon/cmd/token_usage_test.go
+++ b/cli/beacon/cmd/token_usage_test.go
@@ -8,6 +8,7 @@ import (
"strings"
"testing"
+ endpointinventory "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/inventory"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/tokens"
)
@@ -225,3 +226,56 @@ func TestTokenUsageCoverageDoesNotTreatAShellProfileAsAnInstall(t *testing.T) {
}
}
}
+
+// tokens.ConfigKindProfile duplicates inventory.KindProfile so the tokens package stays
+// independent of the scanner. Pinned here, where both are importable, because a silent drift
+// would stop excluding shell profiles and quietly refill the report with false installs.
+func TestTokenCoverageProfileKindConstantMatchesInventory(t *testing.T) {
+ if tokens.ConfigKindProfile != endpointinventory.KindProfile {
+ t.Fatalf("tokens.ConfigKindProfile = %q, inventory.KindProfile = %q; they must agree",
+ tokens.ConfigKindProfile, endpointinventory.KindProfile)
+ }
+}
+
+// A harness scope narrows the events, so it has to narrow the installed list with them. Without
+// that, every other installed runtime has no events in the filtered set and is reported inactive
+// -- which reads as "installed but unused this window" when the truth is only that the reader
+// asked about a different runtime.
+func TestTokenUsageCoverageHarnessScopeDoesNotStrandOtherRuntimes(t *testing.T) {
+ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")
+ lines := []string{
+ `{"timestamp":"2026-06-11T10:00:00Z","vendor":"beacon","product":"endpoint-agent","schema_version":"1.0","event":{"kind":"agent_runtime","action":"token.usage","category":"metric"},"severity":"info","endpoint":{"hostname":"h"},"harness":{"name":"claude_code"},"model":"claude-opus-5","session":{"id":"s1"},"gen_ai":{"usage":{"input_tokens":10}},"message":"usage"}`,
+ `{"timestamp":"2026-06-11T10:01:00Z","vendor":"beacon","product":"endpoint-agent","schema_version":"1.0","event":{"kind":"agent_runtime","action":"command.executed","category":"command"},"severity":"info","endpoint":{"hostname":"h"},"harness":{"name":"codex_cli"},"session":{"id":"s2"},"message":"ls"}`,
+ }
+ if err := os.WriteFile(logPath, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil {
+ t.Fatalf("write log: %v", err)
+ }
+
+ // Hermetic: two runtimes installed, so scoping to one has something to strand. Reading the
+ // real HOME here would make the test pass or fail on what the machine happens to have.
+ home := t.TempDir()
+ t.Setenv("HOME", home)
+ for _, rel := range []string{".claude/settings.json", ".codex/config.toml"} {
+ path := filepath.Join(home, rel)
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ t.Fatalf("mkdir: %v", err)
+ }
+ if err := os.WriteFile(path, []byte("{}\n"), 0o600); err != nil {
+ t.Fatalf("write %s: %v", rel, err)
+ }
+ }
+
+ output := runTokenUsageCommand(t, "--log-path", logPath, "--coverage", "--json", "--harness", "claude_code")
+ var report tokens.CoverageReport
+ if err := json.Unmarshal([]byte(output), &report); err != nil {
+ t.Fatalf("unmarshal coverage report: %v\n%s", err, output)
+ }
+ for _, runtime := range report.Runtimes {
+ if runtime.Harness != "claude_code" {
+ t.Errorf("scoping to claude_code still reported %q as %q", runtime.Harness, runtime.Status)
+ }
+ }
+ if len(report.Runtimes) != 1 || report.Runtimes[0].Status != tokens.CoverageCovered {
+ t.Fatalf("want exactly the scoped runtime, covered; got %+v", report.Runtimes)
+ }
+}
diff --git a/cli/beacon/internal/tokens/coverage.go b/cli/beacon/internal/tokens/coverage.go
index 261e0ac4..8aac5983 100644
--- a/cli/beacon/internal/tokens/coverage.go
+++ b/cli/beacon/internal/tokens/coverage.go
@@ -1,6 +1,7 @@
package tokens
import (
+ "path/filepath"
"sort"
"strings"
@@ -108,6 +109,70 @@ var usageExpectation = map[string]struct {
"openhands": {ExpectNone, "hook payloads carry no token counts"},
}
+// ConfigKindProfile mirrors inventory.KindProfile. It is duplicated rather than imported to keep
+// this package independent of the scanner; cmd pins the two equal in a test.
+const ConfigKindProfile = "profile"
+
+// sharedConfigBasenames are config files that more than one runtime reads, which the scanner
+// nonetheless attributes to a single runtime.
+//
+// `.mcp.json` is the case: a workspace MCP server list that Claude Code, fx and others all read,
+// which the scanner files under fx alone -- deliberately, because attributing one file to several
+// runtimes would report one server several times. That is right for an inventory of MCP servers
+// and wrong as evidence of an install: a repository whose .mcp.json exists for Claude Code would
+// otherwise report fx as installed, and fx would sit in every coverage report as a permanently
+// inactive runtime nobody has.
+var sharedConfigBasenames = map[string]bool{".mcp.json": true}
+
+// InstalledConfig is the part of a config-scanner row that install detection reads.
+type InstalledConfig struct {
+ Runtime string
+ Path string
+ Kind string
+ BeaconManaged bool
+ Exists bool
+}
+
+// InstalledRuntimes selects the scanner rows that are real evidence a runtime is installed.
+//
+// The scanner answers "which config files exist that some runtime might read", which is a
+// different question from "is this runtime installed", and the gap between them is all
+// false positives -- each one a permanently inactive row for a product nobody has. A status that
+// is wrong on most machines is one people learn to skip, which costs more than the rows are worth.
+//
+// A row counts when Beacon itself wired the runtime up, since that is unambiguous whatever file it
+// lives in. Otherwise the file has to be one only that runtime reads: not a shell profile, which
+// exists on nearly every machine, and not a config shared between runtimes.
+//
+// Two exclusions today, both found in review rather than by design, which is why the rule is
+// written as a rule with a named list instead of a condition per file. A third shared path is
+// likelier than not.
+func InstalledRuntimes(configs []InstalledConfig) []string {
+ seen := map[string]bool{}
+ var out []string
+ for _, config := range configs {
+ if !config.Exists {
+ continue
+ }
+ if !config.BeaconManaged {
+ if config.Kind == ConfigKindProfile {
+ continue
+ }
+ if sharedConfigBasenames[strings.ToLower(filepath.Base(config.Path))] {
+ continue
+ }
+ }
+ name := strings.TrimSpace(config.Runtime)
+ if name == "" || seen[name] {
+ continue
+ }
+ seen[name] = true
+ out = append(out, name)
+ }
+ sort.Strings(out)
+ return out
+}
+
// RuntimeCoverage is one runtime's line in the coverage report.
type RuntimeCoverage struct {
Harness string `json:"harness"`
diff --git a/cli/beacon/internal/tokens/coverage_test.go b/cli/beacon/internal/tokens/coverage_test.go
index 34c8787e..76ee0afb 100644
--- a/cli/beacon/internal/tokens/coverage_test.go
+++ b/cli/beacon/internal/tokens/coverage_test.go
@@ -262,3 +262,87 @@ func TestNewHarnessesAreClassifiedNotAlerted(t *testing.T) {
t.Fatalf("got %d rows, want grok and grok_bot kept apart: %+v", len(both.Runtimes), both.Runtimes)
}
}
+
+// Install detection reads the config scanner, which answers "which config files exist that some
+// runtime might read" -- a different question from "is this runtime installed". Every row where
+// those two diverge is a permanently inactive line for a product nobody has, and a status that is
+// wrong on most machines is one people stop reading.
+func TestInstalledRuntimesRequiresEvidenceThatIsActuallyTheRuntimes(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ config InstalledConfig
+ want bool
+ }{
+ {
+ name: "a runtime-exclusive config counts",
+ config: InstalledConfig{Runtime: "claude_code", Path: "/home/u/.claude/settings.json", Kind: "native_config", Exists: true},
+ want: true,
+ },
+ {
+ // ~/.zshrc exists on nearly every machine whether or not the product does.
+ name: "a shell profile alone does not",
+ config: InstalledConfig{Runtime: "copilot_cli", Path: "/home/u/.zshrc", Kind: ConfigKindProfile, Exists: true},
+ want: false,
+ },
+ {
+ // Beacon writing the runtime's OTLP export into the profile is unambiguous.
+ name: "a shell profile Beacon wired up does",
+ config: InstalledConfig{Runtime: "copilot_cli", Path: "/home/u/.zshrc", Kind: ConfigKindProfile, BeaconManaged: true, Exists: true},
+ want: true,
+ },
+ {
+ // The scanner files the workspace .mcp.json under fx alone, though Claude Code and
+ // others read it too. Right for an MCP inventory, wrong as proof fx is installed.
+ name: "a config shared between runtimes does not",
+ config: InstalledConfig{Runtime: "vercel_fx", Path: "/repo/.mcp.json", Kind: "native_config", Exists: true},
+ want: false,
+ },
+ {
+ name: "fx's own MCP file does",
+ config: InstalledConfig{Runtime: "vercel_fx", Path: "/home/u/.fx/mcp.json", Kind: "native_config", Exists: true},
+ want: true,
+ },
+ {
+ name: "a config that does not exist does not",
+ config: InstalledConfig{Runtime: "cline", Path: "/home/u/.cline/config.json", Kind: "native_config", Exists: false},
+ want: false,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ got := len(InstalledRuntimes([]InstalledConfig{tc.config})) == 1
+ if got != tc.want {
+ t.Fatalf("InstalledRuntimes(%+v) counted=%v, want %v", tc.config, got, tc.want)
+ }
+ })
+ }
+}
+
+// A runtime with several scanner rows is one runtime.
+func TestInstalledRuntimesDeduplicates(t *testing.T) {
+ got := InstalledRuntimes([]InstalledConfig{
+ {Runtime: "claude_code", Path: "/home/u/.claude/settings.json", Kind: "native_config", Exists: true},
+ {Runtime: "claude_code", Path: "/home/u/.claude.json", Kind: "native_config", Exists: true},
+ })
+ if len(got) != 1 || got[0] != "claude_code" {
+ t.Fatalf("InstalledRuntimes = %v, want [claude_code]", got)
+ }
+}
+
+// One weak row must not be rescued by another weak row for the same runtime, and a strong row
+// must still count when a weak one is present.
+func TestInstalledRuntimesMixesWeakAndStrongRowsCorrectly(t *testing.T) {
+ onlyWeak := InstalledRuntimes([]InstalledConfig{
+ {Runtime: "vercel_fx", Path: "/repo/.mcp.json", Kind: "native_config", Exists: true},
+ {Runtime: "copilot_cli", Path: "/home/u/.zshrc", Kind: ConfigKindProfile, Exists: true},
+ })
+ if len(onlyWeak) != 0 {
+ t.Fatalf("InstalledRuntimes = %v, want none -- both rows are shared files", onlyWeak)
+ }
+ withStrong := InstalledRuntimes([]InstalledConfig{
+ {Runtime: "vercel_fx", Path: "/repo/.mcp.json", Kind: "native_config", Exists: true},
+ {Runtime: "vercel_fx", Path: "/home/u/.fx/settings.json", Kind: "native_config", Exists: true},
+ })
+ if len(withStrong) != 1 {
+ t.Fatalf("InstalledRuntimes = %v, want vercel_fx counted on its own settings file", withStrong)
+ }
+}