The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Extract shared rule loading into pkg/asymptoteobserve/rulestore
Claude58da25c26c0f
diff --git a/cli/beacon/internal/endpoint/detect/detect.go b/cli/beacon/internal/endpoint/detect/detect.goindex 6702c2e2..fa2a3f85 100644--- a/cli/beacon/internal/endpoint/detect/detect.go+++ b/cli/beacon/internal/endpoint/detect/detect.go@@ -1,44 +1,34 @@// Package detect wires the open threat-rules engine to the local Beacon endpoint: it-// manages a runtime rules store (~/.beacon/endpoint/rules), ships a small frozen baseline-// embedded in the binary, and loads the active rule set for `beacon scan`.+// resolves the runtime rules store (~/.beacon/endpoint/rules) from endpoint config and+// loads the active rule set for `beacon scan`.//-// The engine ships in the binary; the rule corpus is external data. Only the frozen-// baseline below is embedded, so a growing corpus loaded into the store never enlarges-// the binary.+// The shared store mechanics (embedded baseline, load/install/remove) live in+// pkg/asymptoteobserve/rulestore so they can be reused across modules. This package is a+// thin cli/beacon adapter that resolves the store directory from endpoint config and+// delegates, preserving its existing (userMode bool) API for the CLI, scan, and dashboard.package detectimport (- "embed"- "fmt"- "io/fs"- "os"- "path""path/filepath"- "sort"- "strings"endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/rulestore""github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/threatrules")-//go:embed baseline/*.rule.yaml-var baselineFS embed.FS--const ruleFileSuffix = ".rule.yaml"-// Source identifies where an active rule came from.-type Source string+type Source = rulestore.Sourceconst (- SourceBaseline Source = "baseline"- SourceStore Source = "store"+ SourceBaseline = rulestore.SourceBaseline+ SourceStore = rulestore.SourceStore)// LoadedRule is a validated rule paired with its origin.-type LoadedRule struct {- Rule *threatrules.Rule- Source Source-}+type LoadedRule = rulestore.LoadedRule++// Installed reports a rule written into the store.+type Installed = rulestore.Installed// StoreDir returns the local rules store directory for the given mode.func StoreDir(userMode bool) string {@@ -47,299 +37,25 @@ func StoreDir(userMode bool) string {// EnsureStore creates the rules store directory if needed and returns its path.func EnsureStore(userMode bool) (string, error) {- dir := StoreDir(userMode)- if err := os.MkdirAll(dir, 0o755); err != nil {- return "", fmt.Errorf("create rules store: %w", err)- }- return dir, nil+ return rulestore.EnsureStore(StoreDir(userMode))}// Baseline returns the embedded frozen baseline rules, validated.func Baseline() ([]*threatrules.Rule, error) {- entries, err := baselineFS.ReadDir("baseline")- if err != nil {- return nil, err- }- names := make([]string, 0, len(entries))- for _, e := range entries {- if !e.IsDir() && strings.HasSuffix(e.Name(), ruleFileSuffix) {- names = append(names, e.Name())- }- }- sort.Strings(names)-- rules := make([]*threatrules.Rule, 0, len(names))- seen := map[string]bool{}- for _, name := range names {- data, err := baselineFS.ReadFile(baselineRulePath(name))- if err != nil {- return nil, err- }- rule, err := threatrules.DecodeRule(data)- if err != nil {- return nil, fmt.Errorf("baseline %s: %w", name, err)- }- if err := rule.Validate(); err != nil {- return nil, fmt.Errorf("baseline %s: %w", name, err)- }- if seen[rule.ID] {- return nil, fmt.Errorf("baseline duplicate rule id %q", rule.ID)- }- seen[rule.ID] = true- rules = append(rules, rule)- }- return rules, nil-}--func baselineRulePath(name string) string {- return path.Join("baseline", name)+ return rulestore.Baseline()}-// LoadActive resolves the active rule set:-//-// - if rulesDir is non-empty, load only from that directory (explicit override);-// - else load from the store; if the store is empty (or absent), fall back to the-// embedded baseline.-//-// Every returned rule has passed validation (via threatrules.LoadDir / Baseline).+// LoadActive resolves the active rule set for the given mode. See rulestore.LoadActive.func LoadActive(userMode bool, rulesDir string) ([]LoadedRule, error) {- if rulesDir != "" {- rules, err := threatrules.LoadDir(rulesDir)- if err != nil {- return nil, err- }- return tag(rules, SourceStore), nil- }-- store := StoreDir(userMode)- if hasRuleFiles(store) {- rules, err := threatrules.LoadDir(store)- if err != nil {- return nil, err- }- return tag(rules, SourceStore), nil- }-- base, err := Baseline()- if err != nil {- return nil, err- }- return tag(base, SourceBaseline), nil-}--func tag(rules []*threatrules.Rule, src Source) []LoadedRule {- out := make([]LoadedRule, len(rules))- for i, r := range rules {- out[i] = LoadedRule{Rule: r, Source: src}- }- return out-}--func hasRuleFiles(dir string) bool {- entries, err := os.ReadDir(dir)- if err != nil {- return false- }- for _, e := range entries {- if !e.IsDir() && strings.HasSuffix(e.Name(), ruleFileSuffix) {- return true- }- }- return false-}--// Installed reports a rule written into the store.-type Installed struct {- ID string- Path string+ return rulestore.LoadActive(StoreDir(userMode), rulesDir)}-// InstallFiles validates every *.rule.yaml found at src (a file or directory) and writes-// the valid ones into the store. Each rule is validated before any file is written; an-// invalid rule aborts the whole install. A rule whose id already exists in the store is-// rejected unless force is set. Returns the rules installed.+// InstallFiles validates and installs the *.rule.yaml files at src into the store.func InstallFiles(userMode bool, src string, force bool) ([]Installed, error) {- srcPaths, err := ruleFilesAt(src)- if err != nil {- return nil, err- }- if len(srcPaths) == 0 {- return nil, fmt.Errorf("no %s files found at %s", ruleFileSuffix, src)- }-- store, err := EnsureStore(userMode)- if err != nil {- return nil, err- }- existing, err := storeIDs(store)- if err != nil {- return nil, err- }-- // Validate everything first; collect the bytes + target names. Abort on any failure- // so a partial/invalid install never lands.- type pending struct {- id string- dest string- data []byte- }- var todo []pending- staged := map[string]bool{}- for _, p := range srcPaths {- data, err := os.ReadFile(p)- if err != nil {- return nil, err- }- rule, err := threatrules.DecodeRule(data)- if err != nil {- return nil, fmt.Errorf("%s: %w", p, err)- }- results, err := threatrules.CheckRule(rule)- if err != nil {- return nil, fmt.Errorf("%s: %w", p, err)- }- for _, res := range results {- if !res.OK() {- return nil, fmt.Errorf("%s: fixture %s", p, res.String())- }- }- // rule.ID passed CheckRule (id-grammar) above; re-assert it explicitly so the- // store path join below can never be driven outside the store by rule content.- if !threatrules.ValidID(rule.ID) {- return nil, fmt.Errorf("%s: invalid rule id %q", p, rule.ID)- }- if (existing[rule.ID] || staged[rule.ID]) && !force {- return nil, fmt.Errorf("rule %q already installed (use --force to overwrite)", rule.ID)- }- staged[rule.ID] = true- todo = append(todo, pending{id: rule.ID, dest: filepath.Join(store, rule.ID+ruleFileSuffix), data: data})- }-- tmpDir, err := os.MkdirTemp(store, ".install-*")- if err != nil {- return nil, err- }- defer os.RemoveAll(tmpDir)-- for _, p := range todo {- tmpPath := filepath.Join(tmpDir, p.id+ruleFileSuffix)- if err := os.WriteFile(tmpPath, p.data, 0o644); err != nil {- return nil, err- }- }-- installed := make([]Installed, 0, len(todo))- for _, p := range todo {- tmpPath := filepath.Join(tmpDir, p.id+ruleFileSuffix)- backupPath, err := moveExistingRuleAside(p.dest, tmpDir, p.id)- if err != nil {- rollbackInstall(installed, tmpDir)- return nil, err- }- if err := os.Rename(tmpPath, p.dest); err != nil {- if backupPath != "" {- _ = os.Rename(backupPath, p.dest)- }- rollbackInstall(installed, tmpDir)- return nil, err- }- installed = append(installed, Installed{ID: p.id, Path: p.dest})- }- return installed, nil-}--func moveExistingRuleAside(dest, tmpDir, id string) (string, error) {- info, err := os.Lstat(dest)- if err != nil {- if os.IsNotExist(err) {- return "", nil- }- return "", err- }- if info.IsDir() {- return "", fmt.Errorf("%s exists and is a directory", dest)- }- backupPath := filepath.Join(tmpDir, id+".backup")- if err := os.Rename(dest, backupPath); err != nil {- return "", err- }- return backupPath, nil+ return rulestore.InstallFiles(StoreDir(userMode), src, force)}-func rollbackInstall(installed []Installed, tmpDir string) {- for i := len(installed) - 1; i >= 0; i-- {- backupPath := filepath.Join(tmpDir, installed[i].ID+".backup")- _ = os.Remove(installed[i].Path)- if _, err := os.Stat(backupPath); err == nil {- _ = os.Rename(backupPath, installed[i].Path)- }- }-}--// Remove deletes a rule by id from the store. Returns the removed path. The id-// is validated against the rule-id grammar before use so it can never contain-// path separators or ".." segments that would resolve outside the store.+// Remove deletes a rule by id from the store and returns the removed path.func Remove(userMode bool, id string) (string, error) {- if !threatrules.ValidID(id) {- return "", fmt.Errorf("invalid rule id %q", id)- }- path := filepath.Join(StoreDir(userMode), id+ruleFileSuffix)- if err := os.Remove(path); err != nil {- if os.IsNotExist(err) {- return "", fmt.Errorf("rule %q is not installed in the store", id)- }- return "", err- }- return path, nil-}--// ruleFilesAt returns the *.rule.yaml files at src, which may be a single file or a dir.-func ruleFilesAt(src string) ([]string, error) {- info, err := os.Stat(src)- if err != nil {- return nil, err- }- if !info.IsDir() {- if !isRuleFile(src) {- return nil, fmt.Errorf("%s is not a %s file", src, ruleFileSuffix)- }- return []string{src}, nil- }- var paths []string- err = filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {- if err != nil {- return err- }- if !d.IsDir() && isRuleFile(path) {- paths = append(paths, path)- }- return nil- })- if err != nil {- return nil, err- }- sort.Strings(paths)- return paths, nil-}--func isRuleFile(path string) bool {- return strings.HasSuffix(path, ruleFileSuffix) && !strings.HasPrefix(filepath.Base(path), "._")-}--func storeIDs(store string) (map[string]bool, error) {- ids := map[string]bool{}- entries, err := os.ReadDir(store)- if err != nil {- if os.IsNotExist(err) {- return ids, nil- }- return nil, err- }- for _, e := range entries {- if e.IsDir() || !strings.HasSuffix(e.Name(), ruleFileSuffix) {- continue- }- ids[strings.TrimSuffix(e.Name(), ruleFileSuffix)] = true- }- return ids, nil+ return rulestore.Remove(StoreDir(userMode), id)}diff --git a/cli/beacon/internal/endpoint/detect/detect_test.go b/cli/beacon/internal/endpoint/detect/detect_test.goindex 100b9a22..3738dd15 100644--- a/cli/beacon/internal/endpoint/detect/detect_test.go+++ b/cli/beacon/internal/endpoint/detect/detect_test.go@@ -6,32 +6,10 @@ import ("strings""testing"+ endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config""github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/threatrules")-func TestBaselineValid(t *testing.T) {- rules, err := Baseline()- if err != nil {- t.Fatalf("baseline: %v", err)- }- if len(rules) == 0 {- t.Fatal("embedded baseline is empty")- }- // Every embedded rule must pass full conformance (validate + maturity + fixtures).- for _, r := range rules {- if _, err := threatrules.CheckRule(r); err != nil {- t.Errorf("baseline rule %q fails conformance: %v", r.ID, err)- }- }-}--func TestBaselineRulePathUsesEmbedFSSeparator(t *testing.T) {- got := baselineRulePath("example.rule.yaml")- if got != "baseline/example.rule.yaml" {- t.Fatalf("embedded FS paths must use forward slashes, got %q", got)- }-}-const aRule = `id: %ID%version: 1@@ -51,6 +29,31 @@ tests:func ruleWithID(id string) string { return strings.ReplaceAll(aRule, "%ID%", id) }+// The exhaustive store mechanics (baseline conformance, install/remove, rollback,+// path traversal) live in pkg/asymptoteobserve/rulestore. These tests cover the thin+// cli/beacon adapter: store-dir resolution from endpoint config and delegation.++func TestStoreDirUsesBaseDir(t *testing.T) {+ if got, want := StoreDir(true), filepath.Join(endpointconfig.BaseDir(true), "rules"); got != want {+ t.Fatalf("StoreDir(true) = %q, want %q", got, want)+ }+}++func TestBaselineDelegates(t *testing.T) {+ rules, err := Baseline()+ if err != nil {+ t.Fatalf("baseline: %v", err)+ }+ if len(rules) == 0 {+ t.Fatal("embedded baseline is empty")+ }+ for _, r := range rules {+ if _, err := threatrules.CheckRule(r); err != nil {+ t.Errorf("baseline rule %q fails conformance: %v", r.ID, err)+ }+ }+}+func TestLoadActiveFallsBackToBaseline(t *testing.T) {t.Setenv("HOME", t.TempDir()) // empty store -> baselineloaded, err := LoadActive(true, "")@@ -58,13 +61,12 @@ func TestLoadActiveFallsBackToBaseline(t *testing.T) {t.Fatalf("load active: %v", err)}if len(loaded) == 0 || loaded[0].Source != SourceBaseline {- t.Fatalf("expected baseline fallback, got %d rules src=%v", len(loaded), srcOf(loaded))+ t.Fatalf("expected baseline fallback, got %d rules", len(loaded))}}-func TestInstallThenLoadActiveUsesStore(t *testing.T) {- home := t.TempDir()- t.Setenv("HOME", home)+func TestInstallRemoveRoundTripViaUserMode(t *testing.T) {+ t.Setenv("HOME", t.TempDir())src := filepath.Join(t.TempDir(), "my.rule.yaml")if err := os.WriteFile(src, []byte(ruleWithID("custom-rule")), 0o644); err != nil {@@ -85,194 +87,11 @@ func TestInstallThenLoadActiveUsesStore(t *testing.T) {if len(loaded) != 1 || loaded[0].Source != SourceStore || loaded[0].Rule.ID != "custom-rule" {t.Fatalf("expected store rule, got %+v", loaded)}-}--func TestInstallFilesIgnoresAppleDoubleRuleFiles(t *testing.T) {- t.Setenv("HOME", t.TempDir())- srcDir := t.TempDir()- if err := os.WriteFile(filepath.Join(srcDir, "ok.rule.yaml"), []byte(ruleWithID("custom-rule")), 0o644); err != nil {- t.Fatal(err)- }- if err := os.WriteFile(filepath.Join(srcDir, "._ok.rule.yaml"), []byte("\x00\x05metadata"), 0o644); err != nil {- t.Fatal(err)- }- installed, err := InstallFiles(true, srcDir, false)- if err != nil {- t.Fatalf("install: %v", err)- }- if len(installed) != 1 || installed[0].ID != "custom-rule" {- t.Fatalf("unexpected install result: %+v", installed)- }-}--func TestInstallRejectsInvalidRule(t *testing.T) {- t.Setenv("HOME", t.TempDir())- src := filepath.Join(t.TempDir(), "bad.rule.yaml")- // valid YAML, invalid CEL field -> CheckRule fails- bad := strings.Replace(ruleWithID("bad-rule"), `e.event.action == "file.read"`, `e.nope.field == 1`, 1)- if err := os.WriteFile(src, []byte(bad), 0o644); err != nil {- t.Fatal(err)- }- if _, err := InstallFiles(true, src, false); err == nil {- t.Fatal("expected install to reject an invalid rule")- }- // Nothing should have been written.- if hasRuleFiles(StoreDir(true)) {- t.Fatal("invalid rule must not be written to the store")- }-}-func TestInstallRejectsFailingFixture(t *testing.T) {- t.Setenv("HOME", t.TempDir())- src := filepath.Join(t.TempDir(), "mismatch.rule.yaml")- // Valid, compilable rule whose embedded fixture asserts the wrong verdict:- // the match condition only fires on file.read, but the fixture feeds a- // file.write event while expecting a match. CheckRule returns no error here,- // only a failing FixtureResult.- bad := strings.Replace(ruleWithID("mismatch-rule"), "action: file.read }", "action: file.write }", 1)- if err := os.WriteFile(src, []byte(bad), 0o644); err != nil {- t.Fatal(err)- }- if _, err := InstallFiles(true, src, false); err == nil {- t.Fatal("expected install to reject a rule with a failing fixture")- }- if hasRuleFiles(StoreDir(true)) {- t.Fatal("rule with a failing fixture must not be written to the store")- }-}--func TestInstallRejectsDuplicateWithoutForce(t *testing.T) {- t.Setenv("HOME", t.TempDir())- src := filepath.Join(t.TempDir(), "dup.rule.yaml")- if err := os.WriteFile(src, []byte(ruleWithID("dup-rule")), 0o644); err != nil {- t.Fatal(err)- }- if _, err := InstallFiles(true, src, false); err != nil {- t.Fatalf("first install: %v", err)- }- if _, err := InstallFiles(true, src, false); err == nil {- t.Fatal("expected duplicate-id rejection without --force")- }- if _, err := InstallFiles(true, src, true); err != nil {- t.Fatalf("force re-install should succeed: %v", err)- }-}--func TestInstallRollsBackWhenLaterWriteFails(t *testing.T) {- t.Setenv("HOME", t.TempDir())- srcDir := t.TempDir()- if err := os.WriteFile(filepath.Join(srcDir, "a.rule.yaml"), []byte(ruleWithID("a-rule")), 0o644); err != nil {- t.Fatal(err)- }- if err := os.WriteFile(filepath.Join(srcDir, "b.rule.yaml"), []byte(ruleWithID("b-rule")), 0o644); err != nil {- t.Fatal(err)- }-- store, err := EnsureStore(true)- if err != nil {- t.Fatal(err)- }- blockedDest := filepath.Join(store, "b-rule"+ruleFileSuffix)- if err := os.Mkdir(blockedDest, 0o755); err != nil {- t.Fatal(err)- }-- if _, err := InstallFiles(true, srcDir, false); err == nil {- t.Fatal("expected install to fail on blocked destination")- }- if _, err := os.Stat(filepath.Join(store, "a-rule"+ruleFileSuffix)); !os.IsNotExist(err) {- t.Fatalf("first rule should have been rolled back, stat err=%v", err)- }- if info, err := os.Stat(blockedDest); err != nil || !info.IsDir() {- t.Fatalf("blocked destination directory should remain, info=%v err=%v", info, err)- }-}--func TestForceInstallRestoresOverwrittenRuleWhenLaterWriteFails(t *testing.T) {- t.Setenv("HOME", t.TempDir())- initial := strings.Replace(ruleWithID("a-rule"), "title: T", "title: Old", 1)- initialSrc := filepath.Join(t.TempDir(), "a.rule.yaml")- if err := os.WriteFile(initialSrc, []byte(initial), 0o644); err != nil {- t.Fatal(err)- }- if _, err := InstallFiles(true, initialSrc, false); err != nil {- t.Fatalf("initial install: %v", err)- }-- srcDir := t.TempDir()- updated := strings.Replace(ruleWithID("a-rule"), "title: T", "title: New", 1)- if err := os.WriteFile(filepath.Join(srcDir, "a.rule.yaml"), []byte(updated), 0o644); err != nil {- t.Fatal(err)- }- if err := os.WriteFile(filepath.Join(srcDir, "b.rule.yaml"), []byte(ruleWithID("b-rule")), 0o644); err != nil {- t.Fatal(err)- }-- store := StoreDir(true)- blockedDest := filepath.Join(store, "b-rule"+ruleFileSuffix)- if err := os.Mkdir(blockedDest, 0o755); err != nil {- t.Fatal(err)- }-- if _, err := InstallFiles(true, srcDir, true); err == nil {- t.Fatal("expected force install to fail on blocked destination")- }- got, err := os.ReadFile(filepath.Join(store, "a-rule"+ruleFileSuffix))- if err != nil {- t.Fatal(err)- }- if string(got) != initial {- t.Fatalf("overwritten rule was not restored:\n%s", got)- }-}--func TestRemove(t *testing.T) {- t.Setenv("HOME", t.TempDir())- src := filepath.Join(t.TempDir(), "r.rule.yaml")- if err := os.WriteFile(src, []byte(ruleWithID("removable")), 0o644); err != nil {- t.Fatal(err)- }- if _, err := InstallFiles(true, src, false); err != nil {- t.Fatalf("install: %v", err)- }- if _, err := Remove(true, "removable"); err != nil {+ if _, err := Remove(true, "custom-rule"); err != nil {t.Fatalf("remove: %v", err)}- if _, err := Remove(true, "removable"); err == nil {+ if _, err := Remove(true, "custom-rule"); err == nil {t.Fatal("removing a missing rule should error")}}--func TestRemoveRejectsPathTraversal(t *testing.T) {- home := t.TempDir()- t.Setenv("HOME", home)-- // A rule file outside the store that a traversal id could resolve to.- victimDir := t.TempDir()- victim := filepath.Join(victimDir, "victim.rule.yaml")- if err := os.WriteFile(victim, []byte("id: victim\n"), 0o644); err != nil {- t.Fatal(err)- }-- // id crafted so id+suffix == "<victimDir>/victim.rule.yaml" after Join+Clean.- rel, err := filepath.Rel(StoreDir(true), filepath.Join(victimDir, "victim"))- if err != nil {- t.Fatal(err)- }- for _, id := range []string{rel, "../../etc/passwd", "a/b", "."} {- if _, err := Remove(true, id); err == nil {- t.Fatalf("expected Remove to reject unsafe id %q", id)- }- }- if _, err := os.Stat(victim); err != nil {- t.Fatalf("victim file must not be deleted: %v", err)- }-}--func srcOf(l []LoadedRule) []Source {- s := make([]Source, len(l))- for i := range l {- s[i] = l[i].Source- }- return s-}diff --git a/cli/beacon/internal/endpoint/detect/baseline/credentials-in-curl-data.rule.yaml b/pkg/asymptoteobserve/rulestore/baseline/credentials-in-curl-data.rule.yamlsimilarity index 100%rename from cli/beacon/internal/endpoint/detect/baseline/credentials-in-curl-data.rule.yamlrename to pkg/asymptoteobserve/rulestore/baseline/credentials-in-curl-data.rule.yamldiff --git a/cli/beacon/internal/endpoint/detect/baseline/curl-pipe-to-shell.rule.yaml b/pkg/asymptoteobserve/rulestore/baseline/curl-pipe-to-shell.rule.yamlsimilarity index 100%rename from cli/beacon/internal/endpoint/detect/baseline/curl-pipe-to-shell.rule.yamlrename to pkg/asymptoteobserve/rulestore/baseline/curl-pipe-to-shell.rule.yamldiff --git a/cli/beacon/internal/endpoint/detect/baseline/exfiltrate-system-prompt.rule.yaml b/pkg/asymptoteobserve/rulestore/baseline/exfiltrate-system-prompt.rule.yamlsimilarity index 100%rename from cli/beacon/internal/endpoint/detect/baseline/exfiltrate-system-prompt.rule.yamlrename to pkg/asymptoteobserve/rulestore/baseline/exfiltrate-system-prompt.rule.yamldiff --git a/cli/beacon/internal/endpoint/detect/baseline/ignore-previous-instructions.rule.yaml b/pkg/asymptoteobserve/rulestore/baseline/ignore-previous-instructions.rule.yamlsimilarity index 100%rename from cli/beacon/internal/endpoint/detect/baseline/ignore-previous-instructions.rule.yamlrename to pkg/asymptoteobserve/rulestore/baseline/ignore-previous-instructions.rule.yamldiff --git a/cli/beacon/internal/endpoint/detect/baseline/recursive-root-delete.rule.yaml b/pkg/asymptoteobserve/rulestore/baseline/recursive-root-delete.rule.yamlsimilarity index 100%rename from cli/beacon/internal/endpoint/detect/baseline/recursive-root-delete.rule.yamlrename to pkg/asymptoteobserve/rulestore/baseline/recursive-root-delete.rule.yamldiff --git a/cli/beacon/internal/endpoint/detect/baseline/secret-read-then-egress.rule.yaml b/pkg/asymptoteobserve/rulestore/baseline/secret-read-then-egress.rule.yamlsimilarity index 100%rename from cli/beacon/internal/endpoint/detect/baseline/secret-read-then-egress.rule.yamlrename to pkg/asymptoteobserve/rulestore/baseline/secret-read-then-egress.rule.yamldiff --git a/pkg/asymptoteobserve/rulestore/rulestore.go b/pkg/asymptoteobserve/rulestore/rulestore.gonew file mode 100644index 00000000..acaf139d--- /dev/null+++ b/pkg/asymptoteobserve/rulestore/rulestore.go@@ -0,0 +1,344 @@+// Package rulestore manages the open threat-rules corpus shared across Beacon+// components: it ships a small frozen baseline embedded in the binary and loads,+// installs, and removes rules in a local store directory.+//+// The engine ships in the binary; the rule corpus is external data. Only the+// frozen baseline below is embedded, so a growing corpus loaded into the store+// never enlarges the binary.+//+// Unlike the cli/beacon `detect` adapter that wraps it, this package takes an+// explicit store directory rather than resolving one from endpoint config, so it+// can be imported from any module (beacon-hooks, an external policy provider, the+// public CLI) without depending on cli/beacon internals.+package rulestore++import (+ "embed"+ "fmt"+ "io/fs"+ "os"+ "path"+ "path/filepath"+ "sort"+ "strings"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/threatrules"+)++//go:embed baseline/*.rule.yaml+var baselineFS embed.FS++// RuleFileSuffix is the extension every rule corpus file must use.+const RuleFileSuffix = ".rule.yaml"++// Source identifies where an active rule came from.+type Source string++const (+ SourceBaseline Source = "baseline"+ SourceStore Source = "store"+)++// LoadedRule is a validated rule paired with its origin.+type LoadedRule struct {+ Rule *threatrules.Rule+ Source Source+}++// Installed reports a rule written into the store.+type Installed struct {+ ID string+ Path string+}++// EnsureStore creates the store directory if needed and returns its path.+func EnsureStore(storeDir string) (string, error) {+ if err := os.MkdirAll(storeDir, 0o755); err != nil {+ return "", fmt.Errorf("create rules store: %w", err)+ }+ return storeDir, nil+}++// Baseline returns the embedded frozen baseline rules, validated.+func Baseline() ([]*threatrules.Rule, error) {+ entries, err := baselineFS.ReadDir("baseline")+ if err != nil {+ return nil, err+ }+ names := make([]string, 0, len(entries))+ for _, e := range entries {+ if !e.IsDir() && strings.HasSuffix(e.Name(), RuleFileSuffix) {+ names = append(names, e.Name())+ }+ }+ sort.Strings(names)++ rules := make([]*threatrules.Rule, 0, len(names))+ seen := map[string]bool{}+ for _, name := range names {+ data, err := baselineFS.ReadFile(baselineRulePath(name))+ if err != nil {+ return nil, err+ }+ rule, err := threatrules.DecodeRule(data)+ if err != nil {+ return nil, fmt.Errorf("baseline %s: %w", name, err)+ }+ if err := rule.Validate(); err != nil {+ return nil, fmt.Errorf("baseline %s: %w", name, err)+ }+ if seen[rule.ID] {+ return nil, fmt.Errorf("baseline duplicate rule id %q", rule.ID)+ }+ seen[rule.ID] = true+ rules = append(rules, rule)+ }+ return rules, nil+}++func baselineRulePath(name string) string {+ return path.Join("baseline", name)+}++// LoadActive resolves the active rule set:+//+// - if rulesDir is non-empty, load only from that directory (explicit override);+// - else load from storeDir; if the store is empty (or absent), fall back to the+// embedded baseline.+//+// Every returned rule has passed validation (via threatrules.LoadDir / Baseline).+func LoadActive(storeDir, rulesDir string) ([]LoadedRule, error) {+ if rulesDir != "" {+ rules, err := threatrules.LoadDir(rulesDir)+ if err != nil {+ return nil, err+ }+ return tag(rules, SourceStore), nil+ }++ if HasRuleFiles(storeDir) {+ rules, err := threatrules.LoadDir(storeDir)+ if err != nil {+ return nil, err+ }+ return tag(rules, SourceStore), nil+ }++ base, err := Baseline()+ if err != nil {+ return nil, err+ }+ return tag(base, SourceBaseline), nil+}++func tag(rules []*threatrules.Rule, src Source) []LoadedRule {+ out := make([]LoadedRule, len(rules))+ for i, r := range rules {+ out[i] = LoadedRule{Rule: r, Source: src}+ }+ return out+}++// HasRuleFiles reports whether dir contains at least one rule corpus file.+func HasRuleFiles(dir string) bool {+ entries, err := os.ReadDir(dir)+ if err != nil {+ return false+ }+ for _, e := range entries {+ if !e.IsDir() && strings.HasSuffix(e.Name(), RuleFileSuffix) {+ return true+ }+ }+ return false+}++// InstallFiles validates every *.rule.yaml found at src (a file or directory) and+// writes the valid ones into storeDir. Each rule is validated before any file is+// written; an invalid rule aborts the whole install. A rule whose id already exists+// in the store is rejected unless force is set. Returns the rules installed.+func InstallFiles(storeDir, src string, force bool) ([]Installed, error) {+ srcPaths, err := ruleFilesAt(src)+ if err != nil {+ return nil, err+ }+ if len(srcPaths) == 0 {+ return nil, fmt.Errorf("no %s files found at %s", RuleFileSuffix, src)+ }++ store, err := EnsureStore(storeDir)+ if err != nil {+ return nil, err+ }+ existing, err := storeIDs(store)+ if err != nil {+ return nil, err+ }++ // Validate everything first; collect the bytes + target names. Abort on any failure+ // so a partial/invalid install never lands.+ type pending struct {+ id string+ dest string+ data []byte+ }+ var todo []pending+ staged := map[string]bool{}+ for _, p := range srcPaths {+ data, err := os.ReadFile(p)+ if err != nil {+ return nil, err+ }+ rule, err := threatrules.DecodeRule(data)+ if err != nil {+ return nil, fmt.Errorf("%s: %w", p, err)+ }+ results, err := threatrules.CheckRule(rule)+ if err != nil {+ return nil, fmt.Errorf("%s: %w", p, err)+ }+ for _, res := range results {+ if !res.OK() {+ return nil, fmt.Errorf("%s: fixture %s", p, res.String())+ }+ }+ // rule.ID passed CheckRule (id-grammar) above; re-assert it explicitly so the+ // store path join below can never be driven outside the store by rule content.+ if !threatrules.ValidID(rule.ID) {+ return nil, fmt.Errorf("%s: invalid rule id %q", p, rule.ID)+ }+ if (existing[rule.ID] || staged[rule.ID]) && !force {+ return nil, fmt.Errorf("rule %q already installed (use --force to overwrite)", rule.ID)+ }+ staged[rule.ID] = true+ todo = append(todo, pending{id: rule.ID, dest: filepath.Join(store, rule.ID+RuleFileSuffix), data: data})+ }++ tmpDir, err := os.MkdirTemp(store, ".install-*")+ if err != nil {+ return nil, err+ }+ defer os.RemoveAll(tmpDir)++ for _, p := range todo {+ tmpPath := filepath.Join(tmpDir, p.id+RuleFileSuffix)+ if err := os.WriteFile(tmpPath, p.data, 0o644); err != nil {+ return nil, err+ }+ }++ installed := make([]Installed, 0, len(todo))+ for _, p := range todo {+ tmpPath := filepath.Join(tmpDir, p.id+RuleFileSuffix)+ backupPath, err := moveExistingRuleAside(p.dest, tmpDir, p.id)+ if err != nil {+ rollbackInstall(installed, tmpDir)+ return nil, err+ }+ if err := os.Rename(tmpPath, p.dest); err != nil {+ if backupPath != "" {+ _ = os.Rename(backupPath, p.dest)+ }+ rollbackInstall(installed, tmpDir)+ return nil, err+ }+ installed = append(installed, Installed{ID: p.id, Path: p.dest})+ }+ return installed, nil+}++func moveExistingRuleAside(dest, tmpDir, id string) (string, error) {+ info, err := os.Lstat(dest)+ if err != nil {+ if os.IsNotExist(err) {+ return "", nil+ }+ return "", err+ }+ if info.IsDir() {+ return "", fmt.Errorf("%s exists and is a directory", dest)+ }+ backupPath := filepath.Join(tmpDir, id+".backup")+ if err := os.Rename(dest, backupPath); err != nil {+ return "", err+ }+ return backupPath, nil+}++func rollbackInstall(installed []Installed, tmpDir string) {+ for i := len(installed) - 1; i >= 0; i-- {+ backupPath := filepath.Join(tmpDir, installed[i].ID+".backup")+ _ = os.Remove(installed[i].Path)+ if _, err := os.Stat(backupPath); err == nil {+ _ = os.Rename(backupPath, installed[i].Path)+ }+ }+}++// Remove deletes a rule by id from storeDir. Returns the removed path. The id is+// validated against the rule-id grammar before use so it can never contain path+// separators or ".." segments that would resolve outside the store.+func Remove(storeDir, id string) (string, error) {+ if !threatrules.ValidID(id) {+ return "", fmt.Errorf("invalid rule id %q", id)+ }+ target := filepath.Join(storeDir, id+RuleFileSuffix)+ if err := os.Remove(target); err != nil {+ if os.IsNotExist(err) {+ return "", fmt.Errorf("rule %q is not installed in the store", id)+ }+ return "", err+ }+ return target, nil+}++// ruleFilesAt returns the *.rule.yaml files at src, which may be a single file or a dir.+func ruleFilesAt(src string) ([]string, error) {+ info, err := os.Stat(src)+ if err != nil {+ return nil, err+ }+ if !info.IsDir() {+ if !isRuleFile(src) {+ return nil, fmt.Errorf("%s is not a %s file", src, RuleFileSuffix)+ }+ return []string{src}, nil+ }+ var paths []string+ err = filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {+ if err != nil {+ return err+ }+ if !d.IsDir() && isRuleFile(path) {+ paths = append(paths, path)+ }+ return nil+ })+ if err != nil {+ return nil, err+ }+ sort.Strings(paths)+ return paths, nil+}++func isRuleFile(path string) bool {+ return strings.HasSuffix(path, RuleFileSuffix) && !strings.HasPrefix(filepath.Base(path), "._")+}++func storeIDs(store string) (map[string]bool, error) {+ ids := map[string]bool{}+ entries, err := os.ReadDir(store)+ if err != nil {+ if os.IsNotExist(err) {+ return ids, nil+ }+ return nil, err+ }+ for _, e := range entries {+ if e.IsDir() || !strings.HasSuffix(e.Name(), RuleFileSuffix) {+ continue+ }+ ids[strings.TrimSuffix(e.Name(), RuleFileSuffix)] = true+ }+ return ids, nil+}diff --git a/pkg/asymptoteobserve/rulestore/rulestore_test.go b/pkg/asymptoteobserve/rulestore/rulestore_test.gonew file mode 100644index 00000000..c871cac8--- /dev/null+++ b/pkg/asymptoteobserve/rulestore/rulestore_test.go@@ -0,0 +1,285 @@+package rulestore++import (+ "os"+ "path/filepath"+ "strings"+ "testing"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/threatrules"+)++func TestBaselineValid(t *testing.T) {+ rules, err := Baseline()+ if err != nil {+ t.Fatalf("baseline: %v", err)+ }+ if len(rules) == 0 {+ t.Fatal("embedded baseline is empty")+ }+ // Every embedded rule must pass full conformance (validate + maturity + fixtures).+ for _, r := range rules {+ if _, err := threatrules.CheckRule(r); err != nil {+ t.Errorf("baseline rule %q fails conformance: %v", r.ID, err)+ }+ }+}++func TestBaselineRulePathUsesEmbedFSSeparator(t *testing.T) {+ got := baselineRulePath("example.rule.yaml")+ if got != "baseline/example.rule.yaml" {+ t.Fatalf("embedded FS paths must use forward slashes, got %q", got)+ }+}++const aRule = `+id: %ID%+version: 1+title: T+severity: low+status: experimental+posture: detect+match: 'e.event.action == "file.read"'+emit:+ reason: ok+tests:+ - name: p+ verdict: match+ events:+ - event: { action: file.read }+`++func ruleWithID(id string) string { return strings.ReplaceAll(aRule, "%ID%", id) }++func TestLoadActiveFallsBackToBaseline(t *testing.T) {+ store := t.TempDir() // empty store -> baseline+ loaded, err := LoadActive(store, "")+ if err != nil {+ t.Fatalf("load active: %v", err)+ }+ if len(loaded) == 0 || loaded[0].Source != SourceBaseline {+ t.Fatalf("expected baseline fallback, got %d rules src=%v", len(loaded), srcOf(loaded))+ }+}++func TestLoadActiveRulesDirOverride(t *testing.T) {+ dir := t.TempDir()+ if err := os.WriteFile(filepath.Join(dir, "x.rule.yaml"), []byte(ruleWithID("override-rule")), 0o644); err != nil {+ t.Fatal(err)+ }+ loaded, err := LoadActive(t.TempDir(), dir)+ if err != nil {+ t.Fatalf("load active: %v", err)+ }+ if len(loaded) != 1 || loaded[0].Source != SourceStore || loaded[0].Rule.ID != "override-rule" {+ t.Fatalf("expected explicit rulesDir override, got %+v", loaded)+ }+}++func TestInstallThenLoadActiveUsesStore(t *testing.T) {+ store := t.TempDir()++ src := filepath.Join(t.TempDir(), "my.rule.yaml")+ if err := os.WriteFile(src, []byte(ruleWithID("custom-rule")), 0o644); err != nil {+ t.Fatal(err)+ }+ installed, err := InstallFiles(store, src, false)+ if err != nil {+ t.Fatalf("install: %v", err)+ }+ if len(installed) != 1 || installed[0].ID != "custom-rule" {+ t.Fatalf("unexpected install result: %+v", installed)+ }++ loaded, err := LoadActive(store, "")+ if err != nil {+ t.Fatalf("load active: %v", err)+ }+ if len(loaded) != 1 || loaded[0].Source != SourceStore || loaded[0].Rule.ID != "custom-rule" {+ t.Fatalf("expected store rule, got %+v", loaded)+ }+}++func TestInstallFilesIgnoresAppleDoubleRuleFiles(t *testing.T) {+ store := t.TempDir()+ srcDir := t.TempDir()+ if err := os.WriteFile(filepath.Join(srcDir, "ok.rule.yaml"), []byte(ruleWithID("custom-rule")), 0o644); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(filepath.Join(srcDir, "._ok.rule.yaml"), []byte("\x00\x05metadata"), 0o644); err != nil {+ t.Fatal(err)+ }+ installed, err := InstallFiles(store, srcDir, false)+ if err != nil {+ t.Fatalf("install: %v", err)+ }+ if len(installed) != 1 || installed[0].ID != "custom-rule" {+ t.Fatalf("unexpected install result: %+v", installed)+ }+}++func TestInstallRejectsInvalidRule(t *testing.T) {+ store := t.TempDir()+ src := filepath.Join(t.TempDir(), "bad.rule.yaml")+ // valid YAML, invalid CEL field -> CheckRule fails+ bad := strings.Replace(ruleWithID("bad-rule"), `e.event.action == "file.read"`, `e.nope.field == 1`, 1)+ if err := os.WriteFile(src, []byte(bad), 0o644); err != nil {+ t.Fatal(err)+ }+ if _, err := InstallFiles(store, src, false); err == nil {+ t.Fatal("expected install to reject an invalid rule")+ }+ // Nothing should have been written.+ if HasRuleFiles(store) {+ t.Fatal("invalid rule must not be written to the store")+ }+}++func TestInstallRejectsFailingFixture(t *testing.T) {+ store := t.TempDir()+ src := filepath.Join(t.TempDir(), "mismatch.rule.yaml")+ // Valid, compilable rule whose embedded fixture asserts the wrong verdict:+ // the match condition only fires on file.read, but the fixture feeds a+ // file.write event while expecting a match. CheckRule returns no error here,+ // only a failing FixtureResult.+ bad := strings.Replace(ruleWithID("mismatch-rule"), "action: file.read }", "action: file.write }", 1)+ if err := os.WriteFile(src, []byte(bad), 0o644); err != nil {+ t.Fatal(err)+ }+ if _, err := InstallFiles(store, src, false); err == nil {+ t.Fatal("expected install to reject a rule with a failing fixture")+ }+ if HasRuleFiles(store) {+ t.Fatal("rule with a failing fixture must not be written to the store")+ }+}++func TestInstallRejectsDuplicateWithoutForce(t *testing.T) {+ store := t.TempDir()+ src := filepath.Join(t.TempDir(), "dup.rule.yaml")+ if err := os.WriteFile(src, []byte(ruleWithID("dup-rule")), 0o644); err != nil {+ t.Fatal(err)+ }+ if _, err := InstallFiles(store, src, false); err != nil {+ t.Fatalf("first install: %v", err)+ }+ if _, err := InstallFiles(store, src, false); err == nil {+ t.Fatal("expected duplicate-id rejection without --force")+ }+ if _, err := InstallFiles(store, src, true); err != nil {+ t.Fatalf("force re-install should succeed: %v", err)+ }+}++func TestInstallRollsBackWhenLaterWriteFails(t *testing.T) {+ store := t.TempDir()+ srcDir := t.TempDir()+ if err := os.WriteFile(filepath.Join(srcDir, "a.rule.yaml"), []byte(ruleWithID("a-rule")), 0o644); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(filepath.Join(srcDir, "b.rule.yaml"), []byte(ruleWithID("b-rule")), 0o644); err != nil {+ t.Fatal(err)+ }++ blockedDest := filepath.Join(store, "b-rule"+RuleFileSuffix)+ if err := os.Mkdir(blockedDest, 0o755); err != nil {+ t.Fatal(err)+ }++ if _, err := InstallFiles(store, srcDir, false); err == nil {+ t.Fatal("expected install to fail on blocked destination")+ }+ if _, err := os.Stat(filepath.Join(store, "a-rule"+RuleFileSuffix)); !os.IsNotExist(err) {+ t.Fatalf("first rule should have been rolled back, stat err=%v", err)+ }+ if info, err := os.Stat(blockedDest); err != nil || !info.IsDir() {+ t.Fatalf("blocked destination directory should remain, info=%v err=%v", info, err)+ }+}++func TestForceInstallRestoresOverwrittenRuleWhenLaterWriteFails(t *testing.T) {+ store := t.TempDir()+ initial := strings.Replace(ruleWithID("a-rule"), "title: T", "title: Old", 1)+ initialSrc := filepath.Join(t.TempDir(), "a.rule.yaml")+ if err := os.WriteFile(initialSrc, []byte(initial), 0o644); err != nil {+ t.Fatal(err)+ }+ if _, err := InstallFiles(store, initialSrc, false); err != nil {+ t.Fatalf("initial install: %v", err)+ }++ srcDir := t.TempDir()+ updated := strings.Replace(ruleWithID("a-rule"), "title: T", "title: New", 1)+ if err := os.WriteFile(filepath.Join(srcDir, "a.rule.yaml"), []byte(updated), 0o644); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(filepath.Join(srcDir, "b.rule.yaml"), []byte(ruleWithID("b-rule")), 0o644); err != nil {+ t.Fatal(err)+ }++ blockedDest := filepath.Join(store, "b-rule"+RuleFileSuffix)+ if err := os.Mkdir(blockedDest, 0o755); err != nil {+ t.Fatal(err)+ }++ if _, err := InstallFiles(store, srcDir, true); err == nil {+ t.Fatal("expected force install to fail on blocked destination")+ }+ got, err := os.ReadFile(filepath.Join(store, "a-rule"+RuleFileSuffix))+ if err != nil {+ t.Fatal(err)+ }+ if string(got) != initial {+ t.Fatalf("overwritten rule was not restored:\n%s", got)+ }+}++func TestRemove(t *testing.T) {+ store := t.TempDir()+ src := filepath.Join(t.TempDir(), "r.rule.yaml")+ if err := os.WriteFile(src, []byte(ruleWithID("removable")), 0o644); err != nil {+ t.Fatal(err)+ }+ if _, err := InstallFiles(store, src, false); err != nil {+ t.Fatalf("install: %v", err)+ }+ if _, err := Remove(store, "removable"); err != nil {+ t.Fatalf("remove: %v", err)+ }+ if _, err := Remove(store, "removable"); err == nil {+ t.Fatal("removing a missing rule should error")+ }+}++func TestRemoveRejectsPathTraversal(t *testing.T) {+ store := t.TempDir()++ // A rule file outside the store that a traversal id could resolve to.+ victimDir := t.TempDir()+ victim := filepath.Join(victimDir, "victim.rule.yaml")+ if err := os.WriteFile(victim, []byte("id: victim\n"), 0o644); err != nil {+ t.Fatal(err)+ }++ // id crafted so id+suffix == "<victimDir>/victim.rule.yaml" after Join+Clean.+ rel, err := filepath.Rel(store, filepath.Join(victimDir, "victim"))+ if err != nil {+ t.Fatal(err)+ }+ for _, id := range []string{rel, "../../etc/passwd", "a/b", "."} {+ if _, err := Remove(store, id); err == nil {+ t.Fatalf("expected Remove to reject unsafe id %q", id)+ }+ }+ if _, err := os.Stat(victim); err != nil {+ t.Fatalf("victim file must not be deleted: %v", err)+ }+}++func srcOf(l []LoadedRule) []Source {+ s := make([]Source, len(l))+ for i := range l {+ s[i] = l[i].Source+ }+ return s+}