Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

perf(threat-rules): compile constant matches() patterns once (#1138)

kyle-elliott-asymptoteb2bd82dc3d79
diff --git a/pkg/asymptoteobserve/threatrules/cel.go b/pkg/asymptoteobserve/threatrules/cel.go
index 81cef75a..96dfbaf2 100644
--- a/pkg/asymptoteobserve/threatrules/cel.go
+++ b/pkg/asymptoteobserve/threatrules/cel.go
@@ -3,12 +3,14 @@ package threatrules
import (
"fmt"
"reflect"
+ "regexp"
"strings"
"sync"
"github.com/google/cel-go/cel"
celast "github.com/google/cel-go/common/ast"
"github.com/google/cel-go/ext"
+ "github.com/google/cel-go/interpreter"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)
@@ -97,6 +99,24 @@ const MaxEvalCost uint64 = 5_000_000
// stop quickly.
var evalCostLimit = MaxEvalCost
+// constantRegex compiles the constant pattern of a matches() call once, when the program is
+// built. Without it cel-go compiles the pattern on every evaluation, for every rule and every
+// event, which dominated the time and allocation of a scan. A pattern that does not compile is
+// left to the call as written, so a rule with a broken pattern still loads and still fails when
+// it is evaluated, exactly as before (TestInvalidRegexErrorsAtEval): moving that failure to load
+// time would make rulestore skip the rule instead. Results and CEL costs are unchanged
+// (TestConstantRegexChangesNothing).
+var constantRegex = &interpreter.RegexOptimization{
+ Function: interpreter.MatchesRegexOptimization.Function,
+ RegexIndex: interpreter.MatchesRegexOptimization.RegexIndex,
+ Factory: func(call interpreter.InterpretableCall, pattern string) (interpreter.InterpretableCall, error) {
+ if _, err := regexp.Compile(pattern); err != nil {
+ return call, nil
+ }
+ return interpreter.MatchesRegexOptimization.Factory(call, pattern)
+ },
+}
+
// CompileMatch compiles a single match expression and verifies it types to bool. The
// returned program is safe for concurrent reuse across events, and each evaluation is
// bounded by MaxEvalCost: one that exceeds it fails with an error rather than matching. A
@@ -124,7 +144,7 @@ func compileMatch(expr string) (cel.Program, *cel.Ast, error) {
if ast.OutputType() != cel.BoolType {
return nil, nil, fmt.Errorf("match expression must evaluate to bool, got %s", ast.OutputType())
}
- prog, err := env.Program(ast, cel.CostLimit(evalCostLimit))
+ prog, err := env.Program(ast, cel.CostLimit(evalCostLimit), cel.OptimizeRegex(constantRegex))
if err != nil {
return nil, nil, fmt.Errorf("build CEL program: %w", err)
}
diff --git a/pkg/asymptoteobserve/threatrules/conformance_test.go b/pkg/asymptoteobserve/threatrules/conformance_test.go
index 3644401b..3ebb7437 100644
--- a/pkg/asymptoteobserve/threatrules/conformance_test.go
+++ b/pkg/asymptoteobserve/threatrules/conformance_test.go
@@ -14,7 +14,7 @@ import (
// repoRoot locates the repository root by walking up from this test file until it finds
// the spec sentinel (spec/threat-rules/VERSION). This keeps tests hermetic without
// hardcoded absolute paths.
-func repoRoot(t *testing.T) string {
+func repoRoot(t testing.TB) string {
t.Helper()
_, thisFile, _, ok := runtime.Caller(0)
if !ok {
@@ -33,7 +33,7 @@ func repoRoot(t *testing.T) string {
}
}
-func rulesDir(t *testing.T) string { return filepath.Join(repoRoot(t), "rules") }
+func rulesDir(t testing.TB) string { return filepath.Join(repoRoot(t), "rules") }
func specDir(t *testing.T) string { return filepath.Join(repoRoot(t), "spec", "threat-rules") }
diff --git a/pkg/asymptoteobserve/threatrules/limits_test.go b/pkg/asymptoteobserve/threatrules/limits_test.go
index 79f1a337..7dc6df9b 100644
--- a/pkg/asymptoteobserve/threatrules/limits_test.go
+++ b/pkg/asymptoteobserve/threatrules/limits_test.go
@@ -97,7 +97,7 @@ func TestScanNamesTheRuleThatExceededTheCostLimit(t *testing.T) {
}
// corpusRules loads the open corpus and the embedded baseline.
-func corpusRules(t *testing.T) []*Rule {
+func corpusRules(t testing.TB) []*Rule {
t.Helper()
var all []*Rule
for _, dir := range []string{rulesDir(t), filepath.Join(repoRoot(t), "pkg", "asymptoteobserve", "rulestore", "baseline")} {
diff --git a/pkg/asymptoteobserve/threatrules/regex_test.go b/pkg/asymptoteobserve/threatrules/regex_test.go
new file mode 100644
index 00000000..904bfdd9
--- /dev/null
+++ b/pkg/asymptoteobserve/threatrules/regex_test.go
@@ -0,0 +1,100 @@
+package threatrules
+
+import (
+ "reflect"
+ "strings"
+ "testing"
+
+ "github.com/google/cel-go/cel"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+// Compiling constant matches() patterns once must change nothing a rule can observe: every
+// expression in the open corpus and the embedded baseline, on every fixture event and on a
+// pessimistic copy whose every string is 64 KiB, gives the same result, the same error, and the
+// same CEL cost (the cost limit is a safety contract) as the program cel-go builds by default.
+func TestConstantRegexChangesNothing(t *testing.T) {
+ env, err := Env()
+ if err != nil {
+ t.Fatal(err)
+ }
+ big := strings.Repeat("a", asymptoteobserve.DefaultMaxEventBytes)
+ compared := 0
+ for _, r := range corpusRules(t) {
+ var checks []func(what string, e asymptoteobserve.Event)
+ for _, expr := range ruleExpressions(r) {
+ ast, iss := env.Compile(expr)
+ if iss != nil && iss.Err() != nil {
+ t.Fatalf("%s: %v", r.ID, iss.Err())
+ }
+ plain, err := env.Program(ast, cel.CostLimit(evalCostLimit), cel.EvalOptions(cel.OptTrackCost))
+ if err != nil {
+ t.Fatalf("%s: %v", r.ID, err)
+ }
+ optimized, err := env.Program(ast, cel.CostLimit(evalCostLimit), cel.EvalOptions(cel.OptTrackCost), cel.OptimizeRegex(constantRegex))
+ if err != nil {
+ t.Fatalf("%s: %v", r.ID, err)
+ }
+ checks = append(checks, func(what string, e asymptoteobserve.Event) {
+ in := map[string]any{eventVar: e}
+ wantVal, wantDet, wantErr := plain.Eval(in)
+ gotVal, gotDet, gotErr := optimized.Eval(in)
+ if (wantErr == nil) != (gotErr == nil) {
+ t.Fatalf("%s %s: error %v, default %v", r.ID, what, gotErr, wantErr)
+ }
+ if wantErr == nil && !reflect.DeepEqual(gotVal.Value(), wantVal.Value()) {
+ t.Fatalf("%s %s: %v, default %v", r.ID, what, gotVal, wantVal)
+ }
+ if *gotDet.ActualCost() != *wantDet.ActualCost() {
+ t.Fatalf("%s %s: cost %d, default %d", r.ID, what, *gotDet.ActualCost(), *wantDet.ActualCost())
+ }
+ compared++
+ })
+ }
+ // Every step on the fixtures as written first: fillStrings writes through the pointers
+ // EventList's shallow copies share with r.Tests, so the 64 KiB pass comes after.
+ for _, pessimistic := range []bool{false, true} {
+ for _, f := range r.Tests {
+ for _, ev := range f.EventList() {
+ ev = withDerivedFields(ev)
+ what := f.Name
+ if pessimistic {
+ fillStrings(reflect.ValueOf(&ev).Elem(), big, 0)
+ what += " (64 KiB strings)"
+ }
+ for _, check := range checks {
+ check(what, ev)
+ }
+ }
+ }
+ }
+ }
+ if compared < 1000 {
+ t.Fatalf("compared only %d evaluations; the corpus did not load", compared)
+ }
+ t.Logf("%d evaluations agree", compared)
+}
+
+// BenchmarkScanCorpus scans every fixture event of the open corpus and the embedded baseline
+// with every rule, as `beacon scan` does.
+func BenchmarkScanCorpus(b *testing.B) {
+ var compiled []*CompiledRule
+ var events []asymptoteobserve.Event
+ for _, r := range corpusRules(b) {
+ c, err := Compile(r)
+ if err != nil {
+ b.Fatalf("%s: %v", r.ID, err)
+ }
+ compiled = append(compiled, c)
+ for _, f := range r.Tests {
+ events = append(events, f.EventList()...)
+ }
+ }
+ b.ReportAllocs()
+ for b.Loop() {
+ if _, err := ScanEvents(compiled, events); err != nil {
+ b.Fatal(err)
+ }
+ }
+}