Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
perf(harness): a realistic mix in which most events do not alert (depth 4.4)
Beenu Arora6b6370a1b2d1
diff --git a/scripts/perf/load_harness.py b/scripts/perf/load_harness.pyindex a9c18a7b4..6af836ca2 100644--- a/scripts/perf/load_harness.py+++ b/scripts/perf/load_harness.py@@ -85,6 +85,11 @@ from gate_toolkit import repo_root, self_test_main # noqa: E402#: pins the two together, because a silent disagreement would look like a#: pipeline that lost every event.MARKER_PREFIX = "aisoc-load"+#: Must equal ``BenignMarkerPrefix`` in services/demo-producer/main.go.+#: The two prefixes are distinguishable because the alerting one is+#: always followed by a space and the run id, so a `like` on+#: "aisoc-load <run> %" cannot match a benign title.+BENIGN_MARKER_PREFIX = "aisoc-load-benign"#: ``aisoc-load <run-id> seq=<n> t=<unix-nanoseconds>``_TITLE_RE = re.compile(rf"^{re.escape(MARKER_PREFIX)} (?P<run>[A-Za-z0-9_-]+) seq=(?P<seq>\d+) t=(?P<sent>\d+)$")@@ -322,6 +327,26 @@ class ProducerSummary:def accepted_eps(self) -> float:return float(self.raw["accepted_eps"])+ @property+ def alerting_sent(self) -> int | None:+ """How many events the producer sent in the alerting shape.++ ``None`` for a summary written before the realistic mix existed,+ which is the honest answer rather than falling back to the+ accepted count and quietly grading an old run against a+ denominator it never had.+ """+ value = self.raw.get("alerting_events_sent")+ return int(value) if value is not None else None++ @property+ def benign_sent(self) -> int:+ return int(self.raw.get("benign_events_sent") or 0)++ @property+ def alert_every(self) -> int:+ return int(self.raw.get("alert_every") or 1)+@dataclassclass AlertRow:@@ -418,6 +443,7 @@ def run_producer(target_eps: int,run_id: str,summary_path: Path,+ alert_every: int = 1,) -> ProducerSummary:# Where `go run` is invoked from, which is not the repository root. The# producer is its own module (`services/demo-producer/go.mod`) and there@@ -458,6 +484,8 @@ def run_producer(run_id,"--summary",str(summary_path),+ "--alert-every",+ str(max(1, alert_every)),]done = subprocess.run(argv, cwd=cwd, capture_output=True, text=True, check=False) # noqa: S603if done.returncode != 0:@@ -465,6 +493,21 @@ def run_producer(return ProducerSummary(json.loads(summary_path.read_text(encoding="utf-8")))+def count_benign_alerts(target: Target, run_id: str) -> int:+ """Alerts raised by events that were supposed to raise none.++ Counted rather than assumed zero. The whole point of the realistic+ mix is that benign telemetry reaches the lake and produces nothing;+ an all-alerting harness cannot observe that property at all, and a+ mix that never checks it is the same harness with more events.+ """+ raw = target.sql(f"select count(*) from alerts where title like '{BENIGN_MARKER_PREFIX} {run_id} %'")+ try:+ return int(raw or 0)+ except (TypeError, ValueError):+ return 0++def collect_alerts(target: Target, run_id: str) -> list[AlertRow]:query = f"select title, extract(epoch from created_at) from alerts where title like '{MARKER_PREFIX} {run_id} %'"rows: list[AlertRow] = []@@ -538,6 +581,7 @@ def build_report(drain_seconds: float,settled: bool,shape: dict[str, Any] | None = None,+ benign_alerts: int = 0,) -> dict[str, Any]:accepted = summary.acceptedseen: dict[int, int] = {}@@ -575,12 +619,30 @@ def build_report(else:metrics["pipeline_eps"] = unmeasured("pipeline_eps", "alerts/s", "fewer than two alerts landed")+ # Depth plan 4.4. With a realistic mix most accepted events are benign+ # and are *supposed* to produce no alert, so dividing alerts by accepted+ # events would report a delivery ratio of 0.01 for a pipeline that lost+ # nothing. The denominator is the alerting population the producer+ # actually sent, which it counts rather than computing from+ # `total / alert_every` — a run that stops early would otherwise be+ # graded against an expectation it never reached.+ expected_alerting = summary.alerting_sent if summary.alerting_sent is not None else acceptedmetrics["delivery_ratio"] = (- measured("delivery_ratio", "fraction", len(seen) / accepted, accepted)- if accepted > 0- else unmeasured("delivery_ratio", "fraction", "ingest accepted no events")+ measured("delivery_ratio", "fraction", len(seen) / expected_alerting, expected_alerting)+ if expected_alerting > 0+ else unmeasured("delivery_ratio", "fraction", "no alerting events were sent"))+ # The property the realistic mix exists to assert. Benign telemetry+ # reaching the lake and producing no alert is what a real estate looks+ # like; benign telemetry that alerts means the corpus is firing on+ # routine traffic, which is the failure the mix is designed to expose+ # and which an all-alerting harness cannot see at all.+ if summary.benign_sent:+ metrics["benign_alert_rate"] = measured("benign_alert_rate", "fraction", benign_alerts / summary.benign_sent, summary.benign_sent)+ else:+ metrics["benign_alert_rate"] = unmeasured("benign_alert_rate", "fraction", "no benign events were sent (alert-every is 1)")+for key, (value, reason) in (("consumer_lag_in_flight", lag_in_flight), ("consumer_lag_drained", lag_drained)):metrics[key] = measured(key, "messages", float(value), 1) if value is not None else unmeasured(key, "messages", reason)@@ -611,6 +673,10 @@ def build_report("accepted_events": accepted,"alerts_observed": len(alerts),"distinct_events_alerted": len(seen),+ "alerting_events_sent": summary.alerting_sent,+ "benign_events_sent": summary.benign_sent,+ "benign_events_that_alerted": benign_alerts,+ "alert_every": summary.alert_every,"missing_events": missing,"duplicate_alert_rows": duplicates,"push_wall_seconds": summary.wall_seconds,@@ -681,6 +747,23 @@ def main(argv: list[str] | None = None) -> int:parser.add_argument("--assert-eps-floor", type=float, help="fail if pipeline throughput is below this")parser.add_argument("--assert-p95-ceiling-ms", type=float, help="fail if p95 event-to-alert latency exceeds this")parser.add_argument("--assert-no-loss", action="store_true", help="fail if any accepted event produced no alert")+ parser.add_argument(+ "--alert-every",+ type=int,+ default=1,+ help=(+ "send one alerting event every Nth event and benign telemetry in between. 1 (the default) "+ "keeps the historical all-alerting load; 100 is a realistic estate where almost nothing alerts"+ ),+ )+ parser.add_argument(+ "--assert-benign-alert-rate-max",+ type=float,+ help=(+ "fail if more than this fraction of the benign events raised an alert. Only meaningful with "+ "--alert-every above 1; the realistic mix exists to make this measurable"+ ),+ )args = parser.parse_args(argv)root = repo_root()@@ -726,6 +809,7 @@ def main(argv: list[str] | None = None) -> int:target_eps=args.target_eps,run_id=run_id,summary_path=summary_path,+ alert_every=args.alert_every,)# Sampled immediately after the push rather than after the drain: the# queue is at its deepest here, and a lag read once the pipeline has caught@@ -738,7 +822,11 @@ def main(argv: list[str] | None = None) -> int:settled = Falsewhile time.time() - drain_started < args.drain_timeout:count = int(target.sql(f"select count(*) from alerts where title like '{MARKER_PREFIX} {run_id} %'") or 0)- if count >= summary.accepted:+ # With a realistic mix most accepted events are benign and raise+ # nothing, so waiting for one alert per accepted event would always+ # time out and report the drain as unsettled. The target is the+ # alerting population the producer counted.+ if count >= (summary.alerting_sent if summary.alerting_sent is not None else summary.accepted):settled = Truebreakif count == previous and count > 0:@@ -804,6 +892,7 @@ def main(argv: list[str] | None = None) -> int:drain_seconds=drain_seconds,settled=settled,shape=shape,+ benign_alerts=count_benign_alerts(target, run_id),)report["run"]["total_wall_seconds"] = time.time() - push_started@@ -828,6 +917,18 @@ def main(argv: list[str] | None = None) -> int:failures.append(f"p95 latency {body['value']:.0f}ms exceeds the ceiling {args.assert_p95_ceiling_ms:.0f}ms")if args.assert_no_loss and report["run"]["missing_events"] > 0:failures.append(f"{report['run']['missing_events']} accepted events produced no alert")+ if args.assert_benign_alert_rate_max is not None:+ body = report["metrics"]["benign_alert_rate"]+ if not body["measured"]:+ # Not a pass. A ceiling on a figure that was not measured is a+ # gate that reports success for a run it could not grade, which+ # is the shape this repository keeps finding.+ failures.append(f"the benign alert rate was not measured ({body['reason']}), so the ceiling cannot be met")+ elif body["value"] > args.assert_benign_alert_rate_max:+ failures.append(+ f"{body['value']:.4f} of benign events raised an alert, above the ceiling "+ f"{args.assert_benign_alert_rate_max:.4f}; the corpus is firing on routine traffic"+ )for failure in failures:print(f"FAIL: {failure}", file=sys.stderr)diff --git a/services/demo-producer/main.go b/services/demo-producer/main.goindex 6af39c36d..af02dcfae 100644--- a/services/demo-producer/main.go+++ b/services/demo-producer/main.go@@ -408,10 +408,106 @@ func loadEvent(runID string, seq int64, sentNanos int64) map[string]interface{}}}+// BenignMarkerPrefix opens the title of every *non-alerting* load event.+// Distinct from MarkerPrefix so the harness can count the two populations+// separately: the alerting ones are what latency is measured over, and the+// benign ones existing in the lake while producing no alert is the property+// the realistic mix is for.+const BenignMarkerPrefix = "aisoc-load-benign"++// benignLoadEvent is the other 99% of a real estate's telemetry.+//+// Depth plan 4.4. The load harness used to send one event shape, every+// instance of which became an alert. That measures a pipeline nobody runs:+// in a real deployment almost every event is routine, and the work fusion+// actually does is evaluating the corpus against traffic that matches+// nothing. A harness in which every event alerts over-measures the alert+// path and *under*-measures detection, which is the stage parity 6.9 is+// about.+//+// Deliberately shaped like real cloud/identity telemetry rather than as a+// near-miss of the alerting event: a benign event built by mutating the+// malicious one shares its field namespace, so it exercises the same+// handful of rules and tells you nothing about the corpus. These carry the+// fields a CloudTrail or Okta event carries, so the rule index resolves+// them the way production traffic would.+//+// `severity: info` and an ordinary event type keep it below the promoter's+// bar as well as the detection corpus's, so an alert from one of these is+// a real finding about the pipeline rather than an artefact of the fixture.+func benignLoadEvent(runID string, seq int64, sentNanos int64) map[string]interface{} {+ shapes := []map[string]interface{}{+ {+ "event_type": "s3.GetObject",+ "description": "GetObject succeeded",+ "user": fmt.Sprintf("svc-reporting%03d", seq%50),+ "src_ip": fmt.Sprintf("10.%d.%d.%d", seq%250, (seq/250)%250, (seq/60000)%250),+ "action": "allow",+ "outcome": "success",+ "region": "us-east-1",+ "account_id": "123456789012",+ "user_agent": "aws-sdk-go/1.44",+ "resource": "arn:aws:s3:::reports",+ },+ {+ "event_type": "user.session.start",+ "description": "sign-in succeeded",+ "user": fmt.Sprintf("analyst%03d", seq%200),+ "src_ip": fmt.Sprintf("198.51.100.%d", seq%250),+ "action": "allow",+ "outcome": "success",+ "mfa": "true",+ "app": "okta",+ "user_agent": "Mozilla/5.0",+ },+ {+ "event_type": "http.request",+ "description": "200 GET /healthz",+ "src_ip": fmt.Sprintf("10.20.%d.%d", seq%250, (seq/250)%250),+ "status_code": 200,+ "method": "GET",+ "path": "/healthz",+ "bytes_out": 128,+ "outcome": "success",+ },+ }+ event := map[string]interface{}{}+ for k, v := range shapes[seq%int64(len(shapes))] {+ event[k] = v+ }+ event["severity"] = "info"+ event["host"] = fmt.Sprintf("LOADB-%s-%06d", runID, seq)+ event["title"] = fmt.Sprintf("%s %s seq=%d t=%d", BenignMarkerPrefix, runID, seq, sentNanos)+ event["timestamp"] = time.Unix(0, sentNanos).UTC().Format(time.RFC3339Nano)+ return event+}++// loadEventFor picks the shape for this sequence number from the configured+// mix.+//+// Deterministic rather than random: the same seq always produces the same+// shape, so two runs of the same configuration push the same traffic and a+// difference between them is the system rather than the dice. `alertEvery`+// of 0 means every event alerts, which is the previous behaviour and the+// default.+func loadEventFor(runID string, seq int64, sentNanos int64, alertEvery int64) (map[string]interface{}, bool) {+ if alertEvery <= 1 {+ return loadEvent(runID, seq, sentNanos), true+ }+ if seq%alertEvery == 0 {+ return loadEvent(runID, seq, sentNanos), true+ }+ return benignLoadEvent(runID, seq, sentNanos), false+}+type loadSummary struct {RunID string `json:"run_id"`Mode string `json:"mode"`MarkerPrefix string `json:"marker_prefix"`+ BenignPrefix string `json:"benign_marker_prefix"`+ AlertEvery int64 `json:"alert_every"`+ AlertingSent int64 `json:"alerting_events_sent"`+ BenignSent int64 `json:"benign_events_sent"`IngestURL string `json:"ingest_url"`Tenant string `json:"tenant"`StartedAt string `json:"started_at"`@@ -441,6 +537,11 @@ func runLoad(ctx context.Context, opts options) loadSummary {st := newStats()var seq atomic.Int64var issued atomic.Int64+ // Counted here rather than derived from `total / alertEvery`: the run+ // can stop early, and a computed expectation would then disagree with+ // what was sent in exactly the case the harness needs it most.+ var alertingSent atomic.Int64+ var benignSent atomic.Int64// A shared bucket rather than a per-worker ticker: pacing per worker makes// the aggregate rate depend on how many workers happen to be blocked.@@ -489,7 +590,14 @@ func runLoad(ctx context.Context, opts options) loadSummary {batch := make([]map[string]interface{}, 0, size)now := time.Now().UnixNano()for i := 0; i < size; i++ {- batch = append(batch, loadEvent(opts.runID, seq.Add(1)-1, now))+ next := seq.Add(1) - 1+ event, alerting := loadEventFor(opts.runID, next, now, opts.alertEvery)+ if alerting {+ alertingSent.Add(1)+ } else {+ benignSent.Add(1)+ }+ batch = append(batch, event)}err := postBatch(ctx, client, opts, ingestRequest{ConnectorID: "load-" + opts.runID,@@ -522,6 +630,10 @@ func runLoad(ctx context.Context, opts options) loadSummary {RunID: opts.runID,Mode: "load",MarkerPrefix: MarkerPrefix,+ BenignPrefix: BenignMarkerPrefix,+ AlertEvery: opts.alertEvery,+ AlertingSent: alertingSent.Load(),+ BenignSent: benignSent.Load(),IngestURL: opts.ingestURL,Tenant: opts.tenant,StartedAt: started.UTC().Format(time.RFC3339Nano),@@ -565,6 +677,9 @@ type options struct {workers inttargetEPS intsummary string+ // alertEvery shapes the mix: one alerting event every N, benign+ // telemetry in between. 1 is the historical all-alerting load.+ alertEvery int64}func main() {@@ -581,6 +696,8 @@ func main() {flag.IntVar(&opts.workers, "workers", 8, "Load mode: concurrent senders")flag.IntVar(&opts.targetEPS, "target-eps", 0, "Load mode: pace to this many events per second (0 = saturate)")flag.StringVar(&opts.summary, "summary", "", "Load mode: write the run summary as JSON to this path")+ flag.Int64Var(&opts.alertEvery, "alert-every", 1, "Load mode: send an alerting event every Nth event and benign telemetry in between. "++ "1 (the default) is the historical all-alerting mix; 100 is a realistic estate where almost nothing alerts")flag.Parse()ctx, cancel := context.WithCancel(context.Background())diff --git a/tests/test_load_harness_realistic_mix.py b/tests/test_load_harness_realistic_mix.pynew file mode 100644index 000000000..2908210b2--- /dev/null+++ b/tests/test_load_harness_realistic_mix.py@@ -0,0 +1,185 @@+"""The realistic mix is a mix, and the harness grades it as one (depth 4.4).++The load harness used to send one event shape, every instance of which+became an alert. That measures a pipeline nobody runs: in a real estate+almost every event is routine, and the work fusion actually does is+evaluating the corpus against traffic that matches nothing. An+all-alerting harness over-measures the alert path and *under*-measures+detection, which is the stage parity 6.9 is about.++Three things have to hold for the mix to mean anything, and each has+failed in this repository's harnesses before:++* the Go producer and the Python harness must agree on the marker, or+ the harness counts zero of one population;+* the delivery ratio must be taken against the *alerting* population, or+ a lossless pipeline reports 0.01;+* a ceiling on a figure that was **not measured** must fail, not pass.+"""++from __future__ import annotations++import importlib.util+import re+import sys+from pathlib import Path++import pytest++REPO_ROOT = Path(__file__).resolve().parents[1]+PRODUCER_GO = REPO_ROOT / "services" / "demo-producer" / "main.go"+++def _load(name: str, relative: str):+ spec = importlib.util.spec_from_file_location(name, REPO_ROOT / relative)+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module+ spec.loader.exec_module(module)+ return module+++harness = _load("aisoc_load_harness_mix", "scripts/perf/load_harness.py")+++def _go_const(name: str) -> str:+ source = PRODUCER_GO.read_text(encoding="utf-8")+ match = re.search(rf'^const {name} = "([^"]+)"$', source, flags=re.M)+ assert match, f"{name} is not declared in {PRODUCER_GO}"+ return match.group(1)+++def test_the_benign_marker_matches_the_one_the_producer_writes():+ # The sibling of the existing marker-parity test. A harness counting a+ # prefix the producer does not write reports zero benign alerts+ # forever, which looks exactly like a clean run.+ assert harness.BENIGN_MARKER_PREFIX == _go_const("BenignMarkerPrefix")+++def test_the_two_markers_cannot_be_confused_by_the_sql():+ """The benign prefix *contains* the alerting one.++ `aisoc-load-benign` starts with `aisoc-load`, so the alerting query+ would match benign titles were it not for the space and run id that+ follow. This pins the property rather than trusting it.+ """+ assert harness.BENIGN_MARKER_PREFIX.startswith(harness.MARKER_PREFIX)+ run = "run1"+ alerting_pattern = f"{harness.MARKER_PREFIX} {run} "+ benign_title = f"{harness.BENIGN_MARKER_PREFIX} {run} seq=1 t=2"+ assert not benign_title.startswith(alerting_pattern), (+ "a benign alert would be counted as an alerting one and the mix would measure nothing"+ )+++def _summary(**overrides) -> dict:+ base = {+ "run_id": "r",+ "accepted_events": 1000,+ "attempted_events": 1000,+ "wall_seconds": 10.0,+ "accepted_eps": 100.0,+ "alerting_events_sent": 10,+ "benign_events_sent": 990,+ "alert_every": 100,+ }+ base.update(overrides)+ return base+++def _report(alerts, *, benign_alerts=0, **summary_overrides) -> dict:+ target = harness.Target(name="t", psql=[], kafka=[], stats=None, ingest_url="")+ return harness.build_report(+ target=target,+ summary=harness.ProducerSummary(_summary(**summary_overrides)),+ alerts=alerts,+ clock_offset=0.0,+ clock_spread=0.0,+ lag_in_flight=(None, "not measured"),+ lag_drained=(None, "not measured"),+ dead_letters=0,+ dead_letter_reason="",+ resources=[],+ drain_seconds=1.0,+ settled=True,+ benign_alerts=benign_alerts,+ )+++def _alert_rows(count: int) -> list:+ return [harness.AlertRow(seq=i, sent_epoch=1.0 + i, created_epoch=1.5 + i) for i in range(count)]+++def test_delivery_ratio_is_taken_against_the_alerting_population():+ """A lossless pipeline must report 1.0, not 0.01.++ Ten alerting events were sent and ten alerts arrived. Dividing by the+ 1,000 accepted events — 990 of which were supposed to produce nothing+ — would report a catastrophic loss for a perfect run, and a floor set+ against that number would be meaningless.+ """+ report = _report(_alert_rows(10))+ ratio = report["metrics"]["delivery_ratio"]+ assert ratio["measured"] is True+ assert ratio["value"] == pytest.approx(1.0)+++def test_the_benign_alert_rate_is_measured_and_reported():+ report = _report(_alert_rows(10), benign_alerts=0)+ rate = report["metrics"]["benign_alert_rate"]+ assert rate["measured"] is True+ assert rate["value"] == 0.0+ assert report["run"]["benign_events_sent"] == 990+ assert report["run"]["benign_events_that_alerted"] == 0+++def test_benign_events_that_alert_are_counted_not_hidden():+ report = _report(_alert_rows(10), benign_alerts=99)+ assert report["metrics"]["benign_alert_rate"]["value"] == pytest.approx(0.1)+++def test_an_all_alerting_run_says_the_benign_rate_is_not_measured():+ # Honest: with alert-every of 1 there are no benign events, so there is+ # no rate. Reporting 0.0 would read as "the corpus fired on nothing",+ # which is a claim this run cannot support.+ report = _report(_alert_rows(10), alerting_events_sent=1000, benign_events_sent=0, alert_every=1)+ rate = report["metrics"]["benign_alert_rate"]+ assert rate["measured"] is False+ assert "alert-every is 1" in rate["reason"]+++def test_an_old_summary_without_the_mix_fields_still_grades():+ """A result file written before the mix existed must not be graded+ against a denominator it never had."""+ summary = _summary()+ del summary["alerting_events_sent"]+ del summary["benign_events_sent"]+ del summary["alert_every"]+ target = harness.Target(name="t", psql=[], kafka=[], stats=None, ingest_url="")+ report = harness.build_report(+ target=target,+ summary=harness.ProducerSummary(summary),+ alerts=_alert_rows(1000),+ clock_offset=0.0,+ clock_spread=0.0,+ lag_in_flight=(None, "n/a"),+ lag_drained=(None, "n/a"),+ dead_letters=0,+ dead_letter_reason="",+ resources=[],+ drain_seconds=1.0,+ settled=True,+ )+ assert report["metrics"]["delivery_ratio"]["value"] == pytest.approx(1.0)+ assert report["metrics"]["benign_alert_rate"]["measured"] is False+++def test_a_ceiling_on_an_unmeasured_rate_fails_rather_than_passes():+ """The shape this repository keeps finding: a gate reporting success+ for a run it could not grade."""+ source = (REPO_ROOT / "scripts/perf/load_harness.py").read_text(encoding="utf-8")+ block = source.split("if args.assert_benign_alert_rate_max is not None:", 1)[1]+ block = block.split("for failure in failures:", 1)[0]+ assert 'if not body["measured"]:' in block+ assert "failures.append" in block.split('if not body["measured"]:', 1)[1].split("elif", 1)[0], (+ "an unmeasured benign rate does not fail the ceiling"+ )