Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

perf(harness): a realistic mix in which most events do not alert (depth 4.4)

Beenu Arora6b6370a1b2d1
diff --git a/scripts/perf/load_harness.py b/scripts/perf/load_harness.py
index a9c18a7b4..6af836ca2 100644
--- a/scripts/perf/load_harness.py
+++ b/scripts/perf/load_harness.py
@@ -85,6 +85,11 @@ from gate_toolkit import repo_root, self_test_main # noqa: E402
#: pins the two together, because a silent disagreement would look like a
#: pipeline that lost every event.
MARKER_PREFIX = "aisoc-load"
+#: Must equal ``BenignMarkerPrefix`` in services/demo-producer/main.go.
+#: The two prefixes are distinguishable because the alerting one is
+#: always followed by a space and the run id, so a `like` on
+#: "aisoc-load <run> %" cannot match a benign title.
+BENIGN_MARKER_PREFIX = "aisoc-load-benign"
#: ``aisoc-load <run-id> seq=<n> t=<unix-nanoseconds>``
_TITLE_RE = re.compile(rf"^{re.escape(MARKER_PREFIX)} (?P<run>[A-Za-z0-9_-]+) seq=(?P<seq>\d+) t=(?P<sent>\d+)$")
@@ -322,6 +327,26 @@ class ProducerSummary:
def accepted_eps(self) -> float:
return float(self.raw["accepted_eps"])
+ @property
+ def alerting_sent(self) -> int | None:
+ """How many events the producer sent in the alerting shape.
+
+ ``None`` for a summary written before the realistic mix existed,
+ which is the honest answer rather than falling back to the
+ accepted count and quietly grading an old run against a
+ denominator it never had.
+ """
+ value = self.raw.get("alerting_events_sent")
+ return int(value) if value is not None else None
+
+ @property
+ def benign_sent(self) -> int:
+ return int(self.raw.get("benign_events_sent") or 0)
+
+ @property
+ def alert_every(self) -> int:
+ return int(self.raw.get("alert_every") or 1)
+
@dataclass
class AlertRow:
@@ -418,6 +443,7 @@ def run_producer(
target_eps: int,
run_id: str,
summary_path: Path,
+ alert_every: int = 1,
) -> ProducerSummary:
# Where `go run` is invoked from, which is not the repository root. The
# producer is its own module (`services/demo-producer/go.mod`) and there
@@ -458,6 +484,8 @@ def run_producer(
run_id,
"--summary",
str(summary_path),
+ "--alert-every",
+ str(max(1, alert_every)),
]
done = subprocess.run(argv, cwd=cwd, capture_output=True, text=True, check=False) # noqa: S603
if done.returncode != 0:
@@ -465,6 +493,21 @@ def run_producer(
return ProducerSummary(json.loads(summary_path.read_text(encoding="utf-8")))
+def count_benign_alerts(target: Target, run_id: str) -> int:
+ """Alerts raised by events that were supposed to raise none.
+
+ Counted rather than assumed zero. The whole point of the realistic
+ mix is that benign telemetry reaches the lake and produces nothing;
+ an all-alerting harness cannot observe that property at all, and a
+ mix that never checks it is the same harness with more events.
+ """
+ raw = target.sql(f"select count(*) from alerts where title like '{BENIGN_MARKER_PREFIX} {run_id} %'")
+ try:
+ return int(raw or 0)
+ except (TypeError, ValueError):
+ return 0
+
+
def collect_alerts(target: Target, run_id: str) -> list[AlertRow]:
query = f"select title, extract(epoch from created_at) from alerts where title like '{MARKER_PREFIX} {run_id} %'"
rows: list[AlertRow] = []
@@ -538,6 +581,7 @@ def build_report(
drain_seconds: float,
settled: bool,
shape: dict[str, Any] | None = None,
+ benign_alerts: int = 0,
) -> dict[str, Any]:
accepted = summary.accepted
seen: dict[int, int] = {}
@@ -575,12 +619,30 @@ def build_report(
else:
metrics["pipeline_eps"] = unmeasured("pipeline_eps", "alerts/s", "fewer than two alerts landed")
+ # Depth plan 4.4. With a realistic mix most accepted events are benign
+ # and are *supposed* to produce no alert, so dividing alerts by accepted
+ # events would report a delivery ratio of 0.01 for a pipeline that lost
+ # nothing. The denominator is the alerting population the producer
+ # actually sent, which it counts rather than computing from
+ # `total / alert_every` — a run that stops early would otherwise be
+ # graded against an expectation it never reached.
+ expected_alerting = summary.alerting_sent if summary.alerting_sent is not None else accepted
metrics["delivery_ratio"] = (
- measured("delivery_ratio", "fraction", len(seen) / accepted, accepted)
- if accepted > 0
- else unmeasured("delivery_ratio", "fraction", "ingest accepted no events")
+ measured("delivery_ratio", "fraction", len(seen) / expected_alerting, expected_alerting)
+ if expected_alerting > 0
+ else unmeasured("delivery_ratio", "fraction", "no alerting events were sent")
)
+ # The property the realistic mix exists to assert. Benign telemetry
+ # reaching the lake and producing no alert is what a real estate looks
+ # like; benign telemetry that alerts means the corpus is firing on
+ # routine traffic, which is the failure the mix is designed to expose
+ # and which an all-alerting harness cannot see at all.
+ if summary.benign_sent:
+ metrics["benign_alert_rate"] = measured("benign_alert_rate", "fraction", benign_alerts / summary.benign_sent, summary.benign_sent)
+ else:
+ metrics["benign_alert_rate"] = unmeasured("benign_alert_rate", "fraction", "no benign events were sent (alert-every is 1)")
+
for key, (value, reason) in (("consumer_lag_in_flight", lag_in_flight), ("consumer_lag_drained", lag_drained)):
metrics[key] = measured(key, "messages", float(value), 1) if value is not None else unmeasured(key, "messages", reason)
@@ -611,6 +673,10 @@ def build_report(
"accepted_events": accepted,
"alerts_observed": len(alerts),
"distinct_events_alerted": len(seen),
+ "alerting_events_sent": summary.alerting_sent,
+ "benign_events_sent": summary.benign_sent,
+ "benign_events_that_alerted": benign_alerts,
+ "alert_every": summary.alert_every,
"missing_events": missing,
"duplicate_alert_rows": duplicates,
"push_wall_seconds": summary.wall_seconds,
@@ -681,6 +747,23 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument("--assert-eps-floor", type=float, help="fail if pipeline throughput is below this")
parser.add_argument("--assert-p95-ceiling-ms", type=float, help="fail if p95 event-to-alert latency exceeds this")
parser.add_argument("--assert-no-loss", action="store_true", help="fail if any accepted event produced no alert")
+ parser.add_argument(
+ "--alert-every",
+ type=int,
+ default=1,
+ help=(
+ "send one alerting event every Nth event and benign telemetry in between. 1 (the default) "
+ "keeps the historical all-alerting load; 100 is a realistic estate where almost nothing alerts"
+ ),
+ )
+ parser.add_argument(
+ "--assert-benign-alert-rate-max",
+ type=float,
+ help=(
+ "fail if more than this fraction of the benign events raised an alert. Only meaningful with "
+ "--alert-every above 1; the realistic mix exists to make this measurable"
+ ),
+ )
args = parser.parse_args(argv)
root = repo_root()
@@ -726,6 +809,7 @@ def main(argv: list[str] | None = None) -> int:
target_eps=args.target_eps,
run_id=run_id,
summary_path=summary_path,
+ alert_every=args.alert_every,
)
# Sampled immediately after the push rather than after the drain: the
# queue is at its deepest here, and a lag read once the pipeline has caught
@@ -738,7 +822,11 @@ def main(argv: list[str] | None = None) -> int:
settled = False
while time.time() - drain_started < args.drain_timeout:
count = int(target.sql(f"select count(*) from alerts where title like '{MARKER_PREFIX} {run_id} %'") or 0)
- if count >= summary.accepted:
+ # With a realistic mix most accepted events are benign and raise
+ # nothing, so waiting for one alert per accepted event would always
+ # time out and report the drain as unsettled. The target is the
+ # alerting population the producer counted.
+ if count >= (summary.alerting_sent if summary.alerting_sent is not None else summary.accepted):
settled = True
break
if count == previous and count > 0:
@@ -804,6 +892,7 @@ def main(argv: list[str] | None = None) -> int:
drain_seconds=drain_seconds,
settled=settled,
shape=shape,
+ benign_alerts=count_benign_alerts(target, run_id),
)
report["run"]["total_wall_seconds"] = time.time() - push_started
@@ -828,6 +917,18 @@ def main(argv: list[str] | None = None) -> int:
failures.append(f"p95 latency {body['value']:.0f}ms exceeds the ceiling {args.assert_p95_ceiling_ms:.0f}ms")
if args.assert_no_loss and report["run"]["missing_events"] > 0:
failures.append(f"{report['run']['missing_events']} accepted events produced no alert")
+ if args.assert_benign_alert_rate_max is not None:
+ body = report["metrics"]["benign_alert_rate"]
+ if not body["measured"]:
+ # Not a pass. A ceiling on a figure that was not measured is a
+ # gate that reports success for a run it could not grade, which
+ # is the shape this repository keeps finding.
+ failures.append(f"the benign alert rate was not measured ({body['reason']}), so the ceiling cannot be met")
+ elif body["value"] > args.assert_benign_alert_rate_max:
+ failures.append(
+ f"{body['value']:.4f} of benign events raised an alert, above the ceiling "
+ f"{args.assert_benign_alert_rate_max:.4f}; the corpus is firing on routine traffic"
+ )
for failure in failures:
print(f"FAIL: {failure}", file=sys.stderr)
diff --git a/services/demo-producer/main.go b/services/demo-producer/main.go
index 6af39c36d..af02dcfae 100644
--- a/services/demo-producer/main.go
+++ b/services/demo-producer/main.go
@@ -408,10 +408,106 @@ func loadEvent(runID string, seq int64, sentNanos int64) map[string]interface{}
}
}
+// BenignMarkerPrefix opens the title of every *non-alerting* load event.
+// Distinct from MarkerPrefix so the harness can count the two populations
+// separately: the alerting ones are what latency is measured over, and the
+// benign ones existing in the lake while producing no alert is the property
+// the realistic mix is for.
+const BenignMarkerPrefix = "aisoc-load-benign"
+
+// benignLoadEvent is the other 99% of a real estate's telemetry.
+//
+// Depth plan 4.4. The load harness used to send one event shape, every
+// instance of which became an alert. That measures a pipeline nobody runs:
+// in a real deployment almost every event is routine, and the work fusion
+// actually does is evaluating the corpus against traffic that matches
+// nothing. A harness in which every event alerts over-measures the alert
+// path and *under*-measures detection, which is the stage parity 6.9 is
+// about.
+//
+// Deliberately shaped like real cloud/identity telemetry rather than as a
+// near-miss of the alerting event: a benign event built by mutating the
+// malicious one shares its field namespace, so it exercises the same
+// handful of rules and tells you nothing about the corpus. These carry the
+// fields a CloudTrail or Okta event carries, so the rule index resolves
+// them the way production traffic would.
+//
+// `severity: info` and an ordinary event type keep it below the promoter's
+// bar as well as the detection corpus's, so an alert from one of these is
+// a real finding about the pipeline rather than an artefact of the fixture.
+func benignLoadEvent(runID string, seq int64, sentNanos int64) map[string]interface{} {
+ shapes := []map[string]interface{}{
+ {
+ "event_type": "s3.GetObject",
+ "description": "GetObject succeeded",
+ "user": fmt.Sprintf("svc-reporting%03d", seq%50),
+ "src_ip": fmt.Sprintf("10.%d.%d.%d", seq%250, (seq/250)%250, (seq/60000)%250),
+ "action": "allow",
+ "outcome": "success",
+ "region": "us-east-1",
+ "account_id": "123456789012",
+ "user_agent": "aws-sdk-go/1.44",
+ "resource": "arn:aws:s3:::reports",
+ },
+ {
+ "event_type": "user.session.start",
+ "description": "sign-in succeeded",
+ "user": fmt.Sprintf("analyst%03d", seq%200),
+ "src_ip": fmt.Sprintf("198.51.100.%d", seq%250),
+ "action": "allow",
+ "outcome": "success",
+ "mfa": "true",
+ "app": "okta",
+ "user_agent": "Mozilla/5.0",
+ },
+ {
+ "event_type": "http.request",
+ "description": "200 GET /healthz",
+ "src_ip": fmt.Sprintf("10.20.%d.%d", seq%250, (seq/250)%250),
+ "status_code": 200,
+ "method": "GET",
+ "path": "/healthz",
+ "bytes_out": 128,
+ "outcome": "success",
+ },
+ }
+ event := map[string]interface{}{}
+ for k, v := range shapes[seq%int64(len(shapes))] {
+ event[k] = v
+ }
+ event["severity"] = "info"
+ event["host"] = fmt.Sprintf("LOADB-%s-%06d", runID, seq)
+ event["title"] = fmt.Sprintf("%s %s seq=%d t=%d", BenignMarkerPrefix, runID, seq, sentNanos)
+ event["timestamp"] = time.Unix(0, sentNanos).UTC().Format(time.RFC3339Nano)
+ return event
+}
+
+// loadEventFor picks the shape for this sequence number from the configured
+// mix.
+//
+// Deterministic rather than random: the same seq always produces the same
+// shape, so two runs of the same configuration push the same traffic and a
+// difference between them is the system rather than the dice. `alertEvery`
+// of 0 means every event alerts, which is the previous behaviour and the
+// default.
+func loadEventFor(runID string, seq int64, sentNanos int64, alertEvery int64) (map[string]interface{}, bool) {
+ if alertEvery <= 1 {
+ return loadEvent(runID, seq, sentNanos), true
+ }
+ if seq%alertEvery == 0 {
+ return loadEvent(runID, seq, sentNanos), true
+ }
+ return benignLoadEvent(runID, seq, sentNanos), false
+}
+
type loadSummary struct {
RunID string `json:"run_id"`
Mode string `json:"mode"`
MarkerPrefix string `json:"marker_prefix"`
+ BenignPrefix string `json:"benign_marker_prefix"`
+ AlertEvery int64 `json:"alert_every"`
+ AlertingSent int64 `json:"alerting_events_sent"`
+ BenignSent int64 `json:"benign_events_sent"`
IngestURL string `json:"ingest_url"`
Tenant string `json:"tenant"`
StartedAt string `json:"started_at"`
@@ -441,6 +537,11 @@ func runLoad(ctx context.Context, opts options) loadSummary {
st := newStats()
var seq atomic.Int64
var issued atomic.Int64
+ // Counted here rather than derived from `total / alertEvery`: the run
+ // can stop early, and a computed expectation would then disagree with
+ // what was sent in exactly the case the harness needs it most.
+ var alertingSent atomic.Int64
+ var benignSent atomic.Int64
// A shared bucket rather than a per-worker ticker: pacing per worker makes
// the aggregate rate depend on how many workers happen to be blocked.
@@ -489,7 +590,14 @@ func runLoad(ctx context.Context, opts options) loadSummary {
batch := make([]map[string]interface{}, 0, size)
now := time.Now().UnixNano()
for i := 0; i < size; i++ {
- batch = append(batch, loadEvent(opts.runID, seq.Add(1)-1, now))
+ next := seq.Add(1) - 1
+ event, alerting := loadEventFor(opts.runID, next, now, opts.alertEvery)
+ if alerting {
+ alertingSent.Add(1)
+ } else {
+ benignSent.Add(1)
+ }
+ batch = append(batch, event)
}
err := postBatch(ctx, client, opts, ingestRequest{
ConnectorID: "load-" + opts.runID,
@@ -522,6 +630,10 @@ func runLoad(ctx context.Context, opts options) loadSummary {
RunID: opts.runID,
Mode: "load",
MarkerPrefix: MarkerPrefix,
+ BenignPrefix: BenignMarkerPrefix,
+ AlertEvery: opts.alertEvery,
+ AlertingSent: alertingSent.Load(),
+ BenignSent: benignSent.Load(),
IngestURL: opts.ingestURL,
Tenant: opts.tenant,
StartedAt: started.UTC().Format(time.RFC3339Nano),
@@ -565,6 +677,9 @@ type options struct {
workers int
targetEPS int
summary string
+ // alertEvery shapes the mix: one alerting event every N, benign
+ // telemetry in between. 1 is the historical all-alerting load.
+ alertEvery int64
}
func main() {
@@ -581,6 +696,8 @@ func main() {
flag.IntVar(&opts.workers, "workers", 8, "Load mode: concurrent senders")
flag.IntVar(&opts.targetEPS, "target-eps", 0, "Load mode: pace to this many events per second (0 = saturate)")
flag.StringVar(&opts.summary, "summary", "", "Load mode: write the run summary as JSON to this path")
+ flag.Int64Var(&opts.alertEvery, "alert-every", 1, "Load mode: send an alerting event every Nth event and benign telemetry in between. "+
+ "1 (the default) is the historical all-alerting mix; 100 is a realistic estate where almost nothing alerts")
flag.Parse()
ctx, cancel := context.WithCancel(context.Background())
diff --git a/tests/test_load_harness_realistic_mix.py b/tests/test_load_harness_realistic_mix.py
new file mode 100644
index 000000000..2908210b2
--- /dev/null
+++ b/tests/test_load_harness_realistic_mix.py
@@ -0,0 +1,185 @@
+"""The realistic mix is a mix, and the harness grades it as one (depth 4.4).
+
+The load harness used to send one event shape, every instance of which
+became an alert. That measures a pipeline nobody runs: in a real estate
+almost every event is routine, and the work fusion actually does is
+evaluating the corpus against traffic that matches nothing. An
+all-alerting harness over-measures the alert path and *under*-measures
+detection, which is the stage parity 6.9 is about.
+
+Three things have to hold for the mix to mean anything, and each has
+failed in this repository's harnesses before:
+
+* the Go producer and the Python harness must agree on the marker, or
+ the harness counts zero of one population;
+* the delivery ratio must be taken against the *alerting* population, or
+ a lossless pipeline reports 0.01;
+* a ceiling on a figure that was **not measured** must fail, not pass.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import re
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+PRODUCER_GO = REPO_ROOT / "services" / "demo-producer" / "main.go"
+
+
+def _load(name: str, relative: str):
+ spec = importlib.util.spec_from_file_location(name, REPO_ROOT / relative)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+harness = _load("aisoc_load_harness_mix", "scripts/perf/load_harness.py")
+
+
+def _go_const(name: str) -> str:
+ source = PRODUCER_GO.read_text(encoding="utf-8")
+ match = re.search(rf'^const {name} = "([^"]+)"$', source, flags=re.M)
+ assert match, f"{name} is not declared in {PRODUCER_GO}"
+ return match.group(1)
+
+
+def test_the_benign_marker_matches_the_one_the_producer_writes():
+ # The sibling of the existing marker-parity test. A harness counting a
+ # prefix the producer does not write reports zero benign alerts
+ # forever, which looks exactly like a clean run.
+ assert harness.BENIGN_MARKER_PREFIX == _go_const("BenignMarkerPrefix")
+
+
+def test_the_two_markers_cannot_be_confused_by_the_sql():
+ """The benign prefix *contains* the alerting one.
+
+ `aisoc-load-benign` starts with `aisoc-load`, so the alerting query
+ would match benign titles were it not for the space and run id that
+ follow. This pins the property rather than trusting it.
+ """
+ assert harness.BENIGN_MARKER_PREFIX.startswith(harness.MARKER_PREFIX)
+ run = "run1"
+ alerting_pattern = f"{harness.MARKER_PREFIX} {run} "
+ benign_title = f"{harness.BENIGN_MARKER_PREFIX} {run} seq=1 t=2"
+ assert not benign_title.startswith(alerting_pattern), (
+ "a benign alert would be counted as an alerting one and the mix would measure nothing"
+ )
+
+
+def _summary(**overrides) -> dict:
+ base = {
+ "run_id": "r",
+ "accepted_events": 1000,
+ "attempted_events": 1000,
+ "wall_seconds": 10.0,
+ "accepted_eps": 100.0,
+ "alerting_events_sent": 10,
+ "benign_events_sent": 990,
+ "alert_every": 100,
+ }
+ base.update(overrides)
+ return base
+
+
+def _report(alerts, *, benign_alerts=0, **summary_overrides) -> dict:
+ target = harness.Target(name="t", psql=[], kafka=[], stats=None, ingest_url="")
+ return harness.build_report(
+ target=target,
+ summary=harness.ProducerSummary(_summary(**summary_overrides)),
+ alerts=alerts,
+ clock_offset=0.0,
+ clock_spread=0.0,
+ lag_in_flight=(None, "not measured"),
+ lag_drained=(None, "not measured"),
+ dead_letters=0,
+ dead_letter_reason="",
+ resources=[],
+ drain_seconds=1.0,
+ settled=True,
+ benign_alerts=benign_alerts,
+ )
+
+
+def _alert_rows(count: int) -> list:
+ return [harness.AlertRow(seq=i, sent_epoch=1.0 + i, created_epoch=1.5 + i) for i in range(count)]
+
+
+def test_delivery_ratio_is_taken_against_the_alerting_population():
+ """A lossless pipeline must report 1.0, not 0.01.
+
+ Ten alerting events were sent and ten alerts arrived. Dividing by the
+ 1,000 accepted events — 990 of which were supposed to produce nothing
+ — would report a catastrophic loss for a perfect run, and a floor set
+ against that number would be meaningless.
+ """
+ report = _report(_alert_rows(10))
+ ratio = report["metrics"]["delivery_ratio"]
+ assert ratio["measured"] is True
+ assert ratio["value"] == pytest.approx(1.0)
+
+
+def test_the_benign_alert_rate_is_measured_and_reported():
+ report = _report(_alert_rows(10), benign_alerts=0)
+ rate = report["metrics"]["benign_alert_rate"]
+ assert rate["measured"] is True
+ assert rate["value"] == 0.0
+ assert report["run"]["benign_events_sent"] == 990
+ assert report["run"]["benign_events_that_alerted"] == 0
+
+
+def test_benign_events_that_alert_are_counted_not_hidden():
+ report = _report(_alert_rows(10), benign_alerts=99)
+ assert report["metrics"]["benign_alert_rate"]["value"] == pytest.approx(0.1)
+
+
+def test_an_all_alerting_run_says_the_benign_rate_is_not_measured():
+ # Honest: with alert-every of 1 there are no benign events, so there is
+ # no rate. Reporting 0.0 would read as "the corpus fired on nothing",
+ # which is a claim this run cannot support.
+ report = _report(_alert_rows(10), alerting_events_sent=1000, benign_events_sent=0, alert_every=1)
+ rate = report["metrics"]["benign_alert_rate"]
+ assert rate["measured"] is False
+ assert "alert-every is 1" in rate["reason"]
+
+
+def test_an_old_summary_without_the_mix_fields_still_grades():
+ """A result file written before the mix existed must not be graded
+ against a denominator it never had."""
+ summary = _summary()
+ del summary["alerting_events_sent"]
+ del summary["benign_events_sent"]
+ del summary["alert_every"]
+ target = harness.Target(name="t", psql=[], kafka=[], stats=None, ingest_url="")
+ report = harness.build_report(
+ target=target,
+ summary=harness.ProducerSummary(summary),
+ alerts=_alert_rows(1000),
+ clock_offset=0.0,
+ clock_spread=0.0,
+ lag_in_flight=(None, "n/a"),
+ lag_drained=(None, "n/a"),
+ dead_letters=0,
+ dead_letter_reason="",
+ resources=[],
+ drain_seconds=1.0,
+ settled=True,
+ )
+ assert report["metrics"]["delivery_ratio"]["value"] == pytest.approx(1.0)
+ assert report["metrics"]["benign_alert_rate"]["measured"] is False
+
+
+def test_a_ceiling_on_an_unmeasured_rate_fails_rather_than_passes():
+ """The shape this repository keeps finding: a gate reporting success
+ for a run it could not grade."""
+ source = (REPO_ROOT / "scripts/perf/load_harness.py").read_text(encoding="utf-8")
+ block = source.split("if args.assert_benign_alert_rate_max is not None:", 1)[1]
+ block = block.split("for failure in failures:", 1)[0]
+ assert 'if not body["measured"]:' in block
+ assert "failures.append" in block.split('if not body["measured"]:', 1)[1].split("elif", 1)[0], (
+ "an unmeasured benign rate does not fail the ceiling"
+ )