Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
feat(detection): WS-B4 detection-as-code propose-for-review with eval gate
Beenu Arora2ae5c94430ff
diff --git a/apps/web/src/components/detections/RuleEditor.tsx b/apps/web/src/components/detections/RuleEditor.tsxindex 17a15c595..7f19a1608 100644--- a/apps/web/src/components/detections/RuleEditor.tsx+++ b/apps/web/src/components/detections/RuleEditor.tsx@@ -9,7 +9,9 @@ import { clsx } from 'clsx';import toast from 'react-hot-toast';import {detectionApi,+ detectionProposalsApi,type DetectionLanguage,+ type DetectionProposal,type DetectionRule,type HuntResult,} from '@/lib/api';@@ -132,6 +134,26 @@ export function RuleEditor({ mode, ruleId }: RuleEditorProps) {} | null>(null);const [saving, setSaving] = useState(false);+ // Detection-as-Code: propose-for-review state. Wave-2 buyer-value path.+ // The author writes a rule here, hits "Propose for review", and the backend+ // runs `scripts/run_evals.py` against the active baseline. The verdict is+ // attached to the proposal and the regression gate (≥ 1pp MITRE accuracy+ // drop) decides whether the proposal can be promoted to a live rule.+ const [proposing, setProposing] = useState(false);+ const [proposalStage, setProposalStage] = useState<+ 'idle' | 'creating' | 'evaluating' | 'attaching' | 'done'+ >('idle');+ const [evalVerdict, setEvalVerdict] = useState<{+ proposalId: string;+ passed: boolean;+ candidateMitre?: number;+ baselineMitre?: number;+ dropPp?: number;+ maxRegressionPp?: number;+ durationSeconds?: number;+ ranAt?: string;+ } | null>(null);+// Hydrate form when data arrivesuseEffect(() => {if (data) {@@ -289,6 +311,119 @@ export function RuleEditor({ mode, ruleId }: RuleEditorProps) {}};+ // Detection-as-Code propose-for-review flow:+ // 1) POST /detection-proposals → register the candidate+ // 2) POST /detection-proposals/run-eval → run scripts/run_evals.py+ // 3) POST /detection-proposals/:id/eval → attach verdict (server gates)+ // We surface a verdict card on the editor and link to the queue at+ // /detection/proposals so a reviewer can take it from approved → promoted.+ // If any leg fails we keep the proposal in 'proposed' state and surface+ // the error — partial state is fine because the queue ignores stale eval+ // attachments.+ const handlePropose = async () => {+ if (!name.trim()) {+ toast.error('Rule name is required to propose a change');+ return;+ }+ if (!body.trim()) {+ toast.error('Rule body is empty');+ return;+ }+ setProposing(true);+ setEvalVerdict(null);+ setProposalStage('creating');+ try {+ // 1. Create the proposal — base_rule_id links it to the rule being+ // edited so the reviewer queue can show "modified vs. live".+ const created = await detectionProposalsApi.create({+ name: name.trim(),+ description: description.trim() || null,+ rule_language: language,+ rule_body: body,+ // The rule editor has no category picker today; the proposal API+ // requires a non-empty string, so default to a sensible value.+ category: 'detection',+ severity,+ confidence: 0.7,+ mitre_techniques: mitre,+ mitre_tactics: [],+ tags,+ base_rule_id: mode === 'edit' && ruleId ? ruleId : null,+ });++ // 2. Run the offline eval harness server-side. This blocks the request+ // for ~5–15s on the 200-incident corpus, which is acceptable for an+ // explicit "Propose" click.+ setProposalStage('evaluating');+ let runResult: Awaited<ReturnType<typeof detectionProposalsApi.runEval>>;+ try {+ runResult = await detectionProposalsApi.runEval({+ use_active_baseline: true,+ max_regression_pp: 1.0,+ timeout_seconds: 240,+ });+ } catch (evalErr) {+ console.error('Eval harness call failed', evalErr);+ toast.error(+ 'Proposal created, but the eval harness could not run. A reviewer can attach a report manually.',+ );+ setEvalVerdict({+ proposalId: created.id,+ passed: false,+ });+ setProposalStage('done');+ // Best effort: still navigate to the proposal queue so the user+ // can see it queued.+ router.push('/detection/proposals');+ return;+ }++ // 3. Attach the verdict — backend authoritatively decides eval_passed+ // vs. eval_failed using its 1pp regression rule against the active+ // baseline. We pass the same threshold the runner used so the+ // server's recorded `max_regression_pp` matches.+ setProposalStage('attaching');+ const decided = await detectionProposalsApi.attachEval(created.id, {+ eval_report: runResult.report,+ max_regression_pp: 1.0,+ });++ const verdict = decided.eval_result as DetectionProposal['eval_result'];+ const passed = decided.status === 'eval_passed';+ setEvalVerdict({+ proposalId: decided.id,+ passed,+ candidateMitre:+ (verdict as { candidate?: { mitre_accuracy?: number } } | undefined)+ ?.candidate?.mitre_accuracy,+ baselineMitre:+ (verdict as { baseline?: { mitre_accuracy?: number } } | undefined)+ ?.baseline?.mitre_accuracy,+ dropPp: (verdict as { drop_pp?: number } | undefined)?.drop_pp,+ maxRegressionPp:+ (verdict as { max_regression_pp?: number } | undefined)+ ?.max_regression_pp,+ durationSeconds: runResult.duration_seconds,+ ranAt: runResult.ran_at,+ });+ setProposalStage('done');++ if (passed) {+ toast.success('Proposal passed eval — ready for review');+ } else {+ toast.error(+ 'Proposal failed eval — MITRE accuracy regression vs. active baseline',+ );+ }+ } catch (err) {+ console.error('Propose-for-review failed', err);+ toast.error('Could not create proposal — backend unreachable');+ setProposalStage('idle');+ } finally {+ setProposing(false);+ }+ };+// ─── Render ────────────────────────────────────────────────────────────────return (@@ -317,6 +452,29 @@ export function RuleEditor({ mode, ruleId }: RuleEditorProps) {Delete</button>)}+ {/*+ "Propose for review" runs the eval-gated DaC flow. The button is+ distinct from "Save" because it doesn't write to the live rule —+ it creates a queued proposal that a reviewer promotes after the+ eval verdict + a human approval. We show it on both create and+ edit so authors can iterate on a draft and only submit once the+ harness passes.+ */}+ <button+ onClick={handlePropose}+ disabled={proposing || saving}+ className={clsx(+ 'inline-flex items-center gap-2 rounded-md border border-blue-500/40 bg-blue-500/10 px-3 py-1.5 text-sm font-medium text-blue-200 shadow-sm transition-colors',+ proposing || saving ? 'opacity-60' : 'hover:bg-blue-500/20',+ )}+ title="Open a proposal that the eval harness must pass before promotion"+ >+ {proposalStage === 'creating' && 'Creating proposal…'}+ {proposalStage === 'evaluating' && 'Running eval harness…'}+ {proposalStage === 'attaching' && 'Attaching verdict…'}+ {(proposalStage === 'idle' || proposalStage === 'done') &&+ 'Propose for review'}+ </button><buttononClick={handleSave}disabled={saving}@@ -330,6 +488,93 @@ export function RuleEditor({ mode, ruleId }: RuleEditorProps) {</div></div>+ {/*+ Eval verdict card — surfaces the outcome of the most recent+ "Propose for review" click against the active 200-incident+ baseline. We compare candidate vs. baseline MITRE accuracy and+ the absolute drop in percentage points, mirroring the gate in+ services/api/app/api/v1/endpoints/detection_proposals.py. Reviewer+ actions (approve / reject / promote) live on /detection/proposals,+ which we deep-link to once we have a proposal id.+ */}+ {evalVerdict && (+ <div+ className={clsx(+ 'rounded-lg border px-4 py-3 text-sm',+ evalVerdict.passed+ ? 'border-emerald-500/30 bg-emerald-500/5 text-emerald-100'+ : 'border-red-500/30 bg-red-500/5 text-red-100',+ )}+ >+ <div className="flex items-center justify-between gap-3">+ <div className="flex items-center gap-2 font-medium">+ <span+ className={clsx(+ 'inline-flex h-2 w-2 rounded-full',+ evalVerdict.passed ? 'bg-emerald-400' : 'bg-red-400',+ )}+ />+ {evalVerdict.passed+ ? 'Eval passed — proposal ready for review'+ : 'Eval failed — MITRE accuracy regression'}+ </div>+ <Link+ href={`/detection/proposals?highlight=${evalVerdict.proposalId}`}+ className="text-xs underline-offset-2 hover:underline"+ >+ View in proposal queue →+ </Link>+ </div>+ {(evalVerdict.candidateMitre !== undefined ||+ evalVerdict.baselineMitre !== undefined ||+ evalVerdict.dropPp !== undefined) && (+ <div className="mt-2 grid grid-cols-2 gap-x-6 gap-y-1 text-xs text-gray-200 sm:grid-cols-4">+ {evalVerdict.candidateMitre !== undefined && (+ <div>+ <span className="text-gray-400">Candidate MITRE</span>{' '}+ <span className="font-mono">+ {(evalVerdict.candidateMitre * 100).toFixed(1)}%+ </span>+ </div>+ )}+ {evalVerdict.baselineMitre !== undefined && (+ <div>+ <span className="text-gray-400">Baseline MITRE</span>{' '}+ <span className="font-mono">+ {(evalVerdict.baselineMitre * 100).toFixed(1)}%+ </span>+ </div>+ )}+ {evalVerdict.dropPp !== undefined && (+ <div>+ <span className="text-gray-400">Δ pp</span>{' '}+ <span className="font-mono">+ {evalVerdict.dropPp >= 0 ? '+' : ''}+ {(-evalVerdict.dropPp).toFixed(2)}+ </span>+ </div>+ )}+ {evalVerdict.maxRegressionPp !== undefined && (+ <div>+ <span className="text-gray-400">Gate</span>{' '}+ <span className="font-mono">+ ≤ {evalVerdict.maxRegressionPp.toFixed(2)} pp+ </span>+ </div>+ )}+ {evalVerdict.durationSeconds !== undefined && (+ <div>+ <span className="text-gray-400">Runtime</span>{' '}+ <span className="font-mono">+ {evalVerdict.durationSeconds.toFixed(1)}s+ </span>+ </div>+ )}+ </div>+ )}+ </div>+ )}+{/*Ambient Copilot — rule-scoped contextual AI. Only shown in edit mode wherethe rule has a stable id + body to reason about; new-rule drafts have nodiff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.tsindex a37bc5ee7..ec4972ea7 100644--- a/apps/web/src/lib/api.ts+++ b/apps/web/src/lib/api.ts@@ -2369,6 +2369,26 @@ export const detectionProposalsApi = {body: JSON.stringify(body),}),+ // Triggers a synchronous run of `scripts/run_evals.py` server-side and+ // returns the JSON report plus runner metadata. The rule editor calls this+ // from "Propose for review" so the eval gate runs before the proposal is+ // visible for approval. Subprocess takes ~5–15s on the 200-incident corpus.+ runEval: (body?: {+ use_active_baseline?: boolean;+ max_regression_pp?: number;+ timeout_seconds?: number;+ }) =>+ request<{+ report: Record<string, unknown>;+ exit_code: number;+ duration_seconds: number;+ ran_at: string;+ script: string;+ }>('/api/v1/detection-proposals/run-eval', {+ method: 'POST',+ body: JSON.stringify(body ?? {}),+ }),+decide: (id: string,body: { decision: 'approve' | 'reject'; comment?: string | null },diff --git a/services/api/app/api/v1/endpoints/detection_proposals.py b/services/api/app/api/v1/endpoints/detection_proposals.pyindex dca677f79..3c2272761 100644--- a/services/api/app/api/v1/endpoints/detection_proposals.py+++ b/services/api/app/api/v1/endpoints/detection_proposals.py@@ -22,8 +22,15 @@ Endpointsfrom __future__ import annotations+import asyncio+import json+import os+import subprocess+import sys+import tempfileimport uuidfrom datetime import UTC, datetime+from pathlib import Pathfrom typing import Annotated, Any, Literalfrom fastapi import APIRouter, Depends, HTTPException, Query, status@@ -41,6 +48,15 @@ from app.models.detection_rule import DetectionRulerouter = APIRouter(prefix="/detection-proposals", tags=["detection_rules", "dac"])+# Repository root: services/api/app/api/v1/endpoints/detection_proposals.py+# ^ ^ ^ ^ ^ ^ ^+# parents: [0] [1] [2] [3] [4][5] [6] = repo root+_ENDPOINT_FILE = Path(__file__).resolve()+_REPO_ROOT_DEFAULT = _ENDPOINT_FILE.parents[6]+_REPO_ROOT = Path(os.environ.get("AISOC_REPO_ROOT", str(_REPO_ROOT_DEFAULT)))+_EVAL_SCRIPT = _REPO_ROOT / "scripts" / "run_evals.py"++# ────────────────────────────────────────────────────────────────────────────# Pydantic schemas# ────────────────────────────────────────────────────────────────────────────@@ -114,6 +130,49 @@ class DecisionRequest(BaseModel):comment: str | None = Field(default=None, max_length=2000)+class RunEvalRequest(BaseModel):+ """Trigger a local run of the offline eval harness.++ The runner is `scripts/run_evals.py`. It is fully offline and deterministic+ against the 200-incident synthetic benchmark, so this endpoint can be safely+ called from the rule editor on every "Propose for review" action.+ """++ use_active_baseline: bool = Field(+ default=True,+ description=(+ "If true (default), the most recent active MITRE accuracy baseline "+ "is materialised to a temp file and passed to run_evals.py via "+ "--baseline so the runner can compute baseline_compare in-band."+ ),+ )+ max_regression_pp: float = Field(+ default=1.0,+ ge=0.0,+ le=100.0,+ description=(+ "Allowed MITRE accuracy regression vs the active baseline, in "+ "percentage points. Forwarded to run_evals.py."+ ),+ )+ timeout_seconds: int = Field(+ default=180,+ ge=10,+ le=900,+ description="Hard timeout for the eval subprocess.",+ )+++class RunEvalResponse(BaseModel):+ """Wraps the JSON report emitted by `run_evals.py`."""++ report: dict[str, Any]+ exit_code: int+ duration_seconds: float+ ran_at: datetime+ script: str++class BaselineResponse(BaseModel):id: uuid.UUIDtenant_id: uuid.UUID | None@@ -368,6 +427,184 @@ async def comment_on_proposal(return ProposalResponse.model_validate(proposal)+async def _resolve_active_baseline(+ db: Any,+ tenant_id: uuid.UUID,+) -> DetectionEvalBaseline | None:+ """Return the most recent active MITRE accuracy baseline for the tenant.++ Falls back to the platform-wide (tenant_id IS NULL) baseline.+ """+ q = await db.execute(+ select(DetectionEvalBaseline)+ .where(+ DetectionEvalBaseline.suite == "mitre_accuracy",+ DetectionEvalBaseline.is_active.is_(True),+ or_(+ DetectionEvalBaseline.tenant_id == tenant_id,+ DetectionEvalBaseline.tenant_id.is_(None),+ ),+ )+ .order_by(DetectionEvalBaseline.created_at.desc())+ .limit(1)+ )+ return q.scalar_one_or_none()+++def _run_eval_subprocess(+ *,+ baseline_path: Path | None,+ out_path: Path,+ max_regression_pp: float,+ timeout_seconds: int,+) -> tuple[int, str, str]:+ """Run `scripts/run_evals.py` synchronously and return (exit_code, stdout, stderr).++ Designed to be invoked from a thread pool via ``run_in_executor`` so the+ FastAPI event loop is not blocked.+ """+ cmd: list[str] = [+ sys.executable,+ str(_EVAL_SCRIPT),+ "--json",+ "--out",+ str(out_path),+ "--max-regression-pp",+ str(max_regression_pp),+ ]+ if baseline_path is not None:+ cmd.extend(["--baseline", str(baseline_path)])++ env = os.environ.copy()+ # The runner imports modules from services/agents — make sure that path is available.+ env.setdefault("PYTHONPATH", str(_REPO_ROOT / "services" / "agents"))++ proc = subprocess.run( # noqa: S603 — fixed argv, no shell=True+ cmd,+ cwd=str(_REPO_ROOT),+ capture_output=True,+ text=True,+ timeout=timeout_seconds,+ env=env,+ check=False,+ )+ return proc.returncode, proc.stdout, proc.stderr+++ "/run-eval",+ response_model=RunEvalResponse,+ summary="Run the offline eval harness (`scripts/run_evals.py`) and return the JSON report",+)+async def run_eval_harness(+ request: RunEvalRequest,+ current_user: Annotated[AuthUser, Depends(require_permission("rules:write"))],+ db: DBSession,+) -> RunEvalResponse:+ """Execute the offline eval harness and return its JSON output.++ This is the "eval-harness regression on save" hook for the in-app rule editor.+ The harness is deterministic and offline, but takes 5–15s, so the editor+ surfaces it as an explicit "Propose for review" action rather than firing on+ every keystroke.+ """+ _ensure_dac_enabled()++ if not _EVAL_SCRIPT.exists():+ raise HTTPException(+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,+ detail=(+ f"Eval runner missing at {_EVAL_SCRIPT}. Set AISOC_REPO_ROOT or "+ "verify the deployment includes scripts/run_evals.py."+ ),+ )++ baseline_path: Path | None = None+ baseline_tmp: tempfile._TemporaryFileWrapper[str] | None = None+ out_tmp: tempfile._TemporaryFileWrapper[str] | None = None+ started_at = datetime.now(UTC)+ try:+ if request.use_active_baseline:+ baseline = await _resolve_active_baseline(db, current_user.tenant_id)+ if baseline is not None:+ baseline_tmp = tempfile.NamedTemporaryFile( # noqa: SIM115 — closed in finally+ mode="w",+ suffix="_baseline.json",+ delete=False,+ )+ json.dump(baseline.payload or {}, baseline_tmp)+ baseline_tmp.flush()+ baseline_tmp.close()+ baseline_path = Path(baseline_tmp.name)++ out_tmp = tempfile.NamedTemporaryFile( # noqa: SIM115 — closed in finally+ mode="w",+ suffix="_eval_report.json",+ delete=False,+ )+ out_tmp.close()+ out_path = Path(out_tmp.name)++ loop = asyncio.get_event_loop()+ try:+ exit_code, stdout, stderr = await loop.run_in_executor(+ None,+ lambda: _run_eval_subprocess(+ baseline_path=baseline_path,+ out_path=out_path,+ max_regression_pp=request.max_regression_pp,+ timeout_seconds=request.timeout_seconds,+ ),+ )+ except subprocess.TimeoutExpired as exc:+ raise HTTPException(+ status_code=status.HTTP_504_GATEWAY_TIMEOUT,+ detail=f"Eval harness exceeded {request.timeout_seconds}s timeout",+ ) from exc++ # The runner exits with:+ # 0 = pass, 1 = a suite is below floor, 2 = MITRE regressed vs baseline.+ # In all three cases the JSON report is still written to --out.+ if exit_code not in (0, 1, 2):+ raise HTTPException(+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,+ detail=(+ f"Eval harness failed with exit code {exit_code}. "+ f"stderr={stderr[-2000:].strip()}"+ ),+ )++ # Prefer reading the on-disk JSON over parsing stdout — stdout is also+ # JSON when --json is set, but the runner sometimes prefixes a banner.+ try:+ report_text = out_path.read_text()+ report = json.loads(report_text) if report_text.strip() else json.loads(stdout)+ except json.JSONDecodeError as exc:+ raise HTTPException(+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,+ detail=(+ "Eval harness produced unparseable JSON. "+ f"stderr={stderr[-2000:].strip()}"+ ),+ ) from exc++ finished_at = datetime.now(UTC)+ return RunEvalResponse(+ report=report,+ exit_code=exit_code,+ duration_seconds=round((finished_at - started_at).total_seconds(), 3),+ ran_at=started_at,+ script=str(_EVAL_SCRIPT.relative_to(_REPO_ROOT)) if _EVAL_SCRIPT.is_relative_to(_REPO_ROOT) else str(_EVAL_SCRIPT),+ )+ finally:+ for tmp in (baseline_tmp, out_tmp):+ if tmp is not None:+ try:+ Path(tmp.name).unlink(missing_ok=True)+ except OSError:+ pass++@router.post("/{proposal_id}/eval", response_model=ProposalResponse)async def attach_eval_result(proposal_id: uuid.UUID,diff --git a/services/api/tests/api/v1/endpoints/test_detection_proposals_run_eval.py b/services/api/tests/api/v1/endpoints/test_detection_proposals_run_eval.pynew file mode 100644index 000000000..96882c097--- /dev/null+++ b/services/api/tests/api/v1/endpoints/test_detection_proposals_run_eval.py@@ -0,0 +1,161 @@+"""Unit tests for the WS-B4 run-eval bridge.++The `POST /detection-proposals/run-eval` endpoint is mostly orchestration:+it spawns ``scripts/run_evals.py`` via ``run_in_executor`` and surfaces the+JSON report back to the rule editor. The interesting logic is the helper+``_run_eval_subprocess`` — it owns the subprocess argv, working directory,+and ``PYTHONPATH`` env wiring. Getting that wrong silently breaks the+"eval-harness regression on save" gate, so we pin the contract here.++These tests stub ``subprocess.run`` so they remain millisecond-fast and CI-safe+(the real harness takes 5–15s).+"""++from __future__ import annotations++import json+import subprocess+import sys+from pathlib import Path+from unittest.mock import patch++import pytest+from app.api.v1.endpoints import detection_proposals as dp_endpoint+++class _FakeCompleted:+ """Minimal stand-in for ``subprocess.CompletedProcess``."""++ def __init__(self, returncode: int = 0, stdout: str = "{}", stderr: str = "") -> None:+ self.returncode = returncode+ self.stdout = stdout+ self.stderr = stderr+++def test_eval_script_path_resolves_to_real_file():+ """Sanity check: the path math finds the real run_evals.py."""+ assert dp_endpoint._EVAL_SCRIPT.exists(), (+ f"Eval runner not found at {dp_endpoint._EVAL_SCRIPT}; the path math in "+ "detection_proposals.py is wrong or the deployment is missing scripts/."+ )+ assert dp_endpoint._EVAL_SCRIPT.name == "run_evals.py"+++def test_run_eval_subprocess_builds_correct_argv(tmp_path: Path):+ """The subprocess argv must include --json, --out, and --max-regression-pp.++ Without --json the runner emits a banner instead of pure JSON, and without+ --max-regression-pp the gate gets the wrong baseline tolerance.+ """+ out_path = tmp_path / "eval_report.json"+ captured: dict[str, list[str]] = {}++ def fake_run(cmd: list[str], **kwargs): # noqa: ANN001+ captured["cmd"] = cmd+ captured["cwd"] = kwargs.get("cwd")+ captured["env"] = kwargs.get("env", {})+ return _FakeCompleted(returncode=0, stdout="{}")++ with patch.object(subprocess, "run", side_effect=fake_run):+ rc, stdout, stderr = dp_endpoint._run_eval_subprocess(+ baseline_path=None,+ out_path=out_path,+ max_regression_pp=1.5,+ timeout_seconds=60,+ )++ assert rc == 0+ cmd = captured["cmd"]+ assert cmd[0] == sys.executable+ assert cmd[1].endswith("run_evals.py")+ assert "--json" in cmd+ assert "--out" in cmd and str(out_path) in cmd+ assert "--max-regression-pp" in cmd and "1.5" in cmd+ # cwd is the repo root so relative imports inside the runner work.+ assert captured["cwd"] == str(dp_endpoint._REPO_ROOT)+ # PYTHONPATH points at services/agents so the runner can import the eval modules.+ assert "services/agents" in captured["env"].get("PYTHONPATH", "")+++def test_run_eval_subprocess_includes_baseline_when_provided(tmp_path: Path):+ """If a baseline path is supplied, the runner must receive --baseline."""+ baseline_path = tmp_path / "baseline.json"+ baseline_path.write_text(json.dumps({"score": 0.91}))+ out_path = tmp_path / "eval_report.json"+ captured: dict[str, list[str]] = {}++ def fake_run(cmd: list[str], **kwargs): # noqa: ANN001+ captured["cmd"] = cmd+ return _FakeCompleted(returncode=0, stdout="{}")++ with patch.object(subprocess, "run", side_effect=fake_run):+ dp_endpoint._run_eval_subprocess(+ baseline_path=baseline_path,+ out_path=out_path,+ max_regression_pp=1.0,+ timeout_seconds=60,+ )++ cmd = captured["cmd"]+ assert "--baseline" in cmd+ baseline_idx = cmd.index("--baseline")+ assert cmd[baseline_idx + 1] == str(baseline_path)+++def test_run_eval_subprocess_omits_baseline_when_none(tmp_path: Path):+ """Without a baseline, --baseline must NOT appear (runner skips compare)."""+ out_path = tmp_path / "eval_report.json"+ captured: dict[str, list[str]] = {}++ def fake_run(cmd: list[str], **kwargs): # noqa: ANN001+ captured["cmd"] = cmd+ return _FakeCompleted(returncode=0, stdout="{}")++ with patch.object(subprocess, "run", side_effect=fake_run):+ dp_endpoint._run_eval_subprocess(+ baseline_path=None,+ out_path=out_path,+ max_regression_pp=1.0,+ timeout_seconds=60,+ )++ assert "--baseline" not in captured["cmd"]+++def test_run_eval_subprocess_propagates_timeout(tmp_path: Path):+ """Timeout from the runner bubbles up so the API can return 504."""+ out_path = tmp_path / "eval_report.json"++ def fake_run(cmd: list[str], **kwargs): # noqa: ANN001+ raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs.get("timeout", 0))++ with patch.object(subprocess, "run", side_effect=fake_run):+ with pytest.raises(subprocess.TimeoutExpired):+ dp_endpoint._run_eval_subprocess(+ baseline_path=None,+ out_path=out_path,+ max_regression_pp=1.0,+ timeout_seconds=5,+ )+++def test_run_eval_subprocess_passes_through_nonzero_exit(tmp_path: Path):+ """Exit codes 1 and 2 are valid runner outcomes (suite below floor / regressed),+ so the helper must NOT call check=True. They are pass-through to the caller."""+ out_path = tmp_path / "eval_report.json"+ captured: dict[str, object] = {}++ def fake_run(cmd: list[str], **kwargs): # noqa: ANN001+ captured["check"] = kwargs.get("check")+ return _FakeCompleted(returncode=2, stdout='{"all_passed": true}', stderr="")++ with patch.object(subprocess, "run", side_effect=fake_run):+ rc, stdout, stderr = dp_endpoint._run_eval_subprocess(+ baseline_path=None,+ out_path=out_path,+ max_regression_pp=1.0,+ timeout_seconds=60,+ )++ assert rc == 2+ assert captured["check"] is False