Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

feat(detection): WS-B4 detection-as-code propose-for-review with eval gate

Beenu Arora2ae5c94430ff
diff --git a/apps/web/src/components/detections/RuleEditor.tsx b/apps/web/src/components/detections/RuleEditor.tsx
index 17a15c595..7f19a1608 100644
--- a/apps/web/src/components/detections/RuleEditor.tsx
+++ b/apps/web/src/components/detections/RuleEditor.tsx
@@ -9,7 +9,9 @@ import { clsx } from 'clsx';
import toast from 'react-hot-toast';
import {
detectionApi,
+ detectionProposalsApi,
type DetectionLanguage,
+ type DetectionProposal,
type DetectionRule,
type HuntResult,
} from '@/lib/api';
@@ -132,6 +134,26 @@ export function RuleEditor({ mode, ruleId }: RuleEditorProps) {
} | null>(null);
const [saving, setSaving] = useState(false);
+ // Detection-as-Code: propose-for-review state. Wave-2 buyer-value path.
+ // The author writes a rule here, hits "Propose for review", and the backend
+ // runs `scripts/run_evals.py` against the active baseline. The verdict is
+ // attached to the proposal and the regression gate (≥ 1pp MITRE accuracy
+ // drop) decides whether the proposal can be promoted to a live rule.
+ const [proposing, setProposing] = useState(false);
+ const [proposalStage, setProposalStage] = useState<
+ 'idle' | 'creating' | 'evaluating' | 'attaching' | 'done'
+ >('idle');
+ const [evalVerdict, setEvalVerdict] = useState<{
+ proposalId: string;
+ passed: boolean;
+ candidateMitre?: number;
+ baselineMitre?: number;
+ dropPp?: number;
+ maxRegressionPp?: number;
+ durationSeconds?: number;
+ ranAt?: string;
+ } | null>(null);
+
// Hydrate form when data arrives
useEffect(() => {
if (data) {
@@ -289,6 +311,119 @@ export function RuleEditor({ mode, ruleId }: RuleEditorProps) {
}
};
+ // Detection-as-Code propose-for-review flow:
+ // 1) POST /detection-proposals → register the candidate
+ // 2) POST /detection-proposals/run-eval → run scripts/run_evals.py
+ // 3) POST /detection-proposals/:id/eval → attach verdict (server gates)
+ // We surface a verdict card on the editor and link to the queue at
+ // /detection/proposals so a reviewer can take it from approved → promoted.
+ // If any leg fails we keep the proposal in 'proposed' state and surface
+ // the error — partial state is fine because the queue ignores stale eval
+ // attachments.
+ const handlePropose = async () => {
+ if (!name.trim()) {
+ toast.error('Rule name is required to propose a change');
+ return;
+ }
+ if (!body.trim()) {
+ toast.error('Rule body is empty');
+ return;
+ }
+ setProposing(true);
+ setEvalVerdict(null);
+ setProposalStage('creating');
+ try {
+ // 1. Create the proposal — base_rule_id links it to the rule being
+ // edited so the reviewer queue can show "modified vs. live".
+ const created = await detectionProposalsApi.create({
+ name: name.trim(),
+ description: description.trim() || null,
+ rule_language: language,
+ rule_body: body,
+ // The rule editor has no category picker today; the proposal API
+ // requires a non-empty string, so default to a sensible value.
+ category: 'detection',
+ severity,
+ confidence: 0.7,
+ mitre_techniques: mitre,
+ mitre_tactics: [],
+ tags,
+ base_rule_id: mode === 'edit' && ruleId ? ruleId : null,
+ });
+
+ // 2. Run the offline eval harness server-side. This blocks the request
+ // for ~5–15s on the 200-incident corpus, which is acceptable for an
+ // explicit "Propose" click.
+ setProposalStage('evaluating');
+ let runResult: Awaited<ReturnType<typeof detectionProposalsApi.runEval>>;
+ try {
+ runResult = await detectionProposalsApi.runEval({
+ use_active_baseline: true,
+ max_regression_pp: 1.0,
+ timeout_seconds: 240,
+ });
+ } catch (evalErr) {
+ console.error('Eval harness call failed', evalErr);
+ toast.error(
+ 'Proposal created, but the eval harness could not run. A reviewer can attach a report manually.',
+ );
+ setEvalVerdict({
+ proposalId: created.id,
+ passed: false,
+ });
+ setProposalStage('done');
+ // Best effort: still navigate to the proposal queue so the user
+ // can see it queued.
+ router.push('/detection/proposals');
+ return;
+ }
+
+ // 3. Attach the verdict — backend authoritatively decides eval_passed
+ // vs. eval_failed using its 1pp regression rule against the active
+ // baseline. We pass the same threshold the runner used so the
+ // server's recorded `max_regression_pp` matches.
+ setProposalStage('attaching');
+ const decided = await detectionProposalsApi.attachEval(created.id, {
+ eval_report: runResult.report,
+ max_regression_pp: 1.0,
+ });
+
+ const verdict = decided.eval_result as DetectionProposal['eval_result'];
+ const passed = decided.status === 'eval_passed';
+ setEvalVerdict({
+ proposalId: decided.id,
+ passed,
+ candidateMitre:
+ (verdict as { candidate?: { mitre_accuracy?: number } } | undefined)
+ ?.candidate?.mitre_accuracy,
+ baselineMitre:
+ (verdict as { baseline?: { mitre_accuracy?: number } } | undefined)
+ ?.baseline?.mitre_accuracy,
+ dropPp: (verdict as { drop_pp?: number } | undefined)?.drop_pp,
+ maxRegressionPp:
+ (verdict as { max_regression_pp?: number } | undefined)
+ ?.max_regression_pp,
+ durationSeconds: runResult.duration_seconds,
+ ranAt: runResult.ran_at,
+ });
+ setProposalStage('done');
+
+ if (passed) {
+ toast.success('Proposal passed eval — ready for review');
+ } else {
+ toast.error(
+ 'Proposal failed eval — MITRE accuracy regression vs. active baseline',
+ );
+ }
+ } catch (err) {
+ console.error('Propose-for-review failed', err);
+ toast.error('Could not create proposal — backend unreachable');
+ setProposalStage('idle');
+ } finally {
+ setProposing(false);
+ }
+ };
+
// ─── Render ────────────────────────────────────────────────────────────────
return (
@@ -317,6 +452,29 @@ export function RuleEditor({ mode, ruleId }: RuleEditorProps) {
Delete
</button>
)}
+ {/*
+ "Propose for review" runs the eval-gated DaC flow. The button is
+ distinct from "Save" because it doesn't write to the live rule —
+ it creates a queued proposal that a reviewer promotes after the
+ eval verdict + a human approval. We show it on both create and
+ edit so authors can iterate on a draft and only submit once the
+ harness passes.
+ */}
+ <button
+ onClick={handlePropose}
+ disabled={proposing || saving}
+ className={clsx(
+ 'inline-flex items-center gap-2 rounded-md border border-blue-500/40 bg-blue-500/10 px-3 py-1.5 text-sm font-medium text-blue-200 shadow-sm transition-colors',
+ proposing || saving ? 'opacity-60' : 'hover:bg-blue-500/20',
+ )}
+ title="Open a proposal that the eval harness must pass before promotion"
+ >
+ {proposalStage === 'creating' && 'Creating proposal…'}
+ {proposalStage === 'evaluating' && 'Running eval harness…'}
+ {proposalStage === 'attaching' && 'Attaching verdict…'}
+ {(proposalStage === 'idle' || proposalStage === 'done') &&
+ 'Propose for review'}
+ </button>
<button
onClick={handleSave}
disabled={saving}
@@ -330,6 +488,93 @@ export function RuleEditor({ mode, ruleId }: RuleEditorProps) {
</div>
</div>
+ {/*
+ Eval verdict card — surfaces the outcome of the most recent
+ "Propose for review" click against the active 200-incident
+ baseline. We compare candidate vs. baseline MITRE accuracy and
+ the absolute drop in percentage points, mirroring the gate in
+ services/api/app/api/v1/endpoints/detection_proposals.py. Reviewer
+ actions (approve / reject / promote) live on /detection/proposals,
+ which we deep-link to once we have a proposal id.
+ */}
+ {evalVerdict && (
+ <div
+ className={clsx(
+ 'rounded-lg border px-4 py-3 text-sm',
+ evalVerdict.passed
+ ? 'border-emerald-500/30 bg-emerald-500/5 text-emerald-100'
+ : 'border-red-500/30 bg-red-500/5 text-red-100',
+ )}
+ >
+ <div className="flex items-center justify-between gap-3">
+ <div className="flex items-center gap-2 font-medium">
+ <span
+ className={clsx(
+ 'inline-flex h-2 w-2 rounded-full',
+ evalVerdict.passed ? 'bg-emerald-400' : 'bg-red-400',
+ )}
+ />
+ {evalVerdict.passed
+ ? 'Eval passed — proposal ready for review'
+ : 'Eval failed — MITRE accuracy regression'}
+ </div>
+ <Link
+ href={`/detection/proposals?highlight=${evalVerdict.proposalId}`}
+ className="text-xs underline-offset-2 hover:underline"
+ >
+ View in proposal queue →
+ </Link>
+ </div>
+ {(evalVerdict.candidateMitre !== undefined ||
+ evalVerdict.baselineMitre !== undefined ||
+ evalVerdict.dropPp !== undefined) && (
+ <div className="mt-2 grid grid-cols-2 gap-x-6 gap-y-1 text-xs text-gray-200 sm:grid-cols-4">
+ {evalVerdict.candidateMitre !== undefined && (
+ <div>
+ <span className="text-gray-400">Candidate MITRE</span>{' '}
+ <span className="font-mono">
+ {(evalVerdict.candidateMitre * 100).toFixed(1)}%
+ </span>
+ </div>
+ )}
+ {evalVerdict.baselineMitre !== undefined && (
+ <div>
+ <span className="text-gray-400">Baseline MITRE</span>{' '}
+ <span className="font-mono">
+ {(evalVerdict.baselineMitre * 100).toFixed(1)}%
+ </span>
+ </div>
+ )}
+ {evalVerdict.dropPp !== undefined && (
+ <div>
+ <span className="text-gray-400">Δ pp</span>{' '}
+ <span className="font-mono">
+ {evalVerdict.dropPp >= 0 ? '+' : ''}
+ {(-evalVerdict.dropPp).toFixed(2)}
+ </span>
+ </div>
+ )}
+ {evalVerdict.maxRegressionPp !== undefined && (
+ <div>
+ <span className="text-gray-400">Gate</span>{' '}
+ <span className="font-mono">
+ ≤ {evalVerdict.maxRegressionPp.toFixed(2)} pp
+ </span>
+ </div>
+ )}
+ {evalVerdict.durationSeconds !== undefined && (
+ <div>
+ <span className="text-gray-400">Runtime</span>{' '}
+ <span className="font-mono">
+ {evalVerdict.durationSeconds.toFixed(1)}s
+ </span>
+ </div>
+ )}
+ </div>
+ )}
+ </div>
+ )}
+
{/*
Ambient Copilot — rule-scoped contextual AI. Only shown in edit mode where
the rule has a stable id + body to reason about; new-rule drafts have no
diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts
index a37bc5ee7..ec4972ea7 100644
--- a/apps/web/src/lib/api.ts
+++ b/apps/web/src/lib/api.ts
@@ -2369,6 +2369,26 @@ export const detectionProposalsApi = {
body: JSON.stringify(body),
}),
+ // Triggers a synchronous run of `scripts/run_evals.py` server-side and
+ // returns the JSON report plus runner metadata. The rule editor calls this
+ // from "Propose for review" so the eval gate runs before the proposal is
+ // visible for approval. Subprocess takes ~5–15s on the 200-incident corpus.
+ runEval: (body?: {
+ use_active_baseline?: boolean;
+ max_regression_pp?: number;
+ timeout_seconds?: number;
+ }) =>
+ request<{
+ report: Record<string, unknown>;
+ exit_code: number;
+ duration_seconds: number;
+ ran_at: string;
+ script: string;
+ }>('/api/v1/detection-proposals/run-eval', {
+ method: 'POST',
+ body: JSON.stringify(body ?? {}),
+ }),
+
decide: (
id: string,
body: { decision: 'approve' | 'reject'; comment?: string | null },
diff --git a/services/api/app/api/v1/endpoints/detection_proposals.py b/services/api/app/api/v1/endpoints/detection_proposals.py
index dca677f79..3c2272761 100644
--- a/services/api/app/api/v1/endpoints/detection_proposals.py
+++ b/services/api/app/api/v1/endpoints/detection_proposals.py
@@ -22,8 +22,15 @@ Endpoints
from __future__ import annotations
+import asyncio
+import json
+import os
+import subprocess
+import sys
+import tempfile
import uuid
from datetime import UTC, datetime
+from pathlib import Path
from typing import Annotated, Any, Literal
from fastapi import APIRouter, Depends, HTTPException, Query, status
@@ -41,6 +48,15 @@ from app.models.detection_rule import DetectionRule
router = APIRouter(prefix="/detection-proposals", tags=["detection_rules", "dac"])
+# Repository root: services/api/app/api/v1/endpoints/detection_proposals.py
+# ^ ^ ^ ^ ^ ^ ^
+# parents: [0] [1] [2] [3] [4][5] [6] = repo root
+_ENDPOINT_FILE = Path(__file__).resolve()
+_REPO_ROOT_DEFAULT = _ENDPOINT_FILE.parents[6]
+_REPO_ROOT = Path(os.environ.get("AISOC_REPO_ROOT", str(_REPO_ROOT_DEFAULT)))
+_EVAL_SCRIPT = _REPO_ROOT / "scripts" / "run_evals.py"
+
+
# ────────────────────────────────────────────────────────────────────────────
# Pydantic schemas
# ────────────────────────────────────────────────────────────────────────────
@@ -114,6 +130,49 @@ class DecisionRequest(BaseModel):
comment: str | None = Field(default=None, max_length=2000)
+class RunEvalRequest(BaseModel):
+ """Trigger a local run of the offline eval harness.
+
+ The runner is `scripts/run_evals.py`. It is fully offline and deterministic
+ against the 200-incident synthetic benchmark, so this endpoint can be safely
+ called from the rule editor on every "Propose for review" action.
+ """
+
+ use_active_baseline: bool = Field(
+ default=True,
+ description=(
+ "If true (default), the most recent active MITRE accuracy baseline "
+ "is materialised to a temp file and passed to run_evals.py via "
+ "--baseline so the runner can compute baseline_compare in-band."
+ ),
+ )
+ max_regression_pp: float = Field(
+ default=1.0,
+ ge=0.0,
+ le=100.0,
+ description=(
+ "Allowed MITRE accuracy regression vs the active baseline, in "
+ "percentage points. Forwarded to run_evals.py."
+ ),
+ )
+ timeout_seconds: int = Field(
+ default=180,
+ ge=10,
+ le=900,
+ description="Hard timeout for the eval subprocess.",
+ )
+
+
+class RunEvalResponse(BaseModel):
+ """Wraps the JSON report emitted by `run_evals.py`."""
+
+ report: dict[str, Any]
+ exit_code: int
+ duration_seconds: float
+ ran_at: datetime
+ script: str
+
+
class BaselineResponse(BaseModel):
id: uuid.UUID
tenant_id: uuid.UUID | None
@@ -368,6 +427,184 @@ async def comment_on_proposal(
return ProposalResponse.model_validate(proposal)
+async def _resolve_active_baseline(
+ db: Any,
+ tenant_id: uuid.UUID,
+) -> DetectionEvalBaseline | None:
+ """Return the most recent active MITRE accuracy baseline for the tenant.
+
+ Falls back to the platform-wide (tenant_id IS NULL) baseline.
+ """
+ q = await db.execute(
+ select(DetectionEvalBaseline)
+ .where(
+ DetectionEvalBaseline.suite == "mitre_accuracy",
+ DetectionEvalBaseline.is_active.is_(True),
+ or_(
+ DetectionEvalBaseline.tenant_id == tenant_id,
+ DetectionEvalBaseline.tenant_id.is_(None),
+ ),
+ )
+ .order_by(DetectionEvalBaseline.created_at.desc())
+ .limit(1)
+ )
+ return q.scalar_one_or_none()
+
+
+def _run_eval_subprocess(
+ *,
+ baseline_path: Path | None,
+ out_path: Path,
+ max_regression_pp: float,
+ timeout_seconds: int,
+) -> tuple[int, str, str]:
+ """Run `scripts/run_evals.py` synchronously and return (exit_code, stdout, stderr).
+
+ Designed to be invoked from a thread pool via ``run_in_executor`` so the
+ FastAPI event loop is not blocked.
+ """
+ cmd: list[str] = [
+ sys.executable,
+ str(_EVAL_SCRIPT),
+ "--json",
+ "--out",
+ str(out_path),
+ "--max-regression-pp",
+ str(max_regression_pp),
+ ]
+ if baseline_path is not None:
+ cmd.extend(["--baseline", str(baseline_path)])
+
+ env = os.environ.copy()
+ # The runner imports modules from services/agents — make sure that path is available.
+ env.setdefault("PYTHONPATH", str(_REPO_ROOT / "services" / "agents"))
+
+ proc = subprocess.run( # noqa: S603 — fixed argv, no shell=True
+ cmd,
+ cwd=str(_REPO_ROOT),
+ capture_output=True,
+ text=True,
+ timeout=timeout_seconds,
+ env=env,
+ check=False,
+ )
+ return proc.returncode, proc.stdout, proc.stderr
+
+
+ "/run-eval",
+ response_model=RunEvalResponse,
+ summary="Run the offline eval harness (`scripts/run_evals.py`) and return the JSON report",
+)
+async def run_eval_harness(
+ request: RunEvalRequest,
+ current_user: Annotated[AuthUser, Depends(require_permission("rules:write"))],
+ db: DBSession,
+) -> RunEvalResponse:
+ """Execute the offline eval harness and return its JSON output.
+
+ This is the "eval-harness regression on save" hook for the in-app rule editor.
+ The harness is deterministic and offline, but takes 5–15s, so the editor
+ surfaces it as an explicit "Propose for review" action rather than firing on
+ every keystroke.
+ """
+ _ensure_dac_enabled()
+
+ if not _EVAL_SCRIPT.exists():
+ raise HTTPException(
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+ detail=(
+ f"Eval runner missing at {_EVAL_SCRIPT}. Set AISOC_REPO_ROOT or "
+ "verify the deployment includes scripts/run_evals.py."
+ ),
+ )
+
+ baseline_path: Path | None = None
+ baseline_tmp: tempfile._TemporaryFileWrapper[str] | None = None
+ out_tmp: tempfile._TemporaryFileWrapper[str] | None = None
+ started_at = datetime.now(UTC)
+ try:
+ if request.use_active_baseline:
+ baseline = await _resolve_active_baseline(db, current_user.tenant_id)
+ if baseline is not None:
+ baseline_tmp = tempfile.NamedTemporaryFile( # noqa: SIM115 — closed in finally
+ mode="w",
+ suffix="_baseline.json",
+ delete=False,
+ )
+ json.dump(baseline.payload or {}, baseline_tmp)
+ baseline_tmp.flush()
+ baseline_tmp.close()
+ baseline_path = Path(baseline_tmp.name)
+
+ out_tmp = tempfile.NamedTemporaryFile( # noqa: SIM115 — closed in finally
+ mode="w",
+ suffix="_eval_report.json",
+ delete=False,
+ )
+ out_tmp.close()
+ out_path = Path(out_tmp.name)
+
+ loop = asyncio.get_event_loop()
+ try:
+ exit_code, stdout, stderr = await loop.run_in_executor(
+ None,
+ lambda: _run_eval_subprocess(
+ baseline_path=baseline_path,
+ out_path=out_path,
+ max_regression_pp=request.max_regression_pp,
+ timeout_seconds=request.timeout_seconds,
+ ),
+ )
+ except subprocess.TimeoutExpired as exc:
+ raise HTTPException(
+ status_code=status.HTTP_504_GATEWAY_TIMEOUT,
+ detail=f"Eval harness exceeded {request.timeout_seconds}s timeout",
+ ) from exc
+
+ # The runner exits with:
+ # 0 = pass, 1 = a suite is below floor, 2 = MITRE regressed vs baseline.
+ # In all three cases the JSON report is still written to --out.
+ if exit_code not in (0, 1, 2):
+ raise HTTPException(
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+ detail=(
+ f"Eval harness failed with exit code {exit_code}. "
+ f"stderr={stderr[-2000:].strip()}"
+ ),
+ )
+
+ # Prefer reading the on-disk JSON over parsing stdout — stdout is also
+ # JSON when --json is set, but the runner sometimes prefixes a banner.
+ try:
+ report_text = out_path.read_text()
+ report = json.loads(report_text) if report_text.strip() else json.loads(stdout)
+ except json.JSONDecodeError as exc:
+ raise HTTPException(
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+ detail=(
+ "Eval harness produced unparseable JSON. "
+ f"stderr={stderr[-2000:].strip()}"
+ ),
+ ) from exc
+
+ finished_at = datetime.now(UTC)
+ return RunEvalResponse(
+ report=report,
+ exit_code=exit_code,
+ duration_seconds=round((finished_at - started_at).total_seconds(), 3),
+ ran_at=started_at,
+ script=str(_EVAL_SCRIPT.relative_to(_REPO_ROOT)) if _EVAL_SCRIPT.is_relative_to(_REPO_ROOT) else str(_EVAL_SCRIPT),
+ )
+ finally:
+ for tmp in (baseline_tmp, out_tmp):
+ if tmp is not None:
+ try:
+ Path(tmp.name).unlink(missing_ok=True)
+ except OSError:
+ pass
+
+
@router.post("/{proposal_id}/eval", response_model=ProposalResponse)
async def attach_eval_result(
proposal_id: uuid.UUID,
diff --git a/services/api/tests/api/v1/endpoints/test_detection_proposals_run_eval.py b/services/api/tests/api/v1/endpoints/test_detection_proposals_run_eval.py
new file mode 100644
index 000000000..96882c097
--- /dev/null
+++ b/services/api/tests/api/v1/endpoints/test_detection_proposals_run_eval.py
@@ -0,0 +1,161 @@
+"""Unit tests for the WS-B4 run-eval bridge.
+
+The `POST /detection-proposals/run-eval` endpoint is mostly orchestration:
+it spawns ``scripts/run_evals.py`` via ``run_in_executor`` and surfaces the
+JSON report back to the rule editor. The interesting logic is the helper
+``_run_eval_subprocess`` — it owns the subprocess argv, working directory,
+and ``PYTHONPATH`` env wiring. Getting that wrong silently breaks the
+"eval-harness regression on save" gate, so we pin the contract here.
+
+These tests stub ``subprocess.run`` so they remain millisecond-fast and CI-safe
+(the real harness takes 5–15s).
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+from app.api.v1.endpoints import detection_proposals as dp_endpoint
+
+
+class _FakeCompleted:
+ """Minimal stand-in for ``subprocess.CompletedProcess``."""
+
+ def __init__(self, returncode: int = 0, stdout: str = "{}", stderr: str = "") -> None:
+ self.returncode = returncode
+ self.stdout = stdout
+ self.stderr = stderr
+
+
+def test_eval_script_path_resolves_to_real_file():
+ """Sanity check: the path math finds the real run_evals.py."""
+ assert dp_endpoint._EVAL_SCRIPT.exists(), (
+ f"Eval runner not found at {dp_endpoint._EVAL_SCRIPT}; the path math in "
+ "detection_proposals.py is wrong or the deployment is missing scripts/."
+ )
+ assert dp_endpoint._EVAL_SCRIPT.name == "run_evals.py"
+
+
+def test_run_eval_subprocess_builds_correct_argv(tmp_path: Path):
+ """The subprocess argv must include --json, --out, and --max-regression-pp.
+
+ Without --json the runner emits a banner instead of pure JSON, and without
+ --max-regression-pp the gate gets the wrong baseline tolerance.
+ """
+ out_path = tmp_path / "eval_report.json"
+ captured: dict[str, list[str]] = {}
+
+ def fake_run(cmd: list[str], **kwargs): # noqa: ANN001
+ captured["cmd"] = cmd
+ captured["cwd"] = kwargs.get("cwd")
+ captured["env"] = kwargs.get("env", {})
+ return _FakeCompleted(returncode=0, stdout="{}")
+
+ with patch.object(subprocess, "run", side_effect=fake_run):
+ rc, stdout, stderr = dp_endpoint._run_eval_subprocess(
+ baseline_path=None,
+ out_path=out_path,
+ max_regression_pp=1.5,
+ timeout_seconds=60,
+ )
+
+ assert rc == 0
+ cmd = captured["cmd"]
+ assert cmd[0] == sys.executable
+ assert cmd[1].endswith("run_evals.py")
+ assert "--json" in cmd
+ assert "--out" in cmd and str(out_path) in cmd
+ assert "--max-regression-pp" in cmd and "1.5" in cmd
+ # cwd is the repo root so relative imports inside the runner work.
+ assert captured["cwd"] == str(dp_endpoint._REPO_ROOT)
+ # PYTHONPATH points at services/agents so the runner can import the eval modules.
+ assert "services/agents" in captured["env"].get("PYTHONPATH", "")
+
+
+def test_run_eval_subprocess_includes_baseline_when_provided(tmp_path: Path):
+ """If a baseline path is supplied, the runner must receive --baseline."""
+ baseline_path = tmp_path / "baseline.json"
+ baseline_path.write_text(json.dumps({"score": 0.91}))
+ out_path = tmp_path / "eval_report.json"
+ captured: dict[str, list[str]] = {}
+
+ def fake_run(cmd: list[str], **kwargs): # noqa: ANN001
+ captured["cmd"] = cmd
+ return _FakeCompleted(returncode=0, stdout="{}")
+
+ with patch.object(subprocess, "run", side_effect=fake_run):
+ dp_endpoint._run_eval_subprocess(
+ baseline_path=baseline_path,
+ out_path=out_path,
+ max_regression_pp=1.0,
+ timeout_seconds=60,
+ )
+
+ cmd = captured["cmd"]
+ assert "--baseline" in cmd
+ baseline_idx = cmd.index("--baseline")
+ assert cmd[baseline_idx + 1] == str(baseline_path)
+
+
+def test_run_eval_subprocess_omits_baseline_when_none(tmp_path: Path):
+ """Without a baseline, --baseline must NOT appear (runner skips compare)."""
+ out_path = tmp_path / "eval_report.json"
+ captured: dict[str, list[str]] = {}
+
+ def fake_run(cmd: list[str], **kwargs): # noqa: ANN001
+ captured["cmd"] = cmd
+ return _FakeCompleted(returncode=0, stdout="{}")
+
+ with patch.object(subprocess, "run", side_effect=fake_run):
+ dp_endpoint._run_eval_subprocess(
+ baseline_path=None,
+ out_path=out_path,
+ max_regression_pp=1.0,
+ timeout_seconds=60,
+ )
+
+ assert "--baseline" not in captured["cmd"]
+
+
+def test_run_eval_subprocess_propagates_timeout(tmp_path: Path):
+ """Timeout from the runner bubbles up so the API can return 504."""
+ out_path = tmp_path / "eval_report.json"
+
+ def fake_run(cmd: list[str], **kwargs): # noqa: ANN001
+ raise subprocess.TimeoutExpired(cmd=cmd, timeout=kwargs.get("timeout", 0))
+
+ with patch.object(subprocess, "run", side_effect=fake_run):
+ with pytest.raises(subprocess.TimeoutExpired):
+ dp_endpoint._run_eval_subprocess(
+ baseline_path=None,
+ out_path=out_path,
+ max_regression_pp=1.0,
+ timeout_seconds=5,
+ )
+
+
+def test_run_eval_subprocess_passes_through_nonzero_exit(tmp_path: Path):
+ """Exit codes 1 and 2 are valid runner outcomes (suite below floor / regressed),
+ so the helper must NOT call check=True. They are pass-through to the caller."""
+ out_path = tmp_path / "eval_report.json"
+ captured: dict[str, object] = {}
+
+ def fake_run(cmd: list[str], **kwargs): # noqa: ANN001
+ captured["check"] = kwargs.get("check")
+ return _FakeCompleted(returncode=2, stdout='{"all_passed": true}', stderr="")
+
+ with patch.object(subprocess, "run", side_effect=fake_run):
+ rc, stdout, stderr = dp_endpoint._run_eval_subprocess(
+ baseline_path=None,
+ out_path=out_path,
+ max_regression_pp=1.0,
+ timeout_seconds=60,
+ )
+
+ assert rc == 2
+ assert captured["check"] is False