Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
feat(detections): backtest as a proposal signal + opt-in decide gate + rule-editor panel (W2.2/W2.3) (#575)
Beenu Aroraaf652837ef41
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex dc0072257..37a75af5c 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -18,6 +18,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0only testing against hand-crafted fixtures. The source filter isinjection-sanitised and the SELECT is tenant-rewritten via`lake_sql.rewrite_for_tenant`. Read-only (`rules:read`). Gate: `test_backtest.py`.+ Also `POST /detection-proposals/{id}/backtest` attaches the result to the+ proposal's `eval_result["backtest"]` as a third promotion signal, with an+ opt-in `AISOC_BACKTEST_MAX_HIT_RATE` gate that blocks `/decide` approval for+ rules too noisy over history (W2.2, `test_backtest_gate.py`), and the rule+ editor gains a "Backtest over history" panel (W2.3).- **Closed loop: outcomes compound, repeat alerts auto-suppress (Wave 1).** Everydurable auto-triage outcome is now written back as a per-signature institutionalprior (`services/agents/app/memory/outcomes.py`), and a later alert matching adiff --git a/apps/web/src/components/detections/RuleEditor.tsx b/apps/web/src/components/detections/RuleEditor.tsxindex fc2ce92be..12c926103 100644--- a/apps/web/src/components/detections/RuleEditor.tsx+++ b/apps/web/src/components/detections/RuleEditor.tsx@@ -134,6 +134,16 @@ export function RuleEditor({ mode, ruleId }: RuleEditorProps) {} | null>(null);const [saving, setSaving] = useState(false);+ // Wave 2 (W2.3) — backtest a SAVED rule over real historical lake events.+ const [backtestDays, setBacktestDays] = useState(30);+ const [backtesting, setBacktesting] = useState(false);+ const [backtestResult, setBacktestResult] = useState<{+ eventsScanned: number;+ wouldFire: number;+ hitRate: number;+ error: string | null;+ } | null>(null);+// Detection-as-Code: propose-for-review state. Wave-2 buyer-value path.// The author writes a rule here, hits "Propose for review", and the backend// runs `scripts/run_evals.py` against the active baseline. The verdict is@@ -261,6 +271,41 @@ export function RuleEditor({ mode, ruleId }: RuleEditorProps) {}};+ const handleBacktest = async () => {+ if (!data?.id) {+ toast.error('Save the rule first to backtest it over historical data');+ return;+ }+ setBacktesting(true);+ setBacktestResult(null);+ try {+ const result = await detectionApi.backtest(data.id, {+ window_days: backtestDays,+ });+ setBacktestResult({+ eventsScanned: result.events_scanned,+ wouldFire: result.would_fire,+ hitRate: result.hit_rate,+ error: result.error,+ });+ toast.success(+ `Backtest: would fire on ${result.would_fire} of ${result.events_scanned} event(s)`,+ );+ } catch (err) {+ const message =+ err instanceof Error ? err.message : 'Backtest failed';+ setBacktestResult({+ eventsScanned: 0,+ wouldFire: 0,+ hitRate: 0,+ error: message,+ });+ toast.error(message);+ } finally {+ setBacktesting(false);+ }+ };+const handleSave = async () => {if (!name.trim()) {toast.error('Rule name is required');@@ -742,6 +787,68 @@ export function RuleEditor({ mode, ruleId }: RuleEditorProps) {</div>)}</div>+ {/* Wave 2 (W2.3) — backtest a saved rule over historical lake events */}+ <div className="mt-3 rounded-lg border border-gray-800 bg-gray-900/40 p-3">+ <div className="flex items-center justify-between gap-2">+ <span className="text-xs font-medium text-gray-300">+ Backtest over history+ </span>+ <div className="flex items-center gap-2">+ <label className="text-[11px] text-gray-500">days</label>+ <input+ type="number"+ min={1}+ max={365}+ value={backtestDays}+ onChange={(e) =>+ setBacktestDays(+ Math.max(+ 1,+ Math.min(365, Number(e.target.value) || 30),+ ),+ )+ }+ className="w-16 rounded-md border border-gray-800 bg-gray-950 px-2 py-1 text-xs text-gray-200 outline-none focus:border-blue-500/60"+ />+ <button+ type="button"+ onClick={handleBacktest}+ disabled={backtesting || !data?.id}+ className="rounded-md bg-blue-600/80 px-2 py-1 text-xs font-medium text-white disabled:opacity-50"+ >+ {backtesting ? 'Running…' : 'Run backtest'}+ </button>+ </div>+ </div>+ <div className="mt-2">+ {!data?.id ? (+ <p className="text-[11px] text-gray-600">+ Save the rule to backtest it against real historical+ events.+ </p>+ ) : !backtestResult ? (+ <p className="text-[11px] text-gray-600">+ Run a backtest to see how many past events this rule+ would have fired on.+ </p>+ ) : backtestResult.error ? (+ <p className="text-[11px] text-red-400">+ {backtestResult.error}+ </p>+ ) : (+ <p className="text-[11px] text-gray-300">+ Would fire on{' '}+ <span className="font-semibold text-emerald-300">+ {backtestResult.wouldFire}+ </span>{' '}+ of {backtestResult.eventsScanned} event+ {backtestResult.eventsScanned === 1 ? '' : 's'} (+ {(backtestResult.hitRate * 100).toFixed(1)}% hit-rate)+ over the last {backtestDays}d.+ </p>+ )}+ </div>+ </div></div></div></div>diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.tsindex 06129d3f2..fac9ed445 100644--- a/apps/web/src/lib/api.ts+++ b/apps/web/src/lib/api.ts@@ -3509,6 +3509,17 @@ export interface DetectionConfidence {generatedAt: string;}+export interface DetectionBacktestResult {+ rule_id: string;+ rule_language: string;+ window_days: number;+ events_scanned: number;+ would_fire: number;+ hit_rate: number;+ sample_matches: Record<string, unknown>[];+ error: string | null;+}+export const detectionApi = {list: () =>request<{ rules: DetectionRule[]; total: number }>(@@ -3517,6 +3528,17 @@ export const detectionApi = {get: (id: string) => request<DetectionRule>(`/api/v1/detection/rules/${id}`),+ // Wave 2 (W2.3) — backtest a saved rule over the last N days of real lake+ // events. Returns exactly how many events would have fired + sample matches.+ backtest: (+ id: string,+ body: { window_days?: number; limit?: number; source?: string | null },+ ) =>+ request<DetectionBacktestResult>(`/api/v1/rules/${id}/backtest`, {+ method: 'POST',+ body: JSON.stringify(body),+ }),+create: (rule: Partial<DetectionRule>) =>request<DetectionRule>('/api/v1/detection/rules', {method: 'POST',diff --git a/docs/audit/CLAIM_TO_GATE_MATRIX.md b/docs/audit/CLAIM_TO_GATE_MATRIX.mdindex 61ffc1015..e6d2691a9 100644--- a/docs/audit/CLAIM_TO_GATE_MATRIX.md+++ b/docs/audit/CLAIM_TO_GATE_MATRIX.md@@ -52,7 +52,7 @@ Statuses: `GATED` (a CI job fails when the claim stops being true) · `PARTIAL`| Institutional memory nudges fuse-time confidence (bounded, per-tenant) | README (compounding memory) | `ci.yml` fusion job (`test_memory_nudge.py` — distilled per-signature priors nudge new alerts of that signature ±0.10 in `ConfidenceScorer`; unknown signatures unchanged; `MemoryPriorProvider` distils per-tenant priors from disposition history so one tenant never nudges another) | GATED | - || Scheduled hunt findings flow into governed detection proposals | README (proactive hunting → detections) | `ci.yml` api job (`test_wave1_loop_edges.py` — a scheduled hunt that returns hits opens a DRAFT `DetectionRuleProposal` (source `hunt-finding`) into the propose→review→eval-gate→promote lifecycle; zero-hit / flag-off paths open nothing) | GATED | - || Analyst disposition history drives human-approval-gated tuning proposals | README (self-improving detections) | `ci.yml` api job (`test_wave1_loop_edges.py` — the previously-orphaned `suggest_tuning` engine is wired to real per-rule disposition history via `/detection/tuning/auto-suggest`, opening DRAFT proposals; `test_detection_tuner.py` fixture-gates the engine) | PARTIAL (mapping + wiring gated; end-to-end proposal insert exercised in api integration, not a unit DB) | Phase 2 |-| Detections can be backtested over historical lake data before promotion | README (detection engineering) | `ci.yml` api job (`test_backtest.py` — bounded/allow-listed SQL construction with injection-sanitised source filter, lake-row→event mapping with JSON payload expansion, exact would-fire count + hit-rate + samples) | PARTIAL (pure backtest logic gated; the live tenant-scoped ClickHouse fetch is exercised by the integration gate, not a unit) | Phase 2 |+| Detections can be backtested over historical lake data before promotion | README (detection engineering) | `ci.yml` api job (`test_backtest.py` — bounded/allow-listed SQL construction with injection-sanitised source filter, lake-row→event mapping with JSON payload expansion, exact would-fire count + hit-rate + samples; `test_backtest_gate.py` — opt-in `AISOC_BACKTEST_MAX_HIT_RATE` /decide gate) + web type-check (rule-editor backtest panel) | PARTIAL (pure backtest logic + gate gated; the live tenant-scoped ClickHouse fetch is exercised by the integration gate, not a unit) | Phase 2 |## Summarydiff --git a/docs/openapi.yaml b/docs/openapi.yamlindex 13fd2782a..b6e35ad7c 100644--- a/docs/openapi.yaml+++ b/docs/openapi.yaml@@ -3303,6 +3303,55 @@ paths:application/json:schema:$ref: '#/components/schemas/HTTPValidationError'+ /api/v1/detection-proposals/{proposal_id}/backtest:+ post:+ tags:+ - detection_rules+ - dac+ summary: Backtest Proposal+ description: 'Run the candidate rule over the last N days of REAL lake events+ and++ attach the result to ``eval_result["backtest"]`` (Wave 2, W2.2).+++ This is a third, honest signal alongside the fixture gate + MITRE baseline:++ a rule that would have fired on a large fraction of historical events is++ noisy. When ``AISOC_BACKTEST_MAX_HIT_RATE`` is set, ``/decide`` blocks++ approval above that hit-rate (opt-in — informational by default).'+ operationId: backtest_proposal_api_v1_detection_proposals__proposal_id__backtest_post+ security:+ - HTTPBearer: []+ parameters:+ - name: proposal_id+ in: path+ required: true+ schema:+ type: string+ format: uuid+ title: Proposal Id+ requestBody:+ required: true+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/BacktestProposalRequest'+ responses:+ '200':+ description: Successful Response+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/ProposalResponse'+ '422':+ description: Validation Error+ content:+ application/json:+ schema:+ $ref: '#/components/schemas/HTTPValidationError'/api/v1/detection-proposals/{proposal_id}/eval:post:tags:@@ -15135,6 +15184,30 @@ components:- tenant_id- actionstitle: AutonomyPolicyResponse+ BacktestProposalRequest:+ properties:+ window_days:+ type: integer+ maximum: 365.0+ minimum: 1.0+ title: Window Days+ default: 30+ limit:+ type: integer+ maximum: 100000.0+ minimum: 1.0+ title: Limit+ default: 5000+ source:+ anyOf:+ - type: string+ - type: 'null'+ title: Source+ description: Optional connector_type filter.+ type: object+ title: BacktestProposalRequest+ description: Backtest a candidate proposal's rule over historical lake events+ (W2.2).BacktestRequest:properties:window_days:diff --git a/services/api/app/api/v1/endpoints/detection_proposals.py b/services/api/app/api/v1/endpoints/detection_proposals.pyindex 5c9226cec..44ccf7b31 100644--- a/services/api/app/api/v1/endpoints/detection_proposals.py+++ b/services/api/app/api/v1/endpoints/detection_proposals.py@@ -53,12 +53,25 @@ from app.models.detection_proposal import (DetectionRuleProposal,)from app.models.detection_rule import DetectionRule+from app.services.backtest import backtest_rule, fetch_lake_eventsfrom app.services.detection_eval import evaluate_candidate_rulefrom app.services.github import create_detection_prrouter = APIRouter(prefix="/detection-proposals", tags=["detection_rules", "dac"])+def _backtest_max_hit_rate() -> float | None:+ """Optional approval gate: max backtest hit-rate before /decide blocks+ approval. Unset (default) => informational only (W2.2)."""+ raw = os.getenv("AISOC_BACKTEST_MAX_HIT_RATE", "").strip()+ if not raw:+ return None+ try:+ return float(raw)+ except ValueError:+ return None++# Resolve the AiSOC repository root by walking up from this file until we# find the eval harness this module shells out to (``scripts/run_evals.py``).# Hard-coding ``Path(__file__).parents[6]`` only works on the host, where@@ -724,6 +737,63 @@ async def evaluate_rule(return ProposalResponse.model_validate(proposal)+class BacktestProposalRequest(BaseModel):+ """Backtest a candidate proposal's rule over historical lake events (W2.2)."""++ window_days: int = Field(30, ge=1, le=365)+ limit: int = Field(5000, ge=1, le=100_000)+ source: str | None = Field(None, description="Optional connector_type filter.")++[email protected]("/{proposal_id}/backtest", response_model=ProposalResponse)+async def backtest_proposal(+ proposal_id: uuid.UUID,+ request: BacktestProposalRequest,+ current_user: Annotated[AuthUser, Depends(require_permission("rules:write"))],+ db: DBSession,+) -> ProposalResponse:+ """Run the candidate rule over the last N days of REAL lake events and+ attach the result to ``eval_result["backtest"]`` (Wave 2, W2.2).++ This is a third, honest signal alongside the fixture gate + MITRE baseline:+ a rule that would have fired on a large fraction of historical events is+ noisy. When ``AISOC_BACKTEST_MAX_HIT_RATE`` is set, ``/decide`` blocks+ approval above that hit-rate (opt-in — informational by default)."""+ _ensure_dac_enabled()+ from app.db.clickhouse import LakeQueryNotConfiguredError # noqa: PLC0415++ proposal = await _load_proposal(db, proposal_id, current_user.tenant_id)+ if proposal.status in {"promoted", "rejected"}:+ raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=f"Proposal is {proposal.status}; backtest cannot be re-run")++ try:+ events = await fetch_lake_events(+ current_user.tenant_id,+ window_days=request.window_days,+ limit=request.limit,+ source=request.source,+ )+ except LakeQueryNotConfiguredError as exc:+ raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="lake backend not configured") from exc+ except Exception as exc: # noqa: BLE001 — lake failure is a 502, not 500+ raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"lake query failed: {exc}") from exc++ summary = backtest_rule(rule_language=proposal.rule_language, rule_body=proposal.rule_body, events=events)+ merged = dict(proposal.eval_result or {})+ merged["backtest"] = {+ "window_days": request.window_days,+ "events_scanned": summary["events_scanned"],+ "would_fire": summary["would_fire"],+ "hit_rate": summary["hit_rate"],+ "error": summary["error"],+ "ran_at": datetime.now(UTC).isoformat(),+ }+ proposal.eval_result = merged+ await db.commit()+ await db.refresh(proposal)+ return ProposalResponse.model_validate(proposal)++@router.post("/{proposal_id}/eval", response_model=ProposalResponse)async def attach_eval_result(proposal_id: uuid.UUID,@@ -832,6 +902,20 @@ async def decide_proposal(status_code=status.HTTP_412_PRECONDITION_FAILED,detail="Substrate benchmark gate failed (MITRE regression); cannot approve.",)+ # W2.2 — optional backtest hit-rate gate (opt-in; informational by+ # default). A rule that would fire on too large a fraction of real+ # historical events is too noisy to promote.+ gate = _backtest_max_hit_rate()+ if gate is not None:+ backtest = eval_result.get("backtest")+ if isinstance(backtest, dict) and float(backtest.get("hit_rate", 0.0)) > gate:+ raise HTTPException(+ status_code=status.HTTP_412_PRECONDITION_FAILED,+ detail=(+ f"Backtest hit-rate {float(backtest['hit_rate']):.0%} exceeds the configured "+ f"maximum {gate:.0%} — rule is too noisy over history; cannot approve."+ ),+ )proposal.status = "approved"else:proposal.status = "rejected"diff --git a/services/api/app/services/backtest.py b/services/api/app/services/backtest.pyindex 31f360b5f..8bdeed674 100644--- a/services/api/app/services/backtest.py+++ b/services/api/app/services/backtest.py@@ -15,6 +15,7 @@ from __future__ import annotationsimport jsonimport re+import uuidfrom typing import Anyfrom app.services.rule_engine import evaluate_rule_over_events@@ -66,6 +67,24 @@ def rows_to_events(columns: list[str], rows: list[list[Any]]) -> list[dict[str,return events+async def fetch_lake_events(+ tenant_id: uuid.UUID,+ *,+ window_days: int,+ limit: int,+ source: str | None,+) -> list[dict[str, Any]]:+ """Fetch tenant-scoped historical events from the ClickHouse lake for a+ backtest. Shared by the rule + proposal backtest endpoints."""+ from app.db.clickhouse import execute_lake_query # noqa: PLC0415 — optional lake backend+ from app.services.lake_sql import rewrite_for_tenant # noqa: PLC0415++ sql = build_backtest_sql(window_days=window_days, limit=limit, source=source)+ rewrite = rewrite_for_tenant(sql, tenant_id, row_cap=limit)+ result = await execute_lake_query(rewrite.sql, timeout_seconds=30)+ return rows_to_events(result.columns, result.rows)++def backtest_rule(*,rule_language: str,diff --git a/services/api/tests/test_backtest_gate.py b/services/api/tests/test_backtest_gate.pynew file mode 100644index 000000000..34cf0b408--- /dev/null+++ b/services/api/tests/test_backtest_gate.py@@ -0,0 +1,20 @@+"""Wave 2 (W2.2) - the opt-in backtest hit-rate approval gate."""++from __future__ import annotations++from app.api.v1.endpoints.detection_proposals import _backtest_max_hit_rate+++def test_gate_disabled_by_default(monkeypatch):+ monkeypatch.delenv("AISOC_BACKTEST_MAX_HIT_RATE", raising=False)+ assert _backtest_max_hit_rate() is None+++def test_gate_reads_threshold(monkeypatch):+ monkeypatch.setenv("AISOC_BACKTEST_MAX_HIT_RATE", "0.5")+ assert _backtest_max_hit_rate() == 0.5+++def test_gate_ignores_garbage(monkeypatch):+ monkeypatch.setenv("AISOC_BACKTEST_MAX_HIT_RATE", "not-a-number")+ assert _backtest_max_hit_rate() is None