Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

feat(detections): backtest as a proposal signal + opt-in decide gate + rule-editor panel (W2.2/W2.3) (#575)

Beenu Aroraaf652837ef41
diff --git a/CHANGELOG.md b/CHANGELOG.md
index dc0072257..37a75af5c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
only testing against hand-crafted fixtures. The source filter is
injection-sanitised and the SELECT is tenant-rewritten via
`lake_sql.rewrite_for_tenant`. Read-only (`rules:read`). Gate: `test_backtest.py`.
+ Also `POST /detection-proposals/{id}/backtest` attaches the result to the
+ proposal's `eval_result["backtest"]` as a third promotion signal, with an
+ opt-in `AISOC_BACKTEST_MAX_HIT_RATE` gate that blocks `/decide` approval for
+ rules too noisy over history (W2.2, `test_backtest_gate.py`), and the rule
+ editor gains a "Backtest over history" panel (W2.3).
- **Closed loop: outcomes compound, repeat alerts auto-suppress (Wave 1).** Every
durable auto-triage outcome is now written back as a per-signature institutional
prior (`services/agents/app/memory/outcomes.py`), and a later alert matching a
diff --git a/apps/web/src/components/detections/RuleEditor.tsx b/apps/web/src/components/detections/RuleEditor.tsx
index fc2ce92be..12c926103 100644
--- a/apps/web/src/components/detections/RuleEditor.tsx
+++ b/apps/web/src/components/detections/RuleEditor.tsx
@@ -134,6 +134,16 @@ export function RuleEditor({ mode, ruleId }: RuleEditorProps) {
} | null>(null);
const [saving, setSaving] = useState(false);
+ // Wave 2 (W2.3) — backtest a SAVED rule over real historical lake events.
+ const [backtestDays, setBacktestDays] = useState(30);
+ const [backtesting, setBacktesting] = useState(false);
+ const [backtestResult, setBacktestResult] = useState<{
+ eventsScanned: number;
+ wouldFire: number;
+ hitRate: number;
+ error: string | null;
+ } | null>(null);
+
// Detection-as-Code: propose-for-review state. Wave-2 buyer-value path.
// The author writes a rule here, hits "Propose for review", and the backend
// runs `scripts/run_evals.py` against the active baseline. The verdict is
@@ -261,6 +271,41 @@ export function RuleEditor({ mode, ruleId }: RuleEditorProps) {
}
};
+ const handleBacktest = async () => {
+ if (!data?.id) {
+ toast.error('Save the rule first to backtest it over historical data');
+ return;
+ }
+ setBacktesting(true);
+ setBacktestResult(null);
+ try {
+ const result = await detectionApi.backtest(data.id, {
+ window_days: backtestDays,
+ });
+ setBacktestResult({
+ eventsScanned: result.events_scanned,
+ wouldFire: result.would_fire,
+ hitRate: result.hit_rate,
+ error: result.error,
+ });
+ toast.success(
+ `Backtest: would fire on ${result.would_fire} of ${result.events_scanned} event(s)`,
+ );
+ } catch (err) {
+ const message =
+ err instanceof Error ? err.message : 'Backtest failed';
+ setBacktestResult({
+ eventsScanned: 0,
+ wouldFire: 0,
+ hitRate: 0,
+ error: message,
+ });
+ toast.error(message);
+ } finally {
+ setBacktesting(false);
+ }
+ };
+
const handleSave = async () => {
if (!name.trim()) {
toast.error('Rule name is required');
@@ -742,6 +787,68 @@ export function RuleEditor({ mode, ruleId }: RuleEditorProps) {
</div>
)}
</div>
+ {/* Wave 2 (W2.3) — backtest a saved rule over historical lake events */}
+ <div className="mt-3 rounded-lg border border-gray-800 bg-gray-900/40 p-3">
+ <div className="flex items-center justify-between gap-2">
+ <span className="text-xs font-medium text-gray-300">
+ Backtest over history
+ </span>
+ <div className="flex items-center gap-2">
+ <label className="text-[11px] text-gray-500">days</label>
+ <input
+ type="number"
+ min={1}
+ max={365}
+ value={backtestDays}
+ onChange={(e) =>
+ setBacktestDays(
+ Math.max(
+ 1,
+ Math.min(365, Number(e.target.value) || 30),
+ ),
+ )
+ }
+ className="w-16 rounded-md border border-gray-800 bg-gray-950 px-2 py-1 text-xs text-gray-200 outline-none focus:border-blue-500/60"
+ />
+ <button
+ type="button"
+ onClick={handleBacktest}
+ disabled={backtesting || !data?.id}
+ className="rounded-md bg-blue-600/80 px-2 py-1 text-xs font-medium text-white disabled:opacity-50"
+ >
+ {backtesting ? 'Running…' : 'Run backtest'}
+ </button>
+ </div>
+ </div>
+ <div className="mt-2">
+ {!data?.id ? (
+ <p className="text-[11px] text-gray-600">
+ Save the rule to backtest it against real historical
+ events.
+ </p>
+ ) : !backtestResult ? (
+ <p className="text-[11px] text-gray-600">
+ Run a backtest to see how many past events this rule
+ would have fired on.
+ </p>
+ ) : backtestResult.error ? (
+ <p className="text-[11px] text-red-400">
+ {backtestResult.error}
+ </p>
+ ) : (
+ <p className="text-[11px] text-gray-300">
+ Would fire on{' '}
+ <span className="font-semibold text-emerald-300">
+ {backtestResult.wouldFire}
+ </span>{' '}
+ of {backtestResult.eventsScanned} event
+ {backtestResult.eventsScanned === 1 ? '' : 's'} (
+ {(backtestResult.hitRate * 100).toFixed(1)}% hit-rate)
+ over the last {backtestDays}d.
+ </p>
+ )}
+ </div>
+ </div>
</div>
</div>
</div>
diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts
index 06129d3f2..fac9ed445 100644
--- a/apps/web/src/lib/api.ts
+++ b/apps/web/src/lib/api.ts
@@ -3509,6 +3509,17 @@ export interface DetectionConfidence {
generatedAt: string;
}
+export interface DetectionBacktestResult {
+ rule_id: string;
+ rule_language: string;
+ window_days: number;
+ events_scanned: number;
+ would_fire: number;
+ hit_rate: number;
+ sample_matches: Record<string, unknown>[];
+ error: string | null;
+}
+
export const detectionApi = {
list: () =>
request<{ rules: DetectionRule[]; total: number }>(
@@ -3517,6 +3528,17 @@ export const detectionApi = {
get: (id: string) => request<DetectionRule>(`/api/v1/detection/rules/${id}`),
+ // Wave 2 (W2.3) — backtest a saved rule over the last N days of real lake
+ // events. Returns exactly how many events would have fired + sample matches.
+ backtest: (
+ id: string,
+ body: { window_days?: number; limit?: number; source?: string | null },
+ ) =>
+ request<DetectionBacktestResult>(`/api/v1/rules/${id}/backtest`, {
+ method: 'POST',
+ body: JSON.stringify(body),
+ }),
+
create: (rule: Partial<DetectionRule>) =>
request<DetectionRule>('/api/v1/detection/rules', {
method: 'POST',
diff --git a/docs/audit/CLAIM_TO_GATE_MATRIX.md b/docs/audit/CLAIM_TO_GATE_MATRIX.md
index 61ffc1015..e6d2691a9 100644
--- a/docs/audit/CLAIM_TO_GATE_MATRIX.md
+++ b/docs/audit/CLAIM_TO_GATE_MATRIX.md
@@ -52,7 +52,7 @@ Statuses: `GATED` (a CI job fails when the claim stops being true) · `PARTIAL`
| Institutional memory nudges fuse-time confidence (bounded, per-tenant) | README (compounding memory) | `ci.yml` fusion job (`test_memory_nudge.py` — distilled per-signature priors nudge new alerts of that signature ±0.10 in `ConfidenceScorer`; unknown signatures unchanged; `MemoryPriorProvider` distils per-tenant priors from disposition history so one tenant never nudges another) | GATED | - |
| Scheduled hunt findings flow into governed detection proposals | README (proactive hunting → detections) | `ci.yml` api job (`test_wave1_loop_edges.py` — a scheduled hunt that returns hits opens a DRAFT `DetectionRuleProposal` (source `hunt-finding`) into the propose→review→eval-gate→promote lifecycle; zero-hit / flag-off paths open nothing) | GATED | - |
| Analyst disposition history drives human-approval-gated tuning proposals | README (self-improving detections) | `ci.yml` api job (`test_wave1_loop_edges.py` — the previously-orphaned `suggest_tuning` engine is wired to real per-rule disposition history via `/detection/tuning/auto-suggest`, opening DRAFT proposals; `test_detection_tuner.py` fixture-gates the engine) | PARTIAL (mapping + wiring gated; end-to-end proposal insert exercised in api integration, not a unit DB) | Phase 2 |
-| Detections can be backtested over historical lake data before promotion | README (detection engineering) | `ci.yml` api job (`test_backtest.py` — bounded/allow-listed SQL construction with injection-sanitised source filter, lake-row→event mapping with JSON payload expansion, exact would-fire count + hit-rate + samples) | PARTIAL (pure backtest logic gated; the live tenant-scoped ClickHouse fetch is exercised by the integration gate, not a unit) | Phase 2 |
+| Detections can be backtested over historical lake data before promotion | README (detection engineering) | `ci.yml` api job (`test_backtest.py` — bounded/allow-listed SQL construction with injection-sanitised source filter, lake-row→event mapping with JSON payload expansion, exact would-fire count + hit-rate + samples; `test_backtest_gate.py` — opt-in `AISOC_BACKTEST_MAX_HIT_RATE` /decide gate) + web type-check (rule-editor backtest panel) | PARTIAL (pure backtest logic + gate gated; the live tenant-scoped ClickHouse fetch is exercised by the integration gate, not a unit) | Phase 2 |
## Summary
diff --git a/docs/openapi.yaml b/docs/openapi.yaml
index 13fd2782a..b6e35ad7c 100644
--- a/docs/openapi.yaml
+++ b/docs/openapi.yaml
@@ -3303,6 +3303,55 @@ paths:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
+ /api/v1/detection-proposals/{proposal_id}/backtest:
+ post:
+ tags:
+ - detection_rules
+ - dac
+ summary: Backtest Proposal
+ description: 'Run the candidate rule over the last N days of REAL lake events
+ and
+
+ attach the result to ``eval_result["backtest"]`` (Wave 2, W2.2).
+
+
+ This is a third, honest signal alongside the fixture gate + MITRE baseline:
+
+ a rule that would have fired on a large fraction of historical events is
+
+ noisy. When ``AISOC_BACKTEST_MAX_HIT_RATE`` is set, ``/decide`` blocks
+
+ approval above that hit-rate (opt-in — informational by default).'
+ operationId: backtest_proposal_api_v1_detection_proposals__proposal_id__backtest_post
+ security:
+ - HTTPBearer: []
+ parameters:
+ - name: proposal_id
+ in: path
+ required: true
+ schema:
+ type: string
+ format: uuid
+ title: Proposal Id
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/BacktestProposalRequest'
+ responses:
+ '200':
+ description: Successful Response
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ProposalResponse'
+ '422':
+ description: Validation Error
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/HTTPValidationError'
/api/v1/detection-proposals/{proposal_id}/eval:
post:
tags:
@@ -15135,6 +15184,30 @@ components:
- tenant_id
- actions
title: AutonomyPolicyResponse
+ BacktestProposalRequest:
+ properties:
+ window_days:
+ type: integer
+ maximum: 365.0
+ minimum: 1.0
+ title: Window Days
+ default: 30
+ limit:
+ type: integer
+ maximum: 100000.0
+ minimum: 1.0
+ title: Limit
+ default: 5000
+ source:
+ anyOf:
+ - type: string
+ - type: 'null'
+ title: Source
+ description: Optional connector_type filter.
+ type: object
+ title: BacktestProposalRequest
+ description: Backtest a candidate proposal's rule over historical lake events
+ (W2.2).
BacktestRequest:
properties:
window_days:
diff --git a/services/api/app/api/v1/endpoints/detection_proposals.py b/services/api/app/api/v1/endpoints/detection_proposals.py
index 5c9226cec..44ccf7b31 100644
--- a/services/api/app/api/v1/endpoints/detection_proposals.py
+++ b/services/api/app/api/v1/endpoints/detection_proposals.py
@@ -53,12 +53,25 @@ from app.models.detection_proposal import (
DetectionRuleProposal,
)
from app.models.detection_rule import DetectionRule
+from app.services.backtest import backtest_rule, fetch_lake_events
from app.services.detection_eval import evaluate_candidate_rule
from app.services.github import create_detection_pr
router = APIRouter(prefix="/detection-proposals", tags=["detection_rules", "dac"])
+def _backtest_max_hit_rate() -> float | None:
+ """Optional approval gate: max backtest hit-rate before /decide blocks
+ approval. Unset (default) => informational only (W2.2)."""
+ raw = os.getenv("AISOC_BACKTEST_MAX_HIT_RATE", "").strip()
+ if not raw:
+ return None
+ try:
+ return float(raw)
+ except ValueError:
+ return None
+
+
# Resolve the AiSOC repository root by walking up from this file until we
# find the eval harness this module shells out to (``scripts/run_evals.py``).
# Hard-coding ``Path(__file__).parents[6]`` only works on the host, where
@@ -724,6 +737,63 @@ async def evaluate_rule(
return ProposalResponse.model_validate(proposal)
+class BacktestProposalRequest(BaseModel):
+ """Backtest a candidate proposal's rule over historical lake events (W2.2)."""
+
+ window_days: int = Field(30, ge=1, le=365)
+ limit: int = Field(5000, ge=1, le=100_000)
+ source: str | None = Field(None, description="Optional connector_type filter.")
+
+
[email protected]("/{proposal_id}/backtest", response_model=ProposalResponse)
+async def backtest_proposal(
+ proposal_id: uuid.UUID,
+ request: BacktestProposalRequest,
+ current_user: Annotated[AuthUser, Depends(require_permission("rules:write"))],
+ db: DBSession,
+) -> ProposalResponse:
+ """Run the candidate rule over the last N days of REAL lake events and
+ attach the result to ``eval_result["backtest"]`` (Wave 2, W2.2).
+
+ This is a third, honest signal alongside the fixture gate + MITRE baseline:
+ a rule that would have fired on a large fraction of historical events is
+ noisy. When ``AISOC_BACKTEST_MAX_HIT_RATE`` is set, ``/decide`` blocks
+ approval above that hit-rate (opt-in — informational by default)."""
+ _ensure_dac_enabled()
+ from app.db.clickhouse import LakeQueryNotConfiguredError # noqa: PLC0415
+
+ proposal = await _load_proposal(db, proposal_id, current_user.tenant_id)
+ if proposal.status in {"promoted", "rejected"}:
+ raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=f"Proposal is {proposal.status}; backtest cannot be re-run")
+
+ try:
+ events = await fetch_lake_events(
+ current_user.tenant_id,
+ window_days=request.window_days,
+ limit=request.limit,
+ source=request.source,
+ )
+ except LakeQueryNotConfiguredError as exc:
+ raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="lake backend not configured") from exc
+ except Exception as exc: # noqa: BLE001 — lake failure is a 502, not 500
+ raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"lake query failed: {exc}") from exc
+
+ summary = backtest_rule(rule_language=proposal.rule_language, rule_body=proposal.rule_body, events=events)
+ merged = dict(proposal.eval_result or {})
+ merged["backtest"] = {
+ "window_days": request.window_days,
+ "events_scanned": summary["events_scanned"],
+ "would_fire": summary["would_fire"],
+ "hit_rate": summary["hit_rate"],
+ "error": summary["error"],
+ "ran_at": datetime.now(UTC).isoformat(),
+ }
+ proposal.eval_result = merged
+ await db.commit()
+ await db.refresh(proposal)
+ return ProposalResponse.model_validate(proposal)
+
+
@router.post("/{proposal_id}/eval", response_model=ProposalResponse)
async def attach_eval_result(
proposal_id: uuid.UUID,
@@ -832,6 +902,20 @@ async def decide_proposal(
status_code=status.HTTP_412_PRECONDITION_FAILED,
detail="Substrate benchmark gate failed (MITRE regression); cannot approve.",
)
+ # W2.2 — optional backtest hit-rate gate (opt-in; informational by
+ # default). A rule that would fire on too large a fraction of real
+ # historical events is too noisy to promote.
+ gate = _backtest_max_hit_rate()
+ if gate is not None:
+ backtest = eval_result.get("backtest")
+ if isinstance(backtest, dict) and float(backtest.get("hit_rate", 0.0)) > gate:
+ raise HTTPException(
+ status_code=status.HTTP_412_PRECONDITION_FAILED,
+ detail=(
+ f"Backtest hit-rate {float(backtest['hit_rate']):.0%} exceeds the configured "
+ f"maximum {gate:.0%} — rule is too noisy over history; cannot approve."
+ ),
+ )
proposal.status = "approved"
else:
proposal.status = "rejected"
diff --git a/services/api/app/services/backtest.py b/services/api/app/services/backtest.py
index 31f360b5f..8bdeed674 100644
--- a/services/api/app/services/backtest.py
+++ b/services/api/app/services/backtest.py
@@ -15,6 +15,7 @@ from __future__ import annotations
import json
import re
+import uuid
from typing import Any
from app.services.rule_engine import evaluate_rule_over_events
@@ -66,6 +67,24 @@ def rows_to_events(columns: list[str], rows: list[list[Any]]) -> list[dict[str,
return events
+async def fetch_lake_events(
+ tenant_id: uuid.UUID,
+ *,
+ window_days: int,
+ limit: int,
+ source: str | None,
+) -> list[dict[str, Any]]:
+ """Fetch tenant-scoped historical events from the ClickHouse lake for a
+ backtest. Shared by the rule + proposal backtest endpoints."""
+ from app.db.clickhouse import execute_lake_query # noqa: PLC0415 — optional lake backend
+ from app.services.lake_sql import rewrite_for_tenant # noqa: PLC0415
+
+ sql = build_backtest_sql(window_days=window_days, limit=limit, source=source)
+ rewrite = rewrite_for_tenant(sql, tenant_id, row_cap=limit)
+ result = await execute_lake_query(rewrite.sql, timeout_seconds=30)
+ return rows_to_events(result.columns, result.rows)
+
+
def backtest_rule(
*,
rule_language: str,
diff --git a/services/api/tests/test_backtest_gate.py b/services/api/tests/test_backtest_gate.py
new file mode 100644
index 000000000..34cf0b408
--- /dev/null
+++ b/services/api/tests/test_backtest_gate.py
@@ -0,0 +1,20 @@
+"""Wave 2 (W2.2) - the opt-in backtest hit-rate approval gate."""
+
+from __future__ import annotations
+
+from app.api.v1.endpoints.detection_proposals import _backtest_max_hit_rate
+
+
+def test_gate_disabled_by_default(monkeypatch):
+ monkeypatch.delenv("AISOC_BACKTEST_MAX_HIT_RATE", raising=False)
+ assert _backtest_max_hit_rate() is None
+
+
+def test_gate_reads_threshold(monkeypatch):
+ monkeypatch.setenv("AISOC_BACKTEST_MAX_HIT_RATE", "0.5")
+ assert _backtest_max_hit_rate() == 0.5
+
+
+def test_gate_ignores_garbage(monkeypatch):
+ monkeypatch.setenv("AISOC_BACKTEST_MAX_HIT_RATE", "not-a-number")
+ assert _backtest_max_hit_rate() is None