Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
import clsx from 'clsx';
/**
* `legacy` exists because one suite is neither a measurement of this product
* nor a self-consistency gate: the alert-reduction suite groups on four tiers
* of `(rule_id, host, user)`, while the shipping `RawAlert.correlation_key()`
* groups on `{tenant}:{entity}:{tactic}`. It does not merely re-implement
* fusion's grouping, it implements *different* grouping, so its number
* describes an algorithm the product does not run. `apps/docs/docs/benchmark.md`
* withdrew the claim; this surface badged it "Real measurement" regardless.
*/
type Kind = 'measurement' | 'self-consistency' | 'legacy';
interface PerTemplateBlock {
metric: string;
value: number;
target: number;
passed: boolean;
template_count: number;
failing_templates: string[];
}
interface SuiteBlock {
metric: string;
value: number;
target: number;
passed: boolean;
duration_ms?: number;
details?: Record<string, unknown>;
per_template?: PerTemplateBlock;
}
export interface EvalReport {
generated_at: string;
dataset: string;
suites: Record<string, SuiteBlock>;
telemetry?: {
present?: boolean;
events?: number;
sources?: Record<string, number>;
incidents_with_telemetry?: number;
path?: string;
};
all_passed: boolean;
/** Source URL we actually fetched, for the "verify yourself" link. */
_source?: string;
/** Set when the live fetch failed and we fell back to the snapshot below. */
_stale?: boolean;
}
/**
* Snapshot fallback used when the live fetch from the `eval-results` branch
* fails (e.g. the branch hasn't been initialized yet, or the build runs
* offline). Numbers are hand-copied from a `scripts/run_evals.py` run on
* `main`, and `generated_at` below is that run's date, not this file's — the
* page renders it alongside a "Snapshot fallback" badge so a reader can see
* how old it is. The pass/fail gates in `services/agents/tests/test_*.py`
* are the source of truth — if they pass, the headline "≥ target" claim
* still holds even if these exact percentages lag a commit or two.
*
* `alert_reduction` here is the *legacy* four-tier suite, the only one
* `run_evals.py` emits under that key. It is rendered with a `legacy` badge
* rather than a measurement one; see the `Kind` docblock above.
*/
const SNAPSHOT_REPORT: EvalReport = {
generated_at: '2026-05-06T04:32:09.258436+00:00',
dataset: 'synthetic_incidents.json (200 cases, deterministic)',
all_passed: true,
suites: {
mitre_accuracy: {
metric: 'accuracy',
value: 0.97,
target: 0.8,
passed: true,
details: { incidents: 200, correct: 194, f1: 0.7712 },
per_template: {
metric: 'macro_accuracy',
value: 0.9636,
target: 0.8,
passed: true,
template_count: 55,
failing_templates: ['outlook-auto-forward-rule', 'compromised-ci-runner'],
},
},
alert_reduction: {
metric: 'reduction_ratio',
value: 0.753,
target: 0.7,
passed: true,
details: {
alerts_in: 1000,
incidents_out: 247,
storm_incidents: 16,
},
},
investigation_completeness: {
metric: 'mean_keyword_coverage',
value: 0.9425,
target: 0.85,
passed: true,
details: { incidents: 200, fully_covered: 134, fully_covered_pct: 0.67 },
per_template: {
metric: 'macro_completeness',
value: 0.9429,
target: 0.8,
passed: true,
template_count: 55,
failing_templates: [],
},
},
response_quality: {
metric: 'mean_rubric_score',
value: 1.0,
target: 0.8,
passed: true,
details: { incidents: 200 },
per_template: {
metric: 'macro_score',
value: 1.0,
target: 0.75,
passed: true,
template_count: 55,
failing_templates: [],
},
},
},
telemetry: {
present: true,
events: 361,
incidents_with_telemetry: 200,
path: 'services/agents/tests/eval_data/synthetic_telemetry.jsonl',
},
};
const LATEST_URL =
'https://raw.githubusercontent.com/beenuar/AiSOC/eval-results/eval/results/latest.json';
/**
* Server-side fetch of the most recent `eval_report.json` published by the
* `p1-eval` job in `.github/workflows/ci.yml`. The job writes
* `eval/results/<sha>.json` and `eval/results/latest.json` on the
* `eval-results` branch on every successful main-branch run.
*
* We use Next.js fetch revalidation so the page stays fresh between pushes
* without rebuilding the site. On failure we transparently fall back to the
* hand-copied `SNAPSHOT_REPORT` and flag the response as stale.
*/
export async function fetchLatestEvalReport(): Promise<EvalReport> {
try {
const res = await fetch(LATEST_URL, {
// Re-fetch every 5 minutes; CI publishes on every main-branch push.
next: { revalidate: 300 },
});
if (!res.ok) {
throw new Error(`status ${res.status}`);
}
const data = (await res.json()) as EvalReport;
return { ...data, _source: LATEST_URL };
} catch {
return { ...SNAPSHOT_REPORT, _source: LATEST_URL, _stale: true };
}
}
interface SuiteCard {
id: string;
name: string;
metricLabel: string;
blurb: string;
kind: Kind;
}
const SUITE_META: Record<string, SuiteCard> = {
alert_reduction: {
id: 'alert_reduction',
name: 'Alert reduction (legacy suite)',
metricLabel: 'Reduction ratio',
kind: 'legacy',
blurb:
'A four-tier scheme implemented inside the test, keyed on (rule_id, host, user) with 10/30/5-minute windows. RawAlert.correlation_key() — what Correlator actually calls — keys on {tenant}:{entity}:{tactic} over a one-hour window. Different dimensions, different windows, different answer. Retained for continuity; it does not describe this product. The number that does is measured against the real correlation key by services/fusion/tests/test_alert_reduction_real.py and published on the benchmark page.',
},
mitre_accuracy: {
id: 'mitre_accuracy',
name: 'MITRE tactic accuracy',
metricLabel: 'Tactic accuracy (per-case)',
kind: 'self-consistency',
blurb:
'Each synthetic incident is generated with a labeled tactic and a description written to include keywords the hand-curated extractor recognizes. The headline number mostly checks that dataset and extractor agree — useful as a regression sentinel for the extractor, not a measure of LLM agent accuracy.',
},
investigation_completeness: {
id: 'investigation_completeness',
name: 'Investigation completeness',
metricLabel: 'Mean keyword coverage',
kind: 'self-consistency',
blurb:
"The simulator wraps each incident's description in a Markdown report; the judge then looks for evidence keywords drawn from that same description. Close to a string-copy tautology — it confirms the report template includes the description and the judge can find keywords inside it. Catches template breakage, not LLM quality.",
},
response_quality: {
id: 'response_quality',
name: 'Response-plan quality',
metricLabel: 'Mean rubric score',
kind: 'self-consistency',
blurb:
'The synthesizer embeds the expected MITRE techniques and first evidence keyword directly into the templated plan, then a 5-criterion rubric checks for them. By construction the score is ~1.000. Catches a broken templating pipeline; it is not a grade of LLM-written plans.',
},
};
const KIND_LABEL: Record<Kind, { label: string; classes: string }> = {
measurement: {
label: 'Real measurement',
classes: 'border-emerald-500/30 bg-emerald-500/10 text-emerald-200',
},
'self-consistency': {
label: 'Substrate self-consistency',
classes: 'border-amber-500/30 bg-amber-500/10 text-amber-200',
},
legacy: {
label: 'Legacy — does not describe this product',
classes: 'border-rose-500/30 bg-rose-500/10 text-rose-200',
},
};
function formatPercent(value: number): string {
return `${(value * 100).toFixed(1)}%`;
}
function formatTarget(value: number): string {
return `≥${(value * 100).toFixed(0)}%`;
}
function suiteDetailRows(
id: string,
suite: SuiteBlock,
): { label: string; value: string }[] {
const d = suite.details ?? {};
const rows: { label: string; value: string }[] = [];
if (id === 'alert_reduction') {
if (typeof d.alerts_in === 'number') rows.push({ label: 'Alerts in', value: String(d.alerts_in) });
if (typeof d.incidents_out === 'number')
rows.push({ label: 'Incidents out', value: String(d.incidents_out) });
if (typeof d.storm_incidents === 'number')
rows.push({ label: 'Storms', value: String(d.storm_incidents) });
} else if (id === 'mitre_accuracy') {
if (typeof d.incidents === 'number') rows.push({ label: 'Incidents', value: String(d.incidents) });
if (typeof d.correct === 'number') rows.push({ label: 'Correct', value: String(d.correct) });
if (typeof d.f1 === 'number') rows.push({ label: 'F1 (per-case)', value: d.f1.toFixed(2) });
} else if (id === 'investigation_completeness') {
if (typeof d.incidents === 'number') rows.push({ label: 'Incidents', value: String(d.incidents) });
if (typeof d.fully_covered === 'number' && typeof d.fully_covered_pct === 'number') {
rows.push({
label: 'Fully covered',
value: `${d.fully_covered} (${(d.fully_covered_pct * 100).toFixed(0)}%)`,
});
}
rows.push({ label: 'Judge', value: 'Offline keyword' });
} else if (id === 'response_quality') {
if (typeof d.incidents === 'number') rows.push({ label: 'Incidents', value: String(d.incidents) });
rows.push({ label: 'Criteria', value: '5 (all hit by template)' });
rows.push({ label: 'Judge', value: 'Offline keyword' });
}
return rows;
}
interface BenchmarkResultsProps {
report: EvalReport;
}
export function BenchmarkResults({ report }: BenchmarkResultsProps) {
// The legacy suite sorts last: it led the grid while badged as a real
// measurement, which put a withdrawn figure in the first card a reader sees.
const orderedIds: string[] = [
'mitre_accuracy',
'investigation_completeness',
'response_quality',
'alert_reduction',
];
return (
<div className="grid gap-4 md:grid-cols-2">
{orderedIds.map((id) => {
const suite = report.suites[id];
const meta = SUITE_META[id];
if (!suite || !meta) return null;
const passed = suite.passed;
const headroom = ((suite.value - suite.target) * 100).toFixed(1);
const kind = KIND_LABEL[meta.kind];
const details = suiteDetailRows(id, suite);
const perTemplate = suite.per_template;
return (
<div
key={id}
className="group relative overflow-hidden rounded-xl border border-white/10 bg-white/[0.02] p-6 transition-colors hover:border-white/20"
>
<div className="flex items-start justify-between gap-4">
<div>
<p className="text-xs uppercase tracking-wider text-gray-500">
{meta.metricLabel}
</p>
<h3 className="mt-1 text-base font-semibold text-white">
{meta.name}
</h3>
</div>
<span
className={clsx(
'inline-flex items-center gap-1.5 rounded-full px-2.5 py-1 text-xs font-medium',
passed
? 'border border-emerald-500/20 bg-emerald-500/10 text-emerald-300'
: 'border border-rose-500/20 bg-rose-500/10 text-rose-300',
)}
>
<span
className={clsx(
'h-1.5 w-1.5 rounded-full',
passed ? 'bg-emerald-400' : 'bg-rose-400',
)}
/>
{passed ? 'Pass' : 'Fail'}
</span>
</div>
<div className="mt-5 flex items-baseline gap-2">
<span className="font-mono text-4xl font-semibold tabular-nums text-white">
{formatPercent(suite.value)}
</span>
<span className="text-xs text-gray-500">
target {formatTarget(suite.target)}
</span>
</div>
<div className="mt-1 text-xs text-gray-500">
{passed ? `+${headroom} pts above gate` : `${headroom} pts below gate`}
</div>
<span
className={clsx(
'mt-4 inline-flex items-center rounded-full border px-2 py-0.5 text-[10px] font-semibold uppercase tracking-wider',
kind.classes,
)}
>
{kind.label}
</span>
<p className="mt-3 text-sm leading-relaxed text-gray-400">
{meta.blurb}
</p>
{perTemplate && (
<div className="mt-4 rounded-lg border border-white/5 bg-black/20 p-3">
<div className="flex items-center justify-between gap-3">
<div>
<p className="text-[10px] font-semibold uppercase tracking-wider text-gray-500">
Per-template macro
</p>
<p className="mt-0.5 text-xs text-gray-400">
Equal-weight average across {perTemplate.template_count} distinct
incident templates — surfaces a single weak template that the
per-case mean would mask.
</p>
</div>
<div className="text-right">
<span className="font-mono text-base font-semibold tabular-nums text-white">
{formatPercent(perTemplate.value)}
</span>
<p className="mt-0.5 text-[10px] text-gray-500">
target {formatTarget(perTemplate.target)}
</p>
</div>
</div>
{perTemplate.failing_templates.length > 0 && (
<p className="mt-2 text-[11px] text-amber-300/80">
Regressions:{' '}
<span className="font-mono">
{perTemplate.failing_templates.join(', ')}
</span>
</p>
)}
</div>
)}
{details.length > 0 && (
<div className="mt-5 flex flex-wrap gap-x-6 gap-y-2 border-t border-white/5 pt-4 text-xs">
{details.map((d) => (
<div key={d.label}>
<dt className="text-gray-500">{d.label}</dt>
<dd className="mt-0.5 font-mono tabular-nums text-gray-200">
{d.value}
</dd>
</div>
))}
</div>
)}
</div>
);
})}
</div>
);
}