Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
/**
* Tests for the autonomy posture scorecard (Phase C3).
*
* Pins the copilot-default contract: the posture only flips to Autopilot when a
* high/critical-blast action really does reach a vendor unattended; otherwise it
* stays Copilot. Also checks the pure compute (distribution by blast radius,
* auto-exec + unimplemented counts) and the rendered summary.
*
* Issue #1243 rewrote what "auto-executes" means here. It used to be
* `thresholds.auto < 1` — a reading of the confidence threshold beside each
* row, which no dispatch path consults — and on shipped defaults that made
* nineteen of nineteen actions autonomous and put an Autopilot badge on a
* deployment that queues all of them. It is now `effective_auto_execute`, which
* the API resolves from the autonomy tier and the action's capability contract.
*
* The tests that mattered most are the two that encode the difference:
* `a reachable confidence threshold is not an auto-execution` fails on the
* pre-fix compute, and `an action with no verdict is not assumed autonomous`
* pins the fail-closed direction.
*/
import { describe, expect, it } from 'vitest';
import { render, screen } from '@testing-library/react';
import type {
AgreementResponse,
AgreementWindow,
AutonomyActionPolicy,
AutonomyBlastRadius,
AutonomyEffective,
AutonomyGrant,
} from '@/lib/api';
import {
AutonomyScorecard,
computeScorecard,
formatRate,
summariseTrackRecord,
} from './AutonomyScorecard';
function action(
name: string,
blast: AutonomyBlastRadius,
auto: number,
overridden = false,
runtime: Partial<AutonomyActionPolicy> = {},
): AutonomyActionPolicy {
return {
action: name,
blast_radius: blast,
thresholds: { auto, review: Math.max(0, auto - 0.2), escalation: Math.max(0, auto - 0.4) },
default_thresholds: { auto: 1, review: 0.8, escalation: 0.6 },
overridden,
// Default to the posture a stock install actually has: a registered verb
// that a human signs off on.
capability: name,
executable: true,
effective_auto_execute: false,
auto_executes_at_any_tier: false,
effective_tier: 'L1',
effective_reason: 'The action\u2019s contract requires analyst approval even at 100% confidence.',
...runtime,
};
}
const AUTO = { effective_auto_execute: true, auto_executes_at_any_tier: true } as const;
function effective(overrides: Partial<AutonomyEffective> = {}): AutonomyEffective {
return {
tier: 'L1',
tier_label: 'L1-Notify',
tier_source: 'environment',
max_automatic_impact: 'read_only',
auto_executing_actions: [],
high_blast_auto_executing: [],
unimplemented_actions: [],
thresholds_are_advisory: true,
advisory_note:
'These thresholds are advisory: nothing in the response path reads them yet.',
...overrides,
};
}
describe('computeScorecard', () => {
it('a reachable confidence threshold is not an auto-execution', () => {
// The defect, as a test. Every one of these has `auto < 1`, which the
// pre-fix compute read as autonomous — including the high-blast one, which
// is what flipped the badge to Autopilot on a stock install.
const card = computeScorecard([
action('lookup_ip', 'read', 0.0),
action('isolate_host', 'high', 0.92),
action('block_ip', 'high', 0.9),
]);
expect(card.posture).toBe('copilot');
expect(card.autoExecuting).toBe(0);
expect(card.highBlastAuto).toBe(0);
});
it('defaults to copilot when no high-blast action auto-executes', () => {
const card = computeScorecard([
action('search_siem', 'read', 0.5, false, AUTO), // a read running itself is fine
action('isolate_host', 'high', 1.0),
action('block_ip', 'medium', 0.9, false, AUTO),
]);
expect(card.posture).toBe('copilot');
expect(card.total).toBe(3);
expect(card.autoExecuting).toBe(2); // search_siem + block_ip
expect(card.highBlastAuto).toBe(0);
});
it('flips to autopilot when a high-blast action auto-executes', () => {
const card = computeScorecard([action('isolate_host', 'high', 0.85, false, AUTO)]);
expect(card.posture).toBe('autopilot');
expect(card.highBlastAuto).toBe(1);
});
it('an action with no verdict is not assumed autonomous', () => {
// An older API, or a field that failed to serialise. Announcing autonomy
// that is not there is the failure this card exists to stop, so an absent
// answer counts as "a human approves".
const card = computeScorecard([
{
...action('isolate_host', 'high', 0.85),
effective_auto_execute: undefined,
auto_executes_at_any_tier: undefined,
},
]);
expect(card.posture).toBe('copilot');
expect(card.autoExecuting).toBe(0);
});
it('counts a verb with no executor apart from one a human approves', () => {
const card = computeScorecard([
action('delete_object', 'critical', 0.95, false, { capability: null, executable: false }),
action('isolate_host', 'high', 0.92),
]);
expect(card.unimplemented).toBe(1);
expect(card.autoExecuting).toBe(0);
expect(card.highBlastTotal).toBe(2);
});
it('counts overrides and distribution by blast radius', () => {
const card = computeScorecard([
action('a', 'read', 0.5, true),
action('b', 'read', 0.5),
action('c', 'critical', 1.0),
]);
expect(card.overridden).toBe(1);
expect(card.byBlast.read).toBe(2);
expect(card.byBlast.critical).toBe(1);
});
it('handles an empty policy', () => {
const card = computeScorecard([]);
expect(card.total).toBe(0);
expect(card.posture).toBe('copilot');
});
});
describe('AutonomyScorecard', () => {
it('renders the Copilot badge by default', () => {
render(<AutonomyScorecard actions={[action('isolate_host', 'high', 0.92)]} />);
expect(screen.getByText('Copilot')).toBeInTheDocument();
expect(screen.getByText(/always require a human/i)).toBeInTheDocument();
});
it('claims only what the current tier supports when a contract could ever allow it', () => {
// `auto_executes_at_any_tier` is true, so "no tier executes these
// unattended" would be false. The weaker sentence is the true one.
render(
<AutonomyScorecard
actions={[action('isolate_host', 'high', 0.92, false, { auto_executes_at_any_tier: true })]}
/>,
);
expect(screen.getByText('Copilot')).toBeInTheDocument();
expect(screen.getByText(/at the current autonomy tier/i)).toBeInTheDocument();
expect(screen.queryByText(/no autonomy tier/i)).not.toBeInTheDocument();
});
it('renders the Autopilot badge with a warning when high-blast auto-executes', () => {
render(<AutonomyScorecard actions={[action('isolate_host', 'high', 0.8, false, AUTO)]} />);
expect(screen.getByText('Autopilot')).toBeInTheDocument();
expect(screen.getByText(/auto-execute/i)).toBeInTheDocument();
});
it('names the tier the posture was derived from, and where it came from', () => {
// A badge with no stated basis is how a posture read off the wrong table
// went unchallenged for as long as it did.
render(
<AutonomyScorecard actions={[action('isolate_host', 'high', 0.92)]} effective={effective()} />,
);
expect(screen.getByText('L1-Notify')).toBeInTheDocument();
expect(screen.getByText(/deployment default/i)).toBeInTheDocument();
expect(screen.getByText(/up to read only impact/i)).toBeInTheDocument();
});
it('says the thresholds are advisory rather than leaving the sliders to imply otherwise', () => {
render(
<AutonomyScorecard actions={[action('isolate_host', 'high', 0.92)]} effective={effective()} />,
);
expect(screen.getByText(/nothing in the response path reads them/i)).toBeInTheDocument();
});
it('names the verbs with no executor instead of showing them as gated actions', () => {
render(
<AutonomyScorecard
actions={[
action('delete_object', 'critical', 0.95, false, { capability: null, executable: false }),
]}
effective={effective({ unimplemented_actions: ['delete_object'] })}
/>,
);
expect(screen.getByText(/No executor is registered for/i)).toBeInTheDocument();
expect(screen.getByText('delete_object')).toBeInTheDocument();
});
});
// ─── Measured track record (gap-closure Phase 2.2) ───────────────────────────
//
// The posture above is what the tenant configured. This half is what the agent
// earned. The tests worth having are about the two ways this display can lie:
// printing a zero where nothing was measured, and printing a rate without the
// count behind it.
function agreementResponse(overrides: Partial<AgreementWindow> = {}): AgreementResponse {
const window: AgreementWindow = {
resolved: 120,
labelled: 100,
unlabeled: 20,
answered: 90,
abstained: 10,
agreed: 87,
malicious_support: 31,
malicious_caught: 29,
agreement_rate: 87 / 90,
malicious_recall: 29 / 31,
abstention_rate: 10 / 100,
...overrides,
};
return {
tenant_id: 't',
scope_kind: 'tenant',
scope_key: '*',
window,
recent: { ...window, answered: 50, agreed: 40, agreement_rate: 0.8 },
window_start: '2026-08-27T00:00:00+00:00',
window_end: '2026-09-26T00:00:00+00:00',
thresholds: {
min_decisions: 100,
min_malicious: 30,
min_agreement: 0.95,
min_malicious_recall: 0.9,
max_abstention_rate: 0.3,
window_days: 30,
demotion_agreement: 0.9,
demotion_malicious_recall: 0.8,
drift_sample: 50,
drift_min_answered: 20,
},
reconciled: 0,
by_alert_class: [],
by_rule: [],
by_source: [],
by_model: [],
};
}
describe('formatRate', () => {
it('renders a missing denominator as words, never as zero', () => {
// A zero here says the agent was wrong every time. "It was never asked" is
// a different fact, and on a new deployment it is always the true one.
expect(formatRate(null)).toBe('not measured');
expect(formatRate(undefined)).toBe('not measured');
});
it('renders a real zero as a zero', () => {
expect(formatRate(0)).toBe('0.0%');
});
});
describe('summariseTrackRecord', () => {
it('returns nothing when no agreement has been fetched', () => {
expect(summariseTrackRecord(undefined)).toBeNull();
});
it('marks a tenant with no closed decisions as unmeasured', () => {
const summary = summariseTrackRecord(
agreementResponse({ resolved: 0, labelled: 0, answered: 0, abstained: 0, agreed: 0 }),
);
expect(summary?.measured).toBe(false);
});
it('carries the counts each rate was computed over', () => {
const summary = summariseTrackRecord(agreementResponse());
expect(summary?.answered).toBe(90);
expect(summary?.maliciousSupport).toBe(31);
expect(summary?.labelled).toBe(100);
});
it('carries the trailing slice separately from the window', () => {
// The window average is where a gradual decline hides. Folding the two
// into one number would make this card conceal the thing it exists to show.
const summary = summariseTrackRecord(agreementResponse());
expect(summary?.agreement).toBeCloseTo(87 / 90);
expect(summary?.recentAgreement).toBeCloseTo(0.8);
});
});
describe('<AutonomyScorecard /> track record', () => {
it('says there is no measurement rather than rendering zeroes', () => {
render(<AutonomyScorecard actions={[action('block_ip', 'medium', 0.9)]} />);
expect(screen.getByText(/No measured track record yet/i)).toBeInTheDocument();
});
it('shows the sample as a fraction of the floor, not as a percentage', () => {
// On a card where every other figure is a percentage, "47%" would be read
// as a fifth accuracy number rather than as progress toward a threshold.
render(
<AutonomyScorecard
actions={[action('block_ip', 'medium', 0.9)]}
agreement={agreementResponse()}
/>,
);
expect(screen.getByText('100 / 100')).toBeInTheDocument();
expect(screen.getByText('31 / 30 malicious')).toBeInTheDocument();
});
it('prints not measured for a rate with no denominator', () => {
render(
<AutonomyScorecard
actions={[action('block_ip', 'medium', 0.9)]}
agreement={agreementResponse({ malicious_support: 0, malicious_caught: 0, malicious_recall: null })}
/>,
);
expect(screen.getByText('not measured')).toBeInTheDocument();
});
});
// ─── Earned autonomy (gap-closure Phase 2.3) ─────────────────────────────────
//
// One property, and it is the reason the grant carries a `source` at all: an
// override must stay legible as an override. Autonomy somebody earned and
// autonomy somebody overruled a refusal to grant behave identically and are
// very different things to be reading during an incident review.
function grant(overrides: Partial<AutonomyGrant> = {}): AutonomyGrant {
return {
id: 'g1',
scope_kind: 'alert_class',
scope_key: 'identity',
capability: 'auto_close',
state: 'granted',
source: 'earned',
is_override: false,
...overrides,
};
}
describe('<AutonomyScorecard /> earned autonomy', () => {
it('says nothing when the tenant holds no capabilities', () => {
render(<AutonomyScorecard actions={[action('block_ip', 'medium', 0.9)]} grants={[]} />);
expect(screen.queryByText(/Autonomy granted/i)).not.toBeInTheDocument();
});
it('labels an earned grant as earned', () => {
render(<AutonomyScorecard actions={[action('block_ip', 'medium', 0.9)]} grants={[grant()]} />);
expect(screen.getByText('earned')).toBeInTheDocument();
expect(screen.queryByText('operator override')).not.toBeInTheDocument();
});
it('labels an override as an override and shows the stated reason', () => {
render(
<AutonomyScorecard
actions={[action('block_ip', 'medium', 0.9)]}
grants={[
grant({
source: 'operator_override',
is_override: true,
override_reason: 'Accepted for a two-week pilot',
}),
]}
/>,
);
expect(screen.getByText('operator override')).toBeInTheDocument();
expect(screen.getByText('Accepted for a two-week pilot')).toBeInTheDocument();
});
it('keeps a demoted grant visible with why it was taken away', () => {
// A capability that was revoked is more interesting than one never held,
// and hiding it would make "what happened to our auto-close" unanswerable
// from this page.
render(
<AutonomyScorecard
actions={[action('block_ip', 'medium', 0.9)]}
grants={[grant({ state: 'demoted', demoted_reason: 'recent_drift' })]}
/>,
);
expect(screen.getByText('demoted')).toBeInTheDocument();
expect(screen.getByText('recent_drift')).toBeInTheDocument();
});
});