Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
"""Tests for the CodeQL zero-alert gate.
`apps/docs/docs/operations/security.md` has claimed since v8.0 wave-1 that the
CodeQL alert count on `main` is zero and that this is enforced as a CI gate.
There was no such gate. `codeql.yml` uploads SARIF and
`github/codeql-action/analyze` does not fail a build on findings; `main` has no
branch protection, so "Code scanning results" was not a required check either;
and `security.yml`'s only hard job was the claim-to-gate matrix. Alerts #893
and #896 sat open on `main` for hours under a documented invariant of zero, and
nothing anywhere could have noticed.
So these tests assert two different things, and the second matters as much as
the first:
1. The gate catches every shape of failure it claims to catch — in particular a
`note`-severity alert, because both alerts that motivated it were `note` and
a severity filter would have reproduced the original silence exactly.
2. The gate is *reachable*. A gate wired into no workflow, or wired only into
`pull_request`, is the defect this repository keeps rediscovering: it
inspects what contributors propose and never what actually lands. The last
two tests read the workflow file and fail if that wiring disappears.
"""
from __future__ import annotations
import ast
import importlib.util
import subprocess
import sys
from datetime import UTC, datetime, timedelta
from pathlib import Path
import pytest
_REPO = Path(__file__).resolve().parents[1]
_SCRIPT = _REPO / "scripts" / "check_codeql_alerts.py"
_GATE_WORKFLOW = _REPO / ".github" / "workflows" / "codeql-alert-gate.yml"
_VALIDATE_PLAYBOOKS = _REPO / "scripts" / "validate_playbooks.py"
# Run in a subprocess, because the property under test is what happens at
# import time and a module imported once in this process would not show it.
_IMPORT_AND_REPORT_TRIGGERS = """
import sys
sys.path.insert(0, 'scripts')
import validate_playbooks
print(sorted(validate_playbooks.SUPPORTED_TRIGGERS))
"""
# Simulates the broken environment: `app.playbook.models` cannot be imported.
_IMPORT_WITH_MODELS_BLOCKED = """
import sys
class Blocker:
def find_spec(self, name, path=None, target=None):
if name.startswith('app.playbook'):
raise ImportError('blocked for the test')
return None
sys.meta_path.insert(0, Blocker())
sys.path.insert(0, 'scripts')
try:
import validate_playbooks
except Exception as exc:
print('CAUGHT', type(exc).__name__)
"""
def _load_gate():
spec = importlib.util.spec_from_file_location("check_codeql_alerts", _SCRIPT)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
gate = _load_gate()
def _clean(now: datetime) -> dict:
return gate._clean_fixture(_REPO, now)
def _run(data: dict, now: datetime, *, alerts_checked: bool = True) -> set[str]:
return {
code
for code, _ in gate.evaluate(
open_alerts=data["open_alerts"],
analyses=data["analyses"],
workflow=data["workflow"],
now=now,
max_age_days=gate.DEFAULT_MAX_ANALYSIS_AGE_DAYS,
alerts_checked=alerts_checked,
)
}
# --------------------------------------------------------------------------
# The gate bites
# --------------------------------------------------------------------------
def test_clean_ref_passes() -> None:
now = datetime.now(UTC)
assert _run(_clean(now), now) == set()
def test_note_severity_alert_is_caught() -> None:
"""The headline case: #893 and #896 were both `note`.
A gate that counted only `error`/`warning`, or only alerts carrying a
`security_severity_level` (both of these were `None`), would have printed
OK for the entire window in which the invariant was false.
"""
now = datetime.now(UTC)
data = _clean(now)
data["open_alerts"] = [gate._alert(896, "py/print-during-import", "note", "scripts/validate_playbooks.py")]
assert "open-alert" in _run(data, now)
@pytest.mark.parametrize("severity", ["note", "warning", "error"])
def test_every_severity_is_gated(severity: str) -> None:
now = datetime.now(UTC)
data = _clean(now)
data["open_alerts"] = [gate._alert(1, "py/some-rule", severity, "scripts/x.py")]
assert "open-alert" in _run(data, now)
def test_unanalyzed_ref_is_not_a_clean_ref() -> None:
"""Zero alerts because nothing ran is the vacuous pass, not a pass."""
now = datetime.now(UTC)
data = _clean(now)
data["analyses"] = []
assert "no-analysis" in _run(data, now)
def test_stale_analysis_is_caught() -> None:
"""`main` can be stale-green: a frozen scan leaves a frozen alert list."""
now = datetime.now(UTC)
data = _clean(now)
for analysis in data["analyses"]:
analysis["created_at"] = (now - timedelta(days=gate.DEFAULT_MAX_ANALYSIS_AGE_DAYS + 1)).isoformat().replace("+00:00", "Z")
assert "stale-analysis" in _run(data, now)
def test_language_dropped_from_analysis_is_caught() -> None:
now = datetime.now(UTC)
data = _clean(now)
data["analyses"] = data["analyses"][:-1]
assert "language-not-analyzed" in _run(data, now)
def test_mixed_codeql_action_pins_are_caught() -> None:
"""CodeQL refuses to run when init/autobuild/analyze disagree.
Dependabot splitting those three across separate PRs produces exactly this
state, and it removes the analysis rather than the findings.
"""
now = datetime.now(UTC)
data = _clean(now)
data["workflow"] = dict(data["workflow"])
data["workflow"]["action_versions"] = dict(data["workflow"]["action_versions"])
data["workflow"]["action_versions"]["analyze"] = "v3.28.0"
assert "action-version-mismatch" in _run(data, now)
def test_pr_only_scan_is_caught() -> None:
now = datetime.now(UTC)
data = _clean(now)
data["workflow"] = dict(data["workflow"])
data["workflow"]["push_branches"] = []
assert "no-push-to-main-trigger" in _run(data, now)
def test_offline_mode_claims_no_alert_verdict() -> None:
"""`--offline` must abstain, not quietly report a clean alert count."""
now = datetime.now(UTC)
data = _clean(now)
data["open_alerts"] = [gate._alert(1, "py/some-rule", "error", "scripts/x.py")]
assert "open-alert" not in _run(data, now, alerts_checked=False)
def test_analysis_of_a_different_commit_is_not_this_commit() -> None:
"""The push-race: the gate and the scan are triggered by the same push.
Unpinned, the gate reads the alert list left by the *previous* commit and
calls the incoming one clean before anything has looked at it.
"""
now = datetime.now(UTC)
data = _clean(now)
for analysis in data["analyses"]:
analysis["commit_sha"] = "a" * 40
codes = {
code
for code, _ in gate.evaluate(
open_alerts=[],
analyses=data["analyses"],
workflow=data["workflow"],
now=now,
max_age_days=gate.DEFAULT_MAX_ANALYSIS_AGE_DAYS,
alerts_checked=True,
commit="b" * 40,
)
}
assert "no-analysis" in codes
def test_wait_times_out_into_a_failure_not_a_pass(monkeypatch: pytest.MonkeyPatch) -> None:
"""A bounded wait that gives up must hand back an unanalyzed verdict."""
now = datetime.now(UTC)
languages = _clean(now)["workflow"]["languages"]
monkeypatch.setattr(
gate,
"fetch",
lambda repo, ref: {"open_alerts": [], "dismissed_alerts": [], "analyses": []},
)
slept: list[float] = []
data = gate.fetch_when_analyzed(
"example/repo",
"refs/heads/main",
commit="c" * 40,
languages=languages,
wait_seconds=2,
poll_seconds=1,
sleep=slept.append,
)
assert data["analyses"] == []
assert slept, "the gate returned without ever waiting"
def test_bundled_self_test_passes() -> None:
"""The same proof CI runs, run here too."""
assert gate.self_test(_REPO) == 0
# --------------------------------------------------------------------------
# The gate fails closed
# --------------------------------------------------------------------------
def test_missing_workflow_raises_rather_than_passing() -> None:
with pytest.raises(gate.GateError):
gate.parse_workflow(_REPO / ".github" / "workflows" / "does-not-exist.yml")
def test_unreadable_inputs_exit_two_not_zero(tmp_path: Path) -> None:
"""Exit 2 is "could not read", and must never be confused with "clean"."""
result = subprocess.run(
[sys.executable, str(_SCRIPT), "--repo-root", str(tmp_path), "--offline"],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 2, result.stdout + result.stderr
def test_missing_token_fails_closed(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("GITHUB_TOKEN", raising=False)
monkeypatch.delenv("GH_TOKEN", raising=False)
with pytest.raises(gate.GateError):
gate.fetch("beenuar/AiSOC", "refs/heads/main")
def test_gate_refuses_non_github_urls() -> None:
with pytest.raises(gate.GateError):
gate._get("https://example.invalid/alerts", "token")
# --------------------------------------------------------------------------
# The gate is reachable
# --------------------------------------------------------------------------
def test_gate_is_wired_into_a_workflow() -> None:
"""A gate no workflow invokes proves nothing, however good its self-test."""
text = _GATE_WORKFLOW.read_text(encoding="utf-8")
assert "scripts/check_codeql_alerts.py" in text
assert "--self-test" in text, "the self-test must run in CI, not only locally"
assert "tests/test_codeql_alert_gate.py" in text
def test_gate_runs_on_push_to_main_not_only_on_pull_requests() -> None:
"""A PR-only gate inspects what is proposed and never what landed."""
text = _GATE_WORKFLOW.read_text(encoding="utf-8")
on_block = gate._block(text, "on:")
assert on_block is not None
push_block = gate._block(on_block, "push:")
assert push_block is not None and "main" in push_block
assert gate._block(on_block, "pull_request:") is not None
def test_gate_pins_the_pushed_commit_rather_than_the_ref() -> None:
"""Otherwise the push run reads the analysis of the commit before it."""
text = _GATE_WORKFLOW.read_text(encoding="utf-8")
assert "--commit" in text and "--wait-seconds" in text
def test_codeql_workflow_pins_one_action_version() -> None:
"""CodeQL refuses to run when init/autobuild/analyze disagree.
Asserted against the real workflow, not a fixture: a dependabot bump that
lands for one of the three steps would otherwise silently remove the
analysis this gate depends on.
"""
workflow = gate.parse_workflow(_REPO / ".github" / "workflows" / "codeql.yml")
pins = set(workflow["action_versions"].values())
assert len(pins) == 1, f"mixed codeql-action pins: {workflow['action_versions']}"
# --------------------------------------------------------------------------
# The alert this wave fixed stays fixed
# --------------------------------------------------------------------------
def test_importing_validate_playbooks_is_silent_and_does_not_exit() -> None:
"""Regression for alert #896 (`py/print-during-import`).
The module printed to stderr and called `sys.exit(2)` while being
imported. Beyond the CodeQL note that is a live defect:
`scripts/check_playbook_schema_parity.py` imports this module to read
SUPPORTED_TRIGGERS, and `SystemExit` derives from `BaseException`, so its
`except Exception` could not catch it.
"""
result = subprocess.run(
[sys.executable, "-c", _IMPORT_AND_REPORT_TRIGGERS],
cwd=_REPO,
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, result.stderr
assert result.stderr == ""
assert result.stdout.strip() == "['alert', 'case', 'manual', 'schedule']"
def test_broken_model_import_is_catchable_by_the_parity_gate() -> None:
"""It must raise ImportError, which `except Exception` catches."""
result = subprocess.run(
[sys.executable, "-c", _IMPORT_WITH_MODELS_BLOCKED],
cwd=_REPO,
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, result.stderr
assert "CAUGHT ImportError" in result.stdout, result.stdout + result.stderr
def test_validate_playbooks_has_no_module_scope_print() -> None:
"""The property CodeQL's `py/print-during-import` checks, asserted locally."""
tree = ast.parse(_VALIDATE_PLAYBOOKS.read_text(encoding="utf-8"))
offenders: list[int] = []
for node in tree.body:
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
continue
if isinstance(node, ast.If) and ast.unparse(node.test) == "__name__ == '__main__'":
continue # the entrypoint guard does not run on import
for sub in ast.walk(node):
if isinstance(sub, ast.Call) and getattr(sub.func, "id", None) == "print":
offenders.append(sub.lineno)
assert offenders == [], f"print() at module scope on line(s) {offenders}"