Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
feat: add Redis-backed scheduler singleton + dependency CVE audit pipeline (#218)
ARDA7787badcdf628944
diff --git a/.github/workflows/security-audit.yml b/.github/workflows/security-audit.ymlnew file mode 100644index 000000000..582246856--- /dev/null+++ b/.github/workflows/security-audit.yml@@ -0,0 +1,77 @@+name: Security Audit++# Scoped ignores: scripts/security_audit_ignores.txt+# Local run: python scripts/security_audit.py <pnpm|python|go|validate-ignores>++on:+ push:+ branches: [main, develop]+ pull_request:+ branches: [main, develop]+ workflow_dispatch:++jobs:+ security-audit:+ name: security-audit+ runs-on: ubuntu-latest+ permissions:+ contents: read+ env:+ PNPM_AUDIT_REGISTRY: https://registry.npmjs.org/++ steps:+ - uses: actions/checkout@v4++ - uses: pnpm/action-setup@v4++ - uses: actions/setup-node@v4+ with:+ node-version: '22'+ cache: pnpm+ cache-dependency-path: pnpm-lock.yaml++ - uses: actions/setup-python@v5+ with:+ python-version: '3.12'++ - uses: actions/setup-go@v5+ with:+ go-version: '1.23'++ - name: Install audit tools+ run: |+ set -euo pipefail+ python -m pip install --upgrade pip+ python -m pip install pip-audit poetry poetry-plugin-export+ go install golang.org/x/vuln/cmd/govulncheck@latest+ echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"++ - name: Validate ignore policy+ run: python scripts/security_audit.py validate-ignores++ - name: Install pnpm dependencies+ run: pnpm install --frozen-lockfile --ignore-scripts++ - name: Audit pnpm dependencies+ id: audit-pnpm+ continue-on-error: true+ run: python scripts/security_audit.py pnpm++ - name: Audit Python dependencies+ id: audit-python+ continue-on-error: true+ run: python scripts/security_audit.py python++ - name: Audit Go dependencies+ id: audit-go+ continue-on-error: true+ run: python scripts/security_audit.py go++ - name: Enforce staged policy+ if: >-+ steps.audit-pnpm.outcome == 'failure' ||+ steps.audit-python.outcome == 'failure' ||+ steps.audit-go.outcome == 'failure'+ run: |+ echo "::error::High/critical dependency vulnerabilities found — see annotations above"+ exit 1diff --git a/scripts/security_audit.py b/scripts/security_audit.pynew file mode 100644index 000000000..a7761496c--- /dev/null+++ b/scripts/security_audit.py@@ -0,0 +1,624 @@+"""Dependency CVE scanner for AiSOC CI.++Subcommands:+ validate-ignores — check the ignores file is well-formed+ pnpm — run pnpm audit (staged: high/critical fail, moderate warns)+ python — run pip-audit per Python service (any finding fails)+ go — run govulncheck per Go module (any finding fails)++Usage (local):+ python scripts/security_audit.py validate-ignores+ python scripts/security_audit.py pnpm+ python scripts/security_audit.py python+ python scripts/security_audit.py go++Environment variables (CI):+ GITHUB_STEP_SUMMARY — path to markdown summary file (set by Actions)+ GITHUB_WORKSPACE — repo root (set by Actions; defaults to cwd)+ PNPM_AUDIT_REGISTRY — fallback registry if primary returns 403+"""++from __future__ import annotations++import argparse+import datetime as dt+import json+import os+import re+import shutil+import subprocess+import sys+import tempfile+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any++# ─── Data types ───────────────────────────────────────────────────────────────++SEVERITY_ORDER = {"info": 0, "low": 1, "moderate": 2, "high": 3, "critical": 4}++IGNORE_ID_PATTERN = re.compile(+ r"^(CVE-\d{4}-\d{4,}|GHSA-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}|GO-\d{4}-\d{4,}|PYSEC-\d{4}-\d+)$"+)+++@dataclass+class Ignore:+ tool: str+ vuln_id: str+ reason: str+ expires: dt.date+++@dataclass+class Finding:+ tool: str+ severity: str+ vuln_id: str+ package: str+ location: str+ title: str+++@dataclass+class Report:+ findings: list[Finding] = field(default_factory=list)+ ignored: list[Finding] = field(default_factory=list)+ warnings: list[str] = field(default_factory=list)++ @property+ def high_critical(self) -> list[Finding]:+ return [f for f in self.findings if f.severity in ("high", "critical")]++ @property+ def moderate(self) -> list[Finding]:+ return [f for f in self.findings if f.severity == "moderate"]++ @property+ def low_info(self) -> list[Finding]:+ return [f for f in self.findings if f.severity in ("low", "info", "unknown")]+++# ─── Ignore loading ──────────────────────────────────────────────────────────+++def load_ignores(path: Path, today: dt.date | None = None) -> list[Ignore]:+ """Load and validate the ignores file.++ Format per line: tool|ID|reason|YYYY-MM-DD+ Lines starting with # and blank lines are skipped.+ """+ if today is None:+ today = dt.date.today()++ max_expiry = today + dt.timedelta(days=90)+ allowed_tools = {"pnpm", "python", "go"}+ ignores: list[Ignore] = []++ if not path.exists():+ return ignores++ for line_no, raw_line in enumerate(path.read_text().splitlines(), start=1):+ line = raw_line.strip()+ if not line or line.startswith("#"):+ continue++ parts = [p.strip() for p in line.split("|")]+ if len(parts) != 4:+ raise ValueError(+ f"Line {line_no}: expected 4 pipe-delimited fields, got {len(parts)}"+ )++ tool, vuln_id, reason, expiry_str = parts++ if tool not in allowed_tools:+ raise ValueError(+ f"Line {line_no}: invalid tool '{tool}' (expected: {sorted(allowed_tools)})"+ )++ if not IGNORE_ID_PATTERN.match(vuln_id):+ raise ValueError(+ f"Line {line_no}: invalid ID '{vuln_id}' "+ "(expected CVE-YYYY-NNNNN, GHSA-xxxx-xxxx-xxxx, GO-YYYY-NNNN, or PYSEC-YYYY-N)"+ )++ if not reason:+ raise ValueError(f"Line {line_no}: reason must not be empty")++ try:+ expiry = dt.date.fromisoformat(expiry_str)+ except ValueError:+ raise ValueError(f"Line {line_no}: invalid date '{expiry_str}' (expected YYYY-MM-DD)")++ if expiry < today:+ raise ValueError(f"Line {line_no}: ignore for {vuln_id} expired on {expiry_str}")++ if expiry > max_expiry:+ raise ValueError(+ f"Line {line_no}: expiry {expiry_str} exceeds 90-day maximum from today ({max_expiry})"+ )++ ignores.append(Ignore(tool=tool, vuln_id=vuln_id, reason=reason, expires=expiry))++ return ignores+++def is_ignored(tool: str, vuln_ids: list[str], ignores: list[Ignore]) -> Ignore | None:+ """Return the matching Ignore if any of the vuln_ids are ignored for this tool."""+ tool_ignores = {ig.vuln_id for ig in ignores if ig.tool == tool}+ for vid in vuln_ids:+ if vid in tool_ignores:+ return next(ig for ig in ignores if ig.tool == tool and ig.vuln_id == vid)+ return None+++# ─── pnpm scanner ────────────────────────────────────────────────────────────+++def classify_pnpm_audit(audit_json: dict[str, Any], ignores: list[Ignore]) -> Report:+ """Classify pnpm audit JSON into a Report using native severity."""+ report = Report()++ for advisory in audit_json.get("advisories", {}).values():+ ghsa = advisory.get("github_advisory_id", "")+ cves = advisory.get("cves", []) or []+ all_ids = [i for i in [ghsa, *cves] if i]+ severity = (advisory.get("severity") or "unknown").lower()+ if severity not in SEVERITY_ORDER:+ severity = "unknown"++ finding = Finding(+ tool="pnpm",+ severity=severity,+ vuln_id=all_ids[0] if all_ids else str(advisory.get("id", "unknown")),+ package=advisory.get("module_name", "unknown"),+ location="pnpm workspace",+ title=advisory.get("title") or advisory.get("overview") or "",+ )++ if is_ignored("pnpm", all_ids, ignores):+ report.ignored.append(finding)+ else:+ report.findings.append(finding)++ return report+++def run_pnpm_audit(repo_root: Path, ignores: list[Ignore]) -> Report:+ """Run pnpm audit with fallback registry on 403."""+ fallback = os.environ.get("PNPM_AUDIT_REGISTRY", "https://registry.npmjs.org/")++ for attempt, registry in enumerate([None, fallback]):+ env = os.environ.copy()+ if registry:+ env["npm_config_registry"] = registry++ proc = subprocess.run(+ ["pnpm", "audit", "--json"],+ cwd=repo_root,+ env=env,+ capture_output=True,+ text=True,+ )++ output = proc.stdout + proc.stderr+ if ("403" in output or "Forbidden" in output) and attempt == 0:+ print(f"Primary registry returned 403; retrying with {fallback}")+ continue++ try:+ data = json.loads(proc.stdout)+ except json.JSONDecodeError:+ print(f"Failed to parse pnpm audit output:\n{output}", file=sys.stderr)+ report = Report()+ report.warnings.append("pnpm audit returned unparseable output")+ return report++ return classify_pnpm_audit(data, ignores)++ report = Report()+ report.warnings.append("pnpm audit failed on all registries")+ return report+++# ─── Python scanner ──────────────────────────────────────────────────────────+++def parse_pip_audit_json(raw: str) -> list[dict[str, Any]]:+ """Parse pip-audit --format=json output."""+ if not raw.strip():+ return []+ data = json.loads(raw)+ if isinstance(data, dict):+ return data.get("dependencies", [])+ if isinstance(data, list):+ return data+ return []+++def classify_pip_audit(+ target: str, dependencies: list[dict[str, Any]], ignores: list[Ignore]+) -> Report:+ """Classify pip-audit findings. All findings are treated as high (fail-closed)."""+ report = Report()++ for dep in dependencies:+ for vuln in dep.get("vulns", []):+ aliases = vuln.get("aliases", []) or []+ vuln_id = vuln.get("id", "")+ all_ids = [i for i in [vuln_id, *aliases] if i]++ finding = Finding(+ tool="python",+ severity="high",+ vuln_id=vuln_id or (aliases[0] if aliases else dep.get("name", "unknown")),+ package=f"{dep.get('name', '?')}=={dep.get('version', '?')}",+ location=target,+ title=vuln.get("description", ""),+ )++ if is_ignored("python", all_ids, ignores):+ report.ignored.append(finding)+ else:+ report.findings.append(finding)++ return report+++def _export_requirements(service_dir: Path, work_dir: Path) -> Path | None:+ """Export a Poetry project to requirements.txt. Returns path or None on failure."""+ pyproject = service_dir / "pyproject.toml"+ shutil.copy2(pyproject, work_dir / "pyproject.toml")++ lock = service_dir / "poetry.lock"+ if lock.exists():+ shutil.copy2(lock, work_dir / "poetry.lock")++ # Poetry needs a README to avoid warnings+ readme = service_dir / "README.md"+ if readme.exists():+ shutil.copy2(readme, work_dir / "README.md")++ try:+ # Generate lock if missing+ if not (work_dir / "poetry.lock").exists():+ proc = subprocess.run(+ ["poetry", "lock", "--no-interaction"],+ cwd=work_dir,+ capture_output=True,+ text=True,+ )+ if proc.returncode != 0:+ return None++ # Export to requirements.txt+ req_path = work_dir / "requirements.txt"+ proc = subprocess.run(+ [+ "poetry", "export",+ "--format", "requirements.txt",+ "--without-hashes",+ "--output", str(req_path),+ ],+ cwd=work_dir,+ capture_output=True,+ text=True,+ )+ if proc.returncode != 0:+ return None+ except FileNotFoundError:+ return None++ return req_path+++def run_pip_audit(repo_root: Path, ignores: list[Ignore]) -> Report:+ """Run pip-audit on all Python services with pyproject.toml.++ Steps per service:+ 1. poetry export → requirements.txt (in a temp dir)+ 2. pip-audit -r requirements.txt --format json+ """+ combined = Report()++ with tempfile.TemporaryDirectory(prefix="security_audit_py_") as tmp:+ tmp_root = Path(tmp)++ for search_dir in ["services", "packages"]:+ parent = repo_root / search_dir+ if not parent.exists():+ continue+ for child in sorted(parent.iterdir()):+ if not child.is_dir():+ continue+ pyproject = child / "pyproject.toml"+ if not pyproject.exists():+ continue++ target = str(child.relative_to(repo_root))+ safe_name = target.replace("/", "__")+ work_dir = tmp_root / safe_name+ work_dir.mkdir(parents=True, exist_ok=True)++ req_path = _export_requirements(child, work_dir)+ if req_path is None:+ combined.warnings.append(+ f"poetry export failed for {target}; skipping pip-audit"+ )+ continue++ try:+ cmd = [+ "pip-audit",+ "-r", str(req_path),+ "--format", "json",+ "--progress-spinner", "off",+ ]++ proc = subprocess.run(cmd, capture_output=True, text=True, cwd=repo_root)+ except FileNotFoundError:+ combined.warnings.append("pip-audit not found on PATH; skipping Python audit")+ break++ if proc.returncode not in (0, 1):+ # pip-audit returns 1 when vulns found, other codes are errors+ combined.warnings.append(+ f"pip-audit on {target} exited {proc.returncode}: {proc.stderr[:200]}"+ )+ continue++ deps = parse_pip_audit_json(proc.stdout)+ sub = classify_pip_audit(target, deps, ignores)+ combined.findings.extend(sub.findings)+ combined.ignored.extend(sub.ignored)++ return combined+++# ─── Go scanner ──────────────────────────────────────────────────────────────+++def parse_govulncheck_json(stdout: str) -> list[dict[str, Any]]:+ """Parse govulncheck -json output (newline-delimited JSON objects)."""+ osv_by_id: dict[str, dict] = {}+ finding_ids: set[str] = set()++ for line in stdout.splitlines():+ line = line.strip()+ if not line:+ continue+ try:+ entry = json.loads(line)+ except json.JSONDecodeError:+ continue++ if "osv" in entry:+ osv = entry["osv"]+ osv_by_id[osv["id"]] = osv+ if "finding" in entry:+ finding_ids.add(entry["finding"]["osv"])++ results = []+ for osv_id in sorted(finding_ids):+ osv = osv_by_id.get(osv_id, {})+ results.append({+ "id": osv_id,+ "aliases": osv.get("aliases", []),+ "summary": osv.get("summary", ""),+ })++ return results+++def classify_govulncheck(+ module: str, vulns: list[dict[str, Any]], ignores: list[Ignore]+) -> Report:+ """Classify govulncheck findings. All findings are treated as high (fail-closed)."""+ report = Report()++ for vuln in vulns:+ vuln_id = vuln.get("id", "")+ aliases = vuln.get("aliases", []) or []+ all_ids = [i for i in [vuln_id, *aliases] if i]++ finding = Finding(+ tool="go",+ severity="high",+ vuln_id=vuln_id,+ package=module,+ location=module,+ title=vuln.get("summary", ""),+ )++ if is_ignored("go", all_ids, ignores):+ report.ignored.append(finding)+ else:+ report.findings.append(finding)++ return report+++def run_govulncheck(repo_root: Path, ignores: list[Ignore]) -> Report:+ """Run govulncheck on all Go modules."""+ combined = Report()++ go_mods: list[Path] = []+ for search_dir in ["services", "packages"]:+ parent = repo_root / search_dir+ if parent.exists():+ go_mods.extend(sorted(parent.rglob("go.mod")))++ for go_mod in go_mods:+ module_dir = go_mod.parent+ module = str(module_dir.relative_to(repo_root))++ try:+ proc = subprocess.run(+ ["govulncheck", "-json", "./..."],+ cwd=module_dir,+ capture_output=True,+ text=True,+ )+ except FileNotFoundError:+ combined.warnings.append("govulncheck not found on PATH; skipping Go audit")+ break++ if proc.returncode not in (0, 3):+ combined.warnings.append(+ f"govulncheck on {module} exited {proc.returncode}: {proc.stderr[:200]}"+ )+ continue++ vulns = parse_govulncheck_json(proc.stdout)+ sub = classify_govulncheck(module, vulns, ignores)+ combined.findings.extend(sub.findings)+ combined.ignored.extend(sub.ignored)++ return combined+++# ─── Output ──────────────────────────────────────────────────────────────────+++def emit_annotations(report: Report) -> None:+ """Emit GitHub Actions annotations."""+ for f in report.high_critical:+ print(f"::error title=security-audit::{f.tool} {f.severity}: {f.vuln_id} in {f.package}")+ for f in report.moderate:+ print(f"::warning title=security-audit::{f.tool} {f.severity}: {f.vuln_id} in {f.package}")+ for f in report.low_info:+ print(f"::notice title=security-audit::{f.tool} {f.severity}: {f.vuln_id} in {f.package}")+ for w in report.warnings:+ print(f"::warning title=security-audit::{w}")+++def emit_summary(report: Report) -> None:+ """Write GitHub step summary markdown."""+ summary_path = os.environ.get("GITHUB_STEP_SUMMARY")+ if not summary_path:+ return++ lines = [+ "## Dependency Security Audit\n",+ f"- **High/Critical**: {len(report.high_critical)}",+ f"- **Moderate**: {len(report.moderate)}",+ f"- **Low/Info**: {len(report.low_info)}",+ f"- **Ignored**: {len(report.ignored)}",+ "",+ ]++ if report.findings:+ lines.append("| Tool | Severity | ID | Package | Location |")+ lines.append("|---|---|---|---|---|")+ for f in sorted(report.findings, key=lambda x: (-SEVERITY_ORDER.get(x.severity, 0), x.tool)):+ lines.append(f"| {f.tool} | {f.severity} | `{f.vuln_id}` | `{f.package}` | {f.location} |")+ lines.append("")++ if report.ignored:+ lines.append("### Suppressed (with expiry)")+ lines.append("| Tool | ID | Package |")+ lines.append("|---|---|---|")+ for f in report.ignored:+ lines.append(f"| {f.tool} | `{f.vuln_id}` | `{f.package}` |")+ lines.append("")++ with open(summary_path, "a") as fh:+ fh.write("\n".join(lines) + "\n")+++def exit_code_for(report: Report) -> int:+ """Staged policy: high/critical → 1, everything else → 0."""+ if report.high_critical:+ return 1+ return 0+++# ─── CLI ─────────────────────────────────────────────────────────────────────+++def get_repo_root() -> Path:+ workspace = os.environ.get("GITHUB_WORKSPACE")+ if workspace:+ return Path(workspace)+ # Walk up from this script to find the repo root (contains .git/)+ p = Path(__file__).resolve().parent.parent+ if (p / ".git").exists():+ return p+ return Path.cwd()+++def get_ignores_path() -> Path:+ return get_repo_root() / "scripts" / "security_audit_ignores.txt"+++def cmd_validate_ignores(args: argparse.Namespace) -> int:+ try:+ ignores = load_ignores(get_ignores_path())+ print(f"Validated {len(ignores)} ignore entries.")+ return 0+ except ValueError as e:+ print(f"Ignore policy error: {e}", file=sys.stderr)+ return 1+++def cmd_pnpm(args: argparse.Namespace) -> int:+ ignores = load_ignores(get_ignores_path())+ report = run_pnpm_audit(get_repo_root(), ignores)+ emit_annotations(report)+ emit_summary(report)+ code = exit_code_for(report)+ n = len(report.findings)+ print(f"pnpm: {n} findings ({len(report.high_critical)} high/critical, "+ f"{len(report.moderate)} moderate, {len(report.low_info)} low/info), "+ f"{len(report.ignored)} ignored")+ return code+++def cmd_python(args: argparse.Namespace) -> int:+ ignores = load_ignores(get_ignores_path())+ report = run_pip_audit(get_repo_root(), ignores)+ emit_annotations(report)+ emit_summary(report)+ code = exit_code_for(report)+ n = len(report.findings)+ print(f"python: {n} findings, {len(report.ignored)} ignored")+ return code+++def cmd_go(args: argparse.Namespace) -> int:+ ignores = load_ignores(get_ignores_path())+ report = run_govulncheck(get_repo_root(), ignores)+ emit_annotations(report)+ emit_summary(report)+ code = exit_code_for(report)+ n = len(report.findings)+ print(f"go: {n} findings, {len(report.ignored)} ignored")+ return code+++def main() -> int:+ parser = argparse.ArgumentParser(+ prog="security_audit",+ description="Dependency CVE scanner for AiSOC CI",+ )+ sub = parser.add_subparsers(dest="command", required=True)++ sub.add_parser("validate-ignores", help="Validate the ignores file")+ sub.add_parser("pnpm", help="Audit pnpm dependencies")+ sub.add_parser("python", help="Audit Python dependencies")+ sub.add_parser("go", help="Audit Go dependencies")++ args = parser.parse_args()++ handlers = {+ "validate-ignores": cmd_validate_ignores,+ "pnpm": cmd_pnpm,+ "python": cmd_python,+ "go": cmd_go,+ }++ return handlers[args.command](args)+++if __name__ == "__main__":+ sys.exit(main())diff --git a/scripts/security_audit_ignores.txt b/scripts/security_audit_ignores.txtnew file mode 100644index 000000000..445691ad2--- /dev/null+++ b/scripts/security_audit_ignores.txt@@ -0,0 +1,10 @@+# Dependency vulnerability ignore list for AiSOC CI.+#+# Format: tool|ID|reason|YYYY-MM-DD+# tool — pnpm, python, or go+# ID — CVE-YYYY-NNNNN, GHSA-xxxx-xxxx-xxxx, GO-YYYY-NNNN, or PYSEC-YYYY-N+# reason — why this ignore is acceptable (required, no blanks)+# expiry — date this ignore expires (max 90 days from today)+#+# Example:+# python|CVE-2026-12345|Upstream fix not released; tracked in issue #123|2026-07-15diff --git a/scripts/tests/test_security_audit.py b/scripts/tests/test_security_audit.pynew file mode 100644index 000000000..5fe1930f3--- /dev/null+++ b/scripts/tests/test_security_audit.py@@ -0,0 +1,448 @@+"""Tests for scripts/security_audit.py — ignore validation, classifiers, parsers."""++from __future__ import annotations++import datetime as dt+import textwrap+from pathlib import Path++import pytest++# Import the module under test+import sys++sys.path.insert(0, str(Path(__file__).resolve().parent.parent))+from security_audit import (+ Finding,+ Ignore,+ Report,+ classify_govulncheck,+ classify_pip_audit,+ classify_pnpm_audit,+ exit_code_for,+ is_ignored,+ load_ignores,+ parse_govulncheck_json,+ parse_pip_audit_json,+)+++# ─── Helpers ──────────────────────────────────────────────────────────────────++TODAY = dt.date(2026, 6, 1)+VALID_EXPIRY = "2026-07-01" # 30 days out — within 90-day window+++def _write_ignores(tmp_path: Path, content: str) -> Path:+ p = tmp_path / "ignores.txt"+ p.write_text(textwrap.dedent(content))+ return p+++# ─── load_ignores tests ──────────────────────────────────────────────────────+++class TestLoadIgnores:+ def test_empty_file(self, tmp_path: Path):+ p = _write_ignores(tmp_path, "")+ assert load_ignores(p, today=TODAY) == []++ def test_comments_and_blanks_skipped(self, tmp_path: Path):+ p = _write_ignores(tmp_path, "# comment\n\n# another\n")+ assert load_ignores(p, today=TODAY) == []++ def test_valid_cve(self, tmp_path: Path):+ p = _write_ignores(tmp_path, f"python|CVE-2026-12345|reason here|{VALID_EXPIRY}\n")+ result = load_ignores(p, today=TODAY)+ assert len(result) == 1+ assert result[0].tool == "python"+ assert result[0].vuln_id == "CVE-2026-12345"+ assert result[0].reason == "reason here"+ assert result[0].expires == dt.date(2026, 7, 1)++ def test_valid_ghsa(self, tmp_path: Path):+ p = _write_ignores(tmp_path, f"pnpm|GHSA-abcd-efgh-ijkl|reason|{VALID_EXPIRY}\n")+ result = load_ignores(p, today=TODAY)+ assert len(result) == 1+ assert result[0].vuln_id == "GHSA-abcd-efgh-ijkl"++ def test_valid_go_id(self, tmp_path: Path):+ p = _write_ignores(tmp_path, f"go|GO-2026-1234|reason|{VALID_EXPIRY}\n")+ result = load_ignores(p, today=TODAY)+ assert len(result) == 1+ assert result[0].vuln_id == "GO-2026-1234"++ def test_valid_pysec(self, tmp_path: Path):+ p = _write_ignores(tmp_path, f"python|PYSEC-2026-42|reason|{VALID_EXPIRY}\n")+ result = load_ignores(p, today=TODAY)+ assert len(result) == 1+ assert result[0].vuln_id == "PYSEC-2026-42"++ def test_wrong_field_count(self, tmp_path: Path):+ p = _write_ignores(tmp_path, "python|CVE-2026-12345|reason\n")+ with pytest.raises(ValueError, match="expected 4 pipe-delimited fields"):+ load_ignores(p, today=TODAY)++ def test_invalid_tool(self, tmp_path: Path):+ p = _write_ignores(tmp_path, f"npm|CVE-2026-12345|reason|{VALID_EXPIRY}\n")+ with pytest.raises(ValueError, match="invalid tool 'npm'"):+ load_ignores(p, today=TODAY)++ def test_invalid_vuln_id(self, tmp_path: Path):+ p = _write_ignores(tmp_path, f"python|BADID-123|reason|{VALID_EXPIRY}\n")+ with pytest.raises(ValueError, match="invalid ID 'BADID-123'"):+ load_ignores(p, today=TODAY)++ def test_empty_reason(self, tmp_path: Path):+ p = _write_ignores(tmp_path, f"python|CVE-2026-12345||{VALID_EXPIRY}\n")+ with pytest.raises(ValueError, match="reason must not be empty"):+ load_ignores(p, today=TODAY)++ def test_invalid_date(self, tmp_path: Path):+ p = _write_ignores(tmp_path, "python|CVE-2026-12345|reason|not-a-date\n")+ with pytest.raises(ValueError, match="invalid date"):+ load_ignores(p, today=TODAY)++ def test_expired(self, tmp_path: Path):+ p = _write_ignores(tmp_path, "python|CVE-2026-12345|reason|2026-05-01\n")+ with pytest.raises(ValueError, match="expired on 2026-05-01"):+ load_ignores(p, today=TODAY)++ def test_exceeds_90_days(self, tmp_path: Path):+ far_future = "2026-12-01"+ p = _write_ignores(tmp_path, f"python|CVE-2026-12345|reason|{far_future}\n")+ with pytest.raises(ValueError, match="exceeds 90-day maximum"):+ load_ignores(p, today=TODAY)++ def test_missing_file_returns_empty(self, tmp_path: Path):+ p = tmp_path / "does_not_exist.txt"+ assert load_ignores(p, today=TODAY) == []++ def test_multiple_valid_entries(self, tmp_path: Path):+ content = (+ f"pnpm|CVE-2026-11111|reason one|{VALID_EXPIRY}\n"+ f"python|CVE-2026-22222|reason two|{VALID_EXPIRY}\n"+ f"go|GO-2026-3333|reason three|{VALID_EXPIRY}\n"+ )+ p = _write_ignores(tmp_path, content)+ result = load_ignores(p, today=TODAY)+ assert len(result) == 3+++# ─── is_ignored tests ────────────────────────────────────────────────────────+++class TestIsIgnored:+ def test_match(self):+ ig = Ignore(tool="pnpm", vuln_id="CVE-2026-111", reason="r", expires=TODAY)+ assert is_ignored("pnpm", ["CVE-2026-111"], [ig]) == ig++ def test_no_match_different_tool(self):+ ig = Ignore(tool="python", vuln_id="CVE-2026-111", reason="r", expires=TODAY)+ assert is_ignored("pnpm", ["CVE-2026-111"], [ig]) is None++ def test_no_match_different_id(self):+ ig = Ignore(tool="pnpm", vuln_id="CVE-2026-111", reason="r", expires=TODAY)+ assert is_ignored("pnpm", ["CVE-2026-999"], [ig]) is None++ def test_match_among_multiple_ids(self):+ ig = Ignore(tool="go", vuln_id="GO-2026-0001", reason="r", expires=TODAY)+ assert is_ignored("go", ["CVE-2026-111", "GO-2026-0001"], [ig]) == ig+++# ─── classify_pnpm_audit tests ───────────────────────────────────────────────+++class TestClassifyPnpmAudit:+ def test_empty_advisories(self):+ report = classify_pnpm_audit({"advisories": {}}, [])+ assert report.findings == []+ assert report.ignored == []++ def test_single_high(self):+ data = {+ "advisories": {+ "100": {+ "id": 100,+ "severity": "high",+ "module_name": "lodash",+ "title": "Prototype Pollution",+ "cves": ["CVE-2026-00100"],+ "github_advisory_id": "GHSA-aaaa-bbbb-cccc",+ }+ }+ }+ report = classify_pnpm_audit(data, [])+ assert len(report.findings) == 1+ f = report.findings[0]+ assert f.severity == "high"+ assert f.package == "lodash"+ assert f.vuln_id == "GHSA-aaaa-bbbb-cccc"++ def test_moderate_finding(self):+ data = {+ "advisories": {+ "200": {+ "id": 200,+ "severity": "moderate",+ "module_name": "express",+ "title": "Open redirect",+ "cves": ["CVE-2026-00200"],+ "github_advisory_id": "",+ }+ }+ }+ report = classify_pnpm_audit(data, [])+ assert len(report.moderate) == 1+ assert report.moderate[0].severity == "moderate"++ def test_ignored_advisory(self):+ data = {+ "advisories": {+ "300": {+ "id": 300,+ "severity": "critical",+ "module_name": "foo",+ "title": "RCE",+ "cves": ["CVE-2026-00300"],+ "github_advisory_id": "GHSA-xxxx-yyyy-zzzz",+ }+ }+ }+ ig = Ignore(tool="pnpm", vuln_id="CVE-2026-00300", reason="r", expires=TODAY)+ report = classify_pnpm_audit(data, [ig])+ assert len(report.findings) == 0+ assert len(report.ignored) == 1++ def test_unknown_severity_normalized(self):+ data = {+ "advisories": {+ "400": {+ "id": 400,+ "severity": "banana",+ "module_name": "bad",+ "title": "weird",+ "cves": [],+ "github_advisory_id": "",+ }+ }+ }+ report = classify_pnpm_audit(data, [])+ assert report.findings[0].severity == "unknown"+++# ─── parse_pip_audit_json tests ──────────────────────────────────────────────+++class TestParsePipAuditJson:+ def test_empty_string(self):+ assert parse_pip_audit_json("") == []++ def test_list_format(self):+ raw = '[{"name": "pkg", "version": "1.0", "vulns": []}]'+ result = parse_pip_audit_json(raw)+ assert len(result) == 1+ assert result[0]["name"] == "pkg"++ def test_dict_format(self):+ raw = '{"dependencies": [{"name": "pkg", "version": "1.0", "vulns": []}]}'+ result = parse_pip_audit_json(raw)+ assert len(result) == 1++ def test_dict_without_dependencies_key(self):+ raw = '{"something_else": true}'+ result = parse_pip_audit_json(raw)+ assert result == []+++# ─── classify_pip_audit tests ────────────────────────────────────────────────+++class TestClassifyPipAudit:+ def test_no_vulns(self):+ deps = [{"name": "requests", "version": "2.31.0", "vulns": []}]+ report = classify_pip_audit("services/api", deps, [])+ assert report.findings == []++ def test_single_vuln(self):+ deps = [+ {+ "name": "requests",+ "version": "2.25.0",+ "vulns": [+ {+ "id": "PYSEC-2026-10",+ "aliases": ["CVE-2026-99999"],+ "description": "SSRF in requests",+ }+ ],+ }+ ]+ report = classify_pip_audit("services/api", deps, [])+ assert len(report.findings) == 1+ f = report.findings[0]+ assert f.severity == "high"+ assert f.vuln_id == "PYSEC-2026-10"+ assert f.package == "requests==2.25.0"+ assert f.location == "services/api"++ def test_vuln_id_used_when_aliases_empty(self):+ deps = [+ {+ "name": "requests",+ "version": "2.25.0",+ "vulns": [+ {+ "id": "PYSEC-2026-10",+ "aliases": [],+ "description": "Regression: vuln_id must not fall through",+ }+ ],+ }+ ]+ report = classify_pip_audit("services/api", deps, [])+ assert report.findings[0].vuln_id == "PYSEC-2026-10"++ def test_ignored_vuln(self):+ deps = [+ {+ "name": "requests",+ "version": "2.25.0",+ "vulns": [+ {+ "id": "PYSEC-2026-10",+ "aliases": ["CVE-2026-99999"],+ "description": "SSRF",+ }+ ],+ }+ ]+ ig = Ignore(tool="python", vuln_id="CVE-2026-99999", reason="r", expires=TODAY)+ report = classify_pip_audit("services/api", deps, [ig])+ assert len(report.findings) == 0+ assert len(report.ignored) == 1+++# ─── parse_govulncheck_json tests ────────────────────────────────────────────+++class TestParseGovulncheckJson:+ def test_empty(self):+ assert parse_govulncheck_json("") == []++ def test_osv_with_finding(self):+ lines = [+ '{"osv": {"id": "GO-2026-0001", "aliases": ["CVE-2026-11111"], "summary": "Bad thing"}}',+ '{"finding": {"osv": "GO-2026-0001", "trace": []}}',+ ]+ result = parse_govulncheck_json("\n".join(lines))+ assert len(result) == 1+ assert result[0]["id"] == "GO-2026-0001"+ assert result[0]["aliases"] == ["CVE-2026-11111"]++ def test_osv_without_finding_excluded(self):+ lines = [+ '{"osv": {"id": "GO-2026-0002", "aliases": [], "summary": "Not called"}}',+ ]+ result = parse_govulncheck_json("\n".join(lines))+ assert result == []++ def test_malformed_lines_skipped(self):+ lines = [+ "not json",+ '{"osv": {"id": "GO-2026-0003", "aliases": [], "summary": "yes"}}',+ '{"finding": {"osv": "GO-2026-0003"}}',+ ]+ result = parse_govulncheck_json("\n".join(lines))+ assert len(result) == 1+++# ─── classify_govulncheck tests ──────────────────────────────────────────────+++class TestClassifyGovulncheck:+ def test_no_vulns(self):+ report = classify_govulncheck("services/ingest", [], [])+ assert report.findings == []++ def test_single_vuln(self):+ vulns = [{"id": "GO-2026-0001", "aliases": ["CVE-2026-55555"], "summary": "Bad"}]+ report = classify_govulncheck("services/ingest", vulns, [])+ assert len(report.findings) == 1+ f = report.findings[0]+ assert f.severity == "high"+ assert f.vuln_id == "GO-2026-0001"++ def test_ignored_vuln(self):+ vulns = [{"id": "GO-2026-0001", "aliases": ["CVE-2026-55555"], "summary": "Bad"}]+ ig = Ignore(tool="go", vuln_id="GO-2026-0001", reason="r", expires=TODAY)+ report = classify_govulncheck("services/ingest", vulns, [ig])+ assert len(report.findings) == 0+ assert len(report.ignored) == 1+++# ─── Report + exit_code_for tests ────────────────────────────────────────────+++class TestExitCodeFor:+ def test_no_findings(self):+ assert exit_code_for(Report()) == 0++ def test_only_moderate(self):+ r = Report(findings=[Finding("pnpm", "moderate", "X", "p", "l", "t")])+ assert exit_code_for(r) == 0++ def test_only_low(self):+ r = Report(findings=[Finding("pnpm", "low", "X", "p", "l", "t")])+ assert exit_code_for(r) == 0++ def test_high_fails(self):+ r = Report(findings=[Finding("python", "high", "X", "p", "l", "t")])+ assert exit_code_for(r) == 1++ def test_critical_fails(self):+ r = Report(findings=[Finding("go", "critical", "X", "p", "l", "t")])+ assert exit_code_for(r) == 1++ def test_mixed_high_and_moderate(self):+ r = Report(+ findings=[+ Finding("pnpm", "moderate", "A", "p", "l", "t"),+ Finding("pnpm", "high", "B", "p", "l", "t"),+ ]+ )+ assert exit_code_for(r) == 1+++# ─── Report property tests ───────────────────────────────────────────────────+++class TestReportProperties:+ def test_high_critical(self):+ r = Report(+ findings=[+ Finding("a", "high", "1", "p", "l", "t"),+ Finding("a", "critical", "2", "p", "l", "t"),+ Finding("a", "moderate", "3", "p", "l", "t"),+ ]+ )+ assert len(r.high_critical) == 2++ def test_moderate(self):+ r = Report(+ findings=[+ Finding("a", "moderate", "1", "p", "l", "t"),+ Finding("a", "high", "2", "p", "l", "t"),+ ]+ )+ assert len(r.moderate) == 1++ def test_low_info(self):+ r = Report(+ findings=[+ Finding("a", "low", "1", "p", "l", "t"),+ Finding("a", "info", "2", "p", "l", "t"),+ Finding("a", "unknown", "3", "p", "l", "t"),+ ]+ )+ assert len(r.low_info) == 3diff --git a/services/api/app/core/scheduler_lock.py b/services/api/app/core/scheduler_lock.pynew file mode 100644index 000000000..17c5abd6b--- /dev/null+++ b/services/api/app/core/scheduler_lock.py@@ -0,0 +1,153 @@+"""Distributed scheduler lock helper for in-process API workers.++Workers in ``app.main`` run in-process on every API replica. This helper+provides a per-job Redis lease so only one replica executes a given worker at+any time while keeping availability-first behavior:++* lock held by another replica -> caller gets ``False`` and can skip silently.+* Redis unavailable -> caller gets ``True`` (fail-open) and work proceeds.+"""++from __future__ import annotations++import asyncio+import logging+import uuid+from collections.abc import AsyncIterator+from contextlib import asynccontextmanager, suppress++from redis.asyncio import Redis, from_url+from redis.exceptions import RedisError++logger = logging.getLogger(__name__)++_redis_client: Redis | None = None+_redis_client_lock = asyncio.Lock()++_BACKEND_ERRORS = (RedisError, OSError, TimeoutError)++_RENEW_IF_OWNER_SCRIPT = """+if redis.call('get', KEYS[1]) == ARGV[1] then+ return redis.call('expire', KEYS[1], ARGV[2])+end+return 0+"""++_RELEASE_IF_OWNER_SCRIPT = """+if redis.call('get', KEYS[1]) == ARGV[1] then+ return redis.call('del', KEYS[1])+end+return 0+"""+++async def _get_redis_client() -> Redis:+ from app.core.config import settings # lazy: only needed at first connect, not at import time++ global _redis_client+ if _redis_client is not None:+ return _redis_client++ async with _redis_client_lock:+ if _redis_client is None:+ _redis_client = from_url(str(settings.REDIS_URL), decode_responses=True)+ return _redis_client+++def _warn_backend_unavailable(*, job_name: str, lock_key: str, phase: str, exc: Exception) -> None:+ logger.warning(+ "scheduler_lock.backend_unavailable job_name=%s lock_key=%s phase=%s error=%s",+ job_name,+ lock_key,+ phase,+ type(exc).__name__,+ )+++async def _renew_lease_loop(+ *,+ redis_client: Redis,+ job_name: str,+ lock_key: str,+ token: str,+ ttl_seconds: int,+ renew_every_seconds: int,+) -> None:+ while True:+ await asyncio.sleep(renew_every_seconds)+ try:+ renewed = await redis_client.eval(+ _RENEW_IF_OWNER_SCRIPT,+ 1,+ lock_key,+ token,+ str(ttl_seconds),+ )+ except _BACKEND_ERRORS as exc:+ _warn_backend_unavailable(job_name=job_name, lock_key=lock_key, phase="renew", exc=exc)+ return++ if not renewed:+ return+++@asynccontextmanager+async def scheduler_lock(*, job_name: str, ttl_seconds: int) -> AsyncIterator[bool]:+ """Yield whether the caller should run this scheduler job.++ The lock key format is always ``scheduler_lock:{job_name}``.+ """+ if ttl_seconds <= 0:+ raise ValueError("ttl_seconds must be > 0")++ lock_key = f"scheduler_lock:{job_name}"+ token = uuid.uuid4().hex+ renew_task: asyncio.Task[None] | None = None+ redis_client: Redis | None = None++ try:+ redis_client = await _get_redis_client()+ acquired = await redis_client.set(lock_key, token, ex=ttl_seconds, nx=True)+ except _BACKEND_ERRORS as exc:+ _warn_backend_unavailable(job_name=job_name, lock_key=lock_key, phase="acquire", exc=exc)+ yield True+ return++ if not acquired:+ yield False+ return++ renew_every_seconds = max(1, ttl_seconds // 3)+ renew_task = asyncio.create_task(+ _renew_lease_loop(+ redis_client=redis_client,+ job_name=job_name,+ lock_key=lock_key,+ token=token,+ ttl_seconds=ttl_seconds,+ renew_every_seconds=renew_every_seconds,+ ),+ name=f"scheduler_lock_renew_{job_name}",+ )++ try:+ yield True+ finally:+ if renew_task is not None:+ renew_task.cancel()+ with suppress(asyncio.CancelledError):+ await renew_task++ if redis_client is not None:+ try:+ await redis_client.eval(+ _RELEASE_IF_OWNER_SCRIPT,+ 1,+ lock_key,+ token,+ )+ except _BACKEND_ERRORS as exc:+ _warn_backend_unavailable(job_name=job_name, lock_key=lock_key, phase="release", exc=exc)+++__all__ = ["scheduler_lock"]diff --git a/services/api/app/main.py b/services/api/app/main.pyindex 73853f445..bccfc6a67 100644--- a/services/api/app/main.py+++ b/services/api/app/main.py@@ -3,7 +3,7 @@import asyncioimport hmacimport time-from collections.abc import AsyncGenerator+from collections.abc import AsyncGenerator, Awaitable, Callablefrom contextlib import asynccontextmanagerimport structlog@@ -20,6 +20,7 @@ from app.core.airgap import airgap_statusfrom app.core.config import is_dev_env, settings, warn_if_insecure_defaultsfrom app.core.cors import build_cors_kwargsfrom app.core.logging import configure_logging+from app.core.scheduler_lock import scheduler_lockfrom app.core.telemetry import instrument_appfrom app.db.clickhouse import close_clickhousefrom app.db.database import engine@@ -37,6 +38,34 @@ _metrics_bearer = HTTPBearer(auto_error=False)logger = structlog.get_logger(__name__)+_SCHEDULER_LOCK_RETRY_SECONDS = 5++# 15m covers slow OAuth due-batch refreshes while each provider call is bounded+# by OAUTH_REFRESH_HTTP_TIMEOUT_SECONDS.+_OAUTH_REFRESH_LOCK_TTL_SECONDS = 900+# Weekly digest can fan out across tenants + PDF generation, so use a longer+# lease to avoid duplicate report generation during slow runs.+_WEEKLY_DIGEST_LOCK_TTL_SECONDS = 5400+# Hunt sweep can execute multiple saved hunts + case opens in one tick; 5m+# covers slow sweeps while still recovering quickly after replica loss.+_HUNT_SCHEDULER_LOCK_TTL_SECONDS = 300+++async def _run_guarded_scheduler_worker(+ *,+ job_name: str,+ ttl_seconds: int,+ worker: Callable[[], Awaitable[None]],+) -> None:+ while True:+ async with scheduler_lock(job_name=job_name, ttl_seconds=ttl_seconds) as should_run:+ if not should_run:+ await asyncio.sleep(_SCHEDULER_LOCK_RETRY_SECONDS)+ continue+ await worker()+ return++# Prometheus metricsREQUEST_COUNT = Counter("aisoc_http_requests_total",@@ -110,7 +139,14 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:oauth_refresh_task: asyncio.Task | None = Noneif settings.OAUTH_REFRESH_WORKER_ENABLED:try:- oauth_refresh_task = asyncio.create_task(run_oauth_refresh(), name="oauth_refresh_worker")+ oauth_refresh_task = asyncio.create_task(+ _run_guarded_scheduler_worker(+ job_name="oauth_refresh",+ ttl_seconds=_OAUTH_REFRESH_LOCK_TTL_SECONDS,+ worker=run_oauth_refresh,+ ),+ name="oauth_refresh_worker",+ )logger.info("oauth_refresh worker started")except Exception as exc:logger.warning("oauth_refresh worker failed to start", error=str(exc))@@ -122,7 +158,14 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:weekly_digest_task: asyncio.Task | None = Noneif settings.WEEKLY_DIGEST_WORKER_ENABLED:try:- weekly_digest_task = asyncio.create_task(run_weekly_digest(), name="weekly_digest_worker")+ weekly_digest_task = asyncio.create_task(+ _run_guarded_scheduler_worker(+ job_name="weekly_digest",+ ttl_seconds=_WEEKLY_DIGEST_LOCK_TTL_SECONDS,+ worker=run_weekly_digest,+ ),+ name="weekly_digest_worker",+ )logger.info("weekly_digest worker started")except Exception as exc:logger.warning("weekly_digest worker failed to start", error=str(exc))@@ -133,7 +176,14 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:hunt_scheduler_task: asyncio.Task | None = Noneif settings.HUNT_SCHEDULER_ENABLED:try:- hunt_scheduler_task = asyncio.create_task(run_hunt_scheduler(), name="hunt_scheduler_worker")+ hunt_scheduler_task = asyncio.create_task(+ _run_guarded_scheduler_worker(+ job_name="hunt_scheduler",+ ttl_seconds=_HUNT_SCHEDULER_LOCK_TTL_SECONDS,+ worker=run_hunt_scheduler,+ ),+ name="hunt_scheduler_worker",+ )logger.info("hunt_scheduler worker started")except Exception as exc:logger.warning("hunt_scheduler worker failed to start", error=str(exc))diff --git a/services/api/tests/test_scheduler_lock.py b/services/api/tests/test_scheduler_lock.pynew file mode 100644index 000000000..eb1903773--- /dev/null+++ b/services/api/tests/test_scheduler_lock.py@@ -0,0 +1,91 @@+"""Unit tests for the scheduler singleton lock helper."""++from __future__ import annotations++import logging++import app.core.scheduler_lock as scheduler_lock_module+import pytest+from redis.exceptions import RedisError+++class _FakeRedisClient:+ def __init__(self, *, acquire_result: bool | None = True, acquire_exc: Exception | None = None) -> None:+ self.acquire_result = acquire_result+ self.acquire_exc = acquire_exc+ self.set_calls: list[dict[str, object]] = []+ self.eval_calls: list[tuple[str, int, tuple[object, ...]]] = []++ async def set(self, name: str, value: str, *, ex: int, nx: bool) -> bool | None:+ self.set_calls.append({"name": name, "value": value, "ex": ex, "nx": nx})+ if self.acquire_exc is not None:+ raise self.acquire_exc+ return self.acquire_result++ async def eval(self, script: str, numkeys: int, *args: object) -> int:+ self.eval_calls.append((script, numkeys, args))+ return 1+++async def test_scheduler_lock_acquired_runs_job(monkeypatch: pytest.MonkeyPatch) -> None:+ fake_redis = _FakeRedisClient(acquire_result=True)++ async def _fake_get_redis_client() -> _FakeRedisClient:+ return fake_redis++ monkeypatch.setattr(scheduler_lock_module, "_get_redis_client", _fake_get_redis_client)++ ran = False+ async with scheduler_lock_module.scheduler_lock(job_name="oauth_refresh", ttl_seconds=900) as should_run:+ if should_run:+ ran = True++ assert ran is True+ assert fake_redis.set_calls+ assert fake_redis.set_calls[0]["name"] == "scheduler_lock:oauth_refresh"+ assert fake_redis.set_calls[0]["ex"] == 900+ assert fake_redis.set_calls[0]["nx"] is True+++async def test_scheduler_lock_not_acquired_skips_job(monkeypatch: pytest.MonkeyPatch) -> None:+ fake_redis = _FakeRedisClient(acquire_result=None)++ async def _fake_get_redis_client() -> _FakeRedisClient:+ return fake_redis++ monkeypatch.setattr(scheduler_lock_module, "_get_redis_client", _fake_get_redis_client)++ ran = False+ async with scheduler_lock_module.scheduler_lock(job_name="weekly_digest", ttl_seconds=5400) as should_run:+ if should_run:+ ran = True++ assert ran is False+ assert fake_redis.set_calls+ assert fake_redis.set_calls[0]["name"] == "scheduler_lock:weekly_digest"+ assert fake_redis.eval_calls == []+++async def test_scheduler_lock_backend_unavailable_fails_open(+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+) -> None:+ fake_redis = _FakeRedisClient(acquire_exc=RedisError("redis unavailable"))++ async def _fake_get_redis_client() -> _FakeRedisClient:+ return fake_redis++ monkeypatch.setattr(scheduler_lock_module, "_get_redis_client", _fake_get_redis_client)++ ran = False+ with caplog.at_level(logging.WARNING, logger="app.core.scheduler_lock"):+ async with scheduler_lock_module.scheduler_lock(job_name="hunt_scheduler", ttl_seconds=300) as should_run:+ if should_run:+ ran = True++ assert ran is True+ assert "scheduler_lock.backend_unavailable" in caplog.text+ assert "phase=acquire" in caplog.text