Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

feat: add Redis-backed scheduler singleton + dependency CVE audit pipeline (#218)

ARDA7787badcdf628944
diff --git a/.github/workflows/security-audit.yml b/.github/workflows/security-audit.yml
new file mode 100644
index 000000000..582246856
--- /dev/null
+++ b/.github/workflows/security-audit.yml
@@ -0,0 +1,77 @@
+name: Security Audit
+
+# Scoped ignores: scripts/security_audit_ignores.txt
+# Local run: python scripts/security_audit.py <pnpm|python|go|validate-ignores>
+
+on:
+ push:
+ branches: [main, develop]
+ pull_request:
+ branches: [main, develop]
+ workflow_dispatch:
+
+jobs:
+ security-audit:
+ name: security-audit
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ env:
+ PNPM_AUDIT_REGISTRY: https://registry.npmjs.org/
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: pnpm/action-setup@v4
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: '22'
+ cache: pnpm
+ cache-dependency-path: pnpm-lock.yaml
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+
+ - uses: actions/setup-go@v5
+ with:
+ go-version: '1.23'
+
+ - name: Install audit tools
+ run: |
+ set -euo pipefail
+ python -m pip install --upgrade pip
+ python -m pip install pip-audit poetry poetry-plugin-export
+ go install golang.org/x/vuln/cmd/govulncheck@latest
+ echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
+
+ - name: Validate ignore policy
+ run: python scripts/security_audit.py validate-ignores
+
+ - name: Install pnpm dependencies
+ run: pnpm install --frozen-lockfile --ignore-scripts
+
+ - name: Audit pnpm dependencies
+ id: audit-pnpm
+ continue-on-error: true
+ run: python scripts/security_audit.py pnpm
+
+ - name: Audit Python dependencies
+ id: audit-python
+ continue-on-error: true
+ run: python scripts/security_audit.py python
+
+ - name: Audit Go dependencies
+ id: audit-go
+ continue-on-error: true
+ run: python scripts/security_audit.py go
+
+ - name: Enforce staged policy
+ if: >-
+ steps.audit-pnpm.outcome == 'failure' ||
+ steps.audit-python.outcome == 'failure' ||
+ steps.audit-go.outcome == 'failure'
+ run: |
+ echo "::error::High/critical dependency vulnerabilities found — see annotations above"
+ exit 1
diff --git a/scripts/security_audit.py b/scripts/security_audit.py
new file mode 100644
index 000000000..a7761496c
--- /dev/null
+++ b/scripts/security_audit.py
@@ -0,0 +1,624 @@
+"""Dependency CVE scanner for AiSOC CI.
+
+Subcommands:
+ validate-ignores — check the ignores file is well-formed
+ pnpm — run pnpm audit (staged: high/critical fail, moderate warns)
+ python — run pip-audit per Python service (any finding fails)
+ go — run govulncheck per Go module (any finding fails)
+
+Usage (local):
+ python scripts/security_audit.py validate-ignores
+ python scripts/security_audit.py pnpm
+ python scripts/security_audit.py python
+ python scripts/security_audit.py go
+
+Environment variables (CI):
+ GITHUB_STEP_SUMMARY — path to markdown summary file (set by Actions)
+ GITHUB_WORKSPACE — repo root (set by Actions; defaults to cwd)
+ PNPM_AUDIT_REGISTRY — fallback registry if primary returns 403
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+# ─── Data types ───────────────────────────────────────────────────────────────
+
+SEVERITY_ORDER = {"info": 0, "low": 1, "moderate": 2, "high": 3, "critical": 4}
+
+IGNORE_ID_PATTERN = re.compile(
+ r"^(CVE-\d{4}-\d{4,}|GHSA-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}|GO-\d{4}-\d{4,}|PYSEC-\d{4}-\d+)$"
+)
+
+
+@dataclass
+class Ignore:
+ tool: str
+ vuln_id: str
+ reason: str
+ expires: dt.date
+
+
+@dataclass
+class Finding:
+ tool: str
+ severity: str
+ vuln_id: str
+ package: str
+ location: str
+ title: str
+
+
+@dataclass
+class Report:
+ findings: list[Finding] = field(default_factory=list)
+ ignored: list[Finding] = field(default_factory=list)
+ warnings: list[str] = field(default_factory=list)
+
+ @property
+ def high_critical(self) -> list[Finding]:
+ return [f for f in self.findings if f.severity in ("high", "critical")]
+
+ @property
+ def moderate(self) -> list[Finding]:
+ return [f for f in self.findings if f.severity == "moderate"]
+
+ @property
+ def low_info(self) -> list[Finding]:
+ return [f for f in self.findings if f.severity in ("low", "info", "unknown")]
+
+
+# ─── Ignore loading ──────────────────────────────────────────────────────────
+
+
+def load_ignores(path: Path, today: dt.date | None = None) -> list[Ignore]:
+ """Load and validate the ignores file.
+
+ Format per line: tool|ID|reason|YYYY-MM-DD
+ Lines starting with # and blank lines are skipped.
+ """
+ if today is None:
+ today = dt.date.today()
+
+ max_expiry = today + dt.timedelta(days=90)
+ allowed_tools = {"pnpm", "python", "go"}
+ ignores: list[Ignore] = []
+
+ if not path.exists():
+ return ignores
+
+ for line_no, raw_line in enumerate(path.read_text().splitlines(), start=1):
+ line = raw_line.strip()
+ if not line or line.startswith("#"):
+ continue
+
+ parts = [p.strip() for p in line.split("|")]
+ if len(parts) != 4:
+ raise ValueError(
+ f"Line {line_no}: expected 4 pipe-delimited fields, got {len(parts)}"
+ )
+
+ tool, vuln_id, reason, expiry_str = parts
+
+ if tool not in allowed_tools:
+ raise ValueError(
+ f"Line {line_no}: invalid tool '{tool}' (expected: {sorted(allowed_tools)})"
+ )
+
+ if not IGNORE_ID_PATTERN.match(vuln_id):
+ raise ValueError(
+ f"Line {line_no}: invalid ID '{vuln_id}' "
+ "(expected CVE-YYYY-NNNNN, GHSA-xxxx-xxxx-xxxx, GO-YYYY-NNNN, or PYSEC-YYYY-N)"
+ )
+
+ if not reason:
+ raise ValueError(f"Line {line_no}: reason must not be empty")
+
+ try:
+ expiry = dt.date.fromisoformat(expiry_str)
+ except ValueError:
+ raise ValueError(f"Line {line_no}: invalid date '{expiry_str}' (expected YYYY-MM-DD)")
+
+ if expiry < today:
+ raise ValueError(f"Line {line_no}: ignore for {vuln_id} expired on {expiry_str}")
+
+ if expiry > max_expiry:
+ raise ValueError(
+ f"Line {line_no}: expiry {expiry_str} exceeds 90-day maximum from today ({max_expiry})"
+ )
+
+ ignores.append(Ignore(tool=tool, vuln_id=vuln_id, reason=reason, expires=expiry))
+
+ return ignores
+
+
+def is_ignored(tool: str, vuln_ids: list[str], ignores: list[Ignore]) -> Ignore | None:
+ """Return the matching Ignore if any of the vuln_ids are ignored for this tool."""
+ tool_ignores = {ig.vuln_id for ig in ignores if ig.tool == tool}
+ for vid in vuln_ids:
+ if vid in tool_ignores:
+ return next(ig for ig in ignores if ig.tool == tool and ig.vuln_id == vid)
+ return None
+
+
+# ─── pnpm scanner ────────────────────────────────────────────────────────────
+
+
+def classify_pnpm_audit(audit_json: dict[str, Any], ignores: list[Ignore]) -> Report:
+ """Classify pnpm audit JSON into a Report using native severity."""
+ report = Report()
+
+ for advisory in audit_json.get("advisories", {}).values():
+ ghsa = advisory.get("github_advisory_id", "")
+ cves = advisory.get("cves", []) or []
+ all_ids = [i for i in [ghsa, *cves] if i]
+ severity = (advisory.get("severity") or "unknown").lower()
+ if severity not in SEVERITY_ORDER:
+ severity = "unknown"
+
+ finding = Finding(
+ tool="pnpm",
+ severity=severity,
+ vuln_id=all_ids[0] if all_ids else str(advisory.get("id", "unknown")),
+ package=advisory.get("module_name", "unknown"),
+ location="pnpm workspace",
+ title=advisory.get("title") or advisory.get("overview") or "",
+ )
+
+ if is_ignored("pnpm", all_ids, ignores):
+ report.ignored.append(finding)
+ else:
+ report.findings.append(finding)
+
+ return report
+
+
+def run_pnpm_audit(repo_root: Path, ignores: list[Ignore]) -> Report:
+ """Run pnpm audit with fallback registry on 403."""
+ fallback = os.environ.get("PNPM_AUDIT_REGISTRY", "https://registry.npmjs.org/")
+
+ for attempt, registry in enumerate([None, fallback]):
+ env = os.environ.copy()
+ if registry:
+ env["npm_config_registry"] = registry
+
+ proc = subprocess.run(
+ ["pnpm", "audit", "--json"],
+ cwd=repo_root,
+ env=env,
+ capture_output=True,
+ text=True,
+ )
+
+ output = proc.stdout + proc.stderr
+ if ("403" in output or "Forbidden" in output) and attempt == 0:
+ print(f"Primary registry returned 403; retrying with {fallback}")
+ continue
+
+ try:
+ data = json.loads(proc.stdout)
+ except json.JSONDecodeError:
+ print(f"Failed to parse pnpm audit output:\n{output}", file=sys.stderr)
+ report = Report()
+ report.warnings.append("pnpm audit returned unparseable output")
+ return report
+
+ return classify_pnpm_audit(data, ignores)
+
+ report = Report()
+ report.warnings.append("pnpm audit failed on all registries")
+ return report
+
+
+# ─── Python scanner ──────────────────────────────────────────────────────────
+
+
+def parse_pip_audit_json(raw: str) -> list[dict[str, Any]]:
+ """Parse pip-audit --format=json output."""
+ if not raw.strip():
+ return []
+ data = json.loads(raw)
+ if isinstance(data, dict):
+ return data.get("dependencies", [])
+ if isinstance(data, list):
+ return data
+ return []
+
+
+def classify_pip_audit(
+ target: str, dependencies: list[dict[str, Any]], ignores: list[Ignore]
+) -> Report:
+ """Classify pip-audit findings. All findings are treated as high (fail-closed)."""
+ report = Report()
+
+ for dep in dependencies:
+ for vuln in dep.get("vulns", []):
+ aliases = vuln.get("aliases", []) or []
+ vuln_id = vuln.get("id", "")
+ all_ids = [i for i in [vuln_id, *aliases] if i]
+
+ finding = Finding(
+ tool="python",
+ severity="high",
+ vuln_id=vuln_id or (aliases[0] if aliases else dep.get("name", "unknown")),
+ package=f"{dep.get('name', '?')}=={dep.get('version', '?')}",
+ location=target,
+ title=vuln.get("description", ""),
+ )
+
+ if is_ignored("python", all_ids, ignores):
+ report.ignored.append(finding)
+ else:
+ report.findings.append(finding)
+
+ return report
+
+
+def _export_requirements(service_dir: Path, work_dir: Path) -> Path | None:
+ """Export a Poetry project to requirements.txt. Returns path or None on failure."""
+ pyproject = service_dir / "pyproject.toml"
+ shutil.copy2(pyproject, work_dir / "pyproject.toml")
+
+ lock = service_dir / "poetry.lock"
+ if lock.exists():
+ shutil.copy2(lock, work_dir / "poetry.lock")
+
+ # Poetry needs a README to avoid warnings
+ readme = service_dir / "README.md"
+ if readme.exists():
+ shutil.copy2(readme, work_dir / "README.md")
+
+ try:
+ # Generate lock if missing
+ if not (work_dir / "poetry.lock").exists():
+ proc = subprocess.run(
+ ["poetry", "lock", "--no-interaction"],
+ cwd=work_dir,
+ capture_output=True,
+ text=True,
+ )
+ if proc.returncode != 0:
+ return None
+
+ # Export to requirements.txt
+ req_path = work_dir / "requirements.txt"
+ proc = subprocess.run(
+ [
+ "poetry", "export",
+ "--format", "requirements.txt",
+ "--without-hashes",
+ "--output", str(req_path),
+ ],
+ cwd=work_dir,
+ capture_output=True,
+ text=True,
+ )
+ if proc.returncode != 0:
+ return None
+ except FileNotFoundError:
+ return None
+
+ return req_path
+
+
+def run_pip_audit(repo_root: Path, ignores: list[Ignore]) -> Report:
+ """Run pip-audit on all Python services with pyproject.toml.
+
+ Steps per service:
+ 1. poetry export → requirements.txt (in a temp dir)
+ 2. pip-audit -r requirements.txt --format json
+ """
+ combined = Report()
+
+ with tempfile.TemporaryDirectory(prefix="security_audit_py_") as tmp:
+ tmp_root = Path(tmp)
+
+ for search_dir in ["services", "packages"]:
+ parent = repo_root / search_dir
+ if not parent.exists():
+ continue
+ for child in sorted(parent.iterdir()):
+ if not child.is_dir():
+ continue
+ pyproject = child / "pyproject.toml"
+ if not pyproject.exists():
+ continue
+
+ target = str(child.relative_to(repo_root))
+ safe_name = target.replace("/", "__")
+ work_dir = tmp_root / safe_name
+ work_dir.mkdir(parents=True, exist_ok=True)
+
+ req_path = _export_requirements(child, work_dir)
+ if req_path is None:
+ combined.warnings.append(
+ f"poetry export failed for {target}; skipping pip-audit"
+ )
+ continue
+
+ try:
+ cmd = [
+ "pip-audit",
+ "-r", str(req_path),
+ "--format", "json",
+ "--progress-spinner", "off",
+ ]
+
+ proc = subprocess.run(cmd, capture_output=True, text=True, cwd=repo_root)
+ except FileNotFoundError:
+ combined.warnings.append("pip-audit not found on PATH; skipping Python audit")
+ break
+
+ if proc.returncode not in (0, 1):
+ # pip-audit returns 1 when vulns found, other codes are errors
+ combined.warnings.append(
+ f"pip-audit on {target} exited {proc.returncode}: {proc.stderr[:200]}"
+ )
+ continue
+
+ deps = parse_pip_audit_json(proc.stdout)
+ sub = classify_pip_audit(target, deps, ignores)
+ combined.findings.extend(sub.findings)
+ combined.ignored.extend(sub.ignored)
+
+ return combined
+
+
+# ─── Go scanner ──────────────────────────────────────────────────────────────
+
+
+def parse_govulncheck_json(stdout: str) -> list[dict[str, Any]]:
+ """Parse govulncheck -json output (newline-delimited JSON objects)."""
+ osv_by_id: dict[str, dict] = {}
+ finding_ids: set[str] = set()
+
+ for line in stdout.splitlines():
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ entry = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+
+ if "osv" in entry:
+ osv = entry["osv"]
+ osv_by_id[osv["id"]] = osv
+ if "finding" in entry:
+ finding_ids.add(entry["finding"]["osv"])
+
+ results = []
+ for osv_id in sorted(finding_ids):
+ osv = osv_by_id.get(osv_id, {})
+ results.append({
+ "id": osv_id,
+ "aliases": osv.get("aliases", []),
+ "summary": osv.get("summary", ""),
+ })
+
+ return results
+
+
+def classify_govulncheck(
+ module: str, vulns: list[dict[str, Any]], ignores: list[Ignore]
+) -> Report:
+ """Classify govulncheck findings. All findings are treated as high (fail-closed)."""
+ report = Report()
+
+ for vuln in vulns:
+ vuln_id = vuln.get("id", "")
+ aliases = vuln.get("aliases", []) or []
+ all_ids = [i for i in [vuln_id, *aliases] if i]
+
+ finding = Finding(
+ tool="go",
+ severity="high",
+ vuln_id=vuln_id,
+ package=module,
+ location=module,
+ title=vuln.get("summary", ""),
+ )
+
+ if is_ignored("go", all_ids, ignores):
+ report.ignored.append(finding)
+ else:
+ report.findings.append(finding)
+
+ return report
+
+
+def run_govulncheck(repo_root: Path, ignores: list[Ignore]) -> Report:
+ """Run govulncheck on all Go modules."""
+ combined = Report()
+
+ go_mods: list[Path] = []
+ for search_dir in ["services", "packages"]:
+ parent = repo_root / search_dir
+ if parent.exists():
+ go_mods.extend(sorted(parent.rglob("go.mod")))
+
+ for go_mod in go_mods:
+ module_dir = go_mod.parent
+ module = str(module_dir.relative_to(repo_root))
+
+ try:
+ proc = subprocess.run(
+ ["govulncheck", "-json", "./..."],
+ cwd=module_dir,
+ capture_output=True,
+ text=True,
+ )
+ except FileNotFoundError:
+ combined.warnings.append("govulncheck not found on PATH; skipping Go audit")
+ break
+
+ if proc.returncode not in (0, 3):
+ combined.warnings.append(
+ f"govulncheck on {module} exited {proc.returncode}: {proc.stderr[:200]}"
+ )
+ continue
+
+ vulns = parse_govulncheck_json(proc.stdout)
+ sub = classify_govulncheck(module, vulns, ignores)
+ combined.findings.extend(sub.findings)
+ combined.ignored.extend(sub.ignored)
+
+ return combined
+
+
+# ─── Output ──────────────────────────────────────────────────────────────────
+
+
+def emit_annotations(report: Report) -> None:
+ """Emit GitHub Actions annotations."""
+ for f in report.high_critical:
+ print(f"::error title=security-audit::{f.tool} {f.severity}: {f.vuln_id} in {f.package}")
+ for f in report.moderate:
+ print(f"::warning title=security-audit::{f.tool} {f.severity}: {f.vuln_id} in {f.package}")
+ for f in report.low_info:
+ print(f"::notice title=security-audit::{f.tool} {f.severity}: {f.vuln_id} in {f.package}")
+ for w in report.warnings:
+ print(f"::warning title=security-audit::{w}")
+
+
+def emit_summary(report: Report) -> None:
+ """Write GitHub step summary markdown."""
+ summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
+ if not summary_path:
+ return
+
+ lines = [
+ "## Dependency Security Audit\n",
+ f"- **High/Critical**: {len(report.high_critical)}",
+ f"- **Moderate**: {len(report.moderate)}",
+ f"- **Low/Info**: {len(report.low_info)}",
+ f"- **Ignored**: {len(report.ignored)}",
+ "",
+ ]
+
+ if report.findings:
+ lines.append("| Tool | Severity | ID | Package | Location |")
+ lines.append("|---|---|---|---|---|")
+ for f in sorted(report.findings, key=lambda x: (-SEVERITY_ORDER.get(x.severity, 0), x.tool)):
+ lines.append(f"| {f.tool} | {f.severity} | `{f.vuln_id}` | `{f.package}` | {f.location} |")
+ lines.append("")
+
+ if report.ignored:
+ lines.append("### Suppressed (with expiry)")
+ lines.append("| Tool | ID | Package |")
+ lines.append("|---|---|---|")
+ for f in report.ignored:
+ lines.append(f"| {f.tool} | `{f.vuln_id}` | `{f.package}` |")
+ lines.append("")
+
+ with open(summary_path, "a") as fh:
+ fh.write("\n".join(lines) + "\n")
+
+
+def exit_code_for(report: Report) -> int:
+ """Staged policy: high/critical → 1, everything else → 0."""
+ if report.high_critical:
+ return 1
+ return 0
+
+
+# ─── CLI ─────────────────────────────────────────────────────────────────────
+
+
+def get_repo_root() -> Path:
+ workspace = os.environ.get("GITHUB_WORKSPACE")
+ if workspace:
+ return Path(workspace)
+ # Walk up from this script to find the repo root (contains .git/)
+ p = Path(__file__).resolve().parent.parent
+ if (p / ".git").exists():
+ return p
+ return Path.cwd()
+
+
+def get_ignores_path() -> Path:
+ return get_repo_root() / "scripts" / "security_audit_ignores.txt"
+
+
+def cmd_validate_ignores(args: argparse.Namespace) -> int:
+ try:
+ ignores = load_ignores(get_ignores_path())
+ print(f"Validated {len(ignores)} ignore entries.")
+ return 0
+ except ValueError as e:
+ print(f"Ignore policy error: {e}", file=sys.stderr)
+ return 1
+
+
+def cmd_pnpm(args: argparse.Namespace) -> int:
+ ignores = load_ignores(get_ignores_path())
+ report = run_pnpm_audit(get_repo_root(), ignores)
+ emit_annotations(report)
+ emit_summary(report)
+ code = exit_code_for(report)
+ n = len(report.findings)
+ print(f"pnpm: {n} findings ({len(report.high_critical)} high/critical, "
+ f"{len(report.moderate)} moderate, {len(report.low_info)} low/info), "
+ f"{len(report.ignored)} ignored")
+ return code
+
+
+def cmd_python(args: argparse.Namespace) -> int:
+ ignores = load_ignores(get_ignores_path())
+ report = run_pip_audit(get_repo_root(), ignores)
+ emit_annotations(report)
+ emit_summary(report)
+ code = exit_code_for(report)
+ n = len(report.findings)
+ print(f"python: {n} findings, {len(report.ignored)} ignored")
+ return code
+
+
+def cmd_go(args: argparse.Namespace) -> int:
+ ignores = load_ignores(get_ignores_path())
+ report = run_govulncheck(get_repo_root(), ignores)
+ emit_annotations(report)
+ emit_summary(report)
+ code = exit_code_for(report)
+ n = len(report.findings)
+ print(f"go: {n} findings, {len(report.ignored)} ignored")
+ return code
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(
+ prog="security_audit",
+ description="Dependency CVE scanner for AiSOC CI",
+ )
+ sub = parser.add_subparsers(dest="command", required=True)
+
+ sub.add_parser("validate-ignores", help="Validate the ignores file")
+ sub.add_parser("pnpm", help="Audit pnpm dependencies")
+ sub.add_parser("python", help="Audit Python dependencies")
+ sub.add_parser("go", help="Audit Go dependencies")
+
+ args = parser.parse_args()
+
+ handlers = {
+ "validate-ignores": cmd_validate_ignores,
+ "pnpm": cmd_pnpm,
+ "python": cmd_python,
+ "go": cmd_go,
+ }
+
+ return handlers[args.command](args)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/security_audit_ignores.txt b/scripts/security_audit_ignores.txt
new file mode 100644
index 000000000..445691ad2
--- /dev/null
+++ b/scripts/security_audit_ignores.txt
@@ -0,0 +1,10 @@
+# Dependency vulnerability ignore list for AiSOC CI.
+#
+# Format: tool|ID|reason|YYYY-MM-DD
+# tool — pnpm, python, or go
+# ID — CVE-YYYY-NNNNN, GHSA-xxxx-xxxx-xxxx, GO-YYYY-NNNN, or PYSEC-YYYY-N
+# reason — why this ignore is acceptable (required, no blanks)
+# expiry — date this ignore expires (max 90 days from today)
+#
+# Example:
+# python|CVE-2026-12345|Upstream fix not released; tracked in issue #123|2026-07-15
diff --git a/scripts/tests/test_security_audit.py b/scripts/tests/test_security_audit.py
new file mode 100644
index 000000000..5fe1930f3
--- /dev/null
+++ b/scripts/tests/test_security_audit.py
@@ -0,0 +1,448 @@
+"""Tests for scripts/security_audit.py — ignore validation, classifiers, parsers."""
+
+from __future__ import annotations
+
+import datetime as dt
+import textwrap
+from pathlib import Path
+
+import pytest
+
+# Import the module under test
+import sys
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+from security_audit import (
+ Finding,
+ Ignore,
+ Report,
+ classify_govulncheck,
+ classify_pip_audit,
+ classify_pnpm_audit,
+ exit_code_for,
+ is_ignored,
+ load_ignores,
+ parse_govulncheck_json,
+ parse_pip_audit_json,
+)
+
+
+# ─── Helpers ──────────────────────────────────────────────────────────────────
+
+TODAY = dt.date(2026, 6, 1)
+VALID_EXPIRY = "2026-07-01" # 30 days out — within 90-day window
+
+
+def _write_ignores(tmp_path: Path, content: str) -> Path:
+ p = tmp_path / "ignores.txt"
+ p.write_text(textwrap.dedent(content))
+ return p
+
+
+# ─── load_ignores tests ──────────────────────────────────────────────────────
+
+
+class TestLoadIgnores:
+ def test_empty_file(self, tmp_path: Path):
+ p = _write_ignores(tmp_path, "")
+ assert load_ignores(p, today=TODAY) == []
+
+ def test_comments_and_blanks_skipped(self, tmp_path: Path):
+ p = _write_ignores(tmp_path, "# comment\n\n# another\n")
+ assert load_ignores(p, today=TODAY) == []
+
+ def test_valid_cve(self, tmp_path: Path):
+ p = _write_ignores(tmp_path, f"python|CVE-2026-12345|reason here|{VALID_EXPIRY}\n")
+ result = load_ignores(p, today=TODAY)
+ assert len(result) == 1
+ assert result[0].tool == "python"
+ assert result[0].vuln_id == "CVE-2026-12345"
+ assert result[0].reason == "reason here"
+ assert result[0].expires == dt.date(2026, 7, 1)
+
+ def test_valid_ghsa(self, tmp_path: Path):
+ p = _write_ignores(tmp_path, f"pnpm|GHSA-abcd-efgh-ijkl|reason|{VALID_EXPIRY}\n")
+ result = load_ignores(p, today=TODAY)
+ assert len(result) == 1
+ assert result[0].vuln_id == "GHSA-abcd-efgh-ijkl"
+
+ def test_valid_go_id(self, tmp_path: Path):
+ p = _write_ignores(tmp_path, f"go|GO-2026-1234|reason|{VALID_EXPIRY}\n")
+ result = load_ignores(p, today=TODAY)
+ assert len(result) == 1
+ assert result[0].vuln_id == "GO-2026-1234"
+
+ def test_valid_pysec(self, tmp_path: Path):
+ p = _write_ignores(tmp_path, f"python|PYSEC-2026-42|reason|{VALID_EXPIRY}\n")
+ result = load_ignores(p, today=TODAY)
+ assert len(result) == 1
+ assert result[0].vuln_id == "PYSEC-2026-42"
+
+ def test_wrong_field_count(self, tmp_path: Path):
+ p = _write_ignores(tmp_path, "python|CVE-2026-12345|reason\n")
+ with pytest.raises(ValueError, match="expected 4 pipe-delimited fields"):
+ load_ignores(p, today=TODAY)
+
+ def test_invalid_tool(self, tmp_path: Path):
+ p = _write_ignores(tmp_path, f"npm|CVE-2026-12345|reason|{VALID_EXPIRY}\n")
+ with pytest.raises(ValueError, match="invalid tool 'npm'"):
+ load_ignores(p, today=TODAY)
+
+ def test_invalid_vuln_id(self, tmp_path: Path):
+ p = _write_ignores(tmp_path, f"python|BADID-123|reason|{VALID_EXPIRY}\n")
+ with pytest.raises(ValueError, match="invalid ID 'BADID-123'"):
+ load_ignores(p, today=TODAY)
+
+ def test_empty_reason(self, tmp_path: Path):
+ p = _write_ignores(tmp_path, f"python|CVE-2026-12345||{VALID_EXPIRY}\n")
+ with pytest.raises(ValueError, match="reason must not be empty"):
+ load_ignores(p, today=TODAY)
+
+ def test_invalid_date(self, tmp_path: Path):
+ p = _write_ignores(tmp_path, "python|CVE-2026-12345|reason|not-a-date\n")
+ with pytest.raises(ValueError, match="invalid date"):
+ load_ignores(p, today=TODAY)
+
+ def test_expired(self, tmp_path: Path):
+ p = _write_ignores(tmp_path, "python|CVE-2026-12345|reason|2026-05-01\n")
+ with pytest.raises(ValueError, match="expired on 2026-05-01"):
+ load_ignores(p, today=TODAY)
+
+ def test_exceeds_90_days(self, tmp_path: Path):
+ far_future = "2026-12-01"
+ p = _write_ignores(tmp_path, f"python|CVE-2026-12345|reason|{far_future}\n")
+ with pytest.raises(ValueError, match="exceeds 90-day maximum"):
+ load_ignores(p, today=TODAY)
+
+ def test_missing_file_returns_empty(self, tmp_path: Path):
+ p = tmp_path / "does_not_exist.txt"
+ assert load_ignores(p, today=TODAY) == []
+
+ def test_multiple_valid_entries(self, tmp_path: Path):
+ content = (
+ f"pnpm|CVE-2026-11111|reason one|{VALID_EXPIRY}\n"
+ f"python|CVE-2026-22222|reason two|{VALID_EXPIRY}\n"
+ f"go|GO-2026-3333|reason three|{VALID_EXPIRY}\n"
+ )
+ p = _write_ignores(tmp_path, content)
+ result = load_ignores(p, today=TODAY)
+ assert len(result) == 3
+
+
+# ─── is_ignored tests ────────────────────────────────────────────────────────
+
+
+class TestIsIgnored:
+ def test_match(self):
+ ig = Ignore(tool="pnpm", vuln_id="CVE-2026-111", reason="r", expires=TODAY)
+ assert is_ignored("pnpm", ["CVE-2026-111"], [ig]) == ig
+
+ def test_no_match_different_tool(self):
+ ig = Ignore(tool="python", vuln_id="CVE-2026-111", reason="r", expires=TODAY)
+ assert is_ignored("pnpm", ["CVE-2026-111"], [ig]) is None
+
+ def test_no_match_different_id(self):
+ ig = Ignore(tool="pnpm", vuln_id="CVE-2026-111", reason="r", expires=TODAY)
+ assert is_ignored("pnpm", ["CVE-2026-999"], [ig]) is None
+
+ def test_match_among_multiple_ids(self):
+ ig = Ignore(tool="go", vuln_id="GO-2026-0001", reason="r", expires=TODAY)
+ assert is_ignored("go", ["CVE-2026-111", "GO-2026-0001"], [ig]) == ig
+
+
+# ─── classify_pnpm_audit tests ───────────────────────────────────────────────
+
+
+class TestClassifyPnpmAudit:
+ def test_empty_advisories(self):
+ report = classify_pnpm_audit({"advisories": {}}, [])
+ assert report.findings == []
+ assert report.ignored == []
+
+ def test_single_high(self):
+ data = {
+ "advisories": {
+ "100": {
+ "id": 100,
+ "severity": "high",
+ "module_name": "lodash",
+ "title": "Prototype Pollution",
+ "cves": ["CVE-2026-00100"],
+ "github_advisory_id": "GHSA-aaaa-bbbb-cccc",
+ }
+ }
+ }
+ report = classify_pnpm_audit(data, [])
+ assert len(report.findings) == 1
+ f = report.findings[0]
+ assert f.severity == "high"
+ assert f.package == "lodash"
+ assert f.vuln_id == "GHSA-aaaa-bbbb-cccc"
+
+ def test_moderate_finding(self):
+ data = {
+ "advisories": {
+ "200": {
+ "id": 200,
+ "severity": "moderate",
+ "module_name": "express",
+ "title": "Open redirect",
+ "cves": ["CVE-2026-00200"],
+ "github_advisory_id": "",
+ }
+ }
+ }
+ report = classify_pnpm_audit(data, [])
+ assert len(report.moderate) == 1
+ assert report.moderate[0].severity == "moderate"
+
+ def test_ignored_advisory(self):
+ data = {
+ "advisories": {
+ "300": {
+ "id": 300,
+ "severity": "critical",
+ "module_name": "foo",
+ "title": "RCE",
+ "cves": ["CVE-2026-00300"],
+ "github_advisory_id": "GHSA-xxxx-yyyy-zzzz",
+ }
+ }
+ }
+ ig = Ignore(tool="pnpm", vuln_id="CVE-2026-00300", reason="r", expires=TODAY)
+ report = classify_pnpm_audit(data, [ig])
+ assert len(report.findings) == 0
+ assert len(report.ignored) == 1
+
+ def test_unknown_severity_normalized(self):
+ data = {
+ "advisories": {
+ "400": {
+ "id": 400,
+ "severity": "banana",
+ "module_name": "bad",
+ "title": "weird",
+ "cves": [],
+ "github_advisory_id": "",
+ }
+ }
+ }
+ report = classify_pnpm_audit(data, [])
+ assert report.findings[0].severity == "unknown"
+
+
+# ─── parse_pip_audit_json tests ──────────────────────────────────────────────
+
+
+class TestParsePipAuditJson:
+ def test_empty_string(self):
+ assert parse_pip_audit_json("") == []
+
+ def test_list_format(self):
+ raw = '[{"name": "pkg", "version": "1.0", "vulns": []}]'
+ result = parse_pip_audit_json(raw)
+ assert len(result) == 1
+ assert result[0]["name"] == "pkg"
+
+ def test_dict_format(self):
+ raw = '{"dependencies": [{"name": "pkg", "version": "1.0", "vulns": []}]}'
+ result = parse_pip_audit_json(raw)
+ assert len(result) == 1
+
+ def test_dict_without_dependencies_key(self):
+ raw = '{"something_else": true}'
+ result = parse_pip_audit_json(raw)
+ assert result == []
+
+
+# ─── classify_pip_audit tests ────────────────────────────────────────────────
+
+
+class TestClassifyPipAudit:
+ def test_no_vulns(self):
+ deps = [{"name": "requests", "version": "2.31.0", "vulns": []}]
+ report = classify_pip_audit("services/api", deps, [])
+ assert report.findings == []
+
+ def test_single_vuln(self):
+ deps = [
+ {
+ "name": "requests",
+ "version": "2.25.0",
+ "vulns": [
+ {
+ "id": "PYSEC-2026-10",
+ "aliases": ["CVE-2026-99999"],
+ "description": "SSRF in requests",
+ }
+ ],
+ }
+ ]
+ report = classify_pip_audit("services/api", deps, [])
+ assert len(report.findings) == 1
+ f = report.findings[0]
+ assert f.severity == "high"
+ assert f.vuln_id == "PYSEC-2026-10"
+ assert f.package == "requests==2.25.0"
+ assert f.location == "services/api"
+
+ def test_vuln_id_used_when_aliases_empty(self):
+ deps = [
+ {
+ "name": "requests",
+ "version": "2.25.0",
+ "vulns": [
+ {
+ "id": "PYSEC-2026-10",
+ "aliases": [],
+ "description": "Regression: vuln_id must not fall through",
+ }
+ ],
+ }
+ ]
+ report = classify_pip_audit("services/api", deps, [])
+ assert report.findings[0].vuln_id == "PYSEC-2026-10"
+
+ def test_ignored_vuln(self):
+ deps = [
+ {
+ "name": "requests",
+ "version": "2.25.0",
+ "vulns": [
+ {
+ "id": "PYSEC-2026-10",
+ "aliases": ["CVE-2026-99999"],
+ "description": "SSRF",
+ }
+ ],
+ }
+ ]
+ ig = Ignore(tool="python", vuln_id="CVE-2026-99999", reason="r", expires=TODAY)
+ report = classify_pip_audit("services/api", deps, [ig])
+ assert len(report.findings) == 0
+ assert len(report.ignored) == 1
+
+
+# ─── parse_govulncheck_json tests ────────────────────────────────────────────
+
+
+class TestParseGovulncheckJson:
+ def test_empty(self):
+ assert parse_govulncheck_json("") == []
+
+ def test_osv_with_finding(self):
+ lines = [
+ '{"osv": {"id": "GO-2026-0001", "aliases": ["CVE-2026-11111"], "summary": "Bad thing"}}',
+ '{"finding": {"osv": "GO-2026-0001", "trace": []}}',
+ ]
+ result = parse_govulncheck_json("\n".join(lines))
+ assert len(result) == 1
+ assert result[0]["id"] == "GO-2026-0001"
+ assert result[0]["aliases"] == ["CVE-2026-11111"]
+
+ def test_osv_without_finding_excluded(self):
+ lines = [
+ '{"osv": {"id": "GO-2026-0002", "aliases": [], "summary": "Not called"}}',
+ ]
+ result = parse_govulncheck_json("\n".join(lines))
+ assert result == []
+
+ def test_malformed_lines_skipped(self):
+ lines = [
+ "not json",
+ '{"osv": {"id": "GO-2026-0003", "aliases": [], "summary": "yes"}}',
+ '{"finding": {"osv": "GO-2026-0003"}}',
+ ]
+ result = parse_govulncheck_json("\n".join(lines))
+ assert len(result) == 1
+
+
+# ─── classify_govulncheck tests ──────────────────────────────────────────────
+
+
+class TestClassifyGovulncheck:
+ def test_no_vulns(self):
+ report = classify_govulncheck("services/ingest", [], [])
+ assert report.findings == []
+
+ def test_single_vuln(self):
+ vulns = [{"id": "GO-2026-0001", "aliases": ["CVE-2026-55555"], "summary": "Bad"}]
+ report = classify_govulncheck("services/ingest", vulns, [])
+ assert len(report.findings) == 1
+ f = report.findings[0]
+ assert f.severity == "high"
+ assert f.vuln_id == "GO-2026-0001"
+
+ def test_ignored_vuln(self):
+ vulns = [{"id": "GO-2026-0001", "aliases": ["CVE-2026-55555"], "summary": "Bad"}]
+ ig = Ignore(tool="go", vuln_id="GO-2026-0001", reason="r", expires=TODAY)
+ report = classify_govulncheck("services/ingest", vulns, [ig])
+ assert len(report.findings) == 0
+ assert len(report.ignored) == 1
+
+
+# ─── Report + exit_code_for tests ────────────────────────────────────────────
+
+
+class TestExitCodeFor:
+ def test_no_findings(self):
+ assert exit_code_for(Report()) == 0
+
+ def test_only_moderate(self):
+ r = Report(findings=[Finding("pnpm", "moderate", "X", "p", "l", "t")])
+ assert exit_code_for(r) == 0
+
+ def test_only_low(self):
+ r = Report(findings=[Finding("pnpm", "low", "X", "p", "l", "t")])
+ assert exit_code_for(r) == 0
+
+ def test_high_fails(self):
+ r = Report(findings=[Finding("python", "high", "X", "p", "l", "t")])
+ assert exit_code_for(r) == 1
+
+ def test_critical_fails(self):
+ r = Report(findings=[Finding("go", "critical", "X", "p", "l", "t")])
+ assert exit_code_for(r) == 1
+
+ def test_mixed_high_and_moderate(self):
+ r = Report(
+ findings=[
+ Finding("pnpm", "moderate", "A", "p", "l", "t"),
+ Finding("pnpm", "high", "B", "p", "l", "t"),
+ ]
+ )
+ assert exit_code_for(r) == 1
+
+
+# ─── Report property tests ───────────────────────────────────────────────────
+
+
+class TestReportProperties:
+ def test_high_critical(self):
+ r = Report(
+ findings=[
+ Finding("a", "high", "1", "p", "l", "t"),
+ Finding("a", "critical", "2", "p", "l", "t"),
+ Finding("a", "moderate", "3", "p", "l", "t"),
+ ]
+ )
+ assert len(r.high_critical) == 2
+
+ def test_moderate(self):
+ r = Report(
+ findings=[
+ Finding("a", "moderate", "1", "p", "l", "t"),
+ Finding("a", "high", "2", "p", "l", "t"),
+ ]
+ )
+ assert len(r.moderate) == 1
+
+ def test_low_info(self):
+ r = Report(
+ findings=[
+ Finding("a", "low", "1", "p", "l", "t"),
+ Finding("a", "info", "2", "p", "l", "t"),
+ Finding("a", "unknown", "3", "p", "l", "t"),
+ ]
+ )
+ assert len(r.low_info) == 3
diff --git a/services/api/app/core/scheduler_lock.py b/services/api/app/core/scheduler_lock.py
new file mode 100644
index 000000000..17c5abd6b
--- /dev/null
+++ b/services/api/app/core/scheduler_lock.py
@@ -0,0 +1,153 @@
+"""Distributed scheduler lock helper for in-process API workers.
+
+Workers in ``app.main`` run in-process on every API replica. This helper
+provides a per-job Redis lease so only one replica executes a given worker at
+any time while keeping availability-first behavior:
+
+* lock held by another replica -> caller gets ``False`` and can skip silently.
+* Redis unavailable -> caller gets ``True`` (fail-open) and work proceeds.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import uuid
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager, suppress
+
+from redis.asyncio import Redis, from_url
+from redis.exceptions import RedisError
+
+logger = logging.getLogger(__name__)
+
+_redis_client: Redis | None = None
+_redis_client_lock = asyncio.Lock()
+
+_BACKEND_ERRORS = (RedisError, OSError, TimeoutError)
+
+_RENEW_IF_OWNER_SCRIPT = """
+if redis.call('get', KEYS[1]) == ARGV[1] then
+ return redis.call('expire', KEYS[1], ARGV[2])
+end
+return 0
+"""
+
+_RELEASE_IF_OWNER_SCRIPT = """
+if redis.call('get', KEYS[1]) == ARGV[1] then
+ return redis.call('del', KEYS[1])
+end
+return 0
+"""
+
+
+async def _get_redis_client() -> Redis:
+ from app.core.config import settings # lazy: only needed at first connect, not at import time
+
+ global _redis_client
+ if _redis_client is not None:
+ return _redis_client
+
+ async with _redis_client_lock:
+ if _redis_client is None:
+ _redis_client = from_url(str(settings.REDIS_URL), decode_responses=True)
+ return _redis_client
+
+
+def _warn_backend_unavailable(*, job_name: str, lock_key: str, phase: str, exc: Exception) -> None:
+ logger.warning(
+ "scheduler_lock.backend_unavailable job_name=%s lock_key=%s phase=%s error=%s",
+ job_name,
+ lock_key,
+ phase,
+ type(exc).__name__,
+ )
+
+
+async def _renew_lease_loop(
+ *,
+ redis_client: Redis,
+ job_name: str,
+ lock_key: str,
+ token: str,
+ ttl_seconds: int,
+ renew_every_seconds: int,
+) -> None:
+ while True:
+ await asyncio.sleep(renew_every_seconds)
+ try:
+ renewed = await redis_client.eval(
+ _RENEW_IF_OWNER_SCRIPT,
+ 1,
+ lock_key,
+ token,
+ str(ttl_seconds),
+ )
+ except _BACKEND_ERRORS as exc:
+ _warn_backend_unavailable(job_name=job_name, lock_key=lock_key, phase="renew", exc=exc)
+ return
+
+ if not renewed:
+ return
+
+
+@asynccontextmanager
+async def scheduler_lock(*, job_name: str, ttl_seconds: int) -> AsyncIterator[bool]:
+ """Yield whether the caller should run this scheduler job.
+
+ The lock key format is always ``scheduler_lock:{job_name}``.
+ """
+ if ttl_seconds <= 0:
+ raise ValueError("ttl_seconds must be > 0")
+
+ lock_key = f"scheduler_lock:{job_name}"
+ token = uuid.uuid4().hex
+ renew_task: asyncio.Task[None] | None = None
+ redis_client: Redis | None = None
+
+ try:
+ redis_client = await _get_redis_client()
+ acquired = await redis_client.set(lock_key, token, ex=ttl_seconds, nx=True)
+ except _BACKEND_ERRORS as exc:
+ _warn_backend_unavailable(job_name=job_name, lock_key=lock_key, phase="acquire", exc=exc)
+ yield True
+ return
+
+ if not acquired:
+ yield False
+ return
+
+ renew_every_seconds = max(1, ttl_seconds // 3)
+ renew_task = asyncio.create_task(
+ _renew_lease_loop(
+ redis_client=redis_client,
+ job_name=job_name,
+ lock_key=lock_key,
+ token=token,
+ ttl_seconds=ttl_seconds,
+ renew_every_seconds=renew_every_seconds,
+ ),
+ name=f"scheduler_lock_renew_{job_name}",
+ )
+
+ try:
+ yield True
+ finally:
+ if renew_task is not None:
+ renew_task.cancel()
+ with suppress(asyncio.CancelledError):
+ await renew_task
+
+ if redis_client is not None:
+ try:
+ await redis_client.eval(
+ _RELEASE_IF_OWNER_SCRIPT,
+ 1,
+ lock_key,
+ token,
+ )
+ except _BACKEND_ERRORS as exc:
+ _warn_backend_unavailable(job_name=job_name, lock_key=lock_key, phase="release", exc=exc)
+
+
+__all__ = ["scheduler_lock"]
diff --git a/services/api/app/main.py b/services/api/app/main.py
index 73853f445..bccfc6a67 100644
--- a/services/api/app/main.py
+++ b/services/api/app/main.py
@@ -3,7 +3,7 @@
import asyncio
import hmac
import time
-from collections.abc import AsyncGenerator
+from collections.abc import AsyncGenerator, Awaitable, Callable
from contextlib import asynccontextmanager
import structlog
@@ -20,6 +20,7 @@ from app.core.airgap import airgap_status
from app.core.config import is_dev_env, settings, warn_if_insecure_defaults
from app.core.cors import build_cors_kwargs
from app.core.logging import configure_logging
+from app.core.scheduler_lock import scheduler_lock
from app.core.telemetry import instrument_app
from app.db.clickhouse import close_clickhouse
from app.db.database import engine
@@ -37,6 +38,34 @@ _metrics_bearer = HTTPBearer(auto_error=False)
logger = structlog.get_logger(__name__)
+_SCHEDULER_LOCK_RETRY_SECONDS = 5
+
+# 15m covers slow OAuth due-batch refreshes while each provider call is bounded
+# by OAUTH_REFRESH_HTTP_TIMEOUT_SECONDS.
+_OAUTH_REFRESH_LOCK_TTL_SECONDS = 900
+# Weekly digest can fan out across tenants + PDF generation, so use a longer
+# lease to avoid duplicate report generation during slow runs.
+_WEEKLY_DIGEST_LOCK_TTL_SECONDS = 5400
+# Hunt sweep can execute multiple saved hunts + case opens in one tick; 5m
+# covers slow sweeps while still recovering quickly after replica loss.
+_HUNT_SCHEDULER_LOCK_TTL_SECONDS = 300
+
+
+async def _run_guarded_scheduler_worker(
+ *,
+ job_name: str,
+ ttl_seconds: int,
+ worker: Callable[[], Awaitable[None]],
+) -> None:
+ while True:
+ async with scheduler_lock(job_name=job_name, ttl_seconds=ttl_seconds) as should_run:
+ if not should_run:
+ await asyncio.sleep(_SCHEDULER_LOCK_RETRY_SECONDS)
+ continue
+ await worker()
+ return
+
+
# Prometheus metrics
REQUEST_COUNT = Counter(
"aisoc_http_requests_total",
@@ -110,7 +139,14 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
oauth_refresh_task: asyncio.Task | None = None
if settings.OAUTH_REFRESH_WORKER_ENABLED:
try:
- oauth_refresh_task = asyncio.create_task(run_oauth_refresh(), name="oauth_refresh_worker")
+ oauth_refresh_task = asyncio.create_task(
+ _run_guarded_scheduler_worker(
+ job_name="oauth_refresh",
+ ttl_seconds=_OAUTH_REFRESH_LOCK_TTL_SECONDS,
+ worker=run_oauth_refresh,
+ ),
+ name="oauth_refresh_worker",
+ )
logger.info("oauth_refresh worker started")
except Exception as exc:
logger.warning("oauth_refresh worker failed to start", error=str(exc))
@@ -122,7 +158,14 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
weekly_digest_task: asyncio.Task | None = None
if settings.WEEKLY_DIGEST_WORKER_ENABLED:
try:
- weekly_digest_task = asyncio.create_task(run_weekly_digest(), name="weekly_digest_worker")
+ weekly_digest_task = asyncio.create_task(
+ _run_guarded_scheduler_worker(
+ job_name="weekly_digest",
+ ttl_seconds=_WEEKLY_DIGEST_LOCK_TTL_SECONDS,
+ worker=run_weekly_digest,
+ ),
+ name="weekly_digest_worker",
+ )
logger.info("weekly_digest worker started")
except Exception as exc:
logger.warning("weekly_digest worker failed to start", error=str(exc))
@@ -133,7 +176,14 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
hunt_scheduler_task: asyncio.Task | None = None
if settings.HUNT_SCHEDULER_ENABLED:
try:
- hunt_scheduler_task = asyncio.create_task(run_hunt_scheduler(), name="hunt_scheduler_worker")
+ hunt_scheduler_task = asyncio.create_task(
+ _run_guarded_scheduler_worker(
+ job_name="hunt_scheduler",
+ ttl_seconds=_HUNT_SCHEDULER_LOCK_TTL_SECONDS,
+ worker=run_hunt_scheduler,
+ ),
+ name="hunt_scheduler_worker",
+ )
logger.info("hunt_scheduler worker started")
except Exception as exc:
logger.warning("hunt_scheduler worker failed to start", error=str(exc))
diff --git a/services/api/tests/test_scheduler_lock.py b/services/api/tests/test_scheduler_lock.py
new file mode 100644
index 000000000..eb1903773
--- /dev/null
+++ b/services/api/tests/test_scheduler_lock.py
@@ -0,0 +1,91 @@
+"""Unit tests for the scheduler singleton lock helper."""
+
+from __future__ import annotations
+
+import logging
+
+import app.core.scheduler_lock as scheduler_lock_module
+import pytest
+from redis.exceptions import RedisError
+
+
+class _FakeRedisClient:
+ def __init__(self, *, acquire_result: bool | None = True, acquire_exc: Exception | None = None) -> None:
+ self.acquire_result = acquire_result
+ self.acquire_exc = acquire_exc
+ self.set_calls: list[dict[str, object]] = []
+ self.eval_calls: list[tuple[str, int, tuple[object, ...]]] = []
+
+ async def set(self, name: str, value: str, *, ex: int, nx: bool) -> bool | None:
+ self.set_calls.append({"name": name, "value": value, "ex": ex, "nx": nx})
+ if self.acquire_exc is not None:
+ raise self.acquire_exc
+ return self.acquire_result
+
+ async def eval(self, script: str, numkeys: int, *args: object) -> int:
+ self.eval_calls.append((script, numkeys, args))
+ return 1
+
+
+async def test_scheduler_lock_acquired_runs_job(monkeypatch: pytest.MonkeyPatch) -> None:
+ fake_redis = _FakeRedisClient(acquire_result=True)
+
+ async def _fake_get_redis_client() -> _FakeRedisClient:
+ return fake_redis
+
+ monkeypatch.setattr(scheduler_lock_module, "_get_redis_client", _fake_get_redis_client)
+
+ ran = False
+ async with scheduler_lock_module.scheduler_lock(job_name="oauth_refresh", ttl_seconds=900) as should_run:
+ if should_run:
+ ran = True
+
+ assert ran is True
+ assert fake_redis.set_calls
+ assert fake_redis.set_calls[0]["name"] == "scheduler_lock:oauth_refresh"
+ assert fake_redis.set_calls[0]["ex"] == 900
+ assert fake_redis.set_calls[0]["nx"] is True
+
+
+async def test_scheduler_lock_not_acquired_skips_job(monkeypatch: pytest.MonkeyPatch) -> None:
+ fake_redis = _FakeRedisClient(acquire_result=None)
+
+ async def _fake_get_redis_client() -> _FakeRedisClient:
+ return fake_redis
+
+ monkeypatch.setattr(scheduler_lock_module, "_get_redis_client", _fake_get_redis_client)
+
+ ran = False
+ async with scheduler_lock_module.scheduler_lock(job_name="weekly_digest", ttl_seconds=5400) as should_run:
+ if should_run:
+ ran = True
+
+ assert ran is False
+ assert fake_redis.set_calls
+ assert fake_redis.set_calls[0]["name"] == "scheduler_lock:weekly_digest"
+ assert fake_redis.eval_calls == []
+
+
+async def test_scheduler_lock_backend_unavailable_fails_open(
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ fake_redis = _FakeRedisClient(acquire_exc=RedisError("redis unavailable"))
+
+ async def _fake_get_redis_client() -> _FakeRedisClient:
+ return fake_redis
+
+ monkeypatch.setattr(scheduler_lock_module, "_get_redis_client", _fake_get_redis_client)
+
+ ran = False
+ with caplog.at_level(logging.WARNING, logger="app.core.scheduler_lock"):
+ async with scheduler_lock_module.scheduler_lock(job_name="hunt_scheduler", ttl_seconds=300) as should_run:
+ if should_run:
+ ran = True
+
+ assert ran is True
+ assert "scheduler_lock.backend_unavailable" in caplog.text
+ assert "phase=acquire" in caplog.text