Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

10.4 KB
Raw Blame
#!/usr/bin/env python3
"""Generate Prometheus alerts from the SLO file, and gate the two together.

`docs/operations/slos.yaml` declares an availability target and a p95
latency budget for all 17 services. The alert rules alongside it use
hardcoded thresholds picked independently — a 5% error rate, a 2-second
p95 — that correspond to no objective in the file. So the SLOs were, as
the plan put it, objectives nothing alerts on, and the alerts were numbers
nothing justifies.

Generating one from the other makes the threshold traceable: an alert
fires because an objective is being missed, and changing the objective
changes the alert. Two numbers that are supposed to agree and are
maintained separately will not agree for long.

**Only scraped services get alerts.** Five of seventeen expose `/metrics`;
generating for all seventeen would produce twelve alerts that can never
fire, and an alert that can never fire is worse than none — it reads as
coverage. The uncovered services are reported instead.

**Burn-rate, not instantaneous.** A 99.9% target allows roughly 43 minutes
of error per month. Alerting the moment the rate exceeds 0.1% pages on
every transient blip; alerting on how fast the month's budget is being
consumed pages when the objective is actually at risk. Fast burn (2% of
budget in an hour) is a page; slow burn (10% in six hours) is a ticket.

Run:  python3 scripts/generate_slo_alerts.py [--check]
"""

from __future__ import annotations

import argparse
import re
import sys
from pathlib import Path

# `scripts/` is on sys.path when this file is run as a program, but not when a
# test loads it by path with importlib. gate_toolkit sits beside it either way.
sys.path.insert(0, str(Path(__file__).resolve().parent))

from gate_toolkit import repo_root, self_test_if_requested

self_test_if_requested(__file__)

try:
    import yaml
except ImportError:  # pragma: no cover
    print("PyYAML is required: pip install pyyaml", file=sys.stderr)
    raise SystemExit(2) from None

REPO_ROOT = repo_root()
SLOS = REPO_ROOT / "docs" / "operations" / "slos.yaml"
PROMETHEUS = REPO_ROOT / "infra" / "docker" / "prometheus.yml"
OUTPUT = REPO_ROOT / "infra" / "docker" / "alerts" / "slo.rules.yml"

HEADER = """# GENERATED by scripts/generate_slo_alerts.py — do not edit.
#
# Thresholds come from docs/operations/slos.yaml. Editing them here makes
# the alert disagree with the objective it exists to enforce, which is the
# state this file was written to end: the SLOs declared targets nothing
# alerted on, and the alerts used thresholds no objective justified.
#
# Only services that expose /metrics are covered. An alert on a service
# Prometheus does not scrape can never fire, and an alert that can never
# fire reads as coverage.
"""

#: Burn-rate windows. The pair is deliberate: fast burn catches an outage,
#: slow burn catches a persistent low-grade failure that would exhaust the
#: month without ever looking like an incident.
FAST_WINDOW = "1h"
FAST_BUDGET_FRACTION = 0.02
SLOW_WINDOW = "6h"
SLOW_BUDGET_FRACTION = 0.10


class _BlockDumper(yaml.SafeDumper):
    """Emit multi-line strings as literal blocks.

    A PromQL expression escaped into a quoted scalar — `"sum(rate(...))\n
    / clamp_min(...)"` — is unreadable, and these are rules an operator
    reads at three in the morning to work out why they were paged.
    """


def _block_str(dumper: yaml.SafeDumper, value: str):
    style = "|" if "\n" in value else None
    return dumper.represent_scalar("tag:yaml.org,2002:str", value, style=style)


_BlockDumper.add_representer(str, _block_str)


def scraped_jobs() -> set[str]:
    """Service names Prometheus actually scrapes, from the job names."""
    if not PROMETHEUS.exists():
        return set()
    jobs: set[str] = set()
    for match in re.finditer(r"^\s*-?\s*job_name:\s*(\S+)", PROMETHEUS.read_text(), re.M):
        job = match.group(1).strip().strip("\"'")
        jobs.add(job.removeprefix("aisoc-"))
    return jobs


def burn_rate_threshold(availability_target: float, budget_fraction: float, window_hours: float) -> float:
    """Error rate that consumes `budget_fraction` of a 30-day budget in `window_hours`.

    Standard multi-window burn rate: the budget is
    ``1 - target`` over 30 days, so consuming a fraction of it in a
    shorter window means erroring at a proportionally higher rate.
    """
    error_budget = 1.0 - availability_target
    month_hours = 30 * 24
    return error_budget * budget_fraction * (month_hours / window_hours)


def rules_for(service: str, slo: dict) -> list[dict]:
    target = float(slo.get("availability_target", 0.999))
    latency_ms = slo.get("latency_p95_ms")
    job = f"aisoc-{service}"

    fast = burn_rate_threshold(target, FAST_BUDGET_FRACTION, 1)
    slow = burn_rate_threshold(target, SLOW_BUDGET_FRACTION, 6)

    rules: list[dict] = [
        {
            "alert": f"Slo{service.title().replace('-', '')}ErrorBudgetFastBurn",
            "expr": (
                f'sum(rate(aisoc_http_requests_total{{job="{job}",status=~"5.."}}[{FAST_WINDOW}]))\n'
                f'  / clamp_min(sum(rate(aisoc_http_requests_total{{job="{job}"}}[{FAST_WINDOW}])), 0.001)\n'
                f"  > {fast:.6f}"
            ),
            "for": "5m",
            "labels": {"severity": "page", "service": service, "slo": "availability"},
            "annotations": {
                "summary": f"{service} is burning its error budget fast",
                "description": (
                    f"At this rate {service} consumes "
                    f"{FAST_BUDGET_FRACTION:.0%} of its 30-day error budget "
                    f"every {FAST_WINDOW}. Target is {target:.3%} availability "
                    f"(docs/operations/slos.yaml)."
                ),
            },
        },
        {
            "alert": f"Slo{service.title().replace('-', '')}ErrorBudgetSlowBurn",
            "expr": (
                f'sum(rate(aisoc_http_requests_total{{job="{job}",status=~"5.."}}[{SLOW_WINDOW}]))\n'
                f'  / clamp_min(sum(rate(aisoc_http_requests_total{{job="{job}"}}[{SLOW_WINDOW}])), 0.001)\n'
                f"  > {slow:.6f}"
            ),
            "for": "30m",
            "labels": {"severity": "ticket", "service": service, "slo": "availability"},
            "annotations": {
                "summary": f"{service} is steadily burning its error budget",
                "description": (
                    f"Not an outage, which is why this is a ticket rather than "
                    f"a page — but at this rate {service} exhausts its 30-day "
                    f"budget without any single incident to point at."
                ),
            },
        },
    ]

    if latency_ms:
        rules.append(
            {
                "alert": f"Slo{service.title().replace('-', '')}LatencyBudgetMissed",
                "expr": (
                    f"histogram_quantile(0.95, sum by (le) "
                    f'(rate(aisoc_http_request_duration_seconds_bucket{{job="{job}"}}[30m])))\n'
                    f"  > {float(latency_ms) / 1000:.3f}"
                ),
                "for": "15m",
                "labels": {"severity": "ticket", "service": service, "slo": "latency"},
                "annotations": {
                    "summary": f"{service} p95 latency is over its objective",
                    "description": (
                        f"p95 above {latency_ms} ms, the target in "
                        f"docs/operations/slos.yaml. Measured over 30 minutes so "
                        f"a single slow request does not page."
                    ),
                },
            }
        )

    return rules


def build() -> tuple[str, list[str], list[str]]:
    """Returns (yaml text, covered services, services with no metrics)."""
    data = yaml.safe_load(SLOS.read_text(encoding="utf-8")) or {}
    services = data.get("services", {})
    jobs = scraped_jobs()

    covered: list[str] = []
    uncovered: list[str] = []
    rules: list[dict] = []

    for service, entry in sorted(services.items()):
        slo = (entry or {}).get("slo") or entry or {}
        if "availability_target" not in slo:
            # An exempt service declares no objective, so there is nothing
            # to alert on. Not a gap.
            continue
        if service not in jobs:
            uncovered.append(service)
            continue
        covered.append(service)
        rules.extend(rules_for(service, slo))

    document = {"groups": [{"name": "aisoc-slo", "rules": rules}]}
    text = HEADER + "\n" + yaml.dump(document, Dumper=_BlockDumper, sort_keys=False, width=1000)
    return text, covered, uncovered


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--check", action="store_true")
    args = parser.parse_args(argv)

    if not SLOS.exists():
        print("slo-alerts: docs/operations/slos.yaml not found", file=sys.stderr)
        return 2

    text, covered, uncovered = build()

    if not covered:
        print(
            "slo-alerts: no service with an SLO is scraped, so no alert could reference an objective",
            file=sys.stderr,
        )
        return 1

    if args.check:
        if not OUTPUT.exists():
            print(
                f"SLO ALERT GATE FAILED: {OUTPUT.relative_to(REPO_ROOT)} is missing.\n  Run: python3 scripts/generate_slo_alerts.py",
                file=sys.stderr,
            )
            return 1
        if OUTPUT.read_text(encoding="utf-8") != text:
            print(
                "SLO ALERT GATE FAILED: the generated alerts no longer match "
                "docs/operations/slos.yaml.\n"
                "  An objective changed and its alert did not, so the alert "
                "now enforces a number nothing declares.\n"
                "  Run: python3 scripts/generate_slo_alerts.py",
                file=sys.stderr,
            )
            return 1
        print(f"slo-alerts: OK — {len(covered)} service(s) alert against their declared objectives")
        if uncovered:
            print(f"  no alerts for {len(uncovered)} service(s) with an SLO but no /metrics: {', '.join(uncovered)}")
        return 0

    OUTPUT.parent.mkdir(parents=True, exist_ok=True)
    OUTPUT.write_text(text, encoding="utf-8")
    print(f"slo-alerts: wrote {len(covered)} service(s) to {OUTPUT.relative_to(REPO_ROOT)}")
    if uncovered:
        print(f"  skipped (SLO declared, no /metrics): {', '.join(uncovered)}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())