Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

17.2 KB
Raw Blame
#!/usr/bin/env python3
"""External, synthetic-only installed alert probe. No product imports or DB access."""
import argparse
import datetime
import ipaddress
import json
import os
from pathlib import Path
import subprocess
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

SOURCE = "2dcf23b9d75742049f72f7eac21b0bf7266ebf6e"
DIGEST = "sha256:d7d24aec91da45b901e7b1c4094d508b5f6c708dca114c0f4dbbdfddd86a82b4"
TEMPLATE = ('{"id":"{{.ID}}","resource":"{{.ResourceName}}",'
            '"start":"{{.StartTime}}","event":"{{.Event}}"}')


class NoRedirect(urllib.request.HTTPRedirectHandler):
    def redirect_request(self, req, fp, code, msg, headers, newurl):
        return None


def check(condition, message):
    if not condition:
        raise AssertionError(message)


def loopback_url(url):
    parsed = urllib.parse.urlsplit(url)
    check(parsed.scheme == "http" and not parsed.username and not parsed.password
          and not parsed.query and not parsed.fragment, "plain loopback HTTP URL required")
    check(ipaddress.ip_address(parsed.hostname).is_loopback, "non-loopback target refused")
    return url.rstrip("/")


def webhook_start(alert):
    # Published prepareWebhookData uses time.RFC3339, not RFC3339Nano.
    # Keep the full API value for incident queries; only project recipient matching.
    start = datetime.datetime.fromisoformat(alert["startTime"].replace("Z", "+00:00"))
    check(start.tzinfo is not None, "occurrence start lacks timezone")
    return start.isoformat(timespec="seconds").replace("+00:00", "Z")


def distinct_webhook_occurrences(old, current):
    check(webhook_start(old) != webhook_start(current),
          "occurrences indistinguishable at webhook timestamp precision")


def report(name, cpu):
    return {"agent": {"id": name, "type": "unified", "intervalSeconds": 5,
                      "commandsEnabled": False},
            "host": {"hostname": name, "platform": "linux", "cpuCount": 1},
            "metrics": {"cpuUsagePercent": cpu,
                        "memory": {"totalBytes": 1073741824, "usedBytes": 104857600,
                                   "freeBytes": 968884224, "usage": 10}},
            "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat()}


class Evidence:
    def __init__(self, directory):
        self.directory = Path(directory)
        self.directory.mkdir(mode=0o700, parents=True, exist_ok=False)
        self.lock = threading.Lock()
        self.counter = 0

    def save(self, label, data):
        with self.lock:
            self.counter += 1
            path = self.directory / ("%04d-%s.json" % (self.counter, label))
            path.write_text(json.dumps(data, indent=2) + "\n")


class Recipient:
    def __init__(self, evidence):
        self.evidence = evidence
        self.lock = threading.Lock()
        self.failures = 0
        self.receipts = []

    def mode(self, failures):
        with self.lock:
            self.failures = failures  # -1 fails indefinitely

    def accept(self, payload):
        with self.lock:
            status = 503 if self.failures else 200
            if self.failures > 0:
                self.failures -= 1
            record = {"monotonic": time.monotonic(), "status": status, "payload": payload}
            self.receipts.append(record)
            self.evidence.save("recipient", record)
            return status

    def matching(self, name, status=200):
        with self.lock:
            return [r for r in self.receipts if r["status"] == status
                    and r["payload"].get("resource") == name
                    and r["payload"].get("event") == "alert"]


def serve(recipient, port):
    class Handler(BaseHTTPRequestHandler):
        def log_message(self, *_):
            pass

        def do_POST(self):
            if self.path != "/receipt":
                self.send_error(404)
                return
            size = int(self.headers.get("Content-Length", "0"))
            if not 0 < size <= 65536:
                self.send_error(413)
                return
            try:
                payload = json.loads(self.rfile.read(size))
                check(isinstance(payload, dict), "object required")
            except (ValueError, AssertionError):
                self.send_error(400)
                return
            status = recipient.accept(payload)
            self.send_response(status)
            if status == 503:
                self.send_header("Retry-After", "0")
            self.end_headers()
            self.wfile.write(b"controlled recipient\n")

    server = ThreadingHTTPServer(("127.0.0.1", port), Handler)
    threading.Thread(target=server.serve_forever, daemon=True).start()
    return server


class Driver:
    def __init__(self, args, evidence, recipient):
        self.args, self.evidence, self.recipient = args, evidence, recipient
        self.base = loopback_url(args.base)
        self.token = os.environ["PULSE_ACCEPTANCE_TOKEN"]
        self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect())
        self.names = ["acceptance-" + args.run_id + "-" + x for x in ("retry", "terminal", "quiet")]
        self.results = []
        self.record_observations = False

    def api(self, path, body=None, method=None):
        data = None if body is None else json.dumps(body).encode()
        req = urllib.request.Request(self.base + path, data=data, method=method,
                                     headers={"X-API-Token": self.token,
                                              "Content-Type": "application/json"})
        try:
            with self.opener.open(req, timeout=10) as response:
                result = json.load(response)
        except urllib.error.HTTPError as error:
            # Do not retain error bodies that might echo authentication material.
            self.evidence.save("http-error", {"path": path, "status": error.code})
            raise RuntimeError("HTTP %s on %s" % (error.code, path)) from None
        if self.record_observations and path.startswith((
                "/api/alerts/active", "/api/alerts/history", "/api/alerts/incidents",
                "/api/notifications/delivery-log", "/api/notifications/health",
                "/api/notifications/queue/stats")):
            self.evidence.save("observation", {"path": path, "response": result})
        return result

    def snapshot(self, label, alert=None):
        data = {path: self.api("/api/" + path) for path in
                ("alerts/active", "alerts/history", "notifications/delivery-log?limit=200",
                 "notifications/health", "notifications/queue/stats")}
        if alert:
            query = urllib.parse.urlencode({"alertIdentifier": alert["id"],
                                            "started_at": alert["startTime"]})
            data["incident"] = self.api("/api/alerts/incidents?" + query)
        self.evidence.save(label, data)
        return data

    def wait(self, label, predicate, timeout=90, tick=None):
        deadline = time.monotonic() + timeout
        while True:
            if tick:
                tick()
            result = predicate()
            if result:
                self.results.append(label)
                self.evidence.save("assertion", {"check": label, "result": "pass"})
                return result
            if time.monotonic() >= deadline:
                raise TimeoutError(label)
            time.sleep(2)

    def ingest(self, name, cpu):
        response = self.api("/api/agents/agent/report", report(name, cpu))
        check(response.get("success") is True, "ingest rejected")
        check(response.get("serverVersion", "").lstrip("v") == "6.4.4-beta.2",
              "installed server version mismatch")

    def active(self, name):
        return next((a for a in (self.api("/api/alerts/active") or [])
                     if a["resourceName"] == name and a["type"] == "cpu"), None)

    def delivery(self, alert, outcome):
        entries = self.api("/api/notifications/delivery-log?limit=200")["entries"] or []
        return [e for e in entries if alert["id"] in (e["alertIds"] or [])
                and e["outcome"] == outcome]

    def received_occurrence(self, name, alert):
        return any(r["payload"].get("id") == alert["id"]
                   and r["payload"].get("start") == webhook_start(alert)
                   for r in self.recipient.matching(name))

    def retry_terminal(self, alert, terminal_ids):
        retry = self.api("/api/notifications/terminal-failures/retry", {})
        # Preserve the returned result even when the assertion fails.
        self.evidence.save("retry-response", retry)
        check(retry.get("affected", 0) >= 1, "no terminal retry admitted")
        self.wait("terminal-retry-sent", lambda: any(
            entry["notificationId"] in terminal_ids
            for entry in self.delivery(alert, "sent")))

    def run(self):
        # Operator supplies independent immutable image inspection, not an API version claim.
        identity = json.loads(Path(self.args.identity).read_text())
        check(identity["source"] == SOURCE and identity["image_digest"] == DIGEST,
              "wrong inspected artifact")
        self.evidence.save("identity-owner-attestation", identity)
        check(not self.api("/api/notifications/webhooks"), "fixture already has webhooks")
        check(not self.api("/api/alerts/active"), "fixture already has alerts")
        check(not self.api("/api/alerts/history"), "fixture already has alert history")
        check(not self.api("/api/notifications/delivery-log")["entries"], "fixture has deliveries")
        for channel in ("email", "apprise"):
            check(not self.api("/api/notifications/" + channel).get("enabled"),
                  "fixture has enabled external destination")
        self.record_observations = True
        config = self.api("/api/alerts/config")
        config.update(enabled=True, activationState="active", disableAllAgents=False,
                      disableAllAgentsOffline=True, flappingEnabled=False,
                      agentDefaults={"cpu": {"trigger": 80, "clear": 60}},
                      timeThresholds={"agent": 0},
                      metricTimeThresholds={"agent": {"cpu": 0}},
                      metricEvaluationWindows={"agent": {"cpu": 0}})
        config["schedule"] = {"initialNotify": "webhook", "cooldown": 0,
                              "maxAlertsHour": 60, "notifyOnResolve": False,
                              "grouping": {"enabled": False},
                              "quietHours": {"enabled": False},
                              "escalation": {"enabled": False}}
        self.api("/api/alerts/config", config, "PUT")
        self.api("/api/notifications/webhooks",
                 {"id": "acceptance-" + self.args.run_id, "name": "Synthetic acceptance",
                  "url": "http://127.0.0.1:%d/receipt" % self.args.port,
                  "enabled": True, "method": "POST", "service": "generic",
                  "template": TEMPLATE})
        first, terminal, quiet = self.names
        self.recipient.mode(1)
        alert = self.wait("firing", lambda: self.active(first),
                          tick=lambda: self.ingest(first, 95))
        self.wait("ordinary-recipient-after-transient", lambda: self.recipient.matching(first))
        check(self.recipient.matching(first, 503), "transient failure not observed")
        self.wait("sent-audit", lambda: self.delivery(alert, "sent"))
        self.snapshot("firing-transient", alert)

        self.recipient.mode(-1)
        old = self.wait("terminal-firing", lambda: self.active(terminal),
                        tick=lambda: self.ingest(terminal, 95))
        terminal_entries = self.wait("exhaustion", lambda: self.delivery(old, "dead_letter"),
                  timeout=600, tick=lambda: self.ingest(terminal, 95))
        terminal_ids = {entry["notificationId"] for entry in terminal_entries}
        before = self.snapshot("before-restart", old)
        # This hook belongs to Delivery and must restart only the disposable server.
        proc = subprocess.run([self.args.restart_hook], timeout=120, capture_output=True)
        check(proc.returncode == 0, "restart hook failed (retain owner logs separately)")
        receipt = json.loads(proc.stdout)
        check(receipt["before_process"] != receipt["after_process"]
              and receipt["same_volume"] is True and receipt["image_digest"] == DIGEST,
              "restart owner receipt does not establish changed process / preserved volume")
        self.evidence.save("restart-owner-attestation", receipt)
        # Hook must return only after authenticated readiness; an error is not retried away.
        after = self.snapshot("after-restart", old)
        check(after["incident"]["id"] == before["incident"]["id"], "incident changed at restart")
        check(self.delivery(old, "dead_letter"), "terminal audit lost at restart")
        self.results.append("restart-preserved-terminal-and-incident")

        # Retry while the occurrence is still firing. Resolution intentionally
        # cancels obsolete terminal firing rows; a later recurrence cannot revive them.
        self.recipient.mode(0)
        self.retry_terminal(old, terminal_ids)
        self.wait("terminal-retry-recipient", lambda: self.received_occurrence(terminal, old))
        self.snapshot("old-occurrence-after-retry", old)
        self.wait("resolved", lambda: not self.active(terminal),
                  tick=lambda: self.ingest(terminal, 10))
        current = self.wait("recurrence", lambda: self.active(terminal),
                            tick=lambda: self.ingest(terminal, 95))
        check(current["startTime"] != old["startTime"], "recurrence reused old start")
        distinct_webhook_occurrences(old, current)
        self.wait("recurrence-recipient", lambda: self.received_occurrence(terminal, current))
        self.snapshot("old-occurrence-after-recurrence", old)
        self.snapshot("current-occurrence-after-recurrence", current)
        # Occurrence-side-effect correctness requires review of both retained timelines.
        config["disableAllAgents"] = True
        self.api("/api/alerts/config", config, "PUT")
        deadline = time.monotonic() + 30
        while time.monotonic() < deadline:
            self.ingest(quiet, 95)
            check(not self.active(quiet), "disabled-agent alert fired")
            check(not self.recipient.matching(quiet), "disabled-agent notification delivered")
            time.sleep(2)
        self.results.append("disabled-agent-suppression-30s")
        config["disableAllAgents"] = False
        self.api("/api/alerts/config", config, "PUT")
        self.recipient.mode(-1)
        dismissed = self.wait("dismissal-firing", lambda: self.active(quiet),
                              tick=lambda: self.ingest(quiet, 95))
        self.wait("dismissal-exhaustion", lambda: self.delivery(dismissed, "dead_letter"),
                  timeout=600, tick=lambda: self.ingest(quiet, 95))
        saved = self.snapshot("before-dismiss", dismissed)
        dismissal = self.api("/api/notifications/terminal-failures/dismiss", {})
        check(dismissal.get("affected", 0) >= 1, "nothing dismissed")
        self.evidence.save("dismiss-response", dismissal)
        self.wait("dismissal-health", lambda: self.api("/api/notifications/health")["overall_healthy"])
        retained = self.snapshot("after-dismiss", dismissed)
        def audit_keys(snapshot):
            return {(e["notificationId"], e["timestamp"], e["outcome"])
                    for e in snapshot["notifications/delivery-log?limit=200"]["entries"]}
        check(audit_keys(saved) <= audit_keys(retained), "dismissal lost delivery history")
        self.results.append("dismissal-retains-audit")
        self.snapshot("final")
        return {"passed": self.results, "installed_acceptance_complete": False,
                "requires_review": ["occurrence-specific replay side effects",
                                    "system notification-warning retirement",
                                    "email/provider acceptance", "reporter acceptance"]}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--base", required=True)
    parser.add_argument("--identity", required=True)
    parser.add_argument("--restart-hook", required=True)
    parser.add_argument("--evidence", required=True)
    parser.add_argument("--run-id", required=True)
    parser.add_argument("--port", type=int, default=18765)
    parser.add_argument("--disposable-fixture", action="store_true", required=True)
    args = parser.parse_args()
    check(args.run_id.isalnum() and len(args.run_id) <= 24, "short alphanumeric run ID required")
    loopback_url(args.base)
    os.umask(0o077)
    evidence = Evidence(args.evidence)
    recipient = Recipient(evidence)
    server = serve(recipient, args.port)
    driver = None
    try:
        driver = Driver(args, evidence, recipient)
        result = driver.run()
        evidence.save("result", result)
    except Exception as error:
        evidence.save("result", {"installed_acceptance_complete": False,
                                 "passed": driver.results if driver else [],
                                 "failure_type": type(error).__name__})
        raise
    finally:
        server.shutdown()
        server.server_close()


if __name__ == "__main__":
    main()