Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

34.5 KB
Raw Blame
"""Cloudflare Tunnel metrics parsing and shared quality evaluation."""

from __future__ import annotations

from collections import deque
from dataclasses import dataclass
from datetime import datetime, timezone
import math
import statistics
import time
from typing import Any

import requests
from prometheus_client.parser import text_string_to_metric_families


SAMPLE_INTERVAL_SECONDS = 15
RATE_WINDOW_SECONDS = 60
BASELINE_MINUTE_SAMPLES = 15
BASELINE_RETENTION_SAMPLES = 24 * 60
CANDIDATE_MIN_SAMPLES = 4
REQUEST_PATH_WINDOW_SECONDS = 180
REQUEST_PATH_MAX_SAMPLES = 48
REQUEST_PATH_MIN_SAMPLES = 12
REQUEST_PATH_HIGH_CONFIDENCE_SAMPLES = 30
REQUEST_BASELINE_RETENTION_SAMPLES = 2 * 24 * 60


def utc_timestamp(now: float) -> str:
    return datetime.fromtimestamp(now, timezone.utc).isoformat().replace("+00:00", "Z")


@dataclass(frozen=True)
class MetricsSample:
    sampled_at: float
    ready: bool
    # Connections the Cloudflare edge has registered, i.e. able to serve requests.
    ha_connections: int
    edge_locations: tuple[str, ...]
    smoothed_rtt_ms: tuple[float, ...]
    request_errors_total: float
    packet_loss_total: float
    closed_connections_total: float
    timeout_packet_loss_by_connection: tuple[tuple[str, float], ...] = ()


@dataclass(frozen=True)
class RequestPathSample:
    sampled_at: float
    latency_ms: tuple[float, ...]
    successes: tuple[bool, ...]


def parse_metrics(
    text: str,
    *,
    ready: bool = True,
    registered_connections: int | None = None,
    now: float | None = None,
) -> MetricsSample:
    """Parse cloudflared Prometheus metrics without depending on sample order.

    ``registered_connections`` is the edge-registered count from cloudflared's
    ``/ready`` endpoint. The ``cloudflared_tunnel_ha_connections`` gauge is only
    a fallback: cloudflared raises it when a connection starts dialing, so it
    reports four while the edge may have registered just one.
    """

    values: dict[str, list[Any]] = {}
    for family in text_string_to_metric_families(text):
        for sample in family.samples:
            values.setdefault(sample.name, []).append(sample)

    def scalar(*names: str) -> float:
        for name in names:
            if name in values:
                return sum(float(sample.value) for sample in values[name])
        return 0.0

    rtt_values = sorted(
        float(sample.value)
        for sample in values.get("quic_client_smoothed_rtt", [])
        if math.isfinite(float(sample.value)) and float(sample.value) >= 0
    )
    locations = sorted(
        {
            str(sample.labels.get("edge_location", "")).lower()
            for sample in values.get("cloudflared_tunnel_server_locations", [])
            if float(sample.value) > 0 and sample.labels.get("edge_location")
        }
    )
    timeout_losses: dict[str, float] = {}
    loss_samples = values.get("quic_client_lost_packets_total") or values.get("quic_client_lost_packets") or []
    for sample in loss_samples:
        connection = sample.labels.get("conn_index")
        reason = str(sample.labels.get("reason") or "").lower()
        if connection is None or reason != "timeout":
            continue
        connection_key = str(connection)
        timeout_losses[connection_key] = timeout_losses.get(connection_key, 0.0) + float(sample.value)
    return MetricsSample(
        sampled_at=now if now is not None else time.time(),
        ready=ready,
        ha_connections=(
            registered_connections
            if registered_connections is not None
            else max(0, int(round(scalar("cloudflared_tunnel_ha_connections"))))
        ),
        edge_locations=tuple(locations[:4]),
        smoothed_rtt_ms=tuple(rtt_values[:4]),
        request_errors_total=max(
            0.0,
            scalar("cloudflared_tunnel_request_errors_total", "cloudflared_tunnel_request_errors"),
        ),
        packet_loss_total=max(0.0, sum(timeout_losses.values())),
        closed_connections_total=max(
            0.0,
            scalar("quic_client_closed_connections_total", "quic_client_closed_connections"),
        ),
        timeout_packet_loss_by_connection=tuple(sorted(timeout_losses.items())),
    )


def scrape_metrics(metrics_url: str, *, timeout: float = 0.5, now: float | None = None) -> MetricsSample:
    base_url = metrics_url.rstrip("/")
    ready_response = requests.get(f"{base_url}/ready", timeout=timeout)
    metrics_response = requests.get(f"{base_url}/metrics", timeout=timeout)
    metrics_response.raise_for_status()
    return parse_metrics(
        metrics_response.text,
        ready=ready_response.ok,
        registered_connections=_ready_connections(ready_response),
        now=now,
    )


def _ready_connections(response: requests.Response) -> int | None:
    """Read ``readyConnections`` from cloudflared's ``/ready`` JSON body."""

    try:
        payload = response.json()
    except ValueError:
        return None
    value = payload.get("readyConnections") if isinstance(payload, dict) else None
    if isinstance(value, bool) or not isinstance(value, int) or value < 0:
        return None
    return value


def probe_request_path(
    session: requests.Session,
    url: str,
    *,
    attempts: int = 3,
    timeout: float = 3.5,
    now: float | None = None,
) -> RequestPathSample:
    """Measure a bounded public health window through one persistent session."""

    latency_values: list[float] = []
    successes: list[bool] = []
    for _ in range(max(1, attempts)):
        started_at = time.perf_counter()
        success = False
        try:
            response = session.get(
                url,
                timeout=timeout,
                allow_redirects=False,
                headers={"Accept": "application/json", "Cache-Control": "no-cache"},
            )
            success = 200 <= response.status_code < 300
            response.close()
        except requests.RequestException:
            pass
        latency_values.append(round((time.perf_counter() - started_at) * 1000, 1))
        successes.append(success)
    return RequestPathSample(
        sampled_at=now if now is not None else time.time(),
        latency_ms=tuple(latency_values),
        successes=tuple(successes),
    )


def rtt_stats(values: tuple[float, ...] | list[float]) -> dict[str, float] | None:
    if not values:
        return None
    return {
        "min": round(min(values), 1),
        "median": round(float(statistics.median(values)), 1),
        "max": round(max(values), 1),
    }


def latency_grade(rtt: dict[str, float] | None) -> str:
    if rtt is None:
        return "unknown"
    median = rtt["median"]
    maximum = rtt["max"]
    if median < 120 and maximum < 250:
        return "good"
    if median < 200 and maximum < 400:
        return "fair"
    if median < 350 and maximum < 700:
        return "poor"
    return "critical"


def _percentile(values: list[float], percentile: float) -> float | None:
    if not values:
        return None
    ordered = sorted(values)
    index = max(0, min(len(ordered) - 1, math.ceil(percentile * len(ordered)) - 1))
    return round(ordered[index], 1)


def request_path_grade(request_path: dict[str, Any] | None) -> str:
    if not request_path or request_path.get("confidence") == "low":
        return "unknown"
    latency = request_path.get("latency_ms")
    if not isinstance(latency, dict):
        return "critical" if request_path.get("status") == "unavailable" else "unknown"
    p95 = float(latency.get("p95") or 0)
    p99 = float(latency.get("p99") or 0)
    slow = request_path.get("slow_request_rate") or {}
    over_one_second = float(slow.get("over_1000_ms") or 0)
    failure_rate = float(request_path.get("failure_rate") or 0)
    if failure_rate >= 0.10:
        return "critical"
    if p95 < 350 and p99 < 750 and over_one_second < 0.02:
        return "good"
    if p95 < 600 and p99 < 1200 and over_one_second < 0.05:
        return "fair"
    if p95 < 1000 and p99 < 2000 and over_one_second < 0.10:
        return "poor"
    return "critical"


def request_path_is_degraded(request_path: dict[str, Any] | None) -> bool:
    if not request_path or request_path.get("confidence") != "high":
        return False
    if request_path.get("status") == "unavailable":
        return True
    latency = request_path.get("latency_ms")
    if not isinstance(latency, dict):
        return False
    p95 = float(latency.get("p95") or 0)
    p99 = float(latency.get("p99") or 0)
    slow = request_path.get("slow_request_rate") or {}
    over_one_second = float(slow.get("over_1000_ms") or 0)
    failure_rate = float(request_path.get("failure_rate") or 0)
    baseline = request_path.get("baseline_p95_ms")
    baseline_regression = baseline is not None and p95 >= 2 * float(baseline)
    return (
        failure_rate >= 0.10
        or p95 >= 750
        or baseline_regression
        or over_one_second >= 0.05
        or (p99 >= 1500 and over_one_second >= 0.03)
    )


def request_path_needs_recovery(
    request_path: dict[str, Any] | None,
    *,
    profile: str = "balanced",
) -> bool:
    """Apply a user-facing optimization policy to high-confidence path evidence."""

    if not request_path or request_path.get("confidence") != "high":
        return False
    if request_path.get("status") == "unavailable":
        return True
    latency = request_path.get("latency_ms")
    if not isinstance(latency, dict):
        return False
    p95 = float(latency.get("p95") or 0)
    p99 = float(latency.get("p99") or 0)
    slow = request_path.get("slow_request_rate") or {}
    over_500_ms = float(slow.get("over_500_ms") or 0)
    over_one_second = float(slow.get("over_1000_ms") or 0)
    failure_rate = float(request_path.get("failure_rate") or 0)
    baseline = request_path.get("baseline_p95_ms")
    baseline_value = float(baseline) if baseline is not None else None

    if failure_rate >= 0.10:
        return True
    if profile == "stable":
        return (
            p95 >= 1200
            or (baseline_value is not None and p95 >= 3 * baseline_value)
            or over_one_second >= 0.10
            or (p99 >= 2400 and over_one_second >= 0.05)
        )
    if profile == "low_latency":
        return (
            p95 >= 500
            or (baseline_value is not None and p95 >= 1.5 * baseline_value)
            or over_500_ms >= 0.20
            or over_one_second >= 0.03
            or p99 >= 1000
        )
    return request_path_is_degraded(request_path)


def route_unavailable(snapshot: dict[str, Any]) -> bool:
    """Whether the active route serves no traffic.

    cloudflared can keep reporting ready connections after its transport has
    silently died, so a high-confidence window with no successful public probe
    is treated as an outage alongside zero ready connections.
    """

    if int(snapshot.get("ha_connections") or 0) == 0:
        return True
    request_path = snapshot.get("request_path")
    return (
        isinstance(request_path, dict)
        and request_path.get("confidence") == "high"
        and request_path.get("status") == "unavailable"
    )


def request_path_has_usable_latency(request_path: dict[str, Any] | None) -> bool:
    if not request_path or request_path.get("confidence") == "low":
        return False
    if request_path.get("status") == "unavailable" or int(request_path.get("success_count") or 0) == 0:
        return False
    return isinstance(request_path.get("latency_ms"), dict)


def summarize_request_path_samples(
    samples: list[RequestPathSample] | tuple[RequestPathSample, ...],
    *,
    baseline_p95_ms: float | None = None,
) -> dict[str, Any] | None:
    latencies: list[float] = []
    successes: list[bool] = []
    for sample in samples:
        count = min(len(sample.latency_ms), len(sample.successes))
        for index in range(count):
            latency = float(sample.latency_ms[index])
            if not math.isfinite(latency) or latency < 0:
                continue
            latencies.append(latency)
            successes.append(bool(sample.successes[index]))
    sample_count = len(latencies)
    if sample_count == 0:
        return None
    success_count = sum(successes)
    confidence = (
        "high"
        if sample_count >= REQUEST_PATH_HIGH_CONFIDENCE_SAMPLES
        else "medium"
        if sample_count >= REQUEST_PATH_MIN_SAMPLES
        else "low"
    )
    latency = {
        "p50": _percentile(latencies, 0.50),
        "p95": _percentile(latencies, 0.95),
        "p99": _percentile(latencies, 0.99),
        "max": round(max(latencies), 1),
    }
    failure_rate = round((sample_count - success_count) / sample_count, 4)
    slow_request_rate = {
        "over_500_ms": round(sum(value > 500 for value in latencies) / sample_count, 4),
        "over_1000_ms": round(sum(value > 1000 for value in latencies) / sample_count, 4),
        "over_2000_ms": round(sum(value > 2000 for value in latencies) / sample_count, 4),
    }
    payload: dict[str, Any] = {
        "source": "synthetic_local",
        "status": "insufficient",
        "confidence": confidence,
        "window_seconds": REQUEST_PATH_WINDOW_SECONDS,
        "sample_count": sample_count,
        "success_count": success_count,
        "latency_ms": latency,
        "failure_rate": failure_rate,
        "slow_request_rate": slow_request_rate,
        "baseline_p95_ms": baseline_p95_ms,
    }
    if confidence != "low":
        if success_count == 0:
            payload["status"] = "unavailable"
        else:
            payload["status"] = "degraded" if request_path_is_degraded(payload) else "healthy"
    return payload


def request_path_is_better(active: dict[str, Any], candidate: dict[str, Any]) -> bool:
    active_latency = active.get("latency_ms")
    candidate_latency = candidate.get("latency_ms")
    if not isinstance(active_latency, dict) or not isinstance(candidate_latency, dict):
        return False
    candidate_count = int(candidate.get("sample_count") or 0)
    candidate_success = int(candidate.get("success_count") or 0)
    if candidate_count < REQUEST_PATH_MIN_SAMPLES or candidate_success / candidate_count < 0.95:
        return False
    active_p95 = float(active_latency.get("p95") or 0)
    active_p99 = float(active_latency.get("p99") or 0)
    candidate_p95 = float(candidate_latency.get("p95") or 0)
    candidate_p99 = float(candidate_latency.get("p99") or 0)
    active_slow = float((active.get("slow_request_rate") or {}).get("over_1000_ms") or 0)
    candidate_slow = float((candidate.get("slow_request_rate") or {}).get("over_1000_ms") or 0)
    active_failure = float(active.get("failure_rate") or 0)
    candidate_failure = float(candidate.get("failure_rate") or 0)
    p95_improved = active_p95 > 0 and candidate_p95 <= active_p95 * 0.80
    tail_improved = (
        active_p95 > 0
        and candidate_p95 <= active_p95 * 1.25
        and active_p99 > 0
        and candidate_p99 <= active_p99 * 0.60
    )
    slow_rate_improved = (
        active_slow >= 0.03
        and candidate_slow <= min(active_slow - 0.02, active_slow * 0.50)
    )
    failure_rate_improved = (
        active_failure >= 0.10
        and candidate_failure <= min(active_failure - 0.05, active_failure * 0.50)
    )
    return p95_improved or tail_improved or slow_rate_improved or failure_rate_improved


class QualityEvaluator:
    """Stateful evaluator shared by status, Doctor, reporting, and recovery."""

    def __init__(self) -> None:
        self._counter_samples: deque[MetricsSample] = deque(maxlen=8)
        self._baseline_samples: deque[tuple[float, float]] = deque(maxlen=BASELINE_RETENTION_SAMPLES)
        self._request_samples: deque[RequestPathSample] = deque(maxlen=REQUEST_PATH_MAX_SAMPLES)
        self._request_baseline_samples: deque[tuple[float, str, float]] = deque(
            maxlen=REQUEST_BASELINE_RETENTION_SAMPLES
        )
        self._last_baseline_minute: int | None = None
        self._last_request_baseline_minute: dict[str, int] = {}
        self._last_snapshot: dict[str, Any] | None = None
        self._latency_bad_samples = 0
        self._partial_samples = 0
        self._error_windows = 0
        self._loss_windows = 0
        self._metrics_failures = 0
        self._healthy_samples = 0

    @property
    def last_snapshot(self) -> dict[str, Any] | None:
        return self._last_snapshot

    @property
    def healthy_samples(self) -> int:
        return self._healthy_samples

    def reset_healthy_samples(self) -> None:
        self._healthy_samples = 0

    def reset_request_window(self) -> None:
        self._request_samples.clear()

    def export_state(self) -> dict[str, Any]:
        return {
            "baseline_samples": [[timestamp, value] for timestamp, value in self._baseline_samples],
            "last_baseline_minute": self._last_baseline_minute,
            "request_baseline_samples": [
                [timestamp, protocol, value]
                for timestamp, protocol, value in self._request_baseline_samples
            ],
            "last_request_baseline_minute": dict(self._last_request_baseline_minute),
        }

    def load_state(self, payload: dict[str, Any], *, now: float | None = None) -> None:
        cutoff = (now if now is not None else time.time()) - 24 * 60 * 60
        samples = payload.get("baseline_samples")
        if not isinstance(samples, list):
            return
        restored: list[tuple[float, float]] = []
        for item in samples[-BASELINE_RETENTION_SAMPLES:]:
            if not isinstance(item, list) or len(item) != 2:
                continue
            try:
                timestamp = float(item[0])
                value = float(item[1])
            except (TypeError, ValueError):
                continue
            if timestamp >= cutoff and value >= 0 and math.isfinite(value):
                restored.append((timestamp, value))
        self._baseline_samples = deque(restored, maxlen=BASELINE_RETENTION_SAMPLES)
        last_minute = payload.get("last_baseline_minute")
        self._last_baseline_minute = int(last_minute) if isinstance(last_minute, int) else None
        request_samples = payload.get("request_baseline_samples")
        restored_request_samples: list[tuple[float, str, float]] = []
        if isinstance(request_samples, list):
            for item in request_samples[-REQUEST_BASELINE_RETENTION_SAMPLES:]:
                if not isinstance(item, list) or len(item) != 3 or item[1] not in {"quic", "http2"}:
                    continue
                try:
                    timestamp = float(item[0])
                    value = float(item[2])
                except (TypeError, ValueError):
                    continue
                if timestamp >= cutoff and value >= 0 and math.isfinite(value):
                    restored_request_samples.append((timestamp, str(item[1]), value))
        self._request_baseline_samples = deque(
            restored_request_samples,
            maxlen=REQUEST_BASELINE_RETENTION_SAMPLES,
        )
        request_minutes = payload.get("last_request_baseline_minute")
        if isinstance(request_minutes, dict):
            self._last_request_baseline_minute = {
                protocol: int(value)
                for protocol, value in request_minutes.items()
                if protocol in {"quic", "http2"} and isinstance(value, int)
            }

    def baseline(self, now: float | None = None) -> float | None:
        cutoff = (now if now is not None else time.time()) - 24 * 60 * 60
        while self._baseline_samples and self._baseline_samples[0][0] < cutoff:
            self._baseline_samples.popleft()
        if len(self._baseline_samples) < BASELINE_MINUTE_SAMPLES:
            return None
        return _percentile([value for _, value in self._baseline_samples], 0.20)

    def request_baseline(self, protocol: str, now: float | None = None) -> float | None:
        cutoff = (now if now is not None else time.time()) - 24 * 60 * 60
        while self._request_baseline_samples and self._request_baseline_samples[0][0] < cutoff:
            self._request_baseline_samples.popleft()
        values = [
            value
            for _, sample_protocol, value in self._request_baseline_samples
            if sample_protocol == protocol
        ]
        if len(values) < BASELINE_MINUTE_SAMPLES:
            return None
        return _percentile(values, 0.20)

    def _request_path(
        self,
        sample: RequestPathSample | None,
        *,
        protocol: str,
        now: float,
    ) -> dict[str, Any] | None:
        if sample is not None:
            self._request_samples.append(sample)
        cutoff = now - REQUEST_PATH_WINDOW_SECONDS
        while self._request_samples and self._request_samples[0].sampled_at < cutoff:
            self._request_samples.popleft()
        return summarize_request_path_samples(
            list(self._request_samples),
            baseline_p95_ms=self.request_baseline(protocol, now) if protocol in {"quic", "http2"} else None,
        )

    def _record_request_baseline(
        self,
        request_path: dict[str, Any] | None,
        *,
        protocol: str,
        now: float,
    ) -> None:
        if (
            protocol not in {"quic", "http2"}
            or not request_path
            or request_path.get("status") != "healthy"
            or request_path.get("confidence") == "low"
        ):
            return
        latency = request_path.get("latency_ms")
        if not isinstance(latency, dict) or latency.get("p95") is None:
            return
        minute = int(now // 60)
        if self._last_request_baseline_minute.get(protocol) == minute:
            return
        self._request_baseline_samples.append((now, protocol, float(latency["p95"])))
        self._last_request_baseline_minute[protocol] = minute

    def _rates(self, sample: MetricsSample) -> tuple[float, float]:
        self._counter_samples.append(sample)
        cutoff = sample.sampled_at - RATE_WINDOW_SECONDS
        base = next((item for item in self._counter_samples if item.sampled_at >= cutoff), self._counter_samples[0])
        elapsed = sample.sampled_at - base.sampled_at
        if elapsed <= 0:
            return 0.0, 0.0
        scale = 60.0 / elapsed
        errors = max(0.0, sample.request_errors_total - base.request_errors_total) * scale
        current_losses = dict(sample.timeout_packet_loss_by_connection)
        base_losses = dict(base.timeout_packet_loss_by_connection)
        connection_losses = [
            max(0.0, current_losses[connection] - base_losses[connection]) * scale
            for connection in current_losses.keys() & base_losses.keys()
        ]
        losses = sum(connection_losses) if sum(loss > 0 for loss in connection_losses) >= 2 else 0.0
        return round(errors, 2), round(losses, 2)

    def update(
        self,
        sample: MetricsSample | None,
        *,
        connector_count: int = 1,
        recovery: dict[str, Any] | None = None,
        configured_protocol: str = "auto",
        effective_protocol: str | None = None,
        request_path_sample: RequestPathSample | None = None,
        now: float | None = None,
    ) -> dict[str, Any]:
        current_time = now if now is not None else (sample.sampled_at if sample else time.time())
        recovery_payload = recovery or empty_recovery()
        previous_state = str((self._last_snapshot or {}).get("state") or "unknown")
        previous_protocol = str((self._last_snapshot or {}).get("protocol") or "unknown")
        protocol = effective_protocol if effective_protocol in {"quic", "http2"} else previous_protocol
        if sample is not None:
            if effective_protocol not in {"quic", "http2"}:
                protocol = "quic" if sample.smoothed_rtt_ms else "http2" if sample.ready else "unknown"
        request_path = self._request_path(request_path_sample, protocol=protocol, now=current_time)

        if sample is None:
            self._metrics_failures += 1
            state = "unknown" if self._metrics_failures >= 3 else previous_state
            if state not in {"healthy", "degraded", "recovering"}:
                state = "unknown"
            previous_connections = int((self._last_snapshot or {}).get("ha_connections") or 0)
            previous_locations = list((self._last_snapshot or {}).get("edge_locations") or [])
            request_grade = request_path_grade(request_path)
            if request_path_is_degraded(request_path):
                state = "degraded"
            snapshot = {
                "schema_version": 2,
                "state": "recovering" if recovery_payload["state"] in {"evaluating", "draining"} else state,
                "grade": request_grade,
                "sampled_at": utc_timestamp(current_time),
                "protocol": protocol,
                "transport": {"configured": configured_protocol, "effective": protocol},
                "connector_count": max(0, connector_count),
                "ha_connections": previous_connections,
                "rtt_ms": None,
                "baseline_median_rtt_ms": self.baseline(current_time),
                "edge_locations": previous_locations,
                "window_seconds": RATE_WINDOW_SECONDS,
                "request_errors_per_minute": 0.0,
                "packet_loss_per_minute": 0.0,
                "request_path": request_path,
                "recovery": recovery_payload,
            }
            self._last_snapshot = snapshot
            return snapshot

        self._metrics_failures = 0
        rtt = rtt_stats(sample.smoothed_rtt_ms)
        connector_grade = latency_grade(rtt)
        request_grade = request_path_grade(request_path)
        grade = request_grade if request_grade != "unknown" else connector_grade
        errors_per_minute, loss_per_minute = self._rates(sample)

        self._latency_bad_samples = self._latency_bad_samples + 1 if connector_grade in {"poor", "critical"} else 0
        self._partial_samples = self._partial_samples + 1 if sample.ha_connections < 4 else 0
        self._error_windows = self._error_windows + 1 if errors_per_minute >= 3 else 0
        self._loss_windows = self._loss_windows + 1 if loss_per_minute >= 10 else 0

        degraded = (
            not sample.ready
            or sample.ha_connections == 0
            or self._partial_samples >= 4
            or self._error_windows >= 2
            or self._loss_windows >= 2
            or self._latency_bad_samples >= 12
            or request_path_is_degraded(request_path)
        )
        baseline = self.baseline(current_time)
        recovery_state = recovery_payload["state"]
        if recovery_state in {"evaluating", "draining"}:
            state = "recovering"
        elif degraded:
            state = "degraded"
            self._healthy_samples = 0
        elif previous_state in {"degraded", "recovering"}:
            healthy_rtt = (
                rtt is None or rtt["median"] < max(160.0, 1.5 * baseline)
                if baseline is not None
                else connector_grade in {"good", "fair", "unknown"}
            )
            healthy_request_path = not request_path_is_degraded(request_path)
            self._healthy_samples = self._healthy_samples + 1 if sample.ha_connections == 4 and errors_per_minute < 3 and loss_per_minute < 10 and healthy_rtt and healthy_request_path else 0
            state = "healthy" if self._healthy_samples >= 20 else "degraded"
        else:
            state = "healthy"
            self._healthy_samples += 1

        minute = int(current_time // 60)
        if (
            state == "healthy"
            and connector_grade in {"good", "fair"}
            and sample.ha_connections == 4
            and errors_per_minute < 3
            and loss_per_minute < 10
            and rtt is not None
            and minute != self._last_baseline_minute
        ):
            self._baseline_samples.append((current_time, rtt["median"]))
            self._last_baseline_minute = minute
            baseline = self.baseline(current_time)

        if state == "healthy":
            self._record_request_baseline(request_path, protocol=protocol, now=current_time)

        snapshot = {
            "schema_version": 2,
            "state": state,
            "grade": grade,
            "sampled_at": utc_timestamp(current_time),
            "protocol": protocol,
            "transport": {"configured": configured_protocol, "effective": protocol},
            "connector_count": max(0, connector_count),
            "ha_connections": min(4, sample.ha_connections),
            "rtt_ms": rtt,
            "baseline_median_rtt_ms": baseline,
            "edge_locations": list(sample.edge_locations),
            "window_seconds": RATE_WINDOW_SECONDS,
            "request_errors_per_minute": errors_per_minute,
            "packet_loss_per_minute": loss_per_minute,
            "request_path": request_path,
            "recovery": recovery_payload,
        }
        self._last_snapshot = snapshot
        return snapshot

    def recovery_trigger(
        self,
        snapshot: dict[str, Any] | None = None,
        *,
        profile: str = "balanced",
    ) -> str | None:
        current = snapshot or self._last_snapshot
        if not current or profile not in {"stable", "balanced", "low_latency"}:
            return None
        state = current.get("state")
        if state != "degraded" and profile != "low_latency":
            return None
        if state not in {"healthy", "degraded"}:
            return None
        if int(current.get("ha_connections") or 0) < 4 or route_unavailable(current):
            return "availability"
        if float(current.get("request_errors_per_minute") or 0) >= 3 or float(current.get("packet_loss_per_minute") or 0) >= 10:
            return "errors"
        if request_path_needs_recovery(current.get("request_path"), profile=profile):
            return "tail_latency"
        rtt = current.get("rtt_ms")
        if not isinstance(rtt, dict):
            return None
        median = float(rtt.get("median") or 0)
        maximum = float(rtt.get("max") or 0)
        baseline = current.get("baseline_median_rtt_ms")
        if profile == "stable":
            if baseline is None:
                return "latency" if median >= 450 or maximum >= 900 else None
            baseline_value = float(baseline)
            if median >= 450 or (median >= 350 and median >= 2.5 * baseline_value):
                return "latency"
            if maximum >= 900 or (maximum >= 700 and maximum >= 3.5 * baseline_value):
                return "latency"
            return None
        if profile == "low_latency":
            if baseline is None:
                return "latency" if median >= 180 or maximum >= 350 else None
            baseline_value = float(baseline)
            if median >= 250 or (median >= 160 and median >= 1.6 * baseline_value):
                return "latency"
            if maximum >= 450 or (maximum >= 300 and maximum >= 2.2 * baseline_value):
                return "latency"
            return None
        if baseline is None:
            return "latency" if median >= 250 or maximum >= 500 else None
        baseline_value = float(baseline)
        if median >= 350 or (median >= 200 and median >= 2 * baseline_value):
            return "latency"
        if maximum >= 700 or (maximum >= 400 and maximum >= 3 * baseline_value):
            return "latency"
        return None


def empty_recovery() -> dict[str, Any]:
    return {
        "state": "idle",
        "last_attempt_at": None,
        "last_trigger": None,
        "last_result": None,
        "previous_median_rtt_ms": None,
        "result_median_rtt_ms": None,
        "previous_protocol": None,
        "result_protocol": None,
        "previous_p95_ms": None,
        "result_p95_ms": None,
        "previous_p99_ms": None,
        "result_p99_ms": None,
        "next_attempt_at": None,
        "attempt_count_window": 0,
    }


def candidate_is_better(active: dict[str, Any], candidate: dict[str, Any], *, trigger: str) -> bool:
    if int(candidate.get("ha_connections") or 0) < 4:
        return False
    if float(candidate.get("request_errors_per_minute") or 0) > 0:
        return False
    if float(candidate.get("packet_loss_per_minute") or 0) > float(active.get("packet_loss_per_minute") or 0):
        return False
    if trigger == "availability" and int(active.get("ha_connections") or 0) < 4:
        return True
    if trigger == "tail_latency":
        active_path = active.get("request_path")
        candidate_path = candidate.get("request_path")
        return (
            isinstance(active_path, dict)
            and isinstance(candidate_path, dict)
            and request_path_is_better(active_path, candidate_path)
        )
    active_rtt = active.get("rtt_ms")
    candidate_rtt = candidate.get("rtt_ms")
    if trigger == "errors":
        if (
            isinstance(active_rtt, dict)
            and isinstance(candidate_rtt, dict)
            and float(candidate_rtt.get("max") or 0) > float(active_rtt.get("max") or 0)
        ):
            return False
        return (
            float(candidate.get("request_errors_per_minute") or 0)
            < float(active.get("request_errors_per_minute") or 0)
            or float(candidate.get("packet_loss_per_minute") or 0)
            < float(active.get("packet_loss_per_minute") or 0)
        )
    if not isinstance(active_rtt, dict) or not isinstance(candidate_rtt, dict):
        return False
    active_median = float(active_rtt.get("median") or 0)
    candidate_median = float(candidate_rtt.get("median") or 0)
    active_maximum = float(active_rtt.get("max") or 0)
    candidate_maximum = float(candidate_rtt.get("max") or 0)
    improvement = active_median - candidate_median
    return improvement >= 30 and candidate_median <= active_median * 0.75 and candidate_maximum <= active_maximum


def summarize_candidate_snapshots(snapshots: list[dict[str, Any]]) -> dict[str, Any] | None:
    """Conservatively summarize a candidate's evaluation window."""

    if len(snapshots) < CANDIDATE_MIN_SAMPLES:
        return None
    summary = dict(snapshots[-1])
    summary["ha_connections"] = min(int(snapshot.get("ha_connections") or 0) for snapshot in snapshots)
    summary["request_errors_per_minute"] = max(
        float(snapshot.get("request_errors_per_minute") or 0) for snapshot in snapshots
    )
    summary["packet_loss_per_minute"] = max(
        float(snapshot.get("packet_loss_per_minute") or 0) for snapshot in snapshots
    )
    rtt_samples = [snapshot.get("rtt_ms") for snapshot in snapshots]
    if all(isinstance(rtt, dict) for rtt in rtt_samples):
        typed_rtt_samples = [rtt for rtt in rtt_samples if isinstance(rtt, dict)]
        summary["rtt_ms"] = {
            "min": round(float(statistics.median(float(rtt["min"]) for rtt in typed_rtt_samples)), 1),
            "median": round(float(statistics.median(float(rtt["median"]) for rtt in typed_rtt_samples)), 1),
            "max": round(max(float(rtt["max"]) for rtt in typed_rtt_samples), 1),
        }
    else:
        summary["rtt_ms"] = None
    return summary