Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
"""Cloudflare Tunnel metrics parsing and shared quality evaluation."""
from __future__ import annotations
from collections import deque
from dataclasses import dataclass
from datetime import datetime, timezone
import math
import statistics
import time
from typing import Any
import requests
from prometheus_client.parser import text_string_to_metric_families
SAMPLE_INTERVAL_SECONDS = 15
RATE_WINDOW_SECONDS = 60
BASELINE_MINUTE_SAMPLES = 15
BASELINE_RETENTION_SAMPLES = 24 * 60
CANDIDATE_MIN_SAMPLES = 4
REQUEST_PATH_WINDOW_SECONDS = 180
REQUEST_PATH_MAX_SAMPLES = 48
REQUEST_PATH_MIN_SAMPLES = 12
REQUEST_PATH_HIGH_CONFIDENCE_SAMPLES = 30
REQUEST_BASELINE_RETENTION_SAMPLES = 2 * 24 * 60
def utc_timestamp(now: float) -> str:
return datetime.fromtimestamp(now, timezone.utc).isoformat().replace("+00:00", "Z")
@dataclass(frozen=True)
class MetricsSample:
sampled_at: float
ready: bool
# Connections the Cloudflare edge has registered, i.e. able to serve requests.
ha_connections: int
edge_locations: tuple[str, ...]
smoothed_rtt_ms: tuple[float, ...]
request_errors_total: float
packet_loss_total: float
closed_connections_total: float
timeout_packet_loss_by_connection: tuple[tuple[str, float], ...] = ()
@dataclass(frozen=True)
class RequestPathSample:
sampled_at: float
latency_ms: tuple[float, ...]
successes: tuple[bool, ...]
def parse_metrics(
text: str,
*,
ready: bool = True,
registered_connections: int | None = None,
now: float | None = None,
) -> MetricsSample:
"""Parse cloudflared Prometheus metrics without depending on sample order.
``registered_connections`` is the edge-registered count from cloudflared's
``/ready`` endpoint. The ``cloudflared_tunnel_ha_connections`` gauge is only
a fallback: cloudflared raises it when a connection starts dialing, so it
reports four while the edge may have registered just one.
"""
values: dict[str, list[Any]] = {}
for family in text_string_to_metric_families(text):
for sample in family.samples:
values.setdefault(sample.name, []).append(sample)
def scalar(*names: str) -> float:
for name in names:
if name in values:
return sum(float(sample.value) for sample in values[name])
return 0.0
rtt_values = sorted(
float(sample.value)
for sample in values.get("quic_client_smoothed_rtt", [])
if math.isfinite(float(sample.value)) and float(sample.value) >= 0
)
locations = sorted(
{
str(sample.labels.get("edge_location", "")).lower()
for sample in values.get("cloudflared_tunnel_server_locations", [])
if float(sample.value) > 0 and sample.labels.get("edge_location")
}
)
timeout_losses: dict[str, float] = {}
loss_samples = values.get("quic_client_lost_packets_total") or values.get("quic_client_lost_packets") or []
for sample in loss_samples:
connection = sample.labels.get("conn_index")
reason = str(sample.labels.get("reason") or "").lower()
if connection is None or reason != "timeout":
continue
connection_key = str(connection)
timeout_losses[connection_key] = timeout_losses.get(connection_key, 0.0) + float(sample.value)
return MetricsSample(
sampled_at=now if now is not None else time.time(),
ready=ready,
ha_connections=(
registered_connections
if registered_connections is not None
else max(0, int(round(scalar("cloudflared_tunnel_ha_connections"))))
),
edge_locations=tuple(locations[:4]),
smoothed_rtt_ms=tuple(rtt_values[:4]),
request_errors_total=max(
0.0,
scalar("cloudflared_tunnel_request_errors_total", "cloudflared_tunnel_request_errors"),
),
packet_loss_total=max(0.0, sum(timeout_losses.values())),
closed_connections_total=max(
0.0,
scalar("quic_client_closed_connections_total", "quic_client_closed_connections"),
),
timeout_packet_loss_by_connection=tuple(sorted(timeout_losses.items())),
)
def scrape_metrics(metrics_url: str, *, timeout: float = 0.5, now: float | None = None) -> MetricsSample:
base_url = metrics_url.rstrip("/")
ready_response = requests.get(f"{base_url}/ready", timeout=timeout)
metrics_response = requests.get(f"{base_url}/metrics", timeout=timeout)
metrics_response.raise_for_status()
return parse_metrics(
metrics_response.text,
ready=ready_response.ok,
registered_connections=_ready_connections(ready_response),
now=now,
)
def _ready_connections(response: requests.Response) -> int | None:
"""Read ``readyConnections`` from cloudflared's ``/ready`` JSON body."""
try:
payload = response.json()
except ValueError:
return None
value = payload.get("readyConnections") if isinstance(payload, dict) else None
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
return None
return value
def probe_request_path(
session: requests.Session,
url: str,
*,
attempts: int = 3,
timeout: float = 3.5,
now: float | None = None,
) -> RequestPathSample:
"""Measure a bounded public health window through one persistent session."""
latency_values: list[float] = []
successes: list[bool] = []
for _ in range(max(1, attempts)):
started_at = time.perf_counter()
success = False
try:
response = session.get(
url,
timeout=timeout,
allow_redirects=False,
headers={"Accept": "application/json", "Cache-Control": "no-cache"},
)
success = 200 <= response.status_code < 300
response.close()
except requests.RequestException:
pass
latency_values.append(round((time.perf_counter() - started_at) * 1000, 1))
successes.append(success)
return RequestPathSample(
sampled_at=now if now is not None else time.time(),
latency_ms=tuple(latency_values),
successes=tuple(successes),
)
def rtt_stats(values: tuple[float, ...] | list[float]) -> dict[str, float] | None:
if not values:
return None
return {
"min": round(min(values), 1),
"median": round(float(statistics.median(values)), 1),
"max": round(max(values), 1),
}
def latency_grade(rtt: dict[str, float] | None) -> str:
if rtt is None:
return "unknown"
median = rtt["median"]
maximum = rtt["max"]
if median < 120 and maximum < 250:
return "good"
if median < 200 and maximum < 400:
return "fair"
if median < 350 and maximum < 700:
return "poor"
return "critical"
def _percentile(values: list[float], percentile: float) -> float | None:
if not values:
return None
ordered = sorted(values)
index = max(0, min(len(ordered) - 1, math.ceil(percentile * len(ordered)) - 1))
return round(ordered[index], 1)
def request_path_grade(request_path: dict[str, Any] | None) -> str:
if not request_path or request_path.get("confidence") == "low":
return "unknown"
latency = request_path.get("latency_ms")
if not isinstance(latency, dict):
return "critical" if request_path.get("status") == "unavailable" else "unknown"
p95 = float(latency.get("p95") or 0)
p99 = float(latency.get("p99") or 0)
slow = request_path.get("slow_request_rate") or {}
over_one_second = float(slow.get("over_1000_ms") or 0)
failure_rate = float(request_path.get("failure_rate") or 0)
if failure_rate >= 0.10:
return "critical"
if p95 < 350 and p99 < 750 and over_one_second < 0.02:
return "good"
if p95 < 600 and p99 < 1200 and over_one_second < 0.05:
return "fair"
if p95 < 1000 and p99 < 2000 and over_one_second < 0.10:
return "poor"
return "critical"
def request_path_is_degraded(request_path: dict[str, Any] | None) -> bool:
if not request_path or request_path.get("confidence") != "high":
return False
if request_path.get("status") == "unavailable":
return True
latency = request_path.get("latency_ms")
if not isinstance(latency, dict):
return False
p95 = float(latency.get("p95") or 0)
p99 = float(latency.get("p99") or 0)
slow = request_path.get("slow_request_rate") or {}
over_one_second = float(slow.get("over_1000_ms") or 0)
failure_rate = float(request_path.get("failure_rate") or 0)
baseline = request_path.get("baseline_p95_ms")
baseline_regression = baseline is not None and p95 >= 2 * float(baseline)
return (
failure_rate >= 0.10
or p95 >= 750
or baseline_regression
or over_one_second >= 0.05
or (p99 >= 1500 and over_one_second >= 0.03)
)
def request_path_needs_recovery(
request_path: dict[str, Any] | None,
*,
profile: str = "balanced",
) -> bool:
"""Apply a user-facing optimization policy to high-confidence path evidence."""
if not request_path or request_path.get("confidence") != "high":
return False
if request_path.get("status") == "unavailable":
return True
latency = request_path.get("latency_ms")
if not isinstance(latency, dict):
return False
p95 = float(latency.get("p95") or 0)
p99 = float(latency.get("p99") or 0)
slow = request_path.get("slow_request_rate") or {}
over_500_ms = float(slow.get("over_500_ms") or 0)
over_one_second = float(slow.get("over_1000_ms") or 0)
failure_rate = float(request_path.get("failure_rate") or 0)
baseline = request_path.get("baseline_p95_ms")
baseline_value = float(baseline) if baseline is not None else None
if failure_rate >= 0.10:
return True
if profile == "stable":
return (
p95 >= 1200
or (baseline_value is not None and p95 >= 3 * baseline_value)
or over_one_second >= 0.10
or (p99 >= 2400 and over_one_second >= 0.05)
)
if profile == "low_latency":
return (
p95 >= 500
or (baseline_value is not None and p95 >= 1.5 * baseline_value)
or over_500_ms >= 0.20
or over_one_second >= 0.03
or p99 >= 1000
)
return request_path_is_degraded(request_path)
def route_unavailable(snapshot: dict[str, Any]) -> bool:
"""Whether the active route serves no traffic.
cloudflared can keep reporting ready connections after its transport has
silently died, so a high-confidence window with no successful public probe
is treated as an outage alongside zero ready connections.
"""
if int(snapshot.get("ha_connections") or 0) == 0:
return True
request_path = snapshot.get("request_path")
return (
isinstance(request_path, dict)
and request_path.get("confidence") == "high"
and request_path.get("status") == "unavailable"
)
def request_path_has_usable_latency(request_path: dict[str, Any] | None) -> bool:
if not request_path or request_path.get("confidence") == "low":
return False
if request_path.get("status") == "unavailable" or int(request_path.get("success_count") or 0) == 0:
return False
return isinstance(request_path.get("latency_ms"), dict)
def summarize_request_path_samples(
samples: list[RequestPathSample] | tuple[RequestPathSample, ...],
*,
baseline_p95_ms: float | None = None,
) -> dict[str, Any] | None:
latencies: list[float] = []
successes: list[bool] = []
for sample in samples:
count = min(len(sample.latency_ms), len(sample.successes))
for index in range(count):
latency = float(sample.latency_ms[index])
if not math.isfinite(latency) or latency < 0:
continue
latencies.append(latency)
successes.append(bool(sample.successes[index]))
sample_count = len(latencies)
if sample_count == 0:
return None
success_count = sum(successes)
confidence = (
"high"
if sample_count >= REQUEST_PATH_HIGH_CONFIDENCE_SAMPLES
else "medium"
if sample_count >= REQUEST_PATH_MIN_SAMPLES
else "low"
)
latency = {
"p50": _percentile(latencies, 0.50),
"p95": _percentile(latencies, 0.95),
"p99": _percentile(latencies, 0.99),
"max": round(max(latencies), 1),
}
failure_rate = round((sample_count - success_count) / sample_count, 4)
slow_request_rate = {
"over_500_ms": round(sum(value > 500 for value in latencies) / sample_count, 4),
"over_1000_ms": round(sum(value > 1000 for value in latencies) / sample_count, 4),
"over_2000_ms": round(sum(value > 2000 for value in latencies) / sample_count, 4),
}
payload: dict[str, Any] = {
"source": "synthetic_local",
"status": "insufficient",
"confidence": confidence,
"window_seconds": REQUEST_PATH_WINDOW_SECONDS,
"sample_count": sample_count,
"success_count": success_count,
"latency_ms": latency,
"failure_rate": failure_rate,
"slow_request_rate": slow_request_rate,
"baseline_p95_ms": baseline_p95_ms,
}
if confidence != "low":
if success_count == 0:
payload["status"] = "unavailable"
else:
payload["status"] = "degraded" if request_path_is_degraded(payload) else "healthy"
return payload
def request_path_is_better(active: dict[str, Any], candidate: dict[str, Any]) -> bool:
active_latency = active.get("latency_ms")
candidate_latency = candidate.get("latency_ms")
if not isinstance(active_latency, dict) or not isinstance(candidate_latency, dict):
return False
candidate_count = int(candidate.get("sample_count") or 0)
candidate_success = int(candidate.get("success_count") or 0)
if candidate_count < REQUEST_PATH_MIN_SAMPLES or candidate_success / candidate_count < 0.95:
return False
active_p95 = float(active_latency.get("p95") or 0)
active_p99 = float(active_latency.get("p99") or 0)
candidate_p95 = float(candidate_latency.get("p95") or 0)
candidate_p99 = float(candidate_latency.get("p99") or 0)
active_slow = float((active.get("slow_request_rate") or {}).get("over_1000_ms") or 0)
candidate_slow = float((candidate.get("slow_request_rate") or {}).get("over_1000_ms") or 0)
active_failure = float(active.get("failure_rate") or 0)
candidate_failure = float(candidate.get("failure_rate") or 0)
p95_improved = active_p95 > 0 and candidate_p95 <= active_p95 * 0.80
tail_improved = (
active_p95 > 0
and candidate_p95 <= active_p95 * 1.25
and active_p99 > 0
and candidate_p99 <= active_p99 * 0.60
)
slow_rate_improved = (
active_slow >= 0.03
and candidate_slow <= min(active_slow - 0.02, active_slow * 0.50)
)
failure_rate_improved = (
active_failure >= 0.10
and candidate_failure <= min(active_failure - 0.05, active_failure * 0.50)
)
return p95_improved or tail_improved or slow_rate_improved or failure_rate_improved
class QualityEvaluator:
"""Stateful evaluator shared by status, Doctor, reporting, and recovery."""
def __init__(self) -> None:
self._counter_samples: deque[MetricsSample] = deque(maxlen=8)
self._baseline_samples: deque[tuple[float, float]] = deque(maxlen=BASELINE_RETENTION_SAMPLES)
self._request_samples: deque[RequestPathSample] = deque(maxlen=REQUEST_PATH_MAX_SAMPLES)
self._request_baseline_samples: deque[tuple[float, str, float]] = deque(
maxlen=REQUEST_BASELINE_RETENTION_SAMPLES
)
self._last_baseline_minute: int | None = None
self._last_request_baseline_minute: dict[str, int] = {}
self._last_snapshot: dict[str, Any] | None = None
self._latency_bad_samples = 0
self._partial_samples = 0
self._error_windows = 0
self._loss_windows = 0
self._metrics_failures = 0
self._healthy_samples = 0
@property
def last_snapshot(self) -> dict[str, Any] | None:
return self._last_snapshot
@property
def healthy_samples(self) -> int:
return self._healthy_samples
def reset_healthy_samples(self) -> None:
self._healthy_samples = 0
def reset_request_window(self) -> None:
self._request_samples.clear()
def export_state(self) -> dict[str, Any]:
return {
"baseline_samples": [[timestamp, value] for timestamp, value in self._baseline_samples],
"last_baseline_minute": self._last_baseline_minute,
"request_baseline_samples": [
[timestamp, protocol, value]
for timestamp, protocol, value in self._request_baseline_samples
],
"last_request_baseline_minute": dict(self._last_request_baseline_minute),
}
def load_state(self, payload: dict[str, Any], *, now: float | None = None) -> None:
cutoff = (now if now is not None else time.time()) - 24 * 60 * 60
samples = payload.get("baseline_samples")
if not isinstance(samples, list):
return
restored: list[tuple[float, float]] = []
for item in samples[-BASELINE_RETENTION_SAMPLES:]:
if not isinstance(item, list) or len(item) != 2:
continue
try:
timestamp = float(item[0])
value = float(item[1])
except (TypeError, ValueError):
continue
if timestamp >= cutoff and value >= 0 and math.isfinite(value):
restored.append((timestamp, value))
self._baseline_samples = deque(restored, maxlen=BASELINE_RETENTION_SAMPLES)
last_minute = payload.get("last_baseline_minute")
self._last_baseline_minute = int(last_minute) if isinstance(last_minute, int) else None
request_samples = payload.get("request_baseline_samples")
restored_request_samples: list[tuple[float, str, float]] = []
if isinstance(request_samples, list):
for item in request_samples[-REQUEST_BASELINE_RETENTION_SAMPLES:]:
if not isinstance(item, list) or len(item) != 3 or item[1] not in {"quic", "http2"}:
continue
try:
timestamp = float(item[0])
value = float(item[2])
except (TypeError, ValueError):
continue
if timestamp >= cutoff and value >= 0 and math.isfinite(value):
restored_request_samples.append((timestamp, str(item[1]), value))
self._request_baseline_samples = deque(
restored_request_samples,
maxlen=REQUEST_BASELINE_RETENTION_SAMPLES,
)
request_minutes = payload.get("last_request_baseline_minute")
if isinstance(request_minutes, dict):
self._last_request_baseline_minute = {
protocol: int(value)
for protocol, value in request_minutes.items()
if protocol in {"quic", "http2"} and isinstance(value, int)
}
def baseline(self, now: float | None = None) -> float | None:
cutoff = (now if now is not None else time.time()) - 24 * 60 * 60
while self._baseline_samples and self._baseline_samples[0][0] < cutoff:
self._baseline_samples.popleft()
if len(self._baseline_samples) < BASELINE_MINUTE_SAMPLES:
return None
return _percentile([value for _, value in self._baseline_samples], 0.20)
def request_baseline(self, protocol: str, now: float | None = None) -> float | None:
cutoff = (now if now is not None else time.time()) - 24 * 60 * 60
while self._request_baseline_samples and self._request_baseline_samples[0][0] < cutoff:
self._request_baseline_samples.popleft()
values = [
value
for _, sample_protocol, value in self._request_baseline_samples
if sample_protocol == protocol
]
if len(values) < BASELINE_MINUTE_SAMPLES:
return None
return _percentile(values, 0.20)
def _request_path(
self,
sample: RequestPathSample | None,
*,
protocol: str,
now: float,
) -> dict[str, Any] | None:
if sample is not None:
self._request_samples.append(sample)
cutoff = now - REQUEST_PATH_WINDOW_SECONDS
while self._request_samples and self._request_samples[0].sampled_at < cutoff:
self._request_samples.popleft()
return summarize_request_path_samples(
list(self._request_samples),
baseline_p95_ms=self.request_baseline(protocol, now) if protocol in {"quic", "http2"} else None,
)
def _record_request_baseline(
self,
request_path: dict[str, Any] | None,
*,
protocol: str,
now: float,
) -> None:
if (
protocol not in {"quic", "http2"}
or not request_path
or request_path.get("status") != "healthy"
or request_path.get("confidence") == "low"
):
return
latency = request_path.get("latency_ms")
if not isinstance(latency, dict) or latency.get("p95") is None:
return
minute = int(now // 60)
if self._last_request_baseline_minute.get(protocol) == minute:
return
self._request_baseline_samples.append((now, protocol, float(latency["p95"])))
self._last_request_baseline_minute[protocol] = minute
def _rates(self, sample: MetricsSample) -> tuple[float, float]:
self._counter_samples.append(sample)
cutoff = sample.sampled_at - RATE_WINDOW_SECONDS
base = next((item for item in self._counter_samples if item.sampled_at >= cutoff), self._counter_samples[0])
elapsed = sample.sampled_at - base.sampled_at
if elapsed <= 0:
return 0.0, 0.0
scale = 60.0 / elapsed
errors = max(0.0, sample.request_errors_total - base.request_errors_total) * scale
current_losses = dict(sample.timeout_packet_loss_by_connection)
base_losses = dict(base.timeout_packet_loss_by_connection)
connection_losses = [
max(0.0, current_losses[connection] - base_losses[connection]) * scale
for connection in current_losses.keys() & base_losses.keys()
]
losses = sum(connection_losses) if sum(loss > 0 for loss in connection_losses) >= 2 else 0.0
return round(errors, 2), round(losses, 2)
def update(
self,
sample: MetricsSample | None,
*,
connector_count: int = 1,
recovery: dict[str, Any] | None = None,
configured_protocol: str = "auto",
effective_protocol: str | None = None,
request_path_sample: RequestPathSample | None = None,
now: float | None = None,
) -> dict[str, Any]:
current_time = now if now is not None else (sample.sampled_at if sample else time.time())
recovery_payload = recovery or empty_recovery()
previous_state = str((self._last_snapshot or {}).get("state") or "unknown")
previous_protocol = str((self._last_snapshot or {}).get("protocol") or "unknown")
protocol = effective_protocol if effective_protocol in {"quic", "http2"} else previous_protocol
if sample is not None:
if effective_protocol not in {"quic", "http2"}:
protocol = "quic" if sample.smoothed_rtt_ms else "http2" if sample.ready else "unknown"
request_path = self._request_path(request_path_sample, protocol=protocol, now=current_time)
if sample is None:
self._metrics_failures += 1
state = "unknown" if self._metrics_failures >= 3 else previous_state
if state not in {"healthy", "degraded", "recovering"}:
state = "unknown"
previous_connections = int((self._last_snapshot or {}).get("ha_connections") or 0)
previous_locations = list((self._last_snapshot or {}).get("edge_locations") or [])
request_grade = request_path_grade(request_path)
if request_path_is_degraded(request_path):
state = "degraded"
snapshot = {
"schema_version": 2,
"state": "recovering" if recovery_payload["state"] in {"evaluating", "draining"} else state,
"grade": request_grade,
"sampled_at": utc_timestamp(current_time),
"protocol": protocol,
"transport": {"configured": configured_protocol, "effective": protocol},
"connector_count": max(0, connector_count),
"ha_connections": previous_connections,
"rtt_ms": None,
"baseline_median_rtt_ms": self.baseline(current_time),
"edge_locations": previous_locations,
"window_seconds": RATE_WINDOW_SECONDS,
"request_errors_per_minute": 0.0,
"packet_loss_per_minute": 0.0,
"request_path": request_path,
"recovery": recovery_payload,
}
self._last_snapshot = snapshot
return snapshot
self._metrics_failures = 0
rtt = rtt_stats(sample.smoothed_rtt_ms)
connector_grade = latency_grade(rtt)
request_grade = request_path_grade(request_path)
grade = request_grade if request_grade != "unknown" else connector_grade
errors_per_minute, loss_per_minute = self._rates(sample)
self._latency_bad_samples = self._latency_bad_samples + 1 if connector_grade in {"poor", "critical"} else 0
self._partial_samples = self._partial_samples + 1 if sample.ha_connections < 4 else 0
self._error_windows = self._error_windows + 1 if errors_per_minute >= 3 else 0
self._loss_windows = self._loss_windows + 1 if loss_per_minute >= 10 else 0
degraded = (
not sample.ready
or sample.ha_connections == 0
or self._partial_samples >= 4
or self._error_windows >= 2
or self._loss_windows >= 2
or self._latency_bad_samples >= 12
or request_path_is_degraded(request_path)
)
baseline = self.baseline(current_time)
recovery_state = recovery_payload["state"]
if recovery_state in {"evaluating", "draining"}:
state = "recovering"
elif degraded:
state = "degraded"
self._healthy_samples = 0
elif previous_state in {"degraded", "recovering"}:
healthy_rtt = (
rtt is None or rtt["median"] < max(160.0, 1.5 * baseline)
if baseline is not None
else connector_grade in {"good", "fair", "unknown"}
)
healthy_request_path = not request_path_is_degraded(request_path)
self._healthy_samples = self._healthy_samples + 1 if sample.ha_connections == 4 and errors_per_minute < 3 and loss_per_minute < 10 and healthy_rtt and healthy_request_path else 0
state = "healthy" if self._healthy_samples >= 20 else "degraded"
else:
state = "healthy"
self._healthy_samples += 1
minute = int(current_time // 60)
if (
state == "healthy"
and connector_grade in {"good", "fair"}
and sample.ha_connections == 4
and errors_per_minute < 3
and loss_per_minute < 10
and rtt is not None
and minute != self._last_baseline_minute
):
self._baseline_samples.append((current_time, rtt["median"]))
self._last_baseline_minute = minute
baseline = self.baseline(current_time)
if state == "healthy":
self._record_request_baseline(request_path, protocol=protocol, now=current_time)
snapshot = {
"schema_version": 2,
"state": state,
"grade": grade,
"sampled_at": utc_timestamp(current_time),
"protocol": protocol,
"transport": {"configured": configured_protocol, "effective": protocol},
"connector_count": max(0, connector_count),
"ha_connections": min(4, sample.ha_connections),
"rtt_ms": rtt,
"baseline_median_rtt_ms": baseline,
"edge_locations": list(sample.edge_locations),
"window_seconds": RATE_WINDOW_SECONDS,
"request_errors_per_minute": errors_per_minute,
"packet_loss_per_minute": loss_per_minute,
"request_path": request_path,
"recovery": recovery_payload,
}
self._last_snapshot = snapshot
return snapshot
def recovery_trigger(
self,
snapshot: dict[str, Any] | None = None,
*,
profile: str = "balanced",
) -> str | None:
current = snapshot or self._last_snapshot
if not current or profile not in {"stable", "balanced", "low_latency"}:
return None
state = current.get("state")
if state != "degraded" and profile != "low_latency":
return None
if state not in {"healthy", "degraded"}:
return None
if int(current.get("ha_connections") or 0) < 4 or route_unavailable(current):
return "availability"
if float(current.get("request_errors_per_minute") or 0) >= 3 or float(current.get("packet_loss_per_minute") or 0) >= 10:
return "errors"
if request_path_needs_recovery(current.get("request_path"), profile=profile):
return "tail_latency"
rtt = current.get("rtt_ms")
if not isinstance(rtt, dict):
return None
median = float(rtt.get("median") or 0)
maximum = float(rtt.get("max") or 0)
baseline = current.get("baseline_median_rtt_ms")
if profile == "stable":
if baseline is None:
return "latency" if median >= 450 or maximum >= 900 else None
baseline_value = float(baseline)
if median >= 450 or (median >= 350 and median >= 2.5 * baseline_value):
return "latency"
if maximum >= 900 or (maximum >= 700 and maximum >= 3.5 * baseline_value):
return "latency"
return None
if profile == "low_latency":
if baseline is None:
return "latency" if median >= 180 or maximum >= 350 else None
baseline_value = float(baseline)
if median >= 250 or (median >= 160 and median >= 1.6 * baseline_value):
return "latency"
if maximum >= 450 or (maximum >= 300 and maximum >= 2.2 * baseline_value):
return "latency"
return None
if baseline is None:
return "latency" if median >= 250 or maximum >= 500 else None
baseline_value = float(baseline)
if median >= 350 or (median >= 200 and median >= 2 * baseline_value):
return "latency"
if maximum >= 700 or (maximum >= 400 and maximum >= 3 * baseline_value):
return "latency"
return None
def empty_recovery() -> dict[str, Any]:
return {
"state": "idle",
"last_attempt_at": None,
"last_trigger": None,
"last_result": None,
"previous_median_rtt_ms": None,
"result_median_rtt_ms": None,
"previous_protocol": None,
"result_protocol": None,
"previous_p95_ms": None,
"result_p95_ms": None,
"previous_p99_ms": None,
"result_p99_ms": None,
"next_attempt_at": None,
"attempt_count_window": 0,
}
def candidate_is_better(active: dict[str, Any], candidate: dict[str, Any], *, trigger: str) -> bool:
if int(candidate.get("ha_connections") or 0) < 4:
return False
if float(candidate.get("request_errors_per_minute") or 0) > 0:
return False
if float(candidate.get("packet_loss_per_minute") or 0) > float(active.get("packet_loss_per_minute") or 0):
return False
if trigger == "availability" and int(active.get("ha_connections") or 0) < 4:
return True
if trigger == "tail_latency":
active_path = active.get("request_path")
candidate_path = candidate.get("request_path")
return (
isinstance(active_path, dict)
and isinstance(candidate_path, dict)
and request_path_is_better(active_path, candidate_path)
)
active_rtt = active.get("rtt_ms")
candidate_rtt = candidate.get("rtt_ms")
if trigger == "errors":
if (
isinstance(active_rtt, dict)
and isinstance(candidate_rtt, dict)
and float(candidate_rtt.get("max") or 0) > float(active_rtt.get("max") or 0)
):
return False
return (
float(candidate.get("request_errors_per_minute") or 0)
< float(active.get("request_errors_per_minute") or 0)
or float(candidate.get("packet_loss_per_minute") or 0)
< float(active.get("packet_loss_per_minute") or 0)
)
if not isinstance(active_rtt, dict) or not isinstance(candidate_rtt, dict):
return False
active_median = float(active_rtt.get("median") or 0)
candidate_median = float(candidate_rtt.get("median") or 0)
active_maximum = float(active_rtt.get("max") or 0)
candidate_maximum = float(candidate_rtt.get("max") or 0)
improvement = active_median - candidate_median
return improvement >= 30 and candidate_median <= active_median * 0.75 and candidate_maximum <= active_maximum
def summarize_candidate_snapshots(snapshots: list[dict[str, Any]]) -> dict[str, Any] | None:
"""Conservatively summarize a candidate's evaluation window."""
if len(snapshots) < CANDIDATE_MIN_SAMPLES:
return None
summary = dict(snapshots[-1])
summary["ha_connections"] = min(int(snapshot.get("ha_connections") or 0) for snapshot in snapshots)
summary["request_errors_per_minute"] = max(
float(snapshot.get("request_errors_per_minute") or 0) for snapshot in snapshots
)
summary["packet_loss_per_minute"] = max(
float(snapshot.get("packet_loss_per_minute") or 0) for snapshot in snapshots
)
rtt_samples = [snapshot.get("rtt_ms") for snapshot in snapshots]
if all(isinstance(rtt, dict) for rtt in rtt_samples):
typed_rtt_samples = [rtt for rtt in rtt_samples if isinstance(rtt, dict)]
summary["rtt_ms"] = {
"min": round(float(statistics.median(float(rtt["min"]) for rtt in typed_rtt_samples)), 1),
"median": round(float(statistics.median(float(rtt["median"]) for rtt in typed_rtt_samples)), 1),
"max": round(max(float(rtt["max"]) for rtt in typed_rtt_samples), 1),
}
else:
summary["rtt_ms"] = None
return summary