Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.
Python ai-agentsaiopsdashboarddockerfastapihomelabincident-managementllmmonitoringopenaireactroot-cause-analysisself-hosteduptime-monitoring
"""Range-aware history for charts: the last hour to all time, from the right level of detail.
A request names a range (``1h`` ... ``all``). Short ranges are read from the check-in samples, longer ones from the
hourly, weekly or monthly summaries (rollups.py). Whatever the source, the answer is a series of at most about
``MAX_POINTS`` points, each with the average and maximum of every requested metric, so a year costs about as much
as an hour. The newest period, which the rollup job has not closed yet, is built from the finer data below it, so
the right-hand edge is always current. Periods without data have no point: a gap stays a gap, and the response
says how long one point's period is (``bucket_seconds``) so a chart can tell a gap from a neighbour.
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Any
from sqlalchemy import func, select
from . import rollups
from .models import ContainerMetric, ContainerRollup, HostMetric, HostRollup, MountMetric, MountRollup
DETAILED = "detailed"
RANGES: dict[str, timedelta | None] = {
"1h": timedelta(hours=1), "6h": timedelta(hours=6), "24h": timedelta(hours=24), "7d": timedelta(days=7),
"30d": timedelta(days=30), "90d": timedelta(days=90), "1y": timedelta(days=365), "all": None,
}
# Which level of detail answers each range.
SOURCE = {"1h": DETAILED, "6h": DETAILED, "24h": DETAILED, "7d": DETAILED,
"30d": rollups.HOUR, "90d": rollups.WEEK, "1y": rollups.WEEK, "all": rollups.MONTH}
MAX_POINTS = 500
# A backstop: with check-ins spaced by the ingest limit, one series stays well below this in seven days.
MAX_RAW_ROWS = 150_000
NATURAL_SECONDS = {rollups.HOUR: 3600, rollups.DAY: 86400, rollups.WEEK: 7 * 86400, rollups.MONTH: 31 * 86400}
# Bucket sizes a chart can be given; the smallest that keeps a range within MAX_POINTS is used.
NICE_STEPS = (15, 30, 60, 120, 300, 600, 900, 1800, 3600, 7200, 14400, 21600, 43200, 86400, 7 * 86400)
# The tiers a partly finished period of each tier is assembled from, coarsest first.
FINER_CHAIN = {rollups.HOUR: (), rollups.DAY: (rollups.HOUR,), rollups.WEEK: (rollups.DAY, rollups.HOUR), rollups.MONTH: (rollups.DAY, rollups.HOUR)}
@dataclass(frozen=True)
class Metric:
"""How one charted value is read from a raw row and from a rollup row. Either side may be absent."""
raw: str
avg: str | None = None # rollup column holding the average (its count column is n_<name>)
max: str | None = None
def _m(raw, name, *, avg=True, maximum=True):
return Metric(raw, f"avg_{name}" if avg else None, f"max_{name}" if maximum else None)
HOST_METRICS = {
"cpu": _m("cpu_utilization_pct", "cpu"), "load": _m("load_1m", "load"), "ram": _m("ram_pct", "ram"), "swap": _m("swap_pct", "swap"),
"disk": _m("disk_usage_pct", "disk"), "inodes": _m("disk_inodes_pct", "inodes"),
"docker_ms": _m("docker_latency_ms", "docker_ms"), "collect_ms": _m("collect_duration_ms", "collect_ms"),
}
MOUNT_METRICS = {
"used_pct": _m("used_pct", "used_pct"), "used_gb": _m("used_gb", "used_gb", maximum=False), "total_gb": _m("total_gb", "total_gb", avg=False),
"inodes_pct": _m("inodes_pct", "inodes_pct"),
}
CONTAINER_METRICS = {
"cpu": _m("cpu_pct", "cpu"), "memory_mb": _m("memory_used_mb", "memory_mb"), "memory_pct": _m("memory_pct", "memory_pct"),
"swap_mb": _m("swap_mb", "swap_mb"), "memory_limit_mb": _m("memory_limit_mb", "memory_limit_mb", avg=False), "restarts": _m("restart_count", "restarts", avg=False),
}
DEFAULT_HOST_METRICS = ("cpu", "ram", "swap", "load", "disk")
@dataclass(frozen=True)
class Series:
raw: Any
rollup: Any
metrics: dict[str, Metric]
has_down: bool = False
HOST = Series(HostMetric, HostRollup, HOST_METRICS)
MOUNT = Series(MountMetric, MountRollup, MOUNT_METRICS)
CONTAINER = Series(ContainerMetric, ContainerRollup, CONTAINER_METRICS, has_down=True)
class Agg:
"""Sum, count and maximum of each metric over some check-ins; adding two of them gives the exact combined figures."""
__slots__ = ("samples", "down", "values")
def __init__(self, samples: int = 0, down: int = 0):
self.samples, self.down, self.values = samples, down, {}
def add_value(self, name: str, total: float | None, count: int, maximum: float | None) -> None:
current = self.values.setdefault(name, [0.0, 0, None])
if total is not None and count:
current[0] += total
current[1] += count
if maximum is not None and (current[2] is None or maximum > current[2]):
current[2] = maximum
def merge(self, other: "Agg") -> None:
self.samples += other.samples
self.down += other.down
for name, (total, count, maximum) in other.values.items():
self.add_value(name, total, count, maximum)
def _raw_agg(spec: Series, wanted, row) -> Agg:
agg = Agg(samples=1)
for index, name in enumerate(wanted):
value = row[index + 1]
if value is not None:
agg.add_value(name, float(value), 1, float(value))
if spec.has_down:
agg.down = 0 if row[-1] == "running" else 1
return agg
def _rollup_agg(spec: Series, wanted, row) -> Agg:
agg = Agg(samples=row.samples, down=getattr(row, "down_samples", 0) if spec.has_down else 0)
for name in wanted:
metric = spec.metrics[name]
average = getattr(row, metric.avg) if metric.avg else None
count = getattr(row, "n_" + metric.avg[len("avg_"):]) if metric.avg else 0
maximum = getattr(row, metric.max) if metric.max else None
agg.add_value(name, average * count if average is not None and count else None, count if average is not None else 0, maximum)
return agg
def _point(spec: Series, wanted, start: datetime, agg: Agg) -> dict:
point: dict[str, Any] = {"time": start.isoformat(), "samples": agg.samples}
if spec.has_down:
point["down"] = agg.down
for name in wanted:
total, count, maximum = agg.values.get(name, (0.0, 0, None))
metric = spec.metrics[name]
value: dict[str, float] = {}
if metric.avg and count:
value["avg"] = total / count
if metric.max and maximum is not None:
value["max"] = maximum
point[name] = value or None
return point
def nice_step(seconds: float) -> int:
return next((step for step in NICE_STEPS if step >= seconds), NICE_STEPS[-1])
def _where(spec: Series, kind: str, server_id: int, key: str | None):
"""Filters that pick one server's series (and one mount or container) from a raw or rollup table."""
table = spec.raw if kind == "raw" else spec.rollup
conditions = [table.server_id == server_id]
if spec is MOUNT:
conditions.append(table.mount == key)
elif spec is CONTAINER:
conditions.append(table.container_id == key)
return conditions
def _raw_items(db, spec: Series, wanted, server_id, key, since: datetime, until: datetime):
columns = [getattr(spec.raw, spec.metrics[name].raw) for name in wanted]
if spec.has_down:
columns.append(spec.raw.status)
query = select(spec.raw.time, *columns).where(*_where(spec, "raw", server_id, key), spec.raw.time >= since, spec.raw.time < until).order_by(spec.raw.time).limit(MAX_RAW_ROWS)
return [(rollups._utc(row[0]), _raw_agg(spec, wanted, row)) for row in db.execute(query)]
def _rollup_items(db, spec: Series, wanted, server_id, key, tier: str, since: datetime, until: datetime):
if until <= since:
return []
query = select(spec.rollup).where(*_where(spec, "rollup", server_id, key), spec.rollup.tier == tier,
spec.rollup.bucket_start >= since, spec.rollup.bucket_start < until).order_by(spec.rollup.bucket_start)
return [(rollups._utc(row.bucket_start), _rollup_agg(spec, wanted, row)) for row in db.scalars(query)]
def data_since(db, spec: Series, server_id: int, key: str | None) -> datetime | None:
"""The earliest moment this series has data: the first sample if it is still stored, else the start of the
oldest summary that reaches further back than anything finer.
A summary's bucket starts on a boundary (midnight, Monday), which can be long before the first check-in, so a
coarser tier only counts when its oldest bucket ends before the finer data begins; otherwise the finer data
already says exactly when the history starts.
"""
first = db.execute(select(func.min(spec.raw.time)).where(*_where(spec, "raw", server_id, key))).scalar()
since = rollups._utc(first) if first else None
for tier in (rollups.HOUR, rollups.DAY, rollups.WEEK, rollups.MONTH):
oldest = db.execute(select(func.min(spec.rollup.bucket_start)).where(*_where(spec, "rollup", server_id, key), spec.rollup.tier == tier)).scalar()
if oldest is None:
continue
oldest = rollups._utc(oldest)
if since is None or rollups.following(tier, oldest) <= since:
since = oldest
return since
def _group(items, bucket_of):
grouped: dict[datetime, Agg] = {}
for start, agg in items:
grouped.setdefault(bucket_of(start), Agg()).merge(agg)
return grouped
def _tiered_items(db, spec, wanted, server_id, key, tier: str, window_start: datetime, now: datetime):
"""One Agg per bucket of ``tier``: closed buckets from the rollups, the newest ones from the finer data."""
first = rollups.floor(tier, window_start)
done = rollups._read_watermark(db, tier)
closed_until = min(done, now) if done and done > first else first
items = _rollup_items(db, spec, wanted, server_id, key, tier, first, closed_until)
grouped = {start: agg for start, agg in items}
tail: list = []
cursor = closed_until
for finer in FINER_CHAIN[tier]:
finer_done = rollups._read_watermark(db, finer)
if finer_done and finer_done > cursor:
tail += _rollup_items(db, spec, wanted, server_id, key, finer, cursor, min(finer_done, now))
cursor = min(finer_done, now)
tail += _raw_items(db, spec, wanted, server_id, key, cursor, now + timedelta(seconds=1))
for start, agg in _group(tail, lambda moment: rollups.floor(tier, moment)).items():
grouped.setdefault(start, Agg()).merge(agg)
return sorted(grouped.items()), NATURAL_SECONDS[tier]
def previous_end(range_name: str, now: datetime | None = None) -> datetime | None:
"""Where the period before ``range_name`` ends (which is where the range itself starts), for "compare with the
previous 7 days". None for "all", which has nothing before it."""
length = RANGES[range_name]
return rollups._utc(now or datetime.now(timezone.utc)) - length if length else None
def history(db, spec: Series, server_id: int, key: str | None, range_name: str, wanted, interval_seconds: int, now: datetime | None = None) -> dict:
"""The chart series for one server (and mount or container) over ``range_name``, ending at ``now``."""
now = rollups._utc(now or datetime.now(timezone.utc))
since = data_since(db, spec, server_id, key)
length = RANGES[range_name]
window_start = max(now - length, since) if length and since else (since or now - (length or timedelta(0)))
source = SOURCE[range_name]
if since is None:
items, natural = [], NATURAL_SECONDS.get(source, interval_seconds)
elif source == DETAILED:
items, natural = _raw_items(db, spec, wanted, server_id, key, window_start, now + timedelta(seconds=1)), interval_seconds
else:
items, natural = _tiered_items(db, spec, wanted, server_id, key, source, window_start, now)
span = max((now - window_start).total_seconds(), 1)
step = max(nice_step(max(span / MAX_POINTS, interval_seconds if source == DETAILED else 0)), natural if source != DETAILED else 0)
if source == DETAILED or step > natural:
epoch = lambda moment: datetime.fromtimestamp(int(moment.timestamp()) // step * step, tz=timezone.utc) # noqa: E731
items = sorted(_group(items, epoch).items())
else:
step = natural
return {
"range": range_name, "source": source, "bucket_seconds": step,
"start": window_start.isoformat(), "end": now.isoformat(), "data_since": since.isoformat() if since else None,
"metrics": list(wanted), "points": [_point(spec, wanted, start, agg) for start, agg in items],
}