Skip to content

Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.

refact: edits on dashboard UI

karloks2005053d6105d23c
diff --git a/docker-compose-example.yml b/docker-compose-example.yml
index 930de47..94a88f4 100644
--- a/docker-compose-example.yml
+++ b/docker-compose-example.yml
@@ -84,6 +84,8 @@ services:
restart: unless-stopped
environment:
HOMELAB_API_URL: http://backend:8000
+ # Marks this collector as the machine running the dashboard (the charts at the top).
+ HOMELAB_DASHBOARD_HOST: "true"
# Nothing to configure: this machine shows up under "New servers detected"
# on the dashboard. An optional enrollment key skips that approval.
HOMELAB_ENROLL_KEY: ${HOMELAB_ENROLL_KEY:-}
diff --git a/docker-compose.server.yml b/docker-compose.server.yml
index c0daa73..46606c7 100644
--- a/docker-compose.server.yml
+++ b/docker-compose.server.yml
@@ -81,6 +81,8 @@ services:
restart: unless-stopped
environment:
HOMELAB_API_URL: http://backend:8000
+ # Marks this collector as the machine running the dashboard (the charts at the top).
+ HOMELAB_DASHBOARD_HOST: "true"
# Nothing to configure: this machine shows up under "New servers detected"
# on the dashboard. An optional enrollment key skips that approval.
HOMELAB_ENROLL_KEY: ${HOMELAB_ENROLL_KEY:-}
diff --git a/docker-compose.yml b/docker-compose.yml
index 106fc3a..a58017e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -71,6 +71,8 @@ services:
restart: unless-stopped
environment:
HOMELAB_API_URL: http://backend:8000
+ # Marks this collector as the machine running the dashboard (the charts at the top).
+ HOMELAB_DASHBOARD_HOST: "true"
# Nothing to configure: this machine shows up under "New servers detected"
# on the dashboard. An optional enrollment key skips that approval.
HOMELAB_ENROLL_KEY: ${HOMELAB_ENROLL_KEY:-}
diff --git a/src/.claude/settings.json b/src/.claude/settings.json
new file mode 100644
index 0000000..48983f6
--- /dev/null
+++ b/src/.claude/settings.json
@@ -0,0 +1 @@
+{ "includeCoAuthoredBy": false }
\ No newline at end of file
diff --git a/src/agent/collector.py b/src/agent/collector.py
index 5c03d93..240bfd0 100644
--- a/src/agent/collector.py
+++ b/src/agent/collector.py
@@ -69,6 +69,69 @@ def _network_totals() -> tuple[int, int]:
return sent, received
+# Filesystems worth reporting: anything backed by a block device, plus these pooled/network ones.
+NON_DEVICE_FS_TYPES = {"zfs", "nfs", "nfs4", "cifs", "smb3", "fuse.mergerfs", "fuse.sshfs"}
+SKIPPED_FS_TYPES = {"squashfs", "overlay", "tmpfs", "devtmpfs"}
+SKIPPED_MOUNT_PREFIXES = ("/proc", "/sys", "/dev", "/run", "/snap", "/var/snap", "/var/lib/docker", "/var/lib/containerd", "/var/lib/kubelet")
+
+
+def _decode_mount_path(path: str) -> str:
+ """/proc mounts escapes spaces, tabs, newlines and backslashes as octal."""
+ for escaped, char in (("\\040", " "), ("\\011", "\t"), ("\\012", "\n"), ("\\134", "\\")):
+ path = path.replace(escaped, char)
+ return path
+
+
+def _mount_entries() -> list[tuple[str, str, str]]:
+ """(source, mount point, fs type) as the host sees them."""
+ # PID 1's table is the host's; "self" would be this container's own mounts.
+ for candidate in (HOST_PROC / "1/mounts", HOST_PROC / "self/mounts"):
+ try:
+ lines = candidate.read_text().splitlines()
+ except OSError:
+ continue
+ return [(fields[0], _decode_mount_path(fields[1]), fields[2]) for fields in (line.split() for line in lines) if len(fields) >= 3]
+ return []
+
+
+def collect_mounts() -> list[dict[str, Any]]:
+ """Usage of every real filesystem on the host, one entry per device."""
+ by_device: dict[tuple[str, int], dict[str, Any]] = {}
+ for source, mount, fs_type in _mount_entries():
+ if fs_type in SKIPPED_FS_TYPES or not (source.startswith("/dev/") or fs_type in NON_DEVICE_FS_TYPES):
+ continue
+ if mount != "/" and mount.startswith(SKIPPED_MOUNT_PREFIXES):
+ continue
+ local_path = HOST_ROOT / mount.lstrip("/")
+ # A mount made after the collector started is only an empty directory here; skip it rather
+ # than report the parent filesystem under its name.
+ if mount != "/" and not os.path.ismount(local_path):
+ continue
+ try:
+ stats = os.statvfs(local_path)
+ except OSError:
+ continue
+ total = stats.f_blocks * stats.f_frsize
+ if not total:
+ continue
+ used = total - stats.f_bfree * stats.f_frsize
+ # Space available to non-root users is what fills up, so percentages use it like `df` does.
+ usable = used + stats.f_bavail * stats.f_frsize
+ entry = {
+ "mount": mount[:255],
+ "fs_type": fs_type[:64],
+ "total_gb": round(total / GiB, 2),
+ "used_gb": round(used / GiB, 2),
+ "used_pct": round(100 * used / usable, 2) if usable else 0,
+ "inodes_pct": round(100 * (stats.f_files - stats.f_ffree) / stats.f_files, 2) if stats.f_files else None,
+ }
+ # Bind mounts and btrfs subvolumes show the same device several times; keep its shortest path.
+ key = (source, total)
+ if key not in by_device or len(mount) < len(by_device[key]["mount"]):
+ by_device[key] = entry
+ return sorted(by_device.values(), key=lambda entry: entry["mount"])
+
+
def collect_host() -> tuple[dict[str, Any], dict[str, Any]]:
before_total, before_idle = _cpu_times()
time.sleep(0.15)
@@ -79,9 +142,10 @@ def collect_host() -> tuple[dict[str, Any], dict[str, Any]]:
disk = os.statvfs(HOST_ROOT)
inode_total = disk.f_files
inode_pct = (100 * (inode_total - disk.f_ffree) / inode_total) if inode_total else None
- disk_total = disk.f_blocks * disk.f_frsize
- disk_free = disk.f_bavail * disk.f_frsize
- disk_pct = (100 * (disk_total - disk_free) / disk_total) if disk_total else 0
+ disk_used = (disk.f_blocks - disk.f_bfree) * disk.f_frsize
+ disk_usable = disk_used + disk.f_bavail * disk.f_frsize
+ # Same formula as `df` and the per-mount figures.
+ disk_pct = (100 * disk_used / disk_usable) if disk_usable else 0
try:
loads = os.getloadavg()
except (AttributeError, OSError):
@@ -97,6 +161,7 @@ def collect_host() -> tuple[dict[str, Any], dict[str, Any]]:
"net_bytes_sent_mb": round(sent / MiB, 3),
"net_bytes_recv_mb": round(received / MiB, 3),
"system_uptime_seconds": round(uptime, 1),
+ "mounts": collect_mounts(),
}
server = {
"hostname": _host_hostname(),
@@ -104,6 +169,9 @@ def collect_host() -> tuple[dict[str, Any], dict[str, Any]]:
"cpu_cores": _host_cpu_cores(),
"total_ram_gb": round(_memory_total_gb(), 3),
}
+ # Set only on the collector in the dashboard's own compose stack; the dashboard's top charts follow it.
+ if os.getenv("HOMELAB_DASHBOARD_HOST", "").lower() in {"1", "true", "yes"}:
+ server["dashboard_host"] = True
return server, host
diff --git a/src/backend/main.py b/src/backend/main.py
index b731164..8bdb759 100644
--- a/src/backend/main.py
+++ b/src/backend/main.py
@@ -21,7 +21,7 @@ from sqlalchemy import case, inspect, text
from .alerts import CONTAINER_RULE_PREFIX, CRITICAL, SERVER_OFFLINE_RULE, SEVERITIES, active_rule_keys, raise_alert, resolve_alerts, sync_alerts
from .auth import authenticate_user, get_recent_logs, get_user_from_session, logout_user, record_audit_log, register_user
from .db import SessionLocal, check_database, get_db
-from .models import AppSetting, AuditLog, Base, ChatMessage, ContainerMetric, Conversation, EnrollmentKey, HostMetric, IncidentAlert, PendingCollector, Server, User
+from .models import AppSetting, AuditLog, Base, ChatMessage, ContainerMetric, Conversation, EnrollmentKey, HostMetric, IncidentAlert, MountMetric, PendingCollector, Server, User
from .db import engine
from .config import (
@@ -151,6 +151,8 @@ def startup_event():
# duplicated on every check-in with no way to resolve, so close them; the engine re-raises live ones.
connection.execute(text("UPDATE incident_alerts SET rule_key = :rule WHERE rule_key IS NULL AND description = :description"), {"rule": SERVER_OFFLINE_RULE, "description": OFFLINE_ALERT_DESCRIPTION})
connection.execute(text("UPDATE incident_alerts SET status = 'resolved', resolved_at = NOW() WHERE rule_key IS NULL AND status = 'active'"))
+ # Disk and inode alerts became per mount; the old keys were for the root filesystem.
+ connection.execute(text("UPDATE incident_alerts SET rule_key = rule_key || ':/' WHERE rule_key IN ('host:disk', 'host:inodes')"))
if "servers" in inspect(engine).get_table_names():
columns = {column["name"] for column in inspect(engine).get_columns("servers")}
if "collector_token_hash" not in columns:
@@ -159,6 +161,9 @@ def startup_event():
if "machine_id" not in columns:
with engine.begin() as connection:
connection.execute(text("ALTER TABLE servers ADD COLUMN machine_id VARCHAR(64)"))
+ if "is_dashboard_host" not in columns:
+ with engine.begin() as connection:
+ connection.execute(text("ALTER TABLE servers ADD COLUMN is_dashboard_host BOOLEAN NOT NULL DEFAULT FALSE"))
db = SessionLocal()
try:
ensure_admin_exists(db)
@@ -370,6 +375,7 @@ def purge_old_metrics(db, now: datetime | None = None) -> int:
cutoff = (now or datetime.now(timezone.utc)) - timedelta(days=get_metrics_retention_days(db))
deleted = db.query(HostMetric).filter(HostMetric.time < cutoff).delete(synchronize_session=False)
deleted += db.query(ContainerMetric).filter(ContainerMetric.time < cutoff).delete(synchronize_session=False)
+ deleted += db.query(MountMetric).filter(MountMetric.time < cutoff).delete(synchronize_session=False)
db.commit()
return deleted
@@ -423,7 +429,8 @@ OVERVIEW_ALERT_LIMIT = 50
def monitoring_overview(session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):
# Servers belong to the install: everyone signed in can view them, only admins manage them.
user = _current_user(session_id, db)
- servers = db.query(Server).order_by(Server.created_at.desc()).all()
+ # The dashboard's own machine first, then the newest servers.
+ servers = db.query(Server).order_by(Server.is_dashboard_host.desc(), Server.created_at.desc()).all()
result = []
for server in servers:
metric = db.query(HostMetric).filter(HostMetric.server_id == server.id).order_by(HostMetric.time.desc()).first()
@@ -431,12 +438,14 @@ def monitoring_overview(session_id: Annotated[str | None, Cookie(alias=SESSION_C
latest = {}
for item in containers:
latest.setdefault(item.container_id, item)
+ mounts = db.query(MountMetric).filter(MountMetric.server_id == server.id, MountMetric.time == metric.time).order_by(MountMetric.mount).all() if metric else []
result.append({
"id": server.id, "hostname": server.hostname, "ip_address": server.ip_address, "os_info": server.os_info,
"cpu_cores": server.cpu_cores, "total_ram_gb": server.total_ram_gb,
- "check_interval_seconds": server.check_interval_seconds,
+ "check_interval_seconds": server.check_interval_seconds, "is_dashboard_host": server.is_dashboard_host,
"last_seen_at": server.last_seen_at.isoformat() if server.last_seen_at else None,
"host": ({key: getattr(metric, key) for key in ("time", "cpu_utilization_pct", "load_1m", "load_5m", "load_15m", "ram_used_gb", "ram_available_gb", "ram_pct", "swap_pct", "disk_usage_pct", "disk_inodes_pct", "net_bytes_sent_mb", "net_bytes_recv_mb", "system_uptime_seconds")} if metric else None),
+ "mounts": [{key: getattr(m, key) for key in ("mount", "fs_type", "total_gb", "used_gb", "used_pct", "inodes_pct")} for m in mounts],
"containers": [{"container_id": c.container_id, "name": c.name, "image": c.image, "status": c.status, "restart_count": c.restart_count, "cpu_pct": c.cpu_pct, "memory_used_mb": c.memory_used_mb, "memory_pct": c.memory_pct, "healthcheck_status": c.healthcheck_status, "time": c.time.isoformat()} for c in latest.values()],
})
active_alerts = db.query(IncidentAlert).filter(IncidentAlert.status == "active")
@@ -444,13 +453,30 @@ def monitoring_overview(session_id: Annotated[str | None, Cookie(alias=SESSION_C
severity_rank = case({severity: rank for rank, severity in enumerate(reversed(SEVERITIES))}, value=IncidentAlert.severity, else_=len(SEVERITIES))
alerts = active_alerts.order_by(severity_rank, IncidentAlert.created_at.desc()).limit(OVERVIEW_ALERT_LIMIT).all()
hostnames = {server.id: server.hostname for server in servers}
+ thresholds = get_alert_thresholds(db)
pending = visible_pending_collectors(db) if _is_admin(db, user["id"]) else []
return {
"servers": result, "pending_collectors": pending, "alert_count": active_alerts.count(),
+ # Lets the dashboard color disk bars the same way the alert rules judge them.
+ "disk_thresholds": {key: thresholds[key] for key in ("disk_warning_pct", "disk_critical_pct", "inodes_warning_pct", "inodes_critical_pct")},
"alerts": [{"id": a.id, "server_id": a.server_id, "hostname": hostnames.get(a.server_id), "severity": a.severity, "title": a.title, "description": a.description, "created_at": a.created_at.isoformat()} for a in alerts],
}
+HISTORY_MAX_MINUTES = 24 * 60
+
+
[email protected]("/monitoring/servers/{server_id}/history")
+def server_history(server_id: int, minutes: int = 30, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):
+ """CPU, memory and root disk usage of one server over the last ``minutes``, oldest first."""
+ _current_user(session_id, db)
+ if not db.query(Server).filter(Server.id == server_id).first():
+ raise HTTPException(status_code=404, detail="Server not found.")
+ since = datetime.now(timezone.utc) - timedelta(minutes=max(1, min(minutes, HISTORY_MAX_MINUTES)))
+ rows = db.query(HostMetric.time, HostMetric.cpu_utilization_pct, HostMetric.ram_pct, HostMetric.disk_usage_pct).filter(HostMetric.server_id == server_id, HostMetric.time >= since).order_by(HostMetric.time).all()
+ return {"server_id": server_id, "samples": [{"time": row.time.isoformat(), "cpu": row.cpu_utilization_pct, "ram": row.ram_pct, "disk": row.disk_usage_pct} for row in rows]}
+
+
CONTAINER_DETAIL_FIELDS = (
"status", "healthcheck_status", "restart_count", "cpu_pct", "memory_used_mb", "memory_limit_mb", "memory_pct",
"net_rx_mb", "net_tx_mb", "block_read_mb", "block_write_mb", "pids",
@@ -729,6 +755,7 @@ def delete_monitored_server(server_id: int, session_id: Annotated[str | None, Co
db.query(HostMetric).filter(HostMetric.server_id == server.id).delete(synchronize_session=False)
db.query(ContainerMetric).filter(ContainerMetric.server_id == server.id).delete(synchronize_session=False)
+ db.query(MountMetric).filter(MountMetric.server_id == server.id).delete(synchronize_session=False)
db.query(IncidentAlert).filter(IncidentAlert.server_id == server.id).delete(synchronize_session=False)
db.query(PendingCollector).filter(PendingCollector.approved_server_id == server.id).delete(synchronize_session=False)
db.delete(server)
@@ -763,8 +790,17 @@ def ingest_metrics(payload: MetricsIngestRequest, authorization: Annotated[str |
for field in ("hostname", "ip_address", "os_info", "cpu_cores", "total_ram_gb"):
if field in payload.server and payload.server[field] is not None:
setattr(server, field, payload.server[field])
+ # Only one server can be the dashboard's own machine; the latest collector to claim it wins.
+ server.is_dashboard_host = payload.server.get("dashboard_host") is True
+ if server.is_dashboard_host:
+ db.query(Server).filter(Server.id != server.id, Server.is_dashboard_host.is_(True)).update({"is_dashboard_host": False}, synchronize_session=False)
now = datetime.now(timezone.utc)
db.add(HostMetric(server_id=server.id, time=now, **{k: v for k, v in payload.host.items() if k in HostMetric.__table__.columns.keys() and k not in {"id", "server_id", "time"}}))
+ mounts = payload.host.get("mounts")
+ for mount in mounts if isinstance(mounts, list) else []:
+ if isinstance(mount, dict) and isinstance(mount.get("mount"), str) and mount["mount"]:
+ allowed = {k: v for k, v in mount.items() if k in MountMetric.__table__.columns.keys() and k not in {"id", "server_id", "time"}}
+ db.add(MountMetric(server_id=server.id, time=now, **allowed))
containers = [c for c in payload.containers if c.get("container_id") and c.get("status")]
for container in containers:
allowed = {k: v for k, v in container.items() if k in ContainerMetric.__table__.columns.keys() and k not in {"id", "server_id", "time"}}
diff --git a/src/backend/models.py b/src/backend/models.py
index 5ceb4a6..61e8a73 100644
--- a/src/backend/models.py
+++ b/src/backend/models.py
@@ -101,6 +101,8 @@ class Server(Base):
collector_token_hash: Mapped[Optional[str]] = mapped_column(String(64), nullable=True, unique=True)
# Stable host identity reported by self-enrolling collectors, so a reinstall maps back to the same server.
machine_id: Mapped[Optional[str]] = mapped_column(String(64), nullable=True, index=True)
+ # The machine running the dashboard itself, as reported by the collector in its compose stack.
+ is_dashboard_host: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
last_seen_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False)
@@ -154,6 +156,22 @@ class HostMetric(Base):
system_uptime_seconds: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+class MountMetric(Base):
+ """Usage of one filesystem at one check-in; shares its time with that check-in's HostMetric."""
+
+ __tablename__ = "mount_metrics"
+
+ id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
+ server_id: Mapped[int] = mapped_column(ForeignKey("servers.id", ondelete="CASCADE"), index=True, nullable=False)
+ time: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), index=True, nullable=False)
+ mount: Mapped[str] = mapped_column(String(255), nullable=False)
+ fs_type: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)
+ total_gb: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ used_gb: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ used_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+ inodes_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)
+
+
class ContainerMetric(Base):
__tablename__ = "container_metrics"
diff --git a/src/backend/rules.py b/src/backend/rules.py
index 4215f95..62df58e 100644
--- a/src/backend/rules.py
+++ b/src/backend/rules.py
@@ -10,7 +10,7 @@ from datetime import datetime, timedelta
from sqlalchemy import func
from .alerts import CONTAINER_RULE_PREFIX, CRITICAL, WARNING
-from .models import ContainerMetric, HostMetric
+from .models import ContainerMetric, HostMetric, MountMetric
HOST_RULE_PREFIX = "host:"
@@ -66,44 +66,74 @@ def _level(value: float | None, warning: float, critical: float | None) -> int:
return 1 if value >= warning else 0
+def _fire_level(values: list, warning: float, critical: float | None, sustained: int, active: bool) -> int:
+ """Alert level for a metric given its recent values, newest first."""
+ levels = [_level(value, warning, critical) for value in values]
+ # Opening an alert, or escalating it to critical, needs every recent check-in to breach;
+ # an open alert stays open while the latest check-in still breaches.
+ sustained_level = min(levels) if len(levels) >= sustained else 0
+ return max(sustained_level, min(levels[0], 1)) if active and levels else sustained_level
+
+
+def _escape(text: str) -> str:
+ return text.replace("{", "{{").replace("}", "}}")
+
+
+def _gb(value: float | None) -> str:
+ if value is None:
+ return "?"
+ return f"{value / 1024:.1f} TB" if value >= 1024 else f"{value:.0f} GB"
+
+
def host_rules(db, server, thresholds: dict, active_keys: set[str]) -> dict[str, tuple[str, str, str]]:
- """Evaluate the host rules against the server's most recent check-ins (newest first)."""
+ """Evaluate the host rules against the server's most recent check-ins."""
sustained = thresholds["sustained_checks"]
samples = db.query(HostMetric).filter(HostMetric.server_id == server.id).order_by(HostMetric.time.desc(), HostMetric.id.desc()).limit(sustained).all()
if not samples:
return {}
+ # Titles are format strings for the latest value, so escape braces in names we put into them.
+ host = _escape(server.hostname)
cores = server.cpu_cores or None
def load_per_core(sample):
return sample.load_5m / cores if cores and sample.load_5m is not None else None
- checks = (
- # (rule, value getter, warning, critical, title, description)
- ("disk", lambda s: s.disk_usage_pct, thresholds["disk_warning_pct"], thresholds["disk_critical_pct"],
- "Disk usage on {host} is {value:.0f}%", "The root filesystem is {value:.1f}% full. Free up space before it fills completely."),
- ("inodes", lambda s: s.disk_inodes_pct, thresholds["inodes_warning_pct"], thresholds["inodes_critical_pct"],
- "Inode usage on {host} is {value:.0f}%", "{value:.1f}% of inodes on the root filesystem are used. Many small files (caches, logs, mail queues) are the usual cause."),
- ("memory", lambda s: s.ram_pct, thresholds["memory_warning_pct"], thresholds["memory_critical_pct"],
- "Memory usage on {host} is {value:.0f}%", "{value:.1f}% of RAM is in use. Check the containers and processes using the most memory."),
- ("swap", lambda s: s.swap_pct, thresholds["swap_warning_pct"], None,
- "Swap usage on {host} is {value:.0f}%", "{value:.1f}% of swap has been in use for several check-ins, an early sign of memory pressure."),
- ("load", load_per_core, thresholds["load_warning_per_core"], thresholds["load_critical_per_core"],
- "Load on {host} is {value:.1f}× its cores", "The 5-minute load average is {value:.2f} per CPU core across " + f"{cores or '?'} cores. Work is queuing for the CPU."),
- )
+ # (rule key, values newest first, warning, critical, title, description), with {value} filled in from the latest value.
+ checks = [
+ (f"{HOST_RULE_PREFIX}memory", [s.ram_pct for s in samples], thresholds["memory_warning_pct"], thresholds["memory_critical_pct"],
+ f"Memory usage on {host} is {{value:.0f}}%", "{value:.1f}% of RAM is in use. Check the containers and processes using the most memory."),
+ (f"{HOST_RULE_PREFIX}swap", [s.swap_pct for s in samples], thresholds["swap_warning_pct"], None,
+ f"Swap usage on {host} is {{value:.0f}}%", "{value:.1f}% of swap has been in use for several check-ins, an early sign of memory pressure."),
+ (f"{HOST_RULE_PREFIX}load", [load_per_core(s) for s in samples], thresholds["load_warning_per_core"], thresholds["load_critical_per_core"],
+ f"Load on {host} is {{value:.1f}}× its cores", f"The 5-minute load average is {{value:.2f}} per CPU core across {cores or '?'} cores. Work is queuing for the CPU."),
+ ]
+
+ times = [sample.time for sample in samples]
+ by_mount: dict[str, dict] = {}
+ for row in db.query(MountMetric).filter(MountMetric.server_id == server.id, MountMetric.time.in_(times)):
+ by_mount.setdefault(row.mount, {})[row.time] = row
+ current = {mount: rows[times[0]] for mount, rows in by_mount.items() if times[0] in rows}
+ if current:
+ for raw_mount, row in sorted(current.items()):
+ mount = _escape(raw_mount)
+ history = [by_mount[raw_mount].get(time) for time in times]
+ size = f"{_gb(row.used_gb)} of {_gb(row.total_gb)} used"
+ checks.append((f"{HOST_RULE_PREFIX}disk:{raw_mount}", [r.used_pct if r else None for r in history], thresholds["disk_warning_pct"], thresholds["disk_critical_pct"],
+ f"Disk usage on {host}:{mount} is {{value:.0f}}%", f"{mount} is {{value:.1f}}% full ({size}). Free up space before it fills completely."))
+ checks.append((f"{HOST_RULE_PREFIX}inodes:{raw_mount}", [r.inodes_pct if r else None for r in history], thresholds["inodes_warning_pct"], thresholds["inodes_critical_pct"],
+ f"Inode usage on {host}:{mount} is {{value:.0f}}%", f"{{value:.1f}}% of inodes on {mount} are used. Many small files (caches, logs, mail queues) are the usual cause."))
+ else:
+ # Collectors from before per-mount reporting only send the root filesystem.
+ checks.append((f"{HOST_RULE_PREFIX}disk:/", [s.disk_usage_pct for s in samples], thresholds["disk_warning_pct"], thresholds["disk_critical_pct"],
+ f"Disk usage on {host}:/ is {{value:.0f}}%", "/ is {value:.1f}% full. Free up space before it fills completely."))
+ checks.append((f"{HOST_RULE_PREFIX}inodes:/", [s.disk_inodes_pct for s in samples], thresholds["inodes_warning_pct"], thresholds["inodes_critical_pct"],
+ f"Inode usage on {host}:/ is {{value:.0f}}%", "{value:.1f}% of inodes on / are used. Many small files (caches, logs, mail queues) are the usual cause."))
+
firing: dict[str, tuple[str, str, str]] = {}
- for rule, value_of, warning, critical, title, description in checks:
- key = f"{HOST_RULE_PREFIX}{rule}"
- latest_value = value_of(samples[0])
- levels = [_level(value_of(sample), warning, critical) for sample in samples]
- # Opening an alert, or escalating it to critical, needs every recent check-in to breach.
- sustained_level = min(levels) if len(samples) >= sustained else 0
- level = max(sustained_level, min(levels[0], 1)) if key in active_keys else sustained_level
+ for key, values, warning, critical, title, description in checks:
+ level = _fire_level(values, warning, critical, sustained, key in active_keys)
if level:
- firing[key] = (
- CRITICAL if level == 2 else WARNING,
- title.format(host=server.hostname, value=latest_value),
- description.format(value=latest_value),
- )
+ firing[key] = (CRITICAL if level == 2 else WARNING, title.format(value=values[0]), description.format(value=values[0]))
return firing
diff --git a/src/frontend/src/App.jsx b/src/frontend/src/App.jsx
index ced5a0d..0874436 100644
--- a/src/frontend/src/App.jsx
+++ b/src/frontend/src/App.jsx
@@ -431,6 +431,73 @@ function RegisterPage({ onRegister }) {
// Izvan ili na vrhu komponente postavi guard za izbjegavanje paralelnih poziva
let isFetchingOverview = false
+// Collectors from before per-mount reporting only send the root filesystem's usage.
+function serverMounts(server) {
+ if (server.mounts?.length) return server.mounts
+ const host = server.host
+ return host?.disk_usage_pct != null ? [{ mount: '/', used_pct: host.disk_usage_pct, inodes_pct: host.disk_inodes_pct }] : []
+}
+
+function fullestMount(mounts) {
+ return mounts.reduce((fullest, mount) => (mount.used_pct != null && (!fullest || mount.used_pct > fullest.used_pct) ? mount : fullest), null)
+}
+
+const HOST_HISTORY_MINUTES = 30
+
+// Adds the latest check-in unless it's already there (the dashboard polls faster than collectors report).
+function appendHostSample(samples, sample) {
+ const last = samples.at(-1)
+ if (!sample.time || (last && new Date(sample.time) <= new Date(last.time))) return samples
+ const cutoff = new Date(sample.time).getTime() - HOST_HISTORY_MINUTES * 60 * 1000
+ return [...samples, sample].filter((item) => new Date(item.time).getTime() >= cutoff)
+}
+
+function formatGigabytes(value) {
+ if (value == null) return '?'
+ return value >= 1024 ? `${(value / 1024).toFixed(1)} TB` : `${value >= 10 ? value.toFixed(0) : value.toFixed(1)} GB`
+}
+
+const DISK_LEVEL_STYLES = [
+ { bar: 'bg-emerald-400/70', text: 'text-[#cfcfcf]' },
+ { bar: 'bg-amber-400/80', text: 'text-amber-200' },
+ { bar: 'bg-red-400/80', text: 'text-red-300' },
+]
+
+function thresholdLevel(value, warning, critical) {
+ if (value == null) return 0
+ return value >= critical ? 2 : value >= warning ? 1 : 0
+}
+
+function StorageBars({ mounts, thresholds }) {
+ if (!mounts.length) return null
+ const t = { disk_warning_pct: 85, disk_critical_pct: 95, inodes_warning_pct: 85, inodes_critical_pct: 95, ...thresholds }
+ return (
+ <div className="mt-4">
+ <div className="mb-2 text-xs uppercase tracking-[0.2em] text-[#7f8589]">Storage</div>
+ <div className="space-y-2">
+ {mounts.map((mount) => {
+ const style = DISK_LEVEL_STYLES[thresholdLevel(mount.used_pct, t.disk_warning_pct, t.disk_critical_pct)]
+ const inodeLevel = thresholdLevel(mount.inodes_pct, t.inodes_warning_pct, t.inodes_critical_pct)
+ const used = Math.min(100, Math.max(0, mount.used_pct ?? 0))
+ return (
+ <div key={mount.mount} className="grid grid-cols-[minmax(0,9rem)_1fr_auto] items-center gap-3 text-sm sm:grid-cols-[minmax(0,12rem)_1fr_auto]">
+ <span className="truncate font-mono text-xs text-[#bcbcbc]" title={mount.fs_type ? `${mount.mount} (${mount.fs_type})` : mount.mount}>{mount.mount}</span>
+ <div className="h-2 overflow-hidden rounded-full bg-white/5" role="meter" aria-label={`${mount.mount} usage`} aria-valuemin={0} aria-valuemax={100} aria-valuenow={used}>
+ <div className={`h-full rounded-full ${style.bar}`} style={{ width: `${used}%` }} />
+ </div>
+ <span className="whitespace-nowrap text-right text-xs text-[#9aa0a6]">
+ {mount.total_gb != null && <span className="hidden sm:inline">{formatGigabytes(mount.used_gb)} / {formatGigabytes(mount.total_gb)} · </span>}
+ <span className={`font-medium ${style.text}`}>{mount.used_pct == null ? '—' : `${Math.round(mount.used_pct)}%`}</span>
+ {inodeLevel > 0 && <span className={`ml-2 ${DISK_LEVEL_STYLES[inodeLevel].text}`}>inodes {Math.round(mount.inodes_pct)}%</span>}
+ </span>
+ </div>
+ )
+ })}
+ </div>
+ </div>
+ )
+}
+
const ALERT_STYLES = {
CRITICAL: { box: 'border-red-400/25 bg-red-500/5', badge: 'bg-red-500/15 text-red-300', title: 'text-red-200' },
WARNING: { box: 'border-amber-400/20 bg-amber-400/5', badge: 'bg-amber-400/15 text-amber-300', title: 'text-amber-200' },
@@ -442,7 +509,8 @@ function DashboardPage({ user, onLogout }) {
const [error, setError] = useState('')
const [showConnect, setShowConnect] = useState(false)
const [deletingServerId, setDeletingServerId] = useState(null)
- const [history, setHistory] = useState([])
+ // Recent samples of the machine running the dashboard: loaded from the backend, then extended live.
+ const [history, setHistory] = useState({ serverId: null, samples: [] })
const refresh = async () => {
// Ako je zahtjev već u tijeku, preskoči ovaj poziv
@@ -454,19 +522,10 @@ const refresh = async () => {
setOverview(data)
setError('')
- const metrics = (data.servers || []).map((server) => server.host).filter(Boolean)
- if (metrics.length) {
- const mean = (key) => metrics.reduce((sum, metric) => sum + (metric[key] || 0), 0) / metrics.length
-
- setHistory((current) => [
- ...current,
- {
- time: new Date(),
- cpu: mean('cpu_utilization_pct'),
- ram: mean('ram_pct'),
- disk: mean('disk_usage_pct')
- }
- ].slice(-40))
+ const host = (data.servers || []).find((server) => server.is_dashboard_host)
+ if (host?.host) {
+ const sample = { time: host.host.time, cpu: host.host.cpu_utilization_pct, ram: host.host.ram_pct, disk: host.host.disk_usage_pct }
+ setHistory((current) => current.serverId === host.id ? { ...current, samples: appendHostSample(current.samples, sample) } : current)
}
} catch (requestError) {
// Ako backend vrati 401 (neautoriziran/odjavljen korisnik), odjavi ga i prekini petlju
@@ -496,6 +555,19 @@ const refresh = async () => {
return () => clearInterval(timer)
}, [user?.id]) // <-- Stabilna instanca, okida se samo pri promjeni ID-a
+ const servers = overview.servers || []
+ const dashboardHost = servers.find((server) => server.is_dashboard_host)
+ const dashboardHostId = dashboardHost?.id
+
+ useEffect(() => {
+ if (!dashboardHostId) return
+ let cancelled = false
+ apiRequest(`/monitoring/servers/${dashboardHostId}/history?minutes=${HOST_HISTORY_MINUTES}`, { method: 'GET' })
+ .then((data) => { if (!cancelled) setHistory({ serverId: dashboardHostId, samples: data.samples }) })
+ .catch(() => { if (!cancelled) setHistory({ serverId: dashboardHostId, samples: [] }) })
+ return () => { cancelled = true }
+ }, [dashboardHostId])
+
const changeInterval = async (server, value) => {
try {
await apiRequest(`/monitoring/servers/${server.id}/interval`, { method: 'PATCH', body: JSON.stringify({ check_interval_seconds: Number(value) }) })
@@ -514,10 +586,9 @@ const refresh = async () => {
setDeletingServerId(null)
}
}
- const servers = overview.servers || []
- const latestMetrics = servers.map((server) => server.host).filter(Boolean)
- const average = (key) => latestMetrics.length ? (latestMetrics.reduce((sum, metric) => sum + (metric[key] || 0), 0) / latestMetrics.length).toFixed(1) : '—'
const online = servers.filter((server) => server.last_seen_at && Date.now() - new Date(server.last_seen_at).getTime() < server.check_interval_seconds * 3000).length
+ const hostSamples = history.serverId === dashboardHostId ? history.samples : []
+ const hostMetric = dashboardHost?.host
const runningContainers = servers.reduce((sum, server) => sum + server.containers.filter((container) => container.status === 'running').length, 0)
// Regular users get a read-only view; the backend enforces the same rules.
@@ -528,7 +599,7 @@ const refresh = async () => {
<div className="flex flex-col gap-5">
<header className="glass-panel rounded-[28px] p-5 md:p-6">
<div className="flex flex-wrap items-end justify-between gap-4">
- <div><div className="text-xs uppercase tracking-[0.22em] text-[#9aa0a6]">Infrastructure overview</div><h1 className="mt-1 text-3xl font-semibold text-white">Dashboard</h1><p className="mt-2 text-sm text-[#9aa0a6]">Live host and container health · updated every 15 seconds.</p></div>
+ <div><div className="text-xs uppercase tracking-[0.22em] text-[#9aa0a6]">Infrastructure overview</div><h1 className="mt-1 text-3xl font-semibold text-white">Dashboard</h1><p className="mt-2 text-sm text-[#9aa0a6]">Live host and container health · updated every {LIVE_GRAPH_REFRESH_INTERVAL_SECONDS} seconds.</p></div>
{isAdmin && <button type="button" onClick={() => setShowConnect((current) => !current)} aria-expanded={showConnect} className="rounded-xl bg-white px-4 py-2 text-sm font-medium text-slate-950">{showConnect ? 'Close' : 'Connect a server'}</button>}
</div>
{error && <div className="mt-4 rounded-xl border border-amber-400/30 bg-amber-400/5 p-3 text-sm text-amber-200">{error}</div>}
@@ -538,20 +609,23 @@ const refresh = async () => {
<section className="grid gap-4 sm:grid-cols-2 xl:grid-cols-3">
{[
['Servers online', `${online} / ${servers.length}`, 'Based on last check-in'],
- ['Average CPU', latestMetrics.length ? `${average('cpu_utilization_pct')}%` : '—', 'Across reporting servers'],
- ['Average memory', latestMetrics.length ? `${average('ram_pct')}%` : '—', 'RAM utilization'],
- ['Average disk', latestMetrics.length ? `${average('disk_usage_pct')}%` : '—', 'Disk utilization'],
['Containers running', String(runningContainers), 'Across monitored servers'],
['Active incidents', String(overview.alert_count ?? overview.alerts?.length ?? 0), 'Needs attention'],
].map(([label, value, detail]) => <div key={label} className="glass-panel rounded-[24px] p-5"><div className="text-sm text-[#9aa0a6]">{label}</div><div className="mt-3 text-3xl font-semibold text-white">{value}</div><div className="mt-2 text-xs text-[#7f8589]">{detail}</div></div>)}
</section>
- <section className="grid gap-4 xl:grid-cols-3">
- {[
- { key: 'cpu', title: 'CPU utilization', color: '#34d399', latest: average('cpu_utilization_pct') },
- { key: 'ram', title: 'Memory utilization', color: '#60a5fa', latest: average('ram_pct') },
- { key: 'disk', title: 'Disk utilization', color: '#c084fc', latest: average('disk_usage_pct') },
- ].map((chart) => <MetricTrend key={chart.key} title={chart.title} color={chart.color} value={chart.latest} points={history.map((sample) => sample[chart.key])} times={history.map((sample) => sample.time)} />)}
+ <section>
+ <div className="mb-3 flex flex-wrap items-baseline justify-between gap-2 px-1">
+ <div className="text-xs uppercase tracking-[0.2em] text-[#9aa0a6]">This machine{dashboardHost && <span className="ml-2 normal-case tracking-normal text-[#cfcfcf]">{dashboardHost.hostname}</span>}</div>
+ <span className="text-xs text-[#7f8589]">Last {HOST_HISTORY_MINUTES} minutes</span>
+ </div>
+ {dashboardHost ? <div className="grid gap-4 xl:grid-cols-3">
+ {[
+ { key: 'cpu', title: 'CPU utilization', color: '#34d399', latest: hostMetric?.cpu_utilization_pct },
+ { key: 'ram', title: 'Memory utilization', color: '#60a5fa', latest: hostMetric?.ram_pct },
+ { key: 'disk', title: 'Disk usage (/)', color: '#c084fc', latest: hostMetric?.disk_usage_pct },
+ ].map((chart) => <MetricTrend key={chart.key} title={chart.title} color={chart.color} value={chart.latest == null ? '—' : Number(chart.latest).toFixed(1)} points={hostSamples.map((sample) => sample[chart.key])} times={hostSamples.map((sample) => sample.time)} />)}
+ </div> : <div className="glass-panel rounded-[24px] p-5 text-sm text-[#9aa0a6]">The machine running the dashboard isn't reporting yet. Its collector starts with the dashboard's compose stack and shows up here once it's added under <span className="text-white">New servers detected</span>.</div>}
</section>
<section className="glass-panel rounded-[28px] p-5 md:p-6">
@@ -559,10 +633,13 @@ const refresh = async () => {
{isAdmin && <PendingServers pending={overview.pending_collectors || []} onChanged={refresh} />}
{!servers.length ? <div className="rounded-2xl border border-dashed border-neutral-700 p-8 text-center"><div className="text-lg font-medium text-white">No servers registered</div><p className="mt-2 text-sm text-[#9aa0a6]">{isAdmin ? <>Click <span className="text-white">Connect a server</span> above and run the command on each machine you want to monitor.</> : 'An admin needs to connect a server before it shows up here.'}</p></div> : <div className="space-y-4">{servers.map((server) => {
const metric = server.host
+ const mounts = serverMounts(server)
+ const fullest = fullestMount(mounts)
const fresh = server.last_seen_at && Date.now() - new Date(server.last_seen_at).getTime() < server.check_interval_seconds * 3000
return <article key={server.id} className="rounded-2xl border border-neutral-800 bg-[#0d1013]/70 p-4 md:p-5">
- <div className="flex flex-wrap items-start justify-between gap-3"><div><div className="flex items-center gap-2"><span className={`h-2.5 w-2.5 rounded-full ${fresh ? 'bg-emerald-400' : 'bg-neutral-600'}`} /><h3 className="text-lg font-medium text-white">{server.hostname}</h3></div><div className="mt-1 text-sm text-[#9aa0a6]">{server.os_info || server.ip_address || 'Waiting for first collector check-in'}{server.cpu_cores ? ` · ${server.cpu_cores} cores` : ''}</div></div>{isAdmin ? <div className="flex flex-wrap items-center gap-2"><label className="flex items-center gap-2 text-xs text-[#9aa0a6]">Check interval<select value={server.check_interval_seconds} onChange={(e) => changeInterval(server, e.target.value)} className="rounded-lg border border-neutral-700 bg-[#111417] px-2 py-1.5 text-sm text-white"><option value="15">15 sec</option><option value="30">30 sec</option><option value="60">1 min</option><option value="300">5 min</option><option value="900">15 min</option></select></label><button type="button" onClick={() => deleteServer(server)} disabled={deletingServerId === server.id} className="rounded-lg border border-red-400/30 bg-red-500/5 px-3 py-1.5 text-xs text-red-300 transition hover:bg-red-500/10 disabled:opacity-50">{deletingServerId === server.id ? 'Deleting…' : 'Delete server'}</button></div> : <span className="text-xs text-[#9aa0a6]">Checks every {server.check_interval_seconds < 60 ? `${server.check_interval_seconds} sec` : `${Math.round(server.check_interval_seconds / 60)} min`}</span>}</div>
- {metric ? <div className="mt-5 grid grid-cols-2 gap-3 lg:grid-cols-4">{[['CPU', metric.cpu_utilization_pct, '%'], ['RAM', metric.ram_pct, '%'], ['Disk', metric.disk_usage_pct, '%'], ['Load (1m)', metric.load_1m, '']].map(([label, value, unit]) => <div key={label} className="rounded-xl bg-[#171b1f] p-3"><div className="text-xs text-[#9aa0a6]">{label}</div><div className="mt-1 text-xl font-semibold text-white">{value == null ? '—' : `${Number(value).toFixed(1)}${unit}`}</div></div>)}</div> : <div className="mt-4 rounded-xl bg-[#171b1f] p-4 text-sm text-[#9aa0a6]">Collector has not sent a measurement yet.</div>}
+ <div className="flex flex-wrap items-start justify-between gap-3"><div><div className="flex items-center gap-2"><span className={`h-2.5 w-2.5 rounded-full ${fresh ? 'bg-emerald-400' : 'bg-neutral-600'}`} /><h3 className="text-lg font-medium text-white">{server.hostname}</h3>{server.is_dashboard_host && <span className="rounded-full border border-white/10 bg-white/5 px-2 py-0.5 text-[10px] uppercase tracking-wider text-[#cfcfcf]">This machine</span>}</div><div className="mt-1 text-sm text-[#9aa0a6]">{server.os_info || server.ip_address || 'Waiting for first collector check-in'}{server.cpu_cores ? ` · ${server.cpu_cores} cores` : ''}</div></div>{isAdmin ? <div className="flex flex-wrap items-center gap-2"><label className="flex items-center gap-2 text-xs text-[#9aa0a6]">Check interval<select value={server.check_interval_seconds} onChange={(e) => changeInterval(server, e.target.value)} className="rounded-lg border border-neutral-700 bg-[#111417] px-2 py-1.5 text-sm text-white"><option value="15">15 sec</option><option value="30">30 sec</option><option value="60">1 min</option><option value="300">5 min</option><option value="900">15 min</option></select></label><button type="button" onClick={() => deleteServer(server)} disabled={deletingServerId === server.id} className="rounded-lg border border-red-400/30 bg-red-500/5 px-3 py-1.5 text-xs text-red-300 transition hover:bg-red-500/10 disabled:opacity-50">{deletingServerId === server.id ? 'Deleting…' : 'Delete server'}</button></div> : <span className="text-xs text-[#9aa0a6]">Checks every {server.check_interval_seconds < 60 ? `${server.check_interval_seconds} sec` : `${Math.round(server.check_interval_seconds / 60)} min`}</span>}</div>
+ {metric ? <div className="mt-5 grid grid-cols-2 gap-3 lg:grid-cols-4">{[['CPU', metric.cpu_utilization_pct, '%'], ['RAM', metric.ram_pct, '%'], [fullest ? `Disk · ${fullest.mount}` : 'Disk', fullest?.used_pct, '%'], ['Load (1m)', metric.load_1m, '']].map(([label, value, unit]) => <div key={label} className="rounded-xl bg-[#171b1f] p-3"><div className="truncate text-xs text-[#9aa0a6]" title={label}>{label}</div><div className="mt-1 text-xl font-semibold text-white">{value == null ? '—' : `${Number(value).toFixed(1)}${unit}`}</div></div>)}</div> : <div className="mt-4 rounded-xl bg-[#171b1f] p-4 text-sm text-[#9aa0a6]">Collector has not sent a measurement yet.</div>}
+ {metric && <StorageBars mounts={mounts} thresholds={overview.disk_thresholds} />}
<div className="mt-4 flex flex-wrap items-center justify-between gap-2 text-xs text-[#7f8589]"><span>Last check: {server.last_seen_at ? new Date(server.last_seen_at).toLocaleString() : 'Never'}</span><span>{server.containers.length} containers</span></div>
{server.containers.length > 0 && <ContainerTable server={server} />}
</article>
@@ -848,6 +925,8 @@ const TREND_Y_TICKS = [100, 90, 80, 70, 60, 50, 40, 30, 20, 10, 0]
const trendY = (percent) => 3 + (100 - Math.max(0, Math.min(100, Number(percent) || 0))) * 0.94
function MetricTrend({ title, color, value, points, times }) {
+ // Used in url(#…), so keep only characters that are safe there (titles like "Disk usage (/)" aren't).
+ const fillId = `fill-${title.replace(/[^a-zA-Z0-9]+/g, '-')}`
const chartPoints = points.map((point, index) => `${points.length > 1 ? (index / (points.length - 1)) * 320 : 320},${trendY(point)}`).join(' ')
const areaPoints = points.length ? `0,${trendY(0)} ${chartPoints} 320,${trendY(0)}` : ''
return (
@@ -858,7 +937,7 @@ function MetricTrend({ title, color, value, points, times }) {
{TREND_Y_TICKS.filter((tick) => tick % 20 === 0).map((tick) => <span key={tick} className="absolute right-0 -translate-y-1/2 leading-none" style={{ top: `${trendY(tick)}%` }}>{tick}%</span>)}
</div>
<div className="min-w-0 flex-1">
- {points.length > 1 ? <svg viewBox="0 0 320 100" preserveAspectRatio="none" className="h-full w-full" role="img" aria-label={`${title} recent trend`}><defs><linearGradient id={`fill-${title.replaceAll(' ', '-')}`} x1="0" x2="0" y1="0" y2="1"><stop offset="0%" stopColor={color} stopOpacity="0.28"/><stop offset="100%" stopColor={color} stopOpacity="0"/></linearGradient></defs>{TREND_Y_TICKS.map((tick) => <line key={tick} x1="0" x2="320" y1={trendY(tick)} y2={trendY(tick)} stroke="#ffffff" strokeOpacity={tick % 20 === 0 ? 0.12 : 0.05} strokeWidth="1" vectorEffect="non-scaling-stroke" />)}<polygon points={areaPoints} fill={`url(#fill-${title.replaceAll(' ', '-')})`}/><polyline points={chartPoints} fill="none" stroke={color} strokeWidth="2.5" strokeLinecap="round" strokeLinejoin="round" vectorEffect="non-scaling-stroke"/></svg> : <div className="flex h-full items-center justify-center rounded-xl border border-dashed border-neutral-800 text-xs text-[#7f8589]">Trend appears as live samples arrive</div>}
+ {points.length > 1 ? <svg viewBox="0 0 320 100" preserveAspectRatio="none" className="h-full w-full" role="img" aria-label={`${title} recent trend`}><defs><linearGradient id={fillId} x1="0" x2="0" y1="0" y2="1"><stop offset="0%" stopColor={color} stopOpacity="0.28"/><stop offset="100%" stopColor={color} stopOpacity="0"/></linearGradient></defs>{TREND_Y_TICKS.map((tick) => <line key={tick} x1="0" x2="320" y1={trendY(tick)} y2={trendY(tick)} stroke="#ffffff" strokeOpacity={tick % 20 === 0 ? 0.12 : 0.05} strokeWidth="1" vectorEffect="non-scaling-stroke" />)}<polygon points={areaPoints} fill={`url(#${fillId})`}/><polyline points={chartPoints} fill="none" stroke={color} strokeWidth="2.5" strokeLinecap="round" strokeLinejoin="round" vectorEffect="non-scaling-stroke"/></svg> : <div className="flex h-full items-center justify-center rounded-xl border border-dashed border-neutral-800 text-xs text-[#7f8589]">Trend appears as live samples arrive</div>}
</div>
</div>
<div className="mt-2 flex justify-between pl-10 text-[10px] text-[#7f8589]"><span>{times[0] ? new Date(times[0]).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }) : 'Waiting for data'}</span><span>{times.at(-1) ? new Date(times.at(-1)).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }) : '—'}</span></div>
diff --git a/tests/test_collector_mounts.py b/tests/test_collector_mounts.py
new file mode 100644
index 0000000..e288ca0
--- /dev/null
+++ b/tests/test_collector_mounts.py
@@ -0,0 +1,44 @@
+import os
+
+from src.agent import collector
+
+MOUNTS = """\
+/dev/sda1 / ext4 rw 0 0
+proc /proc proc rw 0 0
+tmpfs /run tmpfs rw 0 0
+/dev/sda1 /var/snap/firefox/common ext4 rw 0 0
+/dev/sdb1 /mnt/my\\040data ext4 rw 0 0
+/dev/loop0 /snap/core/1 squashfs ro 0 0
+overlay /var/lib/docker/overlay2/x/merged overlay rw 0 0
+nas:/export /mnt/nfs nfs4 rw 0 0
+/dev/sdc1 /mnt/later ext4 rw 0 0
+"""
+
+
+class FakeStat:
+ def __init__(self, total_blocks):
+ self.f_frsize = 1024**3 // 1000
+ self.f_blocks = total_blocks
+ self.f_bfree = total_blocks // 2
+ self.f_bavail = total_blocks // 2
+ self.f_files = 100
+ self.f_ffree = 75
+
+
+def test_collect_mounts_keeps_real_filesystems_once(tmp_path, monkeypatch):
+ proc = tmp_path / "proc"
+ (proc / "1").mkdir(parents=True)
+ (proc / "1" / "mounts").write_text(MOUNTS)
+ monkeypatch.setattr(collector, "HOST_PROC", proc)
+ root = tmp_path / "root"
+ monkeypatch.setattr(collector, "HOST_ROOT", root)
+ sizes = {".": 1000, "var/snap/firefox/common": 1000, "mnt/my data": 4000, "mnt/nfs": 8000}
+ monkeypatch.setattr(os, "statvfs", lambda path: FakeStat(sizes[os.path.relpath(path, root)]))
+ # /mnt/later was mounted after the collector started, so it's only a directory in the container.
+ monkeypatch.setattr(os.path, "ismount", lambda path: not str(path).endswith("later"))
+
+ mounts = collector.collect_mounts()
+
+ assert [(m["mount"], m["fs_type"]) for m in mounts] == [("/", "ext4"), ("/mnt/my data", "ext4"), ("/mnt/nfs", "nfs4")]
+ assert mounts[0]["used_pct"] == 50.0
+ assert mounts[0]["inodes_pct"] == 25.0
diff --git a/tests/test_dashboard_host.py b/tests/test_dashboard_host.py
new file mode 100644
index 0000000..be5e4ae
--- /dev/null
+++ b/tests/test_dashboard_host.py
@@ -0,0 +1,76 @@
+import os
+from datetime import datetime, timedelta, timezone
+
+os.environ.setdefault("DATABASE_URL", "sqlite://")
+os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0")
+
+import pytest
+from fastapi import HTTPException
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from src.agent import collector
+from src.backend import main
+from src.backend.models import Base, HostMetric, Server, User
+
+
+def db(monkeypatch):
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ session = sessionmaker(bind=engine)()
+ session.add(User(id=1, name="Owner", email="[email protected]", password_hash="x", role="admin"))
+ now = datetime.now(timezone.utc)
+ for server_id, name, age in ((1, "local", 3), (2, "nas", 2), (3, "pi", 1)):
+ session.add(Server(id=server_id, owner_id=1, hostname=name, check_interval_seconds=60, created_at=now - timedelta(days=age), collector_token_hash=main._hash_secret(f"token-{server_id}")))
+ session.commit()
+ monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1})
+ return session
+
+
+def _ingest(db, server_id, **server):
+ main.ingest_metrics(main.MetricsIngestRequest(server_id=server_id, server=server, host={"cpu_utilization_pct": 10, "ram_pct": 20, "disk_usage_pct": 30}), authorization=f"Bearer token-{server_id}", session_id=None, db=db)
+
+
+def test_dashboard_host_is_listed_first_and_flagged(db):
+ _ingest(db, 1, dashboard_host=True)
+ _ingest(db, 2)
+
+ servers = main.monitoring_overview(session_id="s", db=db)["servers"]
+
+ assert [(s["hostname"], s["is_dashboard_host"]) for s in servers] == [("local", True), ("pi", False), ("nas", False)]
+
+
+def test_only_latest_claimant_is_dashboard_host_and_flag_can_be_dropped(db):
+ _ingest(db, 1, dashboard_host=True)
+ _ingest(db, 2, dashboard_host=True)
+ assert [s.id for s in db.query(Server).filter_by(is_dashboard_host=True)] == [2]
+
+ _ingest(db, 2)
+ assert db.query(Server).filter_by(is_dashboard_host=True).count() == 0
+
+
+def test_history_returns_recent_samples_oldest_first(db):
+ now = datetime.now(timezone.utc)
+ for minutes_ago, cpu in ((90, 1), (20, 2), (5, 3)):
+ db.add(HostMetric(server_id=1, time=now - timedelta(minutes=minutes_ago), cpu_utilization_pct=cpu, ram_pct=50, disk_usage_pct=60))
+ db.add(HostMetric(server_id=2, time=now, cpu_utilization_pct=99))
+ db.commit()
+
+ samples = main.server_history(1, minutes=30, session_id="s", db=db)["samples"]
+
+ assert [(s["cpu"], s["ram"], s["disk"]) for s in samples] == [(2, 50, 60), (3, 50, 60)]
+
+
+def test_history_of_unknown_server_is_404(db):
+ with pytest.raises(HTTPException) as exc:
+ main.server_history(99, session_id="s", db=db)
+ assert exc.value.status_code == 404
+
+
+def test_collector_sends_flag_only_when_configured(monkeypatch):
+ monkeypatch.setenv("HOMELAB_DASHBOARD_HOST", "true")
+ assert collector.collect_host()[0]["dashboard_host"] is True
+
+ monkeypatch.delenv("HOMELAB_DASHBOARD_HOST")
+ assert "dashboard_host" not in collector.collect_host()[0]
diff --git a/tests/test_rules.py b/tests/test_rules.py
index d5ea971..d508e1c 100644
--- a/tests/test_rules.py
+++ b/tests/test_rules.py
@@ -10,7 +10,7 @@ from sqlalchemy.orm import sessionmaker
from src.backend import main, rules
from src.backend.alerts import CRITICAL, WARNING
-from src.backend.models import AuditLog, Base, IncidentAlert, Server, User
+from src.backend.models import AuditLog, Base, IncidentAlert, MountMetric, Server, User
TOKEN = "collector-token"
@@ -42,9 +42,9 @@ def test_host_alert_needs_sustained_breach(db):
assert _active(db) == {}
_ingest(db, disk_usage_pct=90)
- alert = _active(db)["host:disk"]
+ alert = _active(db)["host:disk:/"]
assert alert.severity == WARNING
- assert alert.title == "Disk usage on nas is 90%"
+ assert alert.title == "Disk usage on nas:/ is 90%"
def test_single_spike_does_not_alert(db):
@@ -58,14 +58,14 @@ def test_open_alert_stays_open_until_value_drops_and_escalates_only_when_sustain
for _ in range(3):
_ingest(db, disk_usage_pct=90)
_ingest(db, disk_usage_pct=97)
- assert _active(db)["host:disk"].severity == WARNING
+ assert _active(db)["host:disk:/"].severity == WARNING
_ingest(db, disk_usage_pct=97)
_ingest(db, disk_usage_pct=97)
- assert _active(db)["host:disk"].severity == CRITICAL
+ assert _active(db)["host:disk:/"].severity == CRITICAL
_ingest(db, disk_usage_pct=88)
- assert _active(db)["host:disk"].severity == WARNING
+ assert _active(db)["host:disk:/"].severity == WARNING
_ingest(db, disk_usage_pct=50)
assert _active(db) == {}
@@ -120,3 +120,59 @@ def test_corrupt_saved_thresholds_fall_back_to_defaults(db):
db.commit()
assert main.get_alert_thresholds(db) == rules.DEFAULT_THRESHOLDS
+
+
+def _mount(mount, used_pct, inodes_pct=10.0, total_gb=2000.0):
+ return {"mount": mount, "fs_type": "ext4", "total_gb": total_gb, "used_gb": total_gb * used_pct / 100, "used_pct": used_pct, "inodes_pct": inodes_pct}
+
+
+def test_each_mount_gets_its_own_disk_alert(db):
+ for _ in range(3):
+ _ingest(db, disk_usage_pct=40, mounts=[_mount("/", 40), _mount("/mnt/data", 96), _mount("/mnt/backup", 88, inodes_pct=99)])
+
+ active = _active(db)
+ assert set(active) == {"host:disk:/mnt/data", "host:disk:/mnt/backup", "host:inodes:/mnt/backup"}
+ assert active["host:disk:/mnt/data"].severity == CRITICAL
+ assert active["host:disk:/mnt/data"].title == "Disk usage on nas:/mnt/data is 96%"
+ assert "(1.9 TB of 2.0 TB used)" in active["host:disk:/mnt/data"].description
+ assert active["host:disk:/mnt/backup"].severity == WARNING
+
+
+def test_unmounted_disk_alert_resolves(db):
+ for _ in range(3):
+ _ingest(db, mounts=[_mount("/", 40), _mount("/mnt/data", 96)])
+ _ingest(db, mounts=[_mount("/", 40)])
+
+ assert _active(db) == {}
+
+
+def test_mount_needs_sustained_breach_of_its_own(db):
+ _ingest(db, mounts=[_mount("/", 40)])
+ _ingest(db, mounts=[_mount("/", 40)])
+ _ingest(db, mounts=[_mount("/", 40), _mount("/mnt/new", 99)])
+
+ assert _active(db) == {}
+
+
+def test_mount_rows_are_stored_and_bad_entries_ignored(db):
+ _ingest(db, mounts=[_mount("/", 40), {"mount": ""}, "junk", {"used_pct": 5}])
+
+ assert [row.mount for row in db.query(MountMetric)] == ["/"]
+
+
+def test_mount_path_with_braces_does_not_break_titles(db):
+ for _ in range(3):
+ _ingest(db, mounts=[_mount("/mnt/{odd}", 99)])
+
+ assert _active(db)["host:disk:/mnt/{odd}"].title == "Disk usage on nas:/mnt/{odd} is 99%"
+
+
+def test_overview_shows_latest_mounts_and_disk_thresholds(db, monkeypatch):
+ monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1})
+ _ingest(db, mounts=[_mount("/", 10)])
+ _ingest(db, mounts=[_mount("/", 20), _mount("/mnt/data", 30)])
+
+ overview = main.monitoring_overview(session_id="s", db=db)
+
+ assert [(m["mount"], m["used_pct"]) for m in overview["servers"][0]["mounts"]] == [("/", 20), ("/mnt/data", 30)]
+ assert overview["disk_thresholds"]["disk_critical_pct"] == rules.DEFAULT_THRESHOLDS["disk_critical_pct"]