Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.
Python ai-agentsaiopsdashboarddockerfastapihomelabincident-managementllmmonitoringopenaireactroot-cause-analysisself-hosteduptime-monitoring
refact: edits on dashboard UI
karloks2005053d6105d23c
diff --git a/docker-compose-example.yml b/docker-compose-example.ymlindex 930de47..94a88f4 100644--- a/docker-compose-example.yml+++ b/docker-compose-example.yml@@ -84,6 +84,8 @@ services:restart: unless-stoppedenvironment:HOMELAB_API_URL: http://backend:8000+ # Marks this collector as the machine running the dashboard (the charts at the top).+ HOMELAB_DASHBOARD_HOST: "true"# Nothing to configure: this machine shows up under "New servers detected"# on the dashboard. An optional enrollment key skips that approval.HOMELAB_ENROLL_KEY: ${HOMELAB_ENROLL_KEY:-}diff --git a/docker-compose.server.yml b/docker-compose.server.ymlindex c0daa73..46606c7 100644--- a/docker-compose.server.yml+++ b/docker-compose.server.yml@@ -81,6 +81,8 @@ services:restart: unless-stoppedenvironment:HOMELAB_API_URL: http://backend:8000+ # Marks this collector as the machine running the dashboard (the charts at the top).+ HOMELAB_DASHBOARD_HOST: "true"# Nothing to configure: this machine shows up under "New servers detected"# on the dashboard. An optional enrollment key skips that approval.HOMELAB_ENROLL_KEY: ${HOMELAB_ENROLL_KEY:-}diff --git a/docker-compose.yml b/docker-compose.ymlindex 106fc3a..a58017e 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -71,6 +71,8 @@ services:restart: unless-stoppedenvironment:HOMELAB_API_URL: http://backend:8000+ # Marks this collector as the machine running the dashboard (the charts at the top).+ HOMELAB_DASHBOARD_HOST: "true"# Nothing to configure: this machine shows up under "New servers detected"# on the dashboard. An optional enrollment key skips that approval.HOMELAB_ENROLL_KEY: ${HOMELAB_ENROLL_KEY:-}diff --git a/src/.claude/settings.json b/src/.claude/settings.jsonnew file mode 100644index 0000000..48983f6--- /dev/null+++ b/src/.claude/settings.json@@ -0,0 +1 @@+{ "includeCoAuthoredBy": false }\ No newline at end of filediff --git a/src/agent/collector.py b/src/agent/collector.pyindex 5c03d93..240bfd0 100644--- a/src/agent/collector.py+++ b/src/agent/collector.py@@ -69,6 +69,69 @@ def _network_totals() -> tuple[int, int]:return sent, received+# Filesystems worth reporting: anything backed by a block device, plus these pooled/network ones.+NON_DEVICE_FS_TYPES = {"zfs", "nfs", "nfs4", "cifs", "smb3", "fuse.mergerfs", "fuse.sshfs"}+SKIPPED_FS_TYPES = {"squashfs", "overlay", "tmpfs", "devtmpfs"}+SKIPPED_MOUNT_PREFIXES = ("/proc", "/sys", "/dev", "/run", "/snap", "/var/snap", "/var/lib/docker", "/var/lib/containerd", "/var/lib/kubelet")+++def _decode_mount_path(path: str) -> str:+ """/proc mounts escapes spaces, tabs, newlines and backslashes as octal."""+ for escaped, char in (("\\040", " "), ("\\011", "\t"), ("\\012", "\n"), ("\\134", "\\")):+ path = path.replace(escaped, char)+ return path+++def _mount_entries() -> list[tuple[str, str, str]]:+ """(source, mount point, fs type) as the host sees them."""+ # PID 1's table is the host's; "self" would be this container's own mounts.+ for candidate in (HOST_PROC / "1/mounts", HOST_PROC / "self/mounts"):+ try:+ lines = candidate.read_text().splitlines()+ except OSError:+ continue+ return [(fields[0], _decode_mount_path(fields[1]), fields[2]) for fields in (line.split() for line in lines) if len(fields) >= 3]+ return []+++def collect_mounts() -> list[dict[str, Any]]:+ """Usage of every real filesystem on the host, one entry per device."""+ by_device: dict[tuple[str, int], dict[str, Any]] = {}+ for source, mount, fs_type in _mount_entries():+ if fs_type in SKIPPED_FS_TYPES or not (source.startswith("/dev/") or fs_type in NON_DEVICE_FS_TYPES):+ continue+ if mount != "/" and mount.startswith(SKIPPED_MOUNT_PREFIXES):+ continue+ local_path = HOST_ROOT / mount.lstrip("/")+ # A mount made after the collector started is only an empty directory here; skip it rather+ # than report the parent filesystem under its name.+ if mount != "/" and not os.path.ismount(local_path):+ continue+ try:+ stats = os.statvfs(local_path)+ except OSError:+ continue+ total = stats.f_blocks * stats.f_frsize+ if not total:+ continue+ used = total - stats.f_bfree * stats.f_frsize+ # Space available to non-root users is what fills up, so percentages use it like `df` does.+ usable = used + stats.f_bavail * stats.f_frsize+ entry = {+ "mount": mount[:255],+ "fs_type": fs_type[:64],+ "total_gb": round(total / GiB, 2),+ "used_gb": round(used / GiB, 2),+ "used_pct": round(100 * used / usable, 2) if usable else 0,+ "inodes_pct": round(100 * (stats.f_files - stats.f_ffree) / stats.f_files, 2) if stats.f_files else None,+ }+ # Bind mounts and btrfs subvolumes show the same device several times; keep its shortest path.+ key = (source, total)+ if key not in by_device or len(mount) < len(by_device[key]["mount"]):+ by_device[key] = entry+ return sorted(by_device.values(), key=lambda entry: entry["mount"])++def collect_host() -> tuple[dict[str, Any], dict[str, Any]]:before_total, before_idle = _cpu_times()time.sleep(0.15)@@ -79,9 +142,10 @@ def collect_host() -> tuple[dict[str, Any], dict[str, Any]]:disk = os.statvfs(HOST_ROOT)inode_total = disk.f_filesinode_pct = (100 * (inode_total - disk.f_ffree) / inode_total) if inode_total else None- disk_total = disk.f_blocks * disk.f_frsize- disk_free = disk.f_bavail * disk.f_frsize- disk_pct = (100 * (disk_total - disk_free) / disk_total) if disk_total else 0+ disk_used = (disk.f_blocks - disk.f_bfree) * disk.f_frsize+ disk_usable = disk_used + disk.f_bavail * disk.f_frsize+ # Same formula as `df` and the per-mount figures.+ disk_pct = (100 * disk_used / disk_usable) if disk_usable else 0try:loads = os.getloadavg()except (AttributeError, OSError):@@ -97,6 +161,7 @@ def collect_host() -> tuple[dict[str, Any], dict[str, Any]]:"net_bytes_sent_mb": round(sent / MiB, 3),"net_bytes_recv_mb": round(received / MiB, 3),"system_uptime_seconds": round(uptime, 1),+ "mounts": collect_mounts(),}server = {"hostname": _host_hostname(),@@ -104,6 +169,9 @@ def collect_host() -> tuple[dict[str, Any], dict[str, Any]]:"cpu_cores": _host_cpu_cores(),"total_ram_gb": round(_memory_total_gb(), 3),}+ # Set only on the collector in the dashboard's own compose stack; the dashboard's top charts follow it.+ if os.getenv("HOMELAB_DASHBOARD_HOST", "").lower() in {"1", "true", "yes"}:+ server["dashboard_host"] = Truereturn server, hostdiff --git a/src/backend/main.py b/src/backend/main.pyindex b731164..8bdb759 100644--- a/src/backend/main.py+++ b/src/backend/main.py@@ -21,7 +21,7 @@ from sqlalchemy import case, inspect, textfrom .alerts import CONTAINER_RULE_PREFIX, CRITICAL, SERVER_OFFLINE_RULE, SEVERITIES, active_rule_keys, raise_alert, resolve_alerts, sync_alertsfrom .auth import authenticate_user, get_recent_logs, get_user_from_session, logout_user, record_audit_log, register_userfrom .db import SessionLocal, check_database, get_db-from .models import AppSetting, AuditLog, Base, ChatMessage, ContainerMetric, Conversation, EnrollmentKey, HostMetric, IncidentAlert, PendingCollector, Server, User+from .models import AppSetting, AuditLog, Base, ChatMessage, ContainerMetric, Conversation, EnrollmentKey, HostMetric, IncidentAlert, MountMetric, PendingCollector, Server, Userfrom .db import enginefrom .config import (@@ -151,6 +151,8 @@ def startup_event():# duplicated on every check-in with no way to resolve, so close them; the engine re-raises live ones.connection.execute(text("UPDATE incident_alerts SET rule_key = :rule WHERE rule_key IS NULL AND description = :description"), {"rule": SERVER_OFFLINE_RULE, "description": OFFLINE_ALERT_DESCRIPTION})connection.execute(text("UPDATE incident_alerts SET status = 'resolved', resolved_at = NOW() WHERE rule_key IS NULL AND status = 'active'"))+ # Disk and inode alerts became per mount; the old keys were for the root filesystem.+ connection.execute(text("UPDATE incident_alerts SET rule_key = rule_key || ':/' WHERE rule_key IN ('host:disk', 'host:inodes')"))if "servers" in inspect(engine).get_table_names():columns = {column["name"] for column in inspect(engine).get_columns("servers")}if "collector_token_hash" not in columns:@@ -159,6 +161,9 @@ def startup_event():if "machine_id" not in columns:with engine.begin() as connection:connection.execute(text("ALTER TABLE servers ADD COLUMN machine_id VARCHAR(64)"))+ if "is_dashboard_host" not in columns:+ with engine.begin() as connection:+ connection.execute(text("ALTER TABLE servers ADD COLUMN is_dashboard_host BOOLEAN NOT NULL DEFAULT FALSE"))db = SessionLocal()try:ensure_admin_exists(db)@@ -370,6 +375,7 @@ def purge_old_metrics(db, now: datetime | None = None) -> int:cutoff = (now or datetime.now(timezone.utc)) - timedelta(days=get_metrics_retention_days(db))deleted = db.query(HostMetric).filter(HostMetric.time < cutoff).delete(synchronize_session=False)deleted += db.query(ContainerMetric).filter(ContainerMetric.time < cutoff).delete(synchronize_session=False)+ deleted += db.query(MountMetric).filter(MountMetric.time < cutoff).delete(synchronize_session=False)db.commit()return deleted@@ -423,7 +429,8 @@ OVERVIEW_ALERT_LIMIT = 50def monitoring_overview(session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):# Servers belong to the install: everyone signed in can view them, only admins manage them.user = _current_user(session_id, db)- servers = db.query(Server).order_by(Server.created_at.desc()).all()+ # The dashboard's own machine first, then the newest servers.+ servers = db.query(Server).order_by(Server.is_dashboard_host.desc(), Server.created_at.desc()).all()result = []for server in servers:metric = db.query(HostMetric).filter(HostMetric.server_id == server.id).order_by(HostMetric.time.desc()).first()@@ -431,12 +438,14 @@ def monitoring_overview(session_id: Annotated[str | None, Cookie(alias=SESSION_Clatest = {}for item in containers:latest.setdefault(item.container_id, item)+ mounts = db.query(MountMetric).filter(MountMetric.server_id == server.id, MountMetric.time == metric.time).order_by(MountMetric.mount).all() if metric else []result.append({"id": server.id, "hostname": server.hostname, "ip_address": server.ip_address, "os_info": server.os_info,"cpu_cores": server.cpu_cores, "total_ram_gb": server.total_ram_gb,- "check_interval_seconds": server.check_interval_seconds,+ "check_interval_seconds": server.check_interval_seconds, "is_dashboard_host": server.is_dashboard_host,"last_seen_at": server.last_seen_at.isoformat() if server.last_seen_at else None,"host": ({key: getattr(metric, key) for key in ("time", "cpu_utilization_pct", "load_1m", "load_5m", "load_15m", "ram_used_gb", "ram_available_gb", "ram_pct", "swap_pct", "disk_usage_pct", "disk_inodes_pct", "net_bytes_sent_mb", "net_bytes_recv_mb", "system_uptime_seconds")} if metric else None),+ "mounts": [{key: getattr(m, key) for key in ("mount", "fs_type", "total_gb", "used_gb", "used_pct", "inodes_pct")} for m in mounts],"containers": [{"container_id": c.container_id, "name": c.name, "image": c.image, "status": c.status, "restart_count": c.restart_count, "cpu_pct": c.cpu_pct, "memory_used_mb": c.memory_used_mb, "memory_pct": c.memory_pct, "healthcheck_status": c.healthcheck_status, "time": c.time.isoformat()} for c in latest.values()],})active_alerts = db.query(IncidentAlert).filter(IncidentAlert.status == "active")@@ -444,13 +453,30 @@ def monitoring_overview(session_id: Annotated[str | None, Cookie(alias=SESSION_Cseverity_rank = case({severity: rank for rank, severity in enumerate(reversed(SEVERITIES))}, value=IncidentAlert.severity, else_=len(SEVERITIES))alerts = active_alerts.order_by(severity_rank, IncidentAlert.created_at.desc()).limit(OVERVIEW_ALERT_LIMIT).all()hostnames = {server.id: server.hostname for server in servers}+ thresholds = get_alert_thresholds(db)pending = visible_pending_collectors(db) if _is_admin(db, user["id"]) else []return {"servers": result, "pending_collectors": pending, "alert_count": active_alerts.count(),+ # Lets the dashboard color disk bars the same way the alert rules judge them.+ "disk_thresholds": {key: thresholds[key] for key in ("disk_warning_pct", "disk_critical_pct", "inodes_warning_pct", "inodes_critical_pct")},"alerts": [{"id": a.id, "server_id": a.server_id, "hostname": hostnames.get(a.server_id), "severity": a.severity, "title": a.title, "description": a.description, "created_at": a.created_at.isoformat()} for a in alerts],}+HISTORY_MAX_MINUTES = 24 * 60++[email protected]("/monitoring/servers/{server_id}/history")+def server_history(server_id: int, minutes: int = 30, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ """CPU, memory and root disk usage of one server over the last ``minutes``, oldest first."""+ _current_user(session_id, db)+ if not db.query(Server).filter(Server.id == server_id).first():+ raise HTTPException(status_code=404, detail="Server not found.")+ since = datetime.now(timezone.utc) - timedelta(minutes=max(1, min(minutes, HISTORY_MAX_MINUTES)))+ rows = db.query(HostMetric.time, HostMetric.cpu_utilization_pct, HostMetric.ram_pct, HostMetric.disk_usage_pct).filter(HostMetric.server_id == server_id, HostMetric.time >= since).order_by(HostMetric.time).all()+ return {"server_id": server_id, "samples": [{"time": row.time.isoformat(), "cpu": row.cpu_utilization_pct, "ram": row.ram_pct, "disk": row.disk_usage_pct} for row in rows]}++CONTAINER_DETAIL_FIELDS = ("status", "healthcheck_status", "restart_count", "cpu_pct", "memory_used_mb", "memory_limit_mb", "memory_pct","net_rx_mb", "net_tx_mb", "block_read_mb", "block_write_mb", "pids",@@ -729,6 +755,7 @@ def delete_monitored_server(server_id: int, session_id: Annotated[str | None, Codb.query(HostMetric).filter(HostMetric.server_id == server.id).delete(synchronize_session=False)db.query(ContainerMetric).filter(ContainerMetric.server_id == server.id).delete(synchronize_session=False)+ db.query(MountMetric).filter(MountMetric.server_id == server.id).delete(synchronize_session=False)db.query(IncidentAlert).filter(IncidentAlert.server_id == server.id).delete(synchronize_session=False)db.query(PendingCollector).filter(PendingCollector.approved_server_id == server.id).delete(synchronize_session=False)db.delete(server)@@ -763,8 +790,17 @@ def ingest_metrics(payload: MetricsIngestRequest, authorization: Annotated[str |for field in ("hostname", "ip_address", "os_info", "cpu_cores", "total_ram_gb"):if field in payload.server and payload.server[field] is not None:setattr(server, field, payload.server[field])+ # Only one server can be the dashboard's own machine; the latest collector to claim it wins.+ server.is_dashboard_host = payload.server.get("dashboard_host") is True+ if server.is_dashboard_host:+ db.query(Server).filter(Server.id != server.id, Server.is_dashboard_host.is_(True)).update({"is_dashboard_host": False}, synchronize_session=False)now = datetime.now(timezone.utc)db.add(HostMetric(server_id=server.id, time=now, **{k: v for k, v in payload.host.items() if k in HostMetric.__table__.columns.keys() and k not in {"id", "server_id", "time"}}))+ mounts = payload.host.get("mounts")+ for mount in mounts if isinstance(mounts, list) else []:+ if isinstance(mount, dict) and isinstance(mount.get("mount"), str) and mount["mount"]:+ allowed = {k: v for k, v in mount.items() if k in MountMetric.__table__.columns.keys() and k not in {"id", "server_id", "time"}}+ db.add(MountMetric(server_id=server.id, time=now, **allowed))containers = [c for c in payload.containers if c.get("container_id") and c.get("status")]for container in containers:allowed = {k: v for k, v in container.items() if k in ContainerMetric.__table__.columns.keys() and k not in {"id", "server_id", "time"}}diff --git a/src/backend/models.py b/src/backend/models.pyindex 5ceb4a6..61e8a73 100644--- a/src/backend/models.py+++ b/src/backend/models.py@@ -101,6 +101,8 @@ class Server(Base):collector_token_hash: Mapped[Optional[str]] = mapped_column(String(64), nullable=True, unique=True)# Stable host identity reported by self-enrolling collectors, so a reinstall maps back to the same server.machine_id: Mapped[Optional[str]] = mapped_column(String(64), nullable=True, index=True)+ # The machine running the dashboard itself, as reported by the collector in its compose stack.+ is_dashboard_host: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)last_seen_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True)created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False)@@ -154,6 +156,22 @@ class HostMetric(Base):system_uptime_seconds: Mapped[Optional[float]] = mapped_column(Float, nullable=True)+class MountMetric(Base):+ """Usage of one filesystem at one check-in; shares its time with that check-in's HostMetric."""++ __tablename__ = "mount_metrics"++ id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)+ server_id: Mapped[int] = mapped_column(ForeignKey("servers.id", ondelete="CASCADE"), index=True, nullable=False)+ time: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), index=True, nullable=False)+ mount: Mapped[str] = mapped_column(String(255), nullable=False)+ fs_type: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)+ total_gb: Mapped[Optional[float]] = mapped_column(Float, nullable=True)+ used_gb: Mapped[Optional[float]] = mapped_column(Float, nullable=True)+ used_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)+ inodes_pct: Mapped[Optional[float]] = mapped_column(Float, nullable=True)++class ContainerMetric(Base):__tablename__ = "container_metrics"diff --git a/src/backend/rules.py b/src/backend/rules.pyindex 4215f95..62df58e 100644--- a/src/backend/rules.py+++ b/src/backend/rules.py@@ -10,7 +10,7 @@ from datetime import datetime, timedeltafrom sqlalchemy import funcfrom .alerts import CONTAINER_RULE_PREFIX, CRITICAL, WARNING-from .models import ContainerMetric, HostMetric+from .models import ContainerMetric, HostMetric, MountMetricHOST_RULE_PREFIX = "host:"@@ -66,44 +66,74 @@ def _level(value: float | None, warning: float, critical: float | None) -> int:return 1 if value >= warning else 0+def _fire_level(values: list, warning: float, critical: float | None, sustained: int, active: bool) -> int:+ """Alert level for a metric given its recent values, newest first."""+ levels = [_level(value, warning, critical) for value in values]+ # Opening an alert, or escalating it to critical, needs every recent check-in to breach;+ # an open alert stays open while the latest check-in still breaches.+ sustained_level = min(levels) if len(levels) >= sustained else 0+ return max(sustained_level, min(levels[0], 1)) if active and levels else sustained_level+++def _escape(text: str) -> str:+ return text.replace("{", "{{").replace("}", "}}")+++def _gb(value: float | None) -> str:+ if value is None:+ return "?"+ return f"{value / 1024:.1f} TB" if value >= 1024 else f"{value:.0f} GB"++def host_rules(db, server, thresholds: dict, active_keys: set[str]) -> dict[str, tuple[str, str, str]]:- """Evaluate the host rules against the server's most recent check-ins (newest first)."""+ """Evaluate the host rules against the server's most recent check-ins."""sustained = thresholds["sustained_checks"]samples = db.query(HostMetric).filter(HostMetric.server_id == server.id).order_by(HostMetric.time.desc(), HostMetric.id.desc()).limit(sustained).all()if not samples:return {}+ # Titles are format strings for the latest value, so escape braces in names we put into them.+ host = _escape(server.hostname)cores = server.cpu_cores or Nonedef load_per_core(sample):return sample.load_5m / cores if cores and sample.load_5m is not None else None- checks = (- # (rule, value getter, warning, critical, title, description)- ("disk", lambda s: s.disk_usage_pct, thresholds["disk_warning_pct"], thresholds["disk_critical_pct"],- "Disk usage on {host} is {value:.0f}%", "The root filesystem is {value:.1f}% full. Free up space before it fills completely."),- ("inodes", lambda s: s.disk_inodes_pct, thresholds["inodes_warning_pct"], thresholds["inodes_critical_pct"],- "Inode usage on {host} is {value:.0f}%", "{value:.1f}% of inodes on the root filesystem are used. Many small files (caches, logs, mail queues) are the usual cause."),- ("memory", lambda s: s.ram_pct, thresholds["memory_warning_pct"], thresholds["memory_critical_pct"],- "Memory usage on {host} is {value:.0f}%", "{value:.1f}% of RAM is in use. Check the containers and processes using the most memory."),- ("swap", lambda s: s.swap_pct, thresholds["swap_warning_pct"], None,- "Swap usage on {host} is {value:.0f}%", "{value:.1f}% of swap has been in use for several check-ins, an early sign of memory pressure."),- ("load", load_per_core, thresholds["load_warning_per_core"], thresholds["load_critical_per_core"],- "Load on {host} is {value:.1f}× its cores", "The 5-minute load average is {value:.2f} per CPU core across " + f"{cores or '?'} cores. Work is queuing for the CPU."),- )+ # (rule key, values newest first, warning, critical, title, description), with {value} filled in from the latest value.+ checks = [+ (f"{HOST_RULE_PREFIX}memory", [s.ram_pct for s in samples], thresholds["memory_warning_pct"], thresholds["memory_critical_pct"],+ f"Memory usage on {host} is {{value:.0f}}%", "{value:.1f}% of RAM is in use. Check the containers and processes using the most memory."),+ (f"{HOST_RULE_PREFIX}swap", [s.swap_pct for s in samples], thresholds["swap_warning_pct"], None,+ f"Swap usage on {host} is {{value:.0f}}%", "{value:.1f}% of swap has been in use for several check-ins, an early sign of memory pressure."),+ (f"{HOST_RULE_PREFIX}load", [load_per_core(s) for s in samples], thresholds["load_warning_per_core"], thresholds["load_critical_per_core"],+ f"Load on {host} is {{value:.1f}}× its cores", f"The 5-minute load average is {{value:.2f}} per CPU core across {cores or '?'} cores. Work is queuing for the CPU."),+ ]++ times = [sample.time for sample in samples]+ by_mount: dict[str, dict] = {}+ for row in db.query(MountMetric).filter(MountMetric.server_id == server.id, MountMetric.time.in_(times)):+ by_mount.setdefault(row.mount, {})[row.time] = row+ current = {mount: rows[times[0]] for mount, rows in by_mount.items() if times[0] in rows}+ if current:+ for raw_mount, row in sorted(current.items()):+ mount = _escape(raw_mount)+ history = [by_mount[raw_mount].get(time) for time in times]+ size = f"{_gb(row.used_gb)} of {_gb(row.total_gb)} used"+ checks.append((f"{HOST_RULE_PREFIX}disk:{raw_mount}", [r.used_pct if r else None for r in history], thresholds["disk_warning_pct"], thresholds["disk_critical_pct"],+ f"Disk usage on {host}:{mount} is {{value:.0f}}%", f"{mount} is {{value:.1f}}% full ({size}). Free up space before it fills completely."))+ checks.append((f"{HOST_RULE_PREFIX}inodes:{raw_mount}", [r.inodes_pct if r else None for r in history], thresholds["inodes_warning_pct"], thresholds["inodes_critical_pct"],+ f"Inode usage on {host}:{mount} is {{value:.0f}}%", f"{{value:.1f}}% of inodes on {mount} are used. Many small files (caches, logs, mail queues) are the usual cause."))+ else:+ # Collectors from before per-mount reporting only send the root filesystem.+ checks.append((f"{HOST_RULE_PREFIX}disk:/", [s.disk_usage_pct for s in samples], thresholds["disk_warning_pct"], thresholds["disk_critical_pct"],+ f"Disk usage on {host}:/ is {{value:.0f}}%", "/ is {value:.1f}% full. Free up space before it fills completely."))+ checks.append((f"{HOST_RULE_PREFIX}inodes:/", [s.disk_inodes_pct for s in samples], thresholds["inodes_warning_pct"], thresholds["inodes_critical_pct"],+ f"Inode usage on {host}:/ is {{value:.0f}}%", "{value:.1f}% of inodes on / are used. Many small files (caches, logs, mail queues) are the usual cause."))+firing: dict[str, tuple[str, str, str]] = {}- for rule, value_of, warning, critical, title, description in checks:- key = f"{HOST_RULE_PREFIX}{rule}"- latest_value = value_of(samples[0])- levels = [_level(value_of(sample), warning, critical) for sample in samples]- # Opening an alert, or escalating it to critical, needs every recent check-in to breach.- sustained_level = min(levels) if len(samples) >= sustained else 0- level = max(sustained_level, min(levels[0], 1)) if key in active_keys else sustained_level+ for key, values, warning, critical, title, description in checks:+ level = _fire_level(values, warning, critical, sustained, key in active_keys)if level:- firing[key] = (- CRITICAL if level == 2 else WARNING,- title.format(host=server.hostname, value=latest_value),- description.format(value=latest_value),- )+ firing[key] = (CRITICAL if level == 2 else WARNING, title.format(value=values[0]), description.format(value=values[0]))return firingdiff --git a/src/frontend/src/App.jsx b/src/frontend/src/App.jsxindex ced5a0d..0874436 100644--- a/src/frontend/src/App.jsx+++ b/src/frontend/src/App.jsx@@ -431,6 +431,73 @@ function RegisterPage({ onRegister }) {// Izvan ili na vrhu komponente postavi guard za izbjegavanje paralelnih pozivalet isFetchingOverview = false+// Collectors from before per-mount reporting only send the root filesystem's usage.+function serverMounts(server) {+ if (server.mounts?.length) return server.mounts+ const host = server.host+ return host?.disk_usage_pct != null ? [{ mount: '/', used_pct: host.disk_usage_pct, inodes_pct: host.disk_inodes_pct }] : []+}++function fullestMount(mounts) {+ return mounts.reduce((fullest, mount) => (mount.used_pct != null && (!fullest || mount.used_pct > fullest.used_pct) ? mount : fullest), null)+}++const HOST_HISTORY_MINUTES = 30++// Adds the latest check-in unless it's already there (the dashboard polls faster than collectors report).+function appendHostSample(samples, sample) {+ const last = samples.at(-1)+ if (!sample.time || (last && new Date(sample.time) <= new Date(last.time))) return samples+ const cutoff = new Date(sample.time).getTime() - HOST_HISTORY_MINUTES * 60 * 1000+ return [...samples, sample].filter((item) => new Date(item.time).getTime() >= cutoff)+}++function formatGigabytes(value) {+ if (value == null) return '?'+ return value >= 1024 ? `${(value / 1024).toFixed(1)} TB` : `${value >= 10 ? value.toFixed(0) : value.toFixed(1)} GB`+}++const DISK_LEVEL_STYLES = [+ { bar: 'bg-emerald-400/70', text: 'text-[#cfcfcf]' },+ { bar: 'bg-amber-400/80', text: 'text-amber-200' },+ { bar: 'bg-red-400/80', text: 'text-red-300' },+]++function thresholdLevel(value, warning, critical) {+ if (value == null) return 0+ return value >= critical ? 2 : value >= warning ? 1 : 0+}++function StorageBars({ mounts, thresholds }) {+ if (!mounts.length) return null+ const t = { disk_warning_pct: 85, disk_critical_pct: 95, inodes_warning_pct: 85, inodes_critical_pct: 95, ...thresholds }+ return (+ <div className="mt-4">+ <div className="mb-2 text-xs uppercase tracking-[0.2em] text-[#7f8589]">Storage</div>+ <div className="space-y-2">+ {mounts.map((mount) => {+ const style = DISK_LEVEL_STYLES[thresholdLevel(mount.used_pct, t.disk_warning_pct, t.disk_critical_pct)]+ const inodeLevel = thresholdLevel(mount.inodes_pct, t.inodes_warning_pct, t.inodes_critical_pct)+ const used = Math.min(100, Math.max(0, mount.used_pct ?? 0))+ return (+ <div key={mount.mount} className="grid grid-cols-[minmax(0,9rem)_1fr_auto] items-center gap-3 text-sm sm:grid-cols-[minmax(0,12rem)_1fr_auto]">+ <span className="truncate font-mono text-xs text-[#bcbcbc]" title={mount.fs_type ? `${mount.mount} (${mount.fs_type})` : mount.mount}>{mount.mount}</span>+ <div className="h-2 overflow-hidden rounded-full bg-white/5" role="meter" aria-label={`${mount.mount} usage`} aria-valuemin={0} aria-valuemax={100} aria-valuenow={used}>+ <div className={`h-full rounded-full ${style.bar}`} style={{ width: `${used}%` }} />+ </div>+ <span className="whitespace-nowrap text-right text-xs text-[#9aa0a6]">+ {mount.total_gb != null && <span className="hidden sm:inline">{formatGigabytes(mount.used_gb)} / {formatGigabytes(mount.total_gb)} · </span>}+ <span className={`font-medium ${style.text}`}>{mount.used_pct == null ? '—' : `${Math.round(mount.used_pct)}%`}</span>+ {inodeLevel > 0 && <span className={`ml-2 ${DISK_LEVEL_STYLES[inodeLevel].text}`}>inodes {Math.round(mount.inodes_pct)}%</span>}+ </span>+ </div>+ )+ })}+ </div>+ </div>+ )+}+const ALERT_STYLES = {CRITICAL: { box: 'border-red-400/25 bg-red-500/5', badge: 'bg-red-500/15 text-red-300', title: 'text-red-200' },WARNING: { box: 'border-amber-400/20 bg-amber-400/5', badge: 'bg-amber-400/15 text-amber-300', title: 'text-amber-200' },@@ -442,7 +509,8 @@ function DashboardPage({ user, onLogout }) {const [error, setError] = useState('')const [showConnect, setShowConnect] = useState(false)const [deletingServerId, setDeletingServerId] = useState(null)- const [history, setHistory] = useState([])+ // Recent samples of the machine running the dashboard: loaded from the backend, then extended live.+ const [history, setHistory] = useState({ serverId: null, samples: [] })const refresh = async () => {// Ako je zahtjev već u tijeku, preskoči ovaj poziv@@ -454,19 +522,10 @@ const refresh = async () => {setOverview(data)setError('')- const metrics = (data.servers || []).map((server) => server.host).filter(Boolean)- if (metrics.length) {- const mean = (key) => metrics.reduce((sum, metric) => sum + (metric[key] || 0), 0) / metrics.length-- setHistory((current) => [- ...current,- {- time: new Date(),- cpu: mean('cpu_utilization_pct'),- ram: mean('ram_pct'),- disk: mean('disk_usage_pct')- }- ].slice(-40))+ const host = (data.servers || []).find((server) => server.is_dashboard_host)+ if (host?.host) {+ const sample = { time: host.host.time, cpu: host.host.cpu_utilization_pct, ram: host.host.ram_pct, disk: host.host.disk_usage_pct }+ setHistory((current) => current.serverId === host.id ? { ...current, samples: appendHostSample(current.samples, sample) } : current)}} catch (requestError) {// Ako backend vrati 401 (neautoriziran/odjavljen korisnik), odjavi ga i prekini petlju@@ -496,6 +555,19 @@ const refresh = async () => {return () => clearInterval(timer)}, [user?.id]) // <-- Stabilna instanca, okida se samo pri promjeni ID-a+ const servers = overview.servers || []+ const dashboardHost = servers.find((server) => server.is_dashboard_host)+ const dashboardHostId = dashboardHost?.id++ useEffect(() => {+ if (!dashboardHostId) return+ let cancelled = false+ apiRequest(`/monitoring/servers/${dashboardHostId}/history?minutes=${HOST_HISTORY_MINUTES}`, { method: 'GET' })+ .then((data) => { if (!cancelled) setHistory({ serverId: dashboardHostId, samples: data.samples }) })+ .catch(() => { if (!cancelled) setHistory({ serverId: dashboardHostId, samples: [] }) })+ return () => { cancelled = true }+ }, [dashboardHostId])+const changeInterval = async (server, value) => {try {await apiRequest(`/monitoring/servers/${server.id}/interval`, { method: 'PATCH', body: JSON.stringify({ check_interval_seconds: Number(value) }) })@@ -514,10 +586,9 @@ const refresh = async () => {setDeletingServerId(null)}}- const servers = overview.servers || []- const latestMetrics = servers.map((server) => server.host).filter(Boolean)- const average = (key) => latestMetrics.length ? (latestMetrics.reduce((sum, metric) => sum + (metric[key] || 0), 0) / latestMetrics.length).toFixed(1) : '—'const online = servers.filter((server) => server.last_seen_at && Date.now() - new Date(server.last_seen_at).getTime() < server.check_interval_seconds * 3000).length+ const hostSamples = history.serverId === dashboardHostId ? history.samples : []+ const hostMetric = dashboardHost?.hostconst runningContainers = servers.reduce((sum, server) => sum + server.containers.filter((container) => container.status === 'running').length, 0)// Regular users get a read-only view; the backend enforces the same rules.@@ -528,7 +599,7 @@ const refresh = async () => {<div className="flex flex-col gap-5"><header className="glass-panel rounded-[28px] p-5 md:p-6"><div className="flex flex-wrap items-end justify-between gap-4">- <div><div className="text-xs uppercase tracking-[0.22em] text-[#9aa0a6]">Infrastructure overview</div><h1 className="mt-1 text-3xl font-semibold text-white">Dashboard</h1><p className="mt-2 text-sm text-[#9aa0a6]">Live host and container health · updated every 15 seconds.</p></div>+ <div><div className="text-xs uppercase tracking-[0.22em] text-[#9aa0a6]">Infrastructure overview</div><h1 className="mt-1 text-3xl font-semibold text-white">Dashboard</h1><p className="mt-2 text-sm text-[#9aa0a6]">Live host and container health · updated every {LIVE_GRAPH_REFRESH_INTERVAL_SECONDS} seconds.</p></div>{isAdmin && <button type="button" onClick={() => setShowConnect((current) => !current)} aria-expanded={showConnect} className="rounded-xl bg-white px-4 py-2 text-sm font-medium text-slate-950">{showConnect ? 'Close' : 'Connect a server'}</button>}</div>{error && <div className="mt-4 rounded-xl border border-amber-400/30 bg-amber-400/5 p-3 text-sm text-amber-200">{error}</div>}@@ -538,20 +609,23 @@ const refresh = async () => {<section className="grid gap-4 sm:grid-cols-2 xl:grid-cols-3">{[['Servers online', `${online} / ${servers.length}`, 'Based on last check-in'],- ['Average CPU', latestMetrics.length ? `${average('cpu_utilization_pct')}%` : '—', 'Across reporting servers'],- ['Average memory', latestMetrics.length ? `${average('ram_pct')}%` : '—', 'RAM utilization'],- ['Average disk', latestMetrics.length ? `${average('disk_usage_pct')}%` : '—', 'Disk utilization'],['Containers running', String(runningContainers), 'Across monitored servers'],['Active incidents', String(overview.alert_count ?? overview.alerts?.length ?? 0), 'Needs attention'],].map(([label, value, detail]) => <div key={label} className="glass-panel rounded-[24px] p-5"><div className="text-sm text-[#9aa0a6]">{label}</div><div className="mt-3 text-3xl font-semibold text-white">{value}</div><div className="mt-2 text-xs text-[#7f8589]">{detail}</div></div>)}</section>- <section className="grid gap-4 xl:grid-cols-3">- {[- { key: 'cpu', title: 'CPU utilization', color: '#34d399', latest: average('cpu_utilization_pct') },- { key: 'ram', title: 'Memory utilization', color: '#60a5fa', latest: average('ram_pct') },- { key: 'disk', title: 'Disk utilization', color: '#c084fc', latest: average('disk_usage_pct') },- ].map((chart) => <MetricTrend key={chart.key} title={chart.title} color={chart.color} value={chart.latest} points={history.map((sample) => sample[chart.key])} times={history.map((sample) => sample.time)} />)}+ <section>+ <div className="mb-3 flex flex-wrap items-baseline justify-between gap-2 px-1">+ <div className="text-xs uppercase tracking-[0.2em] text-[#9aa0a6]">This machine{dashboardHost && <span className="ml-2 normal-case tracking-normal text-[#cfcfcf]">{dashboardHost.hostname}</span>}</div>+ <span className="text-xs text-[#7f8589]">Last {HOST_HISTORY_MINUTES} minutes</span>+ </div>+ {dashboardHost ? <div className="grid gap-4 xl:grid-cols-3">+ {[+ { key: 'cpu', title: 'CPU utilization', color: '#34d399', latest: hostMetric?.cpu_utilization_pct },+ { key: 'ram', title: 'Memory utilization', color: '#60a5fa', latest: hostMetric?.ram_pct },+ { key: 'disk', title: 'Disk usage (/)', color: '#c084fc', latest: hostMetric?.disk_usage_pct },+ ].map((chart) => <MetricTrend key={chart.key} title={chart.title} color={chart.color} value={chart.latest == null ? '—' : Number(chart.latest).toFixed(1)} points={hostSamples.map((sample) => sample[chart.key])} times={hostSamples.map((sample) => sample.time)} />)}+ </div> : <div className="glass-panel rounded-[24px] p-5 text-sm text-[#9aa0a6]">The machine running the dashboard isn't reporting yet. Its collector starts with the dashboard's compose stack and shows up here once it's added under <span className="text-white">New servers detected</span>.</div>}</section><section className="glass-panel rounded-[28px] p-5 md:p-6">@@ -559,10 +633,13 @@ const refresh = async () => {{isAdmin && <PendingServers pending={overview.pending_collectors || []} onChanged={refresh} />}{!servers.length ? <div className="rounded-2xl border border-dashed border-neutral-700 p-8 text-center"><div className="text-lg font-medium text-white">No servers registered</div><p className="mt-2 text-sm text-[#9aa0a6]">{isAdmin ? <>Click <span className="text-white">Connect a server</span> above and run the command on each machine you want to monitor.</> : 'An admin needs to connect a server before it shows up here.'}</p></div> : <div className="space-y-4">{servers.map((server) => {const metric = server.host+ const mounts = serverMounts(server)+ const fullest = fullestMount(mounts)const fresh = server.last_seen_at && Date.now() - new Date(server.last_seen_at).getTime() < server.check_interval_seconds * 3000return <article key={server.id} className="rounded-2xl border border-neutral-800 bg-[#0d1013]/70 p-4 md:p-5">- <div className="flex flex-wrap items-start justify-between gap-3"><div><div className="flex items-center gap-2"><span className={`h-2.5 w-2.5 rounded-full ${fresh ? 'bg-emerald-400' : 'bg-neutral-600'}`} /><h3 className="text-lg font-medium text-white">{server.hostname}</h3></div><div className="mt-1 text-sm text-[#9aa0a6]">{server.os_info || server.ip_address || 'Waiting for first collector check-in'}{server.cpu_cores ? ` · ${server.cpu_cores} cores` : ''}</div></div>{isAdmin ? <div className="flex flex-wrap items-center gap-2"><label className="flex items-center gap-2 text-xs text-[#9aa0a6]">Check interval<select value={server.check_interval_seconds} onChange={(e) => changeInterval(server, e.target.value)} className="rounded-lg border border-neutral-700 bg-[#111417] px-2 py-1.5 text-sm text-white"><option value="15">15 sec</option><option value="30">30 sec</option><option value="60">1 min</option><option value="300">5 min</option><option value="900">15 min</option></select></label><button type="button" onClick={() => deleteServer(server)} disabled={deletingServerId === server.id} className="rounded-lg border border-red-400/30 bg-red-500/5 px-3 py-1.5 text-xs text-red-300 transition hover:bg-red-500/10 disabled:opacity-50">{deletingServerId === server.id ? 'Deleting…' : 'Delete server'}</button></div> : <span className="text-xs text-[#9aa0a6]">Checks every {server.check_interval_seconds < 60 ? `${server.check_interval_seconds} sec` : `${Math.round(server.check_interval_seconds / 60)} min`}</span>}</div>- {metric ? <div className="mt-5 grid grid-cols-2 gap-3 lg:grid-cols-4">{[['CPU', metric.cpu_utilization_pct, '%'], ['RAM', metric.ram_pct, '%'], ['Disk', metric.disk_usage_pct, '%'], ['Load (1m)', metric.load_1m, '']].map(([label, value, unit]) => <div key={label} className="rounded-xl bg-[#171b1f] p-3"><div className="text-xs text-[#9aa0a6]">{label}</div><div className="mt-1 text-xl font-semibold text-white">{value == null ? '—' : `${Number(value).toFixed(1)}${unit}`}</div></div>)}</div> : <div className="mt-4 rounded-xl bg-[#171b1f] p-4 text-sm text-[#9aa0a6]">Collector has not sent a measurement yet.</div>}+ <div className="flex flex-wrap items-start justify-between gap-3"><div><div className="flex items-center gap-2"><span className={`h-2.5 w-2.5 rounded-full ${fresh ? 'bg-emerald-400' : 'bg-neutral-600'}`} /><h3 className="text-lg font-medium text-white">{server.hostname}</h3>{server.is_dashboard_host && <span className="rounded-full border border-white/10 bg-white/5 px-2 py-0.5 text-[10px] uppercase tracking-wider text-[#cfcfcf]">This machine</span>}</div><div className="mt-1 text-sm text-[#9aa0a6]">{server.os_info || server.ip_address || 'Waiting for first collector check-in'}{server.cpu_cores ? ` · ${server.cpu_cores} cores` : ''}</div></div>{isAdmin ? <div className="flex flex-wrap items-center gap-2"><label className="flex items-center gap-2 text-xs text-[#9aa0a6]">Check interval<select value={server.check_interval_seconds} onChange={(e) => changeInterval(server, e.target.value)} className="rounded-lg border border-neutral-700 bg-[#111417] px-2 py-1.5 text-sm text-white"><option value="15">15 sec</option><option value="30">30 sec</option><option value="60">1 min</option><option value="300">5 min</option><option value="900">15 min</option></select></label><button type="button" onClick={() => deleteServer(server)} disabled={deletingServerId === server.id} className="rounded-lg border border-red-400/30 bg-red-500/5 px-3 py-1.5 text-xs text-red-300 transition hover:bg-red-500/10 disabled:opacity-50">{deletingServerId === server.id ? 'Deleting…' : 'Delete server'}</button></div> : <span className="text-xs text-[#9aa0a6]">Checks every {server.check_interval_seconds < 60 ? `${server.check_interval_seconds} sec` : `${Math.round(server.check_interval_seconds / 60)} min`}</span>}</div>+ {metric ? <div className="mt-5 grid grid-cols-2 gap-3 lg:grid-cols-4">{[['CPU', metric.cpu_utilization_pct, '%'], ['RAM', metric.ram_pct, '%'], [fullest ? `Disk · ${fullest.mount}` : 'Disk', fullest?.used_pct, '%'], ['Load (1m)', metric.load_1m, '']].map(([label, value, unit]) => <div key={label} className="rounded-xl bg-[#171b1f] p-3"><div className="truncate text-xs text-[#9aa0a6]" title={label}>{label}</div><div className="mt-1 text-xl font-semibold text-white">{value == null ? '—' : `${Number(value).toFixed(1)}${unit}`}</div></div>)}</div> : <div className="mt-4 rounded-xl bg-[#171b1f] p-4 text-sm text-[#9aa0a6]">Collector has not sent a measurement yet.</div>}+ {metric && <StorageBars mounts={mounts} thresholds={overview.disk_thresholds} />}<div className="mt-4 flex flex-wrap items-center justify-between gap-2 text-xs text-[#7f8589]"><span>Last check: {server.last_seen_at ? new Date(server.last_seen_at).toLocaleString() : 'Never'}</span><span>{server.containers.length} containers</span></div>{server.containers.length > 0 && <ContainerTable server={server} />}</article>@@ -848,6 +925,8 @@ const TREND_Y_TICKS = [100, 90, 80, 70, 60, 50, 40, 30, 20, 10, 0]const trendY = (percent) => 3 + (100 - Math.max(0, Math.min(100, Number(percent) || 0))) * 0.94function MetricTrend({ title, color, value, points, times }) {+ // Used in url(#…), so keep only characters that are safe there (titles like "Disk usage (/)" aren't).+ const fillId = `fill-${title.replace(/[^a-zA-Z0-9]+/g, '-')}`const chartPoints = points.map((point, index) => `${points.length > 1 ? (index / (points.length - 1)) * 320 : 320},${trendY(point)}`).join(' ')const areaPoints = points.length ? `0,${trendY(0)} ${chartPoints} 320,${trendY(0)}` : ''return (@@ -858,7 +937,7 @@ function MetricTrend({ title, color, value, points, times }) {{TREND_Y_TICKS.filter((tick) => tick % 20 === 0).map((tick) => <span key={tick} className="absolute right-0 -translate-y-1/2 leading-none" style={{ top: `${trendY(tick)}%` }}>{tick}%</span>)}</div><div className="min-w-0 flex-1">- {points.length > 1 ? <svg viewBox="0 0 320 100" preserveAspectRatio="none" className="h-full w-full" role="img" aria-label={`${title} recent trend`}><defs><linearGradient id={`fill-${title.replaceAll(' ', '-')}`} x1="0" x2="0" y1="0" y2="1"><stop offset="0%" stopColor={color} stopOpacity="0.28"/><stop offset="100%" stopColor={color} stopOpacity="0"/></linearGradient></defs>{TREND_Y_TICKS.map((tick) => <line key={tick} x1="0" x2="320" y1={trendY(tick)} y2={trendY(tick)} stroke="#ffffff" strokeOpacity={tick % 20 === 0 ? 0.12 : 0.05} strokeWidth="1" vectorEffect="non-scaling-stroke" />)}<polygon points={areaPoints} fill={`url(#fill-${title.replaceAll(' ', '-')})`}/><polyline points={chartPoints} fill="none" stroke={color} strokeWidth="2.5" strokeLinecap="round" strokeLinejoin="round" vectorEffect="non-scaling-stroke"/></svg> : <div className="flex h-full items-center justify-center rounded-xl border border-dashed border-neutral-800 text-xs text-[#7f8589]">Trend appears as live samples arrive</div>}+ {points.length > 1 ? <svg viewBox="0 0 320 100" preserveAspectRatio="none" className="h-full w-full" role="img" aria-label={`${title} recent trend`}><defs><linearGradient id={fillId} x1="0" x2="0" y1="0" y2="1"><stop offset="0%" stopColor={color} stopOpacity="0.28"/><stop offset="100%" stopColor={color} stopOpacity="0"/></linearGradient></defs>{TREND_Y_TICKS.map((tick) => <line key={tick} x1="0" x2="320" y1={trendY(tick)} y2={trendY(tick)} stroke="#ffffff" strokeOpacity={tick % 20 === 0 ? 0.12 : 0.05} strokeWidth="1" vectorEffect="non-scaling-stroke" />)}<polygon points={areaPoints} fill={`url(#${fillId})`}/><polyline points={chartPoints} fill="none" stroke={color} strokeWidth="2.5" strokeLinecap="round" strokeLinejoin="round" vectorEffect="non-scaling-stroke"/></svg> : <div className="flex h-full items-center justify-center rounded-xl border border-dashed border-neutral-800 text-xs text-[#7f8589]">Trend appears as live samples arrive</div>}</div></div><div className="mt-2 flex justify-between pl-10 text-[10px] text-[#7f8589]"><span>{times[0] ? new Date(times[0]).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }) : 'Waiting for data'}</span><span>{times.at(-1) ? new Date(times.at(-1)).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }) : '—'}</span></div>diff --git a/tests/test_collector_mounts.py b/tests/test_collector_mounts.pynew file mode 100644index 0000000..e288ca0--- /dev/null+++ b/tests/test_collector_mounts.py@@ -0,0 +1,44 @@+import os++from src.agent import collector++MOUNTS = """\+/dev/sda1 / ext4 rw 0 0+proc /proc proc rw 0 0+tmpfs /run tmpfs rw 0 0+/dev/sda1 /var/snap/firefox/common ext4 rw 0 0+/dev/sdb1 /mnt/my\\040data ext4 rw 0 0+/dev/loop0 /snap/core/1 squashfs ro 0 0+overlay /var/lib/docker/overlay2/x/merged overlay rw 0 0+nas:/export /mnt/nfs nfs4 rw 0 0+/dev/sdc1 /mnt/later ext4 rw 0 0+"""+++class FakeStat:+ def __init__(self, total_blocks):+ self.f_frsize = 1024**3 // 1000+ self.f_blocks = total_blocks+ self.f_bfree = total_blocks // 2+ self.f_bavail = total_blocks // 2+ self.f_files = 100+ self.f_ffree = 75+++def test_collect_mounts_keeps_real_filesystems_once(tmp_path, monkeypatch):+ proc = tmp_path / "proc"+ (proc / "1").mkdir(parents=True)+ (proc / "1" / "mounts").write_text(MOUNTS)+ monkeypatch.setattr(collector, "HOST_PROC", proc)+ root = tmp_path / "root"+ monkeypatch.setattr(collector, "HOST_ROOT", root)+ sizes = {".": 1000, "var/snap/firefox/common": 1000, "mnt/my data": 4000, "mnt/nfs": 8000}+ monkeypatch.setattr(os, "statvfs", lambda path: FakeStat(sizes[os.path.relpath(path, root)]))+ # /mnt/later was mounted after the collector started, so it's only a directory in the container.+ monkeypatch.setattr(os.path, "ismount", lambda path: not str(path).endswith("later"))++ mounts = collector.collect_mounts()++ assert [(m["mount"], m["fs_type"]) for m in mounts] == [("/", "ext4"), ("/mnt/my data", "ext4"), ("/mnt/nfs", "nfs4")]+ assert mounts[0]["used_pct"] == 50.0+ assert mounts[0]["inodes_pct"] == 25.0diff --git a/tests/test_dashboard_host.py b/tests/test_dashboard_host.pynew file mode 100644index 0000000..be5e4ae--- /dev/null+++ b/tests/test_dashboard_host.py@@ -0,0 +1,76 @@+import os+from datetime import datetime, timedelta, timezone++os.environ.setdefault("DATABASE_URL", "sqlite://")+os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0")++import pytest+from fastapi import HTTPException+from sqlalchemy import create_engine+from sqlalchemy.orm import sessionmaker++from src.agent import collector+from src.backend import main+from src.backend.models import Base, HostMetric, Server, User+++def db(monkeypatch):+ engine = create_engine("sqlite://")+ Base.metadata.create_all(engine)+ session = sessionmaker(bind=engine)()+ session.add(User(id=1, name="Owner", email="[email protected]", password_hash="x", role="admin"))+ now = datetime.now(timezone.utc)+ for server_id, name, age in ((1, "local", 3), (2, "nas", 2), (3, "pi", 1)):+ session.add(Server(id=server_id, owner_id=1, hostname=name, check_interval_seconds=60, created_at=now - timedelta(days=age), collector_token_hash=main._hash_secret(f"token-{server_id}")))+ session.commit()+ monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1})+ return session+++def _ingest(db, server_id, **server):+ main.ingest_metrics(main.MetricsIngestRequest(server_id=server_id, server=server, host={"cpu_utilization_pct": 10, "ram_pct": 20, "disk_usage_pct": 30}), authorization=f"Bearer token-{server_id}", session_id=None, db=db)+++def test_dashboard_host_is_listed_first_and_flagged(db):+ _ingest(db, 1, dashboard_host=True)+ _ingest(db, 2)++ servers = main.monitoring_overview(session_id="s", db=db)["servers"]++ assert [(s["hostname"], s["is_dashboard_host"]) for s in servers] == [("local", True), ("pi", False), ("nas", False)]+++def test_only_latest_claimant_is_dashboard_host_and_flag_can_be_dropped(db):+ _ingest(db, 1, dashboard_host=True)+ _ingest(db, 2, dashboard_host=True)+ assert [s.id for s in db.query(Server).filter_by(is_dashboard_host=True)] == [2]++ _ingest(db, 2)+ assert db.query(Server).filter_by(is_dashboard_host=True).count() == 0+++def test_history_returns_recent_samples_oldest_first(db):+ now = datetime.now(timezone.utc)+ for minutes_ago, cpu in ((90, 1), (20, 2), (5, 3)):+ db.add(HostMetric(server_id=1, time=now - timedelta(minutes=minutes_ago), cpu_utilization_pct=cpu, ram_pct=50, disk_usage_pct=60))+ db.add(HostMetric(server_id=2, time=now, cpu_utilization_pct=99))+ db.commit()++ samples = main.server_history(1, minutes=30, session_id="s", db=db)["samples"]++ assert [(s["cpu"], s["ram"], s["disk"]) for s in samples] == [(2, 50, 60), (3, 50, 60)]+++def test_history_of_unknown_server_is_404(db):+ with pytest.raises(HTTPException) as exc:+ main.server_history(99, session_id="s", db=db)+ assert exc.value.status_code == 404+++def test_collector_sends_flag_only_when_configured(monkeypatch):+ monkeypatch.setenv("HOMELAB_DASHBOARD_HOST", "true")+ assert collector.collect_host()[0]["dashboard_host"] is True++ monkeypatch.delenv("HOMELAB_DASHBOARD_HOST")+ assert "dashboard_host" not in collector.collect_host()[0]diff --git a/tests/test_rules.py b/tests/test_rules.pyindex d5ea971..d508e1c 100644--- a/tests/test_rules.py+++ b/tests/test_rules.py@@ -10,7 +10,7 @@ from sqlalchemy.orm import sessionmakerfrom src.backend import main, rulesfrom src.backend.alerts import CRITICAL, WARNING-from src.backend.models import AuditLog, Base, IncidentAlert, Server, User+from src.backend.models import AuditLog, Base, IncidentAlert, MountMetric, Server, UserTOKEN = "collector-token"@@ -42,9 +42,9 @@ def test_host_alert_needs_sustained_breach(db):assert _active(db) == {}_ingest(db, disk_usage_pct=90)- alert = _active(db)["host:disk"]+ alert = _active(db)["host:disk:/"]assert alert.severity == WARNING- assert alert.title == "Disk usage on nas is 90%"+ assert alert.title == "Disk usage on nas:/ is 90%"def test_single_spike_does_not_alert(db):@@ -58,14 +58,14 @@ def test_open_alert_stays_open_until_value_drops_and_escalates_only_when_sustainfor _ in range(3):_ingest(db, disk_usage_pct=90)_ingest(db, disk_usage_pct=97)- assert _active(db)["host:disk"].severity == WARNING+ assert _active(db)["host:disk:/"].severity == WARNING_ingest(db, disk_usage_pct=97)_ingest(db, disk_usage_pct=97)- assert _active(db)["host:disk"].severity == CRITICAL+ assert _active(db)["host:disk:/"].severity == CRITICAL_ingest(db, disk_usage_pct=88)- assert _active(db)["host:disk"].severity == WARNING+ assert _active(db)["host:disk:/"].severity == WARNING_ingest(db, disk_usage_pct=50)assert _active(db) == {}@@ -120,3 +120,59 @@ def test_corrupt_saved_thresholds_fall_back_to_defaults(db):db.commit()assert main.get_alert_thresholds(db) == rules.DEFAULT_THRESHOLDS+++def _mount(mount, used_pct, inodes_pct=10.0, total_gb=2000.0):+ return {"mount": mount, "fs_type": "ext4", "total_gb": total_gb, "used_gb": total_gb * used_pct / 100, "used_pct": used_pct, "inodes_pct": inodes_pct}+++def test_each_mount_gets_its_own_disk_alert(db):+ for _ in range(3):+ _ingest(db, disk_usage_pct=40, mounts=[_mount("/", 40), _mount("/mnt/data", 96), _mount("/mnt/backup", 88, inodes_pct=99)])++ active = _active(db)+ assert set(active) == {"host:disk:/mnt/data", "host:disk:/mnt/backup", "host:inodes:/mnt/backup"}+ assert active["host:disk:/mnt/data"].severity == CRITICAL+ assert active["host:disk:/mnt/data"].title == "Disk usage on nas:/mnt/data is 96%"+ assert "(1.9 TB of 2.0 TB used)" in active["host:disk:/mnt/data"].description+ assert active["host:disk:/mnt/backup"].severity == WARNING+++def test_unmounted_disk_alert_resolves(db):+ for _ in range(3):+ _ingest(db, mounts=[_mount("/", 40), _mount("/mnt/data", 96)])+ _ingest(db, mounts=[_mount("/", 40)])++ assert _active(db) == {}+++def test_mount_needs_sustained_breach_of_its_own(db):+ _ingest(db, mounts=[_mount("/", 40)])+ _ingest(db, mounts=[_mount("/", 40)])+ _ingest(db, mounts=[_mount("/", 40), _mount("/mnt/new", 99)])++ assert _active(db) == {}+++def test_mount_rows_are_stored_and_bad_entries_ignored(db):+ _ingest(db, mounts=[_mount("/", 40), {"mount": ""}, "junk", {"used_pct": 5}])++ assert [row.mount for row in db.query(MountMetric)] == ["/"]+++def test_mount_path_with_braces_does_not_break_titles(db):+ for _ in range(3):+ _ingest(db, mounts=[_mount("/mnt/{odd}", 99)])++ assert _active(db)["host:disk:/mnt/{odd}"].title == "Disk usage on nas:/mnt/{odd} is 99%"+++def test_overview_shows_latest_mounts_and_disk_thresholds(db, monkeypatch):+ monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1})+ _ingest(db, mounts=[_mount("/", 10)])+ _ingest(db, mounts=[_mount("/", 20), _mount("/mnt/data", 30)])++ overview = main.monitoring_overview(session_id="s", db=db)++ assert [(m["mount"], m["used_pct"]) for m in overview["servers"][0]["mounts"]] == [("/", 20), ("/mnt/data", 30)]+ assert overview["disk_thresholds"]["disk_critical_pct"] == rules.DEFAULT_THRESHOLDS["disk_critical_pct"]