Skip to content

Self-hosted homelab infrastructure visualizer — interactive network diagram with live status monitoring

22.4 KB
Raw Blame
"""APScheduler setup for background scan and status check jobs."""
import asyncio
import logging
from collections.abc import Awaitable, Callable
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any, NamedTuple

from apscheduler.schedulers.asyncio import AsyncIOScheduler
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession

from app.core.config import settings
from app.db.database import AsyncSessionLocal
from app.db.models import InventoryDevice, Node
from app.services.status_checker import check_node, check_services

if TYPE_CHECKING:
    from app.schemas.zigbee import ZigbeeImportRequest
    from app.schemas.zwave import ZwaveImportRequest

logger = logging.getLogger(__name__)

scheduler: AsyncIOScheduler = AsyncIOScheduler()


async def _nodes_by_device(db: AsyncSession) -> dict[str, list[str]]:
    """Map each device id to the nodes drawing it, across every canvas."""
    rows = (await db.execute(select(Node.id, Node.device_id).where(Node.device_id.is_not(None)))).all()
    out: dict[str, list[str]] = {}
    for node_id, device_id in rows:
        out.setdefault(device_id, []).append(node_id)
    return out


class _CheckOutcome(NamedTuple):
    device_id: str
    status: str
    response_time_ms: int | None
    checked_at: datetime


async def _check_single_device(
    device_id: str,
    node_ids: list[str],
    check_method: str,
    check_target: str | None,
    ip: str | None,
) -> _CheckOutcome | None:
    """Probe one device and broadcast the result; returns None if the probe failed.

    Accepts plain scalars — not an ORM object — so there is no risk of
    DetachedInstanceError when the originating session has already closed.

    It does not write: `_save_check_outcomes` persists the whole cycle at once.
    The broadcast goes out here, as soon as the probe answers, so a canvas is
    not kept waiting on the slowest device or on the database.
    """
    from app.api.routes.status import broadcast_status  # avoid circular import

    try:
        check_result = await check_node(check_method, check_target, ip)
    except Exception as exc:
        logger.error("Status check failed for device %s: %s", device_id, exc)
        return None
    raw_ms = check_result["response_time_ms"]
    outcome = _CheckOutcome(
        device_id=device_id,
        status=str(check_result["status"]),
        response_time_ms=raw_ms if isinstance(raw_ms, int) else None,
        checked_at=datetime.now(timezone.utc),
    )
    try:
        await broadcast_status(
            device_id=device_id,
            node_ids=node_ids,
            status=outcome.status,
            checked_at=outcome.checked_at.isoformat(),
            response_time_ms=outcome.response_time_ms,
        )
    except Exception as exc:
        logger.error("Status broadcast failed for device %s: %s", device_id, exc)
    return outcome


async def _save_check_outcomes(outcomes: list[_CheckOutcome]) -> None:
    """Persist a whole status cycle in one write transaction.

    One commit per device meant as many writers queueing on SQLite's single
    write lock as there are devices, and the ones at the back of the queue gave
    up with "database is locked" (#539). A failure here costs the stored state
    of one cycle, never the live one: the broadcasts have already gone out.
    """
    by_id = {o.device_id: o for o in outcomes}
    try:
        async with AsyncSessionLocal() as db:
            devices = (
                await db.execute(select(InventoryDevice).where(InventoryDevice.id.in_(by_id)))
            ).scalars().all()
            for device in devices:
                outcome = by_id[device.id]
                device.status_live = outcome.status
                device.response_time_ms = outcome.response_time_ms
                if outcome.status == "online":
                    device.last_seen = outcome.checked_at
            await db.commit()
    except Exception as exc:
        logger.error("Could not save the status of %d device(s): %s", len(outcomes), exc)


async def _run_status_checks() -> None:
    """Check every device once and broadcast the result to every canvas.

    Device-scoped, not node-scoped: a host drawn on three canvases used to be
    pinged three times and could report three different states. Hidden devices
    are skipped — hiding one is how a user says "stop showing me this".

    A device with no node at all is still checked, deliberately. The inventory
    row owns the device and shows its live status in the list and the detail
    modal, so a rack mount or an inventory-only entry reports state without
    being drawn anywhere — and deleting a node no longer silently stops
    monitoring the host. `hide` (or clearing the check method) is what ends the
    checks; the broadcast simply carries an empty `node_ids`.
    """
    if not settings.status_checker_enabled:
        return
    async with AsyncSessionLocal() as db:
        devices = (
            await db.execute(
                select(InventoryDevice).where(
                    InventoryDevice.check_method.is_not(None),
                    InventoryDevice.check_method != "",
                    InventoryDevice.status != "hidden",
                )
            )
        ).scalars().all()
        node_map = await _nodes_by_device(db)
        # Extract scalars while the session is open to avoid DetachedInstanceError
        checkable = [
            (d.id, node_map.get(d.id, []), d.check_method or "ping", d.check_target, d.ip)
            for d in devices
        ]

    if not checkable:
        return

    results = await asyncio.gather(*[
        _check_single_device(device_id, node_ids, method, target, ip)
        for device_id, node_ids, method, target, ip in checkable
    ])
    outcomes = [r for r in results if r is not None]
    if outcomes:
        await _save_check_outcomes(outcomes)


def _node_host(ip: str | None, hostname: str | None) -> str | None:
    """Pick the address to probe services on: first IP, else hostname."""
    if ip:
        first = ip.split(",")[0].strip()
        if first:
            return first
    return hostname or None


async def _run_service_checks() -> None:
    """Check every service of every device and broadcast per-service results.

    Device-scoped for the same reason as the status check: the services belong
    to the device, so one pass serves every canvas showing it.
    """
    if not (settings.status_checker_enabled and settings.service_check_enabled):
        return
    from app.api.routes.status import broadcast_service_status  # avoid circular import

    async with AsyncSessionLocal() as db:
        devices = (
            await db.execute(select(InventoryDevice).where(InventoryDevice.status != "hidden"))
        ).scalars().all()
        node_map = await _nodes_by_device(db)
        checkable = [
            (d.id, node_map.get(d.id, []), _node_host(d.ip, d.hostname), list(d.services or []))
            for d in devices
            if d.services
        ]

    now = datetime.now(timezone.utc).isoformat()
    for device_id, node_ids, host, services in checkable:
        try:
            statuses = await check_services(host, services)
            await broadcast_service_status(
                device_id=device_id, node_ids=node_ids, services=statuses, checked_at=now
            )
        except Exception as exc:
            logger.error("Service checks failed for device %s: %s", device_id, exc)


async def _run_proxmox_sync() -> None:
    """Fetch the Proxmox inventory and upsert it into pending (auto-sync).

    Records a ScanRun (kind=proxmox) so the scheduled sync shows in Scan
    history, exactly like the manual /sync-now and /import-pending paths.
    """
    if not settings.proxmox_sync_enabled:
        return
    if not (settings.proxmox_host and settings.proxmox_token_id and settings.proxmox_token_secret):
        logger.warning("Proxmox auto-sync enabled but host/token not configured — skipping")
        return
    # Lazy import to avoid a circular import at module load.
    from app.api.routes.proxmox import _background_proxmox_import
    from app.db.models import ScanRun

    async with AsyncSessionLocal() as db:
        run = ScanRun(
            status="running",
            kind="proxmox",
            ranges=[f"{settings.proxmox_host}:{settings.proxmox_port}"],
        )
        db.add(run)
        await db.commit()
        await db.refresh(run)
        run_id = run.id

    # Shares the manual-sync flow: fetch + persist + mark the run done/error +
    # broadcast the inventory-reload signal.
    await _background_proxmox_import(
        run_id,
        settings.proxmox_host,
        settings.proxmox_port,
        settings.proxmox_token_id,
        settings.proxmox_token_secret,
        settings.proxmox_verify_tls,
    )


async def _run_unraid_sync() -> None:
    """Fetch the Unraid containers and upsert them into pending (auto-sync)."""
    if not settings.unraid_sync_enabled:
        return
    if not (settings.unraid_host and settings.unraid_api_key):
        logger.warning("Unraid auto-sync enabled but host/API key not configured - skipping")
        return
    from app.api.routes.unraid import _background_unraid_import
    from app.db.models import ScanRun

    async with AsyncSessionLocal() as db:
        run = ScanRun(
            status="running",
            kind="unraid",
            ranges=[f"{settings.unraid_host}:{settings.unraid_effective_port}"],
        )
        db.add(run)
        await db.commit()
        await db.refresh(run)
        run_id = run.id

    await _background_unraid_import(
        run_id,
        settings.unraid_host,
        settings.unraid_effective_port,
        settings.unraid_api_key,
        settings.unraid_verify_tls,
        settings.unraid_use_https,
        settings.unraid_sync_include_offline,
    )


async def _run_mesh_sync(kind: str) -> None:
    """Shared auto-sync for the MQTT mesh imports (Zigbee / Z-Wave).

    Records a ScanRun (kind=zigbee|zwave) so the scheduled sync shows in Scan
    history, then delegates to the exact same background import the manual
    /sync-now and /import-pending paths use. Connection config comes from the
    server env only (credentials never leave it).
    """
    from app.db.models import ScanRun

    payload: ZigbeeImportRequest | ZwaveImportRequest
    background: Callable[[str, Any], Awaitable[None]]

    if kind == "zigbee":
        if not settings.zigbee_sync_enabled:
            return
        if not settings.zigbee_mqtt_host:
            logger.warning("Zigbee auto-sync enabled but MQTT host not configured — skipping")
            return
        from app.api.routes.zigbee import _background_zigbee_import
        from app.api.routes.zigbee import env_import_request as _zigbee_env_request
        host, port = settings.zigbee_mqtt_host, settings.zigbee_mqtt_port
        payload = _zigbee_env_request()
        background = _background_zigbee_import
    else:
        if not settings.zwave_sync_enabled:
            return
        if not settings.zwave_mqtt_host:
            logger.warning("Z-Wave auto-sync enabled but MQTT host not configured — skipping")
            return
        from app.api.routes.zwave import _background_zwave_import
        from app.api.routes.zwave import env_import_request as _zwave_env_request
        host, port = settings.zwave_mqtt_host, settings.zwave_mqtt_port
        payload = _zwave_env_request()
        background = _background_zwave_import
    async with AsyncSessionLocal() as db:
        run = ScanRun(status="running", kind=kind, ranges=[f"{host}:{port}"])
        db.add(run)
        await db.commit()
        await db.refresh(run)
        run_id = run.id

    await background(run_id, payload)


async def _run_zigbee_sync() -> None:
    await _run_mesh_sync("zigbee")


async def _run_zwave_sync() -> None:
    await _run_mesh_sync("zwave")


async def _run_unifi_sync() -> None:
    """Fetch the UniFi inventory and upsert it into pending (auto-sync)."""
    if not settings.unifi_sync_enabled:
        return
    if not (settings.unifi_effective_host and settings.unifi_username and settings.unifi_password):
        logger.warning("UniFi auto-sync enabled but host/credentials not configured — skipping")
        return
    from app.api.routes.unifi import _background_unifi_sync
    from app.db.models import ScanRun

    async with AsyncSessionLocal() as db:
        run = ScanRun(
            status="running",
            kind="unifi",
            ranges=[f"{settings.unifi_effective_host}:{settings.unifi_effective_port}"],
        )
        db.add(run)
        await db.commit()
        await db.refresh(run)
        run_id = run.id

    await _background_unifi_sync(run_id)


def _add_service_check_job() -> None:
    scheduler.add_job(
        _run_service_checks,
        "interval",
        seconds=settings.service_check_interval,
        id="service_checks",
        max_instances=1,
        coalesce=True,
    )


def _add_proxmox_sync_job() -> None:
    scheduler.add_job(
        _run_proxmox_sync,
        "interval",
        seconds=settings.proxmox_sync_interval,
        id="proxmox_sync",
        max_instances=1,
        coalesce=True,
    )


def _unraid_sync_interval() -> int:
    # .env and the settings file bypass the >= 300 check on write, so clamp here
    return max(300, settings.unraid_sync_interval)


def _add_unraid_sync_job() -> None:
    scheduler.add_job(
        _run_unraid_sync,
        "interval",
        seconds=_unraid_sync_interval(),
        id="unraid_sync",
        max_instances=1,
        coalesce=True,
    )


def _add_zigbee_sync_job() -> None:
    scheduler.add_job(
        _run_zigbee_sync,
        "interval",
        seconds=settings.zigbee_sync_interval,
        id="zigbee_sync",
        max_instances=1,
        coalesce=True,
    )


def _add_zwave_sync_job() -> None:
    scheduler.add_job(
        _run_zwave_sync,
        "interval",
        seconds=settings.zwave_sync_interval,
        id="zwave_sync",
        max_instances=1,
        coalesce=True,
    )


def _add_unifi_sync_job() -> None:
    scheduler.add_job(
        _run_unifi_sync,
        "interval",
        seconds=settings.unifi_sync_interval,
        id="unifi_sync",
        max_instances=1,
        coalesce=True,
    )


def start_scheduler() -> None:
    global scheduler
    if scheduler.running:
        try:
            scheduler.shutdown(wait=False)
        except Exception as exc:
            logger.warning("Failed to shut down previous scheduler instance: %s", exc)
    scheduler = AsyncIOScheduler()
    if settings.status_checker_enabled:
        scheduler.add_job(
            _run_status_checks,
            "interval",
            seconds=settings.status_checker_interval,
            id="status_checks",
            max_instances=1,
            coalesce=True,
        )
        if settings.service_check_enabled:
            _add_service_check_job()
    if settings.proxmox_sync_enabled:
        _add_proxmox_sync_job()
    if settings.unraid_sync_enabled:
        _add_unraid_sync_job()
    if settings.zigbee_sync_enabled:
        _add_zigbee_sync_job()
    if settings.zwave_sync_enabled:
        _add_zwave_sync_job()
    if settings.unifi_sync_enabled:
        _add_unifi_sync_job()
    scheduler.start()
    if settings.status_checker_enabled:
        logger.info("Scheduler started — status checks every %ds", settings.status_checker_interval)
    else:
        logger.info("Scheduler started — status checks disabled (STATUS_CHECKER_ENABLED=false)")


def reschedule_status_checks(interval_seconds: int) -> None:
    """Update the status check interval on the running scheduler."""
    if interval_seconds < 10:
        raise ValueError(f"interval_seconds must be >= 10, got {interval_seconds}")
    if not scheduler.running:
        logger.warning("Scheduler not running, skipping reschedule")
        return
    if not scheduler.get_job("status_checks"):
        logger.info("Status checks disabled — nothing to reschedule")
        return
    scheduler.reschedule_job("status_checks", trigger="interval", seconds=interval_seconds)
    logger.info("Status checks rescheduled to every %ds", interval_seconds)


def reschedule_service_checks(interval_seconds: int) -> None:
    """Update the service-check interval on the running scheduler (if enabled)."""
    if interval_seconds < 30:
        raise ValueError(f"interval_seconds must be >= 30, got {interval_seconds}")
    if not scheduler.running:
        logger.warning("Scheduler not running, skipping reschedule")
        return
    if scheduler.get_job("service_checks"):
        scheduler.reschedule_job("service_checks", trigger="interval", seconds=interval_seconds)
        logger.info("Service checks rescheduled to every %ds", interval_seconds)


def set_service_checks_enabled(enabled: bool) -> None:
    """Add or remove the service-check job on the running scheduler."""
    if not scheduler.running:
        return
    if enabled and not settings.status_checker_enabled:
        logger.info("Service checks stay off: STATUS_CHECKER_ENABLED=false")
        return
    job = scheduler.get_job("service_checks")
    if enabled and not job:
        _add_service_check_job()
        logger.info("Service checks enabled — every %ds", settings.service_check_interval)
    elif not enabled and job:
        scheduler.remove_job("service_checks")
        logger.info("Service checks disabled")


def reschedule_proxmox_sync(interval_seconds: int) -> None:
    """Update the Proxmox auto-sync interval on the running scheduler (if enabled)."""
    if interval_seconds < 300:
        raise ValueError(f"interval_seconds must be >= 300, got {interval_seconds}")
    if not scheduler.running:
        logger.warning("Scheduler not running, skipping reschedule")
        return
    if scheduler.get_job("proxmox_sync"):
        scheduler.reschedule_job("proxmox_sync", trigger="interval", seconds=interval_seconds)
        logger.info("Proxmox auto-sync rescheduled to every %ds", interval_seconds)


def set_proxmox_sync_enabled(enabled: bool) -> None:
    """Add or remove the Proxmox auto-sync job on the running scheduler."""
    if not scheduler.running:
        return
    job = scheduler.get_job("proxmox_sync")
    if enabled and not job:
        _add_proxmox_sync_job()
        logger.info("Proxmox auto-sync enabled — every %ds", settings.proxmox_sync_interval)
    elif not enabled and job:
        scheduler.remove_job("proxmox_sync")
        logger.info("Proxmox auto-sync disabled")


def reschedule_unraid_sync(interval_seconds: int) -> None:
    """Update the Unraid auto-sync interval on the running scheduler (if enabled)."""
    if interval_seconds < 300:
        raise ValueError(f"interval_seconds must be >= 300, got {interval_seconds}")
    if not scheduler.running:
        logger.warning("Scheduler not running, skipping reschedule")
        return
    if scheduler.get_job("unraid_sync"):
        scheduler.reschedule_job("unraid_sync", trigger="interval", seconds=interval_seconds)
        logger.info("Unraid auto-sync rescheduled to every %ds", interval_seconds)


def set_unraid_sync_enabled(enabled: bool) -> None:
    """Add or remove the Unraid auto-sync job on the running scheduler."""
    if not scheduler.running:
        return
    job = scheduler.get_job("unraid_sync")
    if enabled and not job:
        _add_unraid_sync_job()
        logger.info("Unraid auto-sync enabled - every %ds", _unraid_sync_interval())
    elif not enabled and job:
        scheduler.remove_job("unraid_sync")
        logger.info("Unraid auto-sync disabled")


def reschedule_zigbee_sync(interval_seconds: int) -> None:
    """Update the Zigbee auto-sync interval on the running scheduler (if enabled)."""
    if interval_seconds < 300:
        raise ValueError(f"interval_seconds must be >= 300, got {interval_seconds}")
    if not scheduler.running:
        logger.warning("Scheduler not running, skipping reschedule")
        return
    if scheduler.get_job("zigbee_sync"):
        scheduler.reschedule_job("zigbee_sync", trigger="interval", seconds=interval_seconds)
        logger.info("Zigbee auto-sync rescheduled to every %ds", interval_seconds)


def set_zigbee_sync_enabled(enabled: bool) -> None:
    """Add or remove the Zigbee auto-sync job on the running scheduler."""
    if not scheduler.running:
        return
    job = scheduler.get_job("zigbee_sync")
    if enabled and not job:
        _add_zigbee_sync_job()
        logger.info("Zigbee auto-sync enabled — every %ds", settings.zigbee_sync_interval)
    elif not enabled and job:
        scheduler.remove_job("zigbee_sync")
        logger.info("Zigbee auto-sync disabled")


def reschedule_zwave_sync(interval_seconds: int) -> None:
    """Update the Z-Wave auto-sync interval on the running scheduler (if enabled)."""
    if interval_seconds < 300:
        raise ValueError(f"interval_seconds must be >= 300, got {interval_seconds}")
    if not scheduler.running:
        logger.warning("Scheduler not running, skipping reschedule")
        return
    if scheduler.get_job("zwave_sync"):
        scheduler.reschedule_job("zwave_sync", trigger="interval", seconds=interval_seconds)
        logger.info("Z-Wave auto-sync rescheduled to every %ds", interval_seconds)


def set_zwave_sync_enabled(enabled: bool) -> None:
    """Add or remove the Z-Wave auto-sync job on the running scheduler."""
    if not scheduler.running:
        return
    job = scheduler.get_job("zwave_sync")
    if enabled and not job:
        _add_zwave_sync_job()
        logger.info("Z-Wave auto-sync enabled — every %ds", settings.zwave_sync_interval)
    elif not enabled and job:
        scheduler.remove_job("zwave_sync")
        logger.info("Z-Wave auto-sync disabled")


def reschedule_unifi_sync(interval_seconds: int) -> None:
    if interval_seconds < 300:
        raise ValueError(f"interval_seconds must be >= 300, got {interval_seconds}")
    if not scheduler.running:
        logger.warning("Scheduler not running, skipping reschedule")
        return
    if scheduler.get_job("unifi_sync"):
        scheduler.reschedule_job("unifi_sync", trigger="interval", seconds=interval_seconds)
        logger.info("UniFi auto-sync rescheduled to every %ds", interval_seconds)


def set_unifi_sync_enabled(enabled: bool) -> None:
    if not scheduler.running:
        return
    job = scheduler.get_job("unifi_sync")
    if enabled and not job:
        _add_unifi_sync_job()
        logger.info("UniFi auto-sync enabled — every %ds", settings.unifi_sync_interval)
    elif not enabled and job:
        scheduler.remove_job("unifi_sync")
        logger.info("UniFi auto-sync disabled")


def stop_scheduler() -> None:
    if scheduler.running:
        scheduler.shutdown(wait=False)