Self-hosted homelab infrastructure visualizer — interactive network diagram with live status monitoring
TypeScript diagramhomelabhomelab-infrastructure-observabilitynetworkproxmox-veself-hostedzigbeezwavezwave-js
"""FastAPI router for Zigbee2MQTT import."""
import logging
from datetime import datetime, timezone
from typing import Any
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from sqlalchemy import delete as sa_delete
from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.deps import get_current_user
from app.core.config import settings
from app.core.scheduler import reschedule_zigbee_sync, set_zigbee_sync_enabled
from app.db.database import AsyncSessionLocal, get_db
from app.db.models import InventoryDevice, InventoryDeviceLink, Node, ScanRun
from app.schemas.scan import ScanRunResponse
from app.schemas.zigbee import (
ZigbeeConfig,
ZigbeeCoordinatorOut,
ZigbeeEdgeOut,
ZigbeeImportJob,
ZigbeeImportJobResult,
ZigbeeImportPendingResponse,
ZigbeeImportRequest,
ZigbeeImportResponse,
ZigbeeNodeOut,
ZigbeeSyncConfig,
ZigbeeTestConnectionRequest,
ZigbeeTestConnectionResponse,
)
from app.services.discovery_sources import add_source
from app.services.import_jobs import create_job, fail_job, finish_job, get_job
from app.services.inventory_sync import attach_device_ids
from app.services.node_dedupe import dedupe_nodes_by_device
from app.services.zigbee_service import (
build_zigbee_properties,
fetch_networkmap,
merge_zigbee_properties,
test_mqtt_connection,
)
logger = logging.getLogger(__name__)
async def _is_drawn(db: AsyncSession, device_id: str) -> bool:
"""True while at least one canvas node still draws this device."""
return (
await db.execute(select(Node.id).where(Node.device_id == device_id).limit(1))
).scalar_one_or_none() is not None
router = APIRouter()
def _import_error(exc: BaseException) -> tuple[int, str]:
"""Map a fetch_networkmap failure to the (status, detail) the API reports."""
if isinstance(exc, ImportError):
return 500, str(exc)
if isinstance(exc, ConnectionError):
return 502, str(exc)
if isinstance(exc, TimeoutError):
return 504, str(exc)
if isinstance(exc, ValueError):
return 422, str(exc)
logger.exception("Unexpected error during Zigbee import", exc_info=exc)
return 500, "Unexpected error during Zigbee import"
@router.post("/import", response_model=ZigbeeImportJob, status_code=202)
async def import_zigbee_network(
payload: ZigbeeImportRequest,
background_tasks: BackgroundTasks,
_: str = Depends(get_current_user),
) -> ZigbeeImportJob:
"""Start a canvas import and return a job id to poll.
Connects to the specified MQTT broker, publishes a networkmap request to
``<base_topic>/bridge/request/networkmap`` and waits up to
``ZIGBEE_NETWORKMAP_TIMEOUT`` seconds (default 300) for the response.
The fetch runs in the background and the result is collected from
``GET /zigbee/import/{job_id}``: a 200+ device mesh takes minutes to answer,
which outlives the read timeout of any reverse proxy sitting in front of the
API. Polling keeps each request short.
"""
job = create_job()
background_tasks.add_task(_background_canvas_import, job.id, payload)
return ZigbeeImportJob(job_id=job.id, status=job.status)
@router.get("/import/{job_id}", response_model=ZigbeeImportJobResult)
async def get_zigbee_import_job(
job_id: str,
_: str = Depends(get_current_user),
) -> ZigbeeImportJobResult:
"""Poll a canvas import started by ``POST /zigbee/import``.
While running, returns ``status="running"`` with no payload. On success the
nodes and edges are carried in ``result``. A failed fetch is reported as the
status code the synchronous route used to raise, so the client keeps
distinguishing a bad broker (502) from a slow mesh (504).
"""
job = get_job(job_id)
if job is None:
raise HTTPException(status_code=404, detail="Import job not found or expired")
if job.status == "error":
raise HTTPException(
status_code=job.error_status or 500,
detail=job.error or "Zigbee import failed",
)
result = None
if job.status == "done" and job.result is not None:
result = ZigbeeImportResponse(**job.result)
return ZigbeeImportJobResult(job_id=job.id, status=job.status, result=result)
async def _background_canvas_import(job_id: str, payload: ZigbeeImportRequest) -> None:
"""Fetch the network map and park the canvas-ready payload on the job."""
try:
nodes_raw, edges_raw = await fetch_networkmap(
mqtt_host=payload.mqtt_host,
mqtt_port=payload.mqtt_port,
base_topic=payload.base_topic,
username=payload.mqtt_username,
password=payload.mqtt_password,
tls=payload.mqtt_tls,
tls_insecure=payload.mqtt_tls_insecure,
include_mesh_links=payload.include_mesh_links,
)
except Exception as exc:
status, detail = _import_error(exc)
fail_job(job_id, detail, status)
return
# A canvas import is an import: the devices land in the Device Inventory
# too, and each node carries the id of the row it draws so the canvas save
# links to it instead of minting a second, IEEE-less row.
try:
async with AsyncSessionLocal() as db:
await _persist_pending_import(db, nodes_raw, edges_raw)
nodes_raw = await attach_device_ids(db, nodes_raw)
except Exception as exc:
status, detail = _import_error(exc)
fail_job(job_id, detail, status)
return
nodes = [ZigbeeNodeOut(**n) for n in nodes_raw]
edges = [ZigbeeEdgeOut(**e) for e in edges_raw]
finish_job(
job_id,
ZigbeeImportResponse(
nodes=nodes, edges=edges, device_count=len(nodes)
).model_dump(),
)
@router.post("/import-pending", response_model=ScanRunResponse)
async def import_zigbee_to_pending(
payload: ZigbeeImportRequest,
background_tasks: BackgroundTasks,
db: AsyncSession = Depends(get_db),
_: str = Depends(get_current_user),
) -> ScanRun:
"""Queue a Zigbee2MQTT pending import as a background scan run.
Returns the ScanRun row immediately so the UI can close the import
modal and surface progress under Scan History (kind=zigbee). The
actual MQTT fetch + pending upsert happens in the background.
"""
run = ScanRun(
status="running",
kind="zigbee",
ranges=[f"{payload.mqtt_host}:{payload.mqtt_port}"],
)
db.add(run)
await db.commit()
await db.refresh(run)
background_tasks.add_task(_background_zigbee_import, run.id, payload)
return run
def env_import_request() -> ZigbeeImportRequest:
"""Build an import request from the server env config (for auto-sync).
MQTT credentials live in the env only — never in the request body or any
API response. The scheduled auto-sync job and ``/sync-now`` both source
their connection settings here so there is a single source of truth."""
return ZigbeeImportRequest(
mqtt_host=settings.zigbee_mqtt_host,
mqtt_port=settings.zigbee_mqtt_port,
mqtt_username=settings.zigbee_mqtt_username or None,
mqtt_password=settings.zigbee_mqtt_password or None,
base_topic=settings.zigbee_base_topic,
mqtt_tls=settings.zigbee_mqtt_tls,
mqtt_tls_insecure=settings.zigbee_mqtt_tls_insecure,
# Parent tree only unless the user opted into the mesh in Settings —
# the neighbour links multiply the edge count.
include_mesh_links=settings.zigbee_sync_include_mesh_links,
)
@router.post("/sync-now", response_model=ScanRunResponse)
async def sync_zigbee_now(
background_tasks: BackgroundTasks,
db: AsyncSession = Depends(get_db),
_: str = Depends(get_current_user),
) -> ScanRun:
"""Trigger an immediate Zigbee import using the server env config.
Same background flow as ``/import-pending`` but sources the MQTT connection
from ``settings`` (env) rather than the request body — the manual
counterpart to the scheduled auto-sync job. Requires the env host to be set.
"""
if not settings.zigbee_mqtt_host:
raise HTTPException(
status_code=400,
detail="Cannot sync: no Zigbee MQTT host configured on the server.",
)
payload = env_import_request()
run = ScanRun(
status="running",
kind="zigbee",
ranges=[f"{payload.mqtt_host}:{payload.mqtt_port}"],
)
db.add(run)
await db.commit()
await db.refresh(run)
background_tasks.add_task(_background_zigbee_import, run.id, payload)
return run
async def _background_zigbee_import(run_id: str, payload: ZigbeeImportRequest) -> None:
async with AsyncSessionLocal() as db:
try:
nodes_raw, edges_raw = await fetch_networkmap(
mqtt_host=payload.mqtt_host,
mqtt_port=payload.mqtt_port,
base_topic=payload.base_topic,
username=payload.mqtt_username,
password=payload.mqtt_password,
tls=payload.mqtt_tls,
tls_insecure=payload.mqtt_tls_insecure,
include_mesh_links=payload.include_mesh_links,
)
result = await _persist_pending_import(db, nodes_raw, edges_raw)
run = await db.get(ScanRun, run_id)
if run:
run.status = "done"
run.devices_found = result.device_count
run.finished_at = datetime.now(timezone.utc)
await db.commit()
except Exception as exc:
logger.exception("Zigbee import %s failed", run_id)
await db.rollback()
run = await db.get(ScanRun, run_id)
if run:
run.status = "error"
run.error = str(exc)[:500]
run.finished_at = datetime.now(timezone.utc)
await db.commit()
async def _persist_pending_import(
db: AsyncSession,
nodes_raw: list[dict[str, Any]],
edges_raw: list[dict[str, Any]],
) -> ZigbeeImportPendingResponse:
"""Upsert nodes/edges into device_inventory + device_inventory_links.
Coordinator auto-approves to a canvas Node. Other devices upsert by IEEE.
This network's zigbee-source links are wiped and re-inserted from the new map.
"""
# Repair any pre-existing duplicate nodes (same IEEE) before upserting, so
# the by-IEEE lookups below resolve to a single row.
await dedupe_nodes_by_device(db)
# Coordinator is no longer auto-placed, so the response's coordinator fields
# stay unset — retained for backward-compatible response shape.
coordinator_out: ZigbeeCoordinatorOut | None = None
coordinator_existed = False
pending_created = 0
pending_updated = 0
for n in nodes_raw:
ieee = n.get("ieee_address")
if not ieee:
continue
props = build_zigbee_properties(
ieee, n.get("vendor"), n.get("model"), n.get("lqi")
)
# The coordinator is no longer auto-placed on the canvas — it flows to
# the pending inventory like every other device, so the user approves it
# explicitly. Only the shared paths below run for it.
# Properties belong to the inventory row now, so one upsert serves
# every canvas drawing this device — there is no per-node refresh left
# to do. The row's status is never touched here: an approved device
# stays approved, a hidden one stays hidden.
result = await db.execute(
select(InventoryDevice).where(InventoryDevice.ieee_address == ieee)
)
pending = result.scalar_one_or_none()
if pending is None:
db.add(
InventoryDevice(
ieee_address=ieee,
friendly_name=n.get("friendly_name"),
hostname=n.get("friendly_name"),
suggested_type=n.get("type"),
device_subtype=n.get("device_type"),
model=n.get("model"),
vendor=n.get("vendor"),
lqi=n.get("lqi"),
properties=props,
status="pending",
discovery_source="zigbee",
discovery_sources=["zigbee"],
)
)
pending_created += 1
else:
pending.friendly_name = n.get("friendly_name") or pending.friendly_name
pending.suggested_type = n.get("type") or pending.suggested_type
pending.device_subtype = n.get("device_type") or pending.device_subtype
pending.model = n.get("model") or pending.model
pending.vendor = n.get("vendor") or pending.vendor
if n.get("lqi") is not None:
pending.lqi = n.get("lqi")
pending.properties = merge_zigbee_properties(list(pending.properties or []), props)
# Record the import among the row's sources: the UI files a device
# by that list, so a row a canvas save listed as ["canvas"] only
# would otherwise stay out of the zigbee filter for good.
pending.discovery_sources = add_source(pending.discovery_sources, "zigbee")
if pending.status == "approved" and not await _is_drawn(db, pending.id):
# Approved earlier but no canvas draws it any more — the node was
# deleted. Revive the row to "pending" so it reappears in the list
# on re-import instead of being silently swallowed. (Issue #167)
pending.status = "pending"
elif pending.status == "hidden":
# Re-imported a hidden device → leave it hidden, just refresh fields.
pass
pending_updated += 1
# Replace this network's zigbee-source links with the freshly discovered
# set. Only links touching a device this import returned: one import is one
# Zigbee2MQTT instance, and wiping every zigbee link erased the map of a
# second network imported earlier.
imported = {n["ieee_address"] for n in nodes_raw if n.get("ieee_address")}
await db.execute(
sa_delete(InventoryDeviceLink).where(
# Both sources, or mesh links would pile up import after import:
# the wipe below is what keeps this a replace, not an append.
InventoryDeviceLink.discovery_source.in_(["zigbee", "zigbee_mesh"]),
or_(
InventoryDeviceLink.source_ieee.in_(imported),
InventoryDeviceLink.target_ieee.in_(imported),
),
)
)
links_recorded = 0
seen: set[tuple[str, str]] = set()
for e in edges_raw:
src = e.get("source")
tgt = e.get("target")
if not src or not tgt or (src, tgt) in seen:
continue
seen.add((src, tgt))
db.add(
InventoryDeviceLink(
source_ieee=src,
target_ieee=tgt,
lqi=e.get("lqi"),
# A neighbour link is tagged apart from the parent tree so the
# canvas can draw it in its own edge type on approval.
discovery_source=(
"zigbee_mesh" if e.get("kind") == "mesh" else "zigbee"
),
)
)
links_recorded += 1
await db.commit()
return ZigbeeImportPendingResponse(
pending_created=pending_created,
pending_updated=pending_updated,
coordinator=coordinator_out,
coordinator_already_existed=coordinator_existed,
links_recorded=links_recorded,
device_count=len(nodes_raw),
)
@router.post("/test-connection", response_model=ZigbeeTestConnectionResponse)
async def test_zigbee_connection(
payload: ZigbeeTestConnectionRequest,
_: str = Depends(get_current_user),
) -> ZigbeeTestConnectionResponse:
"""Quick MQTT ping to validate broker connection before importing."""
try:
await test_mqtt_connection(
mqtt_host=payload.mqtt_host,
mqtt_port=payload.mqtt_port,
username=payload.mqtt_username,
password=payload.mqtt_password,
tls=payload.mqtt_tls,
tls_insecure=payload.mqtt_tls_insecure,
)
return ZigbeeTestConnectionResponse(connected=True, message="Connection successful")
except ImportError as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
except (ConnectionError, TimeoutError) as exc:
return ZigbeeTestConnectionResponse(connected=False, message=str(exc))
except Exception:
logger.exception("Unexpected error during connection test")
return ZigbeeTestConnectionResponse(connected=False, message="Unexpected error")
@router.get("/config", response_model=ZigbeeConfig)
async def get_zigbee_config(_: str = Depends(get_current_user)) -> ZigbeeConfig:
"""Return non-secret Zigbee config. Never includes MQTT credentials — only
whether a host is configured on the server for auto-sync."""
return ZigbeeConfig(
mqtt_host=settings.zigbee_mqtt_host,
mqtt_port=settings.zigbee_mqtt_port,
base_topic=settings.zigbee_base_topic,
mqtt_tls=settings.zigbee_mqtt_tls,
sync_enabled=settings.zigbee_sync_enabled,
sync_interval=settings.zigbee_sync_interval,
include_mesh_links=settings.zigbee_sync_include_mesh_links,
host_configured=bool(settings.zigbee_mqtt_host),
)
@router.post("/config", response_model=ZigbeeConfig)
async def save_zigbee_config(
payload: ZigbeeSyncConfig,
_: str = Depends(get_current_user),
) -> ZigbeeConfig:
"""Persist the auto-sync activation (enabled, interval, mesh links) and apply it live.
This is the ONLY Zigbee config the app writes. Connection settings
(host/port/credentials/topic/tls) are env-only and are never accepted or
persisted here — enabling auto-sync requires the MQTT host already set in
the server env, since the scheduled job reads it from there.
"""
if payload.sync_enabled and not settings.zigbee_mqtt_host:
raise HTTPException(
status_code=400,
detail="Cannot enable auto-sync: no Zigbee MQTT host configured in the server env.",
)
try:
settings.zigbee_sync_enabled = payload.sync_enabled
settings.zigbee_sync_interval = payload.sync_interval
settings.zigbee_sync_include_mesh_links = payload.include_mesh_links
settings.save_overrides()
set_zigbee_sync_enabled(payload.sync_enabled)
if payload.sync_enabled:
reschedule_zigbee_sync(payload.sync_interval)
except HTTPException:
raise
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
return await get_zigbee_config()