Self-hosted homelab infrastructure visualizer — interactive network diagram with live status monitoring
TypeScript diagramhomelabhomelab-infrastructure-observabilitynetworkproxmox-veself-hostedzigbeezwavezwave-js
feat: wire deep-scan port ranges and HTTP probe into scanner
Pouzor9d1ebd6c6eb2
diff --git a/backend/app/services/scanner.py b/backend/app/services/scanner.pyindex 4f0ac43..fccb315 100644--- a/backend/app/services/scanner.py+++ b/backend/app/services/scanner.py@@ -7,6 +7,7 @@ import reimport socketimport subprocessimport threading+from dataclasses import dataclass, fieldfrom datetime import datetime, timezonefrom typing import Any@@ -15,6 +16,7 @@ from sqlalchemy.ext.asyncio import AsyncSessionfrom app.db.models import Node, PendingDevice, ScanRunfrom app.services.fingerprint import fingerprint_ports, suggest_node_type+from app.services.http_probe import probe_open_portslogger = logging.getLogger(__name__)@@ -34,6 +36,37 @@ _EXTRA_PORTS = ("16686,34567,37777,51413,64738")+# nmap -p accepts "N" or "N-M"; user ranges are validated against this.+_PORT_RANGE_RE = re.compile(r"^\d{1,5}(-\d{1,5})?$")+++@dataclass+class DeepScanOptions:+ """Per-scan deep-scan settings (None/empty → standard scan, today's behaviour)."""++ http_ranges: list[str] = field(default_factory=list)+ http_probe_enabled: bool = False+ verify_tls: bool = False+++def _valid_port_range(spec: str) -> bool:+ if not _PORT_RANGE_RE.match(spec):+ return False+ parts = [int(p) for p in spec.split("-")]+ if any(p < 1 or p > 65535 for p in parts):+ return False+ return len(parts) == 1 or parts[0] <= parts[1]+++def _build_port_spec(http_ranges: list[str] | None) -> str:+ """Combine the default port list with validated user ranges for nmap -p."""+ if not http_ranges:+ return _EXTRA_PORTS+ extra = [r.strip() for r in http_ranges if _valid_port_range(r.strip())]+ if not extra:+ return _EXTRA_PORTS+ return _EXTRA_PORTS + "," + ",".join(extra)+_MDNS_SERVICE_TYPES = ["_http._tcp.local.","_shelly._tcp.local.",@@ -195,7 +228,7 @@ async def _ping_sweep(target: str) -> dict[str, dict[str, Any]]:return alive-def _nmap_scan_single(host_dict: dict[str, Any]) -> dict[str, Any]:+def _nmap_scan_single(host_dict: dict[str, Any], port_spec: str = _EXTRA_PORTS) -> dict[str, Any]:"""Phase 2 — single-IP port scan with service detection.Runs in a thread (blocking). Returns the host dict enriched with open_ports.@@ -210,11 +243,11 @@ def _nmap_scan_single(host_dict: dict[str, Any]) -> dict[str, Any]:is_root = os.geteuid() == 0if is_root:# SYN scan + version detection (fastest, most accurate)- scan_args = f"-sS -sV --open -T4 -Pn --host-timeout 60s -p {_EXTRA_PORTS}"+ scan_args = f"-sS -sV --open -T4 -Pn --host-timeout 60s -p {port_spec}"else:# TCP connect scan (-sT) — no raw sockets needed, works without root.# nmap auto-selects -sT without root but being explicit avoids edge cases.- scan_args = f"-sT -sV --open -T4 -Pn --host-timeout 60s -p {_EXTRA_PORTS}"+ scan_args = f"-sT -sV --open -T4 -Pn --host-timeout 60s -p {port_spec}"logger.debug("[Phase 2] %s args: %s", ip, scan_args)nm = nmap.PortScanner()@@ -252,7 +285,9 @@ def _nmap_scan_single(host_dict: dict[str, Any]) -> dict[str, Any]:return host_dict-async def _nmap_port_scan(alive: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:+async def _nmap_port_scan(+ alive: dict[str, dict[str, Any]], port_spec: str = _EXTRA_PORTS+) -> list[dict[str, Any]]:"""Phase 2: Per-IP service detection with bounded concurrency.Each host is scanned independently in a thread — no inter-host timeout interference.@@ -266,7 +301,7 @@ async def _nmap_port_scan(alive: dict[str, dict[str, Any]]) -> list[dict[str, Anasync def _scan_with_sem(host_dict: dict[str, Any]) -> dict[str, Any]:async with semaphore:- return await asyncio.to_thread(_nmap_scan_single, host_dict)+ return await asyncio.to_thread(_nmap_scan_single, host_dict, port_spec)raw = await asyncio.gather(*[_scan_with_sem(h) for h in alive.values()], return_exceptions=True)results = []@@ -279,7 +314,7 @@ async def _nmap_port_scan(alive: dict[str, dict[str, Any]]) -> list[dict[str, Anreturn results-async def _nmap_scan(target: str) -> list[dict[str, Any]]:+async def _nmap_scan(target: str, port_spec: str = _EXTRA_PORTS) -> list[dict[str, Any]]:"""Two-phase scan for a CIDR range.Phase 1: Concurrent ping sweep to find alive hosts (fast, no false positives).@@ -296,7 +331,7 @@ async def _nmap_scan(target: str) -> list[dict[str, Any]]:except Exception as exc:logger.error("Phase 1 ping sweep failed: %s", exc)raise RuntimeError(str(exc)) from exc- return await _nmap_port_scan(alive)+ return await _nmap_port_scan(alive, port_spec)async def _mdns_discover(timeout: float = 4.0) -> list[dict[str, Any]]:@@ -375,10 +410,18 @@ def _mock_scan(target: str) -> list[dict[str, Any]]:]-async def run_scan(ranges: list[str], db: AsyncSession, run_id: str) -> None:+async def run_scan(+ ranges: list[str],+ db: AsyncSession,+ run_id: str,+ deep_scan: DeepScanOptions | None = None,+) -> None:"""Execute scan for given CIDR ranges and populate pending_devices."""from app.api.routes.status import broadcast_scan_update+ deep_scan = deep_scan or DeepScanOptions()+ port_spec = _build_port_spec(deep_scan.http_ranges)+devices_found = 0mdns_task: asyncio.Task[list[dict[str, Any]]] | None = Nonetry:@@ -427,8 +470,17 @@ async def run_scan(ranges: list[str], db: AsyncSession, run_id: str) -> None:logger.debug("Skipping %s — hidden by user", ip)return- services = fingerprint_ports(host["open_ports"])- suggested_type = suggest_node_type(host["open_ports"], host.get("mac"))+ open_ports = host["open_ports"]+ # Deep-scan HTTP probe: enrich open ports with title/header signals so+ # fingerprint can confirm services on custom ports. No-op when disabled+ # or when the host has no open ports (e.g. mDNS-only discovery).+ if deep_scan.http_probe_enabled and open_ports:+ open_ports = await probe_open_ports(+ ip, open_ports, verify_tls=deep_scan.verify_tls+ )++ services = fingerprint_ports(open_ports)+ suggested_type = suggest_node_type(open_ports, host.get("mac"))existing_result = await db.execute(select(PendingDevice).where(@@ -463,7 +515,7 @@ async def run_scan(ranges: list[str], db: AsyncSession, run_id: str) -> None:for cidr in ranges:if _is_cancelled(run_id):break- hosts = await _nmap_scan(cidr)+ hosts = await _nmap_scan(cidr, port_spec)for host in hosts:if _is_cancelled(run_id):breakdiff --git a/backend/tests/test_scan.py b/backend/tests/test_scan.pyindex 6feb0b9..56ea3ed 100644--- a/backend/tests/test_scan.py+++ b/backend/tests/test_scan.py@@ -518,7 +518,7 @@ async def test_run_scan_cancelled_mid_scan_skips_remaining_cidrs(db_session: Asycall_count = 0- def nmap_side_effect(target: str):+ def nmap_side_effect(target: str, port_spec: str | None = None):nonlocal call_countcall_count += 1# Signal cancellation after the first CIDR scan completesdiff --git a/backend/tests/test_scanner.py b/backend/tests/test_scanner.pyindex e29e2fc..b7d66fa 100644--- a/backend/tests/test_scanner.py+++ b/backend/tests/test_scanner.py@@ -359,7 +359,7 @@ async def test_nmap_port_scan_tolerates_single_host_exception():call_count = 0- def _flaky_scan(host_dict):+ def _flaky_scan(host_dict, port_spec=None):nonlocal call_countcall_count += 1if host_dict["ip"] == "192.168.1.1":@@ -533,3 +533,135 @@ async def test_run_scan_cancelled_marks_status_cancelled(mem_db):run = await session.get(ScanRun, run_id)assert run is not Noneassert run.status == "cancelled"+++# ---------------------------------------------------------------------------+# Deep scan: port-range plumbing + HTTP probe+# ---------------------------------------------------------------------------++def test_valid_port_range():+ from app.services.scanner import _valid_port_range++ assert _valid_port_range("8080")+ assert _valid_port_range("8000-8100")+ assert not _valid_port_range("8100-8000") # reversed+ assert not _valid_port_range("0") # below 1+ assert not _valid_port_range("70000") # above 65535+ assert not _valid_port_range("abc")+ assert not _valid_port_range("80,443") # not a single range+++def test_build_port_spec_default_when_empty():+ from app.services.scanner import _EXTRA_PORTS, _build_port_spec++ assert _build_port_spec([]) == _EXTRA_PORTS+ assert _build_port_spec(None) == _EXTRA_PORTS+++def test_build_port_spec_appends_valid_ranges():+ from app.services.scanner import _EXTRA_PORTS, _build_port_spec++ spec = _build_port_spec(["8000-8100", "9000"])+ assert spec == _EXTRA_PORTS + ",8000-8100,9000"+++def test_build_port_spec_drops_invalid_ranges():+ from app.services.scanner import _EXTRA_PORTS, _build_port_spec++ # invalid entries silently dropped; only valid kept+ assert _build_port_spec(["bad", "70000"]) == _EXTRA_PORTS+ assert _build_port_spec(["bad", "9000"]) == _EXTRA_PORTS + ",9000"+++async def test_run_scan_deep_scan_passes_port_spec_to_nmap(mem_db):+ from app.services.scanner import DeepScanOptions, run_scan++ run_id = _make_run_id()+ async with mem_db() as session:+ session.add(_make_scan_run(run_id))+ await session.commit()++ captured = {}++ async def fake_nmap(target, port_spec):+ captured["port_spec"] = port_spec+ return []++ async with mem_db() as session:+ with patch("app.services.scanner._nmap_scan", new=fake_nmap), \+ patch("app.services.scanner._mdns_discover", new_callable=AsyncMock, return_value=[]), \+ patch("app.api.routes.status.broadcast_scan_update", new_callable=AsyncMock):+ await run_scan(+ ["192.168.1.0/24"], session, run_id,+ deep_scan=DeepScanOptions(http_ranges=["8000-8100"]),+ )++ assert "8000-8100" in captured["port_spec"]+++async def test_run_scan_probe_enriches_services(mem_db):+ """With probe enabled, a custom-port service is identified via HTTP signals."""+ from app.services.scanner import DeepScanOptions, run_scan++ run_id = _make_run_id()+ async with mem_db() as session:+ session.add(_make_scan_run(run_id))+ await session.commit()++ nmap_hosts = [{+ "ip": "192.168.1.50", "hostname": None, "mac": None, "os": None,+ "open_ports": [{"port": 8096, "protocol": "tcp", "banner": ""}],+ }]+ jellyfin_sig = [{+ "port": 8096, "protocol": "tcp", "banner_regex": None, "http_regex": "Jellyfin",+ "service_name": "Jellyfin", "icon": "🎬", "category": "media", "suggested_node_type": "server",+ }]++ async def fake_probe(ip, ports, verify_tls=False, concurrency=50):+ return [{**p, "http_signals": {"title": "Jellyfin", "headers": {}}} for p in ports]++ async with mem_db() as session:+ with patch("app.services.scanner._nmap_scan", new=AsyncMock(return_value=nmap_hosts)), \+ patch("app.services.scanner._mdns_discover", new_callable=AsyncMock, return_value=[]), \+ patch("app.services.scanner.probe_open_ports", new=fake_probe), \+ patch("app.services.fingerprint._load", return_value=jellyfin_sig), \+ patch("app.api.routes.status.broadcast_scan_update", new_callable=AsyncMock):+ await run_scan(+ ["192.168.1.0/24"], session, run_id,+ deep_scan=DeepScanOptions(http_probe_enabled=True),+ )++ async with mem_db() as session:+ result = await session.execute(sa_select(PendingDevice).where(PendingDevice.ip == "192.168.1.50"))+ device = result.scalar_one_or_none()++ assert device is not None+ assert any(s["service_name"] == "Jellyfin" for s in device.services)+++async def test_run_scan_no_probe_when_disabled(mem_db):+ """Probe must not be called on a standard (non-deep) scan."""+ from app.services.scanner import run_scan++ run_id = _make_run_id()+ async with mem_db() as session:+ session.add(_make_scan_run(run_id))+ await session.commit()++ nmap_hosts = [{+ "ip": "192.168.1.51", "hostname": None, "mac": None, "os": None,+ "open_ports": [{"port": 8096, "protocol": "tcp", "banner": ""}],+ }]+ probe = AsyncMock()++ async with mem_db() as session:+ with patch("app.services.scanner._nmap_scan", new=AsyncMock(return_value=nmap_hosts)), \+ patch("app.services.scanner._mdns_discover", new_callable=AsyncMock, return_value=[]), \+ patch("app.services.scanner.probe_open_ports", new=probe), \+ patch("app.api.routes.status.broadcast_scan_update", new_callable=AsyncMock):+ await run_scan(["192.168.1.0/24"], session, run_id)++ probe.assert_not_called()