Skip to content

Self-hosted homelab infrastructure visualizer — interactive network diagram with live status monitoring

feat: wire deep-scan port ranges and HTTP probe into scanner

Pouzor9d1ebd6c6eb2
diff --git a/backend/app/services/scanner.py b/backend/app/services/scanner.py
index 4f0ac43..fccb315 100644
--- a/backend/app/services/scanner.py
+++ b/backend/app/services/scanner.py
@@ -7,6 +7,7 @@ import re
import socket
import subprocess
import threading
+from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any
@@ -15,6 +16,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models import Node, PendingDevice, ScanRun
from app.services.fingerprint import fingerprint_ports, suggest_node_type
+from app.services.http_probe import probe_open_ports
logger = logging.getLogger(__name__)
@@ -34,6 +36,37 @@ _EXTRA_PORTS = (
"16686,34567,37777,51413,64738"
)
+# nmap -p accepts "N" or "N-M"; user ranges are validated against this.
+_PORT_RANGE_RE = re.compile(r"^\d{1,5}(-\d{1,5})?$")
+
+
+@dataclass
+class DeepScanOptions:
+ """Per-scan deep-scan settings (None/empty → standard scan, today's behaviour)."""
+
+ http_ranges: list[str] = field(default_factory=list)
+ http_probe_enabled: bool = False
+ verify_tls: bool = False
+
+
+def _valid_port_range(spec: str) -> bool:
+ if not _PORT_RANGE_RE.match(spec):
+ return False
+ parts = [int(p) for p in spec.split("-")]
+ if any(p < 1 or p > 65535 for p in parts):
+ return False
+ return len(parts) == 1 or parts[0] <= parts[1]
+
+
+def _build_port_spec(http_ranges: list[str] | None) -> str:
+ """Combine the default port list with validated user ranges for nmap -p."""
+ if not http_ranges:
+ return _EXTRA_PORTS
+ extra = [r.strip() for r in http_ranges if _valid_port_range(r.strip())]
+ if not extra:
+ return _EXTRA_PORTS
+ return _EXTRA_PORTS + "," + ",".join(extra)
+
_MDNS_SERVICE_TYPES = [
"_http._tcp.local.",
"_shelly._tcp.local.",
@@ -195,7 +228,7 @@ async def _ping_sweep(target: str) -> dict[str, dict[str, Any]]:
return alive
-def _nmap_scan_single(host_dict: dict[str, Any]) -> dict[str, Any]:
+def _nmap_scan_single(host_dict: dict[str, Any], port_spec: str = _EXTRA_PORTS) -> dict[str, Any]:
"""
Phase 2 — single-IP port scan with service detection.
Runs in a thread (blocking). Returns the host dict enriched with open_ports.
@@ -210,11 +243,11 @@ def _nmap_scan_single(host_dict: dict[str, Any]) -> dict[str, Any]:
is_root = os.geteuid() == 0
if is_root:
# SYN scan + version detection (fastest, most accurate)
- scan_args = f"-sS -sV --open -T4 -Pn --host-timeout 60s -p {_EXTRA_PORTS}"
+ scan_args = f"-sS -sV --open -T4 -Pn --host-timeout 60s -p {port_spec}"
else:
# TCP connect scan (-sT) — no raw sockets needed, works without root.
# nmap auto-selects -sT without root but being explicit avoids edge cases.
- scan_args = f"-sT -sV --open -T4 -Pn --host-timeout 60s -p {_EXTRA_PORTS}"
+ scan_args = f"-sT -sV --open -T4 -Pn --host-timeout 60s -p {port_spec}"
logger.debug("[Phase 2] %s args: %s", ip, scan_args)
nm = nmap.PortScanner()
@@ -252,7 +285,9 @@ def _nmap_scan_single(host_dict: dict[str, Any]) -> dict[str, Any]:
return host_dict
-async def _nmap_port_scan(alive: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:
+async def _nmap_port_scan(
+ alive: dict[str, dict[str, Any]], port_spec: str = _EXTRA_PORTS
+) -> list[dict[str, Any]]:
"""
Phase 2: Per-IP service detection with bounded concurrency.
Each host is scanned independently in a thread — no inter-host timeout interference.
@@ -266,7 +301,7 @@ async def _nmap_port_scan(alive: dict[str, dict[str, Any]]) -> list[dict[str, An
async def _scan_with_sem(host_dict: dict[str, Any]) -> dict[str, Any]:
async with semaphore:
- return await asyncio.to_thread(_nmap_scan_single, host_dict)
+ return await asyncio.to_thread(_nmap_scan_single, host_dict, port_spec)
raw = await asyncio.gather(*[_scan_with_sem(h) for h in alive.values()], return_exceptions=True)
results = []
@@ -279,7 +314,7 @@ async def _nmap_port_scan(alive: dict[str, dict[str, Any]]) -> list[dict[str, An
return results
-async def _nmap_scan(target: str) -> list[dict[str, Any]]:
+async def _nmap_scan(target: str, port_spec: str = _EXTRA_PORTS) -> list[dict[str, Any]]:
"""
Two-phase scan for a CIDR range.
Phase 1: Concurrent ping sweep to find alive hosts (fast, no false positives).
@@ -296,7 +331,7 @@ async def _nmap_scan(target: str) -> list[dict[str, Any]]:
except Exception as exc:
logger.error("Phase 1 ping sweep failed: %s", exc)
raise RuntimeError(str(exc)) from exc
- return await _nmap_port_scan(alive)
+ return await _nmap_port_scan(alive, port_spec)
async def _mdns_discover(timeout: float = 4.0) -> list[dict[str, Any]]:
@@ -375,10 +410,18 @@ def _mock_scan(target: str) -> list[dict[str, Any]]:
]
-async def run_scan(ranges: list[str], db: AsyncSession, run_id: str) -> None:
+async def run_scan(
+ ranges: list[str],
+ db: AsyncSession,
+ run_id: str,
+ deep_scan: DeepScanOptions | None = None,
+) -> None:
"""Execute scan for given CIDR ranges and populate pending_devices."""
from app.api.routes.status import broadcast_scan_update
+ deep_scan = deep_scan or DeepScanOptions()
+ port_spec = _build_port_spec(deep_scan.http_ranges)
+
devices_found = 0
mdns_task: asyncio.Task[list[dict[str, Any]]] | None = None
try:
@@ -427,8 +470,17 @@ async def run_scan(ranges: list[str], db: AsyncSession, run_id: str) -> None:
logger.debug("Skipping %s — hidden by user", ip)
return
- services = fingerprint_ports(host["open_ports"])
- suggested_type = suggest_node_type(host["open_ports"], host.get("mac"))
+ open_ports = host["open_ports"]
+ # Deep-scan HTTP probe: enrich open ports with title/header signals so
+ # fingerprint can confirm services on custom ports. No-op when disabled
+ # or when the host has no open ports (e.g. mDNS-only discovery).
+ if deep_scan.http_probe_enabled and open_ports:
+ open_ports = await probe_open_ports(
+ ip, open_ports, verify_tls=deep_scan.verify_tls
+ )
+
+ services = fingerprint_ports(open_ports)
+ suggested_type = suggest_node_type(open_ports, host.get("mac"))
existing_result = await db.execute(
select(PendingDevice).where(
@@ -463,7 +515,7 @@ async def run_scan(ranges: list[str], db: AsyncSession, run_id: str) -> None:
for cidr in ranges:
if _is_cancelled(run_id):
break
- hosts = await _nmap_scan(cidr)
+ hosts = await _nmap_scan(cidr, port_spec)
for host in hosts:
if _is_cancelled(run_id):
break
diff --git a/backend/tests/test_scan.py b/backend/tests/test_scan.py
index 6feb0b9..56ea3ed 100644
--- a/backend/tests/test_scan.py
+++ b/backend/tests/test_scan.py
@@ -518,7 +518,7 @@ async def test_run_scan_cancelled_mid_scan_skips_remaining_cidrs(db_session: Asy
call_count = 0
- def nmap_side_effect(target: str):
+ def nmap_side_effect(target: str, port_spec: str | None = None):
nonlocal call_count
call_count += 1
# Signal cancellation after the first CIDR scan completes
diff --git a/backend/tests/test_scanner.py b/backend/tests/test_scanner.py
index e29e2fc..b7d66fa 100644
--- a/backend/tests/test_scanner.py
+++ b/backend/tests/test_scanner.py
@@ -359,7 +359,7 @@ async def test_nmap_port_scan_tolerates_single_host_exception():
call_count = 0
- def _flaky_scan(host_dict):
+ def _flaky_scan(host_dict, port_spec=None):
nonlocal call_count
call_count += 1
if host_dict["ip"] == "192.168.1.1":
@@ -533,3 +533,135 @@ async def test_run_scan_cancelled_marks_status_cancelled(mem_db):
run = await session.get(ScanRun, run_id)
assert run is not None
assert run.status == "cancelled"
+
+
+# ---------------------------------------------------------------------------
+# Deep scan: port-range plumbing + HTTP probe
+# ---------------------------------------------------------------------------
+
+def test_valid_port_range():
+ from app.services.scanner import _valid_port_range
+
+ assert _valid_port_range("8080")
+ assert _valid_port_range("8000-8100")
+ assert not _valid_port_range("8100-8000") # reversed
+ assert not _valid_port_range("0") # below 1
+ assert not _valid_port_range("70000") # above 65535
+ assert not _valid_port_range("abc")
+ assert not _valid_port_range("80,443") # not a single range
+
+
+def test_build_port_spec_default_when_empty():
+ from app.services.scanner import _EXTRA_PORTS, _build_port_spec
+
+ assert _build_port_spec([]) == _EXTRA_PORTS
+ assert _build_port_spec(None) == _EXTRA_PORTS
+
+
+def test_build_port_spec_appends_valid_ranges():
+ from app.services.scanner import _EXTRA_PORTS, _build_port_spec
+
+ spec = _build_port_spec(["8000-8100", "9000"])
+ assert spec == _EXTRA_PORTS + ",8000-8100,9000"
+
+
+def test_build_port_spec_drops_invalid_ranges():
+ from app.services.scanner import _EXTRA_PORTS, _build_port_spec
+
+ # invalid entries silently dropped; only valid kept
+ assert _build_port_spec(["bad", "70000"]) == _EXTRA_PORTS
+ assert _build_port_spec(["bad", "9000"]) == _EXTRA_PORTS + ",9000"
+
+
+async def test_run_scan_deep_scan_passes_port_spec_to_nmap(mem_db):
+ from app.services.scanner import DeepScanOptions, run_scan
+
+ run_id = _make_run_id()
+ async with mem_db() as session:
+ session.add(_make_scan_run(run_id))
+ await session.commit()
+
+ captured = {}
+
+ async def fake_nmap(target, port_spec):
+ captured["port_spec"] = port_spec
+ return []
+
+ async with mem_db() as session:
+ with patch("app.services.scanner._nmap_scan", new=fake_nmap), \
+ patch("app.services.scanner._mdns_discover", new_callable=AsyncMock, return_value=[]), \
+ patch("app.api.routes.status.broadcast_scan_update", new_callable=AsyncMock):
+ await run_scan(
+ ["192.168.1.0/24"], session, run_id,
+ deep_scan=DeepScanOptions(http_ranges=["8000-8100"]),
+ )
+
+ assert "8000-8100" in captured["port_spec"]
+
+
+async def test_run_scan_probe_enriches_services(mem_db):
+ """With probe enabled, a custom-port service is identified via HTTP signals."""
+ from app.services.scanner import DeepScanOptions, run_scan
+
+ run_id = _make_run_id()
+ async with mem_db() as session:
+ session.add(_make_scan_run(run_id))
+ await session.commit()
+
+ nmap_hosts = [{
+ "ip": "192.168.1.50", "hostname": None, "mac": None, "os": None,
+ "open_ports": [{"port": 8096, "protocol": "tcp", "banner": ""}],
+ }]
+ jellyfin_sig = [{
+ "port": 8096, "protocol": "tcp", "banner_regex": None, "http_regex": "Jellyfin",
+ "service_name": "Jellyfin", "icon": "🎬", "category": "media", "suggested_node_type": "server",
+ }]
+
+ async def fake_probe(ip, ports, verify_tls=False, concurrency=50):
+ return [{**p, "http_signals": {"title": "Jellyfin", "headers": {}}} for p in ports]
+
+ async with mem_db() as session:
+ with patch("app.services.scanner._nmap_scan", new=AsyncMock(return_value=nmap_hosts)), \
+ patch("app.services.scanner._mdns_discover", new_callable=AsyncMock, return_value=[]), \
+ patch("app.services.scanner.probe_open_ports", new=fake_probe), \
+ patch("app.services.fingerprint._load", return_value=jellyfin_sig), \
+ patch("app.api.routes.status.broadcast_scan_update", new_callable=AsyncMock):
+ await run_scan(
+ ["192.168.1.0/24"], session, run_id,
+ deep_scan=DeepScanOptions(http_probe_enabled=True),
+ )
+
+ async with mem_db() as session:
+ result = await session.execute(sa_select(PendingDevice).where(PendingDevice.ip == "192.168.1.50"))
+ device = result.scalar_one_or_none()
+
+ assert device is not None
+ assert any(s["service_name"] == "Jellyfin" for s in device.services)
+
+
+async def test_run_scan_no_probe_when_disabled(mem_db):
+ """Probe must not be called on a standard (non-deep) scan."""
+ from app.services.scanner import run_scan
+
+ run_id = _make_run_id()
+ async with mem_db() as session:
+ session.add(_make_scan_run(run_id))
+ await session.commit()
+
+ nmap_hosts = [{
+ "ip": "192.168.1.51", "hostname": None, "mac": None, "os": None,
+ "open_ports": [{"port": 8096, "protocol": "tcp", "banner": ""}],
+ }]
+ probe = AsyncMock()
+
+ async with mem_db() as session:
+ with patch("app.services.scanner._nmap_scan", new=AsyncMock(return_value=nmap_hosts)), \
+ patch("app.services.scanner._mdns_discover", new_callable=AsyncMock, return_value=[]), \
+ patch("app.services.scanner.probe_open_ports", new=probe), \
+ patch("app.api.routes.status.broadcast_scan_update", new_callable=AsyncMock):
+ await run_scan(["192.168.1.0/24"], session, run_id)
+
+ probe.assert_not_called()