ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
fix(dashboard-api): a crash loop is unhealthy, not installing (#8258)
Mike Bradleycf8737d22f63
diff --git a/ods/bin/ods-host-agent.py b/ods/bin/ods-host-agent.pyindex ac85b67ff..93859df6e 100755--- a/ods/bin/ods-host-agent.py+++ b/ods/bin/ods-host-agent.py@@ -8129,11 +8129,14 @@ def _docker_service_health_snapshot() -> dict:service_id = declared_containers.get(container_name, service_id)if not service_id and container_name.startswith("ods-"):service_id = container_name.removeprefix("ods-")+ restarts = item.get("RestartCount")containers.append({"service_id": str(service_id or ""),"container_name": container_name,"state": str(state.get("Status") or "unknown"),"health": str(health.get("Status") or "none"),+ # Restart-policy restarts; tells a crash loop from a start.+ "restart_count": restarts if type(restarts) is int and restarts >= 0 else 0,})payload = {"schema_version": "ods.host-service-health.v1",diff --git a/ods/docs/HOST-AGENT-API.md b/ods/docs/HOST-AGENT-API.mdindex 8ea7d662e..ebb130920 100644--- a/ods/docs/HOST-AGENT-API.md+++ b/ods/docs/HOST-AGENT-API.md@@ -182,13 +182,19 @@ and custom names from non-Docker definitions are excluded."service_id": "dashboard","container_name": "ods-dashboard","state": "running",- "health": "healthy"+ "health": "healthy",+ "restart_count": 0}],"sampled_at": "2026-07-20T22:00:00+00:00"}```+`restart_count` is Docker's count of restart-policy restarts since the+container was last started by hand. The dashboard treats three or more, while+the container is restarting or still in its first health check, as a crash+loop and shows the extension as unhealthy instead of starting.+Returns 503 when Docker cannot provide a complete snapshot.### `GET /v1/update/status`diff --git a/ods/extensions/services/dashboard-api/helpers.py b/ods/extensions/services/dashboard-api/helpers.pyindex 2a87edc99..8735ab83d 100644--- a/ods/extensions/services/dashboard-api/helpers.py+++ b/ods/extensions/services/dashboard-api/helpers.py@@ -1141,6 +1141,25 @@ def _timeout_override_allowed(service_id: str, now: float) -> bool:return anchor is not None and now - anchor <= _TIMEOUT_OVERRIDE_GRACE_S+_CRASH_LOOP_RESTARTS = 3+++def container_crash_looping(item: dict) -> bool:+ """Whether a host service-health row shows a restart policy reviving a crash.++ Each restart begins the container's health check anew, so a crash loop+ reads as "starting" and its card said "installing" indefinitely+ (Workstation06, 2026-10-27: Qdrant and TEI on ARM64). One or two+ restarts can be a start-up race; a third is a loop.+ """+ restarts = item.get("restart_count")+ if type(restarts) is not int or restarts < _CRASH_LOOP_RESTARTS:+ return False+ state = str(item.get("state") or "").casefold()+ return state == "restarting" or (+ state == "running" and str(item.get("health") or "").casefold() == "starting")++async def get_all_services(*, only_service_ids: Optional[set[str]] = None) -> list[ServiceStatus]:"""Get all service health statuses.@@ -1219,10 +1238,14 @@ async def get_all_services(*, only_service_ids: Optional[set[str]] = None) -> liitem = by_service.get(status.id) or by_name.get(str(config.get("container_name") or ""))replacement = status.statusstartup_pending = False+ crash_looping = Falseif item and config.get("type", "docker") == "docker":health = str(item.get("health") or "none").casefold()state = str(item.get("state") or "unknown").casefold()- if health == "healthy" and status.status == "degraded" and _timeout_override_allowed(status.id, now):+ if container_crash_looping(item):+ replacement = "unhealthy"+ crash_looping = True+ elif health == "healthy" and status.status == "degraded" and _timeout_override_allowed(status.id, now):# A successful declared Docker healthcheck is authoritative for# transient dashboard-network timeouts, but never masks HTTP# errors or connection refusal from the application probe.@@ -1244,8 +1267,10 @@ async def get_all_services(*, only_service_ids: Optional[set[str]] = None) -> listartup_pending = len(matches) == 1elif state in {"exited", "dead", "removing"}:replacement = "down"- if replacement != status.status or startup_pending != status.startup_pending:- status = status.model_copy(update={"status": replacement, "startup_pending": startup_pending})+ if (replacement != status.status or startup_pending != status.startup_pending+ or crash_looping != status.crash_looping):+ status = status.model_copy(update={"status": replacement, "startup_pending": startup_pending,+ "crash_looping": crash_looping})reconciled.append(status)if _host_native_llm():diff --git a/ods/extensions/services/dashboard-api/models.py b/ods/extensions/services/dashboard-api/models.pyindex b0597c2f0..88f86696c 100644--- a/ods/extensions/services/dashboard-api/models.py+++ b/ods/extensions/services/dashboard-api/models.py@@ -33,6 +33,8 @@ class ServiceStatus(BaseModel):response_time_ms: Optional[float] = None# Internal health provenance; a slow HTTP probe alone cannot establish startup.startup_pending: bool = Field(default=False, exclude=True)+ # Internal: the restart policy keeps bringing a crashing container back up.+ crash_looping: bool = Field(default=False, exclude=True)class NodeCapabilities(BaseModel):diff --git a/ods/extensions/services/dashboard-api/routers/extensions.py b/ods/extensions/services/dashboard-api/routers/extensions.pyindex e065dd894..1071e881c 100644--- a/ods/extensions/services/dashboard-api/routers/extensions.py+++ b/ods/extensions/services/dashboard-api/routers/extensions.py@@ -550,9 +550,10 @@ def _compute_extension_status(ext: dict, services_by_id: dict) -> str:# show "installing". If older, the user likely stopped the# container afterwards — fall through to normal status logic.if not _is_stale(progress.get("updated_at", ""), max_age_seconds=300):- # Long-running services still need an observed healthy state.+ # Long-running services still need an observed healthy state;+ # a crash loop is a failure, not a slow first health check.svc = services_by_id.get(ext_id)- if not (svc and svc.status == "healthy"):+ if not (svc and (svc.status == "healthy" or getattr(svc, "crash_looping", False))):return "installing"# The process imported SERVICES before a Library action could have@@ -1893,14 +1894,19 @@ async def _inspect_non_http_user_services(configs: dict, statuses: dict) -> Nonecontainers = snapshot["containers"]except (AgentClientError, ValueError):pass+ from helpers import container_crash_looping # noqa: PLC0415 - avoid import cycle+for sid, cfg in candidates.items():matches = [item for item in containers if isinstance(item, dict) and item.get("service_id") == sid]state = "unknown"startup_pending = False+ crash_looping = False# Ambiguous snapshots cannot establish readiness of this extension.if len(matches) == 1:item = matches[0]- if item.get("state") == "running":+ if container_crash_looping(item):+ state, crash_looping = "unhealthy", True+ elif item.get("state") == "running":state = {"healthy": "healthy", "unhealthy": "unhealthy"}.get(item.get("health"), "degraded")startup_pending = item.get("health") == "starting"elif item.get("state") in {"exited", "dead", "removing", "created"}:@@ -1909,7 +1915,7 @@ async def _inspect_non_http_user_services(configs: dict, statuses: dict) -> Noneid=sid, name=cfg.get("name", sid), port=cfg["port"],external_port=cfg.get("external_port", cfg["port"]),status=state, response_time_ms=None,- startup_pending=startup_pending,+ startup_pending=startup_pending, crash_looping=crash_looping,)diff --git a/ods/extensions/services/dashboard-api/tests/test_extension_non_http_health.py b/ods/extensions/services/dashboard-api/tests/test_extension_non_http_health.pyindex fc281934a..0bd314226 100644--- a/ods/extensions/services/dashboard-api/tests/test_extension_non_http_health.py+++ b/ods/extensions/services/dashboard-api/tests/test_extension_non_http_health.py@@ -25,6 +25,20 @@ def test_non_http_readiness_requires_running_container_and_healthcheck(monkeypatassert statuses["broker"].startup_pending is (container == {"state": "running", "health": "starting"})[email protected]("container", [+ {"state": "restarting", "health": "starting", "restart_count": 5},+ {"state": "running", "health": "starting", "restart_count": 3},+])+def test_non_http_crash_loop_is_unhealthy_not_starting(monkeypatch, container):+ monkeypatch.setattr(extensions, "request_agent_json", lambda *a, **k: {+ "schema_version": "ods.host-service-health.v1", "containers": [{"service_id": "broker", **container}],+ })+ statuses: dict[str, ServiceStatus] = {}+ asyncio.run(extensions._inspect_non_http_user_services({"broker": {"port": 1883, "health": ""}}, statuses))+ assert (statuses["broker"].status, statuses["broker"].startup_pending,+ statuses["broker"].crash_looping) == ("unhealthy", False, True)++@pytest.mark.parametrize("snapshot", [{"schema_version": "wrong", "containers": []},{"schema_version": "ods.host-service-health.v1", "containers": "bad"},diff --git a/ods/extensions/services/dashboard-api/tests/test_extension_startup_health.py b/ods/extensions/services/dashboard-api/tests/test_extension_startup_health.pyindex 0e8b9e335..1c4649d75 100644--- a/ods/extensions/services/dashboard-api/tests/test_extension_startup_health.py+++ b/ods/extensions/services/dashboard-api/tests/test_extension_startup_health.py@@ -1,5 +1,6 @@"""Follow real health reconciliation through the cache and Library endpoints."""import asyncio+from datetime import datetime, timezonefrom unittest.mock import AsyncMockimport pytest@@ -95,6 +96,47 @@ def test_startup_requires_unambiguous_evidence_and_never_hides_failure(startup_fassert row["runtime_starting"] is False[email protected]("row", [+ {"service_id": "n8n", "container_name": "ods-n8n", "state": "restarting", "health": "starting", "restart_count": 4},+ {"service_id": "n8n", "container_name": "ods-n8n", "state": "running", "health": "starting", "restart_count": 3},+])+def test_a_crash_loop_is_unhealthy_not_installing(startup_fixture, test_client, monkeypatch, row):+ """Workstation06 (2026-10-27): Qdrant and TEI crash-looped on ARM64 while+ their cards said "installing" for as long as anyone watched."""+ sample, _ = startup_fixture+ status = sample("down", rows=[row])+ assert (status.status, status.startup_pending, status.crash_looping) == ("unhealthy", False, True)+ assert "crash_looping" not in status.model_dump() # Internal provenance, like startup_pending.+ for library_row in read_library(test_client):+ assert library_row["status"] == "unhealthy"+ assert library_row["runtime_starting"] is False+ # A fresh start receipt does not hide it either.+ now = datetime.now(timezone.utc).isoformat()+ monkeypatch.setattr(extensions, "_read_progress", lambda _: {"status": "started", "updated_at": now})+ for library_row in read_library(test_client):+ assert library_row["status"] == "unhealthy"++[email protected]("row,looping", [+ ({"state": "restarting", "restart_count": 1}, False), # One crash can be a start-up race.+ ({"state": "restarting", "restart_count": 3}, True),+ ({"state": "running", "health": "starting", "restart_count": "3"}, False),+ ({"state": "running", "health": "healthy", "restart_count": 9}, False), # Recovered.+ ({"state": "running", "health": "starting"}, False), # An older host agent sends no count.+])+def test_crash_loop_needs_repeated_restarts(row, looping):+ assert helpers.container_crash_looping(row) is looping+++def test_a_few_restarts_while_starting_are_still_a_start(startup_fixture, test_client):+ sample, _ = startup_fixture+ status = sample("down", rows=[{"service_id": "n8n", "container_name": "ods-n8n",+ "state": "running", "health": "starting", "restart_count": 2}])+ assert (status.startup_pending, status.crash_looping) == (True, False)+ for row in read_library(test_client):+ assert row["status"] == "installing"++def test_disabled_selection_and_explicit_install_error_override_startup(startup_fixture, test_client, monkeypatch):sample, builtin = startup_fixturesample()diff --git a/ods/extensions/services/dashboard-api/tests/test_host_agent.py b/ods/extensions/services/dashboard-api/tests/test_host_agent.pyindex 575fc4a0a..6f4c05605 100644--- a/ods/extensions/services/dashboard-api/tests/test_host_agent.py+++ b/ods/extensions/services/dashboard-api/tests/test_host_agent.py@@ -11316,9 +11316,28 @@ class TestDockerServiceHealthSnapshot:"container_name": "ods-dashboard","state": "running","health": "healthy",+ "restart_count": 0,}]assert len(calls) == 2+ @pytest.mark.parametrize(("restarts", "expected"), [(7, 7), (-1, 0), ("7", 0), (True, 0), (None, 0)])+ def test_reports_restart_policy_restarts(self, monkeypatch, restarts, expected):+ """A crash loop (Qdrant and TEI on Workstation06's ARM64, 2026-10-27)+ restarts its health check each time; the restart count tells them apart."""+ monkeypatch.setattr(_mod, "_service_health_cache", (0.0, None))++ def run(cmd, **kwargs):+ if cmd[1] == "ps":+ return types.SimpleNamespace(returncode=0, stdout="ods-qdrant\n", stderr="")+ return types.SimpleNamespace(returncode=0, stdout=json.dumps([{+ "Name": "/ods-qdrant", "RestartCount": restarts,+ "State": {"Status": "restarting", "Health": {"Status": "starting"}},+ "Config": {"Labels": {"com.docker.compose.service": "qdrant"}},+ }]), stderr="")++ monkeypatch.setattr(_mod.subprocess, "run", run)+ assert _mod._docker_service_health_snapshot()["containers"][0]["restart_count"] == expected+# .env values that name no model: the configured model's identities are all None._NO_CONFIGURED_MODEL = {"llmModel": None, "ggufFile": None, "contextSize": None}