Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(dashboard-api): a crash loop is unhealthy, not installing (#8258)

Mike Bradleycf8737d22f63
diff --git a/ods/bin/ods-host-agent.py b/ods/bin/ods-host-agent.py
index ac85b67ff..93859df6e 100755
--- a/ods/bin/ods-host-agent.py
+++ b/ods/bin/ods-host-agent.py
@@ -8129,11 +8129,14 @@ def _docker_service_health_snapshot() -> dict:
service_id = declared_containers.get(container_name, service_id)
if not service_id and container_name.startswith("ods-"):
service_id = container_name.removeprefix("ods-")
+ restarts = item.get("RestartCount")
containers.append({
"service_id": str(service_id or ""),
"container_name": container_name,
"state": str(state.get("Status") or "unknown"),
"health": str(health.get("Status") or "none"),
+ # Restart-policy restarts; tells a crash loop from a start.
+ "restart_count": restarts if type(restarts) is int and restarts >= 0 else 0,
})
payload = {
"schema_version": "ods.host-service-health.v1",
diff --git a/ods/docs/HOST-AGENT-API.md b/ods/docs/HOST-AGENT-API.md
index 8ea7d662e..ebb130920 100644
--- a/ods/docs/HOST-AGENT-API.md
+++ b/ods/docs/HOST-AGENT-API.md
@@ -182,13 +182,19 @@ and custom names from non-Docker definitions are excluded.
"service_id": "dashboard",
"container_name": "ods-dashboard",
"state": "running",
- "health": "healthy"
+ "health": "healthy",
+ "restart_count": 0
}
],
"sampled_at": "2026-07-20T22:00:00+00:00"
}
```
+`restart_count` is Docker's count of restart-policy restarts since the
+container was last started by hand. The dashboard treats three or more, while
+the container is restarting or still in its first health check, as a crash
+loop and shows the extension as unhealthy instead of starting.
+
Returns 503 when Docker cannot provide a complete snapshot.
### `GET /v1/update/status`
diff --git a/ods/extensions/services/dashboard-api/helpers.py b/ods/extensions/services/dashboard-api/helpers.py
index 2a87edc99..8735ab83d 100644
--- a/ods/extensions/services/dashboard-api/helpers.py
+++ b/ods/extensions/services/dashboard-api/helpers.py
@@ -1141,6 +1141,25 @@ def _timeout_override_allowed(service_id: str, now: float) -> bool:
return anchor is not None and now - anchor <= _TIMEOUT_OVERRIDE_GRACE_S
+_CRASH_LOOP_RESTARTS = 3
+
+
+def container_crash_looping(item: dict) -> bool:
+ """Whether a host service-health row shows a restart policy reviving a crash.
+
+ Each restart begins the container's health check anew, so a crash loop
+ reads as "starting" and its card said "installing" indefinitely
+ (Workstation06, 2026-10-27: Qdrant and TEI on ARM64). One or two
+ restarts can be a start-up race; a third is a loop.
+ """
+ restarts = item.get("restart_count")
+ if type(restarts) is not int or restarts < _CRASH_LOOP_RESTARTS:
+ return False
+ state = str(item.get("state") or "").casefold()
+ return state == "restarting" or (
+ state == "running" and str(item.get("health") or "").casefold() == "starting")
+
+
async def get_all_services(*, only_service_ids: Optional[set[str]] = None) -> list[ServiceStatus]:
"""Get all service health statuses.
@@ -1219,10 +1238,14 @@ async def get_all_services(*, only_service_ids: Optional[set[str]] = None) -> li
item = by_service.get(status.id) or by_name.get(str(config.get("container_name") or ""))
replacement = status.status
startup_pending = False
+ crash_looping = False
if item and config.get("type", "docker") == "docker":
health = str(item.get("health") or "none").casefold()
state = str(item.get("state") or "unknown").casefold()
- if health == "healthy" and status.status == "degraded" and _timeout_override_allowed(status.id, now):
+ if container_crash_looping(item):
+ replacement = "unhealthy"
+ crash_looping = True
+ elif health == "healthy" and status.status == "degraded" and _timeout_override_allowed(status.id, now):
# A successful declared Docker healthcheck is authoritative for
# transient dashboard-network timeouts, but never masks HTTP
# errors or connection refusal from the application probe.
@@ -1244,8 +1267,10 @@ async def get_all_services(*, only_service_ids: Optional[set[str]] = None) -> li
startup_pending = len(matches) == 1
elif state in {"exited", "dead", "removing"}:
replacement = "down"
- if replacement != status.status or startup_pending != status.startup_pending:
- status = status.model_copy(update={"status": replacement, "startup_pending": startup_pending})
+ if (replacement != status.status or startup_pending != status.startup_pending
+ or crash_looping != status.crash_looping):
+ status = status.model_copy(update={"status": replacement, "startup_pending": startup_pending,
+ "crash_looping": crash_looping})
reconciled.append(status)
if _host_native_llm():
diff --git a/ods/extensions/services/dashboard-api/models.py b/ods/extensions/services/dashboard-api/models.py
index b0597c2f0..88f86696c 100644
--- a/ods/extensions/services/dashboard-api/models.py
+++ b/ods/extensions/services/dashboard-api/models.py
@@ -33,6 +33,8 @@ class ServiceStatus(BaseModel):
response_time_ms: Optional[float] = None
# Internal health provenance; a slow HTTP probe alone cannot establish startup.
startup_pending: bool = Field(default=False, exclude=True)
+ # Internal: the restart policy keeps bringing a crashing container back up.
+ crash_looping: bool = Field(default=False, exclude=True)
class NodeCapabilities(BaseModel):
diff --git a/ods/extensions/services/dashboard-api/routers/extensions.py b/ods/extensions/services/dashboard-api/routers/extensions.py
index e065dd894..1071e881c 100644
--- a/ods/extensions/services/dashboard-api/routers/extensions.py
+++ b/ods/extensions/services/dashboard-api/routers/extensions.py
@@ -550,9 +550,10 @@ def _compute_extension_status(ext: dict, services_by_id: dict) -> str:
# show "installing". If older, the user likely stopped the
# container afterwards — fall through to normal status logic.
if not _is_stale(progress.get("updated_at", ""), max_age_seconds=300):
- # Long-running services still need an observed healthy state.
+ # Long-running services still need an observed healthy state;
+ # a crash loop is a failure, not a slow first health check.
svc = services_by_id.get(ext_id)
- if not (svc and svc.status == "healthy"):
+ if not (svc and (svc.status == "healthy" or getattr(svc, "crash_looping", False))):
return "installing"
# The process imported SERVICES before a Library action could have
@@ -1893,14 +1894,19 @@ async def _inspect_non_http_user_services(configs: dict, statuses: dict) -> None
containers = snapshot["containers"]
except (AgentClientError, ValueError):
pass
+ from helpers import container_crash_looping # noqa: PLC0415 - avoid import cycle
+
for sid, cfg in candidates.items():
matches = [item for item in containers if isinstance(item, dict) and item.get("service_id") == sid]
state = "unknown"
startup_pending = False
+ crash_looping = False
# Ambiguous snapshots cannot establish readiness of this extension.
if len(matches) == 1:
item = matches[0]
- if item.get("state") == "running":
+ if container_crash_looping(item):
+ state, crash_looping = "unhealthy", True
+ elif item.get("state") == "running":
state = {"healthy": "healthy", "unhealthy": "unhealthy"}.get(item.get("health"), "degraded")
startup_pending = item.get("health") == "starting"
elif item.get("state") in {"exited", "dead", "removing", "created"}:
@@ -1909,7 +1915,7 @@ async def _inspect_non_http_user_services(configs: dict, statuses: dict) -> None
id=sid, name=cfg.get("name", sid), port=cfg["port"],
external_port=cfg.get("external_port", cfg["port"]),
status=state, response_time_ms=None,
- startup_pending=startup_pending,
+ startup_pending=startup_pending, crash_looping=crash_looping,
)
diff --git a/ods/extensions/services/dashboard-api/tests/test_extension_non_http_health.py b/ods/extensions/services/dashboard-api/tests/test_extension_non_http_health.py
index fc281934a..0bd314226 100644
--- a/ods/extensions/services/dashboard-api/tests/test_extension_non_http_health.py
+++ b/ods/extensions/services/dashboard-api/tests/test_extension_non_http_health.py
@@ -25,6 +25,20 @@ def test_non_http_readiness_requires_running_container_and_healthcheck(monkeypat
assert statuses["broker"].startup_pending is (container == {"state": "running", "health": "starting"})
[email protected]("container", [
+ {"state": "restarting", "health": "starting", "restart_count": 5},
+ {"state": "running", "health": "starting", "restart_count": 3},
+])
+def test_non_http_crash_loop_is_unhealthy_not_starting(monkeypatch, container):
+ monkeypatch.setattr(extensions, "request_agent_json", lambda *a, **k: {
+ "schema_version": "ods.host-service-health.v1", "containers": [{"service_id": "broker", **container}],
+ })
+ statuses: dict[str, ServiceStatus] = {}
+ asyncio.run(extensions._inspect_non_http_user_services({"broker": {"port": 1883, "health": ""}}, statuses))
+ assert (statuses["broker"].status, statuses["broker"].startup_pending,
+ statuses["broker"].crash_looping) == ("unhealthy", False, True)
+
+
@pytest.mark.parametrize("snapshot", [
{"schema_version": "wrong", "containers": []},
{"schema_version": "ods.host-service-health.v1", "containers": "bad"},
diff --git a/ods/extensions/services/dashboard-api/tests/test_extension_startup_health.py b/ods/extensions/services/dashboard-api/tests/test_extension_startup_health.py
index 0e8b9e335..1c4649d75 100644
--- a/ods/extensions/services/dashboard-api/tests/test_extension_startup_health.py
+++ b/ods/extensions/services/dashboard-api/tests/test_extension_startup_health.py
@@ -1,5 +1,6 @@
"""Follow real health reconciliation through the cache and Library endpoints."""
import asyncio
+from datetime import datetime, timezone
from unittest.mock import AsyncMock
import pytest
@@ -95,6 +96,47 @@ def test_startup_requires_unambiguous_evidence_and_never_hides_failure(startup_f
assert row["runtime_starting"] is False
+ {"service_id": "n8n", "container_name": "ods-n8n", "state": "restarting", "health": "starting", "restart_count": 4},
+ {"service_id": "n8n", "container_name": "ods-n8n", "state": "running", "health": "starting", "restart_count": 3},
+])
+def test_a_crash_loop_is_unhealthy_not_installing(startup_fixture, test_client, monkeypatch, row):
+ """Workstation06 (2026-10-27): Qdrant and TEI crash-looped on ARM64 while
+ their cards said "installing" for as long as anyone watched."""
+ sample, _ = startup_fixture
+ status = sample("down", rows=[row])
+ assert (status.status, status.startup_pending, status.crash_looping) == ("unhealthy", False, True)
+ assert "crash_looping" not in status.model_dump() # Internal provenance, like startup_pending.
+ for library_row in read_library(test_client):
+ assert library_row["status"] == "unhealthy"
+ assert library_row["runtime_starting"] is False
+ # A fresh start receipt does not hide it either.
+ now = datetime.now(timezone.utc).isoformat()
+ monkeypatch.setattr(extensions, "_read_progress", lambda _: {"status": "started", "updated_at": now})
+ for library_row in read_library(test_client):
+ assert library_row["status"] == "unhealthy"
+
+
[email protected]("row,looping", [
+ ({"state": "restarting", "restart_count": 1}, False), # One crash can be a start-up race.
+ ({"state": "restarting", "restart_count": 3}, True),
+ ({"state": "running", "health": "starting", "restart_count": "3"}, False),
+ ({"state": "running", "health": "healthy", "restart_count": 9}, False), # Recovered.
+ ({"state": "running", "health": "starting"}, False), # An older host agent sends no count.
+])
+def test_crash_loop_needs_repeated_restarts(row, looping):
+ assert helpers.container_crash_looping(row) is looping
+
+
+def test_a_few_restarts_while_starting_are_still_a_start(startup_fixture, test_client):
+ sample, _ = startup_fixture
+ status = sample("down", rows=[{"service_id": "n8n", "container_name": "ods-n8n",
+ "state": "running", "health": "starting", "restart_count": 2}])
+ assert (status.startup_pending, status.crash_looping) == (True, False)
+ for row in read_library(test_client):
+ assert row["status"] == "installing"
+
+
def test_disabled_selection_and_explicit_install_error_override_startup(startup_fixture, test_client, monkeypatch):
sample, builtin = startup_fixture
sample()
diff --git a/ods/extensions/services/dashboard-api/tests/test_host_agent.py b/ods/extensions/services/dashboard-api/tests/test_host_agent.py
index 575fc4a0a..6f4c05605 100644
--- a/ods/extensions/services/dashboard-api/tests/test_host_agent.py
+++ b/ods/extensions/services/dashboard-api/tests/test_host_agent.py
@@ -11316,9 +11316,28 @@ class TestDockerServiceHealthSnapshot:
"container_name": "ods-dashboard",
"state": "running",
"health": "healthy",
+ "restart_count": 0,
}]
assert len(calls) == 2
+ @pytest.mark.parametrize(("restarts", "expected"), [(7, 7), (-1, 0), ("7", 0), (True, 0), (None, 0)])
+ def test_reports_restart_policy_restarts(self, monkeypatch, restarts, expected):
+ """A crash loop (Qdrant and TEI on Workstation06's ARM64, 2026-10-27)
+ restarts its health check each time; the restart count tells them apart."""
+ monkeypatch.setattr(_mod, "_service_health_cache", (0.0, None))
+
+ def run(cmd, **kwargs):
+ if cmd[1] == "ps":
+ return types.SimpleNamespace(returncode=0, stdout="ods-qdrant\n", stderr="")
+ return types.SimpleNamespace(returncode=0, stdout=json.dumps([{
+ "Name": "/ods-qdrant", "RestartCount": restarts,
+ "State": {"Status": "restarting", "Health": {"Status": "starting"}},
+ "Config": {"Labels": {"com.docker.compose.service": "qdrant"}},
+ }]), stderr="")
+
+ monkeypatch.setattr(_mod.subprocess, "run", run)
+ assert _mod._docker_service_health_snapshot()["containers"][0]["restart_count"] == expected
+
# .env values that name no model: the configured model's identities are all None.
_NO_CONFIGURED_MODEL = {"llmModel": None, "ggufFile": None, "contextSize": None}