ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
fix(dashboard-api): hold release checks after a failure; dry run survives an owner-only .env (#8273)
Mike Bradleyb5c5517c31e9
diff --git a/ods/docker-compose.base.yml b/ods/docker-compose.base.ymlindex cee94996d..c5622b6ee 100644--- a/ods/docker-compose.base.yml+++ b/ods/docker-compose.base.yml@@ -285,6 +285,8 @@ services:# UID 1000 leaves the 0600 .env unreadable here, and the status must then# fall back to this copy instead of counting Open WebUI as a core service down.- ENABLE_OPEN_WEBUI=${ENABLE_OPEN_WEBUI:-true}+ # The same fallback keeps --offline's switch off the GitHub release check.+ - DISABLE_UPDATE_CHECK=${DISABLE_UPDATE_CHECK:-false}- ODS_AGENT_HOST=${ODS_AGENT_HOST:-}- ODS_AGENT_PORT=${ODS_AGENT_PORT:-7710}- GPU_BACKEND=${GPU_BACKEND:-nvidia}diff --git a/ods/extensions/services/dashboard-api/routers/updates.py b/ods/extensions/services/dashboard-api/routers/updates.pyindex 9c7edd75c..9ed53ceb2 100644--- a/ods/extensions/services/dashboard-api/routers/updates.py+++ b/ods/extensions/services/dashboard-api/routers/updates.py@@ -3,6 +3,8 @@import asyncioimport jsonimport logging+import math+import osimport reimport timefrom datetime import datetime, timezone@@ -35,6 +37,12 @@ _GITHUB_HEADERS = {"Accept": "application/vnd.github.v3+json"}_VERSION_CACHE_TTL = 300.0_version_cache: dict[str, object] = {"expires_at": 0.0, "payload": None}_version_refresh_task: Optional[asyncio.Task] = None+# A failed check holds further GitHub requests. The dashboard asks again every+# minute while a check is unavailable, and GitHub allows 60 unauthenticated+# requests an hour per address, which every machine behind a shared NAT spends.+_VERSION_RETRY_SECONDS = 300.0+_VERSION_MAX_HOLD_SECONDS = 3600.0+_version_hold: dict[str, object] = {"until": 0.0, "rate_limited": False}def _read_utf8(path: Path) -> str:@@ -144,6 +152,10 @@ def _update_check_disabled() -> bool:if line.startswith("DISABLE_UPDATE_CHECK="):value = parse_env_value(line.split("=", 1)[1]).strip().lower()disabled = value in {"true", "1", "yes"}+ except PermissionError:+ # An install owner other than UID 1000 leaves the 0600 .env unreadable+ # here; compose passes the value it read instead.+ return os.environ.get("DISABLE_UPDATE_CHECK", "").strip().lower() in {"true", "1", "yes"}except OSError:return Falsereturn disabled@@ -225,8 +237,41 @@ def _release_payload(response: httpx.Response) -> dict:}+def _rate_limit_seconds(response: httpx.Response) -> Optional[float]:+ """Seconds GitHub asks this address to wait, or None when it set no limit."""+ if response.status_code not in (403, 429):+ return None+ retry_after = response.headers.get("retry-after", "").strip()+ if retry_after.isdigit():+ return float(retry_after)+ if response.headers.get("x-ratelimit-remaining") == "0":+ reset = response.headers.get("x-ratelimit-reset", "").strip()+ return max(0.0, int(reset) - time.time()) if reset.isdigit() else 0.0+ return 0.0 if response.status_code == 429 else None+++def _hold_release_checks(rate_limit_seconds: Optional[float], reason: str) -> None:+ global _version_hold+ wait = min(max(rate_limit_seconds or 0.0, _VERSION_RETRY_SECONDS), _VERSION_MAX_HOLD_SECONDS)+ _version_hold = {"until": time.monotonic() + wait, "rate_limited": rate_limit_seconds is not None}+ logger.info("Release check failed (%s); next GitHub request in %d s", reason, wait)+++def _release_checks_held(force: bool) -> bool:+ """A manual check retries an ordinary failure, but not a rate limit GitHub would refuse."""+ held = time.monotonic() < float(_version_hold["until"]) # type: ignore[arg-type]+ return held and (bool(_version_hold["rate_limited"]) or not force)+++def _held_check_error() -> str:+ if not _version_hold["rate_limited"]:+ return "Could not reach GitHub"+ minutes = max(1, math.ceil((float(_version_hold["until"]) - time.monotonic()) / 60)) # type: ignore[arg-type]+ return f"GitHub's request limit for this network is used up; try again in {minutes} min"++async def _refresh_release_cache() -> Optional[dict]:- global _version_cache+ global _version_cache, _version_holdtry:async with httpx.AsyncClient(timeout=5.0) as client:response = await client.get(@@ -234,19 +279,28 @@ async def _refresh_release_cache() -> Optional[dict]:headers=_GITHUB_HEADERS,)payload = _release_payload(response)- _version_cache = {- "expires_at": time.monotonic() + _VERSION_CACHE_TTL,- "payload": payload,- }- return payload- except (httpx.HTTPError, httpx.TimeoutException, json.JSONDecodeError, OSError, ValueError):+ except httpx.HTTPStatusError as exc:+ _hold_release_checks(_rate_limit_seconds(exc.response), f"GitHub answered {exc.response.status_code}")+ return None+ except (httpx.HTTPError, httpx.TimeoutException, json.JSONDecodeError, OSError, ValueError) as exc:+ _hold_release_checks(None, type(exc).__name__)return None+ _version_cache = {+ "expires_at": time.monotonic() + _VERSION_CACHE_TTL,+ "payload": payload,+ }+ _version_hold = {"until": 0.0, "rate_limited": False}+ return payload-def _ensure_release_refresh() -> asyncio.Task:+def _ensure_release_refresh(force: bool = False) -> Optional[asyncio.Task]:+ """Start one GitHub check, or return None while a failed one holds them."""global _version_refresh_task- if _version_refresh_task is None or _version_refresh_task.done():- _version_refresh_task = asyncio.create_task(_refresh_release_cache())+ if _version_refresh_task is not None and not _version_refresh_task.done():+ return _version_refresh_task+ if _release_checks_held(force):+ return None+ _version_refresh_task = asyncio.create_task(_refresh_release_cache())return _version_refresh_task@@ -261,10 +315,12 @@ async def get_version(force: bool = False):return _build_version_result(current, cached)stale = _get_cached_release_payload(allow_stale=True)- refresh_task = _ensure_release_refresh()+ refresh_task = _ensure_release_refresh(force)if stale and not force:return _build_version_result(current, stale, "stale")+ if refresh_task is None:+ return _build_version_result(current, stale, "stale" if stale else "unavailable")try:payload = await asyncio.wait_for(asyncio.shield(refresh_task), timeout=6.0 if force else 1.25)@@ -329,21 +385,23 @@ async def get_update_dry_run():update_available = Falseversion_check_error: Optional[str] = None- try:- async with httpx.AsyncClient(timeout=8.0) as client:- resp = await client.get(- f"{_GITHUB_RELEASES_API}/latest",- headers=_GITHUB_HEADERS,- )- version = _build_version_result(current, _release_payload(resp))- latest = version["latest"]- changelog_url = version["changelog_url"]- update_available = version["update_available"]- if version["check_status"] == "current-unknown":- version_check_error = "Installed version is unknown; automatic version comparison is unavailable."- except (httpx.HTTPError, httpx.TimeoutException, OSError, json.JSONDecodeError, ValueError) as e:- logger.warning("update dry-run version check failed: %s", e)- version_check_error = "Could not reach GitHub"+ if await asyncio.to_thread(_update_check_disabled):+ version_check_error = "Update checks are off (DISABLE_UPDATE_CHECK)"+ else:+ # A dry run is a manual check that shares the dashboard's cache and hold.+ payload = _get_cached_release_payload()+ refresh_task = None if payload else _ensure_release_refresh(force=True)+ if refresh_task is not None:+ payload = await asyncio.shield(refresh_task)+ if payload is None:+ version_check_error = _held_check_error()+ else:+ version = _build_version_result(current, payload)+ latest = version["latest"]+ changelog_url = version["changelog_url"]+ update_available = version["update_available"]+ if version["check_status"] == "current-unknown":+ version_check_error = "Installed version is unknown; automatic version comparison is unavailable."# ── configured image tags from compose files ──────────────────────────────images: list[str] = []@@ -360,14 +418,19 @@ async def get_update_dry_run():# ── .env keys relevant to the update path ────────────────────────────────env_snapshot: dict[str, str] = {}- if env_file.exists():- for line in _read_utf8(env_file).splitlines():- line = line.strip()- if not line or line.startswith("#") or "=" not in line:- continue- key, _, val = line.partition("=")- if key in _UPDATE_ENV_KEYS:- env_snapshot[key] = val+ env_readable = True+ try:+ env_lines = _read_utf8(env_file).splitlines() if env_file.exists() else []+ except PermissionError:+ env_readable = False+ env_lines = []+ for line in env_lines:+ line = line.strip()+ if not line or line.startswith("#") or "=" not in line:+ continue+ key, _, val = line.partition("=")+ if key in _UPDATE_ENV_KEYS:+ env_snapshot[key] = valreturn {"dry_run": True,@@ -376,7 +439,10 @@ async def get_update_dry_run():"update_available": update_available,"changelog_url": changelog_url,"images": images,- "env_keys": env_snapshot,+ # An install owner other than UID 1000 leaves the 0600 .env unreadable+ # here. Null keys make `ods dry-run` read the file as the owner instead.+ "env_keys": env_snapshot if env_readable else None,+ "env_keys_error": None if env_readable else "The install's .env is readable only by its owner","version_check_error": version_check_error,}diff --git a/ods/extensions/services/dashboard-api/tests/conftest.py b/ods/extensions/services/dashboard-api/tests/conftest.pyindex c30f718ee..586a9810c 100644--- a/ods/extensions/services/dashboard-api/tests/conftest.py+++ b/ods/extensions/services/dashboard-api/tests/conftest.py@@ -67,6 +67,16 @@ def _no_remembered_portal_status(monkeypatch):monkeypatch.setattr(pixel, "_last_available_status", None)[email protected](autouse=True)+def _no_remembered_release_check(monkeypatch):+ """Each test starts without a release or a failed check remembered by an earlier one."""+ from routers import updates++ monkeypatch.setattr(updates, "_version_cache", {"expires_at": 0.0, "payload": None})+ monkeypatch.setattr(updates, "_version_hold", {"until": 0.0, "rate_limited": False})+ monkeypatch.setattr(updates, "_version_refresh_task", None)++@pytest.fixture(autouse=True)def _no_remembered_model_management(monkeypatch):"""Each test starts without a model management proof remembered by an earlier one."""diff --git a/ods/extensions/services/dashboard-api/tests/test_compose_env_mount.py b/ods/extensions/services/dashboard-api/tests/test_compose_env_mount.pyindex de0d0f3dd..7983e3b06 100644--- a/ods/extensions/services/dashboard-api/tests/test_compose_env_mount.py+++ b/ods/extensions/services/dashboard-api/tests/test_compose_env_mount.py@@ -19,3 +19,10 @@ def test_dashboard_api_receives_the_open_webui_switch_for_an_unreadable_env():service = yaml.safe_load(compose.read_text(encoding="utf-8"))["services"]["dashboard-api"]assert "ENABLE_OPEN_WEBUI=${ENABLE_OPEN_WEBUI:-true}" in service["environment"]+++def test_dashboard_api_receives_the_offline_update_switch_for_an_unreadable_env():+ compose = Path(__file__).resolve().parents[4] / "docker-compose.base.yml"+ service = yaml.safe_load(compose.read_text(encoding="utf-8"))["services"]["dashboard-api"]++ assert "DISABLE_UPDATE_CHECK=${DISABLE_UPDATE_CHECK:-false}" in service["environment"]diff --git a/ods/extensions/services/dashboard-api/tests/test_update_detection.py b/ods/extensions/services/dashboard-api/tests/test_update_detection.pyindex 88e4b4597..2ef86492f 100644--- a/ods/extensions/services/dashboard-api/tests/test_update_detection.py+++ b/ods/extensions/services/dashboard-api/tests/test_update_detection.py@@ -76,3 +76,149 @@ def test_failed_forced_check_does_not_announce_cached_release(monkeypatch):result = asyncio.run(updates.get_version(force=True))assert result['latest'] == '2.7.0' and result['check_status'] == 'stale'assert not result['update_available']+++def _github(monkeypatch, response=None, error=None):+ """Answer every GitHub request with one response or error, and count them."""+ client = AsyncMock()+ client.__aenter__.return_value = client+ if error is not None:+ client.get.side_effect = error+ else:+ client.get.return_value = response+ monkeypatch.setattr(updates.httpx, 'AsyncClient', lambda **_: client)+ return client.get+++def _hold_seconds() -> float:+ until = updates._version_hold['until']+ assert isinstance(until, float)+ return until - time.monotonic()+++def _response(status, headers=None, data=None):+ return httpx.Response(status, json=data if data is not None else {'message': 'x'}, headers=headers or {},+ request=httpx.Request('GET', updates._GITHUB_RELEASES_API))++[email protected]('status,headers,expected', [+ (403, {'x-ratelimit-remaining': '0', 'x-ratelimit-reset': 'RESET+600'}, 600),+ (403, {'retry-after': '30'}, 30), (429, {'retry-after': '120'}, 120),+ (429, {}, 0), (403, {}, None), (404, {}, None), (500, {'retry-after': '30'}, None),+])+def test_rate_limit_seconds_reads_github_headers(status, headers, expected):+ now = int(time.time())+ headers = {k: str(now + 600) if v == 'RESET+600' else v for k, v in headers.items()}+ seconds = updates._rate_limit_seconds(_response(status, headers))+ if expected is None:+ assert seconds is None+ else:+ assert seconds is not None and abs(seconds - expected) <= 2+++def test_rate_limited_check_holds_every_request_until_github_resets(monkeypatch):+ # Workstation06 (2026-10-27): the site's shared address had spent GitHub's+ # 60 unauthenticated requests, and each dashboard poll asked again.+ monkeypatch.setattr(updates, '_read_current_version', lambda: '3.0.0')+ reset = str(int(time.time()) + 1200)+ get = _github(monkeypatch, _response(403, {'x-ratelimit-remaining': '0', 'x-ratelimit-reset': reset}))++ assert asyncio.run(updates.get_version())['check_status'] == 'unavailable'+ assert get.await_count == 1+ assert updates._version_hold['rate_limited'] is True+ assert 1100 < _hold_seconds() <= 1200++ for force in (False, True, False):+ assert asyncio.run(updates.get_version(force=force))['check_status'] == 'unavailable'+ assert get.await_count == 1+++def test_failed_check_holds_polls_but_a_manual_check_retries(monkeypatch):+ monkeypatch.setattr(updates, '_read_current_version', lambda: '3.0.0')+ get = _github(monkeypatch, error=httpx.ConnectError('offline'))++ asyncio.run(updates.get_version())+ asyncio.run(updates.get_version())+ assert get.await_count == 1+ assert updates._version_hold['rate_limited'] is False++ asyncio.run(updates.get_version(force=True))+ assert get.await_count == 2+++def test_hold_is_capped_at_an_hour_and_cleared_by_a_good_check(monkeypatch):+ monkeypatch.setattr(updates, '_read_current_version', lambda: '3.0.0')+ reset = str(int(time.time()) + 7200)+ _github(monkeypatch, _response(403, {'x-ratelimit-remaining': '0', 'x-ratelimit-reset': reset}))+ asyncio.run(updates.get_version())+ assert _hold_seconds() <= 3600++ monkeypatch.setattr(updates, '_version_hold', {'until': 0.0, 'rate_limited': True})+ _github(monkeypatch, _response(200, data={'tag_name': 'v3.0.0', 'html_url': 'https://github.com/x'}))+ assert asyncio.run(updates.get_version())['check_status'] == 'checked'+ assert updates._version_hold == {'until': 0.0, 'rate_limited': False}+++def _owner_only_env(monkeypatch, tmp_path, text):+ """The install owner is not UID 1000, so this container cannot read the 0600 .env."""+ (tmp_path / '.env').write_text(text)+ monkeypatch.setattr(updates, 'INSTALL_DIR', str(tmp_path))+ real_read = updates._read_utf8++ def read(path):+ if path.name == '.env':+ raise PermissionError(13, 'Permission denied', str(path))+ return real_read(path)++ monkeypatch.setattr(updates, '_read_utf8', read)+++def test_offline_switch_holds_when_the_env_file_is_owner_only(monkeypatch, tmp_path):+ _owner_only_env(monkeypatch, tmp_path, 'DISABLE_UPDATE_CHECK=true\n')+ monkeypatch.setenv('DISABLE_UPDATE_CHECK', 'true')+ assert updates._update_check_disabled() is True+ monkeypatch.setenv('DISABLE_UPDATE_CHECK', 'false')+ assert updates._update_check_disabled() is False+++def test_dry_run_reports_an_owner_only_env_instead_of_failing(test_client, monkeypatch, tmp_path):+ # Workstation06 (2026-10-27): this read raised PermissionError and the dry run answered 500.+ _owner_only_env(monkeypatch, tmp_path, 'ODS_VERSION=3.0.0\nTIER=4\n')+ monkeypatch.delenv('DISABLE_UPDATE_CHECK', raising=False)+ _github(monkeypatch, _response(200, data={'tag_name': 'v3.0.0', 'html_url': 'https://github.com/x'}))++ resp = test_client.get('/api/update/dry-run', headers=test_client.auth_headers)++ assert resp.status_code == 200+ data = resp.json()+ assert data['env_keys'] is None+ assert data['env_keys_error'] == "The install's .env is readable only by its owner"+ assert data['latest_version'] == '3.0.0'+++def test_dry_run_honors_the_offline_switch(test_client, monkeypatch, tmp_path):+ (tmp_path / '.env').write_text('ODS_VERSION=3.0.0\nDISABLE_UPDATE_CHECK=true\n')+ monkeypatch.setattr(updates, 'INSTALL_DIR', str(tmp_path))+ get = _github(monkeypatch, error=AssertionError('GitHub must not be contacted when update checks are off'))++ data = test_client.get('/api/update/dry-run', headers=test_client.auth_headers).json()++ assert get.await_count == 0+ assert data['latest_version'] is None+ assert data['version_check_error'] == 'Update checks are off (DISABLE_UPDATE_CHECK)'+ assert data['env_keys'] == {'ODS_VERSION': '3.0.0'}+++def test_dry_run_shares_the_dashboard_cache_and_rate_limit_hold(test_client, monkeypatch, tmp_path):+ monkeypatch.setattr(updates, 'INSTALL_DIR', str(tmp_path))+ get = _github(monkeypatch, error=AssertionError('the cached release answers the dry run'))+ monkeypatch.setattr(updates, '_version_cache', {'expires_at': time.monotonic() + 100,+ 'payload': {'latest': '9.0.0', 'changelog_url': None}})+ assert test_client.get('/api/update/dry-run', headers=test_client.auth_headers).json()['latest_version'] == '9.0.0'++ monkeypatch.setattr(updates, '_version_cache', {'expires_at': 0.0, 'payload': None})+ monkeypatch.setattr(updates, '_version_hold', {'until': time.monotonic() + 600, 'rate_limited': True})+ data = test_client.get('/api/update/dry-run', headers=test_client.auth_headers).json()+ assert get.await_count == 0+ assert data['latest_version'] is None+ assert data['version_check_error'] == "GitHub's request limit for this network is used up; try again in 10 min"diff --git a/ods/ods-cli b/ods/ods-cliindex cd3f0c2df..0312eae8b 100755--- a/ods/ods-cli+++ b/ods/ods-cli@@ -2275,10 +2275,11 @@ cmd_dry_run() {"http://127.0.0.1:${dashboard_api_port}/api/update/dry-run" 2>/dev/null || true)fi- local latest_ver="" changelog_url="" update_status=""+ local latest_ver="" changelog_url="" update_status="" check_error=""if [[ -n "$api_json" ]] && echo "$api_json" | jq -e '.current_version' >/dev/null 2>&1; thenlatest_ver=$(echo "$api_json" | jq -r '.latest_version // empty')changelog_url=$(echo "$api_json" | jq -r '.changelog_url // empty')+ check_error=$(echo "$api_json" | jq -r '.version_check_error // empty')local api_availableapi_available=$(echo "$api_json" | jq -r '.update_available // false')[[ "$api_available" == "true" ]] && update_status="update available" || update_status="up to date"@@ -2311,7 +2312,7 @@ cmd_dry_run() {fi[[ -n "$changelog_url" ]] && echo " Changelog : ${changelog_url}"else- echo " Available : (could not reach GitHub)"+ echo " Available : (${check_error:-could not reach GitHub})"fiecho ""