Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(dashboard-api): hold release checks after a failure; dry run survives an owner-only .env (#8273)

Mike Bradleyb5c5517c31e9
diff --git a/ods/docker-compose.base.yml b/ods/docker-compose.base.yml
index cee94996d..c5622b6ee 100644
--- a/ods/docker-compose.base.yml
+++ b/ods/docker-compose.base.yml
@@ -285,6 +285,8 @@ services:
# UID 1000 leaves the 0600 .env unreadable here, and the status must then
# fall back to this copy instead of counting Open WebUI as a core service down.
- ENABLE_OPEN_WEBUI=${ENABLE_OPEN_WEBUI:-true}
+ # The same fallback keeps --offline's switch off the GitHub release check.
+ - DISABLE_UPDATE_CHECK=${DISABLE_UPDATE_CHECK:-false}
- ODS_AGENT_HOST=${ODS_AGENT_HOST:-}
- ODS_AGENT_PORT=${ODS_AGENT_PORT:-7710}
- GPU_BACKEND=${GPU_BACKEND:-nvidia}
diff --git a/ods/extensions/services/dashboard-api/routers/updates.py b/ods/extensions/services/dashboard-api/routers/updates.py
index 9c7edd75c..9ed53ceb2 100644
--- a/ods/extensions/services/dashboard-api/routers/updates.py
+++ b/ods/extensions/services/dashboard-api/routers/updates.py
@@ -3,6 +3,8 @@
import asyncio
import json
import logging
+import math
+import os
import re
import time
from datetime import datetime, timezone
@@ -35,6 +37,12 @@ _GITHUB_HEADERS = {"Accept": "application/vnd.github.v3+json"}
_VERSION_CACHE_TTL = 300.0
_version_cache: dict[str, object] = {"expires_at": 0.0, "payload": None}
_version_refresh_task: Optional[asyncio.Task] = None
+# A failed check holds further GitHub requests. The dashboard asks again every
+# minute while a check is unavailable, and GitHub allows 60 unauthenticated
+# requests an hour per address, which every machine behind a shared NAT spends.
+_VERSION_RETRY_SECONDS = 300.0
+_VERSION_MAX_HOLD_SECONDS = 3600.0
+_version_hold: dict[str, object] = {"until": 0.0, "rate_limited": False}
def _read_utf8(path: Path) -> str:
@@ -144,6 +152,10 @@ def _update_check_disabled() -> bool:
if line.startswith("DISABLE_UPDATE_CHECK="):
value = parse_env_value(line.split("=", 1)[1]).strip().lower()
disabled = value in {"true", "1", "yes"}
+ except PermissionError:
+ # An install owner other than UID 1000 leaves the 0600 .env unreadable
+ # here; compose passes the value it read instead.
+ return os.environ.get("DISABLE_UPDATE_CHECK", "").strip().lower() in {"true", "1", "yes"}
except OSError:
return False
return disabled
@@ -225,8 +237,41 @@ def _release_payload(response: httpx.Response) -> dict:
}
+def _rate_limit_seconds(response: httpx.Response) -> Optional[float]:
+ """Seconds GitHub asks this address to wait, or None when it set no limit."""
+ if response.status_code not in (403, 429):
+ return None
+ retry_after = response.headers.get("retry-after", "").strip()
+ if retry_after.isdigit():
+ return float(retry_after)
+ if response.headers.get("x-ratelimit-remaining") == "0":
+ reset = response.headers.get("x-ratelimit-reset", "").strip()
+ return max(0.0, int(reset) - time.time()) if reset.isdigit() else 0.0
+ return 0.0 if response.status_code == 429 else None
+
+
+def _hold_release_checks(rate_limit_seconds: Optional[float], reason: str) -> None:
+ global _version_hold
+ wait = min(max(rate_limit_seconds or 0.0, _VERSION_RETRY_SECONDS), _VERSION_MAX_HOLD_SECONDS)
+ _version_hold = {"until": time.monotonic() + wait, "rate_limited": rate_limit_seconds is not None}
+ logger.info("Release check failed (%s); next GitHub request in %d s", reason, wait)
+
+
+def _release_checks_held(force: bool) -> bool:
+ """A manual check retries an ordinary failure, but not a rate limit GitHub would refuse."""
+ held = time.monotonic() < float(_version_hold["until"]) # type: ignore[arg-type]
+ return held and (bool(_version_hold["rate_limited"]) or not force)
+
+
+def _held_check_error() -> str:
+ if not _version_hold["rate_limited"]:
+ return "Could not reach GitHub"
+ minutes = max(1, math.ceil((float(_version_hold["until"]) - time.monotonic()) / 60)) # type: ignore[arg-type]
+ return f"GitHub's request limit for this network is used up; try again in {minutes} min"
+
+
async def _refresh_release_cache() -> Optional[dict]:
- global _version_cache
+ global _version_cache, _version_hold
try:
async with httpx.AsyncClient(timeout=5.0) as client:
response = await client.get(
@@ -234,19 +279,28 @@ async def _refresh_release_cache() -> Optional[dict]:
headers=_GITHUB_HEADERS,
)
payload = _release_payload(response)
- _version_cache = {
- "expires_at": time.monotonic() + _VERSION_CACHE_TTL,
- "payload": payload,
- }
- return payload
- except (httpx.HTTPError, httpx.TimeoutException, json.JSONDecodeError, OSError, ValueError):
+ except httpx.HTTPStatusError as exc:
+ _hold_release_checks(_rate_limit_seconds(exc.response), f"GitHub answered {exc.response.status_code}")
+ return None
+ except (httpx.HTTPError, httpx.TimeoutException, json.JSONDecodeError, OSError, ValueError) as exc:
+ _hold_release_checks(None, type(exc).__name__)
return None
+ _version_cache = {
+ "expires_at": time.monotonic() + _VERSION_CACHE_TTL,
+ "payload": payload,
+ }
+ _version_hold = {"until": 0.0, "rate_limited": False}
+ return payload
-def _ensure_release_refresh() -> asyncio.Task:
+def _ensure_release_refresh(force: bool = False) -> Optional[asyncio.Task]:
+ """Start one GitHub check, or return None while a failed one holds them."""
global _version_refresh_task
- if _version_refresh_task is None or _version_refresh_task.done():
- _version_refresh_task = asyncio.create_task(_refresh_release_cache())
+ if _version_refresh_task is not None and not _version_refresh_task.done():
+ return _version_refresh_task
+ if _release_checks_held(force):
+ return None
+ _version_refresh_task = asyncio.create_task(_refresh_release_cache())
return _version_refresh_task
@@ -261,10 +315,12 @@ async def get_version(force: bool = False):
return _build_version_result(current, cached)
stale = _get_cached_release_payload(allow_stale=True)
- refresh_task = _ensure_release_refresh()
+ refresh_task = _ensure_release_refresh(force)
if stale and not force:
return _build_version_result(current, stale, "stale")
+ if refresh_task is None:
+ return _build_version_result(current, stale, "stale" if stale else "unavailable")
try:
payload = await asyncio.wait_for(asyncio.shield(refresh_task), timeout=6.0 if force else 1.25)
@@ -329,21 +385,23 @@ async def get_update_dry_run():
update_available = False
version_check_error: Optional[str] = None
- try:
- async with httpx.AsyncClient(timeout=8.0) as client:
- resp = await client.get(
- f"{_GITHUB_RELEASES_API}/latest",
- headers=_GITHUB_HEADERS,
- )
- version = _build_version_result(current, _release_payload(resp))
- latest = version["latest"]
- changelog_url = version["changelog_url"]
- update_available = version["update_available"]
- if version["check_status"] == "current-unknown":
- version_check_error = "Installed version is unknown; automatic version comparison is unavailable."
- except (httpx.HTTPError, httpx.TimeoutException, OSError, json.JSONDecodeError, ValueError) as e:
- logger.warning("update dry-run version check failed: %s", e)
- version_check_error = "Could not reach GitHub"
+ if await asyncio.to_thread(_update_check_disabled):
+ version_check_error = "Update checks are off (DISABLE_UPDATE_CHECK)"
+ else:
+ # A dry run is a manual check that shares the dashboard's cache and hold.
+ payload = _get_cached_release_payload()
+ refresh_task = None if payload else _ensure_release_refresh(force=True)
+ if refresh_task is not None:
+ payload = await asyncio.shield(refresh_task)
+ if payload is None:
+ version_check_error = _held_check_error()
+ else:
+ version = _build_version_result(current, payload)
+ latest = version["latest"]
+ changelog_url = version["changelog_url"]
+ update_available = version["update_available"]
+ if version["check_status"] == "current-unknown":
+ version_check_error = "Installed version is unknown; automatic version comparison is unavailable."
# ── configured image tags from compose files ──────────────────────────────
images: list[str] = []
@@ -360,14 +418,19 @@ async def get_update_dry_run():
# ── .env keys relevant to the update path ────────────────────────────────
env_snapshot: dict[str, str] = {}
- if env_file.exists():
- for line in _read_utf8(env_file).splitlines():
- line = line.strip()
- if not line or line.startswith("#") or "=" not in line:
- continue
- key, _, val = line.partition("=")
- if key in _UPDATE_ENV_KEYS:
- env_snapshot[key] = val
+ env_readable = True
+ try:
+ env_lines = _read_utf8(env_file).splitlines() if env_file.exists() else []
+ except PermissionError:
+ env_readable = False
+ env_lines = []
+ for line in env_lines:
+ line = line.strip()
+ if not line or line.startswith("#") or "=" not in line:
+ continue
+ key, _, val = line.partition("=")
+ if key in _UPDATE_ENV_KEYS:
+ env_snapshot[key] = val
return {
"dry_run": True,
@@ -376,7 +439,10 @@ async def get_update_dry_run():
"update_available": update_available,
"changelog_url": changelog_url,
"images": images,
- "env_keys": env_snapshot,
+ # An install owner other than UID 1000 leaves the 0600 .env unreadable
+ # here. Null keys make `ods dry-run` read the file as the owner instead.
+ "env_keys": env_snapshot if env_readable else None,
+ "env_keys_error": None if env_readable else "The install's .env is readable only by its owner",
"version_check_error": version_check_error,
}
diff --git a/ods/extensions/services/dashboard-api/tests/conftest.py b/ods/extensions/services/dashboard-api/tests/conftest.py
index c30f718ee..586a9810c 100644
--- a/ods/extensions/services/dashboard-api/tests/conftest.py
+++ b/ods/extensions/services/dashboard-api/tests/conftest.py
@@ -67,6 +67,16 @@ def _no_remembered_portal_status(monkeypatch):
monkeypatch.setattr(pixel, "_last_available_status", None)
[email protected](autouse=True)
+def _no_remembered_release_check(monkeypatch):
+ """Each test starts without a release or a failed check remembered by an earlier one."""
+ from routers import updates
+
+ monkeypatch.setattr(updates, "_version_cache", {"expires_at": 0.0, "payload": None})
+ monkeypatch.setattr(updates, "_version_hold", {"until": 0.0, "rate_limited": False})
+ monkeypatch.setattr(updates, "_version_refresh_task", None)
+
+
@pytest.fixture(autouse=True)
def _no_remembered_model_management(monkeypatch):
"""Each test starts without a model management proof remembered by an earlier one."""
diff --git a/ods/extensions/services/dashboard-api/tests/test_compose_env_mount.py b/ods/extensions/services/dashboard-api/tests/test_compose_env_mount.py
index de0d0f3dd..7983e3b06 100644
--- a/ods/extensions/services/dashboard-api/tests/test_compose_env_mount.py
+++ b/ods/extensions/services/dashboard-api/tests/test_compose_env_mount.py
@@ -19,3 +19,10 @@ def test_dashboard_api_receives_the_open_webui_switch_for_an_unreadable_env():
service = yaml.safe_load(compose.read_text(encoding="utf-8"))["services"]["dashboard-api"]
assert "ENABLE_OPEN_WEBUI=${ENABLE_OPEN_WEBUI:-true}" in service["environment"]
+
+
+def test_dashboard_api_receives_the_offline_update_switch_for_an_unreadable_env():
+ compose = Path(__file__).resolve().parents[4] / "docker-compose.base.yml"
+ service = yaml.safe_load(compose.read_text(encoding="utf-8"))["services"]["dashboard-api"]
+
+ assert "DISABLE_UPDATE_CHECK=${DISABLE_UPDATE_CHECK:-false}" in service["environment"]
diff --git a/ods/extensions/services/dashboard-api/tests/test_update_detection.py b/ods/extensions/services/dashboard-api/tests/test_update_detection.py
index 88e4b4597..2ef86492f 100644
--- a/ods/extensions/services/dashboard-api/tests/test_update_detection.py
+++ b/ods/extensions/services/dashboard-api/tests/test_update_detection.py
@@ -76,3 +76,149 @@ def test_failed_forced_check_does_not_announce_cached_release(monkeypatch):
result = asyncio.run(updates.get_version(force=True))
assert result['latest'] == '2.7.0' and result['check_status'] == 'stale'
assert not result['update_available']
+
+
+def _github(monkeypatch, response=None, error=None):
+ """Answer every GitHub request with one response or error, and count them."""
+ client = AsyncMock()
+ client.__aenter__.return_value = client
+ if error is not None:
+ client.get.side_effect = error
+ else:
+ client.get.return_value = response
+ monkeypatch.setattr(updates.httpx, 'AsyncClient', lambda **_: client)
+ return client.get
+
+
+def _hold_seconds() -> float:
+ until = updates._version_hold['until']
+ assert isinstance(until, float)
+ return until - time.monotonic()
+
+
+def _response(status, headers=None, data=None):
+ return httpx.Response(status, json=data if data is not None else {'message': 'x'}, headers=headers or {},
+ request=httpx.Request('GET', updates._GITHUB_RELEASES_API))
+
+
[email protected]('status,headers,expected', [
+ (403, {'x-ratelimit-remaining': '0', 'x-ratelimit-reset': 'RESET+600'}, 600),
+ (403, {'retry-after': '30'}, 30), (429, {'retry-after': '120'}, 120),
+ (429, {}, 0), (403, {}, None), (404, {}, None), (500, {'retry-after': '30'}, None),
+])
+def test_rate_limit_seconds_reads_github_headers(status, headers, expected):
+ now = int(time.time())
+ headers = {k: str(now + 600) if v == 'RESET+600' else v for k, v in headers.items()}
+ seconds = updates._rate_limit_seconds(_response(status, headers))
+ if expected is None:
+ assert seconds is None
+ else:
+ assert seconds is not None and abs(seconds - expected) <= 2
+
+
+def test_rate_limited_check_holds_every_request_until_github_resets(monkeypatch):
+ # Workstation06 (2026-10-27): the site's shared address had spent GitHub's
+ # 60 unauthenticated requests, and each dashboard poll asked again.
+ monkeypatch.setattr(updates, '_read_current_version', lambda: '3.0.0')
+ reset = str(int(time.time()) + 1200)
+ get = _github(monkeypatch, _response(403, {'x-ratelimit-remaining': '0', 'x-ratelimit-reset': reset}))
+
+ assert asyncio.run(updates.get_version())['check_status'] == 'unavailable'
+ assert get.await_count == 1
+ assert updates._version_hold['rate_limited'] is True
+ assert 1100 < _hold_seconds() <= 1200
+
+ for force in (False, True, False):
+ assert asyncio.run(updates.get_version(force=force))['check_status'] == 'unavailable'
+ assert get.await_count == 1
+
+
+def test_failed_check_holds_polls_but_a_manual_check_retries(monkeypatch):
+ monkeypatch.setattr(updates, '_read_current_version', lambda: '3.0.0')
+ get = _github(monkeypatch, error=httpx.ConnectError('offline'))
+
+ asyncio.run(updates.get_version())
+ asyncio.run(updates.get_version())
+ assert get.await_count == 1
+ assert updates._version_hold['rate_limited'] is False
+
+ asyncio.run(updates.get_version(force=True))
+ assert get.await_count == 2
+
+
+def test_hold_is_capped_at_an_hour_and_cleared_by_a_good_check(monkeypatch):
+ monkeypatch.setattr(updates, '_read_current_version', lambda: '3.0.0')
+ reset = str(int(time.time()) + 7200)
+ _github(monkeypatch, _response(403, {'x-ratelimit-remaining': '0', 'x-ratelimit-reset': reset}))
+ asyncio.run(updates.get_version())
+ assert _hold_seconds() <= 3600
+
+ monkeypatch.setattr(updates, '_version_hold', {'until': 0.0, 'rate_limited': True})
+ _github(monkeypatch, _response(200, data={'tag_name': 'v3.0.0', 'html_url': 'https://github.com/x'}))
+ assert asyncio.run(updates.get_version())['check_status'] == 'checked'
+ assert updates._version_hold == {'until': 0.0, 'rate_limited': False}
+
+
+def _owner_only_env(monkeypatch, tmp_path, text):
+ """The install owner is not UID 1000, so this container cannot read the 0600 .env."""
+ (tmp_path / '.env').write_text(text)
+ monkeypatch.setattr(updates, 'INSTALL_DIR', str(tmp_path))
+ real_read = updates._read_utf8
+
+ def read(path):
+ if path.name == '.env':
+ raise PermissionError(13, 'Permission denied', str(path))
+ return real_read(path)
+
+ monkeypatch.setattr(updates, '_read_utf8', read)
+
+
+def test_offline_switch_holds_when_the_env_file_is_owner_only(monkeypatch, tmp_path):
+ _owner_only_env(monkeypatch, tmp_path, 'DISABLE_UPDATE_CHECK=true\n')
+ monkeypatch.setenv('DISABLE_UPDATE_CHECK', 'true')
+ assert updates._update_check_disabled() is True
+ monkeypatch.setenv('DISABLE_UPDATE_CHECK', 'false')
+ assert updates._update_check_disabled() is False
+
+
+def test_dry_run_reports_an_owner_only_env_instead_of_failing(test_client, monkeypatch, tmp_path):
+ # Workstation06 (2026-10-27): this read raised PermissionError and the dry run answered 500.
+ _owner_only_env(monkeypatch, tmp_path, 'ODS_VERSION=3.0.0\nTIER=4\n')
+ monkeypatch.delenv('DISABLE_UPDATE_CHECK', raising=False)
+ _github(monkeypatch, _response(200, data={'tag_name': 'v3.0.0', 'html_url': 'https://github.com/x'}))
+
+ resp = test_client.get('/api/update/dry-run', headers=test_client.auth_headers)
+
+ assert resp.status_code == 200
+ data = resp.json()
+ assert data['env_keys'] is None
+ assert data['env_keys_error'] == "The install's .env is readable only by its owner"
+ assert data['latest_version'] == '3.0.0'
+
+
+def test_dry_run_honors_the_offline_switch(test_client, monkeypatch, tmp_path):
+ (tmp_path / '.env').write_text('ODS_VERSION=3.0.0\nDISABLE_UPDATE_CHECK=true\n')
+ monkeypatch.setattr(updates, 'INSTALL_DIR', str(tmp_path))
+ get = _github(monkeypatch, error=AssertionError('GitHub must not be contacted when update checks are off'))
+
+ data = test_client.get('/api/update/dry-run', headers=test_client.auth_headers).json()
+
+ assert get.await_count == 0
+ assert data['latest_version'] is None
+ assert data['version_check_error'] == 'Update checks are off (DISABLE_UPDATE_CHECK)'
+ assert data['env_keys'] == {'ODS_VERSION': '3.0.0'}
+
+
+def test_dry_run_shares_the_dashboard_cache_and_rate_limit_hold(test_client, monkeypatch, tmp_path):
+ monkeypatch.setattr(updates, 'INSTALL_DIR', str(tmp_path))
+ get = _github(monkeypatch, error=AssertionError('the cached release answers the dry run'))
+ monkeypatch.setattr(updates, '_version_cache', {'expires_at': time.monotonic() + 100,
+ 'payload': {'latest': '9.0.0', 'changelog_url': None}})
+ assert test_client.get('/api/update/dry-run', headers=test_client.auth_headers).json()['latest_version'] == '9.0.0'
+
+ monkeypatch.setattr(updates, '_version_cache', {'expires_at': 0.0, 'payload': None})
+ monkeypatch.setattr(updates, '_version_hold', {'until': time.monotonic() + 600, 'rate_limited': True})
+ data = test_client.get('/api/update/dry-run', headers=test_client.auth_headers).json()
+ assert get.await_count == 0
+ assert data['latest_version'] is None
+ assert data['version_check_error'] == "GitHub's request limit for this network is used up; try again in 10 min"
diff --git a/ods/ods-cli b/ods/ods-cli
index cd3f0c2df..0312eae8b 100755
--- a/ods/ods-cli
+++ b/ods/ods-cli
@@ -2275,10 +2275,11 @@ cmd_dry_run() {
"http://127.0.0.1:${dashboard_api_port}/api/update/dry-run" 2>/dev/null || true)
fi
- local latest_ver="" changelog_url="" update_status=""
+ local latest_ver="" changelog_url="" update_status="" check_error=""
if [[ -n "$api_json" ]] && echo "$api_json" | jq -e '.current_version' >/dev/null 2>&1; then
latest_ver=$(echo "$api_json" | jq -r '.latest_version // empty')
changelog_url=$(echo "$api_json" | jq -r '.changelog_url // empty')
+ check_error=$(echo "$api_json" | jq -r '.version_check_error // empty')
local api_available
api_available=$(echo "$api_json" | jq -r '.update_available // false')
[[ "$api_available" == "true" ]] && update_status="update available" || update_status="up to date"
@@ -2311,7 +2312,7 @@ cmd_dry_run() {
fi
[[ -n "$changelog_url" ]] && echo " Changelog : ${changelog_url}"
else
- echo " Available : (could not reach GitHub)"
+ echo " Available : (${check_error:-could not reach GitHub})"
fi
echo ""