Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

feat(runtime): download release assets from the dl.avibe.bot mirror with GitHub fallback (#2286)

Call White2eab40b4c60a
diff --git a/core/dependency_network.py b/core/dependency_network.py
index cfc9f8706..cb1dfd2b4 100644
--- a/core/dependency_network.py
+++ b/core/dependency_network.py
@@ -5,7 +5,7 @@ from __future__ import annotations
import errno
import http.client
import logging
-import shutil
+import re
import socket
import ssl
import time
@@ -30,6 +30,15 @@ class RetryPolicy:
DOWNLOAD_RETRY_POLICY = RetryPolicy(max_attempts=3, initial_delay=1.0, max_delay=4.0)
PROBE_RETRY_POLICY = RetryPolicy(max_attempts=2, initial_delay=0.5, max_delay=0.5)
+# Byte-identical copies of published Avibe release assets
+# (docs/plans/release-download-mirror.md). Trusted for availability only:
+# callers verify what they download, whichever source served it.
+RELEASE_DOWNLOAD_ROOT = "https://github.com/avibe-bot/avibe/releases/download/"
+RELEASE_MIRROR_ROOT = "https://dl.avibe.bot/releases/"
+# Index into download_sources() of the source that last completed a download.
+_preferred_source = 0
+_DOWNLOAD_CHUNK_BYTES = 1024 * 1024
+
class DependencyNetworkError(RuntimeError):
"""A dependency request that exhausted its retry policy."""
@@ -145,26 +154,117 @@ def fetch_bytes(
return _run_with_retry(request, operation, policy=policy)
+def download_sources(url: str) -> list[str]:
+ """Download order for a URL: the mirror, then GitHub, for Avibe release assets.
+
+ The mirror holds ``<tag>/<asset>`` only, so a URL with a query stays on its
+ own host. The mirror URL substitutes the prefix alone, which keeps
+ percent-encoding identical.
+ """
+ if url.startswith(RELEASE_DOWNLOAD_ROOT) and "?" not in url:
+ return [RELEASE_MIRROR_ROOT + url.removeprefix(RELEASE_DOWNLOAD_ROOT), url]
+ return [url]
+
+
def fetch_to_path(
- request: str | urllib.request.Request,
+ url: str,
target: Path,
*,
timeout: float,
+ size: int | None = None,
policy: RetryPolicy = DOWNLOAD_RETRY_POLICY,
opener: Callable[..., Any] | None = None,
) -> None:
+ """Download ``url`` to ``target`` from ``download_sources``, starting with the last one that worked.
+
+ Each source gets up to ``policy.max_attempts`` attempts in rotation; one that
+ fails non-retryably leaves the rotation. A failed or stalled attempt (the
+ socket ``timeout``) keeps the bytes already written, and the next attempt
+ resumes them with ``Range``. The file is complete at the pinned ``size``, or
+ else at the length the source declares; a source that sends more leaves the
+ rotation and its bytes are discarded.
+ """
+ global _preferred_source
+ if policy.max_attempts < 1:
+ raise ValueError("retry policy must allow at least one attempt")
resolved_opener = opener or urllib.request.urlopen
-
- def operation() -> None:
- _unlink_quietly(target)
- with resolved_opener(request, timeout=timeout) as response, target.open("wb") as destination:
- shutil.copyfileobj(response, destination)
-
+ sources = download_sources(url)
+ rotation = [(_preferred_source + shift) % len(sources) for shift in range(len(sources))]
+ attempts = 0
+ _unlink_quietly(target)
try:
- _run_with_retry(request, operation, policy=policy)
+ for round_number in range(1, policy.max_attempts + 1):
+ # The next round waits until every remaining source's retry window passes.
+ delay = 0.0
+ for index in list(rotation):
+ attempts += 1
+ try:
+ _continue_download(resolved_opener, sources[index], target, timeout, size)
+ except Exception as exc: # noqa: BLE001
+ details = dependency_error_details(exc, sources[index], attempts=attempts)
+ close = getattr(exc, "close", None)
+ if callable(close):
+ close()
+ if not details.get("retryable") or round_number >= policy.max_attempts:
+ rotation.remove(index)
+ else:
+ delay = max(delay, _retry_delay(details, round_number, policy))
+ if not rotation:
+ raise DependencyNetworkError(details) from exc
+ logger.warning(
+ "Dependency download attempt %d failed for %s (%s)",
+ attempts,
+ redact_url(sources[index]),
+ details.get("kind"),
+ )
+ continue
+ if len(sources) > 1:
+ _preferred_source = index
+ return
+ time.sleep(delay)
except BaseException:
_unlink_quietly(target)
raise
+ raise AssertionError("download rotation exhausted without returning or raising")
+
+
+def _continue_download(
+ opener: Callable[..., Any],
+ url: str,
+ target: Path,
+ timeout: float,
+ size: int | None,
+) -> None:
+ offset = target.stat().st_size if target.exists() else 0
+ # The mirror's CDN rejects urllib's default agent (Cloudflare error 1010).
+ request = urllib.request.Request(url, headers={"User-Agent": "avibe-dependency-download"})
+ if offset:
+ request.add_header("Range", f"bytes={offset}-")
+ with opener(request, timeout=timeout) as response:
+ response_headers = getattr(response, "headers", None) or {}
+ if offset and getattr(response, "status", None) == 206:
+ match = re.fullmatch(r"bytes (\d+)-\d+/(\d+|\*)", str(response_headers.get("Content-Range") or ""))
+ if not match or int(match[1]) != offset:
+ raise ConnectionError("download resumed at the wrong offset")
+ declared = match[2]
+ else:
+ # A complete response, including one from a source that ignored Range.
+ offset = 0
+ declared = str(response_headers.get("Content-Length") or "")
+ # The file's full length, which a partial response's own length is not.
+ end = size if size is not None else int(declared) if declared.isdigit() else None
+ with target.open("ab" if offset else "wb") as destination:
+ while chunk := response.read(_DOWNLOAD_CHUNK_BYTES):
+ destination.write(chunk)
+ if end is not None and destination.tell() > end:
+ break
+ written = destination.tell()
+ if end is not None and written > end:
+ target.unlink()
+ raise ValueError("download exceeds its expected size")
+ # urllib reports a connection closed before Content-Length as a clean end.
+ if end is not None and written < end:
+ raise http.client.IncompleteRead(b"", end - written)
def probe_url(
@@ -198,6 +298,11 @@ def probe_url(
"download_error": error,
}
+ # A download succeeds through any of its sources, the last being ``url`` itself.
+ for source in download_sources(url)[:-1]:
+ result = probe_url(source, timeout=timeout, policy=policy, opener=opener, user_agent=user_agent)
+ if result["ok"]:
+ return result
request = urllib.request.Request(url, headers={"User-Agent": user_agent}, method="HEAD")
resolved_opener = opener or urllib.request.urlopen
@@ -260,10 +365,7 @@ def _run_with_retry(
close()
if not details.get("retryable") or attempt >= policy.max_attempts:
raise DependencyNetworkError(details) from exc
- delay = details.get("retry_after_seconds")
- if not isinstance(delay, (int, float)):
- delay = min(policy.initial_delay * (2 ** (attempt - 1)), policy.max_delay)
- delay = min(max(0.0, float(delay)), policy.max_delay)
+ delay = _retry_delay(details, attempt, policy)
logger.warning(
"Dependency request attempt %d/%d failed for %s (%s); retrying in %.1fs",
attempt,
@@ -276,6 +378,13 @@ def _run_with_retry(
raise AssertionError("retry loop exhausted without returning or raising")
+def _retry_delay(details: dict[str, Any], attempt: int, policy: RetryPolicy) -> float:
+ delay = details.get("retry_after_seconds")
+ if not isinstance(delay, (int, float)):
+ delay = min(policy.initial_delay * (2 ** (attempt - 1)), policy.max_delay)
+ return min(max(0.0, float(delay)), policy.max_delay)
+
+
def _retry_after_seconds(exc: urllib.error.HTTPError) -> float | None:
headers = getattr(exc, "headers", None)
value = headers.get("Retry-After") if headers is not None else None
diff --git a/core/managed_runtime.py b/core/managed_runtime.py
index d2ddabdd7..77a356089 100644
--- a/core/managed_runtime.py
+++ b/core/managed_runtime.py
@@ -1850,6 +1850,7 @@ class ManagedRuntimeManager:
archive.url,
temporary,
timeout=60,
+ size=archive.size,
opener=urllib.request.urlopen,
)
self._download_error = None
diff --git a/docs/plans/release-download-mirror.md b/docs/plans/release-download-mirror.md
index 7fe6723ea..2ea8a7e13 100644
--- a/docs/plans/release-download-mirror.md
+++ b/docs/plans/release-download-mirror.md
@@ -64,8 +64,8 @@ and fall back to GitHub.
- `sha256` is GitHub's asset digest, re-verified against the downloaded bytes
before upload. `commit` is the peeled tag target.
- Producer: the release mirror workflow. Consumers: the desktop updater.
- Planned: the CLI update checker and the runtime downloaders, replacing their
- `api.github.com` discovery calls.
+ Planned: the CLI update checker, replacing its `api.github.com` discovery
+ call. Runtime downloads use pinned release URLs and need no index.
- The index is unauthenticated metadata. A hostile index can at most withhold
releases or name copies that then fail the consumer's existing verification.
@@ -116,6 +116,29 @@ Cloudflare configuration lives outside the repository. The expected state:
and browser TTL respect the origin. The object headers above carry the
policy; the rule exists because default caching skips extensions such as
`.whl`, `.tgz`, `.json`, and `.sig`.
+- The zone's Browser Integrity Check answers the `Python-urllib` User-Agent
+ with 403 (error 1010). Clients name their own agent; curl, PowerShell, and
+ Rust clients already pass.
+
+## Client downloads
+
+The managed git runtime, the model hub engine, and Show Runtime manifest
+archives download through `core.dependency_network.fetch_to_path`:
+
+- An Avibe release URL without a query tries the mirror, then GitHub. Any other
+ URL (tmux builds, legacy Show Runtime archives, manifest overrides) keeps its
+ single source.
+- Sources rotate, each with the download retry budget. A non-retryable failure,
+ such as a mirror 404, drops that source.
+- A failed or stalled attempt (the caller's socket timeout) keeps its bytes, and
+ the next attempt asks for the rest with `Range`. A `200` restarts the file; a
+ `206` must start at the current length. A body shorter than its
+ `Content-Length` is incomplete, not finished.
+- The source that last completed a download goes first for the rest of the
+ process.
+- The pinned `sha256` check after download is unchanged.
+
+`probe_url` reports a release asset reachable when any of its sources answers.
## Rollout
@@ -123,10 +146,11 @@ Cloudflare configuration lives outside the repository. The expected state:
2. Cache Rule, then real-file tests from the three mainland carriers at the
evening peak.
3. Clients, one at a time: desktop updater (done), then runtime and Show
- Runtime downloads, then `install.sh` and `install.ps1`. Each tries the mirror
- first, then GitHub, with a connect timeout and a stall watchdog, resumes with
- `Range` when switching source, and remembers the last source that worked.
- The desktop shell's `avibe_runtime_host::download` is the Rust reference.
+ Runtime downloads (done), then `install.sh` and `install.ps1`. Each tries the
+ mirror first, then GitHub, with a connect timeout and a stall watchdog,
+ resumes with `Range` when switching source, and remembers the last source
+ that worked. The references are the desktop shell's
+ `avibe_runtime_host::download` (Rust) and Client downloads above (Python).
4. Install-script third-party dependencies (the uv installer and
python-build-standalone) through `UV_INSTALLER_GITHUB_BASE_URL` and
`UV_PYTHON_INSTALL_MIRROR`, if step 2 shows GitHub is still their
diff --git a/tests/test_dependency_network.py b/tests/test_dependency_network.py
index 58382631f..53fbaafb0 100644
--- a/tests/test_dependency_network.py
+++ b/tests/test_dependency_network.py
@@ -1,15 +1,56 @@
from __future__ import annotations
import http.client
+import io
import socket
import ssl
import urllib.error
+import urllib.request
import pytest
from core import dependency_network
+RELEASE_URL = "https://github.com/avibe-bot/avibe/releases/download/git-runtime-v2.55.0-1/git-runtime.tar.gz"
+MIRROR_URL = "https://dl.avibe.bot/releases/git-runtime-v2.55.0-1/git-runtime.tar.gz"
+
+
+class _Stream:
+ """A streamed HTTP response that can end by raising ``cut``."""
+
+ def __init__(self, body: bytes, *, status: int = 200, headers=None, cut: BaseException | None = None) -> None:
+ self._body = io.BytesIO(body)
+ self.status = status
+ self.headers = headers or {}
+ self._cut = cut
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_exc) -> None:
+ return None
+
+ def read(self, size: int = -1) -> bytes:
+ chunk = self._body.read(size)
+ if not chunk and self._cut is not None:
+ raise self._cut
+ return chunk
+
+
+def _scripted_opener(steps: list):
+ calls: list[tuple[str, str | None]] = []
+
+ def opener(request, timeout):
+ calls.append((request.full_url, request.get_header("Range")))
+ step = steps.pop(0)
+ if isinstance(step, BaseException):
+ raise step
+ return step
+
+ return opener, calls
+
+
class _Response:
status = 200
@@ -208,3 +249,199 @@ def test_dependency_error_message_reports_exhausted_attempts() -> None:
)
assert message == "Runtime download failed after 3 attempts: Connection timed out (https://example.test/archive.tgz)"
+
+
+ ("url", "sources"),
+ [
+ (
+ "https://github.com/avibe-bot/avibe/releases/download/v1.0.0/avibe_1.0.0%2Bbuild.tar.gz",
+ [
+ "https://dl.avibe.bot/releases/v1.0.0/avibe_1.0.0%2Bbuild.tar.gz",
+ "https://github.com/avibe-bot/avibe/releases/download/v1.0.0/avibe_1.0.0%2Bbuild.tar.gz",
+ ],
+ ),
+ (f"{RELEASE_URL}?token=secret", [f"{RELEASE_URL}?token=secret"]),
+ (
+ "https://github.com/tmux/tmux-builds/releases/download/v3.5/tmux.tar.gz",
+ ["https://github.com/tmux/tmux-builds/releases/download/v3.5/tmux.tar.gz"],
+ ),
+ ],
+)
+def test_only_avibe_release_assets_have_a_mirror_source(url, sources) -> None:
+ assert dependency_network.download_sources(url) == sources
+
+
+def test_mirror_miss_falls_back_to_github_and_later_downloads_start_there(monkeypatch, tmp_path) -> None:
+ monkeypatch.setattr(dependency_network, "_preferred_source", 0)
+ monkeypatch.setattr(dependency_network.time, "sleep", lambda _delay: pytest.fail("fallback must not wait"))
+ target = tmp_path / "asset.tmp"
+ opener, calls = _scripted_opener(
+ [
+ urllib.error.HTTPError(MIRROR_URL, 404, "Not Found", hdrs={}, fp=None),
+ _Stream(b"asset"),
+ _Stream(b"again"),
+ ]
+ )
+
+ dependency_network.fetch_to_path(RELEASE_URL, target, timeout=5, opener=opener)
+ assert target.read_bytes() == b"asset"
+ dependency_network.fetch_to_path(RELEASE_URL, target, timeout=5, opener=opener)
+
+ assert target.read_bytes() == b"again"
+ assert calls == [(MIRROR_URL, None), (RELEASE_URL, None), (RELEASE_URL, None)]
+
+
+def test_downloads_name_an_agent_because_the_mirror_rejects_the_urllib_default(tmp_path) -> None:
+ agents: list[str | None] = []
+
+ def opener(request, timeout):
+ agents.append(request.get_header("User-agent"))
+ return _Stream(b"asset")
+
+ dependency_network.fetch_to_path(RELEASE_URL, tmp_path / "asset.tmp", timeout=5, opener=opener)
+
+ assert agents == ["avibe-dependency-download"]
+
+
+def test_exhausted_sources_report_the_last_failure_and_remove_the_partial_file(monkeypatch, tmp_path) -> None:
+ monkeypatch.setattr(dependency_network, "_preferred_source", 0)
+ sleeps: list[float] = []
+ monkeypatch.setattr(dependency_network.time, "sleep", sleeps.append)
+ target = tmp_path / "asset.tmp"
+ opener, calls = _scripted_opener(
+ [
+ urllib.error.HTTPError(MIRROR_URL, 404, "Not Found", hdrs={}, fp=None),
+ _Stream(b"ab", cut=TimeoutError("stalled")),
+ TimeoutError("stalled"),
+ TimeoutError("stalled"),
+ ]
+ )
+
+ with pytest.raises(dependency_network.DependencyNetworkError) as raised:
+ dependency_network.fetch_to_path(RELEASE_URL, target, timeout=5, opener=opener)
+
+ assert calls == [(MIRROR_URL, None), (RELEASE_URL, None), (RELEASE_URL, "bytes=2-"), (RELEASE_URL, "bytes=2-")]
+ assert sleeps == [1.0, 2.0]
+ assert raised.value.details["url"] == RELEASE_URL
+ assert raised.value.details["kind"] == "timeout"
+ assert raised.value.details["attempts"] == 4
+ assert not target.exists()
+
+
+def test_next_round_waits_out_every_remaining_source_retry_window(monkeypatch, tmp_path) -> None:
+ monkeypatch.setattr(dependency_network, "_preferred_source", 0)
+ sleeps: list[float] = []
+ monkeypatch.setattr(dependency_network.time, "sleep", sleeps.append)
+ opener, calls = _scripted_opener(
+ [
+ urllib.error.HTTPError(MIRROR_URL, 503, "Unavailable", hdrs={"Retry-After": "3"}, fp=None),
+ TimeoutError("stalled"),
+ _Stream(b"asset"),
+ ]
+ )
+
+ dependency_network.fetch_to_path(RELEASE_URL, tmp_path / "asset.tmp", timeout=5, opener=opener)
+
+ assert sleeps == [3.0]
+ assert [url for url, _range in calls] == [MIRROR_URL, RELEASE_URL, MIRROR_URL]
+
+
+class _Endless:
+ """A source that never stops sending, and fails the test if read without bound."""
+
+ status = 200
+ headers: dict[str, str] = {}
+
+ def __init__(self) -> None:
+ self.reads = 0
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_exc) -> None:
+ return None
+
+ def read(self, size: int = -1) -> bytes:
+ self.reads += 1
+ if self.reads > 3:
+ pytest.fail("download kept reading past its pinned size")
+ return b"x" * size
+
+
+def test_a_source_sending_more_than_the_pinned_size_is_cut_off_and_discarded(monkeypatch, tmp_path) -> None:
+ monkeypatch.setattr(dependency_network, "_preferred_source", 0)
+ target = tmp_path / "asset.tmp"
+ opener, calls = _scripted_opener([_Endless(), _Stream(b"abcdefghij")])
+
+ dependency_network.fetch_to_path(RELEASE_URL, target, timeout=5, size=10, opener=opener)
+
+ assert target.read_bytes() == b"abcdefghij"
+ assert calls == [(MIRROR_URL, None), (RELEASE_URL, None)]
+
+
+def _cut_after_abcd():
+ return _Stream(b"abcd", cut=TimeoutError("stalled"))
+
+
+def _resumed(content_range: str = "bytes 4-9/10", body: bytes = b"efghij"):
+ return _Stream(body, status=206, headers={"Content-Range": content_range, "Content-Length": str(len(body))})
+
+
+ ("steps", "expected_calls"),
+ [
+ pytest.param(
+ [_cut_after_abcd(), _resumed()],
+ [(MIRROR_URL, None), (RELEASE_URL, "bytes=4-")],
+ id="stall-resumes-on-next-source",
+ ),
+ pytest.param(
+ [_Stream(b"abcd", headers={"Content-Length": "10"}), _resumed()],
+ [(MIRROR_URL, None), (RELEASE_URL, "bytes=4-")],
+ id="early-close-is-not-a-complete-file",
+ ),
+ pytest.param(
+ [_cut_after_abcd(), _Stream(b"abcdefghij")],
+ [(MIRROR_URL, None), (RELEASE_URL, "bytes=4-")],
+ id="range-ignored-restarts",
+ ),
+ pytest.param(
+ [_cut_after_abcd(), _resumed("bytes 0-9/10", b"abcdefghij"), _resumed()],
+ [(MIRROR_URL, None), (RELEASE_URL, "bytes=4-"), (MIRROR_URL, "bytes=4-")],
+ id="wrong-offset-is-not-spliced",
+ ),
+ pytest.param(
+ [_cut_after_abcd(), _resumed("bytes 4-5/10", b"ef"), _resumed("bytes 6-9/10", b"ghij")],
+ [(MIRROR_URL, None), (RELEASE_URL, "bytes=4-"), (MIRROR_URL, "bytes=6-")],
+ id="partial-range-continues-to-the-file-end",
+ ),
+ ],
+)
+def test_interrupted_download_resumes_without_corrupting_the_file(monkeypatch, tmp_path, steps, expected_calls) -> None:
+ monkeypatch.setattr(dependency_network, "_preferred_source", 0)
+ monkeypatch.setattr(dependency_network.time, "sleep", lambda _delay: None)
+ target = tmp_path / "asset.tmp"
+ opener, calls = _scripted_opener(list(steps))
+
+ dependency_network.fetch_to_path(RELEASE_URL, target, timeout=5, opener=opener)
+
+ assert target.read_bytes() == b"abcdefghij"
+ assert calls == expected_calls
+
+
+def test_probe_reports_a_release_asset_reachable_through_github_when_the_mirror_is_not(monkeypatch) -> None:
+ monkeypatch.setattr(dependency_network.time, "sleep", lambda _delay: None)
+ opener, calls = _scripted_opener(
+ [
+ urllib.error.URLError(socket.gaierror(-2, "not found")),
+ urllib.error.URLError(socket.gaierror(-2, "not found")),
+ _Response(url=RELEASE_URL),
+ ]
+ )
+
+ result = dependency_network.probe_url(RELEASE_URL, opener=opener)
+
+ assert result["ok"] is True
+ assert result["url"] == RELEASE_URL
+ assert [url for url, _range in calls] == [MIRROR_URL, MIRROR_URL, RELEASE_URL]
diff --git a/tests/test_git_runtime.py b/tests/test_git_runtime.py
index 96d7fb6cf..c7f098305 100644
--- a/tests/test_git_runtime.py
+++ b/tests/test_git_runtime.py
@@ -108,6 +108,31 @@ def test_archive_download_retries_transient_network_failure(tmp_path: Path, monk
assert attempts == 2
+def test_archive_download_stops_at_the_manifest_size(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path / "home"))
+ archive = _write_git_archive(tmp_path)
+ manifest = _write_manifest(tmp_path, archive)
+ payload = json.loads(manifest.read_text(encoding="utf-8"))
+ payload["archives"][managed_runtime.runtime_platform_tag()]["url"] = "https://example.test/git.tar.gz"
+ manifest.write_text(json.dumps(payload), encoding="utf-8")
+ reads = 0
+
+ class Endless(io.BytesIO):
+ def read(self, size: int | None = -1) -> bytes:
+ nonlocal reads
+ reads += 1
+ if reads > 3:
+ pytest.fail("archive download kept reading past the manifest size")
+ return b"x" * (size or 1)
+
+ monkeypatch.setattr(managed_runtime.urllib.request, "urlopen", lambda _request, timeout: Endless())
+
+ result = GitRuntimeManager(manifest_path=manifest).ensure()
+
+ assert result["ok"] is False
+ assert result["reason"] == "git_archive_download_failed"
+
+
def test_packaged_manifest_matches_published_four_platform_release(tmp_path: Path) -> None:
manager = GitRuntimeManager(runtime_dir=tmp_path / "runtime", offline=True)