Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
#!/usr/bin/env python3
"""Mirror published GitHub Release assets into the dl.avibe.bot R2 bucket.
GitHub stays the source of truth. For each mirrored repository the bucket holds
byte-identical copies under ``<root>releases/<tag>/<asset>`` and a release
index at ``<root>index/releases.json``. Avibe's root also holds version-less
``latest/`` copies of the newest stable desktop installers; see
``docs/plans/release-download-mirror.md`` for the contract.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Mapping, Sequence
from urllib.parse import quote
REPO_ROOT = Path(__file__).resolve().parents[1]
# Bucket root of each mirrored repository. Avibe's own releases keep the bucket
# root, where their mirror URLs were first published.
REPOSITORIES = {
"avibe-bot/avibe": "",
"avibe-bot/askill": "askill/",
"avibe-bot/avault": "avault/",
"tmux/tmux-builds": "tmux/",
"BurntSushi/ripgrep": "ripgrep/",
"cloudflare/cloudflared": "cloudflared/",
}
# Repositories mirrored only for the releases packaged managed-runtime
# manifests pin. The mirror is those releases' backup: a pinned archive GitHub
# no longer publishes with the manifest's bytes fails the run, and a release an
# earlier manifest pinned stays, since released Avibe versions still name it.
# ripgrep's older releases also carry no GitHub digest.
PINNED_BY_MANIFEST = {
"BurntSushi/ripgrep": REPO_ROOT / "vibe" / "ripgrep_runtime_manifest.json",
"cloudflare/cloudflared": REPO_ROOT / "vibe" / "cloudflared_runtime_manifest.json",
}
# Repositories mirrored without prereleases. tmux-builds rebuilds its
# ``preview`` prerelease in place, changing bytes under unchanged asset names,
# which the mirror must treat as published bytes changing.
FULL_RELEASES_ONLY = frozenset({"tmux/tmux-builds"})
RELEASE_PREFIX = "releases/"
INDEX_KEY = "index/releases.json"
IMMUTABLE_CACHE_CONTROL = "public, max-age=31536000, immutable"
# The index and the ``latest/`` installers change bytes under one URL.
MUTABLE_CACHE_CONTROL = "public, max-age=60"
# ``latest/Avibe_<target>.<ext>`` copies ``Avibe_<version>_<target>.<ext>`` from
# the highest stable release that ships installers. Only Avibe's root has them.
LATEST_REPOSITORY = "avibe-bot/avibe"
LATEST_PREFIX = "latest/"
INSTALLER_SUFFIXES = (".dmg", ".exe")
DEFAULT_KEEP_PRERELEASES = 20
INDEX_SCHEMA_VERSION = 1
_NAME_RE = re.compile(r"[A-Za-z0-9._+-]+")
_SHA256_RE = re.compile(r"[0-9a-f]{64}")
_COMMIT_RE = re.compile(r"[0-9a-f]{40}")
# The desktop updater's stable channel: canonical ``vX.Y.Z`` full releases.
_STABLE_TAG_RE = re.compile(r"v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)")
# The release-notes marker asking clients not to announce a release, read as
# ``core/update_checker.py`` reads it: the first well-formed marker decides.
_UPDATE_NOTIFICATION_RE = re.compile(
r"<!--\s*(?:avibe|vibe-remote):update-notification\s*=\s*(?P<policy>none|default)\s*-->",
re.IGNORECASE,
)
class MirrorError(RuntimeError):
"""Raised when the mirror cannot be proven to match GitHub."""
def release_prefix(repository: str) -> str:
return f"{REPOSITORIES[repository]}{RELEASE_PREFIX}"
def index_key(repository: str) -> str:
return f"{REPOSITORIES[repository]}{INDEX_KEY}"
@dataclass(frozen=True)
class Asset:
name: str
size: int
sha256: str
url: str
content_type: str
@dataclass(frozen=True)
class Release:
repository: str
tag: str
prerelease: bool
published_at: str
commit: str
assets: tuple[Asset, ...]
# The release notes ask clients not to announce it (docs/release.md).
silent: bool = False
def key(self, asset: Asset) -> str:
return f"{release_prefix(self.repository)}{self.tag}/{asset.name}"
@dataclass(frozen=True)
class Plan:
uploads: tuple[tuple[str, Asset], ...]
# ``(latest key, source key, asset)``, copied in the bucket after uploads.
aliases: tuple[tuple[str, str, Asset], ...]
deletions: tuple[str, ...]
# Keys whose published bytes changed upstream. Edge caches may still hold
# the old copy for up to a year, so these fail the run after repair.
changed: tuple[str, ...]
index: bytes
def parse_releases(
repository: str, payload: Sequence[Mapping[str, Any]], commits: Mapping[str, str]
) -> list[Release]:
"""Return every published release, validated against the mirror contract."""
releases = []
for item in payload:
if item["draft"]:
continue
tag = item["tag_name"]
if not _NAME_RE.fullmatch(tag):
raise MirrorError(f"release tag is not mirrorable: {tag!r}")
commit = commits.get(tag)
if commit is None or not _COMMIT_RE.fullmatch(commit):
raise MirrorError(f"published release has no tag commit: {tag}")
assets = []
for raw in item["assets"]:
name = raw["name"]
if not _NAME_RE.fullmatch(name):
raise MirrorError(f"asset name is not mirrorable: {tag}/{name!r}")
# Clients derive a mirror URL by replacing this exact prefix.
expected = f"https://github.com/{repository}/releases/download/{quote(tag)}/{quote(name)}"
if raw["browser_download_url"] != expected:
raise MirrorError(f"unexpected download URL for {tag}/{name}: {raw['browser_download_url']}")
algorithm, _, digest = str(raw.get("digest") or "").partition(":")
if algorithm != "sha256" or not _SHA256_RE.fullmatch(digest):
raise MirrorError(f"asset has no sha256 digest: {tag}/{name}")
assets.append(
Asset(
name=name,
size=int(raw["size"]),
sha256=digest,
url=expected,
content_type=raw.get("content_type") or "application/octet-stream",
)
)
releases.append(
Release(
repository=repository,
tag=tag,
prerelease=bool(item["prerelease"]),
published_at=item["published_at"],
commit=commit,
assets=tuple(sorted(assets, key=lambda asset: asset.name)),
silent=silent_release_notes(item.get("body")),
)
)
return releases
def silent_release_notes(body: object) -> bool:
match = _UPDATE_NOTIFICATION_RE.search(body) if isinstance(body, str) else None
return match is not None and match.group("policy").lower() == "none"
def pinned_releases(
repository: str, payload: Sequence[Mapping[str, Any]], manifest_path: Path
) -> list[Mapping[str, Any]]:
"""Return the release the manifest pins, once GitHub proves every pinned archive."""
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
tag = manifest["release_tag"]
pinned = [item for item in payload if item["tag_name"] == tag and not item["draft"]]
if not pinned:
raise MirrorError(f"pinned release is not published: {repository} {tag}")
published = {raw["name"]: raw for raw in pinned[0]["assets"]}
for archive in manifest["archives"].values():
raw = published.get(archive["name"]) or {}
if raw.get("size") != archive["size"] or raw.get("digest") != f"sha256:{archive['sha256']}":
raise MirrorError(f"pinned archive differs from its manifest: {repository} {tag}/{archive['name']}")
return pinned
def select(releases: Sequence[Release], keep_prereleases: int) -> list[Release]:
"""Keep every full release and the newest prereleases, newest first."""
ordered = sorted(releases, key=lambda release: (release.published_at, release.tag), reverse=True)
prereleases = [release for release in ordered if release.prerelease][:keep_prereleases]
return [release for release in ordered if not release.prerelease or release in prereleases]
def latest_installers(releases: Sequence[Release]) -> dict[str, tuple[str, Asset]]:
"""Map each ``latest/`` key to its source in the highest stable release that ships installers.
SemVer orders the releases, as in the desktop updater; publication order never counts.
"""
stable = [
(tuple(int(part) for part in match.groups()), release)
for release in releases
if not release.prerelease and (match := _STABLE_TAG_RE.fullmatch(release.tag))
]
for _, release in sorted(stable, key=lambda item: item[0], reverse=True):
prefix = f"Avibe_{release.tag.removeprefix('v')}_"
installers = {
f"{LATEST_PREFIX}Avibe_{asset.name.removeprefix(prefix)}": (release.key(asset), asset)
for asset in release.assets
if asset.name.startswith(prefix) and asset.name.endswith(INSTALLER_SUFFIXES)
}
if installers:
return installers
return {}
def render_index(repository: str, releases: Sequence[Release]) -> bytes:
document = {
"schema_version": INDEX_SCHEMA_VERSION,
"repository": repository,
"releases": [
{
"tag": release.tag,
"prerelease": release.prerelease,
"published_at": release.published_at,
"commit": release.commit,
"assets": [
{"name": asset.name, "size": asset.size, "sha256": asset.sha256}
for asset in release.assets
],
# Written only for a silent release, so other indexes keep their bytes.
**({"update_notification": "none"} if release.silent else {}),
}
for release in releases
],
}
return (json.dumps(document, separators=(",", ":")) + "\n").encode()
def _indexed_digests(repository: str, index: bytes | None) -> dict[str, str]:
if index is None:
return {}
try:
document = json.loads(index)
return {
f"{release_prefix(repository)}{release['tag']}/{asset['name']}": asset["sha256"]
for release in document["releases"]
for asset in release["assets"]
}
except (ValueError, KeyError, TypeError) as exc:
# The index is this job's derived state; re-verify every object instead.
print(f"warning: ignoring unreadable mirror index: {exc}", file=sys.stderr)
return {}
def with_earlier_pins(repository: str, releases: Sequence[Release], index: bytes | None) -> list[Release]:
"""Add the releases the previous index holds that the current pin no longer selects."""
selected = {release.tag for release in releases}
try:
document = json.loads(index) if index is not None else {"releases": []}
earlier = [
Release(
repository=repository,
tag=item["tag"],
prerelease=bool(item["prerelease"]),
published_at=item["published_at"],
commit=item["commit"],
assets=tuple(
Asset(
name=asset["name"],
size=int(asset["size"]),
sha256=asset["sha256"],
url=f"https://github.com/{repository}/releases/download/{quote(item['tag'])}/{quote(asset['name'])}",
content_type="application/octet-stream",
)
for asset in item["assets"]
),
silent=item.get("update_notification") == "none",
)
for item in document["releases"]
if item["tag"] not in selected
]
except (ValueError, KeyError, TypeError) as exc:
# The objects stay either way (``plan`` never deletes a pinned root); only the index omits them.
print(f"warning: ignoring unreadable mirror index: {exc}", file=sys.stderr)
earlier = []
return sorted([*releases, *earlier], key=lambda release: (release.published_at, release.tag), reverse=True)
def plan(
repository: str,
releases: Sequence[Release],
objects: Mapping[str, int],
previous_index: bytes | None,
latest: Mapping[str, str | None] | None = None,
) -> Plan:
"""Compare the selected releases with the repository's ``releases/`` objects.
A manifest-pinned repository's root only grows: its objects are the backup of
every release a published manifest pinned. ``latest`` maps each existing
``latest/`` object to the sha256 it was copied with.
"""
if not releases:
raise MirrorError(f"GitHub returned no published releases for {repository}; refusing to reconcile")
prefix = release_prefix(repository)
indexed = _indexed_digests(repository, previous_index)
uploads, changed, expected = [], [], set()
for release in releases:
for asset in release.assets:
key = release.key(asset)
expected.add(key)
prior = indexed.get(key)
if prior is not None and prior != asset.sha256:
changed.append(key)
uploads.append((key, asset))
elif prior is None or objects.get(key) != asset.size:
# Only an object this job verified and indexed is trusted.
uploads.append((key, asset))
deletions = (
[]
if repository in PINNED_BY_MANIFEST
else sorted(key for key in objects if key.startswith(prefix) and key not in expected)
)
aliases = []
if repository == LATEST_REPOSITORY:
current = latest or {}
installers = latest_installers(releases)
aliases = [
(key, source, asset)
for key, (source, asset) in sorted(installers.items())
if current.get(key) != asset.sha256
]
deletions += sorted(key for key in current if key not in installers)
return Plan(
uploads=tuple(uploads),
aliases=tuple(aliases),
deletions=tuple(deletions),
changed=tuple(changed),
index=render_index(repository, releases),
)
def download(asset: Asset, destination: Path, *, attempts: int = 3) -> None:
"""Download ``asset`` and prove its size and sha256 match GitHub's digest."""
request = urllib.request.Request(asset.url, headers={"User-Agent": "avibe-release-mirror/1"})
for attempt in range(1, attempts + 1):
digest, size = hashlib.sha256(), 0
try:
with urllib.request.urlopen(request, timeout=60) as response, destination.open("wb") as output:
while chunk := response.read(1024 * 1024):
size += len(chunk)
if size > asset.size:
break
digest.update(chunk)
output.write(chunk)
except urllib.error.HTTPError as exc:
if not (exc.code == 429 or 500 <= exc.code < 600) or attempt == attempts:
raise MirrorError(f"download failed ({exc.code}): {asset.url}") from exc
except (OSError, urllib.error.URLError) as exc:
if attempt == attempts:
raise MirrorError(f"download failed: {asset.url}: {exc}") from exc
else:
if size != asset.size or digest.hexdigest() != asset.sha256:
raise MirrorError(f"downloaded bytes do not match GitHub's digest: {asset.url}")
return
time.sleep(float(attempt))
def _run(command: Sequence[str]) -> str:
completed = subprocess.run(command, capture_output=True, text=True, check=False)
if completed.returncode != 0:
detail = completed.stderr.strip() or completed.stdout.strip() or "unknown error"
raise MirrorError(f"{command[0]} {command[1]} failed: {detail}")
return completed.stdout
def github_releases(repository: str) -> list[dict[str, Any]]:
pages = json.loads(_run(["gh", "api", "--paginate", "--slurp", f"repos/{repository}/releases?per_page=100"]))
return [release for page in pages for release in page]
def tag_commits(repository: str) -> dict[str, str]:
commits: dict[str, str] = {}
listing = _run(["git", "ls-remote", "--tags", f"https://github.com/{repository}.git"])
for line in listing.splitlines():
sha, ref = line.split("\t")
tag = ref.removeprefix("refs/tags/")
if tag.endswith("^{}"):
commits[tag[:-3]] = sha
else:
commits.setdefault(tag, sha)
return commits
class Bucket:
def __init__(self, name: str, endpoint_url: str) -> None:
self.name = name
self.endpoint_url = endpoint_url
def _s3api(self, *arguments: str) -> str:
return _run(["aws", "s3api", *arguments, "--bucket", self.name, "--endpoint-url", self.endpoint_url])
def objects(self) -> dict[str, int]:
output = self._s3api("list-objects-v2", "--output", "json")
listing = json.loads(output) if output.strip() else {}
return {item["Key"]: int(item["Size"]) for item in listing.get("Contents") or []}
def read(self, key: str) -> bytes:
with tempfile.TemporaryDirectory(prefix="avibe-release-mirror-") as temporary:
path = Path(temporary) / "object"
self._s3api("get-object", "--key", key, str(path))
return path.read_bytes()
def put(self, key: str, path: Path, *, content_type: str, cache_control: str, sha256: str) -> None:
self._s3api(
"put-object", "--key", key, "--body", str(path),
"--content-type", content_type, "--cache-control", cache_control,
"--metadata", f"sha256={sha256}",
)
def copy(
self, source: str, key: str, *, content_type: str, cache_control: str, content_disposition: str, sha256: str
) -> None:
# A server-side copy; REPLACE gives the copy its own headers.
self._s3api(
"copy-object", "--copy-source", f"{self.name}/{source}", "--key", key,
"--metadata-directive", "REPLACE", "--content-type", content_type,
"--cache-control", cache_control, "--content-disposition", content_disposition,
"--metadata", f"sha256={sha256}",
)
def sha256(self, key: str) -> str | None:
"""Return the sha256 this job recorded on ``key``."""
head = json.loads(self._s3api("head-object", "--key", key, "--output", "json"))
return (head.get("Metadata") or {}).get("sha256")
def delete(self, key: str) -> None:
# DeleteObjects requires a request checksum that R2 may not implement.
self._s3api("delete-object", "--key", key)
def reconcile(repository: str, bucket: Bucket, *, keep_prereleases: int, dry_run: bool) -> int:
payload = github_releases(repository)
if repository in PINNED_BY_MANIFEST:
payload = pinned_releases(repository, payload, PINNED_BY_MANIFEST[repository])
releases = select(parse_releases(repository, payload, tag_commits(repository)), keep_prereleases)
objects = bucket.objects()
key = index_key(repository)
previous_index = bucket.read(key) if key in objects else None
if repository in PINNED_BY_MANIFEST:
releases = with_earlier_pins(repository, releases, previous_index)
latest = (
{name: bucket.sha256(name) for name in objects if name.startswith(LATEST_PREFIX)}
if repository == LATEST_REPOSITORY
else None
)
result = plan(repository, releases, objects, previous_index, latest)
size = sum(asset.size for _, asset in result.uploads)
print(
f"{repository}: {len(releases)} releases mirrored; "
f"{len(result.uploads)} uploads ({size / 1e9:.2f} GB), {len(result.aliases)} latest copies, "
f"{len(result.deletions)} deletions, index {'unchanged' if result.index == previous_index else 'changed'}"
)
for changed in result.changed:
print(f"error: published bytes changed upstream: {changed}", file=sys.stderr)
if dry_run:
for upload, _ in result.uploads:
print(f"would upload {upload}")
for alias, source, _ in result.aliases:
print(f"would copy {source} to {alias}")
for deletion in result.deletions:
print(f"would delete {deletion}")
else:
with tempfile.TemporaryDirectory(prefix="avibe-release-mirror-") as temporary:
path = Path(temporary) / "asset"
for upload, asset in result.uploads:
download(asset, path)
bucket.put(
upload, path, content_type=asset.content_type,
cache_control=IMMUTABLE_CACHE_CONTROL, sha256=asset.sha256,
)
print(f"uploaded {upload}")
# Sources exist now: each was verified earlier or uploaded above.
for alias, source, asset in result.aliases:
bucket.copy(
source, alias, content_type=asset.content_type, cache_control=MUTABLE_CACHE_CONTROL,
content_disposition=f'attachment; filename="{asset.name}"', sha256=asset.sha256,
)
print(f"copied {source} to {alias}")
if result.index != previous_index:
path.write_bytes(result.index)
bucket.put(
key, path, content_type="application/json",
cache_control=MUTABLE_CACHE_CONTROL, sha256=hashlib.sha256(result.index).hexdigest(),
)
print(f"wrote {key}")
# Delete only after the index stops listing these objects.
for deletion in result.deletions:
bucket.delete(deletion)
print(f"deleted {deletion}")
if result.changed:
print("error: purge the changed URLs from the dl.avibe.bot cache", file=sys.stderr)
return 1
return 0
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--repository", action="append", choices=sorted(REPOSITORIES),
help="repository to reconcile (repeatable; default: every mirrored repository)",
)
parser.add_argument("--bucket", required=True)
parser.add_argument("--endpoint-url", required=True)
parser.add_argument("--keep-prereleases", type=int, default=DEFAULT_KEEP_PRERELEASES)
parser.add_argument("--dry-run", action="store_true")
arguments = parser.parse_args(argv)
bucket = Bucket(arguments.bucket, arguments.endpoint_url)
status = 0
# Repositories own disjoint roots, so one failing never blocks the others.
for repository in arguments.repository or REPOSITORIES:
try:
keep_prereleases = 0 if repository in FULL_RELEASES_ONLY else arguments.keep_prereleases
status |= reconcile(repository, bucket, keep_prereleases=keep_prereleases, dry_run=arguments.dry_run)
except MirrorError as exc:
print(f"error: {repository}: {exc}", file=sys.stderr)
status = 1
return status
if __name__ == "__main__":
sys.exit(main())