Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

CI: rank apt mirrors by speed so a slow Azure mirror cannot time out apt steps (#15350)

Daniel Han1cf32bf777f7
diff --git a/.github/scripts/retry-with-apt-lock.sh b/.github/scripts/retry-with-apt-lock.sh
index e2908d120cc..9b9524bb93e 100755
--- a/.github/scripts/retry-with-apt-lock.sh
+++ b/.github/scripts/retry-with-apt-lock.sh
@@ -75,6 +75,94 @@ release_apt_locks() {
sleep 5
}
+# GitHub's Ubuntu images send apt through mirror+file:$APT_MIRRORS with azure.archive.ubuntu.com
+# first. apt fails over only on errors: a mirror trickling at a few KB/s (runner-images #14823)
+# never trips Acquire::*::Timeout, so every retry downloads from the same slow node.
+# This measures each mirror once and puts the fastest first; a failed attempt demotes the leader.
+APT_MIRRORS="${APT_MIRRORS_FILE:-/etc/apt/apt-mirrors.txt}"
+MIRROR_PROBE_SECONDS="${APT_MIRROR_PROBE_SECONDS:-5}"
+
+mirror_urls() {
+ grep -vE '^[[:space:]]*(#|$)' "$APT_MIRRORS" 2> /dev/null | awk '{print $1}'
+}
+
+# bytes/s fetched from a mirror in MIRROR_PROBE_SECONDS; 0 when unreachable. curl prints the rate
+# even when --max-time cuts the transfer, which is exactly the slow case being measured.
+probe_mirror() {
+ speed="$(curl -sS -o /dev/null --max-time "$MIRROR_PROBE_SECONDS" -w '%{speed_download}' \
+ "${1%/}/dists/${2}/main/binary-${3}/Packages.gz" 2> /dev/null)"
+ speed="${speed%%.*}"
+ case "$speed" in '' | *[!0-9]*) speed=0 ;; esac
+ printf '%s' "$speed"
+}
+
+# Rewrites APT_MIRRORS with the given URLs in order, priority:1 first.
+write_mirrors() {
+ body=""
+ n=0
+ for url in "$@"; do
+ n=$((n + 1))
+ body="${body}${url} priority:${n}
+"
+ done
+ if ! printf '%s' "$body" | sudo tee "$APT_MIRRORS" > /dev/null 2>&1; then
+ echo "::warning::could not rewrite ${APT_MIRRORS}; apt keeps its mirror order"
+ return 1
+ fi
+}
+
+# Once per job: later helper calls in the same job reuse the order (and any demotion).
+MIRROR_MARKER="${RUNNER_TEMP:-/tmp}/unsloth-apt-mirror-ranked"
+
+prefer_fast_apt_mirror() {
+ [ "${APT_PREFER_FAST_MIRROR:-1}" = "1" ] || return 0
+ [ -f "$APT_MIRRORS" ] && command -v curl > /dev/null 2>&1 || return 0
+ [ -e "$MIRROR_MARKER" ] && return 0
+ : > "$MIRROR_MARKER" 2> /dev/null || true
+ urls="$(mirror_urls)"
+ [ -n "$urls" ] || return 0
+ codename="$(. /etc/os-release 2> /dev/null && printf '%s' "${VERSION_CODENAME:-}")"
+ arch="$(dpkg --print-architecture 2> /dev/null || echo amd64)"
+ [ -n "$codename" ] || return 0
+ # The global archive carries every amd64 pocket, so it is a valid fallback beside any
+ # *.archive.ubuntu.com mirror. ports.ubuntu.com (arm64) has no such twin and is left alone.
+ if printf '%s\n' "$urls" | grep -q 'archive\.ubuntu\.com/ubuntu' \
+ && ! printf '%s\n' "$urls" | grep -qE '^https?://archive\.ubuntu\.com/ubuntu/?$'; then
+ urls="$(printf '%s\nhttp://archive.ubuntu.com/ubuntu/' "$urls")"
+ fi
+ ranked=""
+ for url in $urls; do
+ speed="$(probe_mirror "$url" "$codename" "$arch")"
+ echo "apt mirror ${url}: $((speed / 1024)) KB/s"
+ ranked="${ranked}${speed} ${url}
+"
+ done
+ # shellcheck disable=SC2046 # one URL per word
+ write_mirrors $(printf '%s' "$ranked" | sort -k1,1nr -s | awk '{print $2}') \
+ && echo "apt mirror order: $(mirror_urls | tr '\n' ' ')"
+}
+
+# After a failed attempt the leading mirror goes last, so the retry does not reuse it.
+demote_apt_mirror() {
+ [ "${APT_PREFER_FAST_MIRROR:-1}" = "1" ] && [ -f "$APT_MIRRORS" ] || return 0
+ urls="$(mirror_urls)"
+ [ "$(printf '%s\n' "$urls" | wc -l)" -gt 1 ] || return 0
+ first="$(printf '%s\n' "$urls" | head -n 1)"
+ # shellcheck disable=SC2046
+ write_mirrors $(printf '%s\n' "$urls" | tail -n +2) "$first" \
+ && echo "apt mirror ${first} moved last for the retry"
+}
+
+# A timed-out attempt can leave apt-get running under sudo (timeout signals sudo, not its
+# grandchildren), still holding the dpkg lock and still downloading from the slow mirror.
+stop_orphaned_apt() {
+ sudo pkill -TERM -x apt-get > /dev/null 2>&1 || true
+ sudo pkill -TERM -x apt > /dev/null 2>&1 || true
+ sleep 2
+ sudo dpkg --configure -a > /dev/null 2>&1 || true
+}
+
+prefer_fast_apt_mirror
configure_apt_fail_fast
for attempt in $(seq 1 "$ATTEMPTS"); do
@@ -95,6 +183,10 @@ for attempt in $(seq 1 "$ATTEMPTS"); do
echo "::warning::attempt ${attempt}/${ATTEMPTS} of '$*' ${reason}"
[ "$attempt" -ge "$ATTEMPTS" ] && break
+ if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then
+ stop_orphaned_apt
+ fi
+ demote_apt_mirror
release_apt_locks
done
diff --git a/tests/studio/test_apt_mirror_preference.py b/tests/studio/test_apt_mirror_preference.py
new file mode 100644
index 00000000000..ff3504dca5f
--- /dev/null
+++ b/tests/studio/test_apt_mirror_preference.py
@@ -0,0 +1,165 @@
+"""retry-with-apt-lock.sh ranks the apt mirrors by measured speed and demotes the leader after a failure.
+
+GitHub's Ubuntu images list azure.archive.ubuntu.com first in /etc/apt/apt-mirrors.txt, and apt only
+fails over on errors, so a mirror trickling at a few KB/s held every retry (runner-images #14823).
+These tests run the real helper against a temporary mirror file with `sudo`, `curl` and friends
+stubbed on PATH, so nothing touches the host's apt.
+"""
+
+import os
+import stat
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+HELPER = REPO_ROOT / ".github" / "scripts" / "retry-with-apt-lock.sh"
+
+pytestmark = pytest.mark.skipif(sys.platform == "win32", reason = "bash helper")
+
+AZURE = "http://azure.archive.ubuntu.com/ubuntu/"
+GLOBAL = "http://archive.ubuntu.com/ubuntu/"
+SECURITY = "http://security.ubuntu.com/ubuntu/"
+
+
+def _stub(bin_dir: Path, name: str, body: str) -> None:
+ path = bin_dir / name
+ path.write_text("#!/bin/bash\n" + body, encoding = "utf-8")
+ path.chmod(path.stat().st_mode | stat.S_IEXEC)
+
+
+def _run(tmp_path: Path, mirrors: str, speeds: dict, command: str, **env) -> tuple:
+ bin_dir = tmp_path / "bin"
+ bin_dir.mkdir(exist_ok = True)
+ mirrors_file = tmp_path / "apt-mirrors.txt"
+ mirrors_file.write_text(mirrors, encoding = "utf-8")
+ log = tmp_path / "orders.log"
+
+ # sudo runs the command as is; tee writes where the helper points it, apt.conf.d is redirected.
+ _stub(
+ bin_dir,
+ "sudo",
+ 'case "$1" in tee) [ "${2#/etc/apt/apt.conf.d}" != "$2" ] && exec cat > /dev/null ;; esac\nexec "$@"\n',
+ )
+ lines = "".join(f' *"{host}"*) echo {bps} ;;\n' for host, bps in speeds.items())
+ _stub(bin_dir, "curl", f'url="${{@: -1}}"\ncase "$url" in\n{lines} *) echo 0 ;;\nesac\n')
+ _stub(bin_dir, "pkill", "exit 1\n")
+ _stub(bin_dir, "dpkg", '[ "$1" = --print-architecture ] && echo amd64\nexit 0\n')
+ _stub(bin_dir, "fuser", "exit 1\n")
+ _stub(bin_dir, "sleep", "exit 0\n")
+ # Records the mirror order each attempt saw.
+ _stub(
+ bin_dir,
+ "attempt",
+ f'awk \'{{print $1}}\' "{mirrors_file}" | paste -sd" " >> "{log}"\n{command}\n',
+ )
+
+ run_env = {
+ **os.environ,
+ "PATH": f"{bin_dir}:{os.environ['PATH']}",
+ "APT_MIRRORS_FILE": str(mirrors_file),
+ "RUNNER_TEMP": str(tmp_path),
+ "RETRY_ATTEMPTS": "3",
+ "RETRY_ATTEMPT_TIMEOUT": "30",
+ **env,
+ }
+ proc = subprocess.run(
+ ["bash", str(HELPER), "attempt"],
+ env = run_env,
+ capture_output = True,
+ text = True,
+ timeout = 60,
+ )
+ orders = log.read_text(encoding = "utf-8").splitlines() if log.exists() else []
+ return proc, orders, mirrors_file.read_text(encoding = "utf-8")
+
+
+def _has_os_release() -> bool:
+ try:
+ return "VERSION_CODENAME=" in Path("/etc/os-release").read_text(encoding = "utf-8")
+ except OSError:
+ return False
+
+
+needs_codename = pytest.mark.skipif(not _has_os_release(), reason = "needs /etc/os-release")
+
+
+@needs_codename
+def test_the_fastest_mirror_goes_first(tmp_path):
+ mirrors = f"{AZURE}\tpriority:1\n{SECURITY}\tpriority:2\n"
+ proc, orders, written = _run(
+ tmp_path,
+ mirrors,
+ {"azure.archive": 30_000, "//archive.ubuntu": 9_000_000, "security": 4_000_000},
+ "exit 0",
+ )
+ assert proc.returncode == 0, proc.stderr
+ # The global archive is added beside an *.archive.ubuntu.com mirror and wins on speed.
+ assert orders == [f"{GLOBAL} {SECURITY} {AZURE}"], proc.stdout
+ assert f"{GLOBAL}\tpriority:1" in written
+ assert f"{AZURE}\tpriority:3" in written
+
+
+@needs_codename
+def test_a_fast_azure_mirror_keeps_its_place(tmp_path):
+ mirrors = f"{AZURE}\tpriority:1\n{GLOBAL}\tpriority:2\n"
+ proc, orders, _ = _run(
+ tmp_path, mirrors, {"azure.archive": 9_000_000, "//archive.ubuntu": 1_000_000}, "exit 0"
+ )
+ assert proc.returncode == 0, proc.stderr
+ assert orders == [f"{AZURE} {GLOBAL}"]
+
+
+@needs_codename
+def test_a_failed_attempt_moves_the_leader_last(tmp_path):
+ mirrors = f"{AZURE}\tpriority:1\n{GLOBAL}\tpriority:2\n"
+ # Fails the first time only.
+ command = f'[ -e "{tmp_path}/done" ] && exit 0; touch "{tmp_path}/done"; exit 100'
+ proc, orders, _ = _run(
+ tmp_path, mirrors, {"azure.archive": 9_000_000, "//archive.ubuntu": 1_000_000}, command
+ )
+ assert proc.returncode == 0, proc.stderr + proc.stdout
+ assert orders == [f"{AZURE} {GLOBAL}", f"{GLOBAL} {AZURE}"], proc.stdout
+
+
+@needs_codename
+def test_mirrors_are_ranked_once_per_job(tmp_path):
+ mirrors = f"{AZURE}\tpriority:1\n{GLOBAL}\tpriority:2\n"
+ _run(tmp_path, mirrors, {"azure.archive": 1_000, "//archive.ubuntu": 9_000_000}, "exit 0")
+ # A second helper call in the same job keeps the order it finds, even if speeds have flipped.
+ proc, orders, _ = _run(
+ tmp_path,
+ f"{GLOBAL}\tpriority:1\n{AZURE}\tpriority:2\n",
+ {"azure.archive": 9_000_000, "//archive.ubuntu": 1_000},
+ "exit 0",
+ )
+ assert proc.returncode == 0
+ assert orders[-1] == f"{GLOBAL} {AZURE}"
+
+
+def test_no_mirror_file_runs_the_command_unchanged(tmp_path):
+ proc, orders, _ = _run(
+ tmp_path, "", {}, "exit 0", APT_MIRRORS_FILE = str(tmp_path / "absent.txt")
+ )
+ assert proc.returncode == 0, proc.stderr
+ assert "apt mirror" not in proc.stdout
+
+
+@needs_codename
+def test_opt_out_leaves_the_mirror_file_alone(tmp_path):
+ mirrors = f"{AZURE}\tpriority:1\n"
+ proc, _, written = _run(
+ tmp_path, mirrors, {"//archive.ubuntu": 9_000_000}, "exit 0", APT_PREFER_FAST_MIRROR = "0"
+ )
+ assert proc.returncode == 0
+ assert written == mirrors
+
+
+@needs_codename
+def test_ports_mirrors_get_no_global_archive(tmp_path):
+ ports = "http://ports.ubuntu.com/ubuntu-ports/"
+ proc, orders, _ = _run(tmp_path, f"{ports}\tpriority:1\n", {"ports": 5_000_000}, "exit 0")
+ assert proc.returncode == 0
+ assert orders == [ports]
diff --git a/tests/studio/test_apt_steps_are_bounded.py b/tests/studio/test_apt_steps_are_bounded.py
index a445e0fdaad..16dab164d65 100644
--- a/tests/studio/test_apt_steps_are_bounded.py
+++ b/tests/studio/test_apt_steps_are_bounded.py
@@ -24,6 +24,11 @@ _APT = re.compile(r"\bapt-get\s+(?:update|install)\b|playwright\s+install\b[^\n]
# `release_dpkg_lock()` waits up to 24 * 5s, then kills the holder and sleeps 5.
LOCK_WAIT_SECONDS = 125
+# `prefer_fast_apt_mirror()` probes at most four mirrors for 5s each, once per job, and a
+# timed-out attempt costs a 2s pause to stop the orphaned apt-get.
+MIRROR_PROBE_SECONDS = 4 * 5
+ORPHAN_STOP_SECONDS = 2
+
# Workflows whose apt calls cannot use the helper, each for a durable reason.
EXEMPT_WORKFLOWS = {
# Runs apt in bare containers and WSL with no checkout, and deliberately without `sudo`.
@@ -90,7 +95,8 @@ def _worst_case_seconds(step: dict, run: str) -> int:
per_attempt = int(per_attempt) if per_attempt is not None else 480
calls = run.count(HELPER)
- return calls * (attempts * per_attempt + (attempts - 1) * LOCK_WAIT_SECONDS)
+ per_call = attempts * per_attempt + (attempts - 1) * (LOCK_WAIT_SECONDS + ORPHAN_STOP_SECONDS)
+ return MIRROR_PROBE_SECONDS + calls * per_call
@pytest.mark.parametrize("path", _workflows(), ids = lambda p: p.name)
@@ -182,6 +188,8 @@ def test_helper_defaults_are_what_the_budgets_assume() -> None:
assert 'ATTEMPT_TIMEOUT="${RETRY_ATTEMPT_TIMEOUT:-480}"' in source
assert "seq 1 24" in source
assert LOCK_WAIT_SECONDS == 24 * 5 + 5
+ assert 'MIRROR_PROBE_SECONDS="${APT_MIRROR_PROBE_SECONDS:-5}"' in source
+ assert "sleep 2" in source
def test_the_helper_makes_apt_fail_fast() -> None: