Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
CI: rank apt mirrors by speed so a slow Azure mirror cannot time out apt steps (#15350)
Daniel Han1cf32bf777f7
diff --git a/.github/scripts/retry-with-apt-lock.sh b/.github/scripts/retry-with-apt-lock.shindex e2908d120cc..9b9524bb93e 100755--- a/.github/scripts/retry-with-apt-lock.sh+++ b/.github/scripts/retry-with-apt-lock.sh@@ -75,6 +75,94 @@ release_apt_locks() {sleep 5}+# GitHub's Ubuntu images send apt through mirror+file:$APT_MIRRORS with azure.archive.ubuntu.com+# first. apt fails over only on errors: a mirror trickling at a few KB/s (runner-images #14823)+# never trips Acquire::*::Timeout, so every retry downloads from the same slow node.+# This measures each mirror once and puts the fastest first; a failed attempt demotes the leader.+APT_MIRRORS="${APT_MIRRORS_FILE:-/etc/apt/apt-mirrors.txt}"+MIRROR_PROBE_SECONDS="${APT_MIRROR_PROBE_SECONDS:-5}"++mirror_urls() {+ grep -vE '^[[:space:]]*(#|$)' "$APT_MIRRORS" 2> /dev/null | awk '{print $1}'+}++# bytes/s fetched from a mirror in MIRROR_PROBE_SECONDS; 0 when unreachable. curl prints the rate+# even when --max-time cuts the transfer, which is exactly the slow case being measured.+probe_mirror() {+ speed="$(curl -sS -o /dev/null --max-time "$MIRROR_PROBE_SECONDS" -w '%{speed_download}' \+ "${1%/}/dists/${2}/main/binary-${3}/Packages.gz" 2> /dev/null)"+ speed="${speed%%.*}"+ case "$speed" in '' | *[!0-9]*) speed=0 ;; esac+ printf '%s' "$speed"+}++# Rewrites APT_MIRRORS with the given URLs in order, priority:1 first.+write_mirrors() {+ body=""+ n=0+ for url in "$@"; do+ n=$((n + 1))+ body="${body}${url} priority:${n}+"+ done+ if ! printf '%s' "$body" | sudo tee "$APT_MIRRORS" > /dev/null 2>&1; then+ echo "::warning::could not rewrite ${APT_MIRRORS}; apt keeps its mirror order"+ return 1+ fi+}++# Once per job: later helper calls in the same job reuse the order (and any demotion).+MIRROR_MARKER="${RUNNER_TEMP:-/tmp}/unsloth-apt-mirror-ranked"++prefer_fast_apt_mirror() {+ [ "${APT_PREFER_FAST_MIRROR:-1}" = "1" ] || return 0+ [ -f "$APT_MIRRORS" ] && command -v curl > /dev/null 2>&1 || return 0+ [ -e "$MIRROR_MARKER" ] && return 0+ : > "$MIRROR_MARKER" 2> /dev/null || true+ urls="$(mirror_urls)"+ [ -n "$urls" ] || return 0+ codename="$(. /etc/os-release 2> /dev/null && printf '%s' "${VERSION_CODENAME:-}")"+ arch="$(dpkg --print-architecture 2> /dev/null || echo amd64)"+ [ -n "$codename" ] || return 0+ # The global archive carries every amd64 pocket, so it is a valid fallback beside any+ # *.archive.ubuntu.com mirror. ports.ubuntu.com (arm64) has no such twin and is left alone.+ if printf '%s\n' "$urls" | grep -q 'archive\.ubuntu\.com/ubuntu' \+ && ! printf '%s\n' "$urls" | grep -qE '^https?://archive\.ubuntu\.com/ubuntu/?$'; then+ urls="$(printf '%s\nhttp://archive.ubuntu.com/ubuntu/' "$urls")"+ fi+ ranked=""+ for url in $urls; do+ speed="$(probe_mirror "$url" "$codename" "$arch")"+ echo "apt mirror ${url}: $((speed / 1024)) KB/s"+ ranked="${ranked}${speed} ${url}+"+ done+ # shellcheck disable=SC2046 # one URL per word+ write_mirrors $(printf '%s' "$ranked" | sort -k1,1nr -s | awk '{print $2}') \+ && echo "apt mirror order: $(mirror_urls | tr '\n' ' ')"+}++# After a failed attempt the leading mirror goes last, so the retry does not reuse it.+demote_apt_mirror() {+ [ "${APT_PREFER_FAST_MIRROR:-1}" = "1" ] && [ -f "$APT_MIRRORS" ] || return 0+ urls="$(mirror_urls)"+ [ "$(printf '%s\n' "$urls" | wc -l)" -gt 1 ] || return 0+ first="$(printf '%s\n' "$urls" | head -n 1)"+ # shellcheck disable=SC2046+ write_mirrors $(printf '%s\n' "$urls" | tail -n +2) "$first" \+ && echo "apt mirror ${first} moved last for the retry"+}++# A timed-out attempt can leave apt-get running under sudo (timeout signals sudo, not its+# grandchildren), still holding the dpkg lock and still downloading from the slow mirror.+stop_orphaned_apt() {+ sudo pkill -TERM -x apt-get > /dev/null 2>&1 || true+ sudo pkill -TERM -x apt > /dev/null 2>&1 || true+ sleep 2+ sudo dpkg --configure -a > /dev/null 2>&1 || true+}++prefer_fast_apt_mirrorconfigure_apt_fail_fastfor attempt in $(seq 1 "$ATTEMPTS"); do@@ -95,6 +183,10 @@ for attempt in $(seq 1 "$ATTEMPTS"); doecho "::warning::attempt ${attempt}/${ATTEMPTS} of '$*' ${reason}"[ "$attempt" -ge "$ATTEMPTS" ] && break+ if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then+ stop_orphaned_apt+ fi+ demote_apt_mirrorrelease_apt_locksdonediff --git a/tests/studio/test_apt_mirror_preference.py b/tests/studio/test_apt_mirror_preference.pynew file mode 100644index 00000000000..ff3504dca5f--- /dev/null+++ b/tests/studio/test_apt_mirror_preference.py@@ -0,0 +1,165 @@+"""retry-with-apt-lock.sh ranks the apt mirrors by measured speed and demotes the leader after a failure.++GitHub's Ubuntu images list azure.archive.ubuntu.com first in /etc/apt/apt-mirrors.txt, and apt only+fails over on errors, so a mirror trickling at a few KB/s held every retry (runner-images #14823).+These tests run the real helper against a temporary mirror file with `sudo`, `curl` and friends+stubbed on PATH, so nothing touches the host's apt.+"""++import os+import stat+import subprocess+import sys+from pathlib import Path++import pytest++REPO_ROOT = Path(__file__).resolve().parents[2]+HELPER = REPO_ROOT / ".github" / "scripts" / "retry-with-apt-lock.sh"++pytestmark = pytest.mark.skipif(sys.platform == "win32", reason = "bash helper")++AZURE = "http://azure.archive.ubuntu.com/ubuntu/"+GLOBAL = "http://archive.ubuntu.com/ubuntu/"+SECURITY = "http://security.ubuntu.com/ubuntu/"+++def _stub(bin_dir: Path, name: str, body: str) -> None:+ path = bin_dir / name+ path.write_text("#!/bin/bash\n" + body, encoding = "utf-8")+ path.chmod(path.stat().st_mode | stat.S_IEXEC)+++def _run(tmp_path: Path, mirrors: str, speeds: dict, command: str, **env) -> tuple:+ bin_dir = tmp_path / "bin"+ bin_dir.mkdir(exist_ok = True)+ mirrors_file = tmp_path / "apt-mirrors.txt"+ mirrors_file.write_text(mirrors, encoding = "utf-8")+ log = tmp_path / "orders.log"++ # sudo runs the command as is; tee writes where the helper points it, apt.conf.d is redirected.+ _stub(+ bin_dir,+ "sudo",+ 'case "$1" in tee) [ "${2#/etc/apt/apt.conf.d}" != "$2" ] && exec cat > /dev/null ;; esac\nexec "$@"\n',+ )+ lines = "".join(f' *"{host}"*) echo {bps} ;;\n' for host, bps in speeds.items())+ _stub(bin_dir, "curl", f'url="${{@: -1}}"\ncase "$url" in\n{lines} *) echo 0 ;;\nesac\n')+ _stub(bin_dir, "pkill", "exit 1\n")+ _stub(bin_dir, "dpkg", '[ "$1" = --print-architecture ] && echo amd64\nexit 0\n')+ _stub(bin_dir, "fuser", "exit 1\n")+ _stub(bin_dir, "sleep", "exit 0\n")+ # Records the mirror order each attempt saw.+ _stub(+ bin_dir,+ "attempt",+ f'awk \'{{print $1}}\' "{mirrors_file}" | paste -sd" " >> "{log}"\n{command}\n',+ )++ run_env = {+ **os.environ,+ "PATH": f"{bin_dir}:{os.environ['PATH']}",+ "APT_MIRRORS_FILE": str(mirrors_file),+ "RUNNER_TEMP": str(tmp_path),+ "RETRY_ATTEMPTS": "3",+ "RETRY_ATTEMPT_TIMEOUT": "30",+ **env,+ }+ proc = subprocess.run(+ ["bash", str(HELPER), "attempt"],+ env = run_env,+ capture_output = True,+ text = True,+ timeout = 60,+ )+ orders = log.read_text(encoding = "utf-8").splitlines() if log.exists() else []+ return proc, orders, mirrors_file.read_text(encoding = "utf-8")+++def _has_os_release() -> bool:+ try:+ return "VERSION_CODENAME=" in Path("/etc/os-release").read_text(encoding = "utf-8")+ except OSError:+ return False+++needs_codename = pytest.mark.skipif(not _has_os_release(), reason = "needs /etc/os-release")+++@needs_codename+def test_the_fastest_mirror_goes_first(tmp_path):+ mirrors = f"{AZURE}\tpriority:1\n{SECURITY}\tpriority:2\n"+ proc, orders, written = _run(+ tmp_path,+ mirrors,+ {"azure.archive": 30_000, "//archive.ubuntu": 9_000_000, "security": 4_000_000},+ "exit 0",+ )+ assert proc.returncode == 0, proc.stderr+ # The global archive is added beside an *.archive.ubuntu.com mirror and wins on speed.+ assert orders == [f"{GLOBAL} {SECURITY} {AZURE}"], proc.stdout+ assert f"{GLOBAL}\tpriority:1" in written+ assert f"{AZURE}\tpriority:3" in written+++@needs_codename+def test_a_fast_azure_mirror_keeps_its_place(tmp_path):+ mirrors = f"{AZURE}\tpriority:1\n{GLOBAL}\tpriority:2\n"+ proc, orders, _ = _run(+ tmp_path, mirrors, {"azure.archive": 9_000_000, "//archive.ubuntu": 1_000_000}, "exit 0"+ )+ assert proc.returncode == 0, proc.stderr+ assert orders == [f"{AZURE} {GLOBAL}"]+++@needs_codename+def test_a_failed_attempt_moves_the_leader_last(tmp_path):+ mirrors = f"{AZURE}\tpriority:1\n{GLOBAL}\tpriority:2\n"+ # Fails the first time only.+ command = f'[ -e "{tmp_path}/done" ] && exit 0; touch "{tmp_path}/done"; exit 100'+ proc, orders, _ = _run(+ tmp_path, mirrors, {"azure.archive": 9_000_000, "//archive.ubuntu": 1_000_000}, command+ )+ assert proc.returncode == 0, proc.stderr + proc.stdout+ assert orders == [f"{AZURE} {GLOBAL}", f"{GLOBAL} {AZURE}"], proc.stdout+++@needs_codename+def test_mirrors_are_ranked_once_per_job(tmp_path):+ mirrors = f"{AZURE}\tpriority:1\n{GLOBAL}\tpriority:2\n"+ _run(tmp_path, mirrors, {"azure.archive": 1_000, "//archive.ubuntu": 9_000_000}, "exit 0")+ # A second helper call in the same job keeps the order it finds, even if speeds have flipped.+ proc, orders, _ = _run(+ tmp_path,+ f"{GLOBAL}\tpriority:1\n{AZURE}\tpriority:2\n",+ {"azure.archive": 9_000_000, "//archive.ubuntu": 1_000},+ "exit 0",+ )+ assert proc.returncode == 0+ assert orders[-1] == f"{GLOBAL} {AZURE}"+++def test_no_mirror_file_runs_the_command_unchanged(tmp_path):+ proc, orders, _ = _run(+ tmp_path, "", {}, "exit 0", APT_MIRRORS_FILE = str(tmp_path / "absent.txt")+ )+ assert proc.returncode == 0, proc.stderr+ assert "apt mirror" not in proc.stdout+++@needs_codename+def test_opt_out_leaves_the_mirror_file_alone(tmp_path):+ mirrors = f"{AZURE}\tpriority:1\n"+ proc, _, written = _run(+ tmp_path, mirrors, {"//archive.ubuntu": 9_000_000}, "exit 0", APT_PREFER_FAST_MIRROR = "0"+ )+ assert proc.returncode == 0+ assert written == mirrors+++@needs_codename+def test_ports_mirrors_get_no_global_archive(tmp_path):+ ports = "http://ports.ubuntu.com/ubuntu-ports/"+ proc, orders, _ = _run(tmp_path, f"{ports}\tpriority:1\n", {"ports": 5_000_000}, "exit 0")+ assert proc.returncode == 0+ assert orders == [ports]diff --git a/tests/studio/test_apt_steps_are_bounded.py b/tests/studio/test_apt_steps_are_bounded.pyindex a445e0fdaad..16dab164d65 100644--- a/tests/studio/test_apt_steps_are_bounded.py+++ b/tests/studio/test_apt_steps_are_bounded.py@@ -24,6 +24,11 @@ _APT = re.compile(r"\bapt-get\s+(?:update|install)\b|playwright\s+install\b[^\n]# `release_dpkg_lock()` waits up to 24 * 5s, then kills the holder and sleeps 5.LOCK_WAIT_SECONDS = 125+# `prefer_fast_apt_mirror()` probes at most four mirrors for 5s each, once per job, and a+# timed-out attempt costs a 2s pause to stop the orphaned apt-get.+MIRROR_PROBE_SECONDS = 4 * 5+ORPHAN_STOP_SECONDS = 2+# Workflows whose apt calls cannot use the helper, each for a durable reason.EXEMPT_WORKFLOWS = {# Runs apt in bare containers and WSL with no checkout, and deliberately without `sudo`.@@ -90,7 +95,8 @@ def _worst_case_seconds(step: dict, run: str) -> int:per_attempt = int(per_attempt) if per_attempt is not None else 480calls = run.count(HELPER)- return calls * (attempts * per_attempt + (attempts - 1) * LOCK_WAIT_SECONDS)+ per_call = attempts * per_attempt + (attempts - 1) * (LOCK_WAIT_SECONDS + ORPHAN_STOP_SECONDS)+ return MIRROR_PROBE_SECONDS + calls * per_call@pytest.mark.parametrize("path", _workflows(), ids = lambda p: p.name)@@ -182,6 +188,8 @@ def test_helper_defaults_are_what_the_budgets_assume() -> None:assert 'ATTEMPT_TIMEOUT="${RETRY_ATTEMPT_TIMEOUT:-480}"' in sourceassert "seq 1 24" in sourceassert LOCK_WAIT_SECONDS == 24 * 5 + 5+ assert 'MIRROR_PROBE_SECONDS="${APT_MIRROR_PROBE_SECONDS:-5}"' in source+ assert "sleep 2" in sourcedef test_the_helper_makes_apt_fail_fast() -> None: