Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix llama.cpp Studio update failures on GitHub API rate limits (#9970) (#9980)
Souravrajvi01a7e11b39f91
diff --git a/studio/backend/tests/test_llama_cpp_update.py b/studio/backend/tests/test_llama_cpp_update.pyindex 82e4ac0d4d8..06cf5f52b3e 100644--- a/studio/backend/tests/test_llama_cpp_update.py+++ b/studio/backend/tests/test_llama_cpp_update.py@@ -759,6 +759,130 @@ def test_start_update_installer_failure_reports_error(monkeypatch, tmp_path):assert "boom" in (job["error"] or "")+def test_start_update_rate_limit_reports_actionable_error(monkeypatch, tmp_path):+ monkeypatch.delenv("GH_TOKEN", raising = False)+ monkeypatch.delenv("GITHUB_TOKEN", raising = False)+ install_dir = tmp_path / "llama.cpp"+ binary = _write_install(install_dir, "b9493")+ monkeypatch.setattr(upd, "_find_binary", lambda: binary)+ monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py")+ monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9518")++ lines = [+ "[llama-prebuilt] prebuilt fallback reason: failed to inspect published "+ "releases in unslothai/llama.cpp: GitHub API returned 403 for "+ "https://api.github.com/repos/unslothai/llama.cpp/releases/tags/b9518; "+ "set GH_TOKEN or GITHUB_TOKEN to avoid GitHub API rate limits\n",+ "windows_runtime_dirs=C:\\\\very\\\\long\\\\path\n",+ ]+ _patch_installer_popen(monkeypatch, returncode = 2, lines = lines)++ res = upd.start_update()+ assert res["started"] is True+ deadline = time.time() + 10+ while time.time() < deadline:+ job = upd.get_update_status()["job"]+ if job["state"] in ("success", "error"):+ break+ time.sleep(0.05)+ assert job["state"] == "error"+ assert "GH_TOKEN" in (job["error"] or "")+ assert "windows_runtime_dirs" not in (job["error"] or "")+++def test_start_update_reports_a_github_429_as_a_rate_limit(monkeypatch, tmp_path):+ # GitHub answers an exceeded rate limit with 403 or 429.+ monkeypatch.delenv("GH_TOKEN", raising = False)+ monkeypatch.delenv("GITHUB_TOKEN", raising = False)+ install_dir = tmp_path / "llama.cpp"+ binary = _write_install(install_dir, "b9493")+ monkeypatch.setattr(upd, "_find_binary", lambda: binary)+ monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py")+ monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9518")++ lines = [+ "[llama-prebuilt] prebuilt fallback reason: failed to inspect published "+ "releases in unslothai/llama.cpp: GitHub API returned 429 for "+ "https://api.github.com/repos/unslothai/llama.cpp/releases/tags/b9518; "+ "set GH_TOKEN or GITHUB_TOKEN to avoid GitHub API rate limits\n",+ ]+ _patch_installer_popen(monkeypatch, returncode = 2, lines = lines)++ res = upd.start_update()+ assert res["started"] is True+ deadline = time.time() + 10+ while time.time() < deadline:+ job = upd.get_update_status()["job"]+ if job["state"] in ("success", "error"):+ break+ time.sleep(0.05)+ assert job["state"] == "error"+ assert "GH_TOKEN" in (job["error"] or "")+++def test_start_update_tells_an_authenticated_run_to_wait_for_the_limit(monkeypatch, tmp_path):+ # An authenticated run has spent the larger quota (or tripped a secondary+ # limit); it already holds the token this message used to ask it to set.+ monkeypatch.setenv("GH_TOKEN", "x")+ install_dir = tmp_path / "llama.cpp"+ binary = _write_install(install_dir, "b9493")+ monkeypatch.setattr(upd, "_find_binary", lambda: binary)+ monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py")+ monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9518")++ lines = [+ "[llama-prebuilt] prebuilt fallback reason: failed to inspect published "+ "releases in unslothai/llama.cpp: GitHub API returned 429 for "+ "https://api.github.com/repos/unslothai/llama.cpp/releases/tags/b9518\n",+ ]+ _patch_installer_popen(monkeypatch, returncode = 2, lines = lines)++ res = upd.start_update()+ assert res["started"] is True+ deadline = time.time() + 10+ while time.time() < deadline:+ job = upd.get_update_status()["job"]+ if job["state"] in ("success", "error"):+ break+ time.sleep(0.05)+ assert job["state"] == "error"+ error = job["error"] or ""+ assert "rate-limiting" in error+ assert "GH_TOKEN" not in error+ assert "Wait for the limit to reset" in error+++def test_start_update_does_not_blame_github_for_a_hugging_face_rate_limit(monkeypatch, tmp_path):+ # The tiny validation model comes from huggingface.co, whose own rate limit+ # also surfaces as installer exit 2. GH_TOKEN cannot fix that one.+ install_dir = tmp_path / "llama.cpp"+ binary = _write_install(install_dir, "b9493")+ monkeypatch.setattr(upd, "_find_binary", lambda: binary)+ monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py")+ monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9518")++ lines = [+ "[llama-prebuilt] prebuilt fallback reason: validation model unavailable: "+ "429 Client Error: Too Many Requests for url "+ "https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K.gguf "+ "(Rate limit reached)\n",+ ]+ _patch_installer_popen(monkeypatch, returncode = 2, lines = lines)++ res = upd.start_update()+ assert res["started"] is True+ deadline = time.time() + 10+ while time.time() < deadline:+ job = upd.get_update_status()["job"]+ if job["state"] in ("success", "error"):+ break+ time.sleep(0.05)+ assert job["state"] == "error"+ error = job["error"] or ""+ assert "GH_TOKEN" not in error+ assert "validation model unavailable" in error++# --- installer-argument construction (mirrors the post-#5963 setup scripts) ---diff --git a/studio/backend/tests/test_update_flow_messages.py b/studio/backend/tests/test_update_flow_messages.pynew file mode 100644index 00000000000..8b8c0822174--- /dev/null+++ b/studio/backend/tests/test_update_flow_messages.py@@ -0,0 +1,230 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import sys++import pytest++from utils.prebuilt import update_flow as flow++[email protected](autouse = True)+def _no_github_token(monkeypatch):+ # The rate-limit advice depends on it, and a CI runner may export either.+ monkeypatch.delenv("GH_TOKEN", raising = False)+ monkeypatch.delenv("GITHUB_TOKEN", raising = False)+++def test_format_installer_failure_prefers_fallback_reason_over_path_noise():+ huge_path = "C:\\Windows\\" + "VeryLongDir\\" * 120 + "torch\\lib"+ lines = [+ "[llama-prebuilt] prebuilt install path failed; falling back to source build\n",+ (+ "[llama-prebuilt] prebuilt fallback reason: failed to inspect published "+ "releases in unslothai/llama.cpp: GitHub API returned 403 for "+ "https://api.github.com/repos/unslothai/llama.cpp/releases/tags/b10679; "+ "set GH_TOKEN or GITHUB_TOKEN to avoid GitHub API rate limits\n"+ ),+ f"windows_runtime_dirs={huge_path}\n",+ ]+ message = flow.format_installer_failure_message(2, lines)+ assert "GH_TOKEN" in message+ assert "rate limit" in message.lower()+ assert "windows_runtime_dirs" not in message+++def test_format_installer_failure_does_not_name_llama_for_a_whisper_install():+ # whisper.cpp updates share stream_installer, and prebuilt_core raises the same+ # 403 text for both, so the message must not name the wrong component.+ lines = [+ "[whisper-prebuilt] prebuilt install failed: GitHub API returned 403 for "+ "https://api.github.com/repos/unslothai/whisper.cpp/releases/latest; "+ "set GH_TOKEN or GITHUB_TOKEN to avoid GitHub API rate limits\n",+ ]+ message = flow.format_installer_failure_message(1, lines)+ assert "GH_TOKEN" in message+ assert "llama" not in message.lower()+++def test_format_installer_failure_falls_back_to_tail_without_actionable_line():+ lines = ["line one\n", "line two\n"]+ message = flow.format_installer_failure_message(1, lines)+ assert message == "installer exited 1: line one\nline two"+++def test_format_installer_failure_prefers_the_verdict_over_an_earlier_rate_limit_retry():+ # The installer retries a rate-limited fetch and can still get through, so a+ # retry line must never outrank the reason it actually exited on.+ lines = [+ "[llama-prebuilt] fetch failed (1/4) for "+ "https://api.github.com/repos/unslothai/llama.cpp/releases/tags/b10679: "+ "HTTP Error 403: rate limit exceeded; retrying\n",+ "[llama-prebuilt] prebuilt install refused: unknown backend request 'cuda14'\n",+ ]+ message = flow.format_installer_failure_message(1, lines)+ assert message == "installer exited 1: unknown backend request 'cuda14'"+++def _run_fake_installer(tmp_path, body: str) -> flow.InstallerExit:+ script = tmp_path / "fake_installer.py"+ script.write_text("import sys\n" + body, encoding = "utf-8")+ with pytest.raises(flow.InstallerExit) as excinfo:+ flow.stream_installer(+ [sys.executable, str(script)],+ {},+ timeout_seconds = 60,+ set_progress = lambda _fraction: None,+ )+ return excinfo.value+++def test_stream_installer_keeps_reason_a_long_system_report_pushes_out_of_the_tail(tmp_path):+ # The report (selection log, nvidia-smi, ldd) outruns the bounded tail on a+ # Linux CUDA host, so the reason has to be kept as it streams, not looked up+ # in the tail afterwards.+ exc = _run_fake_installer(+ tmp_path,+ "print('[llama-prebuilt] prebuilt fallback reason: failed to inspect published "+ "releases in unslothai/llama.cpp: GitHub API returned 403; set GH_TOKEN or "+ "GITHUB_TOKEN to avoid GitHub API rate limits')\n"+ "for i in range(300):\n"+ " print(f'linux_runtime_dirs_{i}=/usr/lib/x86_64-linux-gnu')\n"+ "sys.exit(2)\n",+ )+ assert exc.returncode == 2+ assert "GH_TOKEN" in str(exc)+ assert "linux_runtime_dirs" not in str(exc)+++def test_stream_installer_keeps_the_verdict_behind_a_run_of_rate_limit_retries(tmp_path):+ # Retries are unbounded in principle and the verdict comes last, so the two+ # cannot share one capped buffer.+ exc = _run_fake_installer(+ tmp_path,+ "url = 'https://api.github.com/repos/unslothai/llama.cpp/releases/tags/b10679'\n"+ "for i in range(1, 11):\n"+ " print(f'[llama-prebuilt] fetch failed ({i}/4) for {url}: "+ "HTTP Error 403: rate limit exceeded; retrying')\n"+ "print('[llama-prebuilt] prebuilt fallback reason: no published prebuilt asset "+ "matches this host (windows-rocm gfx803)')\n"+ "for i in range(300):\n"+ " print(f'windows_runtime_dirs_{i}=C:/x')\n"+ "sys.exit(2)\n",+ )+ assert exc.returncode == 2+ assert str(exc) == (+ "installer exited 2: no published prebuilt asset matches this host (windows-rocm gfx803)"+ )+++def test_stream_installer_keeps_a_multiline_verdict_whole(tmp_path):+ # The linux preflight failure names one binary per line; truncating at the+ # first line leaves "preflight failed:" and nothing to act on.+ exc = _run_fake_installer(+ tmp_path,+ "print('[llama-prebuilt] prebuilt install path failed; falling back to source build')\n"+ "for line in ['prebuilt fallback reason: linux extracted binary preflight failed:',\n"+ " 'llama-server: missing=libcuda.so.1 ld_library_path=none',\n"+ " 'llama-quantize: missing=libgomp.so.1 ld_library_path=none']:\n"+ " print('[llama-prebuilt] ' + line)\n"+ "print('platform=Linux machine=x86_64')\n"+ "for i in range(300):\n"+ " print(f'linux_runtime_dirs_{i}=/usr/lib')\n"+ "sys.exit(2)\n",+ )+ assert exc.returncode == 2+ assert "missing=libcuda.so.1" in str(exc)+ assert "missing=libgomp.so.1" in str(exc)+ # The unprefixed system report is where the verdict ends.+ assert "linux_runtime_dirs" not in str(exc)+ assert "platform=Linux" not in str(exc)+++def test_stream_installer_reports_a_fatal_error_that_follows_a_survived_retry(tmp_path):+ # The retry succeeded and the run died of something else; the stale hint must+ # not replace the error the installer actually exited on.+ exc = _run_fake_installer(+ tmp_path,+ "url = 'https://api.github.com/repos/unslothai/llama.cpp/releases/tags/b10679'\n"+ "print(f'[llama-prebuilt] fetch failed (1/4) for {url}: "+ "HTTP Error 429: Too Many Requests; retrying')\n"+ "print('[llama-prebuilt] resolved published release b10679-mix-67dfc8b')\n"+ "print('[llama-prebuilt] fatal helper error: staged install could not be "+ "activated: [Errno 13] Permission denied')\n"+ "sys.exit(1)\n",+ )+ assert exc.returncode == 1+ assert str(exc) == (+ "installer exited 1: staged install could not be activated: [Errno 13] Permission denied"+ )+++def test_stream_installer_keeps_the_output_when_only_a_rate_limit_hint_is_present(tmp_path):+ # No verdict line at all: the hint annotates the output rather than replacing+ # it, so a crash is still readable.+ exc = _run_fake_installer(+ tmp_path,+ "url = 'https://api.github.com/repos/unslothai/llama.cpp/releases/tags/b10679'\n"+ "print(f'[llama-prebuilt] fetch failed (1/4) for {url}: "+ "HTTP Error 403: rate limit exceeded; retrying')\n"+ "print('Traceback (most recent call last):')\n"+ "print('MemoryError')\n"+ "sys.exit(1)\n",+ )+ assert exc.returncode == 1+ assert "GH_TOKEN" in str(exc)+ assert "MemoryError" in str(exc)+++def test_format_installer_failure_tells_an_authenticated_run_to_wait(monkeypatch):+ # The token is already set, so its quota is what ran out: setting it again is+ # advice the user cannot act on. fetch_json omits its own hint here too.+ monkeypatch.setenv("GH_TOKEN", "x")+ lines = [+ "[llama-prebuilt] prebuilt fallback reason: failed to inspect published "+ "releases in unslothai/llama.cpp: GitHub API returned 429 for "+ "https://api.github.com/repos/unslothai/llama.cpp/releases/tags/b10679\n",+ ]+ message = flow.format_installer_failure_message(2, lines)+ assert "rate limit" in message.lower()+ assert "GH_TOKEN" not in message+ assert "Wait for the limit to reset" in message+++def test_stream_installer_reads_the_token_from_the_installer_environment(tmp_path, monkeypatch):+ # The advice must follow the env the child ran with, not this process's.+ monkeypatch.delenv("GH_TOKEN", raising = False)+ script = tmp_path / "fake_installer.py"+ script.write_text(+ "import sys\n"+ "print('[llama-prebuilt] prebuilt fallback reason: GitHub API returned 403 for "+ "https://api.github.com/repos/unslothai/llama.cpp/releases/latest')\n"+ "sys.exit(2)\n",+ encoding = "utf-8",+ )+ with pytest.raises(flow.InstallerExit) as excinfo:+ flow.stream_installer(+ [sys.executable, str(script)],+ {"GH_TOKEN": "x"},+ timeout_seconds = 60,+ set_progress = lambda _fraction: None,+ )+ assert "GH_TOKEN" not in str(excinfo.value)+ assert "Wait for the limit to reset" in str(excinfo.value)+++def test_stream_installer_keeps_a_verdict_a_child_announcement_interrupts(tmp_path):+ # The installer announces the servers it starts on the same stream; that+ # protocol line must not be mistaken for the start of the system report.+ exc = _run_fake_installer(+ tmp_path,+ "print('[llama-prebuilt] prebuilt fallback reason: linux extracted binary "+ "preflight failed:')\n"+ "print('UNSLOTH_INSTALLER_CHILD started 4242')\n"+ "print('[llama-prebuilt] llama-server: missing=libcuda.so.1 ld_library_path=none')\n"+ "print('UNSLOTH_INSTALLER_CHILD stopped 4242')\n"+ "sys.exit(2)\n",+ )+ assert exc.returncode == 2+ assert "missing=libcuda.so.1" in str(exc)+ assert "UNSLOTH_INSTALLER_CHILD" not in str(exc)diff --git a/studio/backend/utils/llama_cpp_update.py b/studio/backend/utils/llama_cpp_update.pyindex 08a9f73ca02..607c3b2965f 100644--- a/studio/backend/utils/llama_cpp_update.py+++ b/studio/backend/utils/llama_cpp_update.py@@ -66,6 +66,8 @@ _INSTALL_TIMEOUT_SECONDS = 1800 # 30 min ceiling for download + build/validate_EXIT_NO_SPACE = 4# A concrete backend selection could not be satisfied._EXIT_BACKEND_UNAVAILABLE = 5+# Prebuilt path failed; setup scripts source-build, but the in-app updater cannot.+_EXIT_FALLBACK = 2class _LlamaPhaseError(RuntimeError):@@ -769,6 +771,30 @@ def _run_llama_phase("The installed backend was kept.",reload_required = model_was_active,) from exc+ if exc.returncode == _EXIT_FALLBACK:+ message = str(exc)+ # Same predicate the formatter uses: exit 2 also carries failures+ # like a rate-limited huggingface.co validation-model fetch, which+ # GH_TOKEN cannot fix.+ if _flow.is_github_rate_limit_text(message):+ token_present = _flow.github_token_present(env)+ logger.warning("llama update: GitHub rate limit", authenticated = token_present)+ advice = (+ "Wait for the limit to reset and try again."+ if token_present+ else "Set GH_TOKEN or GITHUB_TOKEN in your environment and try again."+ )+ raise _LlamaPhaseError(+ "Could not update llama.cpp: GitHub is rate-limiting release downloads. "+ f"{advice}",+ reload_required = model_was_active,+ ) from exc+ logger.warning("llama update: prebuilt fallback", error = message)+ detail = message.split(": ", 1)[-1] if ": " in message else message+ raise _LlamaPhaseError(+ f"Could not update llama.cpp from the prebuilt bundle. {detail}",+ reload_required = model_was_active,+ ) from exclogger.warning("llama update: failed", error = str(exc))raise _LlamaPhaseError(str(exc), reload_required = model_was_active) from excexcept Exception as exc:diff --git a/studio/backend/utils/prebuilt/update_flow.py b/studio/backend/utils/prebuilt/update_flow.pyindex 5a8f27d5026..35720f35cd8 100644--- a/studio/backend/utils/prebuilt/update_flow.py+++ b/studio/backend/utils/prebuilt/update_flow.py@@ -20,7 +20,7 @@ import sysimport threadingimport timefrom pathlib import Path-from typing import Callable, Optional+from typing import Callable, Optional, Sequenceimport structlog@@ -53,6 +53,132 @@ class InstallerExit(RuntimeError):self.returncode = returncode+# Any installer log line. whisper.cpp updates run through this same helper, so+# the component is matched as a pattern rather than llama's alone. The prefix is+# also what separates the installer's own lines from the unprefixed system+# report it prints after them.+_PREBUILT_LOG_RE = re.compile(r"^\[[\w.-]+-prebuilt\]\s?(?P<body>.*)$")+# The verdict line an installer logs before it exits: "prebuilt fallback+# reason:", "prebuilt install failed:", "prebuilt install refused:", "prebuilt+# busy reason:", "fatal helper error:", "fatal helper busy conflict:".+_PREBUILT_VERDICT_RE = re.compile(+ r"^(?:prebuilt\b[^:]*\b(?:reason|failed|refused)"+ r"|fatal helper (?:error|busy conflict)):\s*(?P<detail>.*)$"+)+# A multi-line reason (the preflight failure lists one library per line) is+# logged one prefixed line at a time, so a verdict owns the prefixed lines that+# follow it. Bounded because only the installer's own framing bounds them.+_VERDICT_CONTINUATION_LIMIT = 8+++def is_github_rate_limit_text(text: str) -> bool:+ """Whether installer output blames a GitHub API rate limit.++ Both halves are required. huggingface.co rate-limits the validation model+ download with its own 429, and that failure also reaches the updater as+ installer exit 2, so rate-limit wording alone would hand a user GH_TOKEN+ advice that cannot fix it. GitHub itself answers an exceeded primary or+ secondary limit with 403 or 429.+ """+ lowered = text.lower()+ if "github" not in lowered:+ return False+ return (+ "rate limit" in lowered+ or "too many requests" in lowered+ or "returned 403" in lowered+ or "returned 429" in lowered+ )+++def github_token_present(env: Optional[dict] = None) -> bool:+ """Whether the installer ran with a GitHub token, as fetch_json reads it."""+ source = os.environ if env is None else env+ return bool(source.get("GH_TOKEN") or source.get("GITHUB_TOKEN"))+++def github_rate_limit_advice(token_present: bool) -> str:+ """What the user can actually do about the rate limit they just hit.++ An authenticated run has already spent the larger quota, or tripped a+ secondary limit, and telling it to set the token it is holding is advice it+ cannot act on. The installer draws the same distinction: fetch_json only+ appends the token hint when neither variable is set.+ """+ if token_present:+ return "Wait for the limit to reset and try again."+ return "Set GH_TOKEN or GITHUB_TOKEN to avoid GitHub API rate limits."+++def _installer_verdict(lines: Sequence[str]) -> str | None:+ """The reason the installer exited on, with its continuation lines.++ The installer logs that reason before dumping a long system report (Windows+ PATH lines, nvidia-smi, ldd), so a raw tail hides rate-limit and network+ errors behind noise (#9970). The last verdict wins: it is the one the+ installer exited on, while an earlier rate-limit retry may have succeeded on+ a later attempt. An unprefixed line ends a verdict, which is where the+ system report begins.+ """+ verdict: list[str] | None = None+ open_block: list[str] | None = None+ for line in lines:+ stripped = line.strip()+ if CHILD_PID_LINE_RE.match(stripped) is not None:+ # A grandchild announcement is protocol, not output: it can land+ # between a reason and its continuation lines without ending either.+ continue+ log_line = _PREBUILT_LOG_RE.match(stripped)+ if log_line is None:+ open_block = None+ continue+ body = log_line.group("body").strip()+ match = _PREBUILT_VERDICT_RE.match(body)+ if match is not None:+ verdict = open_block = [match.group("detail").strip()]+ elif open_block is not None and len(open_block) <= _VERDICT_CONTINUATION_LIMIT:+ open_block.append(body)+ if verdict is None:+ return None+ detail = "\n".join(part for part in verdict if part)+ return detail or None+++def format_installer_failure_message(+ returncode: int,+ lines: Sequence[str],+ verdict_lines: Sequence[str] = (),+ hint_lines: Sequence[str] = (),+ env: Optional[dict] = None,+) -> str:+ """Build an installer failure message that prefers the verdict over tail noise.++ *verdict_lines* and *hint_lines* are what stream_installer kept as they+ streamed: the system report is long enough on a Linux CUDA host to push the+ reason out of the bounded tail, so the tail alone is not a reliable place to+ find it. A rate-limit hint only annotates the tail, because the run may have+ retried past it and died of something else entirely. *env* is the+ environment the installer ran with, which decides the rate-limit advice.+ """+ advice = github_rate_limit_advice(github_token_present(env))+ detail = _installer_verdict(verdict_lines) or _installer_verdict(lines)+ if detail:+ if is_github_rate_limit_text(detail):+ return (+ f"installer exited {returncode}: GitHub API rate limit exceeded while "+ f"fetching prebuilt releases. {advice}"+ )+ clipped = detail if len(detail) <= 1500 else detail[:1497] + "..."+ return f"installer exited {returncode}: {clipped}"+ tail = "".join(lines).strip()[-1500:] or "no output"+ if any(is_github_rate_limit_text(line) for line in (*hint_lines, *lines)):+ return (+ f"installer exited {returncode}: GitHub API rate limit exceeded while fetching "+ f"prebuilt releases. {advice} Installer output: {tail}"+ )+ return f"installer exited {returncode}: {tail}"++JOB_IDLE = "idle"JOB_RUNNING = "running"JOB_SUCCESS = "success"@@ -377,13 +503,36 @@ def stream_installer(watchdog.daemon = Truewatchdog.start()tail_lines: list[str] = []+ # Two buckets, not one: a run can log a dozen rate-limit retries before the+ # verdict line, and a single capped list would fill with retries and drop the+ # verdict, which is the line the user needs.+ verdict_lines: list[str] = []+ hint_lines: list[str] = []+ open_verdict = Falsetry:assert proc.stdout is not Nonefor line in proc.stdout:tail_lines.append(line)if len(tail_lines) > 80:del tail_lines[0]- child = CHILD_PID_LINE_RE.match(line.strip())+ stripped = line.strip()+ child_line = CHILD_PID_LINE_RE.match(stripped)+ log_line = None if child_line is not None else _PREBUILT_LOG_RE.match(stripped)+ body = log_line.group("body").strip() if log_line is not None else None+ if body is not None and _PREBUILT_VERDICT_RE.match(body) is not None:+ # Restart the block: this verdict supersedes any earlier one.+ verdict_lines = [line]+ open_verdict = True+ elif open_verdict and body is not None:+ if len(verdict_lines) <= _VERDICT_CONTINUATION_LIMIT:+ verdict_lines.append(line)+ elif body is None and child_line is None:+ # Where the unprefixed system report starts, the verdict ends.+ # A grandchild announcement is protocol, not output, and never ends one.+ open_verdict = False+ if not open_verdict and len(hint_lines) < 4 and is_github_rate_limit_text(line):+ hint_lines.append(line)+ child = child_lineif child is not None:# Recorded while it runs and dropped when the installer says it# stopped; one it never got to report stays for the sweep.@@ -410,8 +559,12 @@ def stream_installer(if timed_out.is_set():raise RuntimeError(f"installer timed out after {timeout_seconds}s")if returncode != 0:- tail = "".join(tail_lines).strip()[-1500:]- raise InstallerExit(returncode, f"installer exited {returncode}: {tail or 'no output'}")+ raise InstallerExit(+ returncode,+ format_installer_failure_message(+ returncode, tail_lines, verdict_lines, hint_lines, env = env+ ),+ )def _new_phase_record(spec: dict) -> dict:diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.pyindex 2618f63870c..ac86b6930b3 100644--- a/studio/install_llama_prebuilt.py+++ b/studio/install_llama_prebuilt.py@@ -1976,13 +1976,17 @@ def _fetch_download_host_json(url: str) -> Any:)-def _download_host_resolved_release(repo: str) -> ResolvedPublishedRelease | None:- """Resolve the latest fork release from the download host with zero- api.github.com calls, reusing the API path's parsing and validation. The latest- tag is the authoritative /releases/latest redirect tag, and the checksum asset's+def _download_host_resolved_release(+ repo: str, published_release_tag: str = ""+) -> ResolvedPublishedRelease | None:+ """Resolve a fork release from the download host with zero api.github.com calls,+ reusing the API path's parsing and validation.++ When *published_release_tag* is set, that tag is fetched directly. Otherwise the+ latest tag comes from the /releases/latest redirect and the checksum asset'sself-reported release_tag is cross-checked against it. Returns None (caller fallsback to the API) on a missing JSON asset or a tag mismatch."""- release_tag = _download_host_latest_release_tag(repo)+ release_tag = (published_release_tag or "").strip() or _download_host_latest_release_tag(repo)if not release_tag:return Nonesha_url = _release_asset_download_url(repo, release_tag, DEFAULT_PUBLISHED_SHA256_ASSET)@@ -2109,43 +2113,62 @@ def iter_resolved_published_releases(repo = published_repo or DEFAULT_PUBLISHED_REPOnormalized_requested = normalized_requested_llama_tag(requested_tag)+ # Fast path: resolve a pinned or latest fork release from the download host (no+ # api.github.com rate limit). Latest surfaces only the single newest release, so+ # the caller disables it when the multi-release walk-back is needed (macOS+ # skipping too-new prebuilts); a broken latest then drops to source build, not+ # an older release. Pinned tags use the same CDN path so in-app updates avoid+ # the API entirely (#9970). Any rejection/network error is non-fatal and falls+ # through to the API.+ fast_path_tag: str | None = Noneif published_release_tag:- bundle = pinned_published_release_bundle(repo, published_release_tag)- if not published_release_matches_request(bundle, normalized_requested):- raise PrebuiltFallback(- "published release "- f"{repo}@{published_release_tag} targeted upstream tag {bundle.upstream_tag}, "- f"but requested {normalized_requested}"- )- yield ResolvedPublishedRelease(- bundle = bundle,- checksums = validated_checksums_for_bundle(repo, bundle),- )- return+ fast_path_tag = published_release_tag+ elif normalized_requested == "latest":+ fast_path_tag = ""- # Fast path: resolve the fork's latest release from the download host (no- # api.github.com rate limit). It surfaces only the single latest release, so the- # caller disables it when the multi-release walk-back is needed (macOS skipping- # too-new prebuilts); a broken latest then drops to source build, not an older- # release. Any rejection/network error is non-fatal and falls through to the API.if (- allow_download_host_fast_path+ fast_path_tag is not None+ and allow_download_host_fast_pathand repo == DEFAULT_PUBLISHED_REPO- and normalized_requested == "latest"and _download_host_resolve_enabled()):+ tag_label = fast_path_tag or "latest"try:- resolved = _download_host_resolved_release(repo)+ resolved = _download_host_resolved_release(repo, fast_path_tag)except PrebuiltFallback as exc:- log(f"download-host latest release rejected for {repo} ({exc}); trying GitHub API")+ log(f"download-host release rejected for {repo}@{tag_label} ({exc}); trying GitHub API")resolved = Noneexcept Exception as exc:- log(f"download-host latest resolve unavailable for {repo} ({exc}); trying GitHub API")+ log(+ f"download-host resolve unavailable for {repo}@{tag_label} ({exc}); trying GitHub API"+ )resolved = Noneif resolved is not None:+ if published_release_tag and not published_release_matches_request(+ resolved.bundle, normalized_requested+ ):+ raise PrebuiltFallback(+ "published release "+ f"{repo}@{resolved.bundle.release_tag} targeted upstream tag "+ f"{resolved.bundle.upstream_tag}, but requested {normalized_requested}"+ )yield resolvedreturn+ if published_release_tag:+ bundle = pinned_published_release_bundle(repo, published_release_tag)+ if not published_release_matches_request(bundle, normalized_requested):+ raise PrebuiltFallback(+ "published release "+ f"{repo}@{published_release_tag} targeted upstream tag {bundle.upstream_tag}, "+ f"but requested {normalized_requested}"+ )+ yield ResolvedPublishedRelease(+ bundle = bundle,+ checksums = validated_checksums_for_bundle(repo, bundle),+ )+ return+matched_any = Falseskipped_invalid = 0yielded_valid = False@@ -8052,8 +8075,8 @@ def install_prebuilt(## Not dead code despite the download-host fast path: macOS skips it# entirely (see allow_download_host_fast_path below), as does a- # pinned UNSLOTH_LLAMA_RELEASE_TAG, a non-latest tag, a non-default- # --published-repo, and any CDN outage.+ # non-latest requested tag without a published-release pin, a+ # non-default --published-repo, and any CDN outage.## Transport shapes only: URLError covers HTTPError and the socket/DNS# errors urllib wraps, JSONDecodeError is a ValueError, and@@ -8287,7 +8310,9 @@ def install_prebuilt(if preserve_backendelse "prebuilt install path failed; falling back to source build")- log(f"prebuilt fallback reason: {exc}")+ # log_lines, not log: a preflight failure lists one library per line, and+ # only prefixed lines are distinguishable from the system report below.+ log_lines(f"prebuilt fallback reason: {exc}".splitlines())# Diagnostics must never change the verdict: a probe that raises here# would replace the fallback with EXIT_ERROR, which never source builds.try:@@ -8732,7 +8757,9 @@ if __name__ == "__main__":fatal = _environment_fatal_reason(exc)if fatal:_fail_no_space(f"prebuilt install failed: {fatal}")- log(textwrap.shorten(str(exc), width = 400, placeholder = "..."))+ log(+ f"prebuilt install failed: {textwrap.shorten(str(exc), width = 400, placeholder = '...')}"+ )raise SystemExit(EXIT_FALLBACK)except Exception as exc:fatal = _environment_fatal_reason(exc)diff --git a/studio/install_whisper_prebuilt.py b/studio/install_whisper_prebuilt.pyindex 749043736ad..a73d4611e3d 100644--- a/studio/install_whisper_prebuilt.py+++ b/studio/install_whisper_prebuilt.py@@ -61,7 +61,7 @@ import subprocessimport sysfrom dataclasses import dataclass, replacefrom pathlib import Path-from typing import Any+from typing import Any, Iterable# Put studio/ on sys.path so install_llama_prebuilt / prebuilt_core resolve# whether run as a script (from any cwd) or imported by the tests.@@ -116,6 +116,11 @@ def log(message: str) -> None:print(f"[whisper-prebuilt] {message}", file = sys.stdout if _LOG_TO_STDOUT else sys.stderr)+def log_lines(lines: Iterable[str]) -> None:+ for line in lines:+ log(line)++EXIT_SUCCESS = 0EXIT_ERROR = 1EXIT_INCOMPATIBLE = 2@@ -1591,7 +1596,9 @@ def main(argv: list[str] | None = None) -> int:log(f"incompatible release: {exc}")return EXIT_INCOMPATIBLEexcept PrebuiltFallback as exc:- log(f"prebuilt install failed: {exc}")+ # One prefixed line each: a multi-line reason is otherwise indistinguishable+ # from unprefixed diagnostics for whoever reads this output back.+ log_lines(f"prebuilt install failed: {exc}".splitlines())return EXIT_ERRORexcept Exception as exc: # noqa: BLE001log(f"unexpected error: {exc}")diff --git a/studio/prebuilt_core.py b/studio/prebuilt_core.pyindex 8a483b34f04..783f547cc77 100644--- a/studio/prebuilt_core.py+++ b/studio/prebuilt_core.py@@ -87,6 +87,8 @@ USER_AGENT = "unsloth-studio-prebuilt"GITHUB_AUTH_HOSTS = {"api.github.com", "github.com"}HF_AUTH_HOSTS = {"huggingface.co", "www.huggingface.co"}RETRYABLE_HTTP_STATUS = {408, 429, 500, 502, 503, 504}+# What an exceeded GitHub API rate limit (primary or secondary) answers with.+GITHUB_RATE_LIMIT_STATUS = {403, 429}HTTP_FETCH_ATTEMPTS = 4HTTP_FETCH_BASE_DELAY_SECONDS = 0.75JSON_FETCH_ATTEMPTS = 3@@ -390,8 +392,9 @@ def is_github_api_url(url: str | None) -> bool:def is_retryable_url_error(exc: Exception) -> bool:if isinstance(exc, urllib.error.HTTPError):- # GitHub returns 403 (not 429) on API rate-limit; anonymous calls share a- # 60-req/hour bucket per runner IP that CI fleets exhaust. Treat 403+ # GitHub answers an API rate-limit with 403 or 429 (429 is already in+ # RETRYABLE_HTTP_STATUS); anonymous calls share a 60-req/hour bucket per+ # runner IP that CI fleets exhaust. Treat 403# against api.github.com as retryable so we get a backoff cycle or two# (honouring Retry-After / X-RateLimit-Reset) before the source-build# fallback fires. 403s on other hosts (private downloads, auth) stay non-retryable.@@ -685,11 +688,15 @@ def fetch_json(ops: ModuleOps, url: str) -> Any:else ops.auth_headers(url),)except urllib.error.HTTPError as exc:- if exc.code == 403 and is_github_api_url(url):+ # Both codes: GitHub answers an exceeded primary or secondary rate+ # limit with 403 or 429 (docs.github.com/rest/using-the-rest-api/+ # rate-limits-for-the-rest-api), and a bare "HTTP Error 429: Too Many+ # Requests" carries none of the token guidance.+ if exc.code in GITHUB_RATE_LIMIT_STATUS and is_github_api_url(url):hint = ""if not (os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN")):hint = "; set GH_TOKEN or GITHUB_TOKEN to avoid GitHub API rate limits"- raise RuntimeError(f"GitHub API returned 403 for {url}{hint}") from exc+ raise RuntimeError(f"GitHub API returned {exc.code} for {url}{hint}") from excraiseif not data:last_decode_exc = RuntimeError(f"downloaded empty JSON payload from {url}")diff --git a/tests/studio/install/test_download_host_resolve.py b/tests/studio/install/test_download_host_resolve.pyindex 5108c84062c..1b8bbca22a1 100644--- a/tests/studio/install/test_download_host_resolve.py+++ b/tests/studio/install/test_download_host_resolve.py@@ -41,11 +41,35 @@ def _resolve(tag = "latest", **kw):def test_fast_path_yields_latest_without_api(monkeypatch):sentinel = object()monkeypatch.delenv("UNSLOTH_LLAMA_DISABLE_DOWNLOAD_HOST_RESOLVE", raising = False)- monkeypatch.setattr(ILP, "_download_host_resolved_release", lambda _repo: sentinel)+ monkeypatch.setattr(ILP, "_download_host_resolved_release", lambda _repo, _tag = "": sentinel)monkeypatch.setattr(ILP, "iter_published_release_bundles", _api_raises)assert _resolve() == [sentinel]+def test_fast_path_yields_pinned_tag_without_api(monkeypatch):+ pin = "b9964-mix-53618c5"+ monkeypatch.delenv("UNSLOTH_LLAMA_DISABLE_DOWNLOAD_HOST_RESOLVE", raising = False)+ seen: dict[str, str] = {}++ class _Bundle:+ release_tag = pin+ upstream_tag = UPSTREAM_TAG++ class _Resolved:+ bundle = _Bundle()++ def _fast(_repo, published_release_tag = ""):+ seen["tag"] = published_release_tag+ return _Resolved()++ monkeypatch.setattr(ILP, "_download_host_resolved_release", _fast)+ monkeypatch.setattr(ILP, "pinned_published_release_bundle", _api_raises)+ resolved = list(ILP.iter_resolved_published_releases("latest", FORK_REPO, pin))+ assert len(resolved) == 1+ assert resolved[0].bundle.release_tag == pin+ assert seen["tag"] == pin++def test_fast_path_disabled_by_caller_uses_api(monkeypatch):# macOS passes allow_download_host_fast_path = False to keep the walk-back.monkeypatch.delenv("UNSLOTH_LLAMA_DISABLE_DOWNLOAD_HOST_RESOLVE", raising = False)@@ -73,7 +97,7 @@ def test_fast_path_skipped_for_non_latest_request(monkeypatch):def test_fast_path_none_falls_back_to_api(monkeypatch):monkeypatch.delenv("UNSLOTH_LLAMA_DISABLE_DOWNLOAD_HOST_RESOLVE", raising = False)- monkeypatch.setattr(ILP, "_download_host_resolved_release", lambda _repo: None)+ monkeypatch.setattr(ILP, "_download_host_resolved_release", lambda _repo, _tag = "": None)used = {"api": False}def _api(*_a, **_k):@@ -182,7 +206,7 @@ def _stub_downloads(def test_resolved_release_adds_tag_pinned_url_for_manifest_only_asset(monkeypatch):_stub_downloads(monkeypatch, _sha_payload(), _manifest_bytes())- resolved = ILP._download_host_resolved_release(FORK_REPO)+ resolved = ILP._download_host_resolved_release(FORK_REPO, RELEASE_TAG)assert resolved is not Noneassert resolved.bundle.release_tag == RELEASE_TAG# Binary is named only in the manifest, yet the fast path must expose a@@ -220,7 +244,7 @@ def test_resolved_release_manifest_404_falls_back(monkeypatch):None,)_stub_downloads(monkeypatch, _sha_payload(), not_found)- assert ILP._download_host_resolved_release(FORK_REPO) is None+ assert ILP._download_host_resolved_release(FORK_REPO, RELEASE_TAG) is None# --- _download_host_latest_release_tag (redirect resolution) -------------------diff --git a/tests/studio/install/test_install_llama_prebuilt_logic.py b/tests/studio/install/test_install_llama_prebuilt_logic.pyindex f2fc4e73762..46bd9544cab 100644--- a/tests/studio/install/test_install_llama_prebuilt_logic.py+++ b/tests/studio/install/test_install_llama_prebuilt_logic.py@@ -5598,6 +5598,70 @@ def test_release_listing_failure_exits_fallback_not_error(tmp_path, monkeypatch,assert caught.value.code == INSTALL_LLAMA_PREBUILT.EXIT_FALLBACK+def test_multiline_fallback_reason_logs_one_prefixed_line_each(tmp_path, monkeypatch, capsys):+ """Every line of the reason has to carry the component prefix.++ The preflight failure lists one binary per line, and the unprefixed system+ report follows immediately, so a reader (the Studio updater) can only tell+ the reason's continuation lines from the report by that prefix.+ """++ def boom(*args, **kwargs):+ raise INSTALL_LLAMA_PREBUILT.PrebuiltFallback(+ "linux extracted binary preflight failed:\n"+ "llama-server: missing=libcuda.so.1 ld_library_path=none\n"+ "llama-quantize: missing=libgomp.so.1 ld_library_path=none"+ )++ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "_LOG_TO_STDOUT", True)+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "_fork_manifest_release_plans", boom)+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "detect_host", linux_host)+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "collect_system_report", lambda *a, **k: "report")++ install_dir = tmp_path / "llama.cpp"+ install_dir.mkdir()+ with pytest.raises(SystemExit):+ install_prebuilt(install_dir, "latest", "unslothai/llama.cpp", "")++ reason = [+ line+ for line in capsys.readouterr().out.splitlines()+ if "preflight failed" in line or "missing=" in line+ ]+ assert len(reason) == 3+ assert all(line.startswith("[llama-prebuilt] ") for line in reason)++[email protected]("status", [403, 429])+def test_github_api_rate_limit_status_carries_the_token_hint(monkeypatch, status):+ """GitHub answers an exceeded rate limit with 403 or 429, and the raw+ "HTTP Error 429: Too Many Requests" says nothing a user can act on."""+ url = "https://api.github.com/repos/unslothai/llama.cpp/releases/tags/b10679-mix-67dfc8b"++ def raise_status(_url, **_kw):+ raise urllib.error.HTTPError(_url, status, "rate limited", {}, io.BytesIO(b""))++ monkeypatch.delenv("GH_TOKEN", raising = False)+ monkeypatch.delenv("GITHUB_TOKEN", raising = False)+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "download_bytes", raise_status)+ with pytest.raises(RuntimeError) as caught:+ INSTALL_LLAMA_PREBUILT.fetch_json(url)+ assert f"GitHub API returned {status}" in str(caught.value)+ assert "GH_TOKEN" in str(caught.value)+++def test_non_github_rate_limit_status_is_not_rewritten(monkeypatch):+ """huggingface.co has its own 429; it must not collect GitHub token advice."""+ url = "https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K.gguf"++ def raise_429(_url, **_kw):+ raise urllib.error.HTTPError(_url, 429, "Too Many Requests", {}, io.BytesIO(b""))++ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "download_bytes", raise_429)+ with pytest.raises(urllib.error.HTTPError):+ INSTALL_LLAMA_PREBUILT.fetch_json(url)++_SHARED_PAYLOAD = {"linux": ["libllama-common.so",