Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop the macOS update re-probing binaries and resolving 16 releases (#11237)

oobaboogadf0d65b805f5
diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py
index b548a9b71e1..ac247ffd9cd 100644
--- a/studio/install_llama_prebuilt.py
+++ b/studio/install_llama_prebuilt.py
@@ -34,6 +34,7 @@ import urllib.error
import urllib.parse
import urllib.request
import zipfile
+from collections.abc import Sequence
from contextlib import contextmanager
from dataclasses import dataclass, field, replace as dataclasses_replace
@@ -1359,7 +1360,7 @@ def resolve_simple_install_release_plans(
published_release_tag: str,
*,
max_release_fallbacks: int = DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS,
-) -> tuple[str, list[InstallReleasePlan]]:
+) -> "tuple[str, Sequence[InstallReleasePlan]]":
repo = published_repo or DEFAULT_PUBLISHED_REPO
# The fork (unslothai) ships a manifest describing every bundle's GPU/arch
# coverage, so all fork hosts select from it. Upstream (ggml-org) ships no
@@ -2264,23 +2265,23 @@ def iter_resolved_published_releases(
published_release_tag: str = "",
*,
allow_download_host_fast_path: bool = True,
+ continue_after_fast_path: bool = False,
) -> Iterable[ResolvedPublishedRelease]:
repo = published_repo or DEFAULT_PUBLISHED_REPO
normalized_requested = normalized_requested_llama_tag(requested_tag)
# Fast path: resolve a pinned or latest fork release from the download host (no
- # api.github.com rate limit). Latest surfaces only the single newest release, so
- # the caller disables it when the multi-release walk-back is needed (macOS
- # skipping too-new prebuilts); a broken latest then drops to source build, not
- # an older release. Pinned tags use the same CDN path so in-app updates avoid
- # the API entirely (#9970). Any rejection/network error is non-fatal and falls
- # through to the API.
+ # api.github.com rate limit). Latest surfaces only the newest release: macOS walks
+ # back past too-new prebuilts via continue_after_fast_path, otherwise a broken
+ # latest drops to source build. Pinned tags use the same CDN path so in-app updates
+ # avoid the API entirely (#9970). Any rejection/network error falls through to the API.
fast_path_tag: str | None = None
if published_release_tag:
fast_path_tag = published_release_tag
elif normalized_requested == "latest":
fast_path_tag = ""
+ fast_path_release_tag: str | None = None
if (
fast_path_tag is not None
and allow_download_host_fast_path
@@ -2308,7 +2309,13 @@ def iter_resolved_published_releases(
f"{resolved.bundle.upstream_tag}, but requested {normalized_requested}"
)
yield resolved
- return
+ # A pin names exactly one release, so there is nothing to walk back to and
+ # continue_after_fast_path has nothing to add. Falling through would re-resolve
+ # the same tag below -- the same release twice, and an api.github.com call to
+ # do it -- so a pin returns here whatever the walk-back flag says.
+ if not continue_after_fast_path or published_release_tag:
+ return
+ fast_path_release_tag = resolved.bundle.release_tag
if published_release_tag:
bundle = pinned_published_release_bundle(repo, published_release_tag)
@@ -2324,12 +2331,15 @@ def iter_resolved_published_releases(
)
return
- matched_any = False
+ matched_any = fast_path_release_tag is not None
skipped_invalid = 0
- yielded_valid = False
+ yielded_valid = fast_path_release_tag is not None
for bundle in iter_published_release_bundles(repo):
if not published_release_matches_request(bundle, normalized_requested):
continue
+ if fast_path_release_tag is not None and bundle.release_tag == fast_path_release_tag:
+ # Already yielded from the download host.
+ continue
matched_any = True
try:
checksums = validated_checksums_for_bundle(repo, bundle)
@@ -5754,10 +5764,18 @@ def macos_binary_minos_issues(
def macos_dyld_load_issues(
- binaries: Iterable[Path], install_dir: Path, host: HostInfo
+ binaries: Iterable[Path],
+ install_dir: Path,
+ host: HostInfo,
+ *,
+ loaded: "set[str] | None" = None,
) -> list[str]:
"""Issue strings for every installed executable dyld refuses to load.
+ *loaded*, when given, collects the binaries dyld actually resolved: this probe
+ fails OPEN, so "no issues" alone cannot tell a clean load from one that never
+ ran. Only a name in here is evidence of a load.
+
`--version` costs a process spawn and still makes dyld resolve the whole link
graph, so it catches a missing dylib, a missing symbol or a too-new slice --
unlike validate_server, which loads a model and is off for checksummed bundles
@@ -5786,6 +5804,8 @@ def macos_dyld_load_issues(
log(f"macos load probe could not run {binary_path.name}: {exc}")
continue
if result.returncode == 0:
+ if loaded is not None:
+ loaded.add(binary_path.name)
continue
output = (result.stdout + result.stderr).strip()
# A non-zero exit is not evidence of a bad link: llama-quantize answers
@@ -5795,6 +5815,10 @@ def macos_dyld_load_issues(
# that asked, which otool -L cannot, since an absolute install name absent
# from disk is the normal case for /usr/lib's shared-cache members.
if not looks_like_macos_loader_failure(output):
+ # llama-quantize answers --version by printing its table and exiting 1;
+ # dyld still resolved the image to let it print anything at all.
+ if loaded is not None:
+ loaded.add(binary_path.name)
continue
detail = " | ".join(output.splitlines()[-5:]) or f"exit {result.returncode}"
issues.append(f"{binary_path.name}: {detail}")
@@ -5802,13 +5826,23 @@ def macos_dyld_load_issues(
def preflight_macos_installed_binaries(
- binaries: Iterable[Path], install_dir: Path, host: HostInfo
-) -> None:
- """Reject a macos prebuilt whose minimum-OS is newer than the host, or that
- dyld will not load at all. The upstream selector pins a loadable release up
+ binaries: Iterable[Path],
+ install_dir: Path,
+ host: HostInfo,
+ *,
+ load_probe: bool = True,
+) -> bool:
+ """Reject a macos prebuilt whose minimum-OS is newer than the host, or (with
+ *load_probe*) that dyld will not load at all. The upstream selector pins a loadable release up
front, so here this is the post-download backstop; the published/fork path
also uses it to advance the walk-back.
+ Returns True only when the load probe actually ran and dyld resolved every
+ binary. False means "no evidence", not "bad": the probe is skipped, or it
+ timed out or could not spawn. Callers record the skip only on True, so an
+ inconclusive probe costs another probe next time instead of being cached as
+ a pass.
+
The load probe is the macOS counterpart of preflight_linux_installed_binaries'
ldd sweep, which this side went without: a bundle whose libggml-rpc.0.dylib
links a /usr/lib/librdma.dylib that exists on the builder and nowhere else
@@ -5820,18 +5854,30 @@ def preflight_macos_installed_binaries(
both on an unparseable ``platform.mac_ver()`` left that host with no check at
all, since the runtime validation it deferred to is off by default (#5854)."""
if not host.is_macos:
- return
+ return False
+ binaries = list(binaries)
if host.macos_version is not None:
issues = macos_binary_minos_issues(binaries, install_dir, host)
if issues:
raise PrebuiltFallback(
"macos prebuilt requires a newer macOS than this host:\n" + "\n".join(issues)
)
- load_issues = macos_dyld_load_issues(binaries, install_dir, host)
+ if not load_probe:
+ log("installed binaries match their recorded digests; skipping the dyld load probe")
+ return False
+ loaded: set[str] = set()
+ load_issues = macos_dyld_load_issues(binaries, install_dir, host, loaded = loaded)
if load_issues:
raise PrebuiltFallback(
"macos prebuilt does not load on this host:\n" + "\n".join(load_issues)
)
+ expected = {path.name for path in binaries if path.is_file()}
+ if not expected or not expected.issubset(loaded):
+ # The probe fails open so a loaded machine cannot reject a healthy bundle;
+ # that is right for INSTALLING, and wrong to remember as a load.
+ log("macos load probe did not run for every binary; not recording it as a pass")
+ return False
+ return True
def preflight_linux_installed_binaries(
@@ -6849,6 +6895,85 @@ def _linux_published_attempts(host: HostInfo, bundle: PublishedReleaseBundle) ->
return attempts
+RELEASE_LISTING_TRANSPORT_ERRORS = (
+ urllib.error.URLError,
+ ssl.SSLError,
+ ConnectionError,
+ TimeoutError,
+ RuntimeError,
+ ValueError,
+)
+
+
+def release_listing_fallback(exc: BaseException, published_repo: str) -> PrebuiltFallback:
+ # Install time only: a resolver probe must fail hard, or update_flow caches "no prebuilt".
+ return PrebuiltFallback(
+ f"failed to inspect published releases in "
+ f"{published_repo or DEFAULT_PUBLISHED_REPO}: {exc}"
+ )
+
+
+def iter_release_plans(
+ plans: "Sequence[InstallReleasePlan]", published_repo: str
+) -> "Iterator[InstallReleasePlan]":
+ try:
+ yield from plans
+ except (BusyInstallConflict, PrebuiltFallback):
+ raise
+ except RELEASE_LISTING_TRANSPORT_ERRORS as exc:
+ raise release_listing_fallback(exc, published_repo) from exc
+
+
+class LazyReleasePlans(Sequence):
+ """Release plans resolved on first access; ``len()`` resolves them all."""
+
+ def __init__(self, plans: "Iterable[InstallReleasePlan]") -> None:
+ self._source: "Iterator[InstallReleasePlan] | None" = iter(plans)
+ self._resolved: list[InstallReleasePlan] = []
+ self._failure: "BaseException | None" = None
+
+ def _resolve_through(self, index: int) -> bool:
+ while len(self._resolved) <= index:
+ # A generator that raised is closed, so without remembering why, a second
+ # traversal would report a short-but-clean sequence and len() would answer
+ # differently than the call that raised. Same inputs, same answer, every time.
+ if self._failure is not None:
+ raise self._failure
+ if self._source is None:
+ return False
+ try:
+ self._resolved.append(next(self._source))
+ except StopIteration:
+ self._source = None
+ return False
+ except BaseException as exc:
+ self._source = None
+ self._failure = exc
+ raise
+ return True
+
+ def __getitem__(self, index):
+ if isinstance(index, slice) or index < 0:
+ self._resolve_through(sys.maxsize)
+ return self._resolved[index]
+ if not self._resolve_through(index):
+ raise IndexError(index)
+ return self._resolved[index]
+
+ def __len__(self) -> int:
+ self._resolve_through(sys.maxsize)
+ return len(self._resolved)
+
+ def __iter__(self):
+ index = 0
+ while self._resolve_through(index):
+ yield self._resolved[index]
+ index += 1
+
+ def __bool__(self) -> bool:
+ return self._resolve_through(0)
+
+
def _fork_manifest_release_plans(
llama_tag: str,
host: HostInfo,
@@ -6856,7 +6981,7 @@ def _fork_manifest_release_plans(
published_release_tag: str,
*,
max_release_fallbacks: int = DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS,
-) -> tuple[str, list[InstallReleasePlan]]:
+) -> "tuple[str, Sequence[InstallReleasePlan]]":
"""Manifest-reading branch of resolve_simple_install_release_plans, used for
every fork host: all of the fork's bundles describe their GPU/arch coverage
in llama-prebuilt-manifest.json rather than in the asset filename (CPU,
@@ -6868,74 +6993,82 @@ def _fork_manifest_release_plans(
# version is known; otherwise keep the default (cannot tell up front).
if host.is_macos and allow_older_release_fallback and host.macos_version is not None:
release_limit = max(release_limit, DEFAULT_MAX_MACOS_RELEASE_FALLBACKS)
- plans: list[InstallReleasePlan] = []
- last_error: PrebuiltFallback | None = None
- # The newest release this host could not take, if the first plan is an older one.
- skipped_newest: str | None = None
- for resolved_release in iter_resolved_published_releases(
- llama_tag,
- published_repo,
- published_release_tag,
- # macOS relies on the multi-release walk-back to skip too-new prebuilts,
- # which the single-latest download-host path cannot provide.
- allow_download_host_fast_path = not host.is_macos,
- ):
- bundle = resolved_release.bundle
- checksums = resolved_release.checksums
- resolved_tag = bundle.upstream_tag
- try:
- if host.is_linux:
- linux_attempts = _linux_published_attempts(host, bundle)
- if not linux_attempts:
- raise PrebuiltFallback("no compatible Linux prebuilt asset was found")
- attempts = apply_approved_hashes(linux_attempts, checksums)
- if not attempts:
- raise PrebuiltFallback("no compatible Linux prebuilt asset was found")
- if attempts[0].selection_log:
- log_lines(attempts[0].selection_log)
- else:
- attempts = resolve_release_asset_choice(
- host,
- resolved_tag,
- bundle,
- checksums,
+ def _plans() -> "Iterator[InstallReleasePlan]":
+ resolved_count = 0
+ last_error: PrebuiltFallback | None = None
+ # The newest release this host could not take, if the first plan is an older one.
+ skipped_newest: str | None = None
+
+ for resolved_release in iter_resolved_published_releases(
+ llama_tag,
+ published_repo,
+ published_release_tag,
+ # macOS takes the newest release from the CDN and walks back through the API.
+ continue_after_fast_path = host.is_macos,
+ ):
+ bundle = resolved_release.bundle
+ checksums = resolved_release.checksums
+ resolved_tag = bundle.upstream_tag
+ try:
+ if host.is_linux:
+ linux_attempts = _linux_published_attempts(host, bundle)
+ if not linux_attempts:
+ raise PrebuiltFallback("no compatible Linux prebuilt asset was found")
+ attempts = apply_approved_hashes(linux_attempts, checksums)
+ if not attempts:
+ raise PrebuiltFallback("no compatible Linux prebuilt asset was found")
+ if attempts[0].selection_log:
+ log_lines(attempts[0].selection_log)
+ else:
+ attempts = resolve_release_asset_choice(
+ host,
+ resolved_tag,
+ bundle,
+ checksums,
+ )
+ if not attempts:
+ raise PrebuiltFallback("no compatible prebuilt asset was found")
+ if attempts[0].selection_log:
+ log_lines(attempts[0].selection_log)
+ except PrebuiltFallback as exc:
+ last_error = exc
+ if not allow_older_release_fallback:
+ raise
+ log(
+ "published release skipped for install planning: "
+ f"{bundle.repo}@{bundle.release_tag} upstream_tag={resolved_tag} ({exc})"
)
- if not attempts:
- raise PrebuiltFallback("no compatible prebuilt asset was found")
- if attempts[0].selection_log:
- log_lines(attempts[0].selection_log)
- except PrebuiltFallback as exc:
- last_error = exc
- if not allow_older_release_fallback:
- raise
- log(
- "published release skipped for install planning: "
- f"{bundle.repo}@{bundle.release_tag} upstream_tag={resolved_tag} ({exc})"
- )
- if not plans and skipped_newest is None:
- skipped_newest = bundle.release_tag
- continue
+ if resolved_count == 0 and skipped_newest is None:
+ skipped_newest = bundle.release_tag
+ continue
- plans.append(
- InstallReleasePlan(
+ yield InstallReleasePlan(
requested_tag = requested_tag,
llama_tag = resolved_tag,
release_tag = bundle.release_tag,
attempts = attempts,
approved_checksums = checksums,
- walk_back = _core.walk_back_for(host, skipped_newest) if not plans else None,
+ walk_back = (
+ _core.walk_back_for(host, skipped_newest) if resolved_count == 0 else None
+ ),
)
- )
+ resolved_count += 1
- if not allow_older_release_fallback or len(plans) >= release_limit:
- break
+ if not allow_older_release_fallback or resolved_count >= release_limit:
+ return
- if plans:
- return requested_tag, plans
- if last_error is not None:
- raise last_error
- raise PrebuiltFallback("no installable published llama.cpp releases were found")
+ if resolved_count:
+ return
+ if last_error is not None:
+ raise last_error
+ raise PrebuiltFallback("no installable published llama.cpp releases were found")
+
+ plans = LazyReleasePlans(_plans())
+ # Resolve the first plan here, so its listing failures still raise from the resolver.
+ if not plans:
+ raise PrebuiltFallback("no installable published llama.cpp releases were found")
+ return requested_tag, plans
def persisted_llama_backend(llama_backend: str | None, choice: AssetChoice) -> str | None:
@@ -7052,6 +7185,7 @@ def write_prebuilt_metadata(
backend_request: str | None = None,
rocm_gfx: str | None = None,
walk_back: "_core.WalkBack | None" = None,
+ macos_load_probe_passed: bool = False,
) -> None:
source_asset_name, source_sha256 = selected_source_archive_metadata(
approved_checksums,
@@ -7071,6 +7205,9 @@ def write_prebuilt_metadata(
if fingerprint is None:
raise PrebuiltFallback(f"cannot compute install fingerprint for {choice.name}")
_persisted_backend = persisted_llama_backend(llama_backend, choice)
+ _load_probe = (
+ macos_load_probe_record(host) if macos_load_probe_passed and host is not None else None
+ )
# An install kind with no allowlist raises; the binary tier alone is still honest evidence.
try:
_runtime_patterns: list[str] | None = runtime_patterns_for_choice(choice)
@@ -7141,6 +7278,10 @@ def write_prebuilt_metadata(
"runtime_files": runtime_file_records(install_dir, host, _runtime_patterns),
# The box this bundle was chosen for. Absent reads as "cannot say": full path.
**({"host_profile": host_profile(host)} if host is not None else {}),
+ # Evidence that dyld really loaded these bytes on this macOS build, which is what
+ # lets a later update skip the probe. Absent reads as "cannot say": probe again.
+ # Not fingerprinted, so it never invalidates an install.
+ **({MACOS_LOAD_PROBE_KEY: _load_probe} if _load_probe is not None else {}),
"prebuilt_fallback_used": prebuilt_fallback_used,
"installed_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
}
@@ -7703,14 +7844,18 @@ def runtime_file_records(
libraries are where most of a CUDA bundle's bytes live, which is exactly the
shape a full disk or an interrupted extract leaves behind. A size comparison
catches it for the price of a stat; hashing 300 MB of kernels on every update
- would not be worth it.
+ would not be worth it. macOS hashes both tiers, since its record stands in for the
+ dyld probe and a Metal bundle is tens of MB.
*patterns* comes from runtime_patterns_for_install_kind for the bundle being
installed; without it only the binary tier is recorded, which is what the callers
that have no bundle in hand (a bare re-record) can honestly say.
+
+ Call only after the platform preflights pass: the macOS probe skip trusts the record.
"""
records: dict[str, dict[str, Any]] = {}
runtime_dir = install_runtime_dir(install_dir, host) if host is not None else None
+ hash_payload = host is not None and host.is_macos
if patterns and runtime_dir is not None:
for pattern in patterns:
try:
@@ -7725,6 +7870,15 @@ def runtime_file_records(
relative = candidate.relative_to(install_dir).as_posix()
except ValueError:
continue
+ if hash_payload:
+ try:
+ record["sha256"] = sha256_file(candidate)
+ except (OSError, MemoryError) as exc:
+ log(
+ f"could not hash {relative} for the runtime record ({exc}); "
+ "not recording"
+ )
+ return {}
records[relative] = record
# Last, so a binary the sweep matched is upgraded to the hashed tier.
for candidate in _runtime_record_paths(install_dir, host):
@@ -7746,6 +7900,137 @@ def runtime_file_records(
return records
+MACOS_LOAD_PROBE_KEY = "macos_load_probe"
+
+
+def macos_product_version() -> str:
+ """The macOS identity the dyld shared cache actually tracks, as one string.
+
+ host_profile records parse_macos_version, which is (major, minor) by design --
+ right for the minos comparison, wrong for "can this image still load": since Big
+ Sur the system libraries are a dyld shared cache blob, and Apple replaces that
+ blob in point releases and in Rapid Security Responses.
+
+ A Rapid Security Response does NOT move ProductVersion, which is all
+ platform.mac_ver() reads: 13.3.1 (a) reports ProductVersion 13.3.1 with the "(a)"
+ in ProductVersionExtra and the build at 22E772610a instead of 22E261. Reading
+ SystemVersion.plist has the same blind spot and is additionally stale for an RSR
+ (osquery/osquery#8008), so ask sw_vers -- bare, since it documents single-dash
+ options and a misspelt flag would silently cost the field this exists to read.
+ """
+ try:
+ done = subprocess.run(
+ ["/usr/bin/sw_vers"],
+ capture_output = True,
+ text = True,
+ timeout = 10,
+ check = False,
+ )
+ if done.returncode == 0:
+ values: "dict[str, str]" = {}
+ for line in done.stdout.splitlines():
+ key, sep, value = line.partition(":")
+ if sep:
+ values[key.strip()] = value.strip()
+ # Absent ProductVersionExtra means no RSR, and predates macOS 13; not a failure.
+ fields = [
+ values.get(key, "")
+ for key in ("ProductVersion", "ProductVersionExtra", "BuildVersion")
+ ]
+ if fields[0]:
+ return " ".join(f for f in fields if f)
+ except Exception: # noqa: BLE001
+ pass
+ try:
+ # sw_vers unavailable: the product version alone still catches a point release.
+ return str(platform.mac_ver()[0] or "").strip()
+ except Exception: # noqa: BLE001
+ return ""
+
+
+def macos_load_probe_record(host: HostInfo) -> "dict[str, Any] | None":
+ """The evidence a completed dyld probe leaves for later runs, or None when the
+ host cannot describe itself finely enough to be worth recording."""
+ version = macos_product_version()
+ if not host.is_macos or not version:
+ return None
+ return {"passed": True, "macos_product_version": version}
+
+
+def persist_macos_load_probe(install_dir: Path, host: HostInfo) -> bool:
+ """Remember a reuse-path probe that just passed, so the next update can skip it.
+
+ Without this the skip is unreachable for everyone it was not born with: a marker
+ written before the record existed, an install whose first probe timed out, and
+ every install after a macOS point release all probe again on EVERY later update
+ and never stop, since nothing writes the evidence outside a fresh install.
+
+ Re-recording the runtime files is safe HERE and nowhere else: the minos check, the
+ digests the marker did carry and dyld itself all passed on these exact bytes
+ moments ago, which is the same evidence the install path records on.
+
+ The record covers the ROOT copies too, so only a caller that has started every one
+ of them may write it. That is _existing_install_runs alone; the two reuse fast paths
+ probe build/bin and never reach a root wrapper, which can be a separate file rather
+ than a symlink.
+ """
+ record = macos_load_probe_record(host)
+ if record is None:
+ return False
+ try:
+ marker = load_prebuilt_metadata(install_dir)
+ if not isinstance(marker, dict) or not marker:
+ return False
+ patch: "dict[str, Any]" = {MACOS_LOAD_PROBE_KEY: record}
+ recorded = marker.get("runtime_files")
+ if not (
+ isinstance(recorded, dict)
+ and recorded
+ and all(isinstance(e, dict) and e.get("sha256") for e in recorded.values())
+ ):
+ # Both macOS kinds share one pattern list, so the host answers this without
+ # a plan; an unreadable file yields {} and no evidence rather than a half record.
+ kind = "macos-arm64" if str(host.machine or "").lower() == "arm64" else "macos-x64"
+ records = runtime_file_records(
+ install_dir, host, runtime_patterns_for_install_kind(kind)
+ )
+ if not records:
+ return False
+ patch["runtime_files"] = records
+ marker.update(patch)
+ return _write_marker(install_dir / "UNSLOTH_PREBUILT_INFO.json", marker)
+ except Exception: # noqa: BLE001
+ # Best effort: a marker we could not rewrite costs another probe, never the reuse.
+ return False
+
+
+def _macos_load_record_is_current(marker: "dict[str, Any] | None", host: HostInfo) -> bool:
+ """Whether the dyld probe these recorded bytes passed still holds; callers must also
+ check _runtime_files_match. A size-only entry misses a same-size rewrite, and an
+ unknown macOS version skips the minos check, leaving the probe as the only load check.
+
+ Requires POSITIVE evidence that a probe ran and passed (MACOS_LOAD_PROBE_KEY), on
+ this exact macOS product version. A marker without it -- every one written before
+ this existed, and every install whose probe timed out or could not spawn -- takes
+ the probe, which is what happened on every update before the skip existed.
+ """
+ if not host.is_macos or host.macos_version is None or prebuilt_full_check_requested():
+ return False
+ marker = marker or {}
+ recorded = marker.get("runtime_files")
+ if not isinstance(recorded, dict) or not recorded:
+ return False
+ if not all(isinstance(entry, dict) and entry.get("sha256") for entry in recorded.values()):
+ return False
+ probe = marker.get(MACOS_LOAD_PROBE_KEY)
+ if not isinstance(probe, dict) or probe.get("passed") is not True:
+ return False
+ version = macos_product_version()
+ if not version or probe.get("macos_product_version") != version:
+ return False
+ return marker.get("host_profile") == host_profile(host)
+
+
def _runtime_files_match(install_dir: Path, host: HostInfo, marker: "dict[str, Any]") -> bool:
"""Whether every recorded runtime file is still the file that was installed.
@@ -8041,11 +8326,10 @@ def _expected_release_tag_without_plan(
api.github.com call, so no rate limit, and no manifest or checksum download.
KNOWN AND ACCEPTED LAG, for "latest" only. The HEAD follows /releases/latest, which
- GitHub resolves by make_latest / created_at. The normal macOS path deliberately
- turns the download-host resolver off (allow_download_host_fast_path = not
- host.is_macos, so it can walk back past a prebuilt whose minimum OS is too new) and
- orders releases by published_at instead. The two disagree when a release is created
- before, but published after, another -- e.g. a re-published or back-dated release.
+ GitHub resolves by make_latest / created_at. _select reads the same pointer through
+ the download host, but falls back to the API listing, ordered by published_at, when
+ that host is unavailable. The two disagree when a release is created before, but
+ published after, another -- e.g. a re-published or back-dated release.
In that window this check can report "current" for an install _select would have
moved off. The consequence is one deferred update, never a wrong install: the very
next run whose pointer has caught up does the move, and every OTHER guard here
@@ -8298,16 +8582,19 @@ def existing_install_current_without_plan(
]
if not all(os.access(binary, os.X_OK) for binary in binaries):
return False
+ # (6) the bytes are the ones installed; before the preflight, whose probe skip trusts them.
+ if not _runtime_files_match(install_dir, host, marker):
+ return False
try:
- # Kept, unlike the --version spawns: a preflight answers whether the OS can LOAD the image,
- # which a hash cannot.
preflight_linux_installed_binaries(binaries, install_dir, host)
- preflight_macos_installed_binaries(binaries, install_dir, host)
+ preflight_macos_installed_binaries(
+ binaries,
+ install_dir,
+ host,
+ load_probe = not _macos_load_record_is_current(marker, host),
+ )
except Exception: # noqa: BLE001
return False
- # (6) and the bytes are the ones that were installed.
- if not _runtime_files_match(install_dir, host, marker):
- return False
# The one backfill that is not a release change, so it has to be asked separately.
if _diffusion_visual_server_missing_for_marker(install_dir, host, marker):
return False
@@ -8475,7 +8762,8 @@ def _existing_install_runs(install_dir: Path, host: HostInfo) -> bool:
return False
if not _kept_install_payload_is_healthy(install_dir, host):
return False
- recorded_runtime_line = (load_prebuilt_metadata(install_dir) or {}).get("runtime_line")
+ marker = load_prebuilt_metadata(install_dir) or {}
+ recorded_runtime_line = marker.get("runtime_line")
if not isinstance(recorded_runtime_line, str):
recorded_runtime_line = None
runtime_dir = install_runtime_dir(install_dir, host)
@@ -8483,12 +8771,20 @@ def _existing_install_runs(install_dir: Path, host: HostInfo) -> bool:
binaries = [runtime_dir / f"llama-{name}{ext}" for name in ("server", "quantize")]
if not all(os.access(binary, os.X_OK) for binary in binaries):
return False
+ # A current macOS load record also replaces the `--version` probes below.
+ recorded_bytes_intact = _macos_load_record_is_current(marker, host) and _runtime_files_match(
+ install_dir, host, marker
+ )
try:
# Each preflight is a no-op outside its platform.
preflight_linux_installed_binaries(binaries, install_dir, host)
- preflight_macos_installed_binaries(binaries, install_dir, host)
+ probed = preflight_macos_installed_binaries(
+ binaries, install_dir, host, load_probe = not recorded_bytes_intact
+ )
except Exception:
return False
+ if recorded_bytes_intact:
+ return True
# Root copies first: _find_llama_server_binary reaches them first, and without a
# symlink they can rot alone.
probes: list[Path] = []
@@ -8504,9 +8800,16 @@ def _existing_install_runs(install_dir: Path, host: HostInfo) -> bool:
continue
seen.add(key)
probes.append(binary)
- return all(
+ if not all(
_binary_image_runs(binary, install_dir, host, recorded_runtime_line) for binary in probes
- )
+ ):
+ return False
+ # Last, not straight after the dyld probe: recording before the root entrypoints ran
+ # would bless a corrupt wrapper into the record, and the next run would read the
+ # record as current, return above, and never start the wrapper either.
+ if probed:
+ persist_macos_load_probe(install_dir, host)
+ return True
def existing_install_matches_choice(
@@ -8545,9 +8848,11 @@ def existing_install_matches_choice(
# Windows has no image-reading preflight, so a truncated llama-server.exe kept its fingerprint
# match. Only a marker carrying the record is held to it.
recorded_files = metadata.get("runtime_files")
+ runtime_files_matched = False
if isinstance(recorded_files, dict) and recorded_files:
if not _runtime_files_match(install_dir, host, metadata):
return False
+ runtime_files_matched = True
elif not (host.is_linux or host.is_macos):
# The migration run, on the one platform with no loader preflight below: a pre-record
# marker has no digest, so damaged bytes would become the reference they are later
@@ -8583,6 +8888,9 @@ def existing_install_matches_choice(
[runtime_dir / "llama-server", runtime_dir / "llama-quantize"],
install_dir,
host,
+ load_probe = not (
+ runtime_files_matched and _macos_load_record_is_current(metadata, host)
+ ),
)
except Exception:
return False
@@ -8690,7 +8998,9 @@ def validate_prebuilt_choice(
f"{choice.install_kind} bundle {choice.name} omitted a required runtime component"
)
preflight_linux_installed_binaries((server_path, quantize_path), install_dir, host)
- preflight_macos_installed_binaries((server_path, quantize_path), install_dir, host)
+ macos_load_probe_passed = preflight_macos_installed_binaries(
+ (server_path, quantize_path), install_dir, host
+ )
ensure_repo_shape(install_dir)
write_prebuilt_metadata(
install_dir,
@@ -8706,6 +9016,7 @@ def validate_prebuilt_choice(
backend_request = backend_request,
rocm_gfx = rocm_gfx,
walk_back = walk_back,
+ macos_load_probe_passed = macos_load_probe_passed,
)
# Hashless external prebuilts are not in the approved-sha256
# manifest and rely on the functional smoke test as their only integrity gate,
@@ -9791,7 +10102,7 @@ class BackendSelection:
published_repo: str
published_release_tag: str
requested_tag: str
- release_plans: list[InstallReleasePlan]
+ release_plans: "Sequence[InstallReleasePlan]"
persist_llama_backend: str | None
persist_rocm_gfx: str | None
@@ -9884,7 +10195,7 @@ def route_backend_request(
def _with_rocm_behind_vulkan(
- plans: list[InstallReleasePlan],
+ plans: "Sequence[InstallReleasePlan]",
llama_tag: str,
rocm_host: HostInfo,
published_repo: str,
@@ -9913,7 +10224,7 @@ def _with_rocm_behind_vulkan(
return plan
return dataclasses_replace(plan, attempts = attempts) if attempts else None
- def _drop_cpu_only(candidates: list[InstallReleasePlan]) -> list[InstallReleasePlan]:
+ def _drop_cpu_only(candidates: "Sequence[InstallReleasePlan]") -> list[InstallReleasePlan]:
"""Apply _without_cpu across releases, or fail rather than install CPU."""
kept = [narrowed for plan in candidates if (narrowed := _without_cpu(plan)) is not None]
if candidates and not kept:
@@ -9924,10 +10235,11 @@ def _with_rocm_behind_vulkan(
_tag, rocm_plans = resolve_simple_install_release_plans(
llama_tag, rocm_host, published_repo, published_release_tag
)
+ # Inside the try: later releases resolve lazily here.
+ rocm_attempts = {plan.release_tag: plan.attempts for plan in rocm_plans}
except Exception:
# Not a reason to fail the Vulkan install, but a reason to drop the CPU tail.
return _drop_cpu_only(plans)
- rocm_attempts = {plan.release_tag: plan.attempts for plan in rocm_plans}
out: list[InstallReleasePlan] = []
for plan in plans:
present = {attempt.install_kind for attempt in plan.attempts}
@@ -10085,10 +10397,10 @@ def install_prebuilt(
# transient 403 as "no prebuilt" for 24h; nonzero exits are never
# cached, so the resolver must keep failing hard.
#
- # Not dead code despite the download-host fast path: macOS skips it
- # entirely (see allow_download_host_fast_path below), as does a
- # non-latest requested tag without a published-release pin, a
- # non-default --published-repo, and any CDN outage.
+ # Not dead code despite the download-host fast path: a macOS walk-back
+ # past the newest release lists the API, as does a non-latest
+ # requested tag without a published-release pin, a non-default
+ # --published-repo, and any CDN outage.
#
# Transport shapes only: URLError covers HTTPError and the socket/DNS
# errors urllib wraps, JSONDecodeError is a ValueError, and
@@ -10126,18 +10438,8 @@ def install_prebuilt(
)
except (BusyInstallConflict, PrebuiltFallback):
raise
- except (
- urllib.error.URLError,
- ssl.SSLError,
- ConnectionError,
- TimeoutError,
- RuntimeError,
- ValueError,
- ) as exc:
- raise PrebuiltFallback(
- f"failed to inspect published releases in "
- f"{published_repo or DEFAULT_PUBLISHED_REPO}: {exc}"
- ) from exc
+ except RELEASE_LISTING_TRANSPORT_ERRORS as exc:
+ raise release_listing_fallback(exc, published_repo) from exc
# Before the listing, manifest and checksum fetches, which on a current install resolve
# back to the bundle on disk. One HEAD instead. Routed once and handed to both, since
@@ -10226,20 +10528,11 @@ def install_prebuilt(
work_dir / "stories260K.gguf",
validation_model_cache_path(install_dir),
)
- # Same reason as the per-candidate guard in validate_prebuilt_attempts,
- # one level up: the per-release handler below also swallows
- # PrebuiltFallback and moves to an older plan, so a probe failure raised
- # inside it would install an older llama.cpp over a transient 429. The
- # probe is independent of which release was picked, so resolve it once
- # here when any plan will smoke-test.
- if staged_validation_enabled() or any(
- attempt.expected_sha256 is None
- for release_plan in release_plans
- for attempt in release_plan.attempts
+ probe_resolved = False
+ last_failure: PrebuiltFallback | None = None
+ for release_index, plan in enumerate(
+ iter_release_plans(release_plans, published_repo)
):
- probe = resolve_validation_model(probe)
- release_count = len(release_plans)
- for release_index, plan in enumerate(release_plans):
choice = plan.attempts[0]
backfill = diffusion_visual_server_backfill_needed(install_dir, host, choice)
if existing_install_matches_plan(install_dir, host, plan):
@@ -10261,6 +10554,13 @@ def install_prebuilt(
f"{choice.name} ({choice.source_label}) from published release "
f"{plan.release_tag} for {host.system} {host.machine}"
)
+ # Outside the handler, so a transient failure cannot demote to an older release.
+ if not probe_resolved and (
+ staged_validation_enabled()
+ or any(attempt.expected_sha256 is None for attempt in plan.attempts)
+ ):
+ probe = resolve_validation_model(probe)
+ probe_resolved = True
try:
choice, selected_staging_dir, _ = validate_prebuilt_attempts(
plan.attempts,
@@ -10290,12 +10590,11 @@ def install_prebuilt(
except PrebuiltFallback as exc:
if _environment_fatal_reason(exc):
raise
- if release_index == release_count - 1:
- raise
+ last_failure = exc
log(
"published release "
f"{plan.release_tag} upstream_tag={plan.llama_tag} failed; "
- "trying the next older published prebuilt "
+ "trying an older published prebuilt if one remains "
f"({textwrap.shorten(str(exc), width = 200, placeholder = '...')})"
)
continue
@@ -10318,6 +10617,8 @@ def install_prebuilt(
f"({textwrap.shorten(str(exc), width = 200, placeholder = '...')})"
)
return
+ if last_failure is not None:
+ raise last_failure
except BusyInstallConflict as exc:
log("prebuilt install path is blocked by an in-use llama.cpp install")
log(f"prebuilt busy reason: {exc}")
diff --git a/tests/studio/install/test_download_host_resolve.py b/tests/studio/install/test_download_host_resolve.py
index 0c3ccb5d22b..ed72a3684e9 100644
--- a/tests/studio/install/test_download_host_resolve.py
+++ b/tests/studio/install/test_download_host_resolve.py
@@ -71,7 +71,6 @@ def test_fast_path_yields_pinned_tag_without_api(monkeypatch):
def test_fast_path_disabled_by_caller_uses_api(monkeypatch):
- # macOS passes allow_download_host_fast_path = False to keep the walk-back.
monkeypatch.delenv("UNSLOTH_LLAMA_DISABLE_DOWNLOAD_HOST_RESOLVE", raising = False)
monkeypatch.setattr(ILP, "_download_host_resolved_release", _fast_path_raises)
monkeypatch.setattr(ILP, "iter_published_release_bundles", _empty_api)
diff --git a/tests/studio/install/test_install_llama_prebuilt_logic.py b/tests/studio/install/test_install_llama_prebuilt_logic.py
index 9904d6b7bdd..0d69b451a95 100644
--- a/tests/studio/install/test_install_llama_prebuilt_logic.py
+++ b/tests/studio/install/test_install_llama_prebuilt_logic.py
@@ -7010,6 +7010,23 @@ def test_the_planner_records_the_newest_release_a_mac_walked_past(monkeypatch):
assert plans[0].walk_back is None
+def test_a_late_rocm_listing_failure_keeps_the_vulkan_plan(monkeypatch):
+ module = INSTALL_LLAMA_PREBUILT
+ vulkan = release_plan([asset_choice(install_kind = "linux-vulkan")], release_checksums())
+
+ def rocm_plans():
+ yield vulkan
+ raise urllib.error.URLError("CDN down, API timed out")
+
+ monkeypatch.setattr(
+ module,
+ "resolve_simple_install_release_plans",
+ lambda *args: ("latest", module.LazyReleasePlans(rocm_plans())),
+ )
+ kept = module._with_rocm_behind_vulkan([vulkan], "latest", linux_host(), "", "")
+ assert [plan.release_tag for plan in kept] == [vulkan.release_tag]
+
+
def test_a_reused_marker_takes_the_walk_back_this_run_made():
"""sync_marker_selection: a kept install on a Mac gains the walk-back record a
marker written before it existed lacks (both keys), and a plan that no longer
diff --git a/tests/studio/install/test_macos_loader_probe_record.py b/tests/studio/install/test_macos_loader_probe_record.py
new file mode 100644
index 00000000000..0cfee06835e
--- /dev/null
+++ b/tests/studio/install/test_macos_loader_probe_record.py
@@ -0,0 +1,574 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests for reusing a recorded macOS dyld probe.
+
+The probe may be skipped only when the runtime digests and host profile still match.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+TEST_DIR = Path(__file__).resolve().parent
+if str(TEST_DIR) not in sys.path:
+ sys.path.insert(0, str(TEST_DIR))
+
+from _pr10648_helpers import POSIX_ONLY, llama_host, load_studio_module # noqa: E402
+
+# Use an isolated module because tests monkeypatch its globals.
+ILP = load_studio_module(
+ "studio_install_llama_prebuilt_macos_loader_record", "install_llama_prebuilt.py"
+)
+
+RELEASE_TAG = "release-1"
+UPSTREAM_TAG = "b9001"
+PUBLISHED_REPO = "unslothai/llama.cpp"
+INSTALL_KIND = "macos-arm64"
+
+
+PRODUCT_VERSION = "15.5.1"
+# Captured before the autouse fixture stubs it, for the tests that exercise the real one.
+_REAL_MACOS_PRODUCT_VERSION = ILP.macos_product_version
+
+
[email protected](autouse = True)
+def _clear_full_check(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_PREBUILT_FULL_CHECK", raising = False)
+ # platform.mac_ver() is empty off macOS, so the probe record would never be written
+ # and every test here would read as "no evidence". Pin the host's own answer instead.
+ monkeypatch.setattr(ILP, "macos_product_version", lambda: PRODUCT_VERSION)
+
+
+def macos_host(**overrides):
+ return llama_host(
+ ILP.HostInfo,
+ system = "Darwin",
+ machine = "arm64",
+ macos_version = overrides.pop("macos_version", (15, 5)),
+ **overrides,
+ )
+
+
+def choice_for(host) -> "ILP.AssetChoice":
+ return ILP.AssetChoice(
+ repo = PUBLISHED_REPO,
+ tag = RELEASE_TAG,
+ name = f"llama-{UPSTREAM_TAG}-bin-{INSTALL_KIND}.tar.gz",
+ url = f"https://example.com/llama-{UPSTREAM_TAG}-bin-{INSTALL_KIND}.tar.gz",
+ source_label = "published",
+ install_kind = INSTALL_KIND,
+ expected_sha256 = "a" * 64,
+ )
+
+
+def checksums_for(choice) -> "ILP.ApprovedReleaseChecksums":
+ logical = ILP.source_archive_logical_name(UPSTREAM_TAG)
+ return ILP.ApprovedReleaseChecksums(
+ repo = PUBLISHED_REPO,
+ release_tag = RELEASE_TAG,
+ upstream_tag = UPSTREAM_TAG,
+ source_commit = "deadbeef",
+ artifacts = {
+ logical: ILP.ApprovedArtifactHash(
+ asset_name = logical,
+ sha256 = "b" * 64,
+ repo = "ggml-org/llama.cpp",
+ kind = "upstream-source",
+ ),
+ choice.name: ILP.ApprovedArtifactHash(
+ asset_name = choice.name,
+ sha256 = choice.expected_sha256,
+ repo = PUBLISHED_REPO,
+ kind = "prebuilt",
+ ),
+ },
+ )
+
+
+def build_install(
+ tmp_path: Path,
+ host,
+ *,
+ load_probe_passed: bool = True,
+) -> Path:
+ """Build an install whose marker passes the real fingerprint checks.
+
+ *load_probe_passed* mirrors what the installer learned from its own preflight:
+ True is the normal install, where dyld resolved every binary. False is the
+ install whose probe timed out or could not spawn, which must not be remembered
+ as a pass.
+ """
+ install_dir = tmp_path / "llama.cpp"
+ runtime_dir = install_dir / "build" / "bin"
+ runtime_dir.mkdir(parents = True)
+ for name in ("llama-server", "llama-quantize", "llama-diffusion-gemma-visual-server"):
+ for directory in (install_dir, runtime_dir):
+ binary = directory / name
+ binary.write_text("#!/bin/sh\nexit 0\n", encoding = "utf-8")
+ binary.chmod(0o755)
+ for dylib in ("libllama.0.dylib", "libggml.0.dylib", "libmtmd.0.dylib"):
+ (runtime_dir / dylib).write_bytes(b"DYLIB")
+ (install_dir / "convert_hf_to_gguf.py").write_text("#!/usr/bin/env python3\n", encoding = "utf-8")
+ (install_dir / "gguf-py" / "gguf").mkdir(parents = True)
+
+ choice = choice_for(host)
+ ILP.write_prebuilt_metadata(
+ install_dir,
+ host = host,
+ requested_tag = "latest",
+ llama_tag = UPSTREAM_TAG,
+ release_tag = RELEASE_TAG,
+ choice = choice,
+ approved_checksums = checksums_for(choice),
+ prebuilt_fallback_used = False,
+ backend_request = "auto",
+ macos_load_probe_passed = load_probe_passed,
+ )
+ return install_dir
+
+
+def marker_of(install_dir: Path) -> dict:
+ return json.loads((install_dir / "UNSLOTH_PREBUILT_INFO.json").read_text(encoding = "utf-8"))
+
+
+def write_marker(install_dir: Path, marker: dict) -> None:
+ (install_dir / "UNSLOTH_PREBUILT_INFO.json").write_text(
+ json.dumps(marker, indent = 2) + "\n", encoding = "utf-8"
+ )
+
+
+def count_spawns(monkeypatch) -> list[int]:
+ """Stand in for a probe that RAN and loaded every binary.
+
+ Reporting the load matters: the real probe fails open, so preflight treats an
+ empty *loaded* as "could not run" and refuses to record a pass. A stub that only
+ returned [] would silently test the no-evidence path instead.
+ """
+ calls = [0]
+
+ def _probe(
+ binaries,
+ install_dir,
+ host,
+ *,
+ loaded = None,
+ **_kwargs,
+ ):
+ calls[0] += 1
+ if loaded is not None:
+ loaded.update(path.name for path in binaries)
+ return []
+
+ monkeypatch.setattr(ILP, "macos_dyld_load_issues", _probe)
+ return calls
+
+
+def matches_choice(install_dir: Path, host) -> bool:
+ choice = choice_for(host)
+ return ILP.existing_install_matches_choice(
+ install_dir,
+ host,
+ llama_tag = UPSTREAM_TAG,
+ release_tag = RELEASE_TAG,
+ choice = choice,
+ approved_checksums = checksums_for(choice),
+ )
+
+
+def fast_path(install_dir: Path, host) -> bool:
+ route = ILP.BackendRoute(
+ backend = None,
+ host = host,
+ published_repo = PUBLISHED_REPO,
+ published_release_tag = RELEASE_TAG,
+ persist_llama_backend = None,
+ persist_rocm_gfx = None,
+ )
+ return ILP.existing_install_current_without_plan(
+ install_dir,
+ llama_tag = UPSTREAM_TAG,
+ published_repo = PUBLISHED_REPO,
+ published_release_tag = RELEASE_TAG,
+ backend_request = "auto",
+ force_cpu = False,
+ route = route,
+ )
+
+
+def test_reuse_with_a_current_record_does_not_probe(tmp_path: Path, monkeypatch):
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ calls = count_spawns(monkeypatch)
+
+ assert matches_choice(install_dir, host) is True
+ assert calls[0] == 0
+
+
+def test_fast_path_with_a_current_record_does_not_probe(tmp_path: Path, monkeypatch):
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ calls = count_spawns(monkeypatch)
+
+ assert fast_path(install_dir, host) is True
+ assert calls[0] == 0
+
+
+def test_check_installed_with_a_current_record_does_not_probe(tmp_path: Path, monkeypatch):
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ calls = count_spawns(monkeypatch)
+ # The other spawn on this path: _existing_install_runs starts each binary itself.
+ monkeypatch.setattr(ILP, "_binary_image_runs", lambda *a, **k: pytest.fail("started a binary"))
+
+ assert ILP._existing_install_runs(install_dir, host) is True
+ assert calls[0] == 0
+
+
+def test_a_marker_without_a_byte_record_still_probes(tmp_path: Path, monkeypatch):
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ marker = marker_of(install_dir)
+ marker.pop("runtime_files")
+ write_marker(install_dir, marker)
+ calls = count_spawns(monkeypatch)
+
+ assert matches_choice(install_dir, host) is True
+ assert calls[0] == 1
+
+
+def test_a_size_only_dylib_record_still_probes(tmp_path: Path, monkeypatch):
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ marker = marker_of(install_dir)
+ # A marker written before macOS hashed its payload tier.
+ marker["runtime_files"]["build/bin/libggml.0.dylib"].pop("sha256")
+ write_marker(install_dir, marker)
+ calls = count_spawns(monkeypatch)
+
+ assert matches_choice(install_dir, host) is True
+ assert calls[0] == 1
+
+
+@POSIX_ONLY
+def test_a_same_size_dylib_rewrite_is_rejected(tmp_path: Path, monkeypatch):
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ dylib = install_dir / "build" / "bin" / "libggml.0.dylib"
+ before = dylib.stat().st_size
+ dylib.write_bytes(b"ROTTE")
+ assert dylib.stat().st_size == before
+ calls = count_spawns(monkeypatch)
+
+ assert matches_choice(install_dir, host) is False
+ assert fast_path(install_dir, host) is False
+ # Rejected on the record, before anything was started.
+ assert calls[0] == 0
+ # _existing_install_runs answers a different question, so it only loses the skip.
+ assert ILP._existing_install_runs(install_dir, host) is True
+ assert calls[0] == 1
+
+
+def test_a_marker_without_a_host_profile_still_probes(tmp_path: Path, monkeypatch):
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ marker = marker_of(install_dir)
+ marker.pop("host_profile")
+ write_marker(install_dir, marker)
+ calls = count_spawns(monkeypatch)
+
+ assert matches_choice(install_dir, host) is True
+ assert calls[0] == 1
+
+
+def test_a_macos_upgrade_probes_again(tmp_path: Path, monkeypatch):
+ install_dir = build_install(tmp_path, macos_host(macos_version = (15, 5)))
+ upgraded = macos_host(macos_version = (26, 0))
+ calls = count_spawns(monkeypatch)
+ # The minos read is the other half of the preflight and reads real Mach-O headers.
+ monkeypatch.setattr(ILP, "macos_binary_minos_issues", lambda *a, **k: [])
+
+ assert matches_choice(install_dir, upgraded) is True
+ assert calls[0] == 1
+
+
+def test_an_unknown_macos_version_still_probes(tmp_path: Path, monkeypatch):
+ host = macos_host(macos_version = None)
+ install_dir = build_install(tmp_path, host)
+ calls = count_spawns(monkeypatch)
+ monkeypatch.setattr(ILP, "_binary_image_runs", lambda *a, **k: True)
+
+ assert matches_choice(install_dir, host) is True
+ assert ILP._existing_install_runs(install_dir, host) is True
+ assert calls[0] == 2
+
+
+def test_full_check_forces_the_probe(tmp_path: Path, monkeypatch):
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ monkeypatch.setenv("UNSLOTH_PREBUILT_FULL_CHECK", "1")
+ calls = count_spawns(monkeypatch)
+
+ assert matches_choice(install_dir, host) is True
+ assert calls[0] == 1
+
+
+def test_a_replaced_binary_is_still_rejected(tmp_path: Path, monkeypatch):
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ (install_dir / "build" / "bin" / "llama-server").write_text(
+ "#!/bin/sh\nexit 1\n# swapped\n", encoding = "utf-8"
+ )
+ calls = count_spawns(monkeypatch)
+
+ assert matches_choice(install_dir, host) is False
+ assert fast_path(install_dir, host) is False
+ # Rejected on the record, before anything was started.
+ assert calls[0] == 0
+
+
+def test_a_bundle_that_cannot_load_is_rejected_when_it_is_probed(tmp_path: Path, monkeypatch):
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ marker = marker_of(install_dir)
+ marker.pop("runtime_files")
+ write_marker(install_dir, marker)
+ monkeypatch.setattr(
+ ILP,
+ "macos_dyld_load_issues",
+ lambda *a, **k: ["llama-server: Library not loaded: /usr/lib/librdma.dylib"],
+ )
+
+ assert matches_choice(install_dir, host) is False
+
+
+@POSIX_ONLY
+def test_an_install_whose_probe_never_ran_is_not_recorded_as_a_pass(tmp_path: Path, monkeypatch):
+ """The probe fails open, so "no issues" is not the same as "it loaded"."""
+ host = macos_host()
+ install_dir = build_install(tmp_path, host, load_probe_passed = False)
+
+ assert ILP.MACOS_LOAD_PROBE_KEY not in marker_of(install_dir)
+ calls = count_spawns(monkeypatch)
+ # Every byte still matches; the missing evidence alone is what costs a probe.
+ assert ILP._existing_install_runs(install_dir, host) is True
+ assert calls[0] == 1
+
+
+def test_a_timed_out_probe_reports_no_pass(tmp_path: Path, monkeypatch):
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ runtime_dir = install_dir / "build" / "bin"
+ binaries = [runtime_dir / "llama-server", runtime_dir / "llama-quantize"]
+
+ def _timeout(*_args, **_kwargs):
+ raise subprocess.TimeoutExpired(cmd = "llama-server", timeout = 60)
+
+ monkeypatch.setattr(ILP, "run_capture", _timeout)
+ monkeypatch.setattr(ILP, "macos_binary_minos_issues", lambda *a, **k: [])
+
+ # Still not a rejection: a loaded machine must not lose a healthy bundle.
+ assert ILP.preflight_macos_installed_binaries(binaries, install_dir, host) is False
+
+
+def test_a_probe_that_ran_reports_a_pass(tmp_path: Path, monkeypatch):
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ runtime_dir = install_dir / "build" / "bin"
+ binaries = [runtime_dir / "llama-server", runtime_dir / "llama-quantize"]
+
+ monkeypatch.setattr(
+ ILP,
+ "run_capture",
+ lambda *a, **k: types.SimpleNamespace(returncode = 0, stdout = "", stderr = ""),
+ )
+ monkeypatch.setattr(ILP, "macos_binary_minos_issues", lambda *a, **k: [])
+
+ assert ILP.preflight_macos_installed_binaries(binaries, install_dir, host) is True
+
+
+@POSIX_ONLY
+def test_a_macos_patch_update_probes_again(tmp_path: Path, monkeypatch):
+ """host_profile is (major, minor), so only the recorded product version sees this."""
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ calls = count_spawns(monkeypatch)
+ # 15.5.1 -> 15.5.2 leaves host_profile identical, but replaces the dyld shared cache.
+ monkeypatch.setattr(ILP, "macos_product_version", lambda: "15.5.2")
+
+ assert ILP._existing_install_runs(install_dir, host) is True
+ assert calls[0] == 1
+
+
+@POSIX_ONLY
+def test_a_reuse_probe_that_passed_is_remembered(tmp_path: Path, monkeypatch):
+ """Otherwise the skip is unreachable for every install not born with the record."""
+ host = macos_host()
+ install_dir = build_install(tmp_path, host, load_probe_passed = False)
+ assert ILP.MACOS_LOAD_PROBE_KEY not in marker_of(install_dir)
+
+ calls = count_spawns(monkeypatch)
+ assert ILP._existing_install_runs(install_dir, host) is True
+ assert calls[0] == 1 # this update pays for the probe
+ assert marker_of(install_dir)[ILP.MACOS_LOAD_PROBE_KEY] == {
+ "passed": True,
+ "macos_product_version": PRODUCT_VERSION,
+ }
+
+ assert ILP._existing_install_runs(install_dir, host) is True
+ assert calls[0] == 1 # the next one does not
+
+
+@POSIX_ONLY
+def test_a_legacy_size_only_marker_is_upgraded_by_a_passing_probe(tmp_path: Path, monkeypatch):
+ """The shape every install written before this PR has: payload recorded, not hashed."""
+ host = macos_host()
+ install_dir = build_install(tmp_path, host, load_probe_passed = False)
+ marker = marker_of(install_dir)
+ for key, entry in marker["runtime_files"].items():
+ if not key.endswith(("llama-server", "llama-quantize")):
+ entry.pop("sha256", None)
+ marker.pop(ILP.MACOS_LOAD_PROBE_KEY, None)
+ write_marker(install_dir, marker)
+
+ calls = count_spawns(monkeypatch)
+ assert ILP._existing_install_runs(install_dir, host) is True
+ assert calls[0] == 1
+ upgraded = marker_of(install_dir)["runtime_files"]
+ assert all(entry.get("sha256") for entry in upgraded.values())
+
+ assert ILP._existing_install_runs(install_dir, host) is True
+ assert calls[0] == 1
+
+
+def test_the_reuse_fast_paths_do_not_record_a_pass(tmp_path: Path, monkeypatch):
+ """They probe build/bin only, so they never reach a root wrapper and cannot vouch
+ for a record that a later run uses to skip starting one."""
+ host = macos_host()
+ install_dir = build_install(tmp_path, host, load_probe_passed = False)
+ calls = count_spawns(monkeypatch)
+
+ assert matches_choice(install_dir, host) is True
+ assert fast_path(install_dir, host) is True
+ assert calls[0] == 2 # both probed
+ assert ILP.MACOS_LOAD_PROBE_KEY not in marker_of(install_dir)
+
+
+@POSIX_ONLY
+def test_a_broken_root_wrapper_is_not_blessed_into_the_record(tmp_path: Path, monkeypatch):
+ """The root entrypoint can be its own file, and it is started AFTER the dyld probe.
+
+ Recording between the two would hash the broken wrapper into the digest record, and
+ the next run would read that record as current and never start the wrapper again.
+ """
+ host = macos_host()
+ install_dir = build_install(tmp_path, host, load_probe_passed = False)
+ marker = marker_of(install_dir)
+ for key, entry in marker["runtime_files"].items(): # the pre-change shape
+ if not key.endswith(("llama-server", "llama-quantize")):
+ entry.pop("sha256", None)
+ write_marker(install_dir, marker)
+ count_spawns(monkeypatch)
+ broken = install_dir / "llama-server"
+
+ def _runs(path, *_a, **_k):
+ return Path(path) != broken
+
+ monkeypatch.setattr(ILP, "_binary_image_runs", _runs)
+
+ assert ILP._existing_install_runs(install_dir, host) is False
+ marker = marker_of(install_dir)
+ assert ILP.MACOS_LOAD_PROBE_KEY not in marker
+ # And the legacy size-only record was not upgraded on the strength of it either.
+ assert not all(entry.get("sha256") for entry in marker["runtime_files"].values())
+
+
+def test_a_probe_that_could_not_run_is_not_remembered(tmp_path: Path, monkeypatch):
+ """Only a real load is evidence, so a fail-open probe must not persist one."""
+ host = macos_host()
+ install_dir = build_install(tmp_path, host, load_probe_passed = False)
+ monkeypatch.setattr(ILP, "macos_binary_minos_issues", lambda *a, **k: [])
+
+ def _timeout(*_args, **_kwargs):
+ raise subprocess.TimeoutExpired(cmd = "llama-server", timeout = 60)
+
+ monkeypatch.setattr(ILP, "run_capture", _timeout)
+ monkeypatch.setattr(ILP, "_binary_image_runs", lambda *a, **k: True)
+
+ assert ILP._existing_install_runs(install_dir, host) is True
+ assert ILP.MACOS_LOAD_PROBE_KEY not in marker_of(install_dir)
+
+
+@POSIX_ONLY
+def test_a_rapid_security_response_probes_again(tmp_path: Path, monkeypatch):
+ """An RSR leaves ProductVersion alone and moves ProductVersionExtra and the build."""
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ calls = count_spawns(monkeypatch)
+ monkeypatch.setattr(ILP, "macos_product_version", lambda: PRODUCT_VERSION + " (a) 22E772610a")
+
+ assert ILP._existing_install_runs(install_dir, host) is True
+ assert calls[0] == 1
+
+
+def test_the_recorded_version_carries_the_rsr_and_build_fields(monkeypatch):
+ """platform.mac_ver() alone cannot see an RSR, so sw_vers is what gets asked."""
+ seen = []
+
+ def _sw_vers(cmd, **_kwargs):
+ seen.append(cmd)
+ return types.SimpleNamespace(
+ returncode = 0,
+ stdout = (
+ "ProductName:\tmacOS\n"
+ "ProductVersion:\t13.3.1\n"
+ "ProductVersionExtra:\t(a)\n"
+ "BuildVersion:\t22E772610a\n"
+ ),
+ stderr = "",
+ )
+
+ monkeypatch.setattr(ILP.subprocess, "run", _sw_vers)
+ assert _REAL_MACOS_PRODUCT_VERSION() == "13.3.1 (a) 22E772610a"
+ # Bare: sw_vers documents single-dash options, so no flag is spelled at all.
+ assert seen == [["/usr/bin/sw_vers"]]
+
+
+def test_no_rsr_installed_still_yields_a_version(monkeypatch):
+ """ProductVersionExtra is simply absent with no RSR, and predates macOS 13."""
+
+ def _sw_vers(_cmd, **_kwargs):
+ return types.SimpleNamespace(
+ returncode = 0,
+ stdout = "ProductName:\tmacOS\nProductVersion:\t15.5.1\nBuildVersion:\t24F74\n",
+ stderr = "",
+ )
+
+ monkeypatch.setattr(ILP.subprocess, "run", _sw_vers)
+ assert _REAL_MACOS_PRODUCT_VERSION() == "15.5.1 24F74"
+
+
+def test_a_failing_sw_vers_falls_back_to_the_product_version(monkeypatch):
+ """A fallback that still catches a point release beats recording nothing."""
+
+ def _boom(_cmd, **_kwargs):
+ raise OSError("sw_vers not found")
+
+ monkeypatch.setattr(ILP.subprocess, "run", _boom)
+ monkeypatch.setattr(ILP.platform, "mac_ver", lambda: ("15.5.1", ("", "", ""), "arm64"))
+ assert _REAL_MACOS_PRODUCT_VERSION() == "15.5.1"
+
+
+def test_the_same_macos_patch_level_still_skips(tmp_path: Path, monkeypatch):
+ host = macos_host()
+ install_dir = build_install(tmp_path, host)
+ calls = count_spawns(monkeypatch)
+
+ assert ILP._existing_install_runs(install_dir, host) is True
+ assert calls[0] == 0
diff --git a/tests/studio/install/test_release_plan_laziness.py b/tests/studio/install/test_release_plan_laziness.py
new file mode 100644
index 00000000000..fac285425b2
--- /dev/null
+++ b/tests/studio/install/test_release_plan_laziness.py
@@ -0,0 +1,359 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests for lazy release walk-back and its CDN fast path."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+TEST_DIR = Path(__file__).resolve().parent
+if str(TEST_DIR) not in sys.path:
+ sys.path.insert(0, str(TEST_DIR))
+
+from _pr10648_helpers import llama_host, load_studio_module # noqa: E402
+
+from test_keep_install_backcompat_9979 import MACOS, S12, build_install # noqa: E402
+
+ILP = load_studio_module("studio_install_llama_prebuilt_plan_laziness", "install_llama_prebuilt.py")
+
+PUBLISHED_REPO = "unslothai/llama.cpp"
+RELEASE_TAGS = [f"b{11000 - index}-mix-abcdef0" for index in range(20)]
+
+
+def macos_host(**overrides):
+ return llama_host(
+ ILP.HostInfo,
+ system = "Darwin",
+ machine = "arm64",
+ macos_version = overrides.pop("macos_version", (15, 5)),
+ **overrides,
+ )
+
+
+def linux_host(**overrides):
+ return llama_host(ILP.HostInfo, system = "Linux", machine = "x86_64", **overrides)
+
+
+class ApiRateLimited(RuntimeError):
+ """A 403 in the form returned by fetch_json."""
+
+
+class FakeReleases:
+ def __init__(
+ self,
+ monkeypatch,
+ *,
+ reject: "set[str] | None" = None,
+ cdn_tag: str | None = None,
+ api_error: "BaseException | None" = None,
+ ):
+ self.resolved: list[str] = []
+ self._api_error = api_error
+ self.api_listing_reads = 0
+ self.cdn_reads = 0
+ self._reject = reject or set()
+ self._cdn_tag = cdn_tag
+
+ monkeypatch.setattr(ILP, "_download_host_resolve_enabled", lambda: cdn_tag is not None)
+ monkeypatch.setattr(ILP, "_download_host_resolved_release", self._cdn_release)
+ monkeypatch.setattr(ILP, "iter_published_release_bundles", self._api_bundles)
+ monkeypatch.setattr(ILP, "validated_checksums_for_bundle", self._checksums)
+ monkeypatch.setattr(ILP, "resolve_release_asset_choice", self._attempts)
+ monkeypatch.setattr(
+ ILP,
+ "_linux_published_attempts",
+ lambda host, bundle: self._attempts(host, bundle.upstream_tag, bundle, None),
+ )
+ monkeypatch.setattr(ILP, "apply_approved_hashes", lambda attempts, checksums: attempts)
+
+ def _bundle(self, release_tag: str):
+ return ILP.PublishedReleaseBundle(
+ repo = PUBLISHED_REPO,
+ release_tag = release_tag,
+ upstream_tag = release_tag.split("-")[0],
+ assets = {},
+ )
+
+ def _cdn_release(
+ self,
+ repo: str,
+ published_release_tag: str = "",
+ ):
+ if self._cdn_tag is None:
+ return None
+ self.cdn_reads += 1
+ bundle = self._bundle(self._cdn_tag)
+ return ILP.ResolvedPublishedRelease(bundle = bundle, checksums = self._checksums(repo, bundle))
+
+ def _api_bundles(self, repo: str):
+ self.api_listing_reads += 1
+ if self._api_error is not None:
+ raise self._api_error
+ for release_tag in RELEASE_TAGS:
+ yield self._bundle(release_tag)
+
+ def _checksums(self, repo: str, bundle):
+ return ILP.ApprovedReleaseChecksums(
+ repo = repo,
+ release_tag = bundle.release_tag,
+ upstream_tag = bundle.upstream_tag,
+ source_commit = "deadbeef",
+ artifacts = {},
+ )
+
+ def _attempts(self, host, resolved_tag, bundle, checksums):
+ self.resolved.append(bundle.release_tag)
+ if bundle.release_tag in self._reject:
+ return []
+ return [
+ ILP.AssetChoice(
+ repo = PUBLISHED_REPO,
+ tag = bundle.release_tag,
+ name = f"llama-{bundle.upstream_tag}-bin-macos-arm64.tar.gz",
+ url = f"https://example.com/{bundle.upstream_tag}.tar.gz",
+ source_label = "published",
+ install_kind = "macos-arm64",
+ expected_sha256 = "a" * 64,
+ )
+ ]
+
+
+def resolve(host):
+ return ILP.resolve_simple_install_release_plans("latest", host, PUBLISHED_REPO, "")
+
+
+def test_a_usable_newest_release_resolves_exactly_one_plan(monkeypatch):
+ releases = FakeReleases(monkeypatch)
+ _tag, plans = resolve(macos_host())
+
+ assert plans[0].release_tag == RELEASE_TAGS[0]
+ assert releases.resolved == [RELEASE_TAGS[0]]
+
+
+def test_the_walk_back_still_reaches_older_releases_in_order(monkeypatch):
+ releases = FakeReleases(monkeypatch, reject = set(RELEASE_TAGS[:3]))
+ _tag, plans = resolve(macos_host())
+
+ assert plans[0].release_tag == RELEASE_TAGS[3]
+ assert releases.resolved == RELEASE_TAGS[:4]
+
+
+def test_asking_for_the_next_plan_resolves_one_more_and_no_further(monkeypatch):
+ releases = FakeReleases(monkeypatch)
+ _tag, plans = resolve(macos_host())
+ assert releases.resolved == [RELEASE_TAGS[0]]
+
+ assert plans[1].release_tag == RELEASE_TAGS[1]
+ assert releases.resolved == RELEASE_TAGS[:2]
+
+
+def test_the_walk_back_is_still_capped(monkeypatch):
+ releases = FakeReleases(monkeypatch)
+ _tag, plans = resolve(macos_host())
+
+ assert len(list(plans)) == ILP.DEFAULT_MAX_MACOS_RELEASE_FALLBACKS
+ assert releases.resolved == RELEASE_TAGS[: ILP.DEFAULT_MAX_MACOS_RELEASE_FALLBACKS]
+
+
+def test_linux_still_takes_the_newest_release_from_the_download_host(monkeypatch):
+ releases = FakeReleases(monkeypatch, cdn_tag = RELEASE_TAGS[0])
+ _tag, plans = resolve(linux_host())
+
+ assert len(list(plans)) == 1 # the CDN surfaces only the newest, as before
+ assert releases.resolved == [RELEASE_TAGS[0]]
+ assert releases.api_listing_reads == 0
+
+
+def test_macos_takes_the_newest_release_from_the_download_host(monkeypatch):
+ releases = FakeReleases(monkeypatch, cdn_tag = RELEASE_TAGS[0])
+ _tag, plans = resolve(macos_host())
+
+ assert plans[0].release_tag == RELEASE_TAGS[0]
+ assert releases.cdn_reads == 1
+ assert releases.api_listing_reads == 0
+
+
+def test_macos_falls_back_to_the_api_only_when_it_must_walk(monkeypatch):
+ releases = FakeReleases(monkeypatch, reject = {RELEASE_TAGS[0]}, cdn_tag = RELEASE_TAGS[0])
+ _tag, plans = resolve(macos_host())
+
+ assert plans[0].release_tag == RELEASE_TAGS[1]
+ assert releases.cdn_reads == 1
+ assert releases.api_listing_reads == 1
+ assert releases.resolved == RELEASE_TAGS[:2]
+
+
+def test_a_walk_that_finds_nothing_usable_still_fails(monkeypatch):
+ FakeReleases(monkeypatch, reject = set(RELEASE_TAGS))
+
+ with pytest.raises(ILP.PrebuiltFallback):
+ resolve(macos_host())
+
+
+def test_the_first_plan_is_resolved_before_the_resolver_returns(monkeypatch):
+ def _explode(repo: str):
+ raise RuntimeError("GitHub API returned 403")
+
+ monkeypatch.setattr(ILP, "_download_host_resolve_enabled", lambda: False)
+ monkeypatch.setattr(ILP, "iter_published_release_bundles", _explode)
+
+ with pytest.raises(RuntimeError, match = "403"):
+ resolve(macos_host())
+
+
+def test_a_403_on_the_deferred_lookup_reaches_the_installer_as_a_fallback(monkeypatch):
+ releases = FakeReleases(
+ monkeypatch,
+ cdn_tag = RELEASE_TAGS[0],
+ api_error = ApiRateLimited("GitHub API returned 403 for .../releases?per_page=100"),
+ )
+ _tag, plans = resolve(macos_host())
+ remaining = ILP.iter_release_plans(plans, PUBLISHED_REPO)
+ assert next(remaining).release_tag == RELEASE_TAGS[0]
+ assert releases.api_listing_reads == 0
+
+ with pytest.raises(ILP.PrebuiltFallback, match = "failed to inspect published releases"):
+ next(remaining)
+ assert releases.api_listing_reads == 1
+
+
+def test_a_pinned_release_resolves_once_even_when_the_walk_back_is_on(monkeypatch):
+ """macOS sets continue_after_fast_path by host, not by whether a tag is pinned.
+
+ A pin names exactly one release, so the download host's answer is the whole
+ answer; falling through would yield it a second time and spend an API call.
+ """
+ releases = FakeReleases(monkeypatch, cdn_tag = RELEASE_TAGS[0])
+ pinned_lookups = []
+ monkeypatch.setattr(
+ ILP,
+ "pinned_published_release_bundle",
+ lambda repo, tag: pinned_lookups.append(tag) or releases._bundle(tag),
+ )
+
+ resolved = list(
+ ILP.iter_resolved_published_releases(
+ "latest",
+ PUBLISHED_REPO,
+ RELEASE_TAGS[0],
+ continue_after_fast_path = True,
+ )
+ )
+
+ assert [item.bundle.release_tag for item in resolved] == [RELEASE_TAGS[0]]
+ assert pinned_lookups == []
+ assert releases.api_listing_reads == 0
+
+
+def test_a_pin_without_the_cdn_still_reaches_the_api(monkeypatch):
+ """The guard must not cost the pinned lookup when the fast path produced nothing."""
+ releases = FakeReleases(monkeypatch)
+ pinned_lookups = []
+ monkeypatch.setattr(
+ ILP,
+ "pinned_published_release_bundle",
+ lambda repo, tag: pinned_lookups.append(tag) or releases._bundle(tag),
+ )
+
+ resolved = list(
+ ILP.iter_resolved_published_releases(
+ "latest",
+ PUBLISHED_REPO,
+ RELEASE_TAGS[0],
+ continue_after_fast_path = True,
+ )
+ )
+
+ assert [item.bundle.release_tag for item in resolved] == [RELEASE_TAGS[0]]
+ assert pinned_lookups == [RELEASE_TAGS[0]]
+
+
+def test_a_failed_walk_reports_the_same_failure_every_time(monkeypatch):
+ """A closed generator must not read as a short, clean sequence on the second pass."""
+
+ def _plans():
+ yield "p0"
+ raise ApiRateLimited("GitHub API returned 403")
+
+ plans = ILP.LazyReleasePlans(_plans())
+ assert plans[0] == "p0"
+
+ with pytest.raises(ApiRateLimited):
+ list(plans)
+ # Same question, same answer: without this the second pass returns ["p0"] quietly.
+ with pytest.raises(ApiRateLimited):
+ list(plans)
+ with pytest.raises(ApiRateLimited):
+ len(plans)
+
+
+def test_the_resolver_itself_still_fails_hard_on_a_403(monkeypatch):
+ FakeReleases(
+ monkeypatch,
+ api_error = ApiRateLimited("GitHub API returned 403 for .../releases?per_page=100"),
+ )
+
+ with pytest.raises(ApiRateLimited):
+ resolve(macos_host())
+
+
+def test_a_deferred_403_still_reaches_the_source_build_end_to_end(tmp_path, monkeypatch):
+ plan = ILP.InstallReleasePlan(
+ requested_tag = "latest",
+ llama_tag = "b11000",
+ release_tag = RELEASE_TAGS[0],
+ attempts = [
+ ILP.AssetChoice(
+ repo = PUBLISHED_REPO,
+ tag = RELEASE_TAGS[0],
+ name = "llama-b11000-bin-macos-arm64.tar.gz",
+ url = "https://example.com/llama-b11000-bin-macos-arm64.tar.gz",
+ source_label = "published",
+ install_kind = "macos-arm64",
+ expected_sha256 = "a" * 64,
+ )
+ ],
+ approved_checksums = ILP.ApprovedReleaseChecksums(
+ repo = PUBLISHED_REPO,
+ release_tag = RELEASE_TAGS[0],
+ upstream_tag = "b11000",
+ source_commit = "deadbeef",
+ artifacts = {},
+ ),
+ )
+
+ def _plans():
+ yield plan
+ raise ApiRateLimited("GitHub API returned 403 for .../releases?per_page=100")
+
+ monkeypatch.setattr(
+ ILP,
+ "_fork_manifest_release_plans",
+ lambda *a, **k: ("latest", ILP.LazyReleasePlans(_plans())),
+ )
+ monkeypatch.setattr(ILP, "detect_host", lambda *a, **k: MACOS)
+ monkeypatch.setattr(ILP, "collect_system_report", lambda *a, **k: "report")
+ monkeypatch.setattr(ILP, "existing_install_matches_plan", lambda *a, **k: False)
+ monkeypatch.setattr(ILP, "resolve_validation_model", lambda probe: probe)
+
+ def _validation_fails(*args, **kwargs):
+ raise ILP.PrebuiltFallback("staged bundle failed validation")
+
+ monkeypatch.setattr(ILP, "validate_prebuilt_attempts", _validation_fails)
+
+ # Keep a runnable install when release lookup fails.
+ install_dir = build_install(tmp_path, host = MACOS, marker = S12, payload_backend = "metal")
+ ILP.install_prebuilt(install_dir, "latest", PUBLISHED_REPO, "")
+ assert (install_dir / "llama-server").exists()
+
+ # Otherwise request a source build.
+ broken = build_install(
+ tmp_path / "broken", host = MACOS, marker = S12, payload_backend = "metal", runnable = False
+ )
+ with pytest.raises(SystemExit) as caught:
+ ILP.install_prebuilt(broken, "latest", PUBLISHED_REPO, "")
+ assert caught.value.code == ILP.EXIT_FALLBACK