Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop the macOS update re-probing binaries and resolving 16 releases (#11237)
oobaboogadf0d65b805f5
diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.pyindex b548a9b71e1..ac247ffd9cd 100644--- a/studio/install_llama_prebuilt.py+++ b/studio/install_llama_prebuilt.py@@ -34,6 +34,7 @@ import urllib.errorimport urllib.parseimport urllib.requestimport zipfile+from collections.abc import Sequencefrom contextlib import contextmanagerfrom dataclasses import dataclass, field, replace as dataclasses_replace@@ -1359,7 +1360,7 @@ def resolve_simple_install_release_plans(published_release_tag: str,*,max_release_fallbacks: int = DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS,-) -> tuple[str, list[InstallReleasePlan]]:+) -> "tuple[str, Sequence[InstallReleasePlan]]":repo = published_repo or DEFAULT_PUBLISHED_REPO# The fork (unslothai) ships a manifest describing every bundle's GPU/arch# coverage, so all fork hosts select from it. Upstream (ggml-org) ships no@@ -2264,23 +2265,23 @@ def iter_resolved_published_releases(published_release_tag: str = "",*,allow_download_host_fast_path: bool = True,+ continue_after_fast_path: bool = False,) -> Iterable[ResolvedPublishedRelease]:repo = published_repo or DEFAULT_PUBLISHED_REPOnormalized_requested = normalized_requested_llama_tag(requested_tag)# Fast path: resolve a pinned or latest fork release from the download host (no- # api.github.com rate limit). Latest surfaces only the single newest release, so- # the caller disables it when the multi-release walk-back is needed (macOS- # skipping too-new prebuilts); a broken latest then drops to source build, not- # an older release. Pinned tags use the same CDN path so in-app updates avoid- # the API entirely (#9970). Any rejection/network error is non-fatal and falls- # through to the API.+ # api.github.com rate limit). Latest surfaces only the newest release: macOS walks+ # back past too-new prebuilts via continue_after_fast_path, otherwise a broken+ # latest drops to source build. Pinned tags use the same CDN path so in-app updates+ # avoid the API entirely (#9970). Any rejection/network error falls through to the API.fast_path_tag: str | None = Noneif published_release_tag:fast_path_tag = published_release_tagelif normalized_requested == "latest":fast_path_tag = ""+ fast_path_release_tag: str | None = Noneif (fast_path_tag is not Noneand allow_download_host_fast_path@@ -2308,7 +2309,13 @@ def iter_resolved_published_releases(f"{resolved.bundle.upstream_tag}, but requested {normalized_requested}")yield resolved- return+ # A pin names exactly one release, so there is nothing to walk back to and+ # continue_after_fast_path has nothing to add. Falling through would re-resolve+ # the same tag below -- the same release twice, and an api.github.com call to+ # do it -- so a pin returns here whatever the walk-back flag says.+ if not continue_after_fast_path or published_release_tag:+ return+ fast_path_release_tag = resolved.bundle.release_tagif published_release_tag:bundle = pinned_published_release_bundle(repo, published_release_tag)@@ -2324,12 +2331,15 @@ def iter_resolved_published_releases()return- matched_any = False+ matched_any = fast_path_release_tag is not Noneskipped_invalid = 0- yielded_valid = False+ yielded_valid = fast_path_release_tag is not Nonefor bundle in iter_published_release_bundles(repo):if not published_release_matches_request(bundle, normalized_requested):continue+ if fast_path_release_tag is not None and bundle.release_tag == fast_path_release_tag:+ # Already yielded from the download host.+ continuematched_any = Truetry:checksums = validated_checksums_for_bundle(repo, bundle)@@ -5754,10 +5764,18 @@ def macos_binary_minos_issues(def macos_dyld_load_issues(- binaries: Iterable[Path], install_dir: Path, host: HostInfo+ binaries: Iterable[Path],+ install_dir: Path,+ host: HostInfo,+ *,+ loaded: "set[str] | None" = None,) -> list[str]:"""Issue strings for every installed executable dyld refuses to load.+ *loaded*, when given, collects the binaries dyld actually resolved: this probe+ fails OPEN, so "no issues" alone cannot tell a clean load from one that never+ ran. Only a name in here is evidence of a load.+`--version` costs a process spawn and still makes dyld resolve the whole linkgraph, so it catches a missing dylib, a missing symbol or a too-new slice --unlike validate_server, which loads a model and is off for checksummed bundles@@ -5786,6 +5804,8 @@ def macos_dyld_load_issues(log(f"macos load probe could not run {binary_path.name}: {exc}")continueif result.returncode == 0:+ if loaded is not None:+ loaded.add(binary_path.name)continueoutput = (result.stdout + result.stderr).strip()# A non-zero exit is not evidence of a bad link: llama-quantize answers@@ -5795,6 +5815,10 @@ def macos_dyld_load_issues(# that asked, which otool -L cannot, since an absolute install name absent# from disk is the normal case for /usr/lib's shared-cache members.if not looks_like_macos_loader_failure(output):+ # llama-quantize answers --version by printing its table and exiting 1;+ # dyld still resolved the image to let it print anything at all.+ if loaded is not None:+ loaded.add(binary_path.name)continuedetail = " | ".join(output.splitlines()[-5:]) or f"exit {result.returncode}"issues.append(f"{binary_path.name}: {detail}")@@ -5802,13 +5826,23 @@ def macos_dyld_load_issues(def preflight_macos_installed_binaries(- binaries: Iterable[Path], install_dir: Path, host: HostInfo-) -> None:- """Reject a macos prebuilt whose minimum-OS is newer than the host, or that- dyld will not load at all. The upstream selector pins a loadable release up+ binaries: Iterable[Path],+ install_dir: Path,+ host: HostInfo,+ *,+ load_probe: bool = True,+) -> bool:+ """Reject a macos prebuilt whose minimum-OS is newer than the host, or (with+ *load_probe*) that dyld will not load at all. The upstream selector pins a loadable release upfront, so here this is the post-download backstop; the published/fork pathalso uses it to advance the walk-back.+ Returns True only when the load probe actually ran and dyld resolved every+ binary. False means "no evidence", not "bad": the probe is skipped, or it+ timed out or could not spawn. Callers record the skip only on True, so an+ inconclusive probe costs another probe next time instead of being cached as+ a pass.+The load probe is the macOS counterpart of preflight_linux_installed_binaries'ldd sweep, which this side went without: a bundle whose libggml-rpc.0.dyliblinks a /usr/lib/librdma.dylib that exists on the builder and nowhere else@@ -5820,18 +5854,30 @@ def preflight_macos_installed_binaries(both on an unparseable ``platform.mac_ver()`` left that host with no check atall, since the runtime validation it deferred to is off by default (#5854)."""if not host.is_macos:- return+ return False+ binaries = list(binaries)if host.macos_version is not None:issues = macos_binary_minos_issues(binaries, install_dir, host)if issues:raise PrebuiltFallback("macos prebuilt requires a newer macOS than this host:\n" + "\n".join(issues))- load_issues = macos_dyld_load_issues(binaries, install_dir, host)+ if not load_probe:+ log("installed binaries match their recorded digests; skipping the dyld load probe")+ return False+ loaded: set[str] = set()+ load_issues = macos_dyld_load_issues(binaries, install_dir, host, loaded = loaded)if load_issues:raise PrebuiltFallback("macos prebuilt does not load on this host:\n" + "\n".join(load_issues))+ expected = {path.name for path in binaries if path.is_file()}+ if not expected or not expected.issubset(loaded):+ # The probe fails open so a loaded machine cannot reject a healthy bundle;+ # that is right for INSTALLING, and wrong to remember as a load.+ log("macos load probe did not run for every binary; not recording it as a pass")+ return False+ return Truedef preflight_linux_installed_binaries(@@ -6849,6 +6895,85 @@ def _linux_published_attempts(host: HostInfo, bundle: PublishedReleaseBundle) ->return attempts+RELEASE_LISTING_TRANSPORT_ERRORS = (+ urllib.error.URLError,+ ssl.SSLError,+ ConnectionError,+ TimeoutError,+ RuntimeError,+ ValueError,+)+++def release_listing_fallback(exc: BaseException, published_repo: str) -> PrebuiltFallback:+ # Install time only: a resolver probe must fail hard, or update_flow caches "no prebuilt".+ return PrebuiltFallback(+ f"failed to inspect published releases in "+ f"{published_repo or DEFAULT_PUBLISHED_REPO}: {exc}"+ )+++def iter_release_plans(+ plans: "Sequence[InstallReleasePlan]", published_repo: str+) -> "Iterator[InstallReleasePlan]":+ try:+ yield from plans+ except (BusyInstallConflict, PrebuiltFallback):+ raise+ except RELEASE_LISTING_TRANSPORT_ERRORS as exc:+ raise release_listing_fallback(exc, published_repo) from exc+++class LazyReleasePlans(Sequence):+ """Release plans resolved on first access; ``len()`` resolves them all."""++ def __init__(self, plans: "Iterable[InstallReleasePlan]") -> None:+ self._source: "Iterator[InstallReleasePlan] | None" = iter(plans)+ self._resolved: list[InstallReleasePlan] = []+ self._failure: "BaseException | None" = None++ def _resolve_through(self, index: int) -> bool:+ while len(self._resolved) <= index:+ # A generator that raised is closed, so without remembering why, a second+ # traversal would report a short-but-clean sequence and len() would answer+ # differently than the call that raised. Same inputs, same answer, every time.+ if self._failure is not None:+ raise self._failure+ if self._source is None:+ return False+ try:+ self._resolved.append(next(self._source))+ except StopIteration:+ self._source = None+ return False+ except BaseException as exc:+ self._source = None+ self._failure = exc+ raise+ return True++ def __getitem__(self, index):+ if isinstance(index, slice) or index < 0:+ self._resolve_through(sys.maxsize)+ return self._resolved[index]+ if not self._resolve_through(index):+ raise IndexError(index)+ return self._resolved[index]++ def __len__(self) -> int:+ self._resolve_through(sys.maxsize)+ return len(self._resolved)++ def __iter__(self):+ index = 0+ while self._resolve_through(index):+ yield self._resolved[index]+ index += 1++ def __bool__(self) -> bool:+ return self._resolve_through(0)++def _fork_manifest_release_plans(llama_tag: str,host: HostInfo,@@ -6856,7 +6981,7 @@ def _fork_manifest_release_plans(published_release_tag: str,*,max_release_fallbacks: int = DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS,-) -> tuple[str, list[InstallReleasePlan]]:+) -> "tuple[str, Sequence[InstallReleasePlan]]":"""Manifest-reading branch of resolve_simple_install_release_plans, used forevery fork host: all of the fork's bundles describe their GPU/arch coveragein llama-prebuilt-manifest.json rather than in the asset filename (CPU,@@ -6868,74 +6993,82 @@ def _fork_manifest_release_plans(# version is known; otherwise keep the default (cannot tell up front).if host.is_macos and allow_older_release_fallback and host.macos_version is not None:release_limit = max(release_limit, DEFAULT_MAX_MACOS_RELEASE_FALLBACKS)- plans: list[InstallReleasePlan] = []- last_error: PrebuiltFallback | None = None- # The newest release this host could not take, if the first plan is an older one.- skipped_newest: str | None = None- for resolved_release in iter_resolved_published_releases(- llama_tag,- published_repo,- published_release_tag,- # macOS relies on the multi-release walk-back to skip too-new prebuilts,- # which the single-latest download-host path cannot provide.- allow_download_host_fast_path = not host.is_macos,- ):- bundle = resolved_release.bundle- checksums = resolved_release.checksums- resolved_tag = bundle.upstream_tag- try:- if host.is_linux:- linux_attempts = _linux_published_attempts(host, bundle)- if not linux_attempts:- raise PrebuiltFallback("no compatible Linux prebuilt asset was found")- attempts = apply_approved_hashes(linux_attempts, checksums)- if not attempts:- raise PrebuiltFallback("no compatible Linux prebuilt asset was found")- if attempts[0].selection_log:- log_lines(attempts[0].selection_log)- else:- attempts = resolve_release_asset_choice(- host,- resolved_tag,- bundle,- checksums,+ def _plans() -> "Iterator[InstallReleasePlan]":+ resolved_count = 0+ last_error: PrebuiltFallback | None = None+ # The newest release this host could not take, if the first plan is an older one.+ skipped_newest: str | None = None++ for resolved_release in iter_resolved_published_releases(+ llama_tag,+ published_repo,+ published_release_tag,+ # macOS takes the newest release from the CDN and walks back through the API.+ continue_after_fast_path = host.is_macos,+ ):+ bundle = resolved_release.bundle+ checksums = resolved_release.checksums+ resolved_tag = bundle.upstream_tag+ try:+ if host.is_linux:+ linux_attempts = _linux_published_attempts(host, bundle)+ if not linux_attempts:+ raise PrebuiltFallback("no compatible Linux prebuilt asset was found")+ attempts = apply_approved_hashes(linux_attempts, checksums)+ if not attempts:+ raise PrebuiltFallback("no compatible Linux prebuilt asset was found")+ if attempts[0].selection_log:+ log_lines(attempts[0].selection_log)+ else:+ attempts = resolve_release_asset_choice(+ host,+ resolved_tag,+ bundle,+ checksums,+ )+ if not attempts:+ raise PrebuiltFallback("no compatible prebuilt asset was found")+ if attempts[0].selection_log:+ log_lines(attempts[0].selection_log)+ except PrebuiltFallback as exc:+ last_error = exc+ if not allow_older_release_fallback:+ raise+ log(+ "published release skipped for install planning: "+ f"{bundle.repo}@{bundle.release_tag} upstream_tag={resolved_tag} ({exc})")- if not attempts:- raise PrebuiltFallback("no compatible prebuilt asset was found")- if attempts[0].selection_log:- log_lines(attempts[0].selection_log)- except PrebuiltFallback as exc:- last_error = exc- if not allow_older_release_fallback:- raise- log(- "published release skipped for install planning: "- f"{bundle.repo}@{bundle.release_tag} upstream_tag={resolved_tag} ({exc})"- )- if not plans and skipped_newest is None:- skipped_newest = bundle.release_tag- continue+ if resolved_count == 0 and skipped_newest is None:+ skipped_newest = bundle.release_tag+ continue- plans.append(- InstallReleasePlan(+ yield InstallReleasePlan(requested_tag = requested_tag,llama_tag = resolved_tag,release_tag = bundle.release_tag,attempts = attempts,approved_checksums = checksums,- walk_back = _core.walk_back_for(host, skipped_newest) if not plans else None,+ walk_back = (+ _core.walk_back_for(host, skipped_newest) if resolved_count == 0 else None+ ),)- )+ resolved_count += 1- if not allow_older_release_fallback or len(plans) >= release_limit:- break+ if not allow_older_release_fallback or resolved_count >= release_limit:+ return- if plans:- return requested_tag, plans- if last_error is not None:- raise last_error- raise PrebuiltFallback("no installable published llama.cpp releases were found")+ if resolved_count:+ return+ if last_error is not None:+ raise last_error+ raise PrebuiltFallback("no installable published llama.cpp releases were found")++ plans = LazyReleasePlans(_plans())+ # Resolve the first plan here, so its listing failures still raise from the resolver.+ if not plans:+ raise PrebuiltFallback("no installable published llama.cpp releases were found")+ return requested_tag, plansdef persisted_llama_backend(llama_backend: str | None, choice: AssetChoice) -> str | None:@@ -7052,6 +7185,7 @@ def write_prebuilt_metadata(backend_request: str | None = None,rocm_gfx: str | None = None,walk_back: "_core.WalkBack | None" = None,+ macos_load_probe_passed: bool = False,) -> None:source_asset_name, source_sha256 = selected_source_archive_metadata(approved_checksums,@@ -7071,6 +7205,9 @@ def write_prebuilt_metadata(if fingerprint is None:raise PrebuiltFallback(f"cannot compute install fingerprint for {choice.name}")_persisted_backend = persisted_llama_backend(llama_backend, choice)+ _load_probe = (+ macos_load_probe_record(host) if macos_load_probe_passed and host is not None else None+ )# An install kind with no allowlist raises; the binary tier alone is still honest evidence.try:_runtime_patterns: list[str] | None = runtime_patterns_for_choice(choice)@@ -7141,6 +7278,10 @@ def write_prebuilt_metadata("runtime_files": runtime_file_records(install_dir, host, _runtime_patterns),# The box this bundle was chosen for. Absent reads as "cannot say": full path.**({"host_profile": host_profile(host)} if host is not None else {}),+ # Evidence that dyld really loaded these bytes on this macOS build, which is what+ # lets a later update skip the probe. Absent reads as "cannot say": probe again.+ # Not fingerprinted, so it never invalidates an install.+ **({MACOS_LOAD_PROBE_KEY: _load_probe} if _load_probe is not None else {}),"prebuilt_fallback_used": prebuilt_fallback_used,"installed_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),}@@ -7703,14 +7844,18 @@ def runtime_file_records(libraries are where most of a CUDA bundle's bytes live, which is exactly theshape a full disk or an interrupted extract leaves behind. A size comparisoncatches it for the price of a stat; hashing 300 MB of kernels on every update- would not be worth it.+ would not be worth it. macOS hashes both tiers, since its record stands in for the+ dyld probe and a Metal bundle is tens of MB.*patterns* comes from runtime_patterns_for_install_kind for the bundle beinginstalled; without it only the binary tier is recorded, which is what the callersthat have no bundle in hand (a bare re-record) can honestly say.++ Call only after the platform preflights pass: the macOS probe skip trusts the record."""records: dict[str, dict[str, Any]] = {}runtime_dir = install_runtime_dir(install_dir, host) if host is not None else None+ hash_payload = host is not None and host.is_macosif patterns and runtime_dir is not None:for pattern in patterns:try:@@ -7725,6 +7870,15 @@ def runtime_file_records(relative = candidate.relative_to(install_dir).as_posix()except ValueError:continue+ if hash_payload:+ try:+ record["sha256"] = sha256_file(candidate)+ except (OSError, MemoryError) as exc:+ log(+ f"could not hash {relative} for the runtime record ({exc}); "+ "not recording"+ )+ return {}records[relative] = record# Last, so a binary the sweep matched is upgraded to the hashed tier.for candidate in _runtime_record_paths(install_dir, host):@@ -7746,6 +7900,137 @@ def runtime_file_records(return records+MACOS_LOAD_PROBE_KEY = "macos_load_probe"+++def macos_product_version() -> str:+ """The macOS identity the dyld shared cache actually tracks, as one string.++ host_profile records parse_macos_version, which is (major, minor) by design --+ right for the minos comparison, wrong for "can this image still load": since Big+ Sur the system libraries are a dyld shared cache blob, and Apple replaces that+ blob in point releases and in Rapid Security Responses.++ A Rapid Security Response does NOT move ProductVersion, which is all+ platform.mac_ver() reads: 13.3.1 (a) reports ProductVersion 13.3.1 with the "(a)"+ in ProductVersionExtra and the build at 22E772610a instead of 22E261. Reading+ SystemVersion.plist has the same blind spot and is additionally stale for an RSR+ (osquery/osquery#8008), so ask sw_vers -- bare, since it documents single-dash+ options and a misspelt flag would silently cost the field this exists to read.+ """+ try:+ done = subprocess.run(+ ["/usr/bin/sw_vers"],+ capture_output = True,+ text = True,+ timeout = 10,+ check = False,+ )+ if done.returncode == 0:+ values: "dict[str, str]" = {}+ for line in done.stdout.splitlines():+ key, sep, value = line.partition(":")+ if sep:+ values[key.strip()] = value.strip()+ # Absent ProductVersionExtra means no RSR, and predates macOS 13; not a failure.+ fields = [+ values.get(key, "")+ for key in ("ProductVersion", "ProductVersionExtra", "BuildVersion")+ ]+ if fields[0]:+ return " ".join(f for f in fields if f)+ except Exception: # noqa: BLE001+ pass+ try:+ # sw_vers unavailable: the product version alone still catches a point release.+ return str(platform.mac_ver()[0] or "").strip()+ except Exception: # noqa: BLE001+ return ""+++def macos_load_probe_record(host: HostInfo) -> "dict[str, Any] | None":+ """The evidence a completed dyld probe leaves for later runs, or None when the+ host cannot describe itself finely enough to be worth recording."""+ version = macos_product_version()+ if not host.is_macos or not version:+ return None+ return {"passed": True, "macos_product_version": version}+++def persist_macos_load_probe(install_dir: Path, host: HostInfo) -> bool:+ """Remember a reuse-path probe that just passed, so the next update can skip it.++ Without this the skip is unreachable for everyone it was not born with: a marker+ written before the record existed, an install whose first probe timed out, and+ every install after a macOS point release all probe again on EVERY later update+ and never stop, since nothing writes the evidence outside a fresh install.++ Re-recording the runtime files is safe HERE and nowhere else: the minos check, the+ digests the marker did carry and dyld itself all passed on these exact bytes+ moments ago, which is the same evidence the install path records on.++ The record covers the ROOT copies too, so only a caller that has started every one+ of them may write it. That is _existing_install_runs alone; the two reuse fast paths+ probe build/bin and never reach a root wrapper, which can be a separate file rather+ than a symlink.+ """+ record = macos_load_probe_record(host)+ if record is None:+ return False+ try:+ marker = load_prebuilt_metadata(install_dir)+ if not isinstance(marker, dict) or not marker:+ return False+ patch: "dict[str, Any]" = {MACOS_LOAD_PROBE_KEY: record}+ recorded = marker.get("runtime_files")+ if not (+ isinstance(recorded, dict)+ and recorded+ and all(isinstance(e, dict) and e.get("sha256") for e in recorded.values())+ ):+ # Both macOS kinds share one pattern list, so the host answers this without+ # a plan; an unreadable file yields {} and no evidence rather than a half record.+ kind = "macos-arm64" if str(host.machine or "").lower() == "arm64" else "macos-x64"+ records = runtime_file_records(+ install_dir, host, runtime_patterns_for_install_kind(kind)+ )+ if not records:+ return False+ patch["runtime_files"] = records+ marker.update(patch)+ return _write_marker(install_dir / "UNSLOTH_PREBUILT_INFO.json", marker)+ except Exception: # noqa: BLE001+ # Best effort: a marker we could not rewrite costs another probe, never the reuse.+ return False+++def _macos_load_record_is_current(marker: "dict[str, Any] | None", host: HostInfo) -> bool:+ """Whether the dyld probe these recorded bytes passed still holds; callers must also+ check _runtime_files_match. A size-only entry misses a same-size rewrite, and an+ unknown macOS version skips the minos check, leaving the probe as the only load check.++ Requires POSITIVE evidence that a probe ran and passed (MACOS_LOAD_PROBE_KEY), on+ this exact macOS product version. A marker without it -- every one written before+ this existed, and every install whose probe timed out or could not spawn -- takes+ the probe, which is what happened on every update before the skip existed.+ """+ if not host.is_macos or host.macos_version is None or prebuilt_full_check_requested():+ return False+ marker = marker or {}+ recorded = marker.get("runtime_files")+ if not isinstance(recorded, dict) or not recorded:+ return False+ if not all(isinstance(entry, dict) and entry.get("sha256") for entry in recorded.values()):+ return False+ probe = marker.get(MACOS_LOAD_PROBE_KEY)+ if not isinstance(probe, dict) or probe.get("passed") is not True:+ return False+ version = macos_product_version()+ if not version or probe.get("macos_product_version") != version:+ return False+ return marker.get("host_profile") == host_profile(host)++def _runtime_files_match(install_dir: Path, host: HostInfo, marker: "dict[str, Any]") -> bool:"""Whether every recorded runtime file is still the file that was installed.@@ -8041,11 +8326,10 @@ def _expected_release_tag_without_plan(api.github.com call, so no rate limit, and no manifest or checksum download.KNOWN AND ACCEPTED LAG, for "latest" only. The HEAD follows /releases/latest, which- GitHub resolves by make_latest / created_at. The normal macOS path deliberately- turns the download-host resolver off (allow_download_host_fast_path = not- host.is_macos, so it can walk back past a prebuilt whose minimum OS is too new) and- orders releases by published_at instead. The two disagree when a release is created- before, but published after, another -- e.g. a re-published or back-dated release.+ GitHub resolves by make_latest / created_at. _select reads the same pointer through+ the download host, but falls back to the API listing, ordered by published_at, when+ that host is unavailable. The two disagree when a release is created before, but+ published after, another -- e.g. a re-published or back-dated release.In that window this check can report "current" for an install _select would havemoved off. The consequence is one deferred update, never a wrong install: the verynext run whose pointer has caught up does the move, and every OTHER guard here@@ -8298,16 +8582,19 @@ def existing_install_current_without_plan(]if not all(os.access(binary, os.X_OK) for binary in binaries):return False+ # (6) the bytes are the ones installed; before the preflight, whose probe skip trusts them.+ if not _runtime_files_match(install_dir, host, marker):+ return Falsetry:- # Kept, unlike the --version spawns: a preflight answers whether the OS can LOAD the image,- # which a hash cannot.preflight_linux_installed_binaries(binaries, install_dir, host)- preflight_macos_installed_binaries(binaries, install_dir, host)+ preflight_macos_installed_binaries(+ binaries,+ install_dir,+ host,+ load_probe = not _macos_load_record_is_current(marker, host),+ )except Exception: # noqa: BLE001return False- # (6) and the bytes are the ones that were installed.- if not _runtime_files_match(install_dir, host, marker):- return False# The one backfill that is not a release change, so it has to be asked separately.if _diffusion_visual_server_missing_for_marker(install_dir, host, marker):return False@@ -8475,7 +8762,8 @@ def _existing_install_runs(install_dir: Path, host: HostInfo) -> bool:return Falseif not _kept_install_payload_is_healthy(install_dir, host):return False- recorded_runtime_line = (load_prebuilt_metadata(install_dir) or {}).get("runtime_line")+ marker = load_prebuilt_metadata(install_dir) or {}+ recorded_runtime_line = marker.get("runtime_line")if not isinstance(recorded_runtime_line, str):recorded_runtime_line = Noneruntime_dir = install_runtime_dir(install_dir, host)@@ -8483,12 +8771,20 @@ def _existing_install_runs(install_dir: Path, host: HostInfo) -> bool:binaries = [runtime_dir / f"llama-{name}{ext}" for name in ("server", "quantize")]if not all(os.access(binary, os.X_OK) for binary in binaries):return False+ # A current macOS load record also replaces the `--version` probes below.+ recorded_bytes_intact = _macos_load_record_is_current(marker, host) and _runtime_files_match(+ install_dir, host, marker+ )try:# Each preflight is a no-op outside its platform.preflight_linux_installed_binaries(binaries, install_dir, host)- preflight_macos_installed_binaries(binaries, install_dir, host)+ probed = preflight_macos_installed_binaries(+ binaries, install_dir, host, load_probe = not recorded_bytes_intact+ )except Exception:return False+ if recorded_bytes_intact:+ return True# Root copies first: _find_llama_server_binary reaches them first, and without a# symlink they can rot alone.probes: list[Path] = []@@ -8504,9 +8800,16 @@ def _existing_install_runs(install_dir: Path, host: HostInfo) -> bool:continueseen.add(key)probes.append(binary)- return all(+ if not all(_binary_image_runs(binary, install_dir, host, recorded_runtime_line) for binary in probes- )+ ):+ return False+ # Last, not straight after the dyld probe: recording before the root entrypoints ran+ # would bless a corrupt wrapper into the record, and the next run would read the+ # record as current, return above, and never start the wrapper either.+ if probed:+ persist_macos_load_probe(install_dir, host)+ return Truedef existing_install_matches_choice(@@ -8545,9 +8848,11 @@ def existing_install_matches_choice(# Windows has no image-reading preflight, so a truncated llama-server.exe kept its fingerprint# match. Only a marker carrying the record is held to it.recorded_files = metadata.get("runtime_files")+ runtime_files_matched = Falseif isinstance(recorded_files, dict) and recorded_files:if not _runtime_files_match(install_dir, host, metadata):return False+ runtime_files_matched = Trueelif not (host.is_linux or host.is_macos):# The migration run, on the one platform with no loader preflight below: a pre-record# marker has no digest, so damaged bytes would become the reference they are later@@ -8583,6 +8888,9 @@ def existing_install_matches_choice([runtime_dir / "llama-server", runtime_dir / "llama-quantize"],install_dir,host,+ load_probe = not (+ runtime_files_matched and _macos_load_record_is_current(metadata, host)+ ),)except Exception:return False@@ -8690,7 +8998,9 @@ def validate_prebuilt_choice(f"{choice.install_kind} bundle {choice.name} omitted a required runtime component")preflight_linux_installed_binaries((server_path, quantize_path), install_dir, host)- preflight_macos_installed_binaries((server_path, quantize_path), install_dir, host)+ macos_load_probe_passed = preflight_macos_installed_binaries(+ (server_path, quantize_path), install_dir, host+ )ensure_repo_shape(install_dir)write_prebuilt_metadata(install_dir,@@ -8706,6 +9016,7 @@ def validate_prebuilt_choice(backend_request = backend_request,rocm_gfx = rocm_gfx,walk_back = walk_back,+ macos_load_probe_passed = macos_load_probe_passed,)# Hashless external prebuilts are not in the approved-sha256# manifest and rely on the functional smoke test as their only integrity gate,@@ -9791,7 +10102,7 @@ class BackendSelection:published_repo: strpublished_release_tag: strrequested_tag: str- release_plans: list[InstallReleasePlan]+ release_plans: "Sequence[InstallReleasePlan]"persist_llama_backend: str | Nonepersist_rocm_gfx: str | None@@ -9884,7 +10195,7 @@ def route_backend_request(def _with_rocm_behind_vulkan(- plans: list[InstallReleasePlan],+ plans: "Sequence[InstallReleasePlan]",llama_tag: str,rocm_host: HostInfo,published_repo: str,@@ -9913,7 +10224,7 @@ def _with_rocm_behind_vulkan(return planreturn dataclasses_replace(plan, attempts = attempts) if attempts else None- def _drop_cpu_only(candidates: list[InstallReleasePlan]) -> list[InstallReleasePlan]:+ def _drop_cpu_only(candidates: "Sequence[InstallReleasePlan]") -> list[InstallReleasePlan]:"""Apply _without_cpu across releases, or fail rather than install CPU."""kept = [narrowed for plan in candidates if (narrowed := _without_cpu(plan)) is not None]if candidates and not kept:@@ -9924,10 +10235,11 @@ def _with_rocm_behind_vulkan(_tag, rocm_plans = resolve_simple_install_release_plans(llama_tag, rocm_host, published_repo, published_release_tag)+ # Inside the try: later releases resolve lazily here.+ rocm_attempts = {plan.release_tag: plan.attempts for plan in rocm_plans}except Exception:# Not a reason to fail the Vulkan install, but a reason to drop the CPU tail.return _drop_cpu_only(plans)- rocm_attempts = {plan.release_tag: plan.attempts for plan in rocm_plans}out: list[InstallReleasePlan] = []for plan in plans:present = {attempt.install_kind for attempt in plan.attempts}@@ -10085,10 +10397,10 @@ def install_prebuilt(# transient 403 as "no prebuilt" for 24h; nonzero exits are never# cached, so the resolver must keep failing hard.#- # Not dead code despite the download-host fast path: macOS skips it- # entirely (see allow_download_host_fast_path below), as does a- # non-latest requested tag without a published-release pin, a- # non-default --published-repo, and any CDN outage.+ # Not dead code despite the download-host fast path: a macOS walk-back+ # past the newest release lists the API, as does a non-latest+ # requested tag without a published-release pin, a non-default+ # --published-repo, and any CDN outage.## Transport shapes only: URLError covers HTTPError and the socket/DNS# errors urllib wraps, JSONDecodeError is a ValueError, and@@ -10126,18 +10438,8 @@ def install_prebuilt()except (BusyInstallConflict, PrebuiltFallback):raise- except (- urllib.error.URLError,- ssl.SSLError,- ConnectionError,- TimeoutError,- RuntimeError,- ValueError,- ) as exc:- raise PrebuiltFallback(- f"failed to inspect published releases in "- f"{published_repo or DEFAULT_PUBLISHED_REPO}: {exc}"- ) from exc+ except RELEASE_LISTING_TRANSPORT_ERRORS as exc:+ raise release_listing_fallback(exc, published_repo) from exc# Before the listing, manifest and checksum fetches, which on a current install resolve# back to the bundle on disk. One HEAD instead. Routed once and handed to both, since@@ -10226,20 +10528,11 @@ def install_prebuilt(work_dir / "stories260K.gguf",validation_model_cache_path(install_dir),)- # Same reason as the per-candidate guard in validate_prebuilt_attempts,- # one level up: the per-release handler below also swallows- # PrebuiltFallback and moves to an older plan, so a probe failure raised- # inside it would install an older llama.cpp over a transient 429. The- # probe is independent of which release was picked, so resolve it once- # here when any plan will smoke-test.- if staged_validation_enabled() or any(- attempt.expected_sha256 is None- for release_plan in release_plans- for attempt in release_plan.attempts+ probe_resolved = False+ last_failure: PrebuiltFallback | None = None+ for release_index, plan in enumerate(+ iter_release_plans(release_plans, published_repo)):- probe = resolve_validation_model(probe)- release_count = len(release_plans)- for release_index, plan in enumerate(release_plans):choice = plan.attempts[0]backfill = diffusion_visual_server_backfill_needed(install_dir, host, choice)if existing_install_matches_plan(install_dir, host, plan):@@ -10261,6 +10554,13 @@ def install_prebuilt(f"{choice.name} ({choice.source_label}) from published release "f"{plan.release_tag} for {host.system} {host.machine}")+ # Outside the handler, so a transient failure cannot demote to an older release.+ if not probe_resolved and (+ staged_validation_enabled()+ or any(attempt.expected_sha256 is None for attempt in plan.attempts)+ ):+ probe = resolve_validation_model(probe)+ probe_resolved = Truetry:choice, selected_staging_dir, _ = validate_prebuilt_attempts(plan.attempts,@@ -10290,12 +10590,11 @@ def install_prebuilt(except PrebuiltFallback as exc:if _environment_fatal_reason(exc):raise- if release_index == release_count - 1:- raise+ last_failure = exclog("published release "f"{plan.release_tag} upstream_tag={plan.llama_tag} failed; "- "trying the next older published prebuilt "+ "trying an older published prebuilt if one remains "f"({textwrap.shorten(str(exc), width = 200, placeholder = '...')})")continue@@ -10318,6 +10617,8 @@ def install_prebuilt(f"({textwrap.shorten(str(exc), width = 200, placeholder = '...')})")return+ if last_failure is not None:+ raise last_failureexcept BusyInstallConflict as exc:log("prebuilt install path is blocked by an in-use llama.cpp install")log(f"prebuilt busy reason: {exc}")diff --git a/tests/studio/install/test_download_host_resolve.py b/tests/studio/install/test_download_host_resolve.pyindex 0c3ccb5d22b..ed72a3684e9 100644--- a/tests/studio/install/test_download_host_resolve.py+++ b/tests/studio/install/test_download_host_resolve.py@@ -71,7 +71,6 @@ def test_fast_path_yields_pinned_tag_without_api(monkeypatch):def test_fast_path_disabled_by_caller_uses_api(monkeypatch):- # macOS passes allow_download_host_fast_path = False to keep the walk-back.monkeypatch.delenv("UNSLOTH_LLAMA_DISABLE_DOWNLOAD_HOST_RESOLVE", raising = False)monkeypatch.setattr(ILP, "_download_host_resolved_release", _fast_path_raises)monkeypatch.setattr(ILP, "iter_published_release_bundles", _empty_api)diff --git a/tests/studio/install/test_install_llama_prebuilt_logic.py b/tests/studio/install/test_install_llama_prebuilt_logic.pyindex 9904d6b7bdd..0d69b451a95 100644--- a/tests/studio/install/test_install_llama_prebuilt_logic.py+++ b/tests/studio/install/test_install_llama_prebuilt_logic.py@@ -7010,6 +7010,23 @@ def test_the_planner_records_the_newest_release_a_mac_walked_past(monkeypatch):assert plans[0].walk_back is None+def test_a_late_rocm_listing_failure_keeps_the_vulkan_plan(monkeypatch):+ module = INSTALL_LLAMA_PREBUILT+ vulkan = release_plan([asset_choice(install_kind = "linux-vulkan")], release_checksums())++ def rocm_plans():+ yield vulkan+ raise urllib.error.URLError("CDN down, API timed out")++ monkeypatch.setattr(+ module,+ "resolve_simple_install_release_plans",+ lambda *args: ("latest", module.LazyReleasePlans(rocm_plans())),+ )+ kept = module._with_rocm_behind_vulkan([vulkan], "latest", linux_host(), "", "")+ assert [plan.release_tag for plan in kept] == [vulkan.release_tag]++def test_a_reused_marker_takes_the_walk_back_this_run_made():"""sync_marker_selection: a kept install on a Mac gains the walk-back record amarker written before it existed lacks (both keys), and a plan that no longerdiff --git a/tests/studio/install/test_macos_loader_probe_record.py b/tests/studio/install/test_macos_loader_probe_record.pynew file mode 100644index 00000000000..0cfee06835e--- /dev/null+++ b/tests/studio/install/test_macos_loader_probe_record.py@@ -0,0 +1,574 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Tests for reusing a recorded macOS dyld probe.++The probe may be skipped only when the runtime digests and host profile still match.+"""++from __future__ import annotations++import json+import subprocess+import sys+import types+from pathlib import Path++import pytest++TEST_DIR = Path(__file__).resolve().parent+if str(TEST_DIR) not in sys.path:+ sys.path.insert(0, str(TEST_DIR))++from _pr10648_helpers import POSIX_ONLY, llama_host, load_studio_module # noqa: E402++# Use an isolated module because tests monkeypatch its globals.+ILP = load_studio_module(+ "studio_install_llama_prebuilt_macos_loader_record", "install_llama_prebuilt.py"+)++RELEASE_TAG = "release-1"+UPSTREAM_TAG = "b9001"+PUBLISHED_REPO = "unslothai/llama.cpp"+INSTALL_KIND = "macos-arm64"+++PRODUCT_VERSION = "15.5.1"+# Captured before the autouse fixture stubs it, for the tests that exercise the real one.+_REAL_MACOS_PRODUCT_VERSION = ILP.macos_product_version++[email protected](autouse = True)+def _clear_full_check(monkeypatch):+ monkeypatch.delenv("UNSLOTH_PREBUILT_FULL_CHECK", raising = False)+ # platform.mac_ver() is empty off macOS, so the probe record would never be written+ # and every test here would read as "no evidence". Pin the host's own answer instead.+ monkeypatch.setattr(ILP, "macos_product_version", lambda: PRODUCT_VERSION)+++def macos_host(**overrides):+ return llama_host(+ ILP.HostInfo,+ system = "Darwin",+ machine = "arm64",+ macos_version = overrides.pop("macos_version", (15, 5)),+ **overrides,+ )+++def choice_for(host) -> "ILP.AssetChoice":+ return ILP.AssetChoice(+ repo = PUBLISHED_REPO,+ tag = RELEASE_TAG,+ name = f"llama-{UPSTREAM_TAG}-bin-{INSTALL_KIND}.tar.gz",+ url = f"https://example.com/llama-{UPSTREAM_TAG}-bin-{INSTALL_KIND}.tar.gz",+ source_label = "published",+ install_kind = INSTALL_KIND,+ expected_sha256 = "a" * 64,+ )+++def checksums_for(choice) -> "ILP.ApprovedReleaseChecksums":+ logical = ILP.source_archive_logical_name(UPSTREAM_TAG)+ return ILP.ApprovedReleaseChecksums(+ repo = PUBLISHED_REPO,+ release_tag = RELEASE_TAG,+ upstream_tag = UPSTREAM_TAG,+ source_commit = "deadbeef",+ artifacts = {+ logical: ILP.ApprovedArtifactHash(+ asset_name = logical,+ sha256 = "b" * 64,+ repo = "ggml-org/llama.cpp",+ kind = "upstream-source",+ ),+ choice.name: ILP.ApprovedArtifactHash(+ asset_name = choice.name,+ sha256 = choice.expected_sha256,+ repo = PUBLISHED_REPO,+ kind = "prebuilt",+ ),+ },+ )+++def build_install(+ tmp_path: Path,+ host,+ *,+ load_probe_passed: bool = True,+) -> Path:+ """Build an install whose marker passes the real fingerprint checks.++ *load_probe_passed* mirrors what the installer learned from its own preflight:+ True is the normal install, where dyld resolved every binary. False is the+ install whose probe timed out or could not spawn, which must not be remembered+ as a pass.+ """+ install_dir = tmp_path / "llama.cpp"+ runtime_dir = install_dir / "build" / "bin"+ runtime_dir.mkdir(parents = True)+ for name in ("llama-server", "llama-quantize", "llama-diffusion-gemma-visual-server"):+ for directory in (install_dir, runtime_dir):+ binary = directory / name+ binary.write_text("#!/bin/sh\nexit 0\n", encoding = "utf-8")+ binary.chmod(0o755)+ for dylib in ("libllama.0.dylib", "libggml.0.dylib", "libmtmd.0.dylib"):+ (runtime_dir / dylib).write_bytes(b"DYLIB")+ (install_dir / "convert_hf_to_gguf.py").write_text("#!/usr/bin/env python3\n", encoding = "utf-8")+ (install_dir / "gguf-py" / "gguf").mkdir(parents = True)++ choice = choice_for(host)+ ILP.write_prebuilt_metadata(+ install_dir,+ host = host,+ requested_tag = "latest",+ llama_tag = UPSTREAM_TAG,+ release_tag = RELEASE_TAG,+ choice = choice,+ approved_checksums = checksums_for(choice),+ prebuilt_fallback_used = False,+ backend_request = "auto",+ macos_load_probe_passed = load_probe_passed,+ )+ return install_dir+++def marker_of(install_dir: Path) -> dict:+ return json.loads((install_dir / "UNSLOTH_PREBUILT_INFO.json").read_text(encoding = "utf-8"))+++def write_marker(install_dir: Path, marker: dict) -> None:+ (install_dir / "UNSLOTH_PREBUILT_INFO.json").write_text(+ json.dumps(marker, indent = 2) + "\n", encoding = "utf-8"+ )+++def count_spawns(monkeypatch) -> list[int]:+ """Stand in for a probe that RAN and loaded every binary.++ Reporting the load matters: the real probe fails open, so preflight treats an+ empty *loaded* as "could not run" and refuses to record a pass. A stub that only+ returned [] would silently test the no-evidence path instead.+ """+ calls = [0]++ def _probe(+ binaries,+ install_dir,+ host,+ *,+ loaded = None,+ **_kwargs,+ ):+ calls[0] += 1+ if loaded is not None:+ loaded.update(path.name for path in binaries)+ return []++ monkeypatch.setattr(ILP, "macos_dyld_load_issues", _probe)+ return calls+++def matches_choice(install_dir: Path, host) -> bool:+ choice = choice_for(host)+ return ILP.existing_install_matches_choice(+ install_dir,+ host,+ llama_tag = UPSTREAM_TAG,+ release_tag = RELEASE_TAG,+ choice = choice,+ approved_checksums = checksums_for(choice),+ )+++def fast_path(install_dir: Path, host) -> bool:+ route = ILP.BackendRoute(+ backend = None,+ host = host,+ published_repo = PUBLISHED_REPO,+ published_release_tag = RELEASE_TAG,+ persist_llama_backend = None,+ persist_rocm_gfx = None,+ )+ return ILP.existing_install_current_without_plan(+ install_dir,+ llama_tag = UPSTREAM_TAG,+ published_repo = PUBLISHED_REPO,+ published_release_tag = RELEASE_TAG,+ backend_request = "auto",+ force_cpu = False,+ route = route,+ )+++def test_reuse_with_a_current_record_does_not_probe(tmp_path: Path, monkeypatch):+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ calls = count_spawns(monkeypatch)++ assert matches_choice(install_dir, host) is True+ assert calls[0] == 0+++def test_fast_path_with_a_current_record_does_not_probe(tmp_path: Path, monkeypatch):+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ calls = count_spawns(monkeypatch)++ assert fast_path(install_dir, host) is True+ assert calls[0] == 0+++def test_check_installed_with_a_current_record_does_not_probe(tmp_path: Path, monkeypatch):+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ calls = count_spawns(monkeypatch)+ # The other spawn on this path: _existing_install_runs starts each binary itself.+ monkeypatch.setattr(ILP, "_binary_image_runs", lambda *a, **k: pytest.fail("started a binary"))++ assert ILP._existing_install_runs(install_dir, host) is True+ assert calls[0] == 0+++def test_a_marker_without_a_byte_record_still_probes(tmp_path: Path, monkeypatch):+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ marker = marker_of(install_dir)+ marker.pop("runtime_files")+ write_marker(install_dir, marker)+ calls = count_spawns(monkeypatch)++ assert matches_choice(install_dir, host) is True+ assert calls[0] == 1+++def test_a_size_only_dylib_record_still_probes(tmp_path: Path, monkeypatch):+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ marker = marker_of(install_dir)+ # A marker written before macOS hashed its payload tier.+ marker["runtime_files"]["build/bin/libggml.0.dylib"].pop("sha256")+ write_marker(install_dir, marker)+ calls = count_spawns(monkeypatch)++ assert matches_choice(install_dir, host) is True+ assert calls[0] == 1+++@POSIX_ONLY+def test_a_same_size_dylib_rewrite_is_rejected(tmp_path: Path, monkeypatch):+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ dylib = install_dir / "build" / "bin" / "libggml.0.dylib"+ before = dylib.stat().st_size+ dylib.write_bytes(b"ROTTE")+ assert dylib.stat().st_size == before+ calls = count_spawns(monkeypatch)++ assert matches_choice(install_dir, host) is False+ assert fast_path(install_dir, host) is False+ # Rejected on the record, before anything was started.+ assert calls[0] == 0+ # _existing_install_runs answers a different question, so it only loses the skip.+ assert ILP._existing_install_runs(install_dir, host) is True+ assert calls[0] == 1+++def test_a_marker_without_a_host_profile_still_probes(tmp_path: Path, monkeypatch):+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ marker = marker_of(install_dir)+ marker.pop("host_profile")+ write_marker(install_dir, marker)+ calls = count_spawns(monkeypatch)++ assert matches_choice(install_dir, host) is True+ assert calls[0] == 1+++def test_a_macos_upgrade_probes_again(tmp_path: Path, monkeypatch):+ install_dir = build_install(tmp_path, macos_host(macos_version = (15, 5)))+ upgraded = macos_host(macos_version = (26, 0))+ calls = count_spawns(monkeypatch)+ # The minos read is the other half of the preflight and reads real Mach-O headers.+ monkeypatch.setattr(ILP, "macos_binary_minos_issues", lambda *a, **k: [])++ assert matches_choice(install_dir, upgraded) is True+ assert calls[0] == 1+++def test_an_unknown_macos_version_still_probes(tmp_path: Path, monkeypatch):+ host = macos_host(macos_version = None)+ install_dir = build_install(tmp_path, host)+ calls = count_spawns(monkeypatch)+ monkeypatch.setattr(ILP, "_binary_image_runs", lambda *a, **k: True)++ assert matches_choice(install_dir, host) is True+ assert ILP._existing_install_runs(install_dir, host) is True+ assert calls[0] == 2+++def test_full_check_forces_the_probe(tmp_path: Path, monkeypatch):+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ monkeypatch.setenv("UNSLOTH_PREBUILT_FULL_CHECK", "1")+ calls = count_spawns(monkeypatch)++ assert matches_choice(install_dir, host) is True+ assert calls[0] == 1+++def test_a_replaced_binary_is_still_rejected(tmp_path: Path, monkeypatch):+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ (install_dir / "build" / "bin" / "llama-server").write_text(+ "#!/bin/sh\nexit 1\n# swapped\n", encoding = "utf-8"+ )+ calls = count_spawns(monkeypatch)++ assert matches_choice(install_dir, host) is False+ assert fast_path(install_dir, host) is False+ # Rejected on the record, before anything was started.+ assert calls[0] == 0+++def test_a_bundle_that_cannot_load_is_rejected_when_it_is_probed(tmp_path: Path, monkeypatch):+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ marker = marker_of(install_dir)+ marker.pop("runtime_files")+ write_marker(install_dir, marker)+ monkeypatch.setattr(+ ILP,+ "macos_dyld_load_issues",+ lambda *a, **k: ["llama-server: Library not loaded: /usr/lib/librdma.dylib"],+ )++ assert matches_choice(install_dir, host) is False+++@POSIX_ONLY+def test_an_install_whose_probe_never_ran_is_not_recorded_as_a_pass(tmp_path: Path, monkeypatch):+ """The probe fails open, so "no issues" is not the same as "it loaded"."""+ host = macos_host()+ install_dir = build_install(tmp_path, host, load_probe_passed = False)++ assert ILP.MACOS_LOAD_PROBE_KEY not in marker_of(install_dir)+ calls = count_spawns(monkeypatch)+ # Every byte still matches; the missing evidence alone is what costs a probe.+ assert ILP._existing_install_runs(install_dir, host) is True+ assert calls[0] == 1+++def test_a_timed_out_probe_reports_no_pass(tmp_path: Path, monkeypatch):+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ runtime_dir = install_dir / "build" / "bin"+ binaries = [runtime_dir / "llama-server", runtime_dir / "llama-quantize"]++ def _timeout(*_args, **_kwargs):+ raise subprocess.TimeoutExpired(cmd = "llama-server", timeout = 60)++ monkeypatch.setattr(ILP, "run_capture", _timeout)+ monkeypatch.setattr(ILP, "macos_binary_minos_issues", lambda *a, **k: [])++ # Still not a rejection: a loaded machine must not lose a healthy bundle.+ assert ILP.preflight_macos_installed_binaries(binaries, install_dir, host) is False+++def test_a_probe_that_ran_reports_a_pass(tmp_path: Path, monkeypatch):+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ runtime_dir = install_dir / "build" / "bin"+ binaries = [runtime_dir / "llama-server", runtime_dir / "llama-quantize"]++ monkeypatch.setattr(+ ILP,+ "run_capture",+ lambda *a, **k: types.SimpleNamespace(returncode = 0, stdout = "", stderr = ""),+ )+ monkeypatch.setattr(ILP, "macos_binary_minos_issues", lambda *a, **k: [])++ assert ILP.preflight_macos_installed_binaries(binaries, install_dir, host) is True+++@POSIX_ONLY+def test_a_macos_patch_update_probes_again(tmp_path: Path, monkeypatch):+ """host_profile is (major, minor), so only the recorded product version sees this."""+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ calls = count_spawns(monkeypatch)+ # 15.5.1 -> 15.5.2 leaves host_profile identical, but replaces the dyld shared cache.+ monkeypatch.setattr(ILP, "macos_product_version", lambda: "15.5.2")++ assert ILP._existing_install_runs(install_dir, host) is True+ assert calls[0] == 1+++@POSIX_ONLY+def test_a_reuse_probe_that_passed_is_remembered(tmp_path: Path, monkeypatch):+ """Otherwise the skip is unreachable for every install not born with the record."""+ host = macos_host()+ install_dir = build_install(tmp_path, host, load_probe_passed = False)+ assert ILP.MACOS_LOAD_PROBE_KEY not in marker_of(install_dir)++ calls = count_spawns(monkeypatch)+ assert ILP._existing_install_runs(install_dir, host) is True+ assert calls[0] == 1 # this update pays for the probe+ assert marker_of(install_dir)[ILP.MACOS_LOAD_PROBE_KEY] == {+ "passed": True,+ "macos_product_version": PRODUCT_VERSION,+ }++ assert ILP._existing_install_runs(install_dir, host) is True+ assert calls[0] == 1 # the next one does not+++@POSIX_ONLY+def test_a_legacy_size_only_marker_is_upgraded_by_a_passing_probe(tmp_path: Path, monkeypatch):+ """The shape every install written before this PR has: payload recorded, not hashed."""+ host = macos_host()+ install_dir = build_install(tmp_path, host, load_probe_passed = False)+ marker = marker_of(install_dir)+ for key, entry in marker["runtime_files"].items():+ if not key.endswith(("llama-server", "llama-quantize")):+ entry.pop("sha256", None)+ marker.pop(ILP.MACOS_LOAD_PROBE_KEY, None)+ write_marker(install_dir, marker)++ calls = count_spawns(monkeypatch)+ assert ILP._existing_install_runs(install_dir, host) is True+ assert calls[0] == 1+ upgraded = marker_of(install_dir)["runtime_files"]+ assert all(entry.get("sha256") for entry in upgraded.values())++ assert ILP._existing_install_runs(install_dir, host) is True+ assert calls[0] == 1+++def test_the_reuse_fast_paths_do_not_record_a_pass(tmp_path: Path, monkeypatch):+ """They probe build/bin only, so they never reach a root wrapper and cannot vouch+ for a record that a later run uses to skip starting one."""+ host = macos_host()+ install_dir = build_install(tmp_path, host, load_probe_passed = False)+ calls = count_spawns(monkeypatch)++ assert matches_choice(install_dir, host) is True+ assert fast_path(install_dir, host) is True+ assert calls[0] == 2 # both probed+ assert ILP.MACOS_LOAD_PROBE_KEY not in marker_of(install_dir)+++@POSIX_ONLY+def test_a_broken_root_wrapper_is_not_blessed_into_the_record(tmp_path: Path, monkeypatch):+ """The root entrypoint can be its own file, and it is started AFTER the dyld probe.++ Recording between the two would hash the broken wrapper into the digest record, and+ the next run would read that record as current and never start the wrapper again.+ """+ host = macos_host()+ install_dir = build_install(tmp_path, host, load_probe_passed = False)+ marker = marker_of(install_dir)+ for key, entry in marker["runtime_files"].items(): # the pre-change shape+ if not key.endswith(("llama-server", "llama-quantize")):+ entry.pop("sha256", None)+ write_marker(install_dir, marker)+ count_spawns(monkeypatch)+ broken = install_dir / "llama-server"++ def _runs(path, *_a, **_k):+ return Path(path) != broken++ monkeypatch.setattr(ILP, "_binary_image_runs", _runs)++ assert ILP._existing_install_runs(install_dir, host) is False+ marker = marker_of(install_dir)+ assert ILP.MACOS_LOAD_PROBE_KEY not in marker+ # And the legacy size-only record was not upgraded on the strength of it either.+ assert not all(entry.get("sha256") for entry in marker["runtime_files"].values())+++def test_a_probe_that_could_not_run_is_not_remembered(tmp_path: Path, monkeypatch):+ """Only a real load is evidence, so a fail-open probe must not persist one."""+ host = macos_host()+ install_dir = build_install(tmp_path, host, load_probe_passed = False)+ monkeypatch.setattr(ILP, "macos_binary_minos_issues", lambda *a, **k: [])++ def _timeout(*_args, **_kwargs):+ raise subprocess.TimeoutExpired(cmd = "llama-server", timeout = 60)++ monkeypatch.setattr(ILP, "run_capture", _timeout)+ monkeypatch.setattr(ILP, "_binary_image_runs", lambda *a, **k: True)++ assert ILP._existing_install_runs(install_dir, host) is True+ assert ILP.MACOS_LOAD_PROBE_KEY not in marker_of(install_dir)+++@POSIX_ONLY+def test_a_rapid_security_response_probes_again(tmp_path: Path, monkeypatch):+ """An RSR leaves ProductVersion alone and moves ProductVersionExtra and the build."""+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ calls = count_spawns(monkeypatch)+ monkeypatch.setattr(ILP, "macos_product_version", lambda: PRODUCT_VERSION + " (a) 22E772610a")++ assert ILP._existing_install_runs(install_dir, host) is True+ assert calls[0] == 1+++def test_the_recorded_version_carries_the_rsr_and_build_fields(monkeypatch):+ """platform.mac_ver() alone cannot see an RSR, so sw_vers is what gets asked."""+ seen = []++ def _sw_vers(cmd, **_kwargs):+ seen.append(cmd)+ return types.SimpleNamespace(+ returncode = 0,+ stdout = (+ "ProductName:\tmacOS\n"+ "ProductVersion:\t13.3.1\n"+ "ProductVersionExtra:\t(a)\n"+ "BuildVersion:\t22E772610a\n"+ ),+ stderr = "",+ )++ monkeypatch.setattr(ILP.subprocess, "run", _sw_vers)+ assert _REAL_MACOS_PRODUCT_VERSION() == "13.3.1 (a) 22E772610a"+ # Bare: sw_vers documents single-dash options, so no flag is spelled at all.+ assert seen == [["/usr/bin/sw_vers"]]+++def test_no_rsr_installed_still_yields_a_version(monkeypatch):+ """ProductVersionExtra is simply absent with no RSR, and predates macOS 13."""++ def _sw_vers(_cmd, **_kwargs):+ return types.SimpleNamespace(+ returncode = 0,+ stdout = "ProductName:\tmacOS\nProductVersion:\t15.5.1\nBuildVersion:\t24F74\n",+ stderr = "",+ )++ monkeypatch.setattr(ILP.subprocess, "run", _sw_vers)+ assert _REAL_MACOS_PRODUCT_VERSION() == "15.5.1 24F74"+++def test_a_failing_sw_vers_falls_back_to_the_product_version(monkeypatch):+ """A fallback that still catches a point release beats recording nothing."""++ def _boom(_cmd, **_kwargs):+ raise OSError("sw_vers not found")++ monkeypatch.setattr(ILP.subprocess, "run", _boom)+ monkeypatch.setattr(ILP.platform, "mac_ver", lambda: ("15.5.1", ("", "", ""), "arm64"))+ assert _REAL_MACOS_PRODUCT_VERSION() == "15.5.1"+++def test_the_same_macos_patch_level_still_skips(tmp_path: Path, monkeypatch):+ host = macos_host()+ install_dir = build_install(tmp_path, host)+ calls = count_spawns(monkeypatch)++ assert ILP._existing_install_runs(install_dir, host) is True+ assert calls[0] == 0diff --git a/tests/studio/install/test_release_plan_laziness.py b/tests/studio/install/test_release_plan_laziness.pynew file mode 100644index 00000000000..fac285425b2--- /dev/null+++ b/tests/studio/install/test_release_plan_laziness.py@@ -0,0 +1,359 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Tests for lazy release walk-back and its CDN fast path."""++from __future__ import annotations++import sys+from pathlib import Path++import pytest++TEST_DIR = Path(__file__).resolve().parent+if str(TEST_DIR) not in sys.path:+ sys.path.insert(0, str(TEST_DIR))++from _pr10648_helpers import llama_host, load_studio_module # noqa: E402++from test_keep_install_backcompat_9979 import MACOS, S12, build_install # noqa: E402++ILP = load_studio_module("studio_install_llama_prebuilt_plan_laziness", "install_llama_prebuilt.py")++PUBLISHED_REPO = "unslothai/llama.cpp"+RELEASE_TAGS = [f"b{11000 - index}-mix-abcdef0" for index in range(20)]+++def macos_host(**overrides):+ return llama_host(+ ILP.HostInfo,+ system = "Darwin",+ machine = "arm64",+ macos_version = overrides.pop("macos_version", (15, 5)),+ **overrides,+ )+++def linux_host(**overrides):+ return llama_host(ILP.HostInfo, system = "Linux", machine = "x86_64", **overrides)+++class ApiRateLimited(RuntimeError):+ """A 403 in the form returned by fetch_json."""+++class FakeReleases:+ def __init__(+ self,+ monkeypatch,+ *,+ reject: "set[str] | None" = None,+ cdn_tag: str | None = None,+ api_error: "BaseException | None" = None,+ ):+ self.resolved: list[str] = []+ self._api_error = api_error+ self.api_listing_reads = 0+ self.cdn_reads = 0+ self._reject = reject or set()+ self._cdn_tag = cdn_tag++ monkeypatch.setattr(ILP, "_download_host_resolve_enabled", lambda: cdn_tag is not None)+ monkeypatch.setattr(ILP, "_download_host_resolved_release", self._cdn_release)+ monkeypatch.setattr(ILP, "iter_published_release_bundles", self._api_bundles)+ monkeypatch.setattr(ILP, "validated_checksums_for_bundle", self._checksums)+ monkeypatch.setattr(ILP, "resolve_release_asset_choice", self._attempts)+ monkeypatch.setattr(+ ILP,+ "_linux_published_attempts",+ lambda host, bundle: self._attempts(host, bundle.upstream_tag, bundle, None),+ )+ monkeypatch.setattr(ILP, "apply_approved_hashes", lambda attempts, checksums: attempts)++ def _bundle(self, release_tag: str):+ return ILP.PublishedReleaseBundle(+ repo = PUBLISHED_REPO,+ release_tag = release_tag,+ upstream_tag = release_tag.split("-")[0],+ assets = {},+ )++ def _cdn_release(+ self,+ repo: str,+ published_release_tag: str = "",+ ):+ if self._cdn_tag is None:+ return None+ self.cdn_reads += 1+ bundle = self._bundle(self._cdn_tag)+ return ILP.ResolvedPublishedRelease(bundle = bundle, checksums = self._checksums(repo, bundle))++ def _api_bundles(self, repo: str):+ self.api_listing_reads += 1+ if self._api_error is not None:+ raise self._api_error+ for release_tag in RELEASE_TAGS:+ yield self._bundle(release_tag)++ def _checksums(self, repo: str, bundle):+ return ILP.ApprovedReleaseChecksums(+ repo = repo,+ release_tag = bundle.release_tag,+ upstream_tag = bundle.upstream_tag,+ source_commit = "deadbeef",+ artifacts = {},+ )++ def _attempts(self, host, resolved_tag, bundle, checksums):+ self.resolved.append(bundle.release_tag)+ if bundle.release_tag in self._reject:+ return []+ return [+ ILP.AssetChoice(+ repo = PUBLISHED_REPO,+ tag = bundle.release_tag,+ name = f"llama-{bundle.upstream_tag}-bin-macos-arm64.tar.gz",+ url = f"https://example.com/{bundle.upstream_tag}.tar.gz",+ source_label = "published",+ install_kind = "macos-arm64",+ expected_sha256 = "a" * 64,+ )+ ]+++def resolve(host):+ return ILP.resolve_simple_install_release_plans("latest", host, PUBLISHED_REPO, "")+++def test_a_usable_newest_release_resolves_exactly_one_plan(monkeypatch):+ releases = FakeReleases(monkeypatch)+ _tag, plans = resolve(macos_host())++ assert plans[0].release_tag == RELEASE_TAGS[0]+ assert releases.resolved == [RELEASE_TAGS[0]]+++def test_the_walk_back_still_reaches_older_releases_in_order(monkeypatch):+ releases = FakeReleases(monkeypatch, reject = set(RELEASE_TAGS[:3]))+ _tag, plans = resolve(macos_host())++ assert plans[0].release_tag == RELEASE_TAGS[3]+ assert releases.resolved == RELEASE_TAGS[:4]+++def test_asking_for_the_next_plan_resolves_one_more_and_no_further(monkeypatch):+ releases = FakeReleases(monkeypatch)+ _tag, plans = resolve(macos_host())+ assert releases.resolved == [RELEASE_TAGS[0]]++ assert plans[1].release_tag == RELEASE_TAGS[1]+ assert releases.resolved == RELEASE_TAGS[:2]+++def test_the_walk_back_is_still_capped(monkeypatch):+ releases = FakeReleases(monkeypatch)+ _tag, plans = resolve(macos_host())++ assert len(list(plans)) == ILP.DEFAULT_MAX_MACOS_RELEASE_FALLBACKS+ assert releases.resolved == RELEASE_TAGS[: ILP.DEFAULT_MAX_MACOS_RELEASE_FALLBACKS]+++def test_linux_still_takes_the_newest_release_from_the_download_host(monkeypatch):+ releases = FakeReleases(monkeypatch, cdn_tag = RELEASE_TAGS[0])+ _tag, plans = resolve(linux_host())++ assert len(list(plans)) == 1 # the CDN surfaces only the newest, as before+ assert releases.resolved == [RELEASE_TAGS[0]]+ assert releases.api_listing_reads == 0+++def test_macos_takes_the_newest_release_from_the_download_host(monkeypatch):+ releases = FakeReleases(monkeypatch, cdn_tag = RELEASE_TAGS[0])+ _tag, plans = resolve(macos_host())++ assert plans[0].release_tag == RELEASE_TAGS[0]+ assert releases.cdn_reads == 1+ assert releases.api_listing_reads == 0+++def test_macos_falls_back_to_the_api_only_when_it_must_walk(monkeypatch):+ releases = FakeReleases(monkeypatch, reject = {RELEASE_TAGS[0]}, cdn_tag = RELEASE_TAGS[0])+ _tag, plans = resolve(macos_host())++ assert plans[0].release_tag == RELEASE_TAGS[1]+ assert releases.cdn_reads == 1+ assert releases.api_listing_reads == 1+ assert releases.resolved == RELEASE_TAGS[:2]+++def test_a_walk_that_finds_nothing_usable_still_fails(monkeypatch):+ FakeReleases(monkeypatch, reject = set(RELEASE_TAGS))++ with pytest.raises(ILP.PrebuiltFallback):+ resolve(macos_host())+++def test_the_first_plan_is_resolved_before_the_resolver_returns(monkeypatch):+ def _explode(repo: str):+ raise RuntimeError("GitHub API returned 403")++ monkeypatch.setattr(ILP, "_download_host_resolve_enabled", lambda: False)+ monkeypatch.setattr(ILP, "iter_published_release_bundles", _explode)++ with pytest.raises(RuntimeError, match = "403"):+ resolve(macos_host())+++def test_a_403_on_the_deferred_lookup_reaches_the_installer_as_a_fallback(monkeypatch):+ releases = FakeReleases(+ monkeypatch,+ cdn_tag = RELEASE_TAGS[0],+ api_error = ApiRateLimited("GitHub API returned 403 for .../releases?per_page=100"),+ )+ _tag, plans = resolve(macos_host())+ remaining = ILP.iter_release_plans(plans, PUBLISHED_REPO)+ assert next(remaining).release_tag == RELEASE_TAGS[0]+ assert releases.api_listing_reads == 0++ with pytest.raises(ILP.PrebuiltFallback, match = "failed to inspect published releases"):+ next(remaining)+ assert releases.api_listing_reads == 1+++def test_a_pinned_release_resolves_once_even_when_the_walk_back_is_on(monkeypatch):+ """macOS sets continue_after_fast_path by host, not by whether a tag is pinned.++ A pin names exactly one release, so the download host's answer is the whole+ answer; falling through would yield it a second time and spend an API call.+ """+ releases = FakeReleases(monkeypatch, cdn_tag = RELEASE_TAGS[0])+ pinned_lookups = []+ monkeypatch.setattr(+ ILP,+ "pinned_published_release_bundle",+ lambda repo, tag: pinned_lookups.append(tag) or releases._bundle(tag),+ )++ resolved = list(+ ILP.iter_resolved_published_releases(+ "latest",+ PUBLISHED_REPO,+ RELEASE_TAGS[0],+ continue_after_fast_path = True,+ )+ )++ assert [item.bundle.release_tag for item in resolved] == [RELEASE_TAGS[0]]+ assert pinned_lookups == []+ assert releases.api_listing_reads == 0+++def test_a_pin_without_the_cdn_still_reaches_the_api(monkeypatch):+ """The guard must not cost the pinned lookup when the fast path produced nothing."""+ releases = FakeReleases(monkeypatch)+ pinned_lookups = []+ monkeypatch.setattr(+ ILP,+ "pinned_published_release_bundle",+ lambda repo, tag: pinned_lookups.append(tag) or releases._bundle(tag),+ )++ resolved = list(+ ILP.iter_resolved_published_releases(+ "latest",+ PUBLISHED_REPO,+ RELEASE_TAGS[0],+ continue_after_fast_path = True,+ )+ )++ assert [item.bundle.release_tag for item in resolved] == [RELEASE_TAGS[0]]+ assert pinned_lookups == [RELEASE_TAGS[0]]+++def test_a_failed_walk_reports_the_same_failure_every_time(monkeypatch):+ """A closed generator must not read as a short, clean sequence on the second pass."""++ def _plans():+ yield "p0"+ raise ApiRateLimited("GitHub API returned 403")++ plans = ILP.LazyReleasePlans(_plans())+ assert plans[0] == "p0"++ with pytest.raises(ApiRateLimited):+ list(plans)+ # Same question, same answer: without this the second pass returns ["p0"] quietly.+ with pytest.raises(ApiRateLimited):+ list(plans)+ with pytest.raises(ApiRateLimited):+ len(plans)+++def test_the_resolver_itself_still_fails_hard_on_a_403(monkeypatch):+ FakeReleases(+ monkeypatch,+ api_error = ApiRateLimited("GitHub API returned 403 for .../releases?per_page=100"),+ )++ with pytest.raises(ApiRateLimited):+ resolve(macos_host())+++def test_a_deferred_403_still_reaches_the_source_build_end_to_end(tmp_path, monkeypatch):+ plan = ILP.InstallReleasePlan(+ requested_tag = "latest",+ llama_tag = "b11000",+ release_tag = RELEASE_TAGS[0],+ attempts = [+ ILP.AssetChoice(+ repo = PUBLISHED_REPO,+ tag = RELEASE_TAGS[0],+ name = "llama-b11000-bin-macos-arm64.tar.gz",+ url = "https://example.com/llama-b11000-bin-macos-arm64.tar.gz",+ source_label = "published",+ install_kind = "macos-arm64",+ expected_sha256 = "a" * 64,+ )+ ],+ approved_checksums = ILP.ApprovedReleaseChecksums(+ repo = PUBLISHED_REPO,+ release_tag = RELEASE_TAGS[0],+ upstream_tag = "b11000",+ source_commit = "deadbeef",+ artifacts = {},+ ),+ )++ def _plans():+ yield plan+ raise ApiRateLimited("GitHub API returned 403 for .../releases?per_page=100")++ monkeypatch.setattr(+ ILP,+ "_fork_manifest_release_plans",+ lambda *a, **k: ("latest", ILP.LazyReleasePlans(_plans())),+ )+ monkeypatch.setattr(ILP, "detect_host", lambda *a, **k: MACOS)+ monkeypatch.setattr(ILP, "collect_system_report", lambda *a, **k: "report")+ monkeypatch.setattr(ILP, "existing_install_matches_plan", lambda *a, **k: False)+ monkeypatch.setattr(ILP, "resolve_validation_model", lambda probe: probe)++ def _validation_fails(*args, **kwargs):+ raise ILP.PrebuiltFallback("staged bundle failed validation")++ monkeypatch.setattr(ILP, "validate_prebuilt_attempts", _validation_fails)++ # Keep a runnable install when release lookup fails.+ install_dir = build_install(tmp_path, host = MACOS, marker = S12, payload_backend = "metal")+ ILP.install_prebuilt(install_dir, "latest", PUBLISHED_REPO, "")+ assert (install_dir / "llama-server").exists()++ # Otherwise request a source build.+ broken = build_install(+ tmp_path / "broken", host = MACOS, marker = S12, payload_backend = "metal", runnable = False+ )+ with pytest.raises(SystemExit) as caught:+ ILP.install_prebuilt(broken, "latest", PUBLISHED_REPO, "")+ assert caught.value.code == ILP.EXIT_FALLBACK