Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: answer current llama.cpp, whisper.cpp and Node installs from their markers (#10648)

Daniel Hana1ca9ec462f5
diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py
index 60885779058..02af02a8e03 100644
--- a/studio/install_llama_prebuilt.py
+++ b/studio/install_llama_prebuilt.py
@@ -519,6 +519,8 @@ class InstallReleasePlan:
release_tag: str
attempts: list[AssetChoice]
approved_checksums: ApprovedReleaseChecksums
+ # None when release_tag is the newest: tells a macOS walk-back from an install that fell behind.
+ walk_back: "_core.WalkBack | None" = None
PrebuiltFallback = _core.PrebuiltFallback
@@ -4128,6 +4130,12 @@ def paired_runtime_dll_patterns(choice: AssetChoice) -> list[str]:
def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]:
+ return runtime_patterns_for_install_kind(choice.install_kind, choice.source_label)
+
+
+def runtime_patterns_for_install_kind(
+ install_kind: str, source_label: "str | None" = None
+) -> list[str]:
# Broad shared-library glob + explicit binary names. Lets upstream
# repackage the SO/DLL set (e.g. ggml-org/llama.cpp#23462 split the
# per-binary entry code into paired ``lib<binary>-impl.so`` shared
@@ -4136,7 +4144,7 @@ def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]:
# DiffusionGemma visual-server (when the bundle ships it, for native
# DiffusionGemma serving); other CLIs upstream ships (llama-cli,
# llama-bench, ...) are skipped.
- if choice.install_kind in {
+ if install_kind in {
"linux-cpu",
"linux-cuda",
"linux-arm64-cuda",
@@ -4145,14 +4153,14 @@ def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]:
"linux-vulkan",
}:
return ["llama-server", "llama-quantize", "llama-diffusion-gemma-visual-server", "lib*.so*"]
- if choice.install_kind in {"macos-arm64", "macos-x64"}:
+ if install_kind in {"macos-arm64", "macos-x64"}:
return [
"llama-server",
"llama-quantize",
"llama-diffusion-gemma-visual-server",
"lib*.dylib",
]
- if choice.install_kind in {
+ if install_kind in {
"windows-cpu",
"windows-cuda",
"windows-hip",
@@ -4166,7 +4174,7 @@ def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]:
"llama-diffusion-gemma-visual-server.exe",
"*.dll",
]
- raise PrebuiltFallback(f"unsupported install kind for runtime overlay: {choice.install_kind}")
+ raise PrebuiltFallback(f"unsupported install kind for runtime overlay: {install_kind}")
def runtime_subdirs_for_choice(choice: AssetChoice) -> list[str]:
@@ -6460,6 +6468,8 @@ def _fork_manifest_release_plans(
release_limit = max(release_limit, DEFAULT_MAX_MACOS_RELEASE_FALLBACKS)
plans: list[InstallReleasePlan] = []
last_error: PrebuiltFallback | None = None
+ # The newest release this host could not take, if the first plan is an older one.
+ skipped_newest: str | None = None
for resolved_release in iter_resolved_published_releases(
llama_tag,
@@ -6501,6 +6511,8 @@ def _fork_manifest_release_plans(
"published release skipped for install planning: "
f"{bundle.repo}@{bundle.release_tag} upstream_tag={resolved_tag} ({exc})"
)
+ if not plans and skipped_newest is None:
+ skipped_newest = bundle.release_tag
continue
plans.append(
@@ -6510,6 +6522,7 @@ def _fork_manifest_release_plans(
release_tag = bundle.release_tag,
attempts = attempts,
approved_checksums = checksums,
+ walk_back = _core.walk_back_for(host, skipped_newest) if not plans else None,
)
)
@@ -6554,9 +6567,78 @@ def persisted_marker_backend_request(backend_request: str | None, choice: AssetC
return backend_request
+def host_profile(host: HostInfo) -> dict[str, Any]:
+ """The hardware facts the backend routing reads, in a shape a marker can hold.
+
+ Recorded so a later run can ask the one question the recorded request cannot answer:
+ is this still the box the bundle was chosen for. `backend_request` records what was
+ ASKED ("auto" on every automatic install), not what detection replied, and the
+ release tag does not move when the hardware does -- so without this an automatic
+ install keeps its CPU bundle after a GPU is added, keeps its CUDA bundle after the
+ driver or the card is removed, and keeps a coverage bundle after a card outside its
+ supported_sms replaces the old one, in every case until the fork happens to publish
+ a new release.
+
+ Only fields the selectors branch on, and only JSON-native types: a tuple reads back
+ from JSON as a list, and a profile that never compares equal to itself would take
+ the full path on every update forever. Computed from the ROUTED host, so the
+ comparison is made against the same HostInfo the selection was made from.
+ """
+
+ def _sorted_labels(values: "Iterable[Any] | None") -> list[str]:
+ # One row per GPU, in either order, describes the same host.
+ return sorted({str(value).strip() for value in (values or []) if str(value).strip()})
+
+ return {
+ # A home directory carried to another arch (or Rosetta) keeps every GPU field equal.
+ "machine": str(host.machine or "").strip().lower(),
+ "has_usable_nvidia": bool(host.has_usable_nvidia),
+ # The Vulkan routes are gated on there being no NVIDIA adapter at all, usable or not.
+ "has_physical_nvidia": bool(host.has_physical_nvidia),
+ "driver_cuda_version": (
+ list(host.driver_cuda_version) if host.driver_cuda_version else None
+ ),
+ "compute_caps": _sorted_labels(host.compute_caps),
+ "has_rocm": bool(host.has_rocm),
+ # Upstream ROCm assets go by the runtime's major.minor, which the GPU fields do not carry.
+ "rocm_runtime": _rocm_runtime_for_profile(host),
+ "rocm_gfx_target": host.rocm_gfx_target or None,
+ "rocm_gfx_targets": _sorted_labels(host.rocm_gfx_targets),
+ "has_intel_gpu": bool(host.has_intel_gpu),
+ "has_amd_gpu_without_rocm": bool(host.has_amd_gpu_without_rocm),
+ "macos_version": list(host.macos_version) if host.macos_version else None,
+ # The CUDA runtimes on disk order the CUDA selectors' bundles; None off CUDA hosts.
+ "cuda_runtime_lines": _detected_cuda_runtime_lines(host),
+ }
+
+
+def _rocm_runtime_for_profile(host: HostInfo) -> "list[int] | None":
+ if not host.has_rocm:
+ return None
+ try:
+ version = _detect_host_rocm_version()
+ except Exception: # noqa: BLE001 - an unreadable runtime is "cannot tell", recorded as such
+ return None
+ return list(version) if version else None
+
+
+def _detected_cuda_runtime_lines(host: HostInfo) -> "list[str] | None":
+ if not (host.has_usable_nvidia and (host.is_linux or host.is_windows)):
+ return None
+ try:
+ detected = (
+ detected_linux_runtime_lines() if host.is_linux else detected_windows_runtime_lines()
+ )
+ except Exception: # noqa: BLE001 - an unreadable scan is "cannot tell", which takes the full path
+ return None
+ lines = detected[0] if isinstance(detected, tuple) else detected
+ return sorted({str(line) for line in lines})
+
+
def write_prebuilt_metadata(
install_dir: Path,
*,
+ host: HostInfo | None = None,
requested_tag: str,
llama_tag: str,
release_tag: str,
@@ -6567,6 +6649,7 @@ def write_prebuilt_metadata(
llama_backend: str | None = None,
backend_request: str | None = None,
rocm_gfx: str | None = None,
+ walk_back: "_core.WalkBack | None" = None,
) -> None:
source_asset_name, source_sha256 = selected_source_archive_metadata(
approved_checksums,
@@ -6586,6 +6669,11 @@ def write_prebuilt_metadata(
if fingerprint is None:
raise PrebuiltFallback(f"cannot compute install fingerprint for {choice.name}")
_persisted_backend = persisted_llama_backend(llama_backend, choice)
+ # An install kind with no allowlist raises; the binary tier alone is still honest evidence.
+ try:
+ _runtime_patterns: list[str] | None = runtime_patterns_for_choice(choice)
+ except PrebuiltFallback:
+ _runtime_patterns = None
metadata = {
"requested_tag": requested_tag,
"tag": llama_tag,
@@ -6630,6 +6718,8 @@ def write_prebuilt_metadata(
"ggml_tree": recorded_ggml_tree(approved_checksums, choice),
"bundle_profile": choice.bundle_profile,
"runtime_line": choice.runtime_line,
+ # Only a MOVED torch preference is movement. Not fingerprinted: existing installs stay valid.
+ "torch_runtime_preference": _torch_runtime_preference_for_marker(host),
"coverage_class": choice.coverage_class,
# ROCm bundles: concrete built archs, so runtime GPU selection can gate
# devices the binary has no kernels for (#7624). Deliberately NOT in the
@@ -6639,7 +6729,16 @@ def write_prebuilt_metadata(
"mapped_targets": list(choice.mapped_targets or []),
# CUDA analog of mapped_targets: SM coverage for the same runtime gate.
"supported_sms": [str(s) for s in (choice.supported_sms or [])],
+ # Fingerprinted since #5106 but never written, so the offline re-check could not recompute it.
+ "runtime_sha256": choice.runtime_sha256,
"install_fingerprint": fingerprint,
+ # The macOS walk-back behind release_tag; absent when it is the newest. Not fingerprinted.
+ **(walk_back.marker_fields() if walk_back is not None else {}),
+ # size + sha256 of the binaries a reuse would otherwise RUN, size + mtime_ns of the rest.
+ # Absent on a pre-PR marker: full path once.
+ "runtime_files": runtime_file_records(install_dir, host, _runtime_patterns),
+ # The box this bundle was chosen for. Absent reads as "cannot say": full path.
+ **({"host_profile": host_profile(host)} if host is not None else {}),
"prebuilt_fallback_used": prebuilt_fallback_used,
"installed_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
}
@@ -6688,15 +6787,15 @@ def _write_marker(marker_path: Path, marker: dict) -> bool:
os.fsync(handle.fileno())
if original is not None:
os.chmod(tmp_path, stat.S_IMODE(original.st_mode))
- # Best effort: os.replace installs the temp file's ownership, so a
- # shared marker would otherwise pick up the invoking user's group.
- # A no-op for a non-root user; the mode above is what keeps the
- # marker readable, and declining the refresh instead would leave a
- # deliberate --force-cpu unrecorded, which is the #7213 crash.
+ # Owner then group, as prebuilt_core.write_live_marker explains: neither call
+ # alone is right for both root and a non-root member of a shared group.
try:
os.chown(tmp_path, original.st_uid, original.st_gid)
except (OSError, AttributeError):
- pass
+ try:
+ os.chown(tmp_path, -1, original.st_gid)
+ except (OSError, AttributeError):
+ pass
atomic_replace_from_tempfile(tmp_path, marker_path)
return True
except OSError:
@@ -6732,6 +6831,10 @@ def _marker_selection_patch(
persist_llama_backend: str | None,
ggml_tree: str | None,
rocm_gfx: str | None,
+ install_dir: Path | None = None,
+ host: HostInfo | None = None,
+ prebuilt_fallback_used: bool | None = None,
+ walk_back: "_core.WalkBack | None" = None,
) -> dict:
"""The fields a reused install must still take from this run.
@@ -6740,6 +6843,11 @@ def _marker_selection_patch(
choice behind it is unchanged: someone can switch a CUDA install to CPU and land
on the very bundle detection would have picked, and only these fields stop the
next update from routing it straight back.
+
+ It is also the only place that ever re-examines a bundle without reinstalling it,
+ which makes it the only place that can catch an OLD marker up to the evidence the
+ no-network re-check needs. Hence *install_dir* and *host*: see the backfill at the
+ end.
"""
patch: dict = {}
if bool(marker.get("force_cpu")) != persist_force_cpu:
@@ -6779,6 +6887,40 @@ def _marker_selection_patch(
# Set, never cleared: reuse needs a fingerprint match, so a pair-less bundle matches.
if choice.runtime_name and marker.get("runtime_asset") != choice.runtime_name:
patch["runtime_asset"] = choice.runtime_name
+ # ADDED only, never corrected: overwriting a present key from a reused bundle would re-bless
+ # bytes this run did not hash. prebuilt_fallback_used is the exception, since a stale False
+ # keeps a fallback forever with the preferred bundle never retried.
+ if prebuilt_fallback_used is not None and marker.get("prebuilt_fallback_used") is not (
+ prebuilt_fallback_used
+ ):
+ patch["prebuilt_fallback_used"] = prebuilt_fallback_used
+ if "runtime_sha256" not in marker:
+ # None is what expected_install_fingerprint hashed: null, not absent.
+ patch["runtime_sha256"] = choice.runtime_sha256
+ # A plan no longer walking back retires the keys.
+ patch.update(_core.walk_back_patch(marker, walk_back))
+ # Empty counts as absent: runtime_file_records answers {} for an unreadable binary, and a
+ # marker carrying that would fail closed on every update.
+ if install_dir is not None and not marker.get("runtime_files"):
+ try:
+ backfill_patterns: list[str] | None = runtime_patterns_for_choice(choice)
+ except PrebuiltFallback:
+ backfill_patterns = None
+ records = runtime_file_records(install_dir, host, backfill_patterns)
+ # An empty record is not evidence.
+ if records:
+ patch["runtime_files"] = records
+ # host_profile is this run's own probe: a stale one sends every later update down the full path.
+ if host is not None and marker.get("host_profile") != host_profile(host):
+ patch["host_profile"] = host_profile(host)
+ # Likewise the torch preference: the fast path compares what the selectors SAW, not where they routed.
+ if host is not None and (host.is_linux or host.is_windows):
+ preference = _torch_runtime_preference_for_marker(host)
+ if (
+ "torch_runtime_preference" not in marker
+ or marker.get("torch_runtime_preference") != preference
+ ):
+ patch["torch_runtime_preference"] = preference
return patch
@@ -6791,6 +6933,9 @@ def sync_marker_selection(
persist_llama_backend: str | None = None,
ggml_tree: str | None = None,
rocm_gfx: str | None = None,
+ host: HostInfo | None = None,
+ prebuilt_fallback_used: bool | None = None,
+ walk_back: "_core.WalkBack | None" = None,
) -> None:
"""Record this run's selection on a marker whose bundle was reused unchanged.
@@ -6815,11 +6960,15 @@ def sync_marker_selection(
persist_llama_backend = persist_llama_backend,
ggml_tree = ggml_tree,
rocm_gfx = rocm_gfx,
+ install_dir = install_dir,
+ host = host,
+ prebuilt_fallback_used = prebuilt_fallback_used,
+ walk_back = walk_back,
)
if not patch:
return
for key, value in patch.items():
- if value is None and key == "llama_backend":
+ if value is None and key in ("llama_backend", *_core.WALK_BACK_KEYS):
marker.pop(key, None)
else:
marker[key] = value
@@ -6831,7 +6980,8 @@ def sync_marker_selection(
"a later update may not re-assert this choice"
)
return
- log(f"existing install reused; recorded {patch} from this run")
+ # Keys, not values: runtime_files can hold dozens of entries.
+ log(f"existing install reused; recorded {sorted(patch)} from this run")
def expected_install_fingerprint(
@@ -7090,6 +7240,718 @@ def runtime_payload_is_healthy(install_dir: Path, host: HostInfo, choice: AssetC
)
+# The binaries a reuse decision would otherwise start; the DiffusionGemma visual server is optional.
+_RUNTIME_RECORD_NAMES = (
+ "llama-server",
+ "llama-quantize",
+ "llama-diffusion-gemma-visual-server",
+)
+
+
+def _runtime_record_paths(install_dir: Path, host: "HostInfo | None" = None) -> "list[Path]":
+ """Both copies of each binary, on every layout this installer produces.
+
+ _find_llama_server_binary reaches the root copy first and, without a symlink, the
+ two can rot independently. Both the Windows Release/ layout and the POSIX one are
+ walked whatever the host says: a record is only ever taken of a file that is
+ actually there, and a host argument that has to be threaded through every caller is
+ how this would go stale on the one platform nobody tests it on.
+ """
+ directories = [
+ install_dir,
+ install_dir / "build" / "bin",
+ install_dir / "build" / "bin" / "Release",
+ ]
+ if host is not None:
+ directories.insert(1, install_runtime_dir(install_dir, host))
+ paths: list[Path] = []
+ for directory in directories:
+ for name in _RUNTIME_RECORD_NAMES:
+ for candidate in (directory / name, directory / f"{name}.exe"):
+ if candidate not in paths:
+ paths.append(candidate)
+ return paths
+
+
+def _stat_record(path: Path) -> "dict[str, Any] | None":
+ """size + mtime_ns for one regular file, or None when it is neither."""
+ try:
+ if not path.is_file():
+ return None
+ info = path.stat()
+ except OSError:
+ return None
+ return {"size": info.st_size, "mtime_ns": info.st_mtime_ns}
+
+
+def runtime_file_records(
+ install_dir: Path,
+ host: "HostInfo | None" = None,
+ patterns: "list[str] | None" = None,
+) -> "dict[str, dict[str, Any]]":
+ """What the runtime is made of, in a form a later run can re-check without running it.
+
+ Two tiers, because the two halves of a bundle fail differently:
+
+ * the binaries a reuse decision would otherwise START get size + sha256. Their
+ bytes decide whether the install runs at all, and three files is ~100 ms.
+ * every other file the bundle's own copy allowlist matches gets size + mtime_ns,
+ stat only. _runtime_payload_has globs for EXISTENCE, so a truncated
+ libggml-cuda.so, ggml-cuda.dll or cudart DLL passes it -- and those shared
+ libraries are where most of a CUDA bundle's bytes live, which is exactly the
+ shape a full disk or an interrupted extract leaves behind. A size comparison
+ catches it for the price of a stat; hashing 300 MB of kernels on every update
+ would not be worth it.
+
+ *patterns* comes from runtime_patterns_for_install_kind for the bundle being
+ installed; without it only the binary tier is recorded, which is what the callers
+ that have no bundle in hand (a bare re-record) can honestly say.
+ """
+ records: dict[str, dict[str, Any]] = {}
+ runtime_dir = install_runtime_dir(install_dir, host) if host is not None else None
+ if patterns and runtime_dir is not None:
+ for pattern in patterns:
+ try:
+ matches = sorted(runtime_dir.glob(pattern))
+ except OSError:
+ continue
+ for candidate in matches:
+ record = _stat_record(candidate)
+ if record is None:
+ continue
+ try:
+ relative = candidate.relative_to(install_dir).as_posix()
+ except ValueError:
+ continue
+ records[relative] = record
+ # Last, so a binary the sweep matched is upgraded to the hashed tier.
+ for candidate in _runtime_record_paths(install_dir, host):
+ try:
+ relative = candidate.relative_to(install_dir).as_posix()
+ except ValueError:
+ continue
+ record = _stat_record(candidate)
+ if record is None:
+ continue
+ try:
+ # Streamed: a CUDA llama-server is large.
+ record["sha256"] = sha256_file(candidate)
+ except (OSError, MemoryError) as exc:
+ # No record at all: the size-only tier would trust a same-size corrupt binary.
+ log(f"could not hash {relative} for the runtime record ({exc}); not recording")
+ return {}
+ records[relative] = record
+ return records
+
+
+def _runtime_files_match(install_dir: Path, host: HostInfo, marker: "dict[str, Any]") -> bool:
+ """Whether every recorded runtime file is still the file that was installed.
+
+ Sizes for everything, digests for the binaries that carry one (see
+ runtime_file_records for why the split). mtime_ns is recorded but deliberately NOT
+ compared: a restore from backup, an rsync or a container layer rewrites it without
+ changing a byte, and the answer to a mismatch here is a 200-400 MB re-download.
+
+ Fails CLOSED, unlike the payload scans: this is the evidence that replaces actually
+ starting the binaries, so an empty or unreadable record is not proof of anything.
+ A marker written before this existed has none, and takes the full path once.
+ """
+ recorded = marker.get("runtime_files")
+ if not isinstance(recorded, dict) or not recorded:
+ return False
+ for relative, expected in recorded.items():
+ if not isinstance(expected, dict):
+ return False
+ candidate = install_dir / relative
+ try:
+ info = candidate.stat()
+ if info.st_size != expected.get("size"):
+ log(f"kept install rejected: {relative} is {info.st_size} bytes")
+ return False
+ digest = expected.get("sha256")
+ if digest is not None and sha256_file(candidate) != digest:
+ log(f"kept install rejected: {relative} does not match the recorded digest")
+ return False
+ except OSError as exc:
+ log(f"kept install rejected: {relative} is unreadable ({exc})")
+ return False
+ # An unrecorded binary is not evidence against the install; a RECORDED one that vanished is.
+ return True
+
+
+def _marker_install_fingerprint(marker: "dict[str, Any]") -> "str | None":
+ """Recompute the fingerprint from the marker's own recorded fields.
+
+ Self-consistency, not a comparison against a fresh plan: it proves the marker was
+ written whole by this installer and has not been hand-edited or truncated, which is
+ what lets the rest of the no-network check trust the fields it reads. A marker
+ missing any input -- every pre-PR one, which never recorded runtime_sha256 -- gets
+ None and takes the full path once.
+ """
+ keys = (
+ "published_repo",
+ "release_tag",
+ "asset",
+ "asset_sha256",
+ "source",
+ "source_asset",
+ "source_sha256",
+ "runtime_line",
+ "runtime_asset",
+ "runtime_sha256",
+ "bundle_profile",
+ "coverage_class",
+ )
+ if "runtime_sha256" not in marker or "tag" not in marker:
+ return None
+ payload = {key: marker.get(key) for key in keys}
+ payload["upstream_tag"] = marker.get("tag")
+ return hashlib.sha256(
+ json.dumps(payload, sort_keys = True, separators = (",", ":")).encode("utf-8")
+ ).hexdigest()
+
+
+_PREBUILT_FULL_CHECK_ENV = "UNSLOTH_PREBUILT_FULL_CHECK"
+
+
+def prebuilt_full_check_requested() -> bool:
+ """The escape hatch for the no-network re-check. A skip nobody can turn off is a bug
+ nobody can work around."""
+ return os.environ.get(_PREBUILT_FULL_CHECK_ENV, "").strip().lower() in (
+ "1",
+ "true",
+ "yes",
+ "on",
+ )
+
+
+def _newest_release_tag_from_releases(releases: "Iterable[Any]") -> "str | None":
+ """The newest published release tag by published_at, the ordering _select uses.
+
+ Mirrors iter_release_payloads_by_time's sort (release_time_sort_key, drafts and
+ prereleases dropped) so the two cannot answer differently from the same payload.
+ """
+ published = [
+ release
+ for release in releases
+ if isinstance(release, dict)
+ and not release.get("draft")
+ and not release.get("prerelease")
+ and isinstance(release.get("tag_name"), str)
+ and release.get("tag_name")
+ ]
+ if not published:
+ return None
+ return max(published, key = release_time_sort_key)["tag_name"]
+
+
+def _api_newest_release_tag(repo: str) -> "str | None":
+ """The API's notion of latest, over the pages the full selector scans.
+
+ published_at ordering, so a release drafted early and published late can sit on a
+ later page of a repository with more than a hundred releases; one page would report
+ an older marker current where iter_release_payloads_by_time would install the newer one.
+ """
+ try:
+ return _newest_release_tag_from_releases(
+ github_releases(repo, max_pages = DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES)
+ )
+ except Exception as exc: # noqa: BLE001 - unreachable is a reason to do the work
+ log(f"could not resolve the latest release from the GitHub API ({exc})")
+ return None
+
+
+def _api_newest_release_tag_for_upstream(
+ repo: str, upstream_tag: str, recorded_release: str
+) -> "str | None":
+ """The newest published fork release packaging *upstream_tag*.
+
+ The fork names its releases after the build (bNNNN, bNNNN-mix-<sha>); the release
+ the marker records is known to package it whatever it is called, so it is a
+ candidate too, and the newest of them all is what the selector would install.
+ """
+ try:
+ releases = github_releases(repo, max_pages = DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES)
+ except Exception as exc: # noqa: BLE001 - unreachable is a reason to do the work
+ log(f"could not list the releases packaging {upstream_tag} ({exc})")
+ return None
+ matching = [
+ release
+ for release in releases
+ if isinstance(release, dict)
+ and isinstance(release.get("tag_name"), str)
+ and (
+ release["tag_name"] == upstream_tag
+ or release["tag_name"].startswith(upstream_tag + "-")
+ or release["tag_name"] == recorded_release
+ )
+ ]
+ return _newest_release_tag_from_releases(matching)
+
+
+def _memoized_api_newest_release_tag(repo: str) -> "str | None":
+ """The API's notion of latest, but only when this process already has the payload.
+
+ _METADATA_MEMO is populated only inside _cached_metadata_fetches (--resolve-backends
+ planning several options from one snapshot); on the install path it is None and this
+ answers None without touching the network. Free when it is there, and there is no
+ reason to prefer the cheaper, staler answer over one already in hand.
+ """
+ memo = _METADATA_MEMO
+ if not memo:
+ return None
+ prefix = f"https://api.github.com/repos/{repo}/releases?"
+ releases: list[Any] = []
+ for key, payload in memo.items():
+ if not (isinstance(key, tuple) and len(key) == 2 and key[0] == "json"):
+ continue
+ url = key[1]
+ if isinstance(url, str) and url.startswith(prefix) and isinstance(payload, list):
+ releases.extend(payload)
+ return _newest_release_tag_from_releases(releases) if releases else None
+
+
+def _runtime_preference_moved(marker: "dict[str, Any]", host: HostInfo) -> bool:
+ """Whether torch now prefers another CUDA runtime line than the one installed.
+
+ The selectors order the CUDA bundles around detect_torch_cuda_runtime_preference
+ (torch.version.cuda), which the host profile does not see: a torch upgrade from a
+ CUDA 12 to a CUDA 13 build leaves the GPU and driver as they were. A marker recorded
+ under the old preference is then not what this run would choose. Only a CUDA install
+ on a CUDA host is asked; a preference torch cannot state keeps the fast path.
+ """
+ if not (host.has_usable_nvidia and (host.is_linux or host.is_windows)):
+ return False
+ recorded_line = marker.get("runtime_line")
+ if not isinstance(recorded_line, str) or not recorded_line.startswith("cuda"):
+ return False
+ preferred = detect_torch_cuda_runtime_preference(host).runtime_line
+ if "torch_runtime_preference" in marker:
+ # The selectors do not always follow the preference (Blackwell routing, a release without
+ # that line), so only the preference itself moving is movement.
+ recorded_preference = marker.get("torch_runtime_preference")
+ if preferred == recorded_preference:
+ return False
+ if not preferred:
+ # Preference gone (torch removed): the selectors fall back to the host's runtime order.
+ fallback = _fallback_runtime_line(host)
+ if fallback is None or fallback == recorded_line:
+ return False
+ log(
+ f"kept install rejected: torch no longer states a CUDA preference and the "
+ f"host's runtime order starts at {fallback}, the install is {recorded_line}"
+ )
+ return True
+ if not _runtime_line_selectable(host, preferred):
+ return False
+ log(
+ f"kept install rejected: torch now prefers the {preferred} runtime line, "
+ f"the install was chosen under {recorded_preference or 'no preference'}"
+ )
+ return True
+ # A marker predating the preference has only the installed line; the full path records it once.
+ if not preferred or preferred == recorded_line:
+ return False
+ # A preference the selectors cannot act on must be ignored here too, or every update is full.
+ if not _runtime_line_selectable(host, preferred):
+ return False
+ log(
+ f"kept install rejected: torch now prefers the {preferred} runtime line, "
+ f"the install is {recorded_line}"
+ )
+ return True
+
+
+def _fallback_runtime_line(host: HostInfo) -> "str | None":
+ """The CUDA line the selectors try first when torch states no preference, or None
+ when that cannot be told from here.
+
+ Mirrors the orderings in linux_cuda_choice_from_release and windows_cuda_attempts
+ without a release in hand: Linux takes the detected lines the driver can run, in
+ detection order; Windows the driver-compatible lines narrowed to the detected ones,
+ or all of them when none is detected. A Blackwell host is routed by its own rule
+ ahead of any preference, so nothing about torch moves its choice.
+ """
+ try:
+ if _host_is_blackwell(host):
+ return None
+ if host.is_linux:
+ detected = detected_linux_runtime_lines()[0]
+ compatible = compatible_linux_runtime_lines(host)
+ order = [line for line in detected if line in compatible]
+ else:
+ detected = detected_windows_runtime_lines()[0]
+ compatible = compatible_windows_runtime_lines(host)
+ order = [line for line in compatible if line in detected] or list(compatible)
+ except Exception: # noqa: BLE001 - an unreadable host answers "cannot tell"
+ return None
+ return order[0] if order else None
+
+
+def _torch_runtime_preference_for_marker(host: "HostInfo | None") -> "str | None":
+ """The torch CUDA preference an install is being recorded under, or None.
+
+ Read only where the selectors read it (a CUDA host on Linux or Windows), so the
+ marker says None wherever no preference took part in the choice. Never raises:
+ the marker is metadata, not the install.
+ """
+ if host is None:
+ return None
+ try:
+ if not (host.has_usable_nvidia and (host.is_linux or host.is_windows)):
+ return None
+ return detect_torch_cuda_runtime_preference(host).runtime_line
+ except Exception: # noqa: BLE001 - metadata only
+ return None
+
+
+def _runtime_line_selectable(host: HostInfo, line: str) -> bool:
+ """Whether the CUDA selectors could pick *line* here: its runtime is on disk and the
+ driver can run it, the two filters they apply before ordering. The Windows selector
+ falls back to every driver-compatible line when no runtime DLL is found at all
+ (a bundle carries its own), so an empty detected set does not veto there."""
+ try:
+ if host.is_linux:
+ detected = detected_linux_runtime_lines()[0]
+ compatible = compatible_linux_runtime_lines(host)
+ else:
+ detected = detected_windows_runtime_lines()[0]
+ compatible = compatible_windows_runtime_lines(host)
+ if not detected:
+ detected = list(compatible)
+ except Exception: # noqa: BLE001 - an unreadable host answers "cannot tell", which is not movement
+ return False
+ return line in detected and line in compatible
+
+
+def _expected_release_tag_without_plan(
+ marker: "dict[str, Any]",
+ llama_tag: str,
+ published_repo: str,
+ published_release_tag: str,
+ host: "HostInfo | None" = None,
+) -> "str | None":
+ """The release this run would install, worked out without listing anything.
+
+ None means "cannot say", and every caller treats that as a reason to take the full
+ path. Three shapes:
+ * a pinned UNSLOTH_LLAMA_RELEASE_TAG names the answer outright;
+ * an upstream bNNNN pin: the marker must record that upstream tag, and on the
+ fork the newest published release packaging that build is the answer (the fork
+ can republish a build, b9596-mix-aaa then b9596-mix-bbb, and the selector takes
+ the newest, as does test_llama_cpp_freshness), so this lists the releases; on the
+ upstream repo the release tag is the upstream tag itself;
+ * "latest" costs ONE HEAD on github.com/<repo>/releases/latest -- no
+ api.github.com call, so no rate limit, and no manifest or checksum download.
+
+ KNOWN AND ACCEPTED LAG, for "latest" only. The HEAD follows /releases/latest, which
+ GitHub resolves by make_latest / created_at. The normal macOS path deliberately
+ turns the download-host resolver off (allow_download_host_fast_path = not
+ host.is_macos, so it can walk back past a prebuilt whose minimum OS is too new) and
+ orders releases by published_at instead. The two disagree when a release is created
+ before, but published after, another -- e.g. a re-published or back-dated release.
+ In that window this check can report "current" for an install _select would have
+ moved off. The consequence is one deferred update, never a wrong install: the very
+ next run whose pointer has caught up does the move, and every OTHER guard here
+ (fingerprint, host profile, payload) still holds. Paying an api.github.com call on
+ every update to close it would reintroduce the rate limit this path exists to avoid.
+ Two things narrow it anyway, below: an explicit request for the API path gets the
+ API's answer, and a payload this process already fetched is compared for free.
+
+ A SECOND DEFERRAL of the same class, in the macOS walk-back rather than the pointer.
+ iter_resolved_published_releases skips a release whose checksum index it cannot fetch
+ and logs it, and those skips are invisible to the planner, so a walk-back records only
+ that the NEWEST release was unusable. If a middle release was passed over for a
+ transient fetch failure rather than for its OS floor, walk_back_stands keeps answering
+ "current" for the older install while the newest is unchanged, and the middle one is
+ not reconsidered until something newer publishes. The pre-PR path re-planned every
+ update and would have taken it on the next run. Same consequence as above and left
+ alone for the same reason: both releases are compatible, so this is one deferred
+ update and never a wrong install, and it clears itself on the next publish. Suppressing
+ the walk-back whenever any release was skipped transiently is the fix if this is ever
+ worth closing; it costs a flag threaded out of the iterator.
+ """
+ pinned = (published_release_tag or "").strip()
+ requested = normalized_requested_llama_tag(llama_tag)
+ if pinned:
+ # The full path checks a pinned release against a concrete upstream pin too.
+ if requested != "latest":
+ recorded_upstream = marker.get("tag")
+ if not isinstance(recorded_upstream, str) or recorded_upstream != requested:
+ return None
+ return pinned
+ repo = published_repo or DEFAULT_PUBLISHED_REPO
+ if requested != "latest":
+ # An upstream pin: a marker naming another build is not current.
+ recorded_upstream = marker.get("tag")
+ if not isinstance(recorded_upstream, str) or recorded_upstream != requested:
+ return None
+ recorded_release = marker.get("release_tag")
+ if not isinstance(recorded_release, str) or not recorded_release:
+ return None
+ if repo == UPSTREAM_REPO:
+ # Upstream publishes one release per build under the build's own tag.
+ return recorded_release
+ if repo != DEFAULT_PUBLISHED_REPO and is_release_tag_like(requested):
+ # Any other repo is fetched as that exact release, never a "<pin>-<packaging>" one.
+ return requested
+ # The fork can package one build more than once, so a pinned build still asks the API.
+ return _api_newest_release_tag_for_upstream(repo, requested, recorded_release)
+ # Below the macOS floor the selector answers the pinned upstream fallback; a current marker names it.
+ if host is not None:
+ pinned_macos = pinned_macos_release_tag(host, repo)
+ if pinned_macos is not None:
+ return pinned_macos
+ if not _download_host_resolve_enabled() or repo != DEFAULT_PUBLISHED_REPO:
+ # The API path, or a repo the selector never resolves through the download host:
+ # /releases/latest for a custom repo could disagree with the selector forever.
+ return _api_newest_release_tag(repo)
+ try:
+ resolved = _download_host_latest_release_tag(repo)
+ except Exception as exc: # noqa: BLE001 - unreachable is a reason to do the work
+ log(f"could not resolve the latest release without the API ({exc})")
+ return None
+ if not resolved:
+ return None
+ cached = _memoized_api_newest_release_tag(repo)
+ if cached is not None and cached != resolved:
+ log(
+ f"/releases/latest points at {resolved} but the newest published release in "
+ f"{repo} is {cached}; using the published_at answer this run already has"
+ )
+ return cached
+ return resolved
+
+
+def _diffusion_visual_server_missing_for_marker(
+ install_dir: Path, host: HostInfo, marker: "dict[str, Any]"
+) -> bool:
+ """The marker-only twin of diffusion_visual_server_backfill_needed.
+
+ An install made before the visual-server entered the copy allowlist matches on tag
+ and is missing the binary, so a tag-match skip never backfills it and DiffusionGemma
+ fails with "runner not found". The plan-based check reads the chosen bundle's
+ pattern list; here the marker's own recorded install kind stands in for it. Gated to
+ the fork ("published") bundles that carry it, so upstream installs -- which never
+ ship it -- cannot thrash on repeated updates.
+ """
+ if marker.get("source") != "published":
+ return False
+ backend = marker_backend(marker)
+ platform_prefix = "windows-" if host.is_windows else "macos-" if host.is_macos else "linux-"
+ kinds = [
+ kind for kind in install_kinds_for_backend(backend) if kind.startswith(platform_prefix)
+ ]
+ if not kinds:
+ # An unknown backend cannot say the bundle ships one; guessing yes re-extracts every update.
+ return False
+ name = "llama-diffusion-gemma-visual-server" + (".exe" if host.is_windows else "")
+ if not any(
+ name in runtime_patterns_for_install_kind(kind, marker.get("source")) for kind in kinds
+ ):
+ return False
+ for candidate in (
+ install_dir / name,
+ install_dir / "build" / "bin" / name,
+ install_dir / "build" / "bin" / "Release" / name,
+ ):
+ if candidate.is_file():
+ return False
+ return True
+
+
+def _marker_backend_fits_host(marker: "dict[str, Any]", host: HostInfo) -> bool:
+ """Whether the marker's own backend/request pair is one this platform can hold.
+
+ Two assertions, both cheap and both about the MARKER rather than the disk:
+
+ * self-consistency. write_prebuilt_metadata records backend_request through
+ persisted_marker_backend_request, which stores "auto" whenever the request and
+ the bundle that landed disagree. So a marker naming a concrete request must name
+ the same backend, or it was not written by this installer -- and the rest of this
+ check reads those two fields to decide it need do no work.
+ * platform fit. backend_for_install_kind maps kinds to backends; a backend with no
+ kind on this platform (a "cuda" marker on macOS, a copied install directory)
+ cannot describe a bundle this run would produce, and _kept_install_payload_is_healthy
+ answers True vacuously for it because there is no shared payload to require.
+ """
+ recorded_backend = marker_backend(marker)
+ if recorded_backend is None:
+ return False
+ recorded_request = marker.get("backend_request")
+ if (
+ isinstance(recorded_request, str)
+ and recorded_request not in ("", "auto")
+ and recorded_request != recorded_backend
+ ):
+ return False
+ platform_prefix = "windows-" if host.is_windows else "macos-" if host.is_macos else "linux-"
+ return any(
+ kind.startswith(platform_prefix) for kind in install_kinds_for_backend(recorded_backend)
+ )
+
+
+def existing_install_current_without_plan(
+ install_dir: Path,
+ *,
+ llama_tag: str,
+ published_repo: str,
+ published_release_tag: str,
+ backend_request: str,
+ force_cpu: bool,
+ override_has_rocm: bool = False,
+ override_rocm_gfx: str | None = None,
+ route: "BackendRoute | None" = None,
+) -> bool:
+ """Whether the install on disk is already the one this run would produce.
+
+ Runs BEFORE the release listing, the manifest fetch and the checksum download, so a
+ no-op update costs one HEAD instead of the whole selection plus a re-validation that
+ starts llama-server and loads the CUDA runtime -- 13-63 s on macOS, ~5 s on Windows,
+ every update, to arrive back where it started.
+
+ Every check is on-disk evidence, a local hardware probe, or a tag comparison. Nothing
+ here reasons about what a bundle SHOULD contain: that is existing_install_matches_choice's
+ job, and it needs the plan. This one asserts that the recorded install is intact, that
+ the HOST is still the host it was chosen for, and that the release it names is the
+ release this run would ask for.
+
+ The host half is not optional. The recorded request is "auto" on every automatic
+ install, so on its own it says a CPU bundle is still right after a GPU is added, a
+ CUDA bundle is still right after the card or the driver is gone, and a coverage
+ bundle is still right for a card outside its supported_sms -- and since the tag does
+ not move when hardware does, that verdict would stand until the fork published a new
+ release. So the host is re-derived here through route_backend_request, the same
+ no-network routing _select uses (identical overrides, identical cpu mechanism), and
+ compared against the profile the install recorded. A marker with no host_profile --
+ every one written before this existed -- cannot answer and takes the full path.
+
+ See _expected_release_tag_without_plan for the one thing this deliberately does NOT
+ close: the documented "latest" pointer lag.
+ """
+ if prebuilt_full_check_requested():
+ return False
+ marker = load_prebuilt_metadata(install_dir)
+ if not marker:
+ return False
+ # (1) the marker describes an install this run would have made.
+ if not isinstance(marker.get("install_fingerprint"), str):
+ return False
+ # --force-cpu or --llama-backend differing from the record is a request to change the install.
+ recorded_request = marker.get("backend_request")
+ if not isinstance(recorded_request, str) or recorded_request != backend_request:
+ return False
+ if bool(marker.get("force_cpu")) != bool(force_cpu):
+ return False
+ # A fallback bundle is a stopgap: its marker keeps taking the full path, which retries the preferred.
+ if marker.get("prebuilt_fallback_used") is True:
+ log("kept install rejected: it is a fallback bundle; the preferred one is retried")
+ return False
+ # (2) the hardware. Local probes only, and the caller's route, so they run once per update.
+ if route is None:
+ route = route_backend_request(
+ backend = backend_request,
+ published_repo = published_repo,
+ published_release_tag = published_release_tag,
+ override_has_rocm = override_has_rocm,
+ override_rocm_gfx = override_rocm_gfx,
+ cpu_mechanism = force_cpu,
+ )
+ host = route.host
+ # The ROUTED repo: a Linux arm64 Vulkan host goes upstream, and the marker records that.
+ if (marker.get("published_repo") or "") != (route.published_repo or DEFAULT_PUBLISHED_REPO):
+ return False
+ if not _marker_backend_fits_host(marker, host):
+ return False
+ recorded_profile = marker.get("host_profile")
+ if not isinstance(recorded_profile, dict):
+ # Written before host_profile: "cannot tell". The full path records it, so this is paid once.
+ return False
+ if recorded_profile != host_profile(host):
+ log("kept install rejected: this host no longer matches the one it was installed for")
+ return False
+ if _runtime_preference_moved(marker, host):
+ return False
+ # (3) the release this run would ask for is the release that is installed.
+ expected_release = _expected_release_tag_without_plan(
+ marker, llama_tag, route.published_repo, route.published_release_tag, host = host
+ )
+ if not _release_expectation_met(
+ marker,
+ expected_release,
+ host,
+ pinned = bool((route.published_release_tag or "").strip()),
+ ):
+ return False
+ # (4) the marker was written whole by this installer, so its fields can be trusted.
+ if _marker_install_fingerprint(marker) != marker.get("install_fingerprint"):
+ return False
+ # (5) the tree is the shape the marker's backend implies, and is executable.
+ if not _install_tree_is_usable(install_dir, host):
+ return False
+ if not _kept_install_payload_is_healthy(install_dir, host):
+ return False
+ extension = ".exe" if host.is_windows else ""
+ binaries = [
+ install_runtime_dir(install_dir, host) / f"llama-{name}{extension}"
+ for name in ("server", "quantize")
+ ]
+ if not all(os.access(binary, os.X_OK) for binary in binaries):
+ return False
+ try:
+ # Kept, unlike the --version spawns: a preflight answers whether the OS can LOAD the image,
+ # which a hash cannot.
+ preflight_linux_installed_binaries(binaries, install_dir, host)
+ preflight_macos_installed_binaries(binaries, install_dir, host)
+ except Exception: # noqa: BLE001
+ return False
+ # (6) and the bytes are the ones that were installed.
+ if not _runtime_files_match(install_dir, host, marker):
+ return False
+ # The one backfill that is not a release change, so it has to be asked separately.
+ if _diffusion_visual_server_missing_for_marker(install_dir, host, marker):
+ return False
+ # "already matches" is what setup.sh and setup.ps1 grep for to report the install up to date.
+ log(
+ "existing llama.cpp install already matches selected release "
+ f"{expected_release} upstream_tag={marker.get('tag')}; skipping download and install"
+ )
+ return True
+
+
+def _release_expectation_met(
+ marker: "dict[str, Any]",
+ expected_release: "str | None",
+ host: HostInfo,
+ *,
+ pinned: bool = False,
+) -> bool:
+ """Whether the release this run would ask for is the one installed.
+
+ On a Mac below the newest bundle's OS floor the planner walks back to an older
+ release and records the one it skipped and the host version that skipped it
+ (prebuilt_core.WalkBack). While the newest published release is still that one and
+ the host is still that macOS version, the walk-back stands and the install is
+ current; a newer release or an OS upgrade takes the full path, which re-decides it.
+
+ *pinned* is the caller naming a release explicitly, and it disables that: the walk-back
+ explains why AUTOMATIC selection settled on an older release, and an explicit request is
+ not automatic selection. Without this, a Mac holding b9998 after walking back from b9999
+ answers "already matches selected release b9999" to a run that asked for b9999, which is
+ both a release the user did not get and a sentence that is not true. The full path
+ disables older-release fallback for a pinned request for the same reason
+ (allow_older_release_fallback in _fork_manifest_release_plans), so it reports the
+ incompatibility instead of quietly serving a different build.
+ """
+ if not expected_release:
+ return False
+ if expected_release == marker.get("release_tag"):
+ return True
+ if pinned:
+ return False
+ return _core.walk_back_stands(marker, host, expected_release)
+
+
def _kept_install_payload_is_healthy(install_dir: Path, host: HostInfo) -> bool:
"""Check the payload shared by install kinds allowed by the tree's marker."""
marker = load_prebuilt_metadata(install_dir)
@@ -7237,11 +8099,37 @@ def existing_install_matches_choice(
if not runtime_payload_is_healthy(install_dir, host, choice):
return False
- # Verify primary executables still exist (catches partial deletion)
+ # Non-empty: a zero-byte llama-server.exe under a pre-record marker was reused as current.
runtime_dir = install_runtime_dir(install_dir, host)
ext = ".exe" if host.is_windows else ""
for binary in ("llama-server", "llama-quantize"):
- if not (runtime_dir / f"{binary}{ext}").exists():
+ try:
+ if (runtime_dir / f"{binary}{ext}").stat().st_size == 0:
+ return False
+ except OSError:
+ return False
+ # Windows has no image-reading preflight, so a truncated llama-server.exe kept its fingerprint
+ # match. Only a marker carrying the record is held to it.
+ recorded_files = metadata.get("runtime_files")
+ if isinstance(recorded_files, dict) and recorded_files:
+ if not _runtime_files_match(install_dir, host, metadata):
+ return False
+ elif not (host.is_linux or host.is_macos):
+ # The migration run, on the one platform with no loader preflight below: a pre-record
+ # marker has no digest, so damaged bytes would become the reference they are later
+ # compared against. Start them once instead. _binary_image_runs treats a timeout or an
+ # ordinary non-zero exit as healthy, so this costs a working install nothing.
+ if not all(
+ _binary_image_runs(
+ runtime_dir / f"{binary}{ext}",
+ install_dir,
+ host,
+ metadata.get("runtime_line")
+ if isinstance(metadata.get("runtime_line"), str)
+ else None,
+ )
+ for binary in ("llama-server", "llama-quantize")
+ ):
return False
if host.is_linux:
try:
@@ -7329,6 +8217,7 @@ def validate_prebuilt_choice(
llama_backend: str | None = None,
backend_request: str | None = None,
rocm_gfx: str | None = None,
+ walk_back: "_core.WalkBack | None" = None,
) -> tuple[Path, Path]:
source_repo, source_ref, source_archive, exact_source = preferred_source_archive(
approved_checksums, llama_tag
@@ -7371,6 +8260,7 @@ def validate_prebuilt_choice(
ensure_repo_shape(install_dir)
write_prebuilt_metadata(
install_dir,
+ host = host,
requested_tag = requested_tag,
llama_tag = llama_tag,
release_tag = release_tag,
@@ -7381,6 +8271,7 @@ def validate_prebuilt_choice(
llama_backend = llama_backend,
backend_request = backend_request,
rocm_gfx = rocm_gfx,
+ walk_back = walk_back,
)
# Hashless external prebuilts are not in the approved-sha256
# manifest and rely on the functional smoke test as their only integrity gate,
@@ -7473,6 +8364,7 @@ def validate_prebuilt_attempts(
llama_backend: str | None = None,
backend_request: str | None = None,
rocm_gfx: str | None = None,
+ walk_back: "_core.WalkBack | None" = None,
) -> tuple[AssetChoice, Path, bool]:
attempt_list = list(attempts)
if not attempt_list:
@@ -7545,6 +8437,7 @@ def validate_prebuilt_attempts(
llama_backend = llama_backend,
backend_request = backend_request,
rocm_gfx = rocm_gfx,
+ walk_back = walk_back,
)
except Exception as exc:
remove_tree(staging_dir)
@@ -8770,20 +9663,24 @@ def install_prebuilt(
# problem a source build only makes worse, so those stay EXIT_ERROR
# alongside resolver bugs. ENOSPC still reaches EXIT_NO_SPACE via
# _environment_fatal_reason in __main__.
- def _select(requested_backend: str | None) -> BackendSelection:
+ def _select(
+ requested_backend: str | None, route: BackendRoute | None = None
+ ) -> BackendSelection:
nonlocal host
# Routing is deliberately outside the conversion below: it makes no
# network call, so letting it be caught here would report a bug in
# host detection as a release-listing failure and source build over
# it.
- route = route_backend_request(
- backend = requested_backend,
- published_repo = published_repo,
- published_release_tag = published_release_tag,
- override_has_rocm = override_has_rocm,
- override_rocm_gfx = override_rocm_gfx,
- cpu_mechanism = force_cpu,
- )
+ # Reused: routing runs nvidia-smi and the ROCm probes, and the re-check needs the same answer.
+ if route is None:
+ route = route_backend_request(
+ backend = requested_backend,
+ published_repo = published_repo,
+ published_release_tag = published_release_tag,
+ override_has_rocm = override_has_rocm,
+ override_rocm_gfx = override_rocm_gfx,
+ cpu_mechanism = force_cpu,
+ )
host = route.host
try:
return select_backend_install(
@@ -8808,8 +9705,31 @@ def install_prebuilt(
f"{published_repo or DEFAULT_PUBLISHED_REPO}: {exc}"
) from exc
+ # Before the listing, manifest and checksum fetches, which on a current install resolve
+ # back to the bundle on disk. One HEAD instead. Routed once and handed to both, since
+ # _select would otherwise repeat the probes.
+ initial_route = route_backend_request(
+ backend = backend,
+ published_repo = published_repo,
+ published_release_tag = published_release_tag,
+ override_has_rocm = override_has_rocm,
+ override_rocm_gfx = override_rocm_gfx,
+ cpu_mechanism = force_cpu,
+ )
+ if existing_install_current_without_plan(
+ install_dir,
+ llama_tag = llama_tag,
+ published_repo = published_repo,
+ published_release_tag = published_release_tag,
+ # As effective_backend_request resolved it: an explicit --llama-backend naming
+ # another is a request to CHANGE the install.
+ backend_request = backend,
+ force_cpu = force_cpu,
+ route = initial_route,
+ ):
+ return
try:
- selection = _select(backend)
+ selection = _select(backend, initial_route)
except BackendUnavailable:
if backend_mandatory:
# The caller named this backend, so installing a different one
@@ -8833,16 +9753,23 @@ def install_prebuilt(
persist_rocm_gfx = selection.persist_rocm_gfx
persist_backend_request = backend
- def _record_reused_selection(plan: InstallReleasePlan, reused: AssetChoice) -> None:
+ def _record_reused_selection(
+ plan: InstallReleasePlan, reused: AssetChoice, used_fallback: bool
+ ) -> None:
"""Update selection fields when the existing bundle is reused."""
sync_marker_selection(
install_dir,
choice = reused,
+ # A later candidate reused after the preferred failed is a fallback to retry.
+ prebuilt_fallback_used = used_fallback,
backend_request = persist_backend_request,
persist_force_cpu = persist_force_cpu,
persist_llama_backend = persisted_llama_backend(persist_llama_backend, reused),
ggml_tree = recorded_ggml_tree(plan.approved_checksums, reused),
rocm_gfx = persist_rocm_gfx,
+ walk_back = plan.walk_back,
+ # The routed host, so an older marker gains host_profile and runtime_files here.
+ host = host,
)
if release_plans and existing_install_matches_plan(install_dir, host, release_plans[0]):
@@ -8858,7 +9785,7 @@ def install_prebuilt(
f"{current.release_tag} upstream_tag={current.llama_tag}; skipping download and install"
)
# Record a changed choice even when the bundle already matches.
- _record_reused_selection(current, current.attempts[0])
+ _record_reused_selection(current, current.attempts[0], False)
return
with scratch_dir("unsloth-llama-prebuilt-") as work_dir:
probe = lazy_validation_model(
@@ -8892,7 +9819,8 @@ def install_prebuilt(
"existing llama.cpp install already matches fallback release "
f"{plan.release_tag} upstream_tag={plan.llama_tag}; skipping reinstall"
)
- _record_reused_selection(plan, choice)
+ # An older release kept after the newest failed is a fallback.
+ _record_reused_selection(plan, choice, release_index > 0)
return
log(
"selected "
@@ -8918,11 +9846,12 @@ def install_prebuilt(
llama_backend = persist_llama_backend,
backend_request = persist_backend_request,
rocm_gfx = persist_rocm_gfx,
+ walk_back = plan.walk_back,
)
except ExistingInstallSatisfied as satisfied:
# Third reuse path: the reinstall was skipped, so
# write_prebuilt_metadata does not run here either.
- _record_reused_selection(plan, satisfied.choice)
+ _record_reused_selection(plan, satisfied.choice, satisfied.used_fallback)
return
except PrebuiltFallback as exc:
if _environment_fatal_reason(exc):
@@ -8984,6 +9913,11 @@ def install_prebuilt(
and not explicit_version_request
# Even "auto" is a live instruction: it asks for re-detection.
and not backend_mandatory
+ # A release that was fetched and found untrustworthy ANSWERED, so "update
+ # unavailable" is the wrong sentence for it: it turns a tamper signal into a
+ # routine offline notice. Pre-dates this branch on the llama side and is closed
+ # here so the two installers cannot disagree about what a keep is allowed to hide.
+ and not isinstance(exc, _core.ReleaseIntegrityError)
and host is not None
and _existing_install_runs(install_dir, host)
):
diff --git a/studio/install_node_prebuilt.py b/studio/install_node_prebuilt.py
index 3c57ebca8da..09625d03389 100644
--- a/studio/install_node_prebuilt.py
+++ b/studio/install_node_prebuilt.py
@@ -23,6 +23,7 @@ import argparse
import hashlib
import json
import os
+import stat
import platform
import random
import shutil
@@ -64,6 +65,10 @@ RETRYABLE_HTTP_STATUS = {408, 429, 500, 502, 503, 504}
HTTP_FETCH_ATTEMPTS = 4
HTTP_FETCH_BASE_DELAY_SECONDS = 0.75
INSTALL_LOCK_TIMEOUT_SECONDS = 300
+# The runtime-verification record is an optimisation, so it waits for the lock only briefly:
+# blocking a launch for the full install timeout to write a record that merely saves two
+# subprocess spawns next time is a worse outcome than never writing it.
+RECORD_LOCK_TIMEOUT_SECONDS = 5
INSTALL_STAGING_ROOT_NAME = ".staging"
METADATA_FILENAME = "UNSLOTH_NODE_PREBUILT_INFO.json"
METADATA_SCHEMA_VERSION = 1
@@ -260,8 +265,10 @@ def fetch_json(url: str) -> object:
def atomic_replace_from_tempfile(tmp_path: Path, destination: Path) -> None:
+ # The same retry the directory renames use: rename-over needs DELETE access on the destination,
+ # so a scanner holding the marker fails the swap outright. A no-op off Windows.
destination.parent.mkdir(parents = True, exist_ok = True)
- os.replace(tmp_path, destination)
+ _replace_with_retry(tmp_path, destination)
def download_file(url: str, destination: Path) -> None:
@@ -524,11 +531,12 @@ def _pid_is_alive(pid: int) -> bool:
@contextmanager
-def install_lock(lock_path: Path) -> Iterator[None]:
+def install_lock(lock_path: Path, *, timeout: float | None = None) -> Iterator[None]:
+ seconds = INSTALL_LOCK_TIMEOUT_SECONDS if timeout is None else timeout
lock_path.parent.mkdir(parents = True, exist_ok = True)
if FileLock is None:
fd: int | None = None
- deadline = time.monotonic() + INSTALL_LOCK_TIMEOUT_SECONDS
+ deadline = time.monotonic() + seconds
while True:
try:
fd = os.open(str(lock_path), os.O_CREAT | os.O_EXCL | os.O_RDWR)
@@ -560,7 +568,7 @@ def install_lock(lock_path: Path) -> Iterator[None]:
continue
if time.monotonic() >= deadline:
raise BusyInstallConflict(
- f"timed out after {INSTALL_LOCK_TIMEOUT_SECONDS}s waiting for install lock: {lock_path}"
+ f"timed out after {seconds}s waiting for install lock: {lock_path}"
)
time.sleep(0.5)
try:
@@ -572,11 +580,11 @@ def install_lock(lock_path: Path) -> Iterator[None]:
return
try:
- with FileLock(str(lock_path), timeout = INSTALL_LOCK_TIMEOUT_SECONDS):
+ with FileLock(str(lock_path), timeout = seconds):
yield
except FileLockTimeout as exc:
raise BusyInstallConflict(
- f"timed out after {INSTALL_LOCK_TIMEOUT_SECONDS}s waiting for install lock: {lock_path}"
+ f"timed out after {seconds}s waiting for install lock: {lock_path}"
) from exc
@@ -658,6 +666,78 @@ def metadata_path(install_dir: Path) -> Path:
return install_dir / METADATA_FILENAME
+def _write_metadata_payload(install_dir: Path, payload: dict) -> None:
+ """Replace the marker atomically, or leave the previous one exactly as it was.
+
+ load_metadata reads a truncated or unparseable file as "no install", so a crash,
+ a full disk or a killed installer partway through a plain write_text retires the
+ install the marker described: the next run refetches ~110 MB of Node. Same shape as
+ download_file -- a sibling temp file, flushed and fsynced, then os.replace, which is
+ atomic within a directory on POSIX and Windows alike.
+
+ The temp file is removed on every failure path. A stranded `.tmp-` sibling would sit
+ inside the install directory forever, and _swap_into_place would carry one written
+ here into the live tree.
+
+ Raises: the caller decides. write_metadata is writing into a staging tree that is
+ discarded on failure, so a raise there aborts an install that never landed.
+ """
+ destination = metadata_path(install_dir)
+ destination.parent.mkdir(parents = True, exist_ok = True)
+ try:
+ original: os.stat_result | None = destination.stat()
+ except OSError:
+ original = None
+ original_mode: int | None = stat.S_IMODE(original.st_mode) if original is not None else None
+ # newline at the default, as write_text had it: the marker's bytes must not change on Windows.
+ handle = tempfile.NamedTemporaryFile(
+ prefix = destination.name + ".tmp-",
+ dir = destination.parent,
+ delete = False,
+ mode = "w",
+ encoding = "utf-8",
+ )
+ tmp_path: Path | None = Path(handle.name)
+ try:
+ with handle:
+ handle.write(json.dumps(payload, indent = 2) + "\n")
+ handle.flush()
+ os.fsync(handle.fileno())
+ # NamedTemporaryFile is 0600 and os.replace keeps it, so a refresh left a shared marker
+ # unreadable to other users.
+ replacing_an_existing_marker = original_mode is not None
+ if original_mode is None:
+ mask = os.umask(0)
+ os.umask(mask)
+ original_mode = 0o666 & ~mask
+ try:
+ os.chmod(tmp_path, original_mode)
+ except OSError:
+ # Only the REFRESH abandons: it is the one with another reader and a mode worth
+ # keeping. Raising on a FIRST write aborts a whole Node install over a cosmetic
+ # chmod, reachable on Windows through the sharing violation the swap already retries.
+ if replacing_an_existing_marker:
+ raise
+ if original is not None:
+ # Owner then group, as prebuilt_core.write_live_marker explains: neither call
+ # alone is right for both root and a non-root member of a shared group.
+ try:
+ os.chown(tmp_path, original.st_uid, original.st_gid)
+ except (OSError, AttributeError):
+ try:
+ os.chown(tmp_path, -1, original.st_gid)
+ except (OSError, AttributeError):
+ pass
+ atomic_replace_from_tempfile(tmp_path, destination)
+ tmp_path = None
+ finally:
+ if tmp_path is not None:
+ try:
+ tmp_path.unlink(missing_ok = True)
+ except Exception: # noqa: BLE001
+ pass
+
+
def write_metadata(install_dir: Path, *, version: str, asset: str, sha256: str) -> None:
payload = {
"schema_version": METADATA_SCHEMA_VERSION,
@@ -666,7 +746,7 @@ def write_metadata(install_dir: Path, *, version: str, asset: str, sha256: str)
"asset": asset,
"sha256": sha256,
}
- metadata_path(install_dir).write_text(json.dumps(payload, indent = 2) + "\n", encoding = "utf-8")
+ _write_metadata_payload(install_dir, payload)
def load_metadata(install_dir: Path) -> dict | None:
@@ -680,12 +760,175 @@ def load_metadata(install_dir: Path) -> dict | None:
return data if isinstance(data, dict) else None
+_PREBUILT_FULL_CHECK_ENV = "UNSLOTH_PREBUILT_FULL_CHECK"
+
+
+def prebuilt_full_check_requested() -> bool:
+ """The escape hatch for the recorded-runtime shortcut, spelled as llama and whisper spell it.
+
+ Defined here rather than imported: this installer bootstraps the managed Node runtime and must
+ not depend on the llama module. A user told to set one variable to force a full revalidation
+ would otherwise still get Node answered from its marker.
+ """
+ return os.environ.get(_PREBUILT_FULL_CHECK_ENV, "").strip().lower() in (
+ "1",
+ "true",
+ "yes",
+ "on",
+ )
+
+
+def _file_record(path: Path) -> dict | None:
+ """size, mtime_ns and sha256 for one file, or None when it cannot be read."""
+ # Streamed: node is ~110 MB.
+ try:
+ info = path.stat()
+ digest = sha256_file(path)
+ except (OSError, MemoryError):
+ return None
+ return {"size": info.st_size, "mtime_ns": info.st_mtime_ns, "sha256": digest}
+
+
+def _file_record_matches(path: Path, recorded: object) -> bool:
+ """Whether *path* is still the file *recorded* describes.
+
+ size and mtime_ns only. The digest is recorded too, and deliberately not compared here:
+ this runs on every launch of the installer and node is ~110 MB, so hashing it would cost
+ more than the `node -v` spawn the record exists to avoid. The digest is there so a support
+ log can say which binary this is.
+
+ What that gives up is narrower than it looks, in both directions. A node that no longer
+ RUNS is still caught, because npm-cli.js is run BY the node binary and that probe is never
+ skipped, so the case an update can actually repair is not the case being skipped. And the
+ record was never a tamper defence: it sits in a marker beside the binary, writable by
+ anyone who could rewrite the binary in place, so an edit careful enough to restore the
+ byte count and the nanosecond timestamp is an edit that can restore the record too. The
+ check before this one, `node -v`, could not prove any more than that either.
+ """
+ if not isinstance(recorded, dict):
+ return False
+ try:
+ info = path.stat()
+ except OSError:
+ return False
+ return info.st_size == recorded.get("size") and info.st_mtime_ns == recorded.get("mtime_ns")
+
+
+def record_runtime_verification(
+ install_dir: Path, host: HostInfo, *, version: str, npm_major: int
+) -> None:
+ """Remember that THESE bytes answered `node -v` and `npm --version`.
+
+ Only the node half is later trusted in place of a spawn: npm_major_checked records that
+ the npm probe cleared the floor at the time, not that it still would. See
+ _recorded_runtime_matches.
+
+ Read-modify-write, never raises, and never leaves a half-written marker: this
+ rewrites a file that already describes a good install, so a torn write here is
+ strictly worse than not writing at all.
+ """
+ meta = load_metadata(install_dir)
+ if meta is None:
+ return
+ node_record = _file_record(node_binary_path(install_dir, host))
+ npm_record = _file_record(npm_cli_path(install_dir, host))
+ if node_record is None or npm_record is None:
+ return
+ fresh = {
+ "node_binary": node_record,
+ "npm_cli": npm_record,
+ "node_version_checked": version,
+ "npm_major_checked": npm_major,
+ }
+ if all(meta.get(key) == value for key, value in fresh.items()):
+ # Nothing learned, so nothing is published. Reached on every run under
+ # UNSLOTH_PREBUILT_FULL_CHECK, which re-proves what the record already says: rewriting
+ # the marker there would move its mtime and inode, and re-apply its mode and owner,
+ # forever, for bytes that do not change.
+ return
+ meta.update(fresh)
+ try:
+ _write_metadata_payload(install_dir, meta)
+ except Exception: # noqa: BLE001
+ # An unrefreshable marker costs the two spawns again, never a torn read of "nothing installed".
+ pass
+
+
+def _recorded_runtime_matches(install_dir: Path, host: HostInfo, meta: dict, version: str) -> bool:
+ """Whether a previous run already proved this exact node binary reports this version.
+
+ `node -v` is an interpreter start of a 110 MB runtime, re-run on every install and
+ every update to re-derive an answer that cannot have changed while the binary has not.
+ Absent records mean an install made before this existed, so it pays the spawn once and
+ then records it.
+
+ What this deliberately does NOT prove is npm. The npm record covers npm-cli.js alone,
+ a launcher that bootstraps thousands of files under npm/lib: deleting npm/lib/cli.js
+ leaves the recorded launcher byte-identical while `npm --version` fails. So the npm
+ record is only used as "this file is still the one the last probe ran", and
+ npm_major_checked only as "that probe cleared the floor" -- the caller still pays the
+ npm probe, because only npm can show npm's own module tree still loads.
+ """
+ if prebuilt_full_check_requested():
+ return False
+ if meta.get("node_version_checked") != version:
+ return False
+ npm_major = meta.get("npm_major_checked")
+ if not isinstance(npm_major, int) or npm_major < NPM_MIN_MAJOR:
+ return False
+ if not _file_record_matches(node_binary_path(install_dir, host), meta.get("node_binary")):
+ return False
+ if not _file_record_matches(npm_cli_path(install_dir, host), meta.get("npm_cli")):
+ return False
+ # chmod -x moves ctime only, so the records still match a node that cannot run. npm-cli.js is
+ # read by node, not executed; Windows has no execute bit.
+ return host.is_windows or os.access(node_binary_path(install_dir, host), os.X_OK)
+
+
+def _record_runtime_verification_under_lock(
+ install_dir: Path, host: HostInfo, meta: dict, *, version: str, npm_major: int
+) -> bool:
+ """record_runtime_verification for a caller that does not hold the install lock.
+
+ The record is a read-modify-write of the marker, and the pre-lock check in
+ install_prebuilt is exactly where another installer can be mid-swap: it reads the
+ old marker, the other process swaps a new tree into place, and the old version and
+ checksum are written over the new tree's marker. So the write takes the lock and
+ goes ahead only if the marker is still the one that was read.
+
+ Returns whether the marker was still the one that was read. False means another
+ installer replaced the tree while this one waited, so what was just verified is no
+ longer what is installed; the caller re-checks rather than reporting it current.
+ A write that failed for any other reason still answers True: the install is the one
+ that was verified, and the record merely costs the two spawns again next time.
+ """
+ try:
+ with install_lock(install_lock_path(install_dir), timeout = RECORD_LOCK_TIMEOUT_SECONDS):
+ current = load_metadata(install_dir)
+ if current is None:
+ return False
+ if any(current.get(key) != meta.get(key) for key in ("version", "sha256", "asset")):
+ return False
+ record_runtime_verification(install_dir, host, version = version, npm_major = npm_major)
+ except BusyInstallConflict:
+ # Busy is not evidence. The marker still being the one that was read IS evidence, and
+ # that is the False above; failing to look at it says only that nothing was written.
+ # Answering False here would send a legacy install on to the outer install lock, which
+ # the same holder also fails, so the first launch beside a running installer would exit
+ # busy where the pre-record path reported the install current and exited 0.
+ log("another installer holds the lock; keeping the verified install without a record")
+ except Exception: # noqa: BLE001
+ pass
+ return True
+
+
def existing_install_matches(
install_dir: Path,
host: HostInfo,
*,
version: str,
expected_sha: str | None = None,
+ under_lock: bool = False,
) -> bool:
"""True iff the on-disk install is exactly this version, runs, and (when
expected_sha is given) was recorded with that digest, so a non-pinned or
@@ -695,10 +938,23 @@ def existing_install_matches(
return False
if expected_sha is not None and meta.get("sha256") != expected_sha:
return False
+ if _recorded_runtime_matches(install_dir, host, meta, version):
+ # Stands in for `node -v` only: npm-cli.js bootstraps thousands of files, so npm is still probed.
+ npm_major = installed_npm_major(install_dir, host)
+ return npm_major is not None and npm_major >= NPM_MIN_MAJOR
if installed_node_version(install_dir, host) != version:
return False
npm_major = installed_npm_major(install_dir, host)
- return npm_major is not None and npm_major >= NPM_MIN_MAJOR
+ if npm_major is None or npm_major < NPM_MIN_MAJOR:
+ return False
+ # Written under the install lock either way: the caller's, or one taken here.
+ if under_lock:
+ record_runtime_verification(install_dir, host, version = version, npm_major = npm_major)
+ return True
+ # A marker that changed hands under the lock is another installer's tree, whose re-check decides.
+ return _record_runtime_verification_under_lock(
+ install_dir, host, meta, version = version, npm_major = npm_major
+ )
def existing_install_usable(install_dir: Path, host: HostInfo) -> bool:
@@ -826,7 +1082,9 @@ def install_prebuilt(install_dir: Path, *, channel: str, min_major: int, force:
if (
not force
and may_keep
- and existing_install_matches(install_dir, host, version = version, expected_sha = pin)
+ and existing_install_matches(
+ install_dir, host, version = version, expected_sha = pin, under_lock = True
+ )
):
log(f"existing Node install already matches v{version}; nothing to do")
return EXIT_SUCCESS
@@ -892,6 +1150,12 @@ def install_prebuilt(install_dir: Path, *, channel: str, min_major: int, force:
raise PrebuiltFallback(
f"post-install verification failed: node={final_version} npm_major={npm_major}"
)
+ # After the swap, since _ensure_npm_floor rewrites npm in the staged tree. The lock was released
+ # above, so the write retakes it and proceeds only over the marker this install wrote.
+ installed_meta = load_metadata(install_dir) or {}
+ _record_runtime_verification_under_lock(
+ install_dir, host, installed_meta, version = final_version, npm_major = npm_major
+ )
log(f"installed isolated Node v{final_version} (npm {npm_major}.x) at {install_dir}")
return EXIT_SUCCESS
diff --git a/studio/install_whisper_prebuilt.py b/studio/install_whisper_prebuilt.py
index 70cd0f858fb..2b92a47ac02 100644
--- a/studio/install_whisper_prebuilt.py
+++ b/studio/install_whisper_prebuilt.py
@@ -47,7 +47,10 @@ bundle of an explicitly pinned pre-slim release.
Mirrors ``install_node_prebuilt.py`` / ``install_llama_prebuilt.py``. Exit codes:
0 success (or already current), 1 error, 2 incompatible paired release, 3 busy. A re-run
that already matches logs "already matches" and returns 0 without downloading
-(the scripts grep it).
+(the scripts grep it). A release lookup that could not answer at all also returns 0 when
+an intact, previously validated install is on disk, logging "update unavailable, existing
+prebuilt kept; keeping the existing complete install" -- llama.cpp's wording for its own
+identical outcome, and the substring the setup scripts grep to report it.
"""
from __future__ import annotations
@@ -108,6 +111,11 @@ _OPS = core.ModuleOps(globals())
_LOG_TO_STDOUT = False
+# Raised by prebuilt_core when a fetched release cannot be trusted; re-exported so this
+# module's callers can name it without importing core.
+ReleaseIntegrityError = core.ReleaseIntegrityError
+
+
class ReleaseCompatibilityError(PrebuiltFallback):
"""A valid slim release cannot pair with this host's installed runtime."""
@@ -1044,6 +1052,164 @@ def metadata_path(install_dir: Path) -> Path:
return install_dir / METADATA_FILENAME
+# Every bin layout this installer produces, walked whatever the host says: the backfill runs from
+# settle_kept_install, which prebuilt_core hands the install directory alone.
+_RUNTIME_RECORD_BIN_DIRS = (("build", "bin"), ("build", "bin", "Release"))
+_RUNTIME_RECORD_SERVER_NAMES = ("whisper-server", "whisper-server.exe")
+
+
+def _stat_record(path: Path) -> "dict[str, Any] | None":
+ """size + mtime_ns for one regular file, or None when it is neither."""
+ try:
+ if not path.is_file():
+ return None
+ info = path.stat()
+ except OSError:
+ return None
+ return {"size": info.st_size, "mtime_ns": info.st_mtime_ns}
+
+
+def _runtime_file_records(
+ install_dir: Path,
+ linked_libraries: "Iterable[str] | None" = None,
+ linked_runtime_directories: "Iterable[str] | None" = None,
+) -> "dict[str, dict[str, Any]]":
+ """What the installed payload is made of, in a form a later run can re-check without
+ running it.
+
+ The same two tiers as the llama installer's runtime_file_records, because the two
+ halves of a whisper install fail differently:
+
+ * whisper-server gets size + sha256. Its bytes decide whether dictation works at
+ all, and every reuse path keeps an install without ever starting it, so the
+ digest is what stands in for "it launched once". One small binary is ~10 ms.
+ * the ggml libraries a slim bundle hardlinks get size + mtime_ns, stat only.
+ installed_tree_is_intact checks those for EXISTENCE by name, so a
+ libggml-base.so truncated by a full disk or an interrupted extract passes it --
+ and on a CUDA or ROCm pairing those libraries are most of the bytes. A size
+ comparison catches that for the price of a stat; hashing hundreds of MB of
+ kernels on every update would not be worth it.
+
+ * the ROCm kernel catalogs a slim bundle links get size + mtime_ns, recursively.
+ installed_tree_is_intact asks only that each of those directories hold ANY file
+ (`rglob("*")`), so a rocblas/ that lost or truncated one TensileLibrary blob and
+ kept the rest satisfied it, and the keep-existing path then reported an install
+ whose catalog no longer loads. Stat only, for the same reason as the libraries:
+ rocblas is hundreds of MB and this runs on every update.
+
+ *linked_libraries* and *linked_runtime_directories* come from the slim selection being
+ installed (or from the marker being backfilled); without them only the binary tier is
+ recorded, which is what a fat bundle -- and a caller with no wiring in hand -- can
+ honestly say.
+ """
+ records: dict[str, dict[str, Any]] = {}
+ for parts in _RUNTIME_RECORD_BIN_DIRS:
+ bin_dir = install_dir.joinpath(*parts)
+ for name in linked_libraries or ():
+ # Bare filenames only: a marker naming ../.. must not send the record outside the install.
+ if not isinstance(name, str) or not name or Path(name).name != name:
+ continue
+ record = _stat_record(bin_dir / name)
+ if record is not None:
+ records[(bin_dir / name).relative_to(install_dir).as_posix()] = record
+ for name in linked_runtime_directories or ():
+ # Same bare-name rule: rglob on a marker-supplied path would walk out of the install.
+ if not isinstance(name, str) or not name or Path(name).name != name:
+ continue
+ runtime_dir = bin_dir / name
+ if not runtime_dir.is_dir():
+ continue
+ for path in sorted(runtime_dir.rglob("*")):
+ # Files only; stat follows symlinks, which is what the loader does too.
+ if not path.is_file():
+ continue
+ record = _stat_record(path)
+ if record is not None:
+ records[path.relative_to(install_dir).as_posix()] = record
+ # Last, so a server the wiring loop happened to match is upgraded to the hashed tier.
+ for name in _RUNTIME_RECORD_SERVER_NAMES:
+ candidate = bin_dir / name
+ record = _stat_record(candidate)
+ if record is None:
+ continue
+ try:
+ record["sha256"] = sha256_file(candidate)
+ except (OSError, MemoryError) as exc:
+ # No record at all: a size-only tier would keep a same-size corrupt server forever.
+ log(f"could not hash {name} for the runtime record ({exc}); not recording")
+ return {}
+ records[candidate.relative_to(install_dir).as_posix()] = record
+ return records
+
+
+def runtime_file_records(install_dir: Path, selection: Any) -> "dict[str, dict[str, Any]]":
+ """prebuilt_core.write_prebuilt_metadata's hook: the record for the tree just staged.
+
+ Slim bundles pass their wired ggml filenames through, so the marker describes the
+ hardlinks it is about to claim as well as the server it shipped.
+ """
+ is_slim = getattr(selection, "install_kind", None) == "slim"
+ linked = selection.linked_libraries if is_slim else None
+ runtime_dirs = getattr(selection, "linked_runtime_directories", None) if is_slim else None
+ return _runtime_file_records(install_dir, linked, runtime_dirs)
+
+
+def _runtime_files_match(install_dir: Path, marker: "dict[str, Any]") -> bool:
+ """Whether every recorded payload file is still the file that was installed.
+
+ Sizes for everything, digests for the server that carries one (see
+ _runtime_file_records for why the split). Copied from the llama installer's
+ _runtime_files_match, including both of its deliberate choices:
+
+ * mtime_ns is recorded but deliberately NOT compared. A restore from backup, an
+ rsync or a container layer rewrites it without changing a byte, and the answer
+ to a mismatch here is a 200-400 MB re-download.
+ * it FAILS CLOSED on a record that is present but not a non-empty mapping of
+ mappings. This is the evidence that replaces actually starting whisper-server,
+ so a record that cannot be read is not proof of anything.
+
+ The one divergence from llama is where "absent" is decided, and backwards
+ compatibility forces it: every whisper marker already on a user's disk predates this
+ key, and rejecting those would re-download an install that is perfectly fine. So an
+ ABSENT record is accepted here and backfilled once, under the install lock, by
+ settle_kept_install -> _backfill_runtime_file_records. It is
+ _existing_install_is_intact -- the no-network fast path, which keeps an install
+ having looked at no bytes at all -- that demands the key and takes the full path once
+ without it, exactly as it already does for paired_llama_ggml_tree.
+ """
+ if "runtime_files" not in marker:
+ return True
+ recorded = marker.get("runtime_files")
+ if not isinstance(recorded, dict) or not recorded:
+ log(f"existing install at {install_dir} has an unusable payload record; reinstalling")
+ return False
+ for relative, expected in recorded.items():
+ if not isinstance(expected, dict):
+ log(f"existing install at {install_dir} has an unusable record for {relative}")
+ return False
+ candidate = install_dir / relative
+ try:
+ info = candidate.stat()
+ if info.st_size != expected.get("size"):
+ log(
+ f"existing install at {install_dir} rejected: {relative} is "
+ f"{info.st_size} bytes"
+ )
+ return False
+ digest = expected.get("sha256")
+ if digest is not None and sha256_file(candidate) != digest:
+ log(
+ f"existing install at {install_dir} rejected: {relative} does not match "
+ f"the recorded digest"
+ )
+ return False
+ except OSError as exc:
+ log(f"existing install at {install_dir} rejected: {relative} is unreadable ({exc})")
+ return False
+ # An unrecorded file is not evidence against the install; a RECORDED one that vanished is.
+ return True
+
+
def selection_from_artifact(
*,
published_repo: str,
@@ -1108,7 +1274,189 @@ def existing_install_matches(
llama dir leaves dictation broken while update reports up to date)."""
if not core.existing_install_matches(_OPS, install_dir, host, selection):
return False
+ return installed_tree_is_intact(install_dir, host)
+
+
+def kept_install_needs_settling(install_dir: Path) -> bool:
+ """Whether settle_kept_install has anything to write: a marker with no payload
+ record, or a slim marker missing either half of its pairing record."""
+ marker = load_prebuilt_metadata(install_dir)
+ if not marker:
+ return False
+ # Empty counts as absent: _runtime_file_records answers {} for a server it could not hash.
+ if not marker.get("runtime_files"):
+ return True
+ if marker.get("install_kind") != "slim":
+ return False
+ return not all(
+ isinstance(marker.get(key), str) and marker.get(key)
+ for key in ("paired_llama_ggml_tree", "paired_llama_runtime_id")
+ )
+
+
+def settle_kept_install(install_dir: Path) -> None:
+ """core.install_selected_prebuilt's hook for a kept install, called under the lock.
+
+ The backfill is a read-modify-write of the marker. Done from existing_install_matches
+ it ran once BEFORE the lock, where another installer swapping in a new release
+ between the read and the replace would have had its fresh marker overwritten with
+ the old release's fields plus the backfilled tree.
+ """
+ _backfill_slim_pairing_record(install_dir)
+ _backfill_runtime_file_records(install_dir)
+
+
+def _backfill_runtime_file_records(install_dir: Path) -> None:
+ """Record the payload's sizes and digests on a marker written before that key existed.
+
+ An install made by an older Unsloth Studio carries no runtime_files, and
+ existing_install_current_without_plan demands one: without this backfill every such
+ install would take the full path on EVERY update -- fetching the release, its
+ manifest and its checksum index each time -- instead of once. Nothing here re-downloads
+ anything: the comparison in installed_tree_is_intact accepts an absent record precisely
+ so that an upgrade never costs 200-400 MB for a tree that is fine.
+
+ Added, never corrected: a marker that already carries a record was written by a run
+ that hashed the bytes it installed, and overwriting it would re-bless bytes this run
+ did not choose. What this writes is what is on disk NOW, which is the only thing an
+ offline run can honestly say -- a legacy install already damaged before this ran is
+ recorded as damaged. That is the inherent limit of a record introduced after the fact;
+ from the moment it is written the bytes are pinned.
+
+ Never raises -- the install is already valid, and a read-only marker must not fail
+ setup over a metadata refresh.
+ """
+ marker = load_prebuilt_metadata(install_dir)
+ if not marker or marker.get("runtime_files"):
+ return
+ is_slim = marker.get("install_kind") == "slim"
+ linked = marker.get("linked_libraries") if is_slim else None
+ runtime_dirs = marker.get("linked_runtime_directories") if is_slim else None
+ records = _runtime_file_records(
+ install_dir,
+ linked if isinstance(linked, list) else None,
+ runtime_dirs if isinstance(runtime_dirs, list) else None,
+ )
+ # An empty record is not evidence, and writing one would fail closed on every later update.
+ if not records:
+ return
+ marker["runtime_files"] = records
+ # llama's writer: same atomic temp-and-replace, mode and owner kept, never raises.
+ if llama._write_marker(metadata_path(install_dir), marker):
+ log(f"existing {COMPONENT} install reused; recorded {len(records)} payload files")
+
+
+def _wiring_matches_live_llama(install_dir: Path, marker: dict) -> bool:
+ """Whether this install's wired libraries ARE the live llama runtime's, right now.
+
+ Recording a pairing the wiring has not been checked against is how a FALSE one gets
+ written: the backfill reads whichever llama marker is live at that moment, and a llama
+ that was replaced before the first migration -- or by another installer holding its own
+ per-directory lock during this one -- leaves whisper hardlinked to the previous libraries
+ while the marker claims the new identity. The next fast check then believes it.
+
+ _link_or_copy hardlinks, so a shared (st_dev, st_ino) PROVES the two names are one file.
+ It falls back to shutil.copy2 across filesystems, where the inodes legitimately differ, so
+ an inode mismatch is not evidence of staleness on its own and the bytes are compared
+ instead. Unverifiable answers False: recording nothing costs the fast path, recording a
+ guess costs the install.
+ """
+ linked = marker.get("linked_libraries")
+ linked_from = marker.get("linked_from")
+ if not isinstance(linked, list) or not linked or not isinstance(linked_from, str):
+ return False
+ source_dir = Path(linked_from)
+ if not source_dir.is_dir():
+ return False
+ bin_dir = installed_server_path(install_dir, detect_host()).parent
+ checked = 0
+ for name in linked:
+ if not isinstance(name, str) or Path(name).name != name:
+ return False
+ ours, theirs = bin_dir / name, source_dir / name
+ try:
+ a, b = ours.stat(), theirs.stat()
+ except OSError:
+ return False
+ if (a.st_dev, a.st_ino) != (b.st_dev, b.st_ino):
+ # Copied rather than hardlinked, or genuinely stale. Only the bytes can say.
+ try:
+ if a.st_size != b.st_size or core.sha256_file(ours) != core.sha256_file(theirs):
+ return False
+ except OSError:
+ return False
+ checked += 1
+ return checked > 0
+
+
+def _backfill_slim_pairing_record(install_dir: Path) -> None:
+ """Record the paired llama runtime on a slim marker written before those keys existed.
+
+ existing_install_current_without_plan refuses a slim install whose marker cannot say
+ which llama runtime it hardlinks, so without this an install made before this PR
+ would fetch the release, its manifest and its checksum index on EVERY update rather
+ than once. This is the only place that re-examines a slim install without
+ reinstalling it, and it runs only after the fingerprint and the wiring have just been
+ confirmed, so what it writes describes a pairing it verified.
+
+ Backfilled rather than re-wired on purpose. An older marker records nothing about which
+ gfx bundle it was paired against, so a swap that already happened cannot be detected
+ from it at all, and reinstalling every such install to find out would re-download the
+ bundle for every existing user to answer a question about a swap that almost never
+ happened. Recording the current pairing makes the NEXT one detectable, which is the
+ same trade paired_llama_ggml_tree already makes.
+
+ Added, never corrected: a marker that already names a pairing was written by a run that
+ installed against it. Never raises -- the install is already valid, and a read-only
+ marker must not fail setup over a metadata refresh.
+ """
+ marker = load_prebuilt_metadata(install_dir)
+ if not marker or marker.get("install_kind") != "slim":
+ return
+ # Only record a pairing this install can be SHOWN to have. Without this the backfill
+ # writes whichever llama is live at the moment it runs, which is a guess whenever llama
+ # changed before whisper's first migration or while another installer held llama's own
+ # lock, and the runtime-id check then trusts the guess forever.
+ if not _wiring_matches_live_llama(install_dir, marker):
+ log(
+ f"existing {COMPONENT} install reused; its wiring does not match the live "
+ "llama.cpp runtime, so no pairing was recorded"
+ )
+ return
+ added: list[str] = []
+ for key, live in (
+ ("paired_llama_ggml_tree", installed_paired_runtime_tree),
+ ("paired_llama_runtime_id", installed_paired_runtime_id),
+ ):
+ recorded = marker.get(key)
+ if isinstance(recorded, str) and recorded:
+ continue
+ value = live()
+ if not value:
+ continue
+ marker[key] = value
+ added.append(f"{key}={value}")
+ if not added:
+ return
+ # llama's writer: same atomic temp-and-replace, mode and owner kept, never raises.
+ if llama._write_marker(metadata_path(install_dir), marker):
+ log(f"existing {COMPONENT} install reused; recorded its pairing ({', '.join(added)})")
+
+
+def installed_tree_is_intact(install_dir: Path, host: HostInfo) -> bool:
+ """The on-disk half of existing_install_matches, without the fingerprint.
+
+ Split out so existing_install_current_without_plan -- which runs before anything is
+ resolved and so has no selection to fingerprint against -- makes exactly the same
+ demands of the tree. Two copies of these rules is how a fast path comes to accept an
+ install the slow path repairs.
+ """
server = installed_server_path(install_dir, host)
+ try:
+ if not server.is_file() or server.stat().st_size == 0:
+ return False
+ except OSError:
+ return False
if not host.is_windows and not os.access(server, os.X_OK):
log(f"existing install at {install_dir} has a non-executable server; reinstalling")
return False
@@ -1136,6 +1484,18 @@ def existing_install_matches(
f"libraries ({', '.join(missing[:4])}); reinstalling"
)
return False
+ # Which llama INSTALL, not which source tree: a per-gfx ROCm reselection swaps the asset
+ # within one release, and the hardlinks survive llama's directory swap on purpose, so the
+ # previous GPU's kernels stay wired while every other check passes. Absent is a pre-key
+ # marker, which settle_kept_install records rather than re-downloading.
+ recorded_runtime_id = marker.get("paired_llama_runtime_id")
+ if isinstance(recorded_runtime_id, str) and recorded_runtime_id:
+ if recorded_runtime_id != installed_paired_runtime_id():
+ log(
+ f"existing slim install at {install_dir} is wired to a superseded llama "
+ "runtime; reinstalling"
+ )
+ return False
runtime_dirs = marker.get("linked_runtime_directories")
# Subset plus required, not equality: a target without hipBLASLt kernels wires rocblas alone and is
# complete (#8364), while an unknown name or a missing rocblas still means stale wiring.
@@ -1163,7 +1523,10 @@ def existing_install_matches(
f"({', '.join(str(name) for name in missing_dirs[:4])}); reinstalling"
)
return False
- return True
+ # The only check here that reads the payload's BYTES: a whisper-server truncated to a non-zero
+ # length is still a non-empty executable. Absent on a pre-record marker, which is kept and
+ # backfilled under the lock, so upgrading from an older Studio never re-downloads.
+ return _runtime_files_match(install_dir, marker)
# ── Orchestration ──
@@ -1180,7 +1543,18 @@ def resolve_release_tag(published_repo: str, *, published_release_tag: str | Non
def fetch_release_for_install(
repo: str, *, published_release_tag: str | None
) -> tuple[ReleaseBundle, dict[str, str]]:
- return core.fetch_release_for_install(_OPS, repo, published_release_tag = published_release_tag)
+ try:
+ return core.fetch_release_for_install(
+ _OPS, repo, published_release_tag = published_release_tag
+ )
+ except PrebuiltFallback:
+ raise
+ except (OSError, ValueError, RuntimeError) as exc:
+ # install_prebuilt's keep path reads only PrebuiltFallback, so without this wrap an offline
+ # update printed "prebuilt install failed" over an intact tree.
+ raise PrebuiltFallback(
+ f"could not fetch release {repo}@{published_release_tag or 'latest'}: {exc}"
+ ) from exc
@dataclass(frozen = True)
@@ -1190,6 +1564,8 @@ class WhisperReleasePlan:
artifact: dict[str, Any]
resolved_backend: str
used_fallback: bool
+ # The macOS release walk-back behind bundle (core.WalkBack); None when bundle is the newest.
+ walk_back: core.WalkBack | None = None
def _normalized_upstream_tag(value: str) -> str:
@@ -1337,7 +1713,12 @@ def _release_plan_for_host(
assert first_error is not None
raise first_error
- for release_tag in _published_release_tags(published_repo):
+ # An API limit or network failure is "could not answer", which the keep path handles.
+ try:
+ compatible_tags = _published_release_tags(published_repo)
+ except (OSError, RuntimeError) as exc:
+ raise PrebuiltFallback(f"could not list {published_repo} releases: {exc}") from exc
+ for release_tag in compatible_tags:
if first_bundle is not None and release_tag == first_bundle.release_tag:
continue
try:
@@ -1365,6 +1746,22 @@ def _release_plan_for_host(
f"selected compatible published release {bundle.release_tag} "
f"(upstream {bundle.manifest.get('upstream_tag')})"
)
+ walk_back = (
+ core.walk_back_for(host, first_bundle.release_tag)
+ if first_bundle is not None and not requested_specific_tag
+ else None
+ )
+ if walk_back is not None:
+ # So the marker-only check holds the install while that release is newest for this macOS.
+ plan = replace(
+ plan,
+ walk_back = walk_back,
+ selection = (
+ replace(plan.selection, walk_back = walk_back)
+ if plan.selection is not None
+ else None
+ ),
+ )
return plan
if requested_specific_tag:
@@ -1399,6 +1796,224 @@ def plan_selection(
)
+def installed_paired_runtime_tree() -> str | None:
+ """The ggml tree of the llama runtime a slim bundle would hardlink right now.
+
+ Read from the live llama marker, so prebuilt_core can record it beside
+ paired_llama_tag without knowing anything about llama.cpp.
+ """
+ tree = installed_llama_ggml_tree()
+ return tree if isinstance(tree, str) and tree else None
+
+
+def installed_paired_runtime_id(install_dir: "Path | None" = None) -> str | None:
+ """Which llama INSTALL the hardlinks point into, not which source tree built it.
+
+ ggml_tree cannot answer this: llama publishes a per-gfx ROCm bundle per release, so
+ re-selecting for another gfx target swaps the asset while the tree id stays put. Its
+ install_fingerprint covers asset, asset_sha256 and runtime_sha256, so it moves whenever
+ the bytes behind the hardlinks are superseded.
+ """
+ root = install_dir if install_dir is not None else llama.default_managed_llama_dir()
+ metadata = llama.load_prebuilt_metadata(root)
+ if not metadata:
+ return None
+ recorded = metadata.get("install_fingerprint")
+ return recorded if isinstance(recorded, str) and recorded else None
+
+
+def _existing_install_is_intact(
+ install_dir: Path, host: HostInfo, *, published_repo: str, requested_backend: str
+) -> dict[str, Any] | None:
+ """The marker of a whisper.cpp install worth keeping, or None if there is none.
+
+ Everything existing_install_current_without_plan can establish WITHOUT asking which
+ release is newest: that a previous run of this installer finished and wrote the
+ marker, that it wrote it for this repo and this backend, that a slim install still
+ hardlinks the llama ggml tree it was wired against, and that the tree on disk is the
+ shape the marker describes.
+
+ Split out because install_prebuilt's failed-lookup path has to make exactly these
+ demands and no others -- it runs precisely when "is there a newer release" is the one
+ question nothing can answer -- and a second copy of the remaining rules is how a keep
+ path comes to hold an install the install path would have repaired.
+ """
+ marker = load_prebuilt_metadata(install_dir)
+ if not marker:
+ return None
+ if marker.get("schema_version") != SCHEMA_VERSION or marker.get("component") != COMPONENT:
+ return None
+ if (marker.get("published_repo") or "") != published_repo:
+ return None
+ if marker.get("backend") != requested_backend:
+ return None
+ # A bundle for another architecture (a home directory carried between machines) is not intact.
+ # A marker predating the recorded os/arch is read from its asset name, a convention a custom
+ # repository need not follow.
+ os_token, arch_token = host_platform_tokens(host)
+ recorded_os, recorded_arch = marker.get("os"), marker.get("arch")
+ if isinstance(recorded_os, str) and isinstance(recorded_arch, str):
+ if (recorded_os, recorded_arch) != (os_token, arch_token):
+ return None
+ else:
+ recorded_asset = marker.get("asset")
+ if (
+ not isinstance(recorded_asset, str)
+ or f"-{os_token}-{arch_token}-" not in recorded_asset
+ ):
+ return None
+ # An install restored onto an older Mac has the right tokens and fails at load time.
+ min_os = marker.get("min_os")
+ coverage = marker.get("coverage")
+ if min_os is None and isinstance(coverage, dict):
+ min_os = coverage.get("min_os")
+ if host.is_macos and min_os is not None:
+ if not _macos_min_os_ok(host, min_os):
+ return None
+ recorded_release = marker.get("release_tag")
+ if not isinstance(recorded_release, str) or not recorded_release:
+ return None
+ # A marker without a fingerprint is not a record of a finished install.
+ recorded_fingerprint = marker.get("install_fingerprint")
+ if not isinstance(recorded_fingerprint, str) or not recorded_fingerprint:
+ return None
+ # ...and self-consistent: with no plan to compare against, this is what stands between a
+ # release_tag edited over an old binary and "current". Predating it recomputes to None.
+ if core.marker_install_fingerprint(marker) != recorded_fingerprint:
+ return None
+ # A slim install is only as intact as the llama ggml it hardlinks; predating the key is full path once.
+ if marker.get("install_kind") == "slim":
+ recorded_tree = marker.get("paired_llama_ggml_tree")
+ if not isinstance(recorded_tree, str) or not recorded_tree:
+ return None
+ if recorded_tree != installed_llama_ggml_tree():
+ return None
+ # ...and the payload record itself: this path never asks the network and never starts
+ # whisper-server, so the recorded digests are the only evidence the bytes are the installed
+ # ones. Predating the record is full path once, as with paired_llama_ggml_tree above.
+ if not marker.get("runtime_files") or not isinstance(marker.get("runtime_files"), dict):
+ return None
+ if not installed_tree_is_intact(install_dir, host):
+ return None
+ return marker
+
+
+# What the fork appends to the upstream tag it packages: v1.9.2 -> v1.9.2-unsloth.17.
+_PACKAGING_SUFFIX = "-unsloth."
+
+
+def _api_newest_release_tag_for_upstream(
+ repo: str, whisper_tag: str, recorded_release: str
+) -> "str | None":
+ """The newest published release packaging *whisper_tag* (v1.9.2-unsloth.17, .18, ...).
+
+ The release the marker records is known to package it, so it is a candidate too;
+ the newest of them is what _release_plan_for_host would take.
+ """
+ wanted = _normalized_upstream_tag(whisper_tag)
+ try:
+ releases = llama.github_releases(
+ repo, max_pages = llama.DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES
+ )
+ except Exception as exc: # noqa: BLE001 - unreachable is a reason to do the work
+ log(f"could not list the {COMPONENT} releases packaging {whisper_tag} ({exc})")
+ return None
+ matching = []
+ for release in releases:
+ tag = release.get("tag_name") if isinstance(release, dict) else None
+ if not isinstance(tag, str):
+ continue
+ # The upstream part can carry a hyphen (v1.9.2-rc1), so the tag is neither cut at its first
+ # hyphen nor matched on any: both read v1.9.2-rc1-unsloth.2 as a packaging of v1.9.2.
+ packaged = _normalized_upstream_tag(tag)
+ if (
+ tag == recorded_release
+ or packaged == wanted
+ or packaged.startswith(wanted + _PACKAGING_SUFFIX)
+ ):
+ matching.append(release)
+ return llama._newest_release_tag_from_releases(matching)
+
+
+def existing_install_current_without_plan(
+ install_dir: Path,
+ host: HostInfo,
+ *,
+ whisper_tag: str,
+ published_repo: str,
+ published_release_tag: str | None,
+ requested_backend: str,
+) -> bool:
+ """Whether the whisper.cpp install on disk is already the one this run would make.
+
+ Runs BEFORE _release_plan_for_host, which fetches the release, its manifest and its
+ checksum index on every update to conclude that nothing moved. Every check is on-disk
+ evidence or a tag comparison; the one network call is the same HEAD the llama
+ pre-check makes, and only when the requested tag is "latest".
+
+ The slim pairing is what makes this component's version of the check different: a
+ slim whisper bundle hardlinks the llama runtime's ggml libraries, so a llama install
+ that moved underneath it invalidates a whisper install whose own release did not.
+ That half is _existing_install_is_intact, and it runs FIRST, for the reason llama's
+ pre-check orders its own probes that way: a box whose install is damaged should not
+ pay a network round trip to learn it must reinstall anyway.
+ """
+ if llama.prebuilt_full_check_requested():
+ return False
+ marker = _existing_install_is_intact(
+ install_dir,
+ host,
+ published_repo = published_repo,
+ requested_backend = requested_backend,
+ )
+ if marker is None:
+ return False
+ recorded_release = str(marker.get("release_tag"))
+ pinned = (published_release_tag or "").strip()
+ requested = (whisper_tag or "latest").strip().lower()
+ if requested not in ("", "latest"):
+ # An upstream pin, checked even with a release pin: the full path refuses a pinned release
+ # targeting another upstream version (_bundle_matches_whisper_tag). This fork publishes
+ # several packagings of one upstream tag and takes the newest, so without a release pin the
+ # upstream_tag alone cannot answer.
+ if _normalized_upstream_tag(
+ str(marker.get("upstream_tag") or "")
+ ) != _normalized_upstream_tag(whisper_tag):
+ return False
+ if pinned:
+ if pinned != recorded_release:
+ return False
+ elif requested not in ("", "latest"):
+ # The newest packaging of THAT version, which /releases/latest cannot name once a newer
+ # upstream ships: ask the release list, by tag name, which only the fork's convention
+ # supports; a custom repository is matched by manifest on the full path.
+ if published_repo != DEFAULT_PUBLISHED_REPO:
+ return False
+ newest = _api_newest_release_tag_for_upstream(published_repo, whisper_tag, recorded_release)
+ if not newest or newest != recorded_release:
+ return False
+ else:
+ if not llama._download_host_resolve_enabled():
+ return False
+ try:
+ latest = llama._download_host_latest_release_tag(published_repo)
+ except Exception as exc: # noqa: BLE001 - unreachable is a reason to do the work
+ log(f"could not resolve the latest {COMPONENT} release without the API ({exc})")
+ return False
+ if not latest:
+ return False
+ if latest != recorded_release and not core.walk_back_stands(marker, host, latest):
+ # A recorded macOS walk-back stands while the newest release is still the skipped one on
+ # the same OS version; a newer release or an OS upgrade re-decides it.
+ return False
+ # "already matches" is the substring setup.sh:3635 and setup.ps1:6109 grep for.
+ log(
+ f"existing {COMPONENT} install already matches {recorded_release} "
+ f"({marker.get('backend')}); nothing to do"
+ )
+ return True
+
+
def install_prebuilt(
install_dir: Path,
*,
@@ -1420,13 +2035,68 @@ def install_prebuilt(
f"target {COMPONENT} from {published_repo} "
f"({os_token}-{arch_token}, backend {requested_backend})"
)
- plan = _release_plan_for_host(
+ if not force and existing_install_current_without_plan(
+ install_dir,
host,
+ whisper_tag = whisper_tag,
published_repo = published_repo,
published_release_tag = published_release_tag,
- whisper_tag = whisper_tag,
requested_backend = requested_backend,
- )
+ ):
+ return 0
+ try:
+ plan = _release_plan_for_host(
+ host,
+ published_repo = published_repo,
+ published_release_tag = published_release_tag,
+ whisper_tag = whisper_tag,
+ requested_backend = requested_backend,
+ )
+ except (ReleaseCompatibilityError, core.ReleaseIntegrityError):
+ # The lookup ANSWERED, so keeping would paper over a real answer. Two kinds:
+ # ReleaseCompatibilityError, no published bundle pairs with this host's llama.cpp runtime
+ # (real release skew, setup names both tags from exit 2); and ReleaseIntegrityError, the
+ # release was fetched and found untrustworthy -- an asset outside the checksum index, or a
+ # manifest digest disagreeing with it. Reporting "update unavailable, existing prebuilt
+ # kept" over a tamper signal would turn it into a routine offline notice.
+ raise
+ except PrebuiltFallback as exc:
+ # llama.cpp's rule: a lookup that could not answer says nothing about the tree on disk. A
+ # strict offline update used to print "prebuilt install failed" over a healthy install.
+ # Anything this RUN asked for that keeping would ignore fails instead. Not --has-rocm or
+ # --rocm-gfx (both entrypoints forward DETECTED hardware on every AMD host); not
+ # --published-repo, --backend or --cpu-fallback (a backend request), which
+ # _existing_install_is_intact compares against the marker. UNSLOTH_WHISPER_FORCE_COMPILE
+ # counts: setup.sh runs the source build only after a nonzero exit.
+ explicit_release_request = (
+ force
+ or bool((published_release_tag or "").strip())
+ or (whisper_tag or "latest").strip().lower() not in ("", "latest")
+ or os.environ.get("UNSLOTH_WHISPER_FORCE_COMPILE", "").strip() == "1"
+ )
+ marker = (
+ None
+ if explicit_release_request
+ else _existing_install_is_intact(
+ install_dir,
+ host,
+ published_repo = published_repo,
+ requested_backend = requested_backend,
+ )
+ )
+ if marker is None:
+ raise
+ # "keeping the existing complete install" is the substring setup.sh and setup.ps1 grep for
+ # llama's identical outcome. Not "already matches" or "installed": both name a release this
+ # run never fetched.
+ log(
+ f"{COMPONENT} update unavailable, existing prebuilt kept; keeping the "
+ f"existing complete install of {marker.get('release_tag')}"
+ )
+ # llama.cpp's wording, so update_flow reads both installers alike. log_lines: a multi-line
+ # reason is otherwise indistinguishable from unprefixed diagnostics.
+ log_lines(f"prebuilt update reason: {exc}".splitlines())
+ return EXIT_SUCCESS
if plan.selection is None: # pragma: no cover - install plans always verify
raise PrebuiltFallback("install plan did not validate its checksum entry")
return core.install_selected_prebuilt(
diff --git a/studio/prebuilt_core.py b/studio/prebuilt_core.py
index 4db8a6d022f..6a227d8ebf4 100644
--- a/studio/prebuilt_core.py
+++ b/studio/prebuilt_core.py
@@ -32,6 +32,7 @@ import functools
import hashlib
import json
import os
+import stat
import random
import re
import shutil
@@ -80,6 +81,19 @@ class PrebuiltFallback(RuntimeError):
pass
+class ReleaseIntegrityError(PrebuiltFallback):
+ """A release that ANSWERED, and the answer was that it cannot be trusted.
+
+ A PrebuiltFallback means "this lookup could not answer", which says nothing about the tree
+ already on disk, so the keep-existing paths treat it as a reason to hold the install and
+ report success. An asset missing from the checksum index, or a manifest digest that
+ disagrees with it, is not that: the release was fetched and found untrustworthy, and
+ reporting "update unavailable" over it turns a tamper signal into a routine offline notice.
+ A subclass rather than a sibling so every existing `except PrebuiltFallback` still catches
+ it; only the keep paths single it out and re-raise.
+ """
+
+
class BusyInstallConflict(RuntimeError):
pass
@@ -96,6 +110,11 @@ HTTP_FETCH_BASE_DELAY_SECONDS = 0.75
JSON_FETCH_ATTEMPTS = 3
TTY_PROGRESS_START_DELAY_SECONDS = 0.5
INSTALL_LOCK_TIMEOUT_SECONDS = 300
+# The metadata catch-up over a kept install is an optimisation, and it runs on the FIRST update
+# after an upgrade, for every existing user. Waiting the full install timeout for it would hold
+# an otherwise finished launch for five minutes to write fields whose only effect is to spare
+# the next run some work, so it asks briefly and gives up.
+SETTLE_LOCK_TIMEOUT_SECONDS = 5
INSTALL_STAGING_ROOT_NAME = ".staging"
SCHEMA_VERSION = 1
# Backend to retry when the preferred one has no covering asset; None disables it.
@@ -166,6 +185,14 @@ _OPS_FIRST_NAMES = {
"write_prebuilt_metadata",
"load_prebuilt_metadata",
"existing_install_matches",
+ "marker_install_fingerprint",
+ "WalkBack",
+ "WALK_BACK_KEYS",
+ "macos_version_label",
+ "walk_back_for",
+ "marker_walk_back",
+ "walk_back_stands",
+ "walk_back_patch",
"metadata_path",
"selection_from_artifact",
"plan_selection",
@@ -462,9 +489,83 @@ def atomic_write_bytes(destination: Path, data: bytes) -> None:
os.replace(tmp_path, destination)
-def atomic_replace_from_tempfile(tmp_path: Path, destination: Path) -> None:
+def atomic_replace_from_tempfile(
+ tmp_path: Path,
+ destination: Path,
+ *,
+ attempts: int = 8,
+) -> None:
+ """os.replace, retried against transient Windows sharing violations.
+
+ Rename-over needs DELETE access on the destination, so a scanner or the indexer holding the
+ marker open fails the swap outright -- where the in-place write this replaced would only
+ have contended for write access. Handles clear in a second or two, so a bounded backoff
+ turns the failure into a pause; anything else raises at once rather than stalling on a real
+ problem. A no-op off Windows, where the rename does not care about open handles.
+ """
destination.parent.mkdir(parents = True, exist_ok = True)
- os.replace(tmp_path, destination)
+ delay = 0.25
+ for attempt in range(attempts):
+ try:
+ os.replace(tmp_path, destination)
+ return
+ except OSError as exc:
+ transient = os.name == "nt" and getattr(exc, "winerror", None) in (5, 32, 145)
+ if not transient or attempt == attempts - 1:
+ raise
+ log(f"marker swap blocked ({exc.winerror}), retrying in {delay:.2f}s")
+ time.sleep(delay)
+ delay = min(delay * 2, 4.0)
+
+
+def write_live_marker(marker_path: Path, marker: dict[str, Any]) -> None:
+ """Rewrite a marker that is already in service, keeping its mode and owner.
+
+ Temp-and-replace, so a write that fails part-way leaves the valid marker it found;
+ and the mode and owner restored on the temp file BEFORE the swap, since os.replace
+ keeps the source file's, and NamedTemporaryFile's 0600 would leave a group-shared
+ install's marker readable only by whoever ran this update. Mirrors the llama
+ installer's marker writer. Raises on failure; callers decide what a failed
+ refresh costs.
+ """
+ data = (json.dumps(marker, indent = 2) + "\n").encode("utf-8")
+ try:
+ original = marker_path.stat()
+ except OSError:
+ original = None
+ tmp_path: Path | None = None
+ try:
+ with tempfile.NamedTemporaryFile(
+ prefix = marker_path.name + ".tmp-",
+ dir = marker_path.parent,
+ delete = False,
+ ) as handle:
+ tmp_path = Path(handle.name)
+ handle.write(data)
+ handle.flush()
+ os.fsync(handle.fileno())
+ if original is not None:
+ os.chmod(tmp_path, stat.S_IMODE(original.st_mode))
+ # Owner AND group when the caller can, group alone when it cannot. chown is
+ # all-or-nothing, so the combined call is refused outright for a non-root member of
+ # a shared group and os.replace then installs the member's own uid and gid, which is
+ # how the group was lost (e8d128d24); group-only alone leaves a root-refreshed marker
+ # owned by root, unreadable through an 0600 mode to the user who owns the install.
+ try:
+ os.chown(tmp_path, original.st_uid, original.st_gid)
+ except (OSError, AttributeError):
+ try:
+ os.chown(tmp_path, -1, original.st_gid)
+ except (OSError, AttributeError):
+ pass
+ atomic_replace_from_tempfile(tmp_path, marker_path)
+ except BaseException:
+ if tmp_path is not None:
+ try:
+ tmp_path.unlink()
+ except OSError:
+ pass
+ raise
def sha256_file(path: Path) -> str:
@@ -1079,14 +1180,15 @@ def restore_tar_exec_bits(archive_path: Path, destination: Path) -> None:
# ── Install lock ──
@contextmanager
-def install_lock(lock_path: Path) -> Iterator[None]:
+def install_lock(lock_path: Path, *, timeout: float | None = None) -> Iterator[None]:
+ seconds = INSTALL_LOCK_TIMEOUT_SECONDS if timeout is None else timeout
lock_path.parent.mkdir(parents = True, exist_ok = True)
if FileLock is None:
# Fallback lock: exclusive file creation, writing our PID so stale locks
# from crashed processes can be detected.
fd: int | None = None
- deadline = time.monotonic() + INSTALL_LOCK_TIMEOUT_SECONDS
+ deadline = time.monotonic() + seconds
while True:
try:
fd = os.open(str(lock_path), os.O_CREAT | os.O_EXCL | os.O_RDWR)
@@ -1112,7 +1214,7 @@ def install_lock(lock_path: Path) -> Iterator[None]:
# Exists but PID not yet written; wait for the write to land.
if time.monotonic() >= deadline:
raise BusyInstallConflict(
- f"timed out after {INSTALL_LOCK_TIMEOUT_SECONDS}s waiting for concurrent install lock: {lock_path}"
+ f"timed out after {seconds}s waiting for concurrent install lock: {lock_path}"
)
time.sleep(0.1)
continue
@@ -1130,7 +1232,7 @@ def install_lock(lock_path: Path) -> Iterator[None]:
continue
if time.monotonic() >= deadline:
raise BusyInstallConflict(
- f"timed out after {INSTALL_LOCK_TIMEOUT_SECONDS}s waiting for concurrent install lock: {lock_path}"
+ f"timed out after {seconds}s waiting for concurrent install lock: {lock_path}"
)
time.sleep(0.5)
try:
@@ -1142,11 +1244,11 @@ def install_lock(lock_path: Path) -> Iterator[None]:
return
try:
- with FileLock(lock_path, timeout = INSTALL_LOCK_TIMEOUT_SECONDS):
+ with FileLock(lock_path, timeout = seconds):
yield
except FileLockTimeout as exc:
raise BusyInstallConflict(
- f"timed out after {INSTALL_LOCK_TIMEOUT_SECONDS}s waiting for concurrent install lock: {lock_path}"
+ f"timed out after {seconds}s waiting for concurrent install lock: {lock_path}"
) from exc
@@ -1760,13 +1862,13 @@ def expected_sha256_for(
for the asset must agree with the index (a mismatch means a tampered manifest)."""
digest = checksums.get(asset_name)
if digest is None:
- raise PrebuiltFallback(
+ raise ReleaseIntegrityError(
f"{asset_name} is not covered by {ops.SHA256_ASSET_NAME}; "
f"refusing an unverifiable download"
)
embedded = valid_sha256(manifest_sha256)
if embedded is not None and embedded != digest:
- raise PrebuiltFallback(
+ raise ReleaseIntegrityError(
f"manifest sha256 for {asset_name} disagrees with {ops.SHA256_ASSET_NAME}; "
f"refusing a possibly tampered release"
)
@@ -2009,6 +2111,12 @@ class InstallSelection:
linked_libraries: tuple[str, ...] | None = None
runtime_wiring_version: int | None = None
linked_runtime_directories: tuple[str, ...] | None = None
+ # None when release_tag is the newest. Describes the choice, not the bundle: never fingerprinted.
+ walk_back: "WalkBack | None" = None
+ # The manifest's os/arch, so a keep decision need not infer it from an asset name a custom
+ # repository may spell freely. Outside the fingerprint.
+ platform_os: str | None = None
+ platform_arch: str | None = None
def fingerprint(self) -> str:
return compute_install_fingerprint(
@@ -2024,6 +2132,90 @@ class InstallSelection:
)
+@dataclass(frozen = True)
+class WalkBack:
+ """A macOS release walk-back: the newest published release the planner skipped
+ because the host was below its OS floor, and the host version that decided it.
+
+ Recorded on the marker as walk_back and walked_back_on_macos, outside the
+ fingerprint (it describes the choice, not the bundle). The marker-only re-check
+ holds an install current while BOTH still stand: the newest published release is
+ the one skipped, and the host is the macOS version that skipped it. A newer release
+ or an OS upgrade takes the full path, which re-decides the walk-back.
+ """
+
+ release_tag: str
+ macos_version: str
+
+ def marker_fields(self) -> dict[str, str]:
+ return {
+ "walked_back_from": self.release_tag,
+ "walked_back_on_macos": self.macos_version,
+ }
+
+
+WALK_BACK_KEYS = ("walked_back_from", "walked_back_on_macos")
+
+
+def macos_version_label(host: Any) -> str | None:
+ """The host's macOS version as recorded beside a walk-back ("14.7"); None off
+ macOS or when the version is unknown."""
+ if not getattr(host, "is_macos", False):
+ return None
+ version = getattr(host, "macos_version", None)
+ if not version:
+ return None
+ return ".".join(str(part) for part in version)
+
+
+def walk_back_for(host: Any, skipped_release_tag: str | None) -> WalkBack | None:
+ """The walk-back to record when the planner settled below *skipped_release_tag*
+ on this host; None when there is nothing to record (no skipped release, not
+ macOS, or a host version the marker-only re-check could not compare)."""
+ if not skipped_release_tag:
+ return None
+ label = macos_version_label(host)
+ if label is None:
+ return None
+ return WalkBack(release_tag = skipped_release_tag, macos_version = label)
+
+
+def marker_walk_back(marker: dict[str, Any]) -> WalkBack | None:
+ """The walk-back a marker records, or None when it records none or only half of
+ one (a marker written before the host version was kept beside the tag)."""
+ release_tag = marker.get("walked_back_from")
+ macos_version = marker.get("walked_back_on_macos")
+ if not (isinstance(release_tag, str) and release_tag):
+ return None
+ if not (isinstance(macos_version, str) and macos_version):
+ return None
+ return WalkBack(release_tag = release_tag, macos_version = macos_version)
+
+
+def walk_back_stands(marker: dict[str, Any], host: Any, expected_release: str | None) -> bool:
+ """Whether the marker's walk-back still explains why *expected_release*, the newest
+ published release, is not the installed one: same skipped release, same host
+ macOS version."""
+ recorded = marker_walk_back(marker)
+ if recorded is None or not expected_release:
+ return False
+ return (
+ recorded.release_tag == expected_release
+ and recorded.macos_version == macos_version_label(host)
+ )
+
+
+def walk_back_patch(marker: dict[str, Any], walk_back: WalkBack | None) -> dict[str, Any]:
+ """The marker keys a reused install must take from this run's walk-back: both
+ fields when the plan walked back and the marker says otherwise, None for each
+ one present when the plan no longer walks back."""
+ if walk_back is None:
+ return {key: None for key in WALK_BACK_KEYS if marker.get(key) is not None}
+ return {
+ key: value for key, value in walk_back.marker_fields().items() if marker.get(key) != value
+ }
+
+
def selection_from_artifact(
ops: ModuleOps,
*,
@@ -2047,6 +2239,8 @@ def selection_from_artifact(
else None,
coverage = ops.artifact_coverage(artifact),
studio_protocol = manifest.get("studio_protocol"),
+ platform_os = artifact.get("os") if isinstance(artifact.get("os"), str) else None,
+ platform_arch = artifact.get("arch") if isinstance(artifact.get("arch"), str) else None,
)
@@ -2072,24 +2266,144 @@ def write_prebuilt_metadata(ops: ModuleOps, install_dir: Path, selection: Instal
"min_os": coverage.get("min_os"),
"studio_protocol": selection.studio_protocol,
"install_fingerprint": selection.fingerprint(),
+ # The one fingerprint input the top-level fields do not carry whole, so a later run can
+ # recompute and tell a whole marker from an edited one.
+ "fingerprint_coverage": coverage,
"installed_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
}
+ if selection.walk_back is not None:
+ payload.update(selection.walk_back.marker_fields())
+ if selection.platform_os and selection.platform_arch:
+ payload["os"] = selection.platform_os
+ payload["arch"] = selection.platform_arch
if selection.install_kind == "slim":
# Additive slim fields; fat markers keep the legacy payload exactly.
payload["install_kind"] = "slim"
payload["paired_llama_tag"] = selection.paired_llama_tag
payload["linked_from"] = selection.linked_from
+ # The ggml tree these hardlinks point into: the tag alone cannot say whether a later llama
+ # build still backs this bundle. Absent reads as "cannot say": full path.
+ paired_tree = getattr(ops, "installed_paired_runtime_tree", None)
+ paired_tree = paired_tree() if callable(paired_tree) else None
+ if isinstance(paired_tree, str) and paired_tree:
+ payload["paired_llama_ggml_tree"] = paired_tree
+ # ...and WHICH install of that tree, since one release publishes a bundle per gfx target:
+ # the tree id survives a reselection that replaces every byte behind these hardlinks.
+ paired_id = getattr(ops, "installed_paired_runtime_id", None)
+ paired_id = paired_id() if callable(paired_id) else None
+ if isinstance(paired_id, str) and paired_id:
+ payload["paired_llama_runtime_id"] = paired_id
if selection.linked_libraries is not None:
payload["linked_libraries"] = list(selection.linked_libraries)
if selection.runtime_wiring_version is not None:
payload["runtime_wiring_version"] = selection.runtime_wiring_version
if selection.linked_runtime_directories is not None:
payload["linked_runtime_directories"] = list(selection.linked_runtime_directories)
+ # Optional per component: size + sha256 of the payload a reuse would otherwise have to RUN to
+ # trust. Written last so it covers the wiring above. Absent means full path once.
+ records = getattr(ops, "runtime_file_records", None)
+ records = records(install_dir, selection) if callable(records) else None
+ if records:
+ payload["runtime_files"] = records
ops.metadata_path(install_dir).write_text(
json.dumps(payload, indent = 2) + "\n", encoding = "utf-8"
)
+def marker_install_fingerprint(metadata: dict[str, Any]) -> str | None:
+ """The fingerprint recomputed from the marker's own recorded fields.
+
+ Self-consistency, not a comparison against a fresh plan: equal to the recorded
+ install_fingerprint only when every field it was computed from is still the one
+ written with it, which is what lets a no-network check trust the release_tag it
+ reads. None for a marker that predates fingerprint_coverage, which then takes the
+ full path once and is settled there (_backfill_fingerprint_inputs).
+ """
+ coverage = metadata.get("fingerprint_coverage")
+ if not isinstance(coverage, dict):
+ return None
+ try:
+ return compute_install_fingerprint(
+ published_repo = str(metadata.get("published_repo")),
+ release_tag = str(metadata.get("release_tag")),
+ upstream_tag = metadata.get("upstream_tag"),
+ source_commit = metadata.get("source_commit"),
+ asset = str(metadata.get("asset")),
+ asset_sha256 = str(metadata.get("asset_sha256")),
+ backend = str(metadata.get("backend")),
+ runtime_line = metadata.get("runtime_line"),
+ coverage = coverage,
+ )
+ except (TypeError, ValueError):
+ return None
+
+
+def _kept_marker_patch(
+ ops: ModuleOps, install_dir: Path, selection: InstallSelection
+) -> dict[str, Any] | None:
+ """What a kept install's marker still has to take from this run's selection.
+
+ fingerprint_coverage and the platform on a marker written before those keys
+ existed, and the walk-back this run's plan made (or retired). None when nothing
+ is owed, or when the component has no marker readers (optional, like the settle
+ hooks).
+ """
+ load = getattr(ops, "load_prebuilt_metadata", None)
+ if load is None or getattr(ops, "metadata_path", None) is None:
+ return None
+ metadata = load(install_dir)
+ if not metadata:
+ return None
+ patch: dict[str, Any] = {}
+ if not isinstance(metadata.get("fingerprint_coverage"), dict):
+ patch["fingerprint_coverage"] = selection.coverage
+ patch.update(walk_back_patch(metadata, selection.walk_back))
+ # Added only: a platform already recorded was written by the run that selected it.
+ if (
+ selection.platform_os
+ and selection.platform_arch
+ and not (isinstance(metadata.get("os"), str) and isinstance(metadata.get("arch"), str))
+ ):
+ patch["os"] = selection.platform_os
+ patch["arch"] = selection.platform_arch
+ return patch or None
+
+
+def _kept_marker_needs_settle(
+ ops: ModuleOps, install_dir: Path, selection: InstallSelection
+) -> bool:
+ return _kept_marker_patch(ops, install_dir, selection) is not None
+
+
+def _backfill_fingerprint_inputs(
+ ops: ModuleOps, install_dir: Path, selection: InstallSelection
+) -> None:
+ """Catch a kept marker up to this run: fingerprint_coverage and the platform where
+ the marker predates those keys, and the walk-back the plan made or retired.
+
+ Only for a marker whose fingerprint this run's selection reproduces, so the
+ coverage written is the one it was computed from; anything else is left for the
+ install path to rewrite whole.
+ """
+ patch = _kept_marker_patch(ops, install_dir, selection)
+ if not patch:
+ return
+ metadata = ops.load_prebuilt_metadata(install_dir)
+ # A second read: another installer swapping the tree in between leaves None, and falling out is
+ # right, since the run that replaced the tree wrote its own marker.
+ if not metadata:
+ return
+ if metadata.get("install_fingerprint") != selection.fingerprint():
+ return
+ for key, value in patch.items():
+ if value is None:
+ metadata.pop(key, None)
+ else:
+ metadata[key] = value
+ # Over a LIVE marker: temp-and-replace, so a failed write leaves the valid one in place.
+ write_live_marker(ops.metadata_path(install_dir), metadata)
+
+
def load_prebuilt_metadata(ops: ModuleOps, install_dir: Path) -> dict[str, Any] | None:
path = ops.metadata_path(install_dir)
if not path.is_file():
@@ -2362,11 +2676,14 @@ def install_selected_prebuilt(
"""
if not force and ops.existing_install_matches(install_dir, host, selection):
- return 0
+ if _settle_kept_install(ops, install_dir, host, selection, locked = False):
+ return 0
+ # Changed under the lock: the locked path re-checks rather than reporting a replaced release.
with ops.install_lock(ops.install_lock_path(install_dir)):
# Re-check under the lock: a concurrent run may have just finished.
if not force and ops.existing_install_matches(install_dir, host, selection):
+ _settle_kept_install(ops, install_dir, host, selection, locked = True)
return 0
ops._install_from_bundle(install_dir, host, bundle, selection)
@@ -2379,6 +2696,51 @@ def install_selected_prebuilt(
return 0
+def _settle_kept_install(
+ ops: ModuleOps, install_dir: Path, host: Any, selection: InstallSelection, *, locked: bool
+) -> bool:
+ """Let a component catch up the marker of an install it is keeping, under the lock.
+
+ Returns False only when the pre-lock keep's re-check under the lock found another
+ install on disk; everything else, a settle that fails included, keeps the install.
+
+ Optional: a component that has nothing to write defines neither hook. One that
+ does (whisper's slim pairing backfill) rewrites the marker, and a rewrite outside
+ the install lock races a concurrent installer swapping in a new release: the old
+ marker is read, the tree is replaced, the old fields are written over the new
+ marker. So the pre-lock keep takes the lock for the write, re-checks that the
+ install it read is still the one on disk, and only then settles it. Never raises:
+ the install is already valid, and a lock that cannot be had or a write that fails
+ costs the next run the same settle, not the install.
+ """
+ settle = getattr(ops, "settle_kept_install", None)
+ needs_settling = getattr(ops, "kept_install_needs_settling", None)
+ try:
+ if locked:
+ _backfill_fingerprint_inputs(ops, install_dir, selection)
+ if settle is not None:
+ settle(install_dir)
+ return True
+ needs = _kept_marker_needs_settle(ops, install_dir, selection) or (
+ needs_settling is not None and needs_settling(install_dir)
+ )
+ if not needs:
+ return True
+ with ops.install_lock(
+ ops.install_lock_path(install_dir), timeout = SETTLE_LOCK_TIMEOUT_SECONDS
+ ):
+ if ops.existing_install_matches(install_dir, host, selection):
+ _backfill_fingerprint_inputs(ops, install_dir, selection)
+ if settle is not None:
+ settle(install_dir)
+ return True
+ except Exception as exc: # noqa: BLE001 - a metadata catch-up must never fail a kept install
+ ops.log(f"kept {ops.COMPONENT} install not settled: {exc}")
+ return True
+ ops.log(f"kept {ops.COMPONENT} install changed under the lock; re-validating")
+ return False
+
+
def resolve_prebuilt(
ops: ModuleOps,
host: Any,
diff --git a/studio/setup.ps1 b/studio/setup.ps1
index a6dc272e683..4fdcc73753f 100644
--- a/studio/setup.ps1
+++ b/studio/setup.ps1
@@ -6624,6 +6624,11 @@ if ($env:WHISPER_SERVER_PATH -or $env:UNSLOTH_WHISPER_CPP_PATH) {
if ($whisperExit -eq 0) {
if ($whisperOutput -match "already matches") {
step "whisper.cpp" "prebuilt up to date"
+ } elseif ($whisperOutput -match "keeping the existing complete install") {
+ # Exit 0 can also mean a kept tree after a lookup that could not answer; "prebuilt
+ # installed" would name a release nothing fetched. Same wording and token as llama's
+ # arm.
+ step "whisper.cpp" "update unavailable, existing prebuilt kept" "Yellow"
} else {
step "whisper.cpp" "prebuilt installed"
}
diff --git a/studio/setup.sh b/studio/setup.sh
index 08287c89094..45613f3b4b8 100755
--- a/studio/setup.sh
+++ b/studio/setup.sh
@@ -3871,6 +3871,11 @@ else
if [ "$_WHISPER_STATUS" -eq 0 ]; then
if grep -Fq "already matches" "$_WHISPER_LOG"; then
step "whisper.cpp" "prebuilt up to date"
+ elif grep -Fq "keeping the existing complete install" "$_WHISPER_LOG"; then
+ # Exit 0 can also mean a kept tree after a lookup that could not answer; "prebuilt
+ # installed" would name a release nothing fetched. Same wording and token as llama's
+ # arm.
+ step "whisper.cpp" "update unavailable, existing prebuilt kept" "$C_WARN"
else
step "whisper.cpp" "prebuilt installed"
fi
diff --git a/tests/studio/install/_pr10648_helpers.py b/tests/studio/install/_pr10648_helpers.py
new file mode 100644
index 00000000000..174f42da2b9
--- /dev/null
+++ b/tests/studio/install/_pr10648_helpers.py
@@ -0,0 +1,181 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The preamble the six ``test_pr10648_*`` suites genuinely share.
+
+Only what was BYTE-IDENTICAL in more than one suite lives here. Anything describing one
+suite's scenario stays in that suite, even where another file uses the same name --
+``_install``, ``_host``, ``_fast_path`` and ``_marker`` each mean different things in
+different files, and merging them would be a bug rather than a cleanup.
+
+Every suite runs under ``pytest -n 4``, so: nothing here memoises or caches, and
+``load_studio_module`` takes the ``sys.modules`` name as an argument rather than picking
+one. The suites monkeypatch module globals, so two files sharing a key would see each
+other's patches inside an xdist worker; the distinct per-suite names are what keeps them
+apart, and they stay spelled out at each call site.
+
+llama, whisper and node each have their OWN ``HostInfo`` dataclass with different fields,
+so the factories below stay separate and there is deliberately no fourth that unifies
+them. Each takes the dataclass, because every suite loads its own module instance.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+PACKAGE_ROOT = Path(__file__).resolve().parents[3]
+STUDIO_DIR = PACKAGE_ROOT / "studio"
+if str(STUDIO_DIR) not in sys.path:
+ # The installers import each other by module name; spec-based loading needs studio/ reachable.
+ sys.path.insert(0, str(STUDIO_DIR))
+
+
+WINDOWS_HOST = os.name == "nt"
+IS_ROOT = hasattr(os, "geteuid") and os.geteuid() == 0
+
+# os.access(path, os.X_OK) answers "does this exist" on Windows, and os.chmod cannot clear
+# an execute bit Windows does not have, so the mode-bit rows are indistinguishable there.
+POSIX_ONLY = pytest.mark.skipif(
+ WINDOWS_HOST,
+ reason = "mode bits and os.access(X_OK) are POSIX only",
+)
+# root reads and executes regardless of the bits, so "unreadable" and "not executable"
+# stop being the corruptions those tests apply.
+NOT_ROOT = pytest.mark.skipif(
+ IS_ROOT,
+ reason = "root bypasses the permission bits this asserts on",
+)
+NEEDS_CHOWN = pytest.mark.skipif(
+ not hasattr(os, "chown"),
+ reason = "os.chown does not exist on this platform",
+)
+
+
+def load_studio_module(module_name: str, filename: str):
+ """Load one ``studio/`` installer under *module_name*.
+
+ Every install test loads these modules. ``module_name`` is the caller's to choose and
+ must be unique to the calling suite: sharing a ``sys.modules`` key with another test
+ file would mean one file's module-level monkeypatching could be observed by another
+ under ``pytest -n``.
+ """
+ spec = importlib.util.spec_from_file_location(module_name, STUDIO_DIR / filename)
+ assert spec is not None and spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[module_name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+def llama_host(host_cls, **overrides):
+ """A simulated llama.cpp host. Mirrors ``test_selection_logic.make_host``: the platform
+ booleans are derived from ``system``/``machine``, so no caller can hand out a host whose
+ flags contradict the two strings that name it. Accelerator fields default to a bare CPU
+ box, and ``os.name`` is never patched -- that changes ``pathlib`` underneath the trees.
+ """
+ system = overrides.pop("system", "Linux")
+ machine = overrides.pop("machine", "x86_64")
+ defaults = dict(
+ system = system,
+ machine = machine,
+ is_linux = system == "Linux",
+ is_windows = system == "Windows",
+ is_macos = system == "Darwin",
+ is_x86_64 = machine.lower() in {"x86_64", "amd64"},
+ is_arm64 = machine.lower() in {"arm64", "aarch64"},
+ nvidia_smi = None,
+ driver_cuda_version = None,
+ compute_caps = [],
+ visible_cuda_devices = None,
+ has_physical_nvidia = False,
+ has_usable_nvidia = False,
+ )
+ defaults.update(overrides)
+ return host_cls(**defaults)
+
+
+def whisper_host(host_cls, **overrides):
+ """A simulated whisper.cpp host. A different dataclass from the llama one: it carries
+ ``whisper_os``/``whisper_arch``/``archive_ext``/``is_apple_silicon`` and no GPU fields."""
+ fields = dict(
+ system = "Linux",
+ machine = "x64",
+ whisper_os = "linux",
+ whisper_arch = "x64",
+ archive_ext = ".tar.gz",
+ is_windows = False,
+ is_macos = False,
+ is_apple_silicon = False,
+ )
+ fields.update(overrides)
+ return host_cls(**fields)
+
+
+WHISPER_RELEASE_TAG = "v1.9.1-unsloth.1"
+
+
+def whisper_selection_fields(whisper, **overrides) -> dict:
+ """The fields of a plain CPU Linux ``InstallSelection``.
+
+ Returned as a dict rather than an instance: the suites construct it from different
+ classes (``prebuilt_core.InstallSelection`` in one, ``install_whisper_prebuilt``'s
+ re-export in another) and which instance of ``prebuilt_core`` a suite is exercising is
+ part of what that suite is saying.
+ """
+ fields = dict(
+ published_repo = whisper.DEFAULT_PUBLISHED_REPO,
+ release_tag = WHISPER_RELEASE_TAG,
+ upstream_tag = "v1.9.1",
+ source_commit = "0" * 40,
+ asset = f"whisper-{WHISPER_RELEASE_TAG}-linux-x64-cpu.tar.gz",
+ asset_sha256 = "c" * 64,
+ backend = "cpu",
+ runtime_line = None,
+ coverage = {"min_os": None},
+ studio_protocol = "inference/multipart-v1",
+ platform_os = "linux",
+ platform_arch = "x64",
+ )
+ fields.update(overrides)
+ return fields
+
+
+def whisper_install_is_intact(
+ whisper,
+ install_dir: Path,
+ host,
+ *,
+ backend: str = "cpu",
+) -> bool:
+ """The on-disk half of the whisper precheck: marker identity, the slim pairing, and
+ ``installed_tree_is_intact``. No release lookup, so no network."""
+ return (
+ whisper._existing_install_is_intact(
+ install_dir,
+ host,
+ published_repo = whisper.DEFAULT_PUBLISHED_REPO,
+ requested_backend = backend,
+ )
+ is not None
+ )
+
+
+def git(
+ *args: str,
+ text: bool = False,
+ timeout: int = 300,
+) -> subprocess.CompletedProcess:
+ """Read-only git against this checkout. Never mutates the working tree."""
+ return subprocess.run(
+ ["git", "-C", str(PACKAGE_ROOT), *args],
+ capture_output = True,
+ text = text,
+ timeout = timeout,
+ check = False,
+ )
diff --git a/tests/studio/install/test_install_llama_prebuilt_logic.py b/tests/studio/install/test_install_llama_prebuilt_logic.py
index 221acd9fd2d..0c7a1875f1c 100644
--- a/tests/studio/install/test_install_llama_prebuilt_logic.py
+++ b/tests/studio/install/test_install_llama_prebuilt_logic.py
@@ -1,12 +1,14 @@
import errno
import importlib.util
import io
+import dataclasses
import json
import os
import shutil
import stat
import subprocess
import sys
+import types
import tarfile
import urllib.error
import zipfile
@@ -2295,6 +2297,7 @@ def test_install_prebuilt_falls_back_to_older_release_plan(
llama_backend = None,
backend_request = None,
rocm_gfx = None,
+ walk_back = None,
):
call_log.append((llama_tag, initial_fallback_used))
if llama_tag == "b9002":
@@ -2475,6 +2478,33 @@ def test_existing_install_matches_plan_windows_cpu_requires_llama_dll(tmp_path:
assert existing_install_matches_plan(install_dir, host, plan) is False
+def test_existing_install_matches_plan_windows_rejects_an_empty_executable(tmp_path: Path):
+ """Windows has no executable-image preflight and an older marker carries no runtime
+ record, so a zero-byte llama-server.exe passed every other check and was reused."""
+ install_dir = tmp_path / "llama.cpp"
+ install_dir.mkdir()
+ write_windows_install_shape(install_dir, include_llama_dll = True)
+
+ host = windows_host()
+ choice = asset_choice(
+ name = "llama-b9001-bin-win-cpu-x64.zip",
+ url = "https://example.com/x.zip",
+ source_label = "published",
+ install_kind = "windows-cpu",
+ )
+ checksums = release_checksums((choice.name, choice.expected_sha256, PREBUILT))
+ plan = release_plan([choice], checksums)
+ write_metadata(install_dir, choice, checksums)
+
+ assert existing_install_matches_plan(install_dir, host, plan) is True
+ server = install_dir / "build" / "bin" / "Release" / "llama-server.exe"
+ saved = server.read_bytes()
+ server.write_bytes(b"")
+ assert existing_install_matches_plan(install_dir, host, plan) is False
+ server.write_bytes(saved)
+ assert existing_install_matches_plan(install_dir, host, plan) is True
+
+
def test_existing_install_matches_plan_windows_cuda_requires_cuda_dll(tmp_path: Path):
install_dir = tmp_path / "llama.cpp"
install_dir.mkdir()
@@ -2709,6 +2739,37 @@ def test_reused_install_backfills_the_arch_coverage(tmp_path: Path):
assert marker["llama_backend"] == "auto"
+def test_a_reused_bundle_records_whether_it_is_this_runs_fallback(tmp_path: Path):
+ """Attempt ordering moves with the recorded torch runtime preference. A reuse of a
+ later candidate after the preferred one failed is a fallback, and a marker still
+ saying False would let the no-network pre-check keep it on every update with the
+ preferred bundle never retried; a preferred bundle reused clears an old True."""
+ install_dir = tmp_path / "llama.cpp"
+ install_dir.mkdir()
+ marker_path = install_dir / "UNSLOTH_PREBUILT_INFO.json"
+ marker_path.write_text(
+ json.dumps({"release_tag": "b10360", "prebuilt_fallback_used": False}) + "\n",
+ encoding = "utf-8",
+ )
+ INSTALL_LLAMA_PREBUILT.sync_marker_selection(
+ install_dir, choice = _rocm_choice(), backend_request = None, prebuilt_fallback_used = True
+ )
+ assert json.loads(marker_path.read_text(encoding = "utf-8"))["prebuilt_fallback_used"] is True
+ # Not asked: left alone.
+ INSTALL_LLAMA_PREBUILT.sync_marker_selection(
+ install_dir, choice = _rocm_choice(), backend_request = None
+ )
+ assert json.loads(marker_path.read_text(encoding = "utf-8"))["prebuilt_fallback_used"] is True
+ INSTALL_LLAMA_PREBUILT.sync_marker_selection(
+ install_dir, choice = _rocm_choice(), backend_request = None, prebuilt_fallback_used = False
+ )
+ assert json.loads(marker_path.read_text(encoding = "utf-8"))["prebuilt_fallback_used"] is False
+ # And the kept-install pre-check refuses a marker that says True.
+ source = open(INSTALL_LLAMA_PREBUILT.__file__, encoding = "utf-8").read()
+ assert 'marker.get("prebuilt_fallback_used") is True' in source
+ assert "_record_reused_selection(plan, satisfied.choice, satisfied.used_fallback)" in source
+
+
def test_reused_install_refreshes_corrected_arch_coverage(tmp_path: Path):
"""A manifest that corrects mapped_targets for an unchanged asset must reach the
marker. Stale coverage is worse than none: too narrow forces a supported GPU to
@@ -3267,6 +3328,7 @@ def test_install_prebuilt_skips_when_older_release_fallback_matches_existing_ins
llama_backend = None,
backend_request = None,
rocm_gfx = None,
+ walk_back = None,
):
call_log.append(llama_tag)
raise PrebuiltFallback("validation failed for latest release")
@@ -3356,6 +3418,7 @@ def test_install_prebuilt_skips_same_release_fallback_attempt_when_installed(
llama_backend = None,
backend_request = None,
rocm_gfx = None,
+ walk_back = None,
):
attempted_names.append(choice.name)
if choice.name == first_choice.name:
@@ -3441,6 +3504,7 @@ def test_install_prebuilt_same_tag_upstream_failure_uses_older_unsloth_release_p
llama_backend = None,
backend_request = None,
rocm_gfx = None,
+ walk_back = None,
):
attempted.append((llama_tag, release_tag, attempts[0].source_label))
if llama_tag == "b9002":
@@ -5643,6 +5707,7 @@ _VALIDATOR_KEYWORD_ONLY = {
"llama_backend",
"backend_request",
"rocm_gfx",
+ "walk_back",
),
"validate_prebuilt_choice": (
"requested_tag",
@@ -5655,6 +5720,7 @@ _VALIDATOR_KEYWORD_ONLY = {
"llama_backend",
"backend_request",
"rocm_gfx",
+ "walk_back",
),
}
@@ -5911,3 +5977,1074 @@ def test_a_fresh_windows_install_is_payload_checked_not_just_vulkan():
'choice.install_kind.startswith("windows-")' in gate
), "fresh Windows installs are not payload checked"
assert "VULKAN_INSTALL_KINDS" in gate, "the Vulkan check must not be dropped"
+
+
+# The no-network re-check. An update of a current install used to list, fetch and re-validate by
+# STARTING llama-server (13-63 s per macOS update, ~5 s on Windows); this answers from the marker
+# plus one HEAD, so every case that cannot prove currency must take the full path.
+existing_install_current_without_plan = INSTALL_LLAMA_PREBUILT.existing_install_current_without_plan
+runtime_file_records = INSTALL_LLAMA_PREBUILT.runtime_file_records
+host_profile = INSTALL_LLAMA_PREBUILT.host_profile
+sync_marker_selection = INSTALL_LLAMA_PREBUILT.sync_marker_selection
+
+# A CUDA box as detect_host reports it.
+_CUDA_HOST_FIELDS = dict(
+ nvidia_smi = "nvidia-smi",
+ has_physical_nvidia = True,
+ has_usable_nvidia = True,
+ driver_cuda_version = (12, 8),
+ compute_caps = ["8.9"],
+)
+
+
+def _detects(monkeypatch, host: HostInfo) -> None:
+ """What the hardware probes say on THIS run, whatever the install recorded."""
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "detect_host", lambda **_k: host)
+
+
+def _current_install(
+ tmp_path: Path,
+ monkeypatch,
+ *,
+ install_host: HostInfo | None = None,
+ published_repo: str = "unslothai/llama.cpp",
+ **marker_overrides,
+) -> Path:
+ install_dir = tmp_path / "llama.cpp"
+ install_dir.mkdir(parents = True)
+ write_linux_install_shape(install_dir)
+ for name in ("llama-server", "llama-quantize"):
+ for parent in (install_dir, install_dir / "build" / "bin"):
+ (parent / name).chmod(0o755)
+ choice = asset_choice(repo = published_repo)
+ checksums = release_checksums((choice.name, choice.expected_sha256, UPSTREAM))
+ if published_repo != checksums.repo:
+ checksums = dataclasses.replace(checksums, repo = published_repo)
+ host = install_host if install_host is not None else linux_host()
+ write_prebuilt_metadata(
+ install_dir,
+ host = host,
+ requested_tag = "latest",
+ llama_tag = "b9001",
+ release_tag = "release-1",
+ choice = choice,
+ approved_checksums = checksums,
+ prebuilt_fallback_used = False,
+ backend_request = "auto",
+ )
+ if marker_overrides:
+ marker_path = install_dir / "UNSLOTH_PREBUILT_INFO.json"
+ payload = json.loads(marker_path.read_text(encoding = "utf-8"))
+ for key, value in marker_overrides.items():
+ if value is None:
+ payload.pop(key, None)
+ else:
+ payload[key] = value
+ marker_path.write_text(json.dumps(payload, indent = 2), encoding = "utf-8")
+ _detects(monkeypatch, host)
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT, "_download_host_latest_release_tag", lambda _repo: "release-1"
+ )
+ monkeypatch.delenv("UNSLOTH_PREBUILT_FULL_CHECK", raising = False)
+ monkeypatch.delenv("UNSLOTH_LLAMA_DISABLE_DOWNLOAD_HOST_RESOLVE", raising = False)
+ monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)
+ monkeypatch.delenv("UNSLOTH_ROCM_GFX_REMEMBERED", raising = False)
+ return install_dir
+
+
+def _check(install_dir: Path, **overrides) -> bool:
+ kwargs = dict(
+ llama_tag = "latest",
+ published_repo = "unslothai/llama.cpp",
+ published_release_tag = "",
+ backend_request = "auto",
+ force_cpu = False,
+ )
+ kwargs.update(overrides)
+ return existing_install_current_without_plan(install_dir, **kwargs)
+
+
+def test_a_current_install_is_recognised_without_listing_the_release(tmp_path, monkeypatch):
+ install_dir = _current_install(tmp_path, monkeypatch)
+
+ def boom(*_a, **_k):
+ raise AssertionError("the no-network check must not reach the GitHub API")
+
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "fetch_release_bundle", boom, raising = False)
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "resolve_release_tag", boom, raising = False)
+ assert _check(install_dir) is True
+
+
+def test_the_latest_lookup_is_one_head_on_the_download_host(tmp_path, monkeypatch):
+ install_dir = _current_install(tmp_path, monkeypatch)
+ calls: list[str] = []
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT,
+ "_download_host_latest_release_tag",
+ lambda repo: (calls.append(repo), "release-1")[1],
+ )
+ assert _check(install_dir) is True
+ assert calls == ["unslothai/llama.cpp"]
+
+
+def test_a_newer_release_upstream_is_not_current(tmp_path, monkeypatch):
+ install_dir = _current_install(tmp_path, monkeypatch)
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT, "_download_host_latest_release_tag", lambda _repo: "release-2"
+ )
+ assert _check(install_dir) is False
+
+
+def test_a_lookup_that_cannot_answer_takes_the_full_path(tmp_path, monkeypatch):
+ install_dir = _current_install(tmp_path, monkeypatch)
+ for answer in (None, ""):
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT, "_download_host_latest_release_tag", lambda _repo: answer
+ )
+ assert _check(install_dir) is False
+
+ def raises(_repo):
+ raise OSError("github.com unreachable")
+
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "_download_host_latest_release_tag", raises)
+ assert _check(install_dir) is False
+
+
+def _release(
+ tag: str,
+ published_at: str,
+ release_id: int = 1,
+ **extra,
+) -> dict:
+ payload = {
+ "tag_name": tag,
+ "published_at": published_at,
+ "created_at": published_at,
+ "id": release_id,
+ "draft": False,
+ "prerelease": False,
+ }
+ payload.update(extra)
+ return payload
+
+
+def test_the_api_only_escape_hatch_uses_the_api_notion_of_latest(tmp_path, monkeypatch):
+ """UNSLOTH_LLAMA_DISABLE_DOWNLOAD_HOST_RESOLVE asks for the API path.
+
+ Declining outright would make the escape hatch mean "never skip", which is not what
+ it says: it asks for the API's answer, so give it the API's answer -- ordered by
+ published_at, exactly as iter_release_payloads_by_time orders it.
+ """
+ install_dir = _current_install(tmp_path, monkeypatch)
+ monkeypatch.setenv("UNSLOTH_LLAMA_DISABLE_DOWNLOAD_HOST_RESOLVE", "1")
+
+ def boom(_repo):
+ raise AssertionError("the download-host HEAD is exactly what was turned off")
+
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "_download_host_latest_release_tag", boom)
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT,
+ "github_releases",
+ lambda _repo, **_k: [
+ _release("release-0", "2026-01-01T00:00:00Z", 1),
+ _release("release-1", "2026-02-01T00:00:00Z", 2),
+ ],
+ )
+ assert _check(install_dir) is True
+ # Newest by published_at, not by position or creation order.
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT,
+ "github_releases",
+ lambda _repo, **_k: [
+ _release("release-2", "2026-03-01T00:00:00Z", 3),
+ _release("release-1", "2026-02-01T00:00:00Z", 2),
+ ],
+ )
+ assert _check(install_dir) is False
+ # Drafts and prereleases are not the answer; an API that cannot answer is a reason to do the
+ # work.
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT,
+ "github_releases",
+ lambda _repo, **_k: [_release("release-9", "2026-04-01T00:00:00Z", 4, draft = True)],
+ )
+ assert _check(install_dir) is False
+
+ def raises(_repo, **_k):
+ raise OSError("api.github.com unreachable")
+
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "github_releases", raises)
+ assert _check(install_dir) is False
+
+
+def test_a_custom_repo_is_answered_the_way_the_selector_orders_it(tmp_path, monkeypatch):
+ """iter_resolved_published_releases takes the download-host fast path for the
+ default repo only; any other repo's releases are ordered by published_at through
+ the API. A pre-check that followed /releases/latest for such a repo could stay
+ current forever while the selector wanted the newer release GitHub never marked
+ latest."""
+ install_dir = _current_install(tmp_path, monkeypatch, published_repo = "someone/llama.cpp")
+
+ def boom(_repo):
+ raise AssertionError("a custom repo never resolves through the download host")
+
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "_download_host_latest_release_tag", boom)
+ listed = {"tags": ["release-1"]}
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT,
+ "github_releases",
+ lambda _repo, **_k: [
+ _release(tag, f"2026-0{i + 1}-01T00:00:00Z", i + 1)
+ for i, tag in enumerate(listed["tags"])
+ ],
+ )
+ assert _check(install_dir, published_repo = "someone/llama.cpp") is True
+ listed["tags"] = ["release-1", "release-2"]
+ assert _check(install_dir, published_repo = "someone/llama.cpp") is False
+
+
+def test_a_release_payload_this_run_already_has_overrides_the_latest_pointer(tmp_path, monkeypatch):
+ """/releases/latest resolves by make_latest/created_at; the macOS path orders by
+ published_at. When a payload already in hand disagrees with the pointer, the
+ published_at answer wins -- for free, because it costs no request."""
+ install_dir = _current_install(tmp_path, monkeypatch)
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT, "_download_host_latest_release_tag", lambda _repo: "release-1"
+ )
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT,
+ "_METADATA_MEMO",
+ {
+ (
+ "json",
+ "https://api.github.com/repos/unslothai/llama.cpp/releases?per_page=100&page=1",
+ ): [
+ _release("release-1", "2026-02-01T00:00:00Z", 2),
+ _release("release-2", "2026-03-01T00:00:00Z", 3),
+ ]
+ },
+ )
+ assert _check(install_dir) is False
+ # Agreeing costs nothing and changes nothing.
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT,
+ "_METADATA_MEMO",
+ {
+ (
+ "json",
+ "https://api.github.com/repos/unslothai/llama.cpp/releases?per_page=100&page=1",
+ ): [
+ _release("release-1", "2026-02-01T00:00:00Z", 2),
+ ]
+ },
+ )
+ assert _check(install_dir) is True
+
+
+def test_an_empty_metadata_memo_never_reaches_the_api(tmp_path, monkeypatch):
+ """The install path leaves _METADATA_MEMO at None; only --resolve-backends fills it,
+ so the cross-check must never turn into a request of its own."""
+ install_dir = _current_install(tmp_path, monkeypatch)
+
+ def boom(_repo, **_k):
+ raise AssertionError("the memo cross-check must not fetch anything")
+
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "github_releases", boom)
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "_METADATA_MEMO", None)
+ assert _check(install_dir) is True
+
+
+def test_a_pinned_release_tag_needs_no_lookup_at_all(tmp_path, monkeypatch):
+ install_dir = _current_install(tmp_path, monkeypatch)
+
+ def boom(_repo):
+ raise AssertionError("a pinned release tag names the answer outright")
+
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "_download_host_latest_release_tag", boom)
+ assert _check(install_dir, published_release_tag = "release-1") is True
+ assert _check(install_dir, published_release_tag = "release-9") is False
+
+
+def test_an_upstream_pin_is_answered_by_the_recorded_upstream_tag(tmp_path, monkeypatch):
+ """The marker must record the pinned build, and the release installed must be the
+ fork's newest packaging of it (the fork can republish a build); /releases/latest
+ has nothing to say about a pin."""
+ install_dir = _current_install(tmp_path, monkeypatch)
+ marker = json.loads((install_dir / "UNSLOTH_PREBUILT_INFO.json").read_text(encoding = "utf-8"))
+
+ def boom(_repo):
+ raise AssertionError("an upstream pin never asks /releases/latest")
+
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "_download_host_latest_release_tag", boom)
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT,
+ "github_releases",
+ lambda repo, **kw: [
+ {"tag_name": marker["release_tag"], "published_at": "2026-01-01T00:00:00Z", "id": 1}
+ ],
+ )
+ assert _check(install_dir, llama_tag = "b9001") is True
+ assert _check(install_dir, llama_tag = "b9999") is False
+ # A newer packaging of the same build supersedes the installed one.
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT,
+ "github_releases",
+ lambda repo, **kw: [
+ {"tag_name": marker["release_tag"], "published_at": "2026-01-01T00:00:00Z", "id": 1},
+ {"tag_name": "b9001-mix-newer", "published_at": "2026-02-01T00:00:00Z", "id": 2},
+ ],
+ )
+ assert _check(install_dir, llama_tag = "b9001") is False
+
+
+def test_a_full_check_request_always_does_the_work(tmp_path, monkeypatch):
+ install_dir = _current_install(tmp_path, monkeypatch)
+ for value in ("1", "true", "YES", "on"):
+ monkeypatch.setenv("UNSLOTH_PREBUILT_FULL_CHECK", value)
+ assert _check(install_dir) is False
+ for value in ("0", "false", "", "maybe"):
+ monkeypatch.setenv("UNSLOTH_PREBUILT_FULL_CHECK", value)
+ assert _check(install_dir) is True
+
+
+def test_a_different_repo_or_backend_request_is_not_current(tmp_path, monkeypatch):
+ install_dir = _current_install(tmp_path, monkeypatch)
+ assert _check(install_dir, published_repo = "someone/else") is False
+ # An explicit --llama-backend naming something else must reach the selector.
+ assert _check(install_dir, backend_request = "vulkan") is False
+ assert _check(install_dir, force_cpu = True) is False
+
+
+def test_a_marker_written_before_this_existed_takes_the_full_path_once(tmp_path, monkeypatch):
+ """runtime_sha256, runtime_files and host_profile are new keys; without them the
+ marker cannot be verified against itself, against the disk, or against the box it
+ is running on, and guessing yes would keep an install nobody has checked."""
+ for missing in ("runtime_sha256", "runtime_files", "install_fingerprint", "host_profile"):
+ install_dir = _current_install(tmp_path / missing, monkeypatch, **{missing: None})
+ assert _check(install_dir) is False, missing
+
+
+# The hardware half: backend_request is "auto" and the release tag does not move with the hardware,
+# so without a recorded host profile each box would keep its bundle until a new release.
+
+
+def test_a_gpu_added_since_the_install_is_not_current(tmp_path, monkeypatch):
+ install_dir = _current_install(tmp_path, monkeypatch)
+ assert _check(install_dir) is True
+ _detects(monkeypatch, linux_host(**_CUDA_HOST_FIELDS))
+ assert _check(install_dir) is False
+
+
+def test_a_gpu_removed_since_the_install_is_not_current(tmp_path, monkeypatch):
+ install_dir = _current_install(
+ tmp_path, monkeypatch, install_host = linux_host(**_CUDA_HOST_FIELDS)
+ )
+ assert _check(install_dir) is True
+ # The card is gone, or the driver is: either way the CUDA bundle cannot run.
+ _detects(monkeypatch, linux_host())
+ assert _check(install_dir) is False
+ # A driver downgrade with the card still present is the same question.
+ _detects(monkeypatch, linux_host(**{**_CUDA_HOST_FIELDS, "driver_cuda_version": (12, 4)}))
+ assert _check(install_dir) is False
+
+
+def test_a_new_gpu_outside_the_recorded_coverage_is_not_current(tmp_path, monkeypatch):
+ """A bundle carries kernels for a fixed set of SMs (supported_sms). Swapping the
+ card for one outside that set leaves an install that starts and then cannot run a
+ model, which is worse than a reinstall."""
+ install_dir = _current_install(
+ tmp_path, monkeypatch, install_host = linux_host(**_CUDA_HOST_FIELDS)
+ )
+ _detects(monkeypatch, linux_host(**{**_CUDA_HOST_FIELDS, "compute_caps": ["12.0"]}))
+ assert _check(install_dir) is False
+ # Order and duplicates across cards are not a hardware change.
+ _detects(
+ monkeypatch, linux_host(**{**_CUDA_HOST_FIELDS, "compute_caps": ["8.9", "8.9", " 8.9 "]})
+ )
+ assert _check(install_dir) is True
+
+
+def test_a_rocm_gfx_change_is_not_current(tmp_path, monkeypatch):
+ rocm = dict(has_rocm = True, rocm_gfx_target = "gfx1100", rocm_gfx_targets = ["gfx1100"])
+ install_dir = _current_install(tmp_path, monkeypatch, install_host = linux_host(**rocm))
+ assert _check(install_dir) is True
+ _detects(
+ monkeypatch,
+ linux_host(has_rocm = True, rocm_gfx_target = "gfx1030", rocm_gfx_targets = ["gfx1030"]),
+ )
+ assert _check(install_dir) is False
+ # A second AMD card the ROCm bundle has no kernels for is a change too.
+ _detects(
+ monkeypatch,
+ linux_host(
+ has_rocm = True,
+ rocm_gfx_target = "gfx1100",
+ rocm_gfx_targets = ["gfx1100", "gfx1030"],
+ ),
+ )
+ assert _check(install_dir) is False
+
+
+def test_the_recorded_profile_is_the_routed_host_not_the_raw_one(tmp_path, monkeypatch):
+ """The check re-derives the host through route_backend_request, the same no-network
+ routing _select uses. Comparing a raw detect_host against a routed profile would
+ never match and would take the slow path forever."""
+ install_dir = _current_install(tmp_path, monkeypatch)
+ marker = json.loads((install_dir / "UNSLOTH_PREBUILT_INFO.json").read_text(encoding = "utf-8"))
+ route = INSTALL_LLAMA_PREBUILT.route_backend_request(
+ backend = "auto",
+ published_repo = "unslothai/llama.cpp",
+ published_release_tag = "",
+ )
+ assert marker["host_profile"] == host_profile(route.host)
+
+
+def test_a_marker_naming_a_backend_this_platform_cannot_hold_is_not_current(tmp_path, monkeypatch):
+ """A directory copied off another machine keeps its marker. backend "metal" has no
+ Linux install kind, so _kept_install_payload_is_healthy has no shared payload to
+ require and answers yes to a tree it has never seen."""
+ install_dir = _current_install(tmp_path, monkeypatch, backend = "metal")
+ assert _check(install_dir) is False
+
+
+def test_a_marker_whose_request_and_backend_disagree_is_not_current(tmp_path, monkeypatch):
+ """persisted_marker_backend_request stores "auto" whenever the request and the
+ bundle that landed disagree, so a concrete request must name the bundle's own
+ backend. Anything else was not written by this installer."""
+ install_dir = _current_install(tmp_path, monkeypatch, backend_request = "vulkan")
+ assert _check(install_dir, backend_request = "vulkan") is False
+
+
+def test_a_truncated_shared_library_is_not_current(tmp_path, monkeypatch):
+ """The payload scan only globs for existence, so a half-written libggml-*.so keeps
+ its name and passes it. Most of a CUDA bundle's bytes live in those libraries, and a
+ full disk or an interrupted extract leaves exactly this shape."""
+ install_dir = _current_install(tmp_path, monkeypatch)
+ library = install_dir / "build" / "bin" / "libggml-cpu-x64.so.0"
+ assert library.is_file()
+ library.write_bytes(b"")
+ assert _check(install_dir) is False
+
+
+def test_the_payload_records_cover_the_bundles_own_allowlist(tmp_path, monkeypatch):
+ """Not just the three binaries: every file runtime_patterns_for_install_kind names."""
+ install_dir = tmp_path / "llama.cpp"
+ install_dir.mkdir()
+ write_linux_install_shape(install_dir)
+ patterns = INSTALL_LLAMA_PREBUILT.runtime_patterns_for_install_kind("linux-cpu")
+ records = runtime_file_records(install_dir, linux_host(), patterns)
+ assert "build/bin/libggml-base.so.0" in records
+ assert records["build/bin/libggml-base.so.0"]["size"] == 3
+ # Size and mtime only for the payload; the binaries keep their digest.
+ assert "sha256" not in records["build/bin/libggml-base.so.0"]
+ assert len(records["build/bin/llama-server"]["sha256"]) == 64
+ # A binary that stats but cannot be read leaves no usable record: the fast path fails closed.
+ real_sha256_file = INSTALL_LLAMA_PREBUILT.sha256_file
+
+ def denied(path):
+ if path.name == "llama-server":
+ raise PermissionError("held by a scanner")
+ return real_sha256_file(path)
+
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "sha256_file", denied)
+ assert runtime_file_records(install_dir, linux_host(), patterns) == {}
+
+
+def test_the_reuse_path_backfills_what_the_no_network_check_needs(tmp_path, monkeypatch):
+ """An install made before this PR carries none of the new evidence, so it would take
+ the slow path on EVERY update. The reuse path is the only place that sees the bundle
+ again without reinstalling it, so it is where the record gets caught up -- once."""
+ install_dir = _current_install(
+ tmp_path,
+ monkeypatch,
+ runtime_sha256 = None,
+ runtime_files = None,
+ host_profile = None,
+ )
+ assert _check(install_dir) is False
+ sync_marker_selection(
+ install_dir,
+ choice = asset_choice(),
+ backend_request = "auto",
+ host = linux_host(),
+ )
+ marker = json.loads((install_dir / "UNSLOTH_PREBUILT_INFO.json").read_text(encoding = "utf-8"))
+ assert "runtime_sha256" in marker
+ assert marker["host_profile"] == host_profile(linux_host())
+ assert "build/bin/libggml-base.so.0" in marker["runtime_files"]
+ assert _check(install_dir) is True
+
+
+def test_an_empty_runtime_record_is_backfilled_like_an_absent_one(tmp_path, monkeypatch):
+ """runtime_file_records answers {} when a binary could not be read; a marker that
+ persisted it would fail the no-network check closed forever unless the reuse path
+ treats it as missing and records it again once the bytes can be read."""
+ install_dir = _current_install(tmp_path, monkeypatch, runtime_files = {})
+ assert _check(install_dir) is False
+ sync_marker_selection(
+ install_dir,
+ choice = asset_choice(),
+ backend_request = "auto",
+ host = linux_host(),
+ )
+ marker = json.loads((install_dir / "UNSLOTH_PREBUILT_INFO.json").read_text(encoding = "utf-8"))
+ assert marker["runtime_files"]
+ assert _check(install_dir) is True
+
+
+def test_the_backfill_never_overwrites_evidence_a_run_already_recorded(tmp_path, monkeypatch):
+ """Added only, for the hashes: a key already present was written by a run that
+ hashed the bytes, and a reuse decision must not re-bless them."""
+ install_dir = _current_install(tmp_path, monkeypatch)
+ before = json.loads((install_dir / "UNSLOTH_PREBUILT_INFO.json").read_text(encoding = "utf-8"))
+ (install_dir / "build" / "bin" / "libggml-base.so.0").write_bytes(b"replaced after install")
+ sync_marker_selection(
+ install_dir,
+ choice = asset_choice(),
+ backend_request = "auto",
+ host = linux_host(),
+ )
+ after = json.loads((install_dir / "UNSLOTH_PREBUILT_INFO.json").read_text(encoding = "utf-8"))
+ assert after["runtime_files"] == before["runtime_files"]
+ assert after["runtime_sha256"] == before["runtime_sha256"]
+
+
+def test_the_reuse_path_refreshes_a_stale_host_profile(tmp_path, monkeypatch):
+ """The profile is this run's own probe of the box the reuse was decided on. If the
+ reuse path left it alone after a hardware change, the mismatch would send every
+ later update down the full path: only the no-network check reads the profile, and
+ nothing else would ever write it again."""
+ install_dir = _current_install(
+ tmp_path, monkeypatch, install_host = linux_host(**_CUDA_HOST_FIELDS)
+ )
+ _detects(monkeypatch, linux_host())
+ assert _check(install_dir) is False
+ sync_marker_selection(
+ install_dir,
+ choice = asset_choice(),
+ backend_request = "auto",
+ host = linux_host(),
+ )
+ marker = json.loads((install_dir / "UNSLOTH_PREBUILT_INFO.json").read_text(encoding = "utf-8"))
+ assert marker["host_profile"] == host_profile(linux_host())
+ assert _check(install_dir) is True
+
+
+def test_a_fallback_bundle_is_retried_rather_than_kept(tmp_path, monkeypatch):
+ """A bundle that landed because the preferred one could not be fetched is recorded
+ with prebuilt_fallback_used; the full path is what retries the preferred bundle."""
+ install_dir = _current_install(tmp_path, monkeypatch, prebuilt_fallback_used = True)
+ assert _check(install_dir) is False
+ install_dir = _current_install(tmp_path / "again", monkeypatch, prebuilt_fallback_used = False)
+ assert _check(install_dir) is True
+
+
+def test_a_release_pin_still_honours_an_upstream_pin(tmp_path, monkeypatch):
+ """The full path checks a pinned release against a concrete UNSLOTH_LLAMA_TAG too."""
+ install_dir = _current_install(tmp_path, monkeypatch)
+ assert _check(install_dir, published_release_tag = "release-1", llama_tag = "b9001") is True
+ assert _check(install_dir, published_release_tag = "release-1", llama_tag = "b9999") is False
+
+
+def test_a_different_cpu_architecture_is_not_current(tmp_path, monkeypatch):
+ """Every GPU field can be equal across an x86_64 and an arm64 box; the profile has
+ to carry the architecture the bundle was built for."""
+ assert host_profile(linux_host())["machine"] == "x86_64"
+ stale = dict(host_profile(linux_host()), machine = "aarch64")
+ install_dir = _current_install(tmp_path, monkeypatch, host_profile = stale)
+ assert _check(install_dir) is False
+
+
+def test_a_hand_edited_marker_is_not_trusted(tmp_path, monkeypatch):
+ """The fingerprint is recomputed from the marker's own fields, so a field changed
+ without the fingerprint no longer adds up."""
+ install_dir = _current_install(tmp_path, monkeypatch, asset = "some-other-bundle.tar.gz")
+ assert _check(install_dir) is False
+
+
+def test_a_truncated_binary_is_not_current(tmp_path, monkeypatch):
+ """The check this replaces started llama-server. Hashing the recorded binaries
+ answers the same question in ~100 ms."""
+ install_dir = _current_install(tmp_path, monkeypatch)
+ assert _check(install_dir) is True
+ (install_dir / "build" / "bin" / "llama-server").write_text("", encoding = "utf-8")
+ assert _check(install_dir) is False
+
+
+def test_a_swapped_binary_of_the_same_size_is_not_current(tmp_path, monkeypatch):
+ install_dir = _current_install(tmp_path, monkeypatch)
+ target = install_dir / "build" / "bin" / "llama-quantize"
+ original = target.read_text(encoding = "utf-8")
+ target.write_text("X" * len(original), encoding = "utf-8")
+ target.chmod(0o755)
+ assert _check(install_dir) is False
+
+
+def test_a_deleted_binary_is_not_current(tmp_path, monkeypatch):
+ install_dir = _current_install(tmp_path, monkeypatch)
+ (install_dir / "llama-server").unlink()
+ assert _check(install_dir) is False
+
+
+def test_a_gutted_runtime_payload_is_not_current(tmp_path, monkeypatch):
+ install_dir = _current_install(tmp_path, monkeypatch)
+ (install_dir / "build" / "bin" / "libggml-base.so.0").unlink()
+ assert _check(install_dir) is False
+
+
[email protected](os.name == "nt", reason = "os.chmod cannot clear an execute bit Windows lacks")
+def test_a_non_executable_binary_is_not_current(tmp_path, monkeypatch):
+ install_dir = _current_install(tmp_path, monkeypatch)
+ (install_dir / "build" / "bin" / "llama-server").chmod(0o644)
+ assert _check(install_dir) is False
+
+
+def test_no_marker_at_all_is_not_current(tmp_path, monkeypatch):
+ install_dir = tmp_path / "llama.cpp"
+ install_dir.mkdir()
+ write_linux_install_shape(install_dir)
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "detect_host", lambda **_k: linux_host())
+ assert _check(install_dir) is False
+
+
+def test_the_records_cover_both_copies_of_each_binary(tmp_path):
+ """_find_llama_server_binary reaches the root copy first and, without a symlink,
+ the two can rot independently."""
+ install_dir = tmp_path / "llama.cpp"
+ install_dir.mkdir()
+ write_linux_install_shape(install_dir)
+ records = runtime_file_records(install_dir, linux_host())
+ assert set(records) == {
+ "llama-server",
+ "llama-quantize",
+ "build/bin/llama-server",
+ "build/bin/llama-quantize",
+ }
+ for record in records.values():
+ assert record["size"] > 0 and len(record["sha256"]) == 64
+
+
+def test_the_skip_log_carries_the_string_the_setup_scripts_grep_for(tmp_path, monkeypatch, capsys):
+ """setup.sh and setup.ps1 both match the loose substring "already matches" to
+ report "prebuilt up to date and validated" rather than "installed and validated"."""
+ install_dir = _current_install(tmp_path, monkeypatch)
+ assert _check(install_dir) is True
+ captured = capsys.readouterr()
+ assert "already matches" in captured.out + captured.err
+
+
+def test_a_route_the_caller_already_made_is_not_recomputed(tmp_path, monkeypatch):
+ """Routing runs nvidia-smi and the ROCm probes. install_prebuilt routes once and
+ hands the same answer to this check and to the selector, so an update that does find
+ a new release does not probe the host twice."""
+ install_dir = _current_install(tmp_path, monkeypatch)
+ route = INSTALL_LLAMA_PREBUILT.route_backend_request(
+ backend = "auto",
+ published_repo = "unslothai/llama.cpp",
+ published_release_tag = "",
+ )
+
+ def boom(**_kwargs):
+ raise AssertionError("the caller already routed this host")
+
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "route_backend_request", boom)
+ assert _check(install_dir, route = route) is True
+
+
+def test_the_profile_sees_an_nvidia_adapter_that_is_present_but_not_usable():
+ """The Intel and ROCm-less AMD routes go to Vulkan only when there is NO NVIDIA
+ adapter at all, so an adapter that appears or goes without ever becoming usable
+ changes the selection while has_usable_nvidia and every other field stay equal."""
+ host = linux_host(has_intel_gpu = True)
+ assert host_profile(host)["has_physical_nvidia"] is False
+ with_adapter = dataclasses.replace(host, has_physical_nvidia = True)
+ assert host_profile(with_adapter)["has_physical_nvidia"] is True
+ assert host_profile(with_adapter) != host_profile(host)
+
+
+def test_the_full_path_holds_a_marker_to_the_runtime_files_it_recorded(tmp_path, monkeypatch):
+ """The Linux and macOS preflights read the executable images; Windows has no such
+ probe, so a truncated llama-server.exe used to keep its fingerprint match on the
+ full path and be reused. The recorded sizes and digests answer on every platform."""
+ install_dir = _current_install(tmp_path, monkeypatch)
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT, "preflight_linux_installed_binaries", lambda *a, **k: None
+ )
+ choice = asset_choice()
+ checksums = release_checksums((choice.name, choice.expected_sha256, UPSTREAM))
+ kwargs = dict(
+ llama_tag = "b9001", release_tag = "release-1", choice = choice, approved_checksums = checksums
+ )
+ assert existing_install_matches_choice(install_dir, linux_host(), **kwargs) is True
+ server = install_dir / "build" / "bin" / "llama-server"
+ original = server.read_bytes()
+ server.write_bytes(original[: max(1, len(original) // 2)])
+ assert existing_install_matches_choice(install_dir, linux_host(), **kwargs) is False
+ server.write_bytes(original)
+ assert existing_install_matches_choice(install_dir, linux_host(), **kwargs) is True
+ # A marker from before the record existed is held to the probes alone, as before.
+ marker_path = install_dir / "UNSLOTH_PREBUILT_INFO.json"
+ marker = json.loads(marker_path.read_text(encoding = "utf-8"))
+ marker.pop("runtime_files", None)
+ marker_path.write_text(json.dumps(marker), encoding = "utf-8")
+ server.write_bytes(original[: max(1, len(original) // 2)])
+ assert existing_install_matches_choice(install_dir, linux_host(), **kwargs) is True
+
+
+def test_the_api_latest_lookup_scans_the_pages_the_selector_scans(monkeypatch):
+ """published_at ordering: a release drafted early and published late sits on a later
+ page of a big repository, and one page would report an older marker current."""
+ seen = {}
+
+ def fake_releases(repo, max_pages = 0):
+ seen["max_pages"] = max_pages
+ return [{"tag_name": "release-2", "published_at": "2026-01-02T00:00:00Z"}]
+
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "github_releases", fake_releases)
+ assert INSTALL_LLAMA_PREBUILT._api_newest_release_tag("someone/else") == "release-2"
+ assert seen["max_pages"] == INSTALL_LLAMA_PREBUILT.DEFAULT_GITHUB_RELEASE_SCAN_MAX_PAGES
+
+
+def test_latest_on_an_older_mac_expects_the_pinned_upstream_fallback(monkeypatch):
+ """The selector rewrites "latest" for the upstream repo on a Mac below the floor to
+ the pinned fallback release; the marker check expects that release too, so a current
+ pinned install takes the fast path instead of repeating the release work on every
+ update."""
+ host = macos_host(macos_version = (15, 5))
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT,
+ "_api_newest_release_tag",
+ lambda repo: pytest.fail("listed releases"),
+ )
+ monkeypatch.setattr(
+ INSTALL_LLAMA_PREBUILT,
+ "_download_host_latest_release_tag",
+ lambda repo: pytest.fail("resolved latest"),
+ )
+ marker = {"release_tag": INSTALL_LLAMA_PREBUILT._PINNED_MACOS_FALLBACK_TAG}
+ assert (
+ INSTALL_LLAMA_PREBUILT._expected_release_tag_without_plan(
+ marker, "latest", INSTALL_LLAMA_PREBUILT.UPSTREAM_REPO, "", host = host
+ )
+ == INSTALL_LLAMA_PREBUILT._PINNED_MACOS_FALLBACK_TAG
+ )
+ # At or above the floor the pin does not apply and the repository's newest release is the answer.
+ newer = macos_host(macos_version = (26, 0))
+ monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "_api_newest_release_tag", lambda repo: "b9999")
+ assert (
+ INSTALL_LLAMA_PREBUILT._expected_release_tag_without_plan(
+ marker, "latest", INSTALL_LLAMA_PREBUILT.UPSTREAM_REPO, "", host = newer
+ )
+ == "b9999"
+ )
+
+
+def test_a_pinned_upstream_build_expects_the_newest_fork_packaging_of_it(monkeypatch):
+ """The fork can republish a build (b9596-mix-aaa, then b9596-mix-bbb) and the selector
+ installs the newest packaging; the marker check asks the same question instead of
+ reading the recorded release as current forever."""
+ M = INSTALL_LLAMA_PREBUILT
+ releases = [
+ {"tag_name": "b9600-mix-ccc", "published_at": "2026-03-01T00:00:00Z", "id": 3},
+ {"tag_name": "b9596-mix-bbb", "published_at": "2026-02-01T00:00:00Z", "id": 2},
+ {"tag_name": "b9596-mix-aaa", "published_at": "2026-01-01T00:00:00Z", "id": 1},
+ ]
+ monkeypatch.setattr(M, "github_releases", lambda repo, **kw: releases)
+ marker = {"tag": "b9596", "release_tag": "b9596-mix-aaa"}
+ expected = M._expected_release_tag_without_plan(marker, "b9596", M.DEFAULT_PUBLISHED_REPO, "")
+ assert expected == "b9596-mix-bbb"
+ assert expected != marker["release_tag"]
+
+ # A marker for another build is not current whatever the listing says.
+ assert (
+ M._expected_release_tag_without_plan(
+ {"tag": "b9500", "release_tag": "x"}, "b9596", M.DEFAULT_PUBLISHED_REPO, ""
+ )
+ is None
+ )
+ # Upstream publishes one release per build under the build's own tag: no listing.
+ monkeypatch.setattr(M, "github_releases", lambda repo, **kw: pytest.fail("listed upstream"))
+ assert (
+ M._expected_release_tag_without_plan(
+ {"tag": "b9596", "release_tag": "b9596"}, "b9596", M.UPSTREAM_REPO, ""
+ )
+ == "b9596"
+ )
+ # Any other repository fetches a release-tag-like pin as that exact release: a newer b9596-*
+ # there is not what the selector installs, and nothing is listed.
+ monkeypatch.setattr(M, "github_releases", lambda repo, **kw: pytest.fail("listed the repo"))
+ assert (
+ M._expected_release_tag_without_plan(
+ {"tag": "b9596", "release_tag": "b9596"}, "b9596", "someone/else", ""
+ )
+ == "b9596"
+ )
+ # A commit pin is found by scanning releases in published_at order; the newest packaging wins.
+ monkeypatch.setattr(
+ M,
+ "github_releases",
+ lambda repo, **kw: [
+ {"tag_name": "abc1234-mix-1", "published_at": "2026-01-01T00:00:00Z"},
+ {"tag_name": "abc1234-mix-2", "published_at": "2026-01-02T00:00:00Z"},
+ ],
+ )
+ assert (
+ M._expected_release_tag_without_plan(
+ {"tag": "abc1234", "release_tag": "abc1234-mix-1"}, "abc1234", "someone/else", ""
+ )
+ == "abc1234-mix-2"
+ )
+
+
+def test_a_moved_torch_cuda_preference_declines_the_marker_fast_path(monkeypatch):
+ """torch moving from a CUDA 12 to a CUDA 13 build leaves the GPU and driver, and so
+ the host profile, unchanged; the selector would still reorder the bundles around it."""
+ M = INSTALL_LLAMA_PREBUILT
+ from types import SimpleNamespace
+
+ host = SimpleNamespace(has_usable_nvidia = True, is_linux = True, is_windows = False)
+ monkeypatch.setattr(
+ M,
+ "detect_torch_cuda_runtime_preference",
+ lambda _host: SimpleNamespace(runtime_line = "cuda13", selection_log = []),
+ )
+ monkeypatch.setattr(M, "detected_linux_runtime_lines", lambda: (["cuda12", "cuda13"], {}))
+ monkeypatch.setattr(M, "compatible_linux_runtime_lines", lambda _h: ["cuda12", "cuda13"])
+ assert M._runtime_preference_moved({"runtime_line": "cuda12"}, host) is True
+ assert M._runtime_preference_moved({"runtime_line": "cuda13"}, host) is False
+ # The selectors may have routed away from the recorded preference (Blackwell, a release without
+ # that line), so only the preference itself moving counts.
+ assert (
+ M._runtime_preference_moved(
+ {"runtime_line": "cuda12", "torch_runtime_preference": "cuda13"}, host
+ )
+ is False
+ )
+ assert (
+ M._runtime_preference_moved(
+ {"runtime_line": "cuda13", "torch_runtime_preference": "cuda12"}, host
+ )
+ is True
+ )
+ assert (
+ M._runtime_preference_moved(
+ {"runtime_line": "cuda12", "torch_runtime_preference": None}, host
+ )
+ is True
+ )
+ # The preference vanished: movement only when the host's runtime order starts elsewhere.
+ monkeypatch.setattr(
+ M,
+ "detect_torch_cuda_runtime_preference",
+ lambda _host: SimpleNamespace(runtime_line = None, selection_log = []),
+ )
+ monkeypatch.setattr(M, "_host_is_blackwell", lambda _h: False)
+ monkeypatch.setattr(M, "detected_linux_runtime_lines", lambda: (["cuda13", "cuda12"], {}))
+ assert (
+ M._runtime_preference_moved(
+ {"runtime_line": "cuda12", "torch_runtime_preference": "cuda12"}, host
+ )
+ is True
+ )
+ assert (
+ M._runtime_preference_moved(
+ {"runtime_line": "cuda13", "torch_runtime_preference": "cuda12"}, host
+ )
+ is False
+ )
+ monkeypatch.setattr(M, "_host_is_blackwell", lambda _h: True)
+ assert (
+ M._runtime_preference_moved(
+ {"runtime_line": "cuda12", "torch_runtime_preference": "cuda12"}, host
+ )
+ is False
+ )
+ monkeypatch.setattr(M, "_host_is_blackwell", lambda _h: False)
+ monkeypatch.setattr(
+ M,
+ "detect_torch_cuda_runtime_preference",
+ lambda _host: SimpleNamespace(runtime_line = "cuda13", selection_log = []),
+ )
+ # A preference the selectors cannot act on is not movement.
+ monkeypatch.setattr(M, "detected_linux_runtime_lines", lambda: (["cuda12"], {}))
+ assert M._runtime_preference_moved({"runtime_line": "cuda12"}, host) is False
+ monkeypatch.setattr(M, "detected_linux_runtime_lines", lambda: (["cuda12", "cuda13"], {}))
+ monkeypatch.setattr(M, "compatible_linux_runtime_lines", lambda _h: ["cuda12"])
+ assert M._runtime_preference_moved({"runtime_line": "cuda12"}, host) is False
+ monkeypatch.setattr(M, "compatible_linux_runtime_lines", lambda _h: ["cuda12", "cuda13"])
+ # Windows with no runtime DLL: the selector falls back to every line the driver can run.
+ windows = SimpleNamespace(has_usable_nvidia = True, is_linux = False, is_windows = True)
+ monkeypatch.setattr(M, "detected_windows_runtime_lines", lambda: ([], {}))
+ monkeypatch.setattr(M, "compatible_windows_runtime_lines", lambda _h: ["cuda12", "cuda13"])
+ assert M._runtime_preference_moved({"runtime_line": "cuda12"}, windows) is True
+ monkeypatch.setattr(M, "compatible_windows_runtime_lines", lambda _h: ["cuda12"])
+ assert M._runtime_preference_moved({"runtime_line": "cuda12"}, windows) is False
+ # A non-CUDA install, or a preference torch cannot state, keeps the fast path.
+ assert M._runtime_preference_moved({"runtime_line": "vulkan"}, host) is False
+ monkeypatch.setattr(
+ M,
+ "detect_torch_cuda_runtime_preference",
+ lambda _host: SimpleNamespace(runtime_line = None, selection_log = []),
+ )
+ assert M._runtime_preference_moved({"runtime_line": "cuda12"}, host) is False
+ cpu_host = SimpleNamespace(has_usable_nvidia = False, is_linux = True, is_windows = False)
+ assert M._runtime_preference_moved({"runtime_line": "cuda12"}, cpu_host) is False
+
+
+def test_the_host_profile_carries_the_cuda_runtimes_on_disk(monkeypatch):
+ """The CUDA selectors order the bundles by the runtimes on disk as well as by torch's
+ preference; a runtime that appeared since the install changes the selection with the
+ GPU and driver unchanged, so it has to move the profile the marker check compares."""
+ M = INSTALL_LLAMA_PREBUILT
+ host = linux_host(compute_caps = ["8.9"], has_physical_nvidia = True, has_usable_nvidia = True)
+ monkeypatch.setattr(M, "detected_linux_runtime_lines", lambda: (["cuda12"], {}))
+ before = M.host_profile(host)
+ assert before["cuda_runtime_lines"] == ["cuda12"]
+ monkeypatch.setattr(M, "detected_linux_runtime_lines", lambda: (["cuda13", "cuda12"], {}))
+ after = M.host_profile(host)
+ assert after["cuda_runtime_lines"] == ["cuda12", "cuda13"]
+ assert before != after
+ # Off CUDA hosts no selector reads it, and the key is a constant None.
+ cpu = linux_host(has_physical_nvidia = False, has_usable_nvidia = False)
+ assert M.host_profile(cpu)["cuda_runtime_lines"] is None
+
+
+def test_the_host_profile_records_the_rocm_runtime_the_upstream_selector_reads(monkeypatch):
+ """resolve_upstream_asset_choice picks among ROCm 6.4 and 7.2 assets by the runtime's
+ major.minor; a profile without it kept the old asset current across a ROCm move."""
+ import dataclasses
+ from types import SimpleNamespace
+
+ M = INSTALL_LLAMA_PREBUILT
+ host = linux_host()
+ try:
+ rocm_host = dataclasses.replace(host, has_rocm = True)
+ except TypeError:
+ rocm_host = SimpleNamespace(
+ **{k: getattr(host, k) for k in dir(host) if not k.startswith("_")}
+ )
+ rocm_host.has_rocm = True
+ monkeypatch.setattr(M, "_detect_host_rocm_version", lambda: (6, 4))
+ assert M.host_profile(rocm_host)["rocm_runtime"] == [6, 4]
+ monkeypatch.setattr(M, "_detect_host_rocm_version", lambda: (7, 2))
+ assert M.host_profile(rocm_host)["rocm_runtime"] == [7, 2]
+ monkeypatch.setattr(M, "_detect_host_rocm_version", lambda: None)
+ assert M.host_profile(rocm_host)["rocm_runtime"] is None
+ # Never probed off ROCm hosts.
+ monkeypatch.setattr(M, "_detect_host_rocm_version", lambda: pytest.fail("probed the runtime"))
+ assert M.host_profile(host)["rocm_runtime"] is None
+
+
+def test_the_marker_fast_path_accepts_a_recorded_macos_walk_back():
+ """On a Mac below the newest bundle's OS floor the planner installs an older
+ release and records the one it skipped. The no-network re-check asks for the
+ newest release, so it must read that record rather than fail every such install
+ into the full path; a newer release than the recorded one still does."""
+ met = INSTALL_LLAMA_PREBUILT._release_expectation_met
+ marker = {"release_tag": "r1", "walked_back_from": "r2", "walked_back_on_macos": "14.7"}
+ mac = macos_host(macos_version = (14, 7))
+ assert met(marker, "r1", mac) is True
+ assert met(marker, "r2", mac) is True
+ assert met(marker, "r3", mac) is False
+ assert met(marker, None, mac) is False
+ # An OS upgrade may satisfy the skipped release's floor: decided on 14.7, not standing on 15.0.
+ assert met(marker, "r2", macos_host(macos_version = (15, 0))) is False
+ assert met(marker, "r2", macos_host(macos_version = None)) is False
+ # Only macOS walks back; anywhere else a release mismatch is a release mismatch.
+ assert met(marker, "r2", linux_host()) is False
+ assert met({"release_tag": "r1"}, "r2", mac) is False
+ # A tag without the host version (an older marker) is not trusted: the full path settles it.
+ assert met({"release_tag": "r1", "walked_back_from": "r2"}, "r2", mac) is False
+ assert (
+ met({"release_tag": "r1", "walked_back_from": "", "walked_back_on_macos": "14.7"}, "", mac)
+ is False
+ )
+
+
+def test_the_planner_records_the_newest_release_a_mac_walked_past(monkeypatch):
+ module = INSTALL_LLAMA_PREBUILT
+ bundles = [
+ module.PublishedReleaseBundle(
+ repo = "unslothai/llama.cpp", release_tag = "r2", upstream_tag = "b9002"
+ ),
+ module.PublishedReleaseBundle(
+ repo = "unslothai/llama.cpp", release_tag = "r1", upstream_tag = "b9001"
+ ),
+ ]
+ monkeypatch.setattr(
+ module,
+ "iter_resolved_published_releases",
+ lambda *args, **kwargs: [
+ module.ResolvedPublishedRelease(
+ bundle = bundle,
+ checksums = release_checksums(
+ release_tag = bundle.release_tag, upstream_tag = bundle.upstream_tag
+ ),
+ )
+ for bundle in bundles
+ ],
+ )
+
+ def choose(host, resolved_tag, bundle, checksums):
+ if bundle.release_tag == "r2":
+ raise module.PrebuiltFallback("macOS 15.0 or newer required")
+ return [types.SimpleNamespace(selection_log = [])]
+
+ monkeypatch.setattr(module, "resolve_release_asset_choice", choose)
+ host = macos_host(macos_version = (14, 7))
+ _, plans = module._fork_manifest_release_plans("latest", host, "unslothai/llama.cpp", "")
+ assert [plan.release_tag for plan in plans] == ["r1"]
+ assert plans[0].walk_back == module._core.WalkBack(release_tag = "r2", macos_version = "14.7")
+
+ # A host the newest release fits records no walk-back.
+ monkeypatch.setattr(
+ module,
+ "resolve_release_asset_choice",
+ lambda *args, **kwargs: [types.SimpleNamespace(selection_log = [])],
+ )
+ _, plans = module._fork_manifest_release_plans("latest", host, "unslothai/llama.cpp", "")
+ assert plans[0].release_tag == "r2"
+ assert plans[0].walk_back is None
+
+
+def test_a_reused_marker_takes_the_walk_back_this_run_made():
+ """sync_marker_selection: a kept install on a Mac gains the walk-back record a
+ marker written before it existed lacks (both keys), and a plan that no longer
+ walks back retires a stale one."""
+ choice = asset_choice(
+ name = "llama-b9001-bin-macos-arm64.tar.gz", tag = "r1", source_label = "published"
+ )
+ walk_back = INSTALL_LLAMA_PREBUILT._core.WalkBack(release_tag = "r2", macos_version = "14.7")
+
+ def patch(marker, walk_back):
+ return INSTALL_LLAMA_PREBUILT._marker_selection_patch(
+ marker,
+ choice = choice,
+ backend_request = None,
+ persist_force_cpu = False,
+ persist_llama_backend = None,
+ ggml_tree = None,
+ rocm_gfx = None,
+ walk_back = walk_back,
+ )
+
+ marker = {"release_tag": "r1", "runtime_sha256": None}
+ gained = patch(marker, walk_back)
+ assert gained["walked_back_from"] == "r2" and gained["walked_back_on_macos"] == "14.7"
+ recorded = {**marker, **walk_back.marker_fields()}
+ assert not set(patch(recorded, walk_back)) & {"walked_back_from", "walked_back_on_macos"}
+ keys = {"walked_back_from", "walked_back_on_macos"}
+
+ def walk_back_part(result):
+ return {key: value for key, value in result.items() if key in keys}
+
+ # A half record (tag without host version) is completed, not left alone.
+ assert walk_back_part(patch({**marker, "walked_back_from": "r2"}, walk_back)) == {
+ "walked_back_on_macos": "14.7"
+ }
+ # An OS upgrade re-decides the same walk-back on the new version.
+ upgraded = INSTALL_LLAMA_PREBUILT._core.WalkBack(release_tag = "r2", macos_version = "15.0")
+ assert walk_back_part(patch(recorded, upgraded)) == {"walked_back_on_macos": "15.0"}
+ assert not set(patch(marker, None)) & {"walked_back_from", "walked_back_on_macos"}
+ retired = patch(recorded, None)
+ assert retired["walked_back_from"] is None and retired["walked_back_on_macos"] is None
diff --git a/tests/studio/install/test_install_node_prebuilt_logic.py b/tests/studio/install/test_install_node_prebuilt_logic.py
index 08a9b8fcf27..060266f42e0 100644
--- a/tests/studio/install/test_install_node_prebuilt_logic.py
+++ b/tests/studio/install/test_install_node_prebuilt_logic.py
@@ -4,6 +4,7 @@
import importlib.util
import io
+import contextlib
import json
import os
import sys
@@ -318,6 +319,55 @@ def test_install_prebuilt_short_circuits_when_version_matches(tmp_path: Path, mo
assert rc == M.EXIT_SUCCESS
+def test_a_matching_install_beside_a_running_installer_still_exits_0(tmp_path: Path, monkeypatch):
+ """End to end, through install_prebuilt, with a REAL lock file held by a live PID.
+
+ The whole exit code is the thing under test: the record is an optimisation, and a
+ legacy install that cannot write it because another installer is running must still
+ report the install current. Exit 3 is fatal in setup.sh, so getting this wrong turns
+ "Studio was already installed" into a failed launch on the first update after upgrading.
+ """
+ host = _host("linux", "x64")
+ install_dir = tmp_path / "node"
+ install_dir.mkdir()
+ _real_node_tree(install_dir, host)
+ version = M.pinned_default_version(M.load_pins())
+ asset = M.node_asset_name(version, host)
+ # A marker written before the record existed: the one shape with something to backfill.
+ M.write_metadata(
+ install_dir,
+ version = version,
+ asset = asset,
+ sha256 = M.pinned_sha256(M.load_pins(), version, asset),
+ )
+ assert "node_version_checked" not in M.load_metadata(install_dir)
+
+ monkeypatch.setattr(M, "detect_host", lambda: host)
+ monkeypatch.setattr(M, "fetch_json", lambda url: INDEX)
+ monkeypatch.setattr(M, "installed_node_version", lambda d, h: version)
+ monkeypatch.setattr(M, "installed_npm_major", lambda d, h: 11)
+
+ def boom(*a, **k):
+ raise AssertionError("must not download when the install already matches")
+
+ monkeypatch.setattr(M, "download_file", boom)
+ monkeypatch.setattr(M, "download_bytes", boom)
+ # Held by this process, which is alive, so the stale-lock reclaim does not fire.
+ lock_path = M.install_lock_path(install_dir)
+ lock_path.parent.mkdir(parents = True, exist_ok = True)
+ lock_path.write_text(f"{os.getpid()}\n", encoding = "utf-8")
+ # The PID-file implementation, so the holder is a file this test can really write.
+ monkeypatch.setattr(M, "FileLock", None)
+ monkeypatch.setattr(M, "RECORD_LOCK_TIMEOUT_SECONDS", 0)
+ monkeypatch.setattr(M, "INSTALL_LOCK_TIMEOUT_SECONDS", 0)
+
+ assert M.install_prebuilt(install_dir, channel = "lts", min_major = 24, force = False) == (
+ M.EXIT_SUCCESS
+ )
+ # Nothing was written under a lock this run never held.
+ assert "node_version_checked" not in M.load_metadata(install_dir)
+
+
def test_existing_install_usable_is_version_agnostic(tmp_path: Path, monkeypatch):
host = _host("linux", "x64")
assert M.existing_install_usable(tmp_path, host) is False
@@ -848,3 +898,527 @@ def test_swap_into_place_survives_a_transient_lock(monkeypatch, tmp_path):
monkeypatch.setattr(M.os, "replace", flaky)
M._swap_into_place(extracted, install_dir)
assert (install_dir / "marker.txt").read_text(encoding = "utf-8") == "node"
+
+
+# The recorded runtime check: the 110 MB interpreter start it saves per run.
+def _real_node_tree(root: Path, host) -> None:
+ """The two files existing_install_matches spawns, as real bytes on disk.
+
+ The execute bit is not decoration: the recorded fast path refuses a node it could not
+ run, so a tree built without it never reaches the short circuit under test.
+ """
+ node = M.node_binary_path(root, host)
+ npm = M.npm_cli_path(root, host)
+ node.parent.mkdir(parents = True, exist_ok = True)
+ npm.parent.mkdir(parents = True, exist_ok = True)
+ node.write_bytes(b"node" * 64)
+ npm.write_bytes(b"npm" * 64)
+ node.chmod(0o755)
+
+
+def test_a_verified_install_is_not_re_probed(tmp_path: Path, monkeypatch):
+ """`node -v` is an interpreter start of a 110 MB runtime, run on every install and
+ every update to re-derive an answer that cannot have changed while the binary has
+ not."""
+ host = _host("linux", "x64")
+ _real_node_tree(tmp_path, host)
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+
+ spawns = []
+ monkeypatch.setattr(
+ M, "installed_node_version", lambda d, h: (spawns.append("node"), "24.17.0")[1]
+ )
+ monkeypatch.setattr(M, "installed_npm_major", lambda d, h: (spawns.append("npm"), 11)[1])
+ # First call has nothing recorded, so it spawns -- and writes down what it learned.
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is True
+ assert spawns == ["node", "npm"]
+
+ spawns.clear()
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is True
+ # `node -v` is saved; npm is re-probed, since the record covers the launcher, not the tree.
+ assert spawns == ["npm"], "the recorded verification was not believed"
+
+
+def _runnable_node_tree(
+ root: Path,
+ host,
+ *,
+ version = "v24.17.0",
+ npm = "11.0.0",
+) -> Path:
+ """A node that really executes, so the probes under test are real subprocess runs.
+
+ _run_node invokes the node binary for BOTH probes -- `node -v` and, through it, npm-cli.js
+ -- which is the whole reason the record is allowed to skip only the first.
+ """
+ node = M.node_binary_path(root, host)
+ npm_cli = M.npm_cli_path(root, host)
+ node.parent.mkdir(parents = True, exist_ok = True)
+ npm_cli.parent.mkdir(parents = True, exist_ok = True)
+ # Padded to a fixed length so a later rewrite can preserve the byte count exactly.
+ script = f'#!/bin/sh\ncase "$1" in -v) echo {version} ;; *) echo {npm} ;; esac\n'
+ node.write_text(script.ljust(512), encoding = "utf-8")
+ node.chmod(0o755)
+ npm_cli.write_text("// npm launcher\n", encoding = "utf-8")
+ return node
+
+
[email protected](os.name == "nt", reason = "the stand-in node is a shell script")
+def test_a_node_that_no_longer_runs_is_caught_even_at_the_recorded_size_and_mtime(
+ tmp_path: Path, monkeypatch
+):
+ """The record compares size and mtime_ns, not the digest, so a rewrite that preserves
+ both is invisible TO THE RECORD. It is not invisible to the check: npm-cli.js is run BY
+ the node binary, and that probe is never skipped, so a node that cannot run is still
+ rejected and repaired. This pins the reason the digest is affordable to skip.
+ """
+ host = _host("linux", "x64")
+ node = _runnable_node_tree(tmp_path, host)
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is True
+ recorded = M.load_metadata(tmp_path)["node_binary"]
+
+ original = node.stat()
+ node.write_bytes(b"\x00" * original.st_size)
+ os.utime(node, ns = (original.st_atime_ns, original.st_mtime_ns))
+ after = node.stat()
+ assert (after.st_size, after.st_mtime_ns) == (
+ recorded["size"],
+ recorded["mtime_ns"],
+ ), "the rewrite must be invisible to the record for this test to mean anything"
+
+ # Non-vacuity: the record itself is satisfied, so the rejection comes from the probe.
+ assert M._recorded_runtime_matches(tmp_path, host, M.load_metadata(tmp_path), "24.17.0") is True
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is False
+
+
[email protected](os.name == "nt", reason = "the stand-in node is a shell script")
+def test_a_full_check_over_an_unchanged_install_does_not_rewrite_the_marker(
+ tmp_path: Path, monkeypatch
+):
+ """UNSLOTH_PREBUILT_FULL_CHECK re-proves what the record already says, on every run.
+ Publishing an identical marker each time moves its mtime and inode and re-applies its
+ mode and owner, which is a write, a chmod and a chown for bytes that did not change."""
+ host = _host("linux", "x64")
+ _runnable_node_tree(tmp_path, host)
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is True
+
+ monkeypatch.setenv("UNSLOTH_PREBUILT_FULL_CHECK", "1")
+ marker = M.metadata_path(tmp_path)
+ before = marker.stat()
+ for _ in range(3):
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is True
+ after = marker.stat()
+ assert (after.st_ino, after.st_mtime_ns) == (
+ before.st_ino,
+ before.st_mtime_ns,
+ ), "an unchanged record was republished"
+
+
+def test_a_replaced_binary_is_probed_again(tmp_path: Path, monkeypatch):
+ """The record describes bytes, not a directory: a node swapped underneath us has
+ to answer for itself."""
+ host = _host("linux", "x64")
+ _real_node_tree(tmp_path, host)
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ M.record_runtime_verification(tmp_path, host, version = "24.17.0", npm_major = 11)
+
+ spawns = []
+ monkeypatch.setattr(
+ M, "installed_node_version", lambda d, h: (spawns.append("node"), "24.17.0")[1]
+ )
+ monkeypatch.setattr(M, "installed_npm_major", lambda d, h: (spawns.append("npm"), 11)[1])
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is True
+ assert spawns == ["npm"]
+
+ spawns.clear()
+ M.node_binary_path(tmp_path, host).write_bytes(b"a different node")
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is True
+ assert spawns == ["node", "npm"]
+
+
+def test_a_deleted_binary_is_not_a_match(tmp_path: Path, monkeypatch):
+ host = _host("linux", "x64")
+ _real_node_tree(tmp_path, host)
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ M.record_runtime_verification(tmp_path, host, version = "24.17.0", npm_major = 11)
+ M.npm_cli_path(tmp_path, host).unlink()
+ monkeypatch.setattr(M, "installed_node_version", lambda d, h: "24.17.0")
+ monkeypatch.setattr(M, "installed_npm_major", lambda d, h: None)
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is False
+
+
+def test_a_recorded_verification_never_outranks_the_version_or_the_pin(tmp_path: Path, monkeypatch):
+ """The short-circuit is only about the two spawns. Version and digest are still
+ what decide whether this is the install that was asked for."""
+ host = _host("linux", "x64")
+ _real_node_tree(tmp_path, host)
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "pinned")
+ M.record_runtime_verification(tmp_path, host, version = "24.17.0", npm_major = 11)
+ monkeypatch.setattr(M, "installed_node_version", lambda d, h: "24.17.0")
+ monkeypatch.setattr(M, "installed_npm_major", lambda d, h: 11)
+ assert M.existing_install_matches(tmp_path, host, version = "24.18.0") is False
+ assert (
+ M.existing_install_matches(tmp_path, host, version = "24.17.0", expected_sha = "other") is False
+ )
+ assert (
+ M.existing_install_matches(tmp_path, host, version = "24.17.0", expected_sha = "pinned") is True
+ )
+
+
+def test_a_recorded_npm_below_the_floor_is_probed_again(tmp_path: Path, monkeypatch):
+ host = _host("linux", "x64")
+ _real_node_tree(tmp_path, host)
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ M.record_runtime_verification(tmp_path, host, version = "24.17.0", npm_major = 10)
+ monkeypatch.setattr(M, "installed_node_version", lambda d, h: "24.17.0")
+ monkeypatch.setattr(M, "installed_npm_major", lambda d, h: 10)
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is False
+
+
+# os.access X_OK reads POSIX mode bits, which root ignores and Windows lacks.
+_EXECUTE_BIT_IS_ENFORCED = os.name != "nt" and getattr(os, "geteuid", lambda: 0)() != 0
+
+
+ not _EXECUTE_BIT_IS_ENFORCED,
+ reason = "needs an unprivileged POSIX user: root and Windows both ignore the execute bit",
+)
+def test_a_node_that_lost_its_execute_bit_is_not_a_match(tmp_path: Path, monkeypatch):
+ """Dropping the execute bit is invisible to the record: it moves ctime, and nothing else.
+
+ Size and mtime_ns both survive it, so the file record still matches a node that can no
+ longer be started. The spawn the record stands in for would have failed here and the
+ install would have been repaired, which is the outcome that has to be kept.
+ """
+ host = _host("linux", "x64")
+ _real_node_tree(tmp_path, host)
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ M.record_runtime_verification(tmp_path, host, version = "24.17.0", npm_major = 11)
+ node = M.node_binary_path(tmp_path, host)
+ before = node.stat()
+
+ node.chmod(0o644)
+ after = node.stat()
+ assert (after.st_size, after.st_mtime_ns) == (before.st_size, before.st_mtime_ns)
+ # The record itself still matches, which is exactly why the mode has to be asked for.
+ assert M._file_record_matches(node, M.load_metadata(tmp_path)["node_binary"]) is True
+ assert (
+ M._recorded_runtime_matches(tmp_path, host, M.load_metadata(tmp_path), "24.17.0") is False
+ )
+
+ monkeypatch.setattr(M, "installed_node_version", lambda d, h: None)
+ monkeypatch.setattr(M, "installed_npm_major", lambda d, h: 11)
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is False
+
+
+def test_an_execute_bit_is_not_demanded_of_the_npm_launcher(tmp_path: Path, monkeypatch):
+ """npm-cli.js is read by node, not executed, so its mode says nothing about npm."""
+ host = _host("linux", "x64")
+ _real_node_tree(tmp_path, host)
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ M.record_runtime_verification(tmp_path, host, version = "24.17.0", npm_major = 11)
+ M.npm_cli_path(tmp_path, host).chmod(0o644)
+
+ monkeypatch.setattr(M, "installed_npm_major", lambda d, h: 11)
+ assert M._recorded_runtime_matches(tmp_path, host, M.load_metadata(tmp_path), "24.17.0") is True
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is True
+
+
+def test_a_recorded_install_whose_npm_tree_was_gutted_is_not_a_match(tmp_path: Path, monkeypatch):
+ """The record covers npm-cli.js, a launcher that bootstraps ../lib/cli.js.
+
+ Deleting npm/lib/cli.js leaves the recorded launcher byte for byte identical while
+ `npm --version` fails, so nothing about the record can notice it. That is why the npm
+ probe is still paid on the recorded path, and this pins that it is.
+ """
+ host = _host("linux", "x64")
+ _real_node_tree(tmp_path, host)
+ cli_js = M.npm_cli_path(tmp_path, host).parent.parent / "lib" / "cli.js"
+ cli_js.parent.mkdir(parents = True, exist_ok = True)
+ cli_js.write_bytes(b"module.exports = () => {};\n")
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ M.record_runtime_verification(tmp_path, host, version = "24.17.0", npm_major = 11)
+ launcher_before = M.npm_cli_path(tmp_path, host).read_bytes()
+
+ # Stands in for the real spawn: node loads the launcher, which requires ../lib/cli.js.
+ monkeypatch.setattr(M, "installed_npm_major", lambda d, h: 11 if cli_js.exists() else None)
+ monkeypatch.setattr(
+ M, "installed_node_version", lambda d, h: pytest.fail("`node -v` should stay saved")
+ )
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is True
+
+ cli_js.unlink()
+ assert M.npm_cli_path(tmp_path, host).read_bytes() == launcher_before
+ # The record cannot tell the difference, so only the probe can.
+ assert M._recorded_runtime_matches(tmp_path, host, M.load_metadata(tmp_path), "24.17.0") is True
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is False
+
+
+def test_the_healthy_recorded_case_never_spawns_node_v(tmp_path: Path, monkeypatch):
+ """The saved `node -v` is the whole point of the record; losing it is a silent revert."""
+ host = _host("linux", "x64")
+ _real_node_tree(tmp_path, host)
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ M.record_runtime_verification(tmp_path, host, version = "24.17.0", npm_major = 11)
+
+ probes = []
+ monkeypatch.setattr(
+ M, "installed_node_version", lambda d, h: pytest.fail("`node -v` was spawned")
+ )
+ monkeypatch.setattr(M, "installed_npm_major", lambda d, h: (probes.append("npm"), 11)[1])
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is True
+ assert probes == ["npm"]
+
+
+def test_the_record_survives_an_unwritable_marker(tmp_path: Path):
+ """A marker that cannot be refreshed costs the two spawns again, never the install."""
+ host = _host("linux", "x64")
+ _real_node_tree(tmp_path, host)
+ M.record_runtime_verification(tmp_path, host, version = "24.17.0", npm_major = 11)
+ assert M.load_metadata(tmp_path) is None
+
+
+def test_a_failed_marker_refresh_leaves_the_old_marker_intact(tmp_path: Path, monkeypatch):
+ """The read-modify-write rewrites a marker that already describes a good install.
+
+ A truncated one reads as "no install" (load_metadata returns None on a parse
+ error), so a crash or a full disk mid-write would retire a working 110 MB runtime
+ and buy a full re-download. The replace is atomic: the marker is either the one
+ that was there or the new one, never half of either.
+ """
+ host = _host("linux", "x64")
+ _real_node_tree(tmp_path, host)
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ before = M.metadata_path(tmp_path).read_bytes()
+
+ def boom(tmp, destination):
+ raise OSError(28, "No space left on device")
+
+ monkeypatch.setattr(M, "atomic_replace_from_tempfile", boom)
+ M.record_runtime_verification(tmp_path, host, version = "24.17.0", npm_major = 11)
+ assert M.metadata_path(tmp_path).read_bytes() == before
+ assert M.load_metadata(tmp_path) is not None
+ # A stranded sibling would sit there forever, and _swap_into_place would carry it live.
+ assert list(tmp_path.glob(M.METADATA_FILENAME + ".tmp-*")) == []
+
+
+def test_the_marker_is_never_written_in_place(tmp_path: Path, monkeypatch):
+ """What makes the guarantee above true: the payload is complete on disk in a
+ sibling before the destination is touched at all."""
+ host = _host("linux", "x64")
+ _real_node_tree(tmp_path, host)
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ original = M.metadata_path(tmp_path).read_bytes()
+ seen = {}
+ real = M.atomic_replace_from_tempfile
+
+ def observe(tmp, destination):
+ seen["destination"] = Path(destination).read_bytes()
+ seen["staged"] = json.loads(Path(tmp).read_text(encoding = "utf-8"))
+ return real(tmp, destination)
+
+ monkeypatch.setattr(M, "atomic_replace_from_tempfile", observe)
+ M.record_runtime_verification(tmp_path, host, version = "24.17.0", npm_major = 11)
+ assert seen["destination"] == original, "the destination was written before the replace"
+ assert seen["staged"]["npm_major_checked"] == 11
+ assert M.load_metadata(tmp_path)["npm_major_checked"] == 11
+
+
+def test_the_marker_bytes_are_unchanged_by_the_atomic_writer(tmp_path: Path):
+ """The setup fast path and the idempotency harness both compare this file byte for
+ byte, so moving the writer must not re-spell it."""
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ payload = {
+ "schema_version": M.METADATA_SCHEMA_VERSION,
+ "kind": "node",
+ "version": "24.17.0",
+ "asset": "x",
+ "sha256": "y",
+ }
+ expected = json.dumps(payload, indent = 2) + "\n"
+ assert M.metadata_path(tmp_path).read_text(encoding = "utf-8") == expected
+
+
+def test_a_failed_install_marker_write_strands_nothing(tmp_path: Path, monkeypatch):
+ """write_metadata still raises -- it writes into a staging tree the caller discards
+ -- but it must not leave the temp file behind in it either."""
+
+ def boom(tmp, destination):
+ raise OSError(28, "No space left on device")
+
+ monkeypatch.setattr(M, "atomic_replace_from_tempfile", boom)
+ with pytest.raises(OSError):
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ assert not M.metadata_path(tmp_path).exists()
+ assert list(tmp_path.glob(M.METADATA_FILENAME + ".tmp-*")) == []
+
+
+def test_both_marker_writers_share_one_atomic_path() -> None:
+ """Two spellings of "write the marker" is how one of them would stay non-atomic."""
+ source = MODULE_PATH.read_text(encoding = "utf-8")
+ assert source.count("_write_metadata_payload(") == 3 # def + write_metadata + refresh
+ assert "metadata_path(install_dir).write_text(" not in source
+
+
+def test_the_record_is_written_after_the_swap_not_before() -> None:
+ """_ensure_npm_floor rewrites npm inside the staged tree, so a record taken there
+ describes bytes that are about to be replaced."""
+ source = MODULE_PATH.read_text(encoding = "utf-8")
+ swap = source.index("_swap_into_place(extracted_root, install_dir)")
+ record = source.index(
+ "_record_runtime_verification_under_lock(\n install_dir, host, installed_meta, version = final_version"
+ )
+ assert swap < record
+
+
[email protected](sys.platform == "win32", reason = "POSIX modes")
+def test_a_marker_refresh_keeps_the_marker_readable_to_other_users(tmp_path: Path):
+ """NamedTemporaryFile is 0600 and os.replace keeps the source file's mode, so the
+ first runtime-verification refresh used to leave a shared install's marker readable
+ only by whoever ran it, and every other user's update read "nothing installed"."""
+ import stat as _stat
+
+ host = _host("linux", "x64")
+ _real_node_tree(tmp_path, host)
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ marker = M.metadata_path(tmp_path)
+ mask = os.umask(0)
+ os.umask(mask)
+ assert _stat.S_IMODE(marker.stat().st_mode) == 0o666 & ~mask
+ marker.chmod(0o664)
+ M.record_runtime_verification(tmp_path, host, version = "24.17.0", npm_major = 11)
+ assert M.load_metadata(tmp_path)["node_version_checked"] == "24.17.0"
+ assert _stat.S_IMODE(marker.stat().st_mode) == 0o664
+
+
+def test_the_pre_lock_record_is_written_under_the_lock_and_only_over_the_marker_it_read(
+ tmp_path: Path, monkeypatch
+):
+ """The pre-lock check in install_prebuilt records the spawns it just paid for. That
+ record is a read-modify-write of the marker, so it takes the install lock for the
+ write and goes ahead only if the marker is still the one it read: a concurrent
+ installer that swapped a new tree in between must keep its own marker."""
+ host = _host("linux", "x64")
+ _real_node_tree(tmp_path, host)
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ monkeypatch.setattr(M, "installed_node_version", lambda d, h: "24.17.0")
+ monkeypatch.setattr(M, "installed_npm_major", lambda d, h: 11)
+ held = {"depth": 0, "writes_under_lock": 0}
+ real_lock = M.install_lock
+ real_record = M.record_runtime_verification
+
+ @contextlib.contextmanager
+ def counting_lock(path, **kwargs):
+ with real_lock(path, **kwargs):
+ held["depth"] += 1
+ try:
+ yield
+ finally:
+ held["depth"] -= 1
+
+ def counting_record(*args, **kwargs):
+ assert held["depth"], "the record was written outside the install lock"
+ held["writes_under_lock"] += 1
+ real_record(*args, **kwargs)
+
+ monkeypatch.setattr(M, "install_lock", counting_lock)
+ monkeypatch.setattr(M, "record_runtime_verification", counting_record)
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is True
+ assert held["writes_under_lock"] == 1
+ assert M.load_metadata(tmp_path)["node_version_checked"] == "24.17.0"
+
+ # The marker changes hands while the lock is being taken: no record over it.
+ M.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+
+ @contextlib.contextmanager
+ def swapping_lock(path, **kwargs):
+ with real_lock(path, **kwargs):
+ M.write_metadata(tmp_path, version = "24.18.0", asset = "z", sha256 = "w")
+ yield
+
+ monkeypatch.setattr(M, "install_lock", swapping_lock)
+ # ...and the tree that changed hands is not reported as the one that was asked for.
+ assert M.existing_install_matches(tmp_path, host, version = "24.17.0") is False
+ after = M.load_metadata(tmp_path)
+ assert after["version"] == "24.18.0"
+ assert "node_version_checked" not in after
+
+
[email protected](not hasattr(os, "chown"), reason = "os.chown is POSIX only")
+def test_a_refreshed_marker_keeps_its_owner_and_group(tmp_path, monkeypatch):
+ """os.replace installs the temp file's ownership; a group-shared marker refreshed by
+ another member must not take that member's group and stop being readable."""
+ install_dir = tmp_path / "node"
+ install_dir.mkdir()
+ marker = M.metadata_path(install_dir)
+ marker.write_text("{}", encoding = "utf-8")
+ original = marker.stat()
+ chowned = []
+
+ def refusing(path, uid, gid):
+ chowned.append((uid, gid))
+ if uid != -1:
+ raise PermissionError("a non-root member may not give a file away")
+
+ monkeypatch.setattr(M.os, "chown", refusing)
+ M._write_metadata_payload(install_dir, {"kind": "node"})
+ # Owner and group first, since root can restore both; group alone when that is refused,
+ # which is the non-root member of a group-shared install.
+ assert chowned == [(original.st_uid, original.st_gid), (-1, original.st_gid)]
+ chowned.clear()
+ # A marker written for the first time has no owner to preserve.
+ marker.unlink()
+ M._write_metadata_payload(install_dir, {"kind": "node"})
+ assert chowned == []
+
+
+def test_a_busy_install_lock_keeps_the_verified_install(tmp_path, monkeypatch):
+ """Busy says nothing about the tree. The evidence that another installer replaced it is
+ the marker no longer being the one that was read, and that check answers False on its own.
+
+ Answering False for a lock that was merely busy would send a legacy install -- the one
+ case that has a record to write -- on to the outer install lock, which the same holder
+ also fails. A first launch beside a running installer would then exit busy, where the
+ path before this record reported the install current and exited 0.
+ """
+ waited = {}
+
+ def busy(_path, *, timeout = None):
+ waited["timeout"] = timeout
+ raise M.BusyInstallConflict("held elsewhere")
+
+ monkeypatch.setattr(M, "install_lock", busy)
+ assert (
+ M._record_runtime_verification_under_lock(
+ tmp_path, object(), {"version": "v22.0.0"}, version = "v22.0.0", npm_major = 10
+ )
+ is True
+ )
+ # ...and it does not hold the launch for the install timeout to write an optimisation.
+ assert waited["timeout"] == M.RECORD_LOCK_TIMEOUT_SECONDS
+ assert M.RECORD_LOCK_TIMEOUT_SECONDS < M.INSTALL_LOCK_TIMEOUT_SECONDS
+
+
+def test_a_tree_replaced_under_the_lock_is_still_not_a_match(tmp_path, monkeypatch):
+ """The half that DOES answer: the lock was taken, and the marker had changed hands."""
+ install_dir = tmp_path / "node"
+ install_dir.mkdir()
+ M.metadata_path(install_dir).write_text(
+ json.dumps({"version": "v22.9.9", "sha256": "b" * 64, "asset": "other.tar.xz"}),
+ encoding = "utf-8",
+ )
+ recorded = []
+ monkeypatch.setattr(M, "record_runtime_verification", lambda *a, **k: recorded.append(a))
+ assert (
+ M._record_runtime_verification_under_lock(
+ install_dir,
+ object(),
+ {"version": "v22.0.0", "sha256": "a" * 64, "asset": "node.tar.xz"},
+ version = "v22.0.0",
+ npm_major = 10,
+ )
+ is False
+ )
+ assert recorded == [], "a record was written over another installer's tree"
diff --git a/tests/studio/install/test_install_whisper_prebuilt_logic.py b/tests/studio/install/test_install_whisper_prebuilt_logic.py
index 722827b7098..45efdede23a 100644
--- a/tests/studio/install/test_install_whisper_prebuilt_logic.py
+++ b/tests/studio/install/test_install_whisper_prebuilt_logic.py
@@ -4,7 +4,10 @@
import importlib.util
import io
+import contextlib
import json
+import stat
+import os
import sys
import tarfile
import zipfile
@@ -2176,6 +2179,20 @@ def test_macos_walks_back_to_newest_compatible_release(monkeypatch):
)
assert payload["prebuilt_available"] is True
assert payload["release_tag"] == compatible.release_tag
+ # Recorded on the plan and its selection, so the marker carries it.
+ plan = M._release_plan_for_host(
+ _host("macos", "arm64", macos_version = (14, 7)),
+ published_repo = "unslothai/whisper.cpp",
+ published_release_tag = None,
+ whisper_tag = "latest",
+ requested_backend = "cpu",
+ )
+ assert plan.bundle.release_tag == compatible.release_tag
+ walk_back = M.core.WalkBack(release_tag = latest.release_tag, macos_version = "14.7")
+ assert plan.walk_back == walk_back
+ assert plan.selection is not None
+ assert plan.selection.walk_back == walk_back
+ assert "walked_back_from" not in plan.selection.coverage
def test_macos_walkback_never_masks_checksum_failure(monkeypatch):
@@ -2232,3 +2249,884 @@ def test_resolver_reports_metal_slim_on_macos(tmp_path, monkeypatch, capsys):
assert payload["asset"] == MAC_SLIM_ASSET
assert payload["backend"] == "metal"
assert payload["requested_backend"] == "metal"
+
+
+# The no-network re-check: answers from the marker plus at most one HEAD, and still notices a llama
+# runtime that moved under a slim bundle whose own release did not.
+def _installed_cpu_tree(
+ tmp_path,
+ monkeypatch,
+ *,
+ backend = "cpu",
+):
+ host = _host("linux", "x64")
+ archive, asset, sha256 = _build_cpu_bundle(tmp_path, host)
+ calls = {"n": 0}
+
+ def fake_download(url, destination):
+ calls["n"] += 1
+ destination.parent.mkdir(parents = True, exist_ok = True)
+ destination.write_bytes(archive.read_bytes())
+
+ monkeypatch.setattr(M, "detect_host", lambda: host)
+ monkeypatch.setattr(M, "download_file", fake_download)
+ _install_env(monkeypatch, tmp_path, host, asset = asset, sha256 = sha256)
+ install_dir = tmp_path / "whisper.cpp"
+ assert M.install_prebuilt(install_dir, backend = backend) == M.EXIT_SUCCESS
+ assert calls["n"] == 1
+ monkeypatch.setattr(
+ M.llama, "_download_host_latest_release_tag", lambda _repo: RELEASE_TAG, raising = False
+ )
+ # The release list an upstream pin is answered from: one packaging of the tag.
+ monkeypatch.setattr(
+ M.llama,
+ "github_releases",
+ lambda _repo, max_pages = 1: [
+ {"tag_name": RELEASE_TAG, "published_at": "2026-01-01T00:00:00Z"}
+ ],
+ )
+ monkeypatch.delenv("UNSLOTH_PREBUILT_FULL_CHECK", raising = False)
+ return install_dir, host, calls
+
+
+def _whisper_check(install_dir, host, **overrides) -> bool:
+ kwargs = dict(
+ whisper_tag = "latest",
+ published_repo = M.DEFAULT_PUBLISHED_REPO,
+ published_release_tag = None,
+ requested_backend = "cpu",
+ )
+ kwargs.update(overrides)
+ return M.existing_install_current_without_plan(install_dir, host, **kwargs)
+
+
+def test_a_hyphenated_upstream_pin_keeps_its_suffix_when_matching_packagings(monkeypatch):
+ """v1.9.2-rc1-unsloth.2 packages v1.9.2-rc1, not v1.9.2: cutting the packaged tag at
+ its first hyphen left the newer revision out and reported the recorded one current."""
+ releases = [
+ {"tag_name": "v1.9.2-unsloth.5", "published_at": "2026-03-01T00:00:00Z"},
+ {"tag_name": "v1.9.2-rc1-unsloth.2", "published_at": "2026-02-01T00:00:00Z"},
+ {"tag_name": "v1.9.2-rc1-unsloth.1", "published_at": "2026-01-01T00:00:00Z"},
+ ]
+ monkeypatch.setattr(M.llama, "github_releases", lambda _repo, max_pages = 1: releases)
+ newest = M._api_newest_release_tag_for_upstream("r/w", "v1.9.2-rc1", "v1.9.2-rc1-unsloth.1")
+ assert newest == "v1.9.2-rc1-unsloth.2"
+ # A plain pin takes neither a longer tag starting with it nor a prerelease of it.
+ releases.append({"tag_name": "v1.9.20-unsloth.1", "published_at": "2026-04-01T00:00:00Z"})
+ releases.append({"tag_name": "v1.9.2-rc1-unsloth.9", "published_at": "2026-05-01T00:00:00Z"})
+ assert (
+ M._api_newest_release_tag_for_upstream("r/w", "v1.9.2", "v1.9.2-unsloth.5")
+ == "v1.9.2-unsloth.5"
+ )
+
+
+def test_whisper_current_install_needs_no_release_fetch(tmp_path, monkeypatch):
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+
+ def boom(*_a, **_k):
+ raise AssertionError("the no-network check must not fetch the release")
+
+ monkeypatch.setattr(M, "fetch_release_for_install", boom)
+ assert _whisper_check(install_dir, host) is True
+
+
+def test_whisper_keep_paths_reject_an_empty_server_and_a_marker_without_a_fingerprint(
+ tmp_path, monkeypatch
+):
+ """The lookup-failure keep and the marker-only check share one predicate. A zero-byte
+ server keeps its mode and its marker, and neither the POSIX execute-bit check nor
+ the Windows existence check saw it; a marker missing the fingerprint the full path
+ holds it to is not a record of a finished install."""
+ import json
+
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ assert _whisper_check(install_dir, host) is True
+ server = M.installed_server_path(install_dir, host)
+ saved = server.read_bytes()
+ server.write_bytes(b"")
+ assert _whisper_check(install_dir, host) is False
+ server.write_bytes(saved)
+ assert _whisper_check(install_dir, host) is True
+
+ markers = [
+ p
+ for p in install_dir.rglob("*.json")
+ if "install_fingerprint" in p.read_text(encoding = "utf-8")
+ ]
+ assert markers, "the install wrote no marker carrying a fingerprint"
+ marker_path = markers[0]
+ original = marker_path.read_text(encoding = "utf-8")
+ payload = json.loads(original)
+ payload.pop("install_fingerprint")
+ marker_path.write_text(json.dumps(payload), encoding = "utf-8")
+ assert _whisper_check(install_dir, host) is False
+ marker_path.write_text(original, encoding = "utf-8")
+ assert _whisper_check(install_dir, host) is True
+
+
+def test_whisper_marker_fields_edited_under_a_kept_fingerprint_take_the_full_path(
+ tmp_path, monkeypatch
+):
+ """A release_tag edited to the current tag over an old binary and its old
+ fingerprint used to read as current from the marker alone; the full path compared
+ the fingerprint against the plan's and reinstalled. The marker-only path now
+ recomputes the fingerprint from the marker's own fields."""
+ install_dir, host, calls = _installed_cpu_tree(tmp_path, monkeypatch)
+ marker_path = install_dir / M.METADATA_FILENAME
+ original = marker_path.read_text(encoding = "utf-8")
+ payload = json.loads(original)
+ assert isinstance(payload.get("fingerprint_coverage"), dict)
+ assert M.core.marker_install_fingerprint(payload) == payload["install_fingerprint"]
+ # release_tag moved to the current latest over the old asset: the tag comparison alone passes
+ # this.
+ edited = dict(payload)
+ edited["release_tag"] = "v9.9.9-unsloth.1"
+ marker_path.write_text(json.dumps(edited), encoding = "utf-8")
+ monkeypatch.setattr(
+ M.llama,
+ "_download_host_latest_release_tag",
+ lambda _repo: "v9.9.9-unsloth.1",
+ raising = False,
+ )
+ assert _whisper_check(install_dir, host) is False
+ monkeypatch.setattr(
+ M.llama, "_download_host_latest_release_tag", lambda _repo: RELEASE_TAG, raising = False
+ )
+ for field, value in (("asset_sha256", "0" * 64), ("upstream_tag", "v0.0.1")):
+ edited = dict(payload)
+ edited[field] = value
+ marker_path.write_text(json.dumps(edited), encoding = "utf-8")
+ assert _whisper_check(install_dir, host) is False, field
+ marker_path.write_text(original, encoding = "utf-8")
+ assert _whisper_check(install_dir, host) is True
+ # A marker predating fingerprint_coverage: the full path keeps the install and settles the key.
+ legacy = dict(payload)
+ legacy.pop("fingerprint_coverage")
+ marker_path.write_text(json.dumps(legacy), encoding = "utf-8")
+ # A group-shared install's marker: the settle must not narrow its mode.
+ if os.name != "nt":
+ os.chmod(marker_path, 0o664)
+ assert _whisper_check(install_dir, host) is False
+ downloads = calls["n"]
+ assert M.install_prebuilt(install_dir, backend = "cpu") == M.EXIT_SUCCESS
+ assert calls["n"] == downloads
+ settled = json.loads(marker_path.read_text(encoding = "utf-8"))
+ assert settled.get("fingerprint_coverage") == payload["fingerprint_coverage"]
+ # Temp-and-replace: nothing left beside the marker, mode kept.
+ assert not list(install_dir.glob(M.METADATA_FILENAME + ".tmp-*"))
+ if os.name != "nt":
+ assert stat.S_IMODE(marker_path.stat().st_mode) == 0o664
+ assert settled["install_fingerprint"] == payload["install_fingerprint"]
+ assert _whisper_check(install_dir, host) is True
+
+
+def test_whisper_second_install_run_downloads_and_fetches_nothing(tmp_path, monkeypatch):
+ """The end-to-end shape: a repeat install_prebuilt is now a marker read, not a
+ release fetch followed by a fingerprint comparison."""
+ install_dir, host, calls = _installed_cpu_tree(tmp_path, monkeypatch)
+ fetches = {"n": 0}
+ real_fetch = M.fetch_release_for_install
+
+ def counting(*args, **kwargs):
+ fetches["n"] += 1
+ return real_fetch(*args, **kwargs)
+
+ monkeypatch.setattr(M, "fetch_release_for_install", counting)
+ assert M.install_prebuilt(install_dir, backend = "cpu") == M.EXIT_SUCCESS
+ assert calls["n"] == 1
+ assert fetches["n"] == 0
+ # --force is still the way to make it do the work anyway.
+ assert M.install_prebuilt(install_dir, backend = "cpu", force = True) == M.EXIT_SUCCESS
+ assert fetches["n"] == 1
+
+
+def test_whisper_newer_release_or_unreachable_lookup_is_not_current(tmp_path, monkeypatch):
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ monkeypatch.setattr(
+ M.llama, "_download_host_latest_release_tag", lambda _repo: "v1.9.2-unsloth.1"
+ )
+ assert _whisper_check(install_dir, host) is False
+ monkeypatch.setattr(M.llama, "_download_host_latest_release_tag", lambda _repo: None)
+ assert _whisper_check(install_dir, host) is False
+
+ def raises(_repo):
+ raise OSError("github.com unreachable")
+
+ monkeypatch.setattr(M.llama, "_download_host_latest_release_tag", raises)
+ assert _whisper_check(install_dir, host) is False
+
+
+def test_whisper_a_pinned_release_tag_answers_without_a_lookup(tmp_path, monkeypatch):
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+
+ def boom(_repo):
+ raise AssertionError("a pinned tag names the answer outright")
+
+ monkeypatch.setattr(M.llama, "_download_host_latest_release_tag", boom)
+ assert _whisper_check(install_dir, host, published_release_tag = RELEASE_TAG) is True
+ assert _whisper_check(install_dir, host, published_release_tag = "v9.9.9") is False
+
+
+def test_whisper_an_upstream_pin_still_takes_a_newer_packaging_revision(tmp_path, monkeypatch):
+ """The fork publishes several packaging revisions of one upstream tag and the full
+ path takes the newest that matches, so an upstream pin is current only when the
+ installed release is also the newest one; a wrong pin is refused outright."""
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ listed = {"tags": [RELEASE_TAG]}
+ monkeypatch.setattr(
+ M.llama,
+ "github_releases",
+ lambda _repo, max_pages = 1: [
+ {"tag_name": tag, "published_at": f"2026-01-0{i + 1}T00:00:00Z"}
+ for i, tag in enumerate(listed["tags"])
+ ],
+ )
+
+ def no_head(_repo):
+ raise AssertionError("an upstream pin is answered from the release list, not global latest")
+
+ monkeypatch.setattr(M.llama, "_download_host_latest_release_tag", no_head)
+ assert _whisper_check(install_dir, host, whisper_tag = UPSTREAM_TAG) is True
+ assert _whisper_check(install_dir, host, whisper_tag = "v1.0.0") is False
+ # A newer packaging revision of the same upstream tag is what the full path takes.
+ listed["tags"] = [RELEASE_TAG, RELEASE_TAG[:-1] + "2"]
+ assert _whisper_check(install_dir, host, whisper_tag = UPSTREAM_TAG) is False
+ # A newer UPSTREAM version being the repository's latest changes nothing for a pin.
+ listed["tags"] = [RELEASE_TAG, "v9.9.9-unsloth.1"]
+ assert _whisper_check(install_dir, host, whisper_tag = UPSTREAM_TAG) is True
+
+
+def test_whisper_a_different_repo_or_backend_is_not_current(tmp_path, monkeypatch):
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ assert _whisper_check(install_dir, host, published_repo = "someone/else") is False
+ assert _whisper_check(install_dir, host, requested_backend = "cuda") is False
+
+
+def test_whisper_full_check_request_declines_the_fast_path(tmp_path, monkeypatch):
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ monkeypatch.setenv("UNSLOTH_PREBUILT_FULL_CHECK", "1")
+ assert _whisper_check(install_dir, host) is False
+
+
[email protected](os.name == "nt", reason = "os.chmod cannot clear an execute bit Windows lacks")
+def test_whisper_a_damaged_tree_is_not_current(tmp_path, monkeypatch):
+ """installed_tree_is_intact is the shared on-disk half, so the fast path refuses
+ exactly what the slow path would have repaired."""
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ server = M.installed_server_path(install_dir, host)
+ assert _whisper_check(install_dir, host) is True
+ server.chmod(0o644)
+ assert _whisper_check(install_dir, host) is False
+ server.chmod(0o755)
+ assert _whisper_check(install_dir, host) is True
+ server.unlink()
+ assert _whisper_check(install_dir, host) is False
+
+
+def test_whisper_no_marker_is_not_current(tmp_path, monkeypatch):
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ (install_dir / M.METADATA_FILENAME).unlink()
+ assert _whisper_check(install_dir, host) is False
+
+
+def _wire_from_llama(install_dir: Path, name: str) -> Path:
+ """Give the slim marker the wiring it claims: one library hardlinked out of a llama bin
+ dir, which is what link_ggml_runtime really leaves behind. The pairing backfill verifies
+ the wiring before recording anything, so a marker that merely SAYS "slim" is not enough."""
+ source_dir = install_dir.parent / "llama.cpp" / "build" / "bin"
+ source_dir.mkdir(parents = True, exist_ok = True)
+ source = source_dir / name
+ if not source.exists():
+ source.write_bytes(b"ggml-payload")
+ ours = M.installed_server_path(install_dir, M.detect_host()).parent / name
+ ours.unlink(missing_ok = True)
+ os.link(source, ours)
+ return source_dir
+
+
+def _slim_marker(install_dir: Path, **overrides) -> None:
+ marker_path = install_dir / M.METADATA_FILENAME
+ payload = json.loads(marker_path.read_text(encoding = "utf-8"))
+ payload["install_kind"] = "slim"
+ payload["runtime_wiring_version"] = M.SLIM_RUNTIME_WIRING_VERSION
+ payload["linked_libraries"] = ["libggml-base.so"]
+ payload["linked_from"] = str(_wire_from_llama(install_dir, "libggml-base.so"))
+ payload["paired_llama_tag"] = "b9001"
+ payload["paired_llama_ggml_tree"] = "ggml-abc"
+ for key, value in overrides.items():
+ if value is None:
+ payload.pop(key, None)
+ else:
+ payload[key] = value
+ marker_path.write_text(json.dumps(payload, indent = 2), encoding = "utf-8")
+
+
+def test_a_slim_install_whose_llama_runtime_moved_is_not_current(tmp_path, monkeypatch):
+ """The reason this check cannot just be the llama one: whisper's slim bundle
+ hardlinks llama's ggml libraries, so a llama update invalidates a whisper install
+ at an unchanged whisper release. selection_from_artifact is what normally notices,
+ and it does not run on this path."""
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ _slim_marker(install_dir)
+ monkeypatch.setattr(M, "installed_llama_ggml_tree", lambda: "ggml-abc")
+ assert _whisper_check(install_dir, host) is True
+ monkeypatch.setattr(M, "installed_llama_ggml_tree", lambda: "ggml-def")
+ assert _whisper_check(install_dir, host) is False
+ # A llama install that vanished entirely reads as no tree at all.
+ monkeypatch.setattr(M, "installed_llama_ggml_tree", lambda: None)
+ assert _whisper_check(install_dir, host) is False
+
+
+def test_a_slim_marker_written_before_the_tree_was_recorded_takes_the_full_path(
+ tmp_path, monkeypatch
+):
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ _slim_marker(install_dir, paired_llama_ggml_tree = None)
+ monkeypatch.setattr(M, "installed_llama_ggml_tree", lambda: "ggml-abc")
+ assert _whisper_check(install_dir, host) is False
+
+
+def test_a_slim_install_missing_a_wired_library_is_not_current(tmp_path, monkeypatch):
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ _slim_marker(install_dir)
+ monkeypatch.setattr(M, "installed_llama_ggml_tree", lambda: "ggml-abc")
+ assert _whisper_check(install_dir, host) is True
+ (M.installed_server_path(install_dir, host).parent / "libggml-base.so").unlink()
+ assert _whisper_check(install_dir, host) is False
+
+
+def test_installed_paired_runtime_tree_reads_the_live_llama_marker(monkeypatch):
+ monkeypatch.setattr(M, "installed_llama_ggml_tree", lambda: "ggml-abc")
+ assert M.installed_paired_runtime_tree() == "ggml-abc"
+ for empty in (None, "", 7):
+ monkeypatch.setattr(M, "installed_llama_ggml_tree", lambda: empty)
+ assert M.installed_paired_runtime_tree() is None
+
+
+def test_the_whisper_reuse_path_backfills_the_paired_ggml_tree(tmp_path, monkeypatch):
+ """A slim install made before the tree was recorded would otherwise fetch the
+ release, its manifest and its checksum index on EVERY update rather than once.
+
+ The reuse path is the only place that re-examines a slim install without
+ reinstalling it, and it gets there having just confirmed the fingerprint and the
+ wiring, so it is where the record is caught up."""
+ install_dir, host, calls = _installed_cpu_tree(tmp_path, monkeypatch)
+ _slim_marker(install_dir, paired_llama_ggml_tree = None)
+ monkeypatch.setattr(M, "installed_llama_ggml_tree", lambda: "ggml-abc")
+ assert _whisper_check(install_dir, host) is False
+ assert M.install_prebuilt(install_dir, backend = "cpu") == M.EXIT_SUCCESS
+ # Reused, not reinstalled: the archive is downloaded exactly once, at install time.
+ assert calls["n"] == 1
+ marker = json.loads((install_dir / M.METADATA_FILENAME).read_text(encoding = "utf-8"))
+ assert marker["paired_llama_ggml_tree"] == "ggml-abc"
+ assert _whisper_check(install_dir, host) is True
+
+
+def test_the_whisper_backfill_never_rewrites_a_recorded_tree(tmp_path, monkeypatch):
+ """Added, never corrected: a marker that already names a tree was written by the run
+ that installed against it, and a llama runtime that moved since is exactly what the
+ pre-check is supposed to notice."""
+ install_dir, _host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ _slim_marker(install_dir)
+ monkeypatch.setattr(M, "installed_llama_ggml_tree", lambda: "ggml-def")
+ M._backfill_slim_pairing_record(install_dir)
+ marker = json.loads((install_dir / M.METADATA_FILENAME).read_text(encoding = "utf-8"))
+ assert marker["paired_llama_ggml_tree"] == "ggml-abc"
+
+
+def test_a_fat_install_gains_no_pairing_record(tmp_path, monkeypatch):
+ """Only slim bundles hardlink anything, so a fat marker must stay a fat marker."""
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ monkeypatch.setattr(M, "installed_llama_ggml_tree", lambda: "ggml-abc")
+ M._backfill_slim_pairing_record(install_dir)
+ marker = json.loads((install_dir / M.METADATA_FILENAME).read_text(encoding = "utf-8"))
+ assert "paired_llama_ggml_tree" not in marker
+
+
+# A release lookup that could not answer, over an install that is fine. A strict offline update
+# exited 0 and changed nothing, yet setup printed "prebuilt install failed" for whisper while llama
+# printed "existing prebuilt kept". Only the EVIDENCE on disk decides.
+KEPT_LINE = "whisper.cpp update unavailable, existing prebuilt kept"
+# The substring setup.sh and setup.ps1 already grep to choose that wording for llama.cpp.
+KEPT_GREP = "keeping the existing complete install"
+FAILED_LINE = "prebuilt install failed"
+
+
+def _no_network(monkeypatch, *, release_error = None):
+ """No answer from anywhere: not the pre-check HEAD, not the release, not the listing.
+
+ The environment variables are cleared because main() reads its tag defaults from them,
+ and a pin is exactly what must NOT be answered by keeping a tree.
+ """
+
+ def no_head(_repo):
+ raise OSError("github.com unreachable")
+
+ def no_release(*_args, **_kwargs):
+ raise release_error or PrebuiltFallback(
+ "could not fetch release unslothai/whisper.cpp@latest: <urlopen error refused>"
+ )
+
+ def no_listing(_repo):
+ raise PrebuiltFallback("unexpected releases payload: <urlopen error refused>")
+
+ for name in ("UNSLOTH_WHISPER_TAG", "UNSLOTH_WHISPER_RELEASE_TAG", "UNSLOTH_WHISPER_BACKEND"):
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setattr(M.llama, "_download_host_latest_release_tag", no_head)
+ monkeypatch.setattr(M, "fetch_release_for_install", no_release)
+ monkeypatch.setattr(M, "_published_release_tags", no_listing)
+
+
+def _cli_install(capsys, install_dir, *extra) -> tuple[int, str]:
+ """Through main(), so the exit status the setup scripts branch on is what is asserted:
+ the false failure line is printed by setup.sh for any status that is not 0, 2 or 3."""
+ rc = M.main(["--install-dir", str(install_dir), "--backend", "cpu", *extra])
+ captured = capsys.readouterr()
+ return rc, captured.out + captured.err
+
+
+def test_an_unreachable_lookup_keeps_a_validated_install(tmp_path, monkeypatch, capsys):
+ install_dir, host, calls = _installed_cpu_tree(tmp_path, monkeypatch)
+ marker_path = install_dir / M.METADATA_FILENAME
+ server = M.installed_server_path(install_dir, host)
+ before = (marker_path.read_bytes(), server.read_bytes(), server.stat().st_mtime_ns)
+ _no_network(monkeypatch)
+
+ rc, output = _cli_install(capsys, install_dir)
+
+ assert rc == M.EXIT_SUCCESS
+ assert KEPT_LINE in output
+ assert KEPT_GREP in output
+ # llama.cpp's wording for WHY, so update_flow reads both installers the same way.
+ assert "prebuilt update reason: could not fetch release" in output
+ assert FAILED_LINE not in output
+ # Kept, not reinstalled: one download, and the marker byte for byte as written.
+ assert calls["n"] == 1
+ assert (marker_path.read_bytes(), server.read_bytes(), server.stat().st_mtime_ns) == before
+
+
+def test_whisper_an_asset_for_another_architecture_is_not_current(tmp_path, monkeypatch):
+ """The marker's asset name carries the platform tokens; a bundle for another
+ architecture is not intact on this host whatever the tree looks like."""
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ marker_path = install_dir / M.METADATA_FILENAME
+ marker = json.loads(marker_path.read_text(encoding = "utf-8"))
+ assert "-linux-x64-" in marker["asset"]
+ assert _whisper_check(install_dir, host) is True
+ marker["asset"] = marker["asset"].replace("-linux-x64-", "-linux-arm64-")
+ marker_path.write_text(json.dumps(marker, indent = 2), encoding = "utf-8")
+ assert _whisper_check(install_dir, host) is False
+
+
+def test_a_rate_limited_release_fetch_still_keeps_the_install(tmp_path, monkeypatch, capsys):
+ """fetch_json turns an HTTP 403/429 from api.github.com into RuntimeError, not
+ URLError; the keep path has to read that shape too."""
+ install_dir, host, calls = _installed_cpu_tree(tmp_path, monkeypatch)
+ _no_network(monkeypatch)
+
+ def limited(*_args, **_kwargs):
+ raise RuntimeError("GitHub API returned 403 for https://api.github.com/repos/x/releases")
+
+ monkeypatch.setattr(M, "fetch_release_for_install", M.fetch_release_for_install)
+ monkeypatch.setattr(M.core, "fetch_release_for_install", limited)
+
+ rc, output = _cli_install(capsys, install_dir)
+
+ assert rc == M.EXIT_SUCCESS
+ assert KEPT_LINE in output
+ assert "prebuilt update reason: could not fetch release" in output
+ assert "unexpected error" not in output
+ assert FAILED_LINE not in output
+ assert calls["n"] == 1
+
+
+def test_a_raw_network_error_from_the_release_fetch_still_keeps_the_install(
+ tmp_path, monkeypatch, capsys
+):
+ """The real fetch raises URLError, not PrebuiltFallback: a refused connection or a
+ proxy answering 403 reached install_prebuilt as "unexpected error" and the keep path
+ never ran. Patched at the shared core seam, so the wrapper in this module is what is
+ under test rather than a stand-in that already speaks PrebuiltFallback."""
+ import urllib.error
+
+ install_dir, host, calls = _installed_cpu_tree(tmp_path, monkeypatch)
+ marker_path = install_dir / M.METADATA_FILENAME
+ before = marker_path.read_bytes()
+ _no_network(monkeypatch)
+
+ def refused(*_args, **_kwargs):
+ raise urllib.error.URLError(ConnectionRefusedError(111, "Connection refused"))
+
+ monkeypatch.setattr(M, "fetch_release_for_install", M.fetch_release_for_install)
+ monkeypatch.setattr(M.core, "fetch_release_for_install", refused)
+
+ rc, output = _cli_install(capsys, install_dir)
+
+ assert rc == M.EXIT_SUCCESS
+ assert KEPT_LINE in output
+ assert "prebuilt update reason: could not fetch release" in output
+ assert "unexpected error" not in output
+ assert FAILED_LINE not in output
+ assert calls["n"] == 1
+ assert marker_path.read_bytes() == before
+
+
+ sys.platform == "win32",
+ reason = "os.access(X_OK) is always true on Windows, so the unusable-tree half is POSIX only",
+)
+def test_an_unreachable_lookup_still_fails_on_a_broken_install(tmp_path, monkeypatch, capsys):
+ """Fail-open is the right answer for a tree that cannot serve dictation, and the
+ message that says so sends the user to browser and Transformers STT. Only the
+ evidence a previous run left behind may replace it."""
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ marker_path = install_dir / M.METADATA_FILENAME
+ recorded = marker_path.read_bytes()
+ server = M.installed_server_path(install_dir, host)
+ _no_network(monkeypatch)
+
+ # (1) No marker: nothing on disk says this tree was ever installed and validated here.
+ marker_path.unlink()
+ rc, output = _cli_install(capsys, install_dir)
+ assert rc == M.EXIT_ERROR
+ assert FAILED_LINE in output
+ assert KEPT_LINE not in output
+
+ # (2) Marker back, but the server is not executable, the shape the sidecar refuses.
+ marker_path.write_bytes(recorded)
+ server.chmod(0o644)
+ rc, output = _cli_install(capsys, install_dir)
+ assert rc == M.EXIT_ERROR
+ assert FAILED_LINE in output
+ assert KEPT_LINE not in output
+
+ # (3) Both repaired, same unreachable lookup: now the tree answers for itself.
+ server.chmod(0o755)
+ rc, output = _cli_install(capsys, install_dir)
+ assert rc == M.EXIT_SUCCESS
+ assert KEPT_LINE in output
+
+
+def test_the_kept_path_refuses_a_mismatched_ggml_pairing(tmp_path, monkeypatch, capsys):
+ """A slim whisper tree hardlinks llama's ggml libraries, so a llama runtime that moved
+ underneath it is broken with every whisper byte still in place. The keep path has no
+ release in hand, so the paired tree this branch records on the whisper marker is the
+ only thing that can notice; a weaker "the wired files are present" check would keep a
+ pairing that no longer exists."""
+ install_dir, _host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ _slim_marker(install_dir)
+ _no_network(monkeypatch)
+
+ monkeypatch.setattr(M, "installed_llama_ggml_tree", lambda: "ggml-abc")
+ rc, output = _cli_install(capsys, install_dir)
+ assert rc == M.EXIT_SUCCESS
+ assert KEPT_LINE in output
+
+ monkeypatch.setattr(M, "installed_llama_ggml_tree", lambda: "ggml-def")
+ rc, output = _cli_install(capsys, install_dir)
+ assert rc == M.EXIT_ERROR
+ assert FAILED_LINE in output
+ assert KEPT_LINE not in output
+
+ # A slim marker written before the pairing was recorded cannot claim one either.
+ _slim_marker(install_dir, paired_llama_ggml_tree = None)
+ monkeypatch.setattr(M, "installed_llama_ggml_tree", lambda: "ggml-abc")
+ rc, output = _cli_install(capsys, install_dir)
+ assert rc == M.EXIT_ERROR
+ assert KEPT_LINE not in output
+
+
+def test_a_pairing_gap_still_reports_itself(tmp_path, monkeypatch, capsys):
+ """The one lookup failure that ANSWERED. Exit 2 is how setup learns to print the
+ installed-versus-required tags and tell the user to publish the paired releases; a
+ keep would replace a release skew nobody would then fix with a soothing warning."""
+ install_dir, _host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ _no_network(monkeypatch)
+ rc, output = _cli_install(capsys, install_dir)
+ assert rc == M.EXIT_SUCCESS
+ assert KEPT_LINE in output
+
+ _no_network(
+ monkeypatch,
+ release_error = ReleaseCompatibilityError(
+ "slim bundle requires llama.cpp b9002, installed b9001"
+ ),
+ )
+ rc, output = _cli_install(capsys, install_dir)
+ assert rc == M.EXIT_INCOMPATIBLE
+ assert "slim bundle requires llama.cpp b9002" in output
+ assert KEPT_LINE not in output
+
+
+def test_an_explicit_release_request_is_never_answered_by_keeping(tmp_path, monkeypatch, capsys):
+ """Keeping the tree silently ignores whatever this run asked for, so anything that
+ names a release -- or demands the work be done anyway -- takes the failure instead.
+ --has-rocm and --rocm-gfx are deliberately not on that list: both entrypoints forward
+ DETECTED hardware on every AMD host, and the observed failure was on one."""
+ install_dir, _host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ _no_network(monkeypatch)
+ rc, output = _cli_install(capsys, install_dir, "--rocm-gfx", "gfx1151")
+ assert rc == M.EXIT_SUCCESS
+ assert KEPT_LINE in output
+
+ for extra in (
+ ["--published-release-tag", "v9.9.9"],
+ ["--whisper-tag", "v1.9.9"],
+ ["--force"],
+ ):
+ rc, output = _cli_install(capsys, install_dir, *extra)
+ assert rc == M.EXIT_ERROR, extra
+ assert KEPT_LINE not in output, extra
+
+
+def test_both_setup_scripts_report_the_kept_install_as_kept():
+ """The installer's exit status is 0 on the kept path, and a non-verbose run discards
+ its log, so without an arm of their own the shells print "prebuilt installed" for a
+ release nothing fetched. Wording and grep token are the llama arm's, verbatim, so a
+ strictly offline update reads the same for both components and one step matcher
+ covers them."""
+ kept = "update unavailable, existing prebuilt kept"
+ token = "keeping the existing complete install"
+ sh = (PACKAGE_ROOT / "studio" / "setup.sh").read_text(encoding = "utf-8")
+ ps1 = (PACKAGE_ROOT / "studio" / "setup.ps1").read_text(encoding = "utf-8")
+ # Two arms each now: llama's and whisper's.
+ assert sh.count(f'step "llama.cpp" "{kept}"') == 1
+ assert sh.count(f'step "whisper.cpp" "{kept}"') == 1
+ assert sh.count(f'grep -Fq "{token}" "$_WHISPER_LOG"') == 1
+ assert ps1.count(f'step "whisper.cpp" "{kept}"') == 1
+ assert ps1.count(f'$whisperOutput -match "{token}"') == 1
+ # ...and the installer really emits the token the shells select on.
+ source = MODULE_PATH.read_text(encoding = "utf-8")
+ assert token in source
+ assert kept in source
+
+
+def test_whisper_a_release_pin_does_not_excuse_a_wrong_upstream_pin(tmp_path, monkeypatch):
+ """The full path refuses a pinned release whose bundle targets another upstream
+ version, so the shortcut checks the upstream pin whether or not the release is
+ pinned too."""
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ monkeypatch.setattr(
+ M.llama,
+ "_download_host_latest_release_tag",
+ lambda _repo: pytest.fail("a pinned tag names the answer outright"),
+ )
+ assert (
+ _whisper_check(
+ install_dir, host, published_release_tag = RELEASE_TAG, whisper_tag = UPSTREAM_TAG
+ )
+ is True
+ )
+ assert (
+ _whisper_check(install_dir, host, published_release_tag = RELEASE_TAG, whisper_tag = "v1.0.0")
+ is False
+ )
+
+
+def test_an_unreachable_lookup_keeps_a_cpu_install_asked_for_with_cpu_fallback(
+ tmp_path, monkeypatch, capsys
+):
+ """--cpu-fallback is a backend request (resolve_backend makes it "cpu"), and the
+ intact check compares the marker's backend against it, so a kept CPU tree honours
+ the flag exactly as --backend cpu does; treating it as an explicit release request
+ turned a recoverable offline update into a failure."""
+ install_dir, _host, calls = _installed_cpu_tree(tmp_path, monkeypatch)
+ _no_network(monkeypatch)
+ rc, output = _cli_install(capsys, install_dir, "--cpu-fallback")
+ assert rc == M.EXIT_SUCCESS
+ assert KEPT_GREP in output
+ assert calls["n"] == 1
+
+
+def test_the_whisper_backfill_is_written_under_the_install_lock(tmp_path, monkeypatch):
+ """The backfill is a read-modify-write of the marker. Outside the lock it raced a
+ concurrent installer swapping in a new release: old marker read, tree replaced, old
+ fields written over the new marker. The pre-lock keep now takes the lock, re-checks
+ the install and only then writes."""
+ install_dir, host, calls = _installed_cpu_tree(tmp_path, monkeypatch)
+ _slim_marker(install_dir, paired_llama_ggml_tree = None)
+ monkeypatch.setattr(M, "installed_llama_ggml_tree", lambda: "ggml-abc")
+ held = {"depth": 0, "writes_under_lock": 0, "writes_outside": 0}
+ real_lock = M.install_lock
+
+ @contextlib.contextmanager
+ def counting_lock(path, **kwargs):
+ with real_lock(path, **kwargs):
+ held["depth"] += 1
+ try:
+ yield
+ finally:
+ held["depth"] -= 1
+
+ real_backfill = M._backfill_slim_pairing_record
+
+ def counting_backfill(directory):
+ if held["depth"]:
+ held["writes_under_lock"] += 1
+ else:
+ held["writes_outside"] += 1
+ real_backfill(directory)
+
+ monkeypatch.setattr(M, "install_lock", counting_lock)
+ monkeypatch.setattr(M, "_backfill_slim_pairing_record", counting_backfill)
+ assert M.install_prebuilt(install_dir, backend = "cpu") == M.EXIT_SUCCESS
+ assert calls["n"] == 1
+ assert held == {"depth": 0, "writes_under_lock": 1, "writes_outside": 0}
+ marker = json.loads((install_dir / M.METADATA_FILENAME).read_text(encoding = "utf-8"))
+ assert marker["paired_llama_ggml_tree"] == "ggml-abc"
+ # Nothing left to settle: the second keep takes no lock for a write.
+ held["writes_under_lock"] = 0
+ assert M.install_prebuilt(install_dir, backend = "cpu") == M.EXIT_SUCCESS
+ assert held["writes_under_lock"] == 0 and held["writes_outside"] == 0
+
+
+def test_whisper_an_upstream_pin_without_the_v_prefix_is_the_same_pin(tmp_path, monkeypatch):
+ """The full selector treats v1.9.2 and 1.9.2 as one tag (_normalized_upstream_tag);
+ a raw comparison sent every such pin down the release listing on every update."""
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ assert _whisper_check(install_dir, host, whisper_tag = UPSTREAM_TAG) is True
+ assert _whisper_check(install_dir, host, whisper_tag = UPSTREAM_TAG.lstrip("v")) is True
+
+
+def test_whisper_a_bundle_needing_a_newer_macos_is_not_intact(tmp_path, monkeypatch):
+ """An install restored onto an older same-architecture Mac has the right platform
+ tokens and the execute bit; the marker's recorded min_os is what the selector
+ would refuse it on, so the intact check refuses it too."""
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ marker_path = install_dir / M.METADATA_FILENAME
+ marker = json.loads(marker_path.read_text(encoding = "utf-8"))
+ # Where write_prebuilt_metadata puts it: the marker's top level, not under coverage.
+ marker.pop("coverage", None)
+ marker["min_os"] = "15.0"
+ marker_path.write_text(json.dumps(marker), encoding = "utf-8")
+ calls = []
+
+ def min_os_ok(_host, min_os):
+ calls.append(min_os)
+ return False
+
+ monkeypatch.setattr(M, "_macos_min_os_ok", min_os_ok)
+ # A Linux host never asks.
+ assert _whisper_check(install_dir, host) is True
+ assert calls == []
+ mac = M.HostInfo(**{**host.__dict__, "system": "Darwin"}) if hasattr(host, "__dict__") else host
+ monkeypatch.setattr(M, "host_platform_tokens", lambda _h: ("linux", "x64"))
+ monkeypatch.setattr(type(mac), "is_macos", property(lambda self: True), raising = False)
+ assert _whisper_check(install_dir, mac) is False
+ assert calls == ["15.0"]
+
+
+def test_a_failing_compatibility_listing_keeps_an_intact_install(tmp_path, monkeypatch, capsys):
+ """On macOS the newest release can be fetched and still not pair with this host, and
+ the listing that follows is a second API seam; a 403 there is a lookup that could
+ not answer, not a failed update."""
+ install_dir, host, calls = _installed_cpu_tree(tmp_path, monkeypatch)
+ _no_network(monkeypatch)
+
+ def listing_denied(_repo):
+ raise RuntimeError("HTTP 403: rate limit")
+
+ monkeypatch.setattr(M, "_published_release_tags", listing_denied)
+ rc, output = _cli_install(capsys, install_dir)
+ assert rc == M.EXIT_SUCCESS
+ assert KEPT_GREP in output
+ assert calls["n"] == 1
+
+
+def test_force_compile_lets_a_lookup_failure_reach_the_source_build(tmp_path, monkeypatch, capsys):
+ """setup.sh runs the opt-in source build only after this installer exits nonzero; a
+ lookup failure kept as success under UNSLOTH_WHISPER_FORCE_COMPILE=1 would take that
+ path away, so the switch counts as an explicit request."""
+ install_dir, host, _calls = _installed_cpu_tree(tmp_path, monkeypatch)
+ _no_network(monkeypatch)
+
+ def listing_denied(_repo):
+ raise RuntimeError("HTTP 403: rate limit")
+
+ monkeypatch.setattr(M, "_published_release_tags", listing_denied)
+ monkeypatch.setenv("UNSLOTH_WHISPER_FORCE_COMPILE", "1")
+ rc, output = _cli_install(capsys, install_dir)
+ assert rc != M.EXIT_SUCCESS
+ assert KEPT_GREP not in output
+
+
+def test_whisper_fast_path_accepts_a_recorded_macos_walk_back(tmp_path, monkeypatch):
+ """A Mac below the newest release's OS floor installs an older release; the
+ marker-only re-check asks the download host for the newest and must recognise the
+ recorded walk-back rather than send every such install down the full path. A
+ release newer than the recorded one, a macOS upgrade since the walk-back (the
+ newest release may fit now), or any other OS still does."""
+ install_dir, _, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ marker = {
+ "release_tag": "old",
+ "walked_back_from": "new",
+ "walked_back_on_macos": "14.7",
+ "backend": "cpu",
+ }
+ monkeypatch.setattr(M, "_existing_install_is_intact", lambda *a, **k: dict(marker))
+ monkeypatch.setattr(M.llama, "_download_host_resolve_enabled", lambda: True, raising = False)
+ monkeypatch.setattr(
+ M.llama, "_download_host_latest_release_tag", lambda _repo: "new", raising = False
+ )
+ mac = _host("macos", "arm64", macos_version = (14, 7))
+ assert _whisper_check(install_dir, mac) is True
+ assert _whisper_check(install_dir, _host("macos", "arm64", macos_version = (15, 0))) is False
+ assert _whisper_check(install_dir, _host("linux", "x64")) is False
+ monkeypatch.setattr(
+ M.llama, "_download_host_latest_release_tag", lambda _repo: "newer", raising = False
+ )
+ assert _whisper_check(install_dir, mac) is False
+ monkeypatch.setattr(
+ M.llama, "_download_host_latest_release_tag", lambda _repo: "old", raising = False
+ )
+ assert _whisper_check(install_dir, mac) is True
+ # A marker predating the host version record takes the full path once, which settles it.
+ del marker["walked_back_on_macos"]
+ monkeypatch.setattr(
+ M.llama, "_download_host_latest_release_tag", lambda _repo: "new", raising = False
+ )
+ assert _whisper_check(install_dir, mac) is False
+
+
+def _rewrite_marker(install_dir: Path, **overrides) -> dict:
+ marker_path = install_dir / M.METADATA_FILENAME
+ payload = json.loads(marker_path.read_text(encoding = "utf-8"))
+ for key, value in overrides.items():
+ if value is None:
+ payload.pop(key, None)
+ else:
+ payload[key] = value
+ # Self-consistent: the fast path recomputes the fingerprint from the recorded fields.
+ payload["install_fingerprint"] = M.core.marker_install_fingerprint(payload)
+ marker_path.write_text(json.dumps(payload, indent = 2), encoding = "utf-8")
+ return payload
+
+
+def test_the_marker_records_the_platform_it_was_selected_for(tmp_path, monkeypatch):
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ marker = json.loads((install_dir / M.METADATA_FILENAME).read_text(encoding = "utf-8"))
+ assert (marker["os"], marker["arch"]) == M.host_platform_tokens(host)
+
+
+def test_a_custom_repository_asset_name_is_judged_by_the_recorded_platform(tmp_path, monkeypatch):
+ """A custom --published-repo's manifest may name its assets freely; the platform
+ check reads the os/arch the marker records and only falls back to the fork's
+ asset naming for a marker written before they were recorded. Otherwise every such
+ install took the full path on every update, and the keep-existing path (which runs
+ when the newest release cannot be looked up) dropped a valid install."""
+ install_dir, host, _ = _installed_cpu_tree(tmp_path, monkeypatch)
+ os_token, arch_token = M.host_platform_tokens(host)
+ _rewrite_marker(install_dir, asset = "server-bundle.tar.gz")
+ assert _whisper_check(install_dir, host) is True
+ # ...and the recorded platform is what is checked: another one is not intact here.
+ _rewrite_marker(
+ install_dir, asset = "server-bundle.tar.gz", arch = "arm64" if arch_token != "arm64" else "x64"
+ )
+ assert _whisper_check(install_dir, host) is False
+ # No recorded platform: the fork's naming is the only evidence, and this has none.
+ _rewrite_marker(install_dir, asset = "server-bundle.tar.gz", os = None, arch = None)
+ assert _whisper_check(install_dir, host) is False
+ _rewrite_marker(
+ install_dir, asset = f"whisper-v1.9.1-{os_token}-{arch_token}-cpu.tar.gz", os = None, arch = None
+ )
+ assert _whisper_check(install_dir, host) is True
diff --git a/tests/studio/install/test_pr10648_host_matrix.py b/tests/studio/install/test_pr10648_host_matrix.py
new file mode 100644
index 00000000000..acb7b293b3c
--- /dev/null
+++ b/tests/studio/install/test_pr10648_host_matrix.py
@@ -0,0 +1,916 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The host half of the marker fast path, across every OS x accelerator cell (#10648).
+
+``existing_install_current_without_plan`` answers "the install on disk is already what
+this run would produce" without listing a release. ``backend_request`` is ``"auto"`` on
+every automatic install and the release tag does not move when the hardware does, so the
+only thing in that answer that can notice a hardware change is ``host_profile``: recorded
+into the marker at install time and compared for whole-dict equality on every later run.
+
+Without that comparison a box that gains a GPU keeps its CPU bundle, a box whose card or
+driver is gone keeps a CUDA bundle that cannot load, and a box that changes vendor keeps
+an unusable backend -- in each case until the fork happens to publish a new release. Each
+of those is a named test below, per platform, one axis flipped at a time.
+
+Two halves, and they matter equally. The rejection half is above; the acceptance half is
+that every reachable cell must answer True for the UNCHANGED host, or the "rejections"
+below would be proving nothing but a broken fixture. The markers here are therefore
+written by the real ``write_prebuilt_metadata`` against the routed host, so their
+``install_fingerprint`` reproduces; a hand-written marker fails the fingerprint guard and
+would make every case pass for the wrong reason.
+
+Platforms are simulated by constructing ``HostInfo`` (``os.name`` is never patched -- it
+changes ``pathlib``), and the three probes ``host_profile`` reads that a ``HostInfo`` does
+not carry (the CUDA runtime scan, the ROCm runtime version, the torch CUDA preference) are
+pinned per test on the module object, so no test reads this machine's real hardware.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+TEST_DIR = Path(__file__).resolve().parent
+if str(TEST_DIR) not in sys.path:
+ # Same idiom as conftest.py: these test modules are not a package, so the directory
+ # is what makes a sibling importable.
+ sys.path.insert(0, str(TEST_DIR))
+
+from _pr10648_helpers import llama_host, load_studio_module # noqa: E402
+
+# The install-tree writer the back-compat suite already maintains. Reused rather than
+# re-implemented so a change to what a real tree contains reaches this file too.
+from test_keep_install_backcompat_9979 import build_install # noqa: E402
+
+# A distinct sys.modules name: the sibling suites load this file under
+# "studio_install_llama_prebuilt", and a shared module leaks monkeypatches across xdist tests.
+ILP = load_studio_module(
+ "studio_install_llama_prebuilt_pr10648_matrix", "install_llama_prebuilt.py"
+)
+
+HostInfo = ILP.HostInfo
+host_profile = ILP.host_profile
+existing_install_current_without_plan = ILP.existing_install_current_without_plan
+
+# Every variable that can move routing or the fast path, cleared per test: CUDA_VISIBLE_DEVICES
+# in particular is set on any GPU box, so the answer would depend on the calling shell.
+_ENV_KEYS = (
+ "UNSLOTH_PREBUILT_FULL_CHECK",
+ "UNSLOTH_LLAMA_DISABLE_DOWNLOAD_HOST_RESOLVE",
+ "UNSLOTH_ROCM_GFX_ARCH",
+ "UNSLOTH_ROCM_GFX_REMEMBERED",
+ "UNSLOTH_LLAMA_CPP_BACKEND",
+ "UNSLOTH_FORCE_VULKAN",
+ "CUDA_VISIBLE_DEVICES",
+ "HIP_VISIBLE_DEVICES",
+ "ROCR_VISIBLE_DEVICES",
+ "GPU_DEVICE_ORDINAL",
+ "GGML_VK_VISIBLE_DEVICES",
+)
+
+RELEASE_TAG = "release-1"
+UPSTREAM_TAG = "b9001"
+PUBLISHED_REPO = "unslothai/llama.cpp"
+
+
+def make_host(**overrides) -> HostInfo:
+ return llama_host(HostInfo, **overrides)
+
+
+# The matrix. Rows are operating systems, columns are accelerators.
+
+# WSL reports itself as Linux and HostInfo carries no WSL flag, deliberately (see
+# test_keep_install_backcompat_9979.test_wsl_is_treated_exactly_like_linux_by_the_keep_path).
+# A row anyway, so "WSL is judged by the Linux tables" is stated rather than assumed.
+ROWS = {
+ "linux": dict(system = "Linux", machine = "x86_64"),
+ "windows": dict(system = "Windows", machine = "AMD64"),
+ "wsl": dict(system = "Linux", machine = "x86_64"),
+ "macos-arm64": dict(system = "Darwin", machine = "arm64", macos_version = (15, 5)),
+ "macos-x86_64": dict(system = "Darwin", machine = "x86_64", macos_version = (14, 6)),
+}
+
+# The rows that can carry a discrete accelerator, i.e. everything the hardware transitions
+# below apply to. macOS is Metal or nothing.
+GPU_ROWS = ("linux", "windows", "wsl")
+
+NVIDIA_CUDA12 = dict(
+ nvidia_smi = "nvidia-smi",
+ has_physical_nvidia = True,
+ has_usable_nvidia = True,
+ driver_cuda_version = (12, 8),
+ compute_caps = ["8.9"],
+)
+NVIDIA_CUDA13 = dict(
+ nvidia_smi = "nvidia-smi",
+ has_physical_nvidia = True,
+ has_usable_nvidia = True,
+ driver_cuda_version = (13, 0),
+ compute_caps = ["8.9"],
+)
+# gfx1100 (Navi 31, discrete): _should_prefer_vulkan_for_amd_igpu routes the integrated archs
+# to Vulkan, which would make the cell test the router instead of the profile.
+AMD_ROCM = dict(has_rocm = True, rocm_gfx_target = "gfx1100", rocm_gfx_targets = ["gfx1100"])
+AMD_NO_ROCM = dict(has_amd_gpu_without_rocm = True)
+INTEL = dict(has_intel_gpu = True)
+
+ACCELERATORS = {
+ "nvidia-cuda12": NVIDIA_CUDA12,
+ "nvidia-cuda13": NVIDIA_CUDA13,
+ "amd-rocm": AMD_ROCM,
+ "amd-no-rocm": AMD_NO_ROCM,
+ "intel": INTEL,
+ "cpu": {},
+ "metal": {},
+}
+
+COLUMNS = tuple(ACCELERATORS)
+
+# (install_kind, the payload_backend key build_install writes) per platform and column.
+_KINDS = {
+ "linux": {
+ "nvidia-cuda12": ("linux-cuda", "cuda"),
+ "nvidia-cuda13": ("linux-cuda", "cuda"),
+ "amd-rocm": ("linux-rocm", "rocm"),
+ "amd-no-rocm": ("linux-vulkan", "vulkan"),
+ "intel": ("linux-vulkan", "vulkan"),
+ "cpu": ("linux-cpu", None),
+ },
+ "windows": {
+ "nvidia-cuda12": ("windows-cuda", "cuda"),
+ "nvidia-cuda13": ("windows-cuda", "cuda"),
+ "amd-rocm": ("windows-rocm", "rocm"),
+ "amd-no-rocm": ("windows-vulkan", "vulkan"),
+ "intel": ("windows-vulkan", "vulkan"),
+ "cpu": ("windows-cpu", None),
+ },
+}
+
+_MACOS_KIND = {"macos-arm64": "macos-arm64", "macos-x86_64": "macos-x64"}
+
+# The CUDA runtimes the on-disk scan reports, per column. host_profile records them, and
+# they are what moves when a box's CUDA toolkit does.
+_CUDA_LINES = {"nvidia-cuda12": ("cuda12",), "nvidia-cuda13": ("cuda13",)}
+# What _detect_host_rocm_version answers; upstream ROCm assets are chosen by it.
+_ROCM_RUNTIME = (6, 2)
+
+
[email protected](frozen = True)
+class Cell:
+ """One (OS, accelerator) box: the host, the bundle it would be given, and the probe
+ answers ``host_profile`` reads that ``HostInfo`` does not carry."""
+
+ cell_id: str
+ row: str
+ column: str
+ host: HostInfo
+ install_kind: str
+ payload_backend: "str | None"
+ runtime_line: "str | None"
+ cuda_lines: "tuple[str, ...]"
+ rocm_runtime: "tuple[int, int] | None"
+
+
+def _skip_reason(row: str, column: str) -> "str | None":
+ """Why an (OS, accelerator) pair does not exist, or None when it does."""
+ is_macos = row.startswith("macos")
+ if column == "metal" and not is_macos:
+ return "Metal is the macOS backend; there is no Metal install kind off Darwin"
+ if is_macos and column != "metal":
+ return (
+ "macOS publishes one universal Metal bundle and no CUDA, ROCm or Vulkan "
+ "install kind, so this accelerator column has no cell on Darwin"
+ )
+ return None
+
+
+def _cell(row: str, column: str) -> Cell:
+ host = make_host(**ROWS[row], **ACCELERATORS[column])
+ if row.startswith("macos"):
+ install_kind, payload_backend = _MACOS_KIND[row], None
+ else:
+ install_kind, payload_backend = _KINDS["windows" if row == "windows" else "linux"][column]
+ cuda_lines = _CUDA_LINES.get(column, ())
+ return Cell(
+ cell_id = f"{row}-{column}",
+ row = row,
+ column = column,
+ host = host,
+ install_kind = install_kind,
+ payload_backend = payload_backend,
+ runtime_line = cuda_lines[0] if cuda_lines else None,
+ cuda_lines = cuda_lines,
+ rocm_runtime = _ROCM_RUNTIME if column == "amd-rocm" else None,
+ )
+
+
+MATRIX = [(row, column) for row in ROWS for column in COLUMNS]
+MATRIX_IDS = [f"{row}-{column}" for row, column in MATRIX]
+REACHABLE = [(row, column) for row, column in MATRIX if _skip_reason(row, column) is None]
+REACHABLE_IDS = [f"{row}-{column}" for row, column in REACHABLE]
+
+
+class Probe:
+ """The hardware answers this run gives, pinned on the loaded module.
+
+ ``host_profile`` reads three things a ``HostInfo`` does not carry -- the CUDA runtime
+ scan, the ROCm runtime version and (for the marker) the torch CUDA preference -- and
+ all three would otherwise read this machine. ``set`` moves them together with the
+ detected host, which is exactly what a hardware change does.
+ """
+
+ def __init__(self, monkeypatch):
+ self.host: "HostInfo | None" = None
+ self.cuda_lines: "tuple[str, ...]" = ()
+ self.rocm_runtime: "tuple[int, int] | None" = None
+ self.torch_line: "str | None" = None
+ self.latest_release = RELEASE_TAG
+ monkeypatch.setattr(ILP, "detect_host", lambda **_kwargs: self.host)
+ monkeypatch.setattr(
+ ILP, "detected_linux_runtime_lines", lambda: (list(self.cuda_lines), {})
+ )
+ monkeypatch.setattr(
+ ILP, "detected_windows_runtime_lines", lambda: (list(self.cuda_lines), {})
+ )
+ monkeypatch.setattr(ILP, "_detect_host_rocm_version", lambda: self.rocm_runtime)
+ # torch.cuda.is_available() is slow and beside the point here; the preference only has
+ # to be STABLE, since a moved one is _runtime_preference_moved's subject.
+ monkeypatch.setattr(
+ ILP,
+ "detect_torch_cuda_runtime_preference",
+ lambda _host: SimpleNamespace(runtime_line = self.torch_line, selection_log = []),
+ )
+ monkeypatch.setattr(
+ ILP, "_download_host_latest_release_tag", lambda _repo: self.latest_release
+ )
+
+ def set(
+ self,
+ host: HostInfo,
+ *,
+ cuda_lines: "tuple[str, ...]" = (),
+ rocm_runtime: "tuple[int, int] | None" = None,
+ torch_line: "str | None" = None,
+ ) -> HostInfo:
+ self.host = host
+ self.cuda_lines = cuda_lines
+ self.rocm_runtime = rocm_runtime
+ self.torch_line = torch_line
+ return host
+
+ def set_from(
+ self,
+ cell: Cell,
+ host: "HostInfo | None" = None,
+ **overrides,
+ ) -> HostInfo:
+ """Point the probes at ``cell``'s answers, optionally for a different host."""
+ answers = dict(
+ cuda_lines = cell.cuda_lines,
+ rocm_runtime = cell.rocm_runtime,
+ torch_line = cell.runtime_line,
+ )
+ answers.update(overrides)
+ return self.set(cell.host if host is None else host, **answers)
+
+
+def probe(monkeypatch):
+ for key in _ENV_KEYS:
+ monkeypatch.delenv(key, raising = False)
+ return Probe(monkeypatch)
+
+
+def route_for(host: HostInfo, repo: str = PUBLISHED_REPO):
+ """The routed host and repo, the same no-network routing the fast path re-derives.
+
+ ``host`` is passed explicitly rather than left to the patched ``detect_host`` so a
+ fixture can never route one box while the probes answer for another."""
+ return ILP.route_backend_request(
+ backend = "auto",
+ published_repo = repo,
+ published_release_tag = "",
+ host = host,
+ )
+
+
+def install_cell(
+ tmp_path: Path,
+ probe: Probe,
+ cell: Cell,
+ *,
+ name: str = "install",
+) -> Path:
+ """An install tree plus a marker genuinely written for ``cell``'s host.
+
+ The marker goes through the real ``write_prebuilt_metadata`` against the ROUTED host,
+ so its ``install_fingerprint``, ``runtime_files`` and ``host_profile`` are the ones a
+ real install of this bundle on this box would carry. Anything hand-written here would
+ fail the fingerprint guard and make a later "rejected" assertion meaningless.
+ """
+ probe.set_from(cell)
+ route = route_for(cell.host)
+ install_dir = build_install(
+ tmp_path / name,
+ host = route.host,
+ marker = None,
+ payload_backend = cell.payload_backend,
+ )
+ runtime_dir = ILP.install_runtime_dir(install_dir, route.host)
+ # The fork bundles ship the DiffusionGemma visual server and the marker-only backfill
+ # check looks for it; build_install writes it on Linux and Windows but not on macOS.
+ visual_server = runtime_dir / (
+ "llama-diffusion-gemma-visual-server" + (".exe" if route.host.is_windows else "")
+ )
+ if not visual_server.exists():
+ visual_server.write_text("#!/bin/sh\nexit 0\n", encoding = "utf-8")
+ choice = ILP.AssetChoice(
+ repo = route.published_repo or PUBLISHED_REPO,
+ tag = RELEASE_TAG,
+ name = f"llama-{UPSTREAM_TAG}-{cell.install_kind}.tar.gz",
+ url = f"https://example.com/llama-{UPSTREAM_TAG}-{cell.install_kind}.tar.gz",
+ source_label = "published",
+ install_kind = cell.install_kind,
+ expected_sha256 = "a" * 64,
+ runtime_line = cell.runtime_line,
+ coverage_class = "targeted" if cell.runtime_line else None,
+ supported_sms = ["89"] if cell.runtime_line else None,
+ bundle_profile = f"{cell.runtime_line}-newer" if cell.runtime_line else None,
+ gfx_target = cell.host.rocm_gfx_target,
+ mapped_targets = list(cell.host.rocm_gfx_targets or []),
+ )
+ checksums = _checksums(choice, repo = route.published_repo or PUBLISHED_REPO)
+ ILP.write_prebuilt_metadata(
+ install_dir,
+ host = route.host,
+ requested_tag = "latest",
+ llama_tag = UPSTREAM_TAG,
+ release_tag = RELEASE_TAG,
+ choice = choice,
+ approved_checksums = checksums,
+ prebuilt_fallback_used = False,
+ backend_request = "auto",
+ )
+ return install_dir
+
+
+def _checksums(choice, *, repo: str):
+ logical = ILP.source_archive_logical_name(UPSTREAM_TAG)
+ artifacts = {
+ logical: ILP.ApprovedArtifactHash(
+ asset_name = logical,
+ sha256 = "b" * 64,
+ repo = "ggml-org/llama.cpp",
+ kind = "upstream-source",
+ ),
+ choice.name: ILP.ApprovedArtifactHash(
+ asset_name = choice.name,
+ sha256 = choice.expected_sha256,
+ repo = repo,
+ kind = "prebuilt",
+ ),
+ }
+ return ILP.ApprovedReleaseChecksums(
+ repo = repo,
+ release_tag = RELEASE_TAG,
+ upstream_tag = UPSTREAM_TAG,
+ source_commit = "deadbeef",
+ artifacts = artifacts,
+ )
+
+
+def check(install_dir: Path, **overrides) -> bool:
+ kwargs = dict(
+ llama_tag = "latest",
+ published_repo = PUBLISHED_REPO,
+ published_release_tag = "",
+ backend_request = "auto",
+ force_cpu = False,
+ )
+ kwargs.update(overrides)
+ return existing_install_current_without_plan(install_dir, **kwargs)
+
+
+def marker_of(install_dir: Path) -> dict:
+ return json.loads((install_dir / "UNSLOTH_PREBUILT_INFO.json").read_text(encoding = "utf-8"))
+
+
+def moved(tmp_path: Path, probe: Probe, cell: Cell, after: HostInfo, **answers) -> Path:
+ """Install on ``cell``'s box, prove it is kept, then move the hardware under it."""
+ install_dir = install_cell(tmp_path, probe, cell)
+ assert check(install_dir) is True, f"fixture is not current on its own host: {cell.cell_id}"
+ probe.set_from(cell, after, **answers)
+ return install_dir
+
+
+def assert_reinstall_forced(monkeypatch, install_dir: Path) -> None:
+ """The fast path refuses, and the host profile is the guard that refused.
+
+ A bare ``is False`` would also be satisfied by a broken tree, a moved release tag or a
+ marker that does not add up -- so every hardware transition proves the attribution as
+ well: with the profile comparison neutralised, and nothing else touched, the same call
+ answers True again. That is the install the user would have been left holding.
+
+ Patches last, so no assertion after this one is trustworthy; each caller ends here.
+ """
+ assert check(install_dir) is False
+ recorded = marker_of(install_dir)["host_profile"]
+ monkeypatch.setattr(ILP, "host_profile", lambda _host: recorded)
+ assert check(install_dir) is True, (
+ "the install was refused by some other guard, so this case does not show the "
+ "host profile closing anything"
+ )
+
+
+# (A) the matrix itself: which cells exist.
+
+
[email protected](("row", "column"), MATRIX, ids = MATRIX_IDS)
+def test_every_cell_of_the_matrix_is_reachable_or_explicitly_impossible(row, column):
+ """The user population this fast path runs for: every OS crossed with every
+ accelerator. A cell that does not exist is skipped with the reason, so a column
+ quietly losing a platform shows up as a skip rather than as absence."""
+ reason = _skip_reason(row, column)
+ if reason is not None:
+ pytest.skip(reason)
+ cell = _cell(row, column)
+ assert cell.install_kind in ILP.INSTALL_KIND_BACKENDS, cell.cell_id
+ prefix = "windows-" if cell.host.is_windows else "macos-" if cell.host.is_macos else "linux-"
+ assert cell.install_kind.startswith(prefix), cell.cell_id
+
+
+# (B) the acceptance half: an unchanged box keeps its install, in every reachable cell.
+
+
[email protected](("row", "column"), REACHABLE, ids = REACHABLE_IDS)
+def test_an_unchanged_box_keeps_its_install_in_every_cell(tmp_path, probe, row, column):
+ """Nothing happened to the user's machine since the install: no download, no listing,
+ no re-validation. This is the half the fast path exists for, and every rejection test
+ below is only meaningful because this one passes with the same fixtures."""
+ cell = _cell(row, column)
+ install_dir = install_cell(tmp_path, probe, cell)
+ assert check(install_dir) is True, cell.cell_id
+ # And the recorded profile really is this box, not a default that would match anything.
+ assert marker_of(install_dir)["host_profile"] == host_profile(route_for(cell.host).host)
+
+
[email protected](("row", "column"), REACHABLE, ids = REACHABLE_IDS)
+def test_a_second_update_on_the_same_box_is_still_current(tmp_path, probe, row, column):
+ """Updates are not one-shot: the user runs setup again next week with nothing changed.
+ A profile that failed to compare equal to itself would be a silent permanent
+ regression -- every update back to the full path, forever, with no error to report."""
+ cell = _cell(row, column)
+ install_dir = install_cell(tmp_path, probe, cell)
+ assert check(install_dir) is True, cell.cell_id
+ assert check(install_dir) is True, cell.cell_id
+ assert check(install_dir) is True, cell.cell_id
+
+
+# (C) the rejection half: one axis flipped at a time, each its own user scenario.
+
+
+def _cpu_cell(row: str) -> Cell:
+ return _cell(row, "cpu")
+
+
[email protected]("row", GPU_ROWS)
+def test_a_gpu_added_to_a_cpu_only_box_is_not_current(tmp_path, probe, monkeypatch, row):
+ """The user drops an NVIDIA card into a CPU-only machine and re-runs setup. The
+ request recorded is "auto" and the release tag has not moved, so only the host
+ profile can notice; without it the box keeps its CPU bundle and stays slow."""
+ cell = _cpu_cell(row)
+ after = make_host(**ROWS[row], **NVIDIA_CUDA12)
+ install_dir = moved(tmp_path, probe, cell, after, cuda_lines = ("cuda12",), torch_line = None)
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", GPU_ROWS)
+def test_a_gpu_removed_since_the_install_is_not_current(tmp_path, probe, monkeypatch, row):
+ """The card was pulled, or the driver uninstalled. A CUDA bundle on a box with no
+ CUDA does not load at all, so keeping it is worse than a reinstall."""
+ cell = _cell(row, "nvidia-cuda12")
+ after = make_host(**ROWS[row])
+ install_dir = moved(tmp_path, probe, cell, after, cuda_lines = (), torch_line = None)
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", GPU_ROWS)
+def test_a_cuda_12_to_cuda_13_move_is_not_current(tmp_path, probe, monkeypatch, row):
+ """The user upgrades the NVIDIA driver and the CUDA toolkit. The card is the same and
+ the release tag has not moved; what moved is the driver's CUDA version and the CUDA
+ runtimes on disk, which are what orders the CUDA bundles."""
+ cell = _cell(row, "nvidia-cuda12")
+ after = make_host(**ROWS[row], **NVIDIA_CUDA13)
+ install_dir = moved(
+ tmp_path, probe, cell, after, cuda_lines = ("cuda13",), torch_line = cell.runtime_line
+ )
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", GPU_ROWS)
+def test_only_the_cuda_runtimes_on_disk_moving_is_not_current(tmp_path, probe, monkeypatch, row):
+ """A CUDA 13 toolkit installed beside the existing one, driver untouched. The HostInfo
+ is byte-identical; cuda_runtime_lines is the only field that can carry it."""
+ cell = _cell(row, "nvidia-cuda12")
+ install_dir = moved(
+ tmp_path,
+ probe,
+ cell,
+ cell.host,
+ cuda_lines = ("cuda12", "cuda13"),
+ torch_line = cell.runtime_line,
+ )
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", GPU_ROWS)
+def test_swapping_an_nvidia_card_for_an_amd_one_is_not_current(tmp_path, probe, monkeypatch, row):
+ """The user replaces the NVIDIA card with a Radeon and installs ROCm. A CUDA bundle
+ on an AMD box is an unusable backend, not a slow one."""
+ cell = _cell(row, "nvidia-cuda12")
+ after = make_host(**ROWS[row], **AMD_ROCM)
+ install_dir = moved(tmp_path, probe, cell, after, cuda_lines = (), rocm_runtime = _ROCM_RUNTIME)
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", GPU_ROWS)
+def test_swapping_an_amd_card_for_an_nvidia_one_is_not_current(tmp_path, probe, monkeypatch, row):
+ """The mirror image: a ROCm/HIP bundle left on a box that now has an NVIDIA card."""
+ cell = _cell(row, "amd-rocm")
+ after = make_host(**ROWS[row], **NVIDIA_CUDA12)
+ install_dir = moved(tmp_path, probe, cell, after, cuda_lines = ("cuda12",), rocm_runtime = None)
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", GPU_ROWS)
+def test_a_rocm_runtime_version_bump_is_not_current(tmp_path, probe, monkeypatch, row):
+ """The user upgrades ROCm 6.2 to 6.4 with the same card. Upstream ROCm assets are
+ chosen by the runtime's major.minor, which no GPU field carries -- so the bundle this
+ run would pick has moved even though every other hardware fact is identical."""
+ cell = _cell(row, "amd-rocm")
+ install_dir = moved(tmp_path, probe, cell, cell.host, rocm_runtime = (6, 4))
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", GPU_ROWS)
+def test_a_rocm_gfx_target_change_is_not_current(tmp_path, probe, monkeypatch, row):
+ """A different AMD card: gfx1100 out, gfx1030 in. A HIP bundle carries kernels for the
+ archs it was built for and nothing else, so the install starts and then cannot run a
+ model."""
+ cell = _cell(row, "amd-rocm")
+ after = make_host(
+ **ROWS[row],
+ has_rocm = True,
+ rocm_gfx_target = "gfx1030",
+ rocm_gfx_targets = ["gfx1030"],
+ )
+ install_dir = moved(tmp_path, probe, cell, after, rocm_runtime = _ROCM_RUNTIME)
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", GPU_ROWS)
+def test_a_second_amd_card_the_bundle_has_no_kernels_for_is_not_current(
+ tmp_path, probe, monkeypatch, row
+):
+ """The active gfx is unchanged, but a second Radeon appeared. rocm_gfx_targets is the
+ inventory the runtime device gate reads, so it has to be part of the comparison."""
+ cell = _cell(row, "amd-rocm")
+ after = make_host(
+ **ROWS[row],
+ has_rocm = True,
+ rocm_gfx_target = "gfx1100",
+ rocm_gfx_targets = ["gfx1100", "gfx1030"],
+ )
+ install_dir = moved(tmp_path, probe, cell, after, rocm_runtime = _ROCM_RUNTIME)
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", GPU_ROWS)
+def test_an_intel_gpu_that_appeared_is_not_current(tmp_path, probe, monkeypatch, row):
+ """The user enables the integrated Intel GPU (a BIOS change, or a new machine restored
+ from the same home directory). An Intel GPU with no NVIDIA and no ROCm is the whole
+ purpose of the has_intel_gpu probe: it routes the install to Vulkan."""
+ cell = _cpu_cell(row)
+ after = make_host(**ROWS[row], **INTEL)
+ install_dir = moved(tmp_path, probe, cell, after)
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", GPU_ROWS)
+def test_an_intel_gpu_that_disappeared_is_not_current(tmp_path, probe, monkeypatch, row):
+ """The mirror: the Vulkan bundle chosen for an Intel GPU, on a box that no longer
+ reports one."""
+ cell = _cell(row, "intel")
+ after = make_host(**ROWS[row])
+ install_dir = moved(tmp_path, probe, cell, after)
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", GPU_ROWS)
+def test_a_physical_nvidia_disappearing_behind_an_intel_gpu_is_not_current(
+ tmp_path, probe, monkeypatch, row
+):
+ """The Vulkan-route gate. The box has an Intel GPU and an NVIDIA card hidden by
+ CUDA_VISIBLE_DEVICES, so has_usable_nvidia is already False and the Intel auto-route is
+ held off by has_physical_nvidia alone. Pull the card and the same run routes to Vulkan
+ -- a transition invisible to every other field in the profile."""
+ cell = dataclasses.replace(
+ _cpu_cell(row),
+ host = make_host(
+ **ROWS[row],
+ **INTEL,
+ has_physical_nvidia = True,
+ has_usable_nvidia = False,
+ nvidia_smi = "nvidia-smi",
+ visible_cuda_devices = "",
+ ),
+ )
+ probe.set_from(cell)
+ before_profile = host_profile(route_for(cell.host).host)
+ assert before_profile["has_physical_nvidia"] is True
+ assert before_profile["has_usable_nvidia"] is False
+ assert before_profile["has_intel_gpu"] is True
+ after = make_host(**ROWS[row], **INTEL)
+ install_dir = moved(tmp_path, probe, cell, after)
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", GPU_ROWS)
+def test_a_home_directory_carried_to_another_cpu_architecture_is_not_current(
+ tmp_path, probe, monkeypatch, row
+):
+ """A home directory rsynced from an x86_64 box onto an arm64 one (or opened under
+ Rosetta). Every GPU field is equal and the release tag is the same; only machine
+ differs, and the bundle on disk cannot be executed at all."""
+ cell = _cpu_cell(row)
+ arm = "ARM64" if row == "windows" else "aarch64"
+ after = make_host(**{**ROWS[row], "machine": arm})
+ install_dir = moved(tmp_path, probe, cell, after)
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", GPU_ROWS)
+def test_a_new_card_with_a_different_sm_is_not_current(tmp_path, probe, monkeypatch, row):
+ """An RTX 4090 (sm_89) swapped for a Blackwell part (sm_120), same driver. A bundle
+ carries kernels for a fixed supported_sms set, so an install outside it starts and
+ then fails on the first model load."""
+ cell = _cell(row, "nvidia-cuda12")
+ after = make_host(**ROWS[row], **{**NVIDIA_CUDA12, "compute_caps": ["12.0"]})
+ install_dir = moved(
+ tmp_path, probe, cell, after, cuda_lines = cell.cuda_lines, torch_line = cell.runtime_line
+ )
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", GPU_ROWS)
+def test_the_same_cards_reported_in_another_order_is_not_a_hardware_change(tmp_path, probe, row):
+ """The other direction, and the reason compute_caps is sorted and de-duplicated: two
+ identical cards enumerated in either order, with whitespace, are the same box. Calling
+ that a change would send every multi-GPU user down the full path on every update."""
+ cell = _cell(row, "nvidia-cuda12")
+ after = make_host(**ROWS[row], **{**NVIDIA_CUDA12, "compute_caps": [" 8.9 ", "8.9"]})
+ install_dir = moved(
+ tmp_path, probe, cell, after, cuda_lines = cell.cuda_lines, torch_line = cell.runtime_line
+ )
+ assert check(install_dir) is True
+
+
+MACOS_ROWS = ("macos-arm64", "macos-x86_64")
+
+
[email protected]("row", MACOS_ROWS)
+def test_a_macos_upgrade_across_the_min_os_floor_is_not_current(tmp_path, probe, monkeypatch, row):
+ """The user upgrades macOS past the floor the newest bundle needs. The bundle the
+ planner would pick changes with the OS version (walk-back), so the recorded profile
+ has to carry macos_version or a Mac would keep a walked-back release forever."""
+ cell = _cell(row, "metal")
+ before = cell.host.macos_version
+ assert before is not None and before < ILP._PINNED_MACOS_LATEST_FLOOR
+ after = make_host(**{**ROWS[row], "macos_version": ILP._PINNED_MACOS_LATEST_FLOOR})
+ install_dir = moved(tmp_path, probe, cell, after)
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
[email protected]("row", MACOS_ROWS)
+def test_a_macos_point_release_is_not_current(tmp_path, probe, monkeypatch, row):
+ """Not only the floor: any macOS version move re-decides the bundle, because the
+ minimum-OS preflight is what accepted this one."""
+ cell = _cell(row, "metal")
+ major, minor = cell.host.macos_version
+ after = make_host(**{**ROWS[row], "macos_version": (major, minor + 1)})
+ install_dir = moved(tmp_path, probe, cell, after)
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
+def test_a_mac_home_directory_restored_onto_apple_silicon_is_not_current(
+ tmp_path, probe, monkeypatch
+):
+ """The Intel Mac's home directory restored onto an Apple Silicon Mac, or the same one
+ opened under Rosetta. macos_version and every GPU field are equal; machine is not."""
+ cell = _cell("macos-x86_64", "metal")
+ after = make_host(**{**ROWS["macos-x86_64"], "machine": "arm64"})
+ install_dir = moved(tmp_path, probe, cell, after)
+ assert_reinstall_forced(monkeypatch, install_dir)
+
+
+# (D) the guard is load-bearing, not incidental.
+
+
+def _assert_everything_but_the_host_still_matches(install_dir: Path, after: HostInfo) -> dict:
+ """Everything the fast path checks APART from the host profile still holds.
+
+ Without this, a "rejected" assertion could be passing because the tree was broken or
+ the release moved, and the host guard could be removed with the tests still green.
+ """
+ marker = marker_of(install_dir)
+ route = route_for(after)
+ assert marker["release_tag"] == RELEASE_TAG
+ assert marker["published_repo"] == (route.published_repo or ILP.DEFAULT_PUBLISHED_REPO)
+ assert marker["backend_request"] == "auto"
+ assert marker["prebuilt_fallback_used"] is False
+ # The marker is intact and self-consistent: the fingerprint recomputes from its own
+ # fields, so nothing has been hand-edited.
+ assert ILP._marker_install_fingerprint(marker) == marker["install_fingerprint"]
+ # The release this run would ask for is the release that is installed.
+ assert (
+ ILP._expected_release_tag_without_plan(
+ marker, "latest", PUBLISHED_REPO, "", host = route.host
+ )
+ == RELEASE_TAG
+ )
+ # The tree is intact, executable, and the recorded bytes are still on disk.
+ assert ILP._marker_backend_fits_host(marker, route.host) is True
+ assert ILP._install_tree_is_usable(install_dir, route.host) is True
+ assert ILP._kept_install_payload_is_healthy(install_dir, route.host) is True
+ assert ILP._runtime_files_match(install_dir, route.host, marker) is True
+ assert ILP._diffusion_visual_server_missing_for_marker(install_dir, route.host, marker) is False
+ # And torch has not moved either, so the CUDA-preference guard is not the one talking.
+ assert ILP._runtime_preference_moved(marker, route.host) is False
+ # The one thing that HAS moved.
+ assert marker["host_profile"] != host_profile(route.host)
+ return marker
+
+
[email protected]("row", GPU_ROWS)
+def test_adding_a_gpu_is_rejected_by_the_host_profile_and_nothing_else(
+ tmp_path, probe, monkeypatch, row
+):
+ """The CPU-to-NVIDIA case again, but proving WHICH guard rejected it. Same release,
+ same fingerprint, intact tree -- and with the profile comparison neutralised the fast
+ path keeps the CPU bundle, which is the bug the guard exists to prevent."""
+ cell = _cpu_cell(row)
+ after = make_host(**ROWS[row], **NVIDIA_CUDA12)
+ install_dir = moved(tmp_path, probe, cell, after, cuda_lines = ("cuda12",), torch_line = None)
+ assert check(install_dir) is False
+ marker = _assert_everything_but_the_host_still_matches(install_dir, after)
+ # Neutralise only the profile comparison: everything else is untouched.
+ monkeypatch.setattr(ILP, "host_profile", lambda _host: marker["host_profile"])
+ assert (
+ check(install_dir) is True
+ ), "without the host_profile guard this box keeps its CPU bundle after gaining a GPU"
+
+
[email protected]("row", GPU_ROWS)
+def test_a_cuda_line_move_is_rejected_by_the_host_profile_and_nothing_else(
+ tmp_path, probe, monkeypatch, row
+):
+ """The cuda12-to-cuda13 case, same proof. The release did not move, the marker is
+ whole and the bundle is intact, so the profile is the only thing that can say the
+ installed CUDA bundle is no longer the one this run would pick."""
+ cell = _cell(row, "nvidia-cuda12")
+ after = make_host(**ROWS[row], **NVIDIA_CUDA13)
+ install_dir = moved(
+ tmp_path, probe, cell, after, cuda_lines = ("cuda13",), torch_line = cell.runtime_line
+ )
+ assert check(install_dir) is False
+ marker = _assert_everything_but_the_host_still_matches(install_dir, after)
+ monkeypatch.setattr(ILP, "host_profile", lambda _host: marker["host_profile"])
+ assert (
+ check(install_dir) is True
+ ), "without the host_profile guard this box keeps its cuda12 bundle after moving to cuda13"
+
+
+# (E) the JSON round trip. A profile that never equals itself takes the full path forever.
+
+
+def _tuples_in(value) -> bool:
+ if isinstance(value, tuple):
+ return True
+ if isinstance(value, dict):
+ return any(_tuples_in(item) for item in value.values())
+ if isinstance(value, list):
+ return any(_tuples_in(item) for item in value)
+ return False
+
+
[email protected](("row", "column"), REACHABLE, ids = REACHABLE_IDS)
+def test_a_profile_written_to_a_marker_reads_back_equal_to_a_fresh_one(
+ tmp_path, probe, row, column
+):
+ """Written by one run, compared by the next through json.loads. Whole-dict equality is
+ what the fast path asks, so a single field that does not survive serialisation would
+ send this cell down the full path on every update forever -- with no error, just a
+ permanently slow setup."""
+ cell = _cell(row, column)
+ install_dir = install_cell(tmp_path, probe, cell)
+ recorded = marker_of(install_dir)["host_profile"]
+ fresh = host_profile(route_for(cell.host).host)
+ assert recorded == fresh, cell.cell_id
+ # The hazard named in the docstring at host_profile: a tuple reads back as a list.
+ assert not _tuples_in(fresh), f"{cell.cell_id}: a tuple would never read back equal"
+ assert json.loads(json.dumps(fresh)) == fresh, cell.cell_id
+
+
[email protected](("row", "column"), REACHABLE, ids = REACHABLE_IDS)
+def test_every_profile_field_is_present_in_every_cell(tmp_path, probe, row, column):
+ """Absent and null are different answers to the comparison. Every field the routing
+ branches on must be written in every cell, or two different boxes could record the
+ same profile."""
+ cell = _cell(row, column)
+ probe.set_from(cell)
+ profile = host_profile(route_for(cell.host).host)
+ assert set(profile) == {
+ "machine",
+ "has_usable_nvidia",
+ "has_physical_nvidia",
+ "driver_cuda_version",
+ "compute_caps",
+ "has_rocm",
+ "rocm_runtime",
+ "rocm_gfx_target",
+ "rocm_gfx_targets",
+ "has_intel_gpu",
+ "has_amd_gpu_without_rocm",
+ "macos_version",
+ "cuda_runtime_lines",
+ }, cell.cell_id
+
+
+def test_the_tuple_valued_fields_survive_the_marker_round_trip(tmp_path, probe):
+ """driver_cuda_version, macos_version, compute_caps, rocm_gfx_targets and
+ cuda_runtime_lines are the fields built from tuples or sets. Each is checked on a cell
+ that actually populates it, against the value read back out of the marker file."""
+ cuda = _cell("linux", "nvidia-cuda12")
+ cuda_marker = marker_of(install_cell(tmp_path, probe, cuda, name = "cuda"))["host_profile"]
+ assert cuda_marker["driver_cuda_version"] == [12, 8]
+ assert cuda_marker["compute_caps"] == ["8.9"]
+ assert cuda_marker["cuda_runtime_lines"] == ["cuda12"]
+
+ rocm = _cell("linux", "amd-rocm")
+ rocm_marker = marker_of(install_cell(tmp_path, probe, rocm, name = "rocm"))["host_profile"]
+ assert rocm_marker["rocm_runtime"] == [6, 2]
+ assert rocm_marker["rocm_gfx_target"] == "gfx1100"
+ assert rocm_marker["rocm_gfx_targets"] == ["gfx1100"]
+
+ mac = _cell("macos-arm64", "metal")
+ mac_marker = marker_of(install_cell(tmp_path, probe, mac, name = "mac"))["host_profile"]
+ assert mac_marker["macos_version"] == [15, 5]
+ assert mac_marker["machine"] == "arm64"
+
+ # The round trip is what turns each of those tuples into a list; a profile computed
+ # fresh from the same host must still compare equal to the parsed one.
+ for cell, parsed in ((cuda, cuda_marker), (rocm, rocm_marker), (mac, mac_marker)):
+ probe.set_from(cell)
+ assert parsed == host_profile(route_for(cell.host).host), cell.cell_id
+
+
+def test_a_marker_with_no_host_profile_cannot_answer_and_takes_the_full_path(tmp_path, probe):
+ """Every marker written before #10648 has no profile. "Cannot tell" must not read as
+ "unchanged", or the very installs this guard was added for would skip it."""
+ cell = _cell("linux", "nvidia-cuda12")
+ install_dir = install_cell(tmp_path, probe, cell)
+ assert check(install_dir) is True
+ marker = marker_of(install_dir)
+ marker.pop("host_profile")
+ (install_dir / "UNSLOTH_PREBUILT_INFO.json").write_text(
+ json.dumps(marker, indent = 2) + "\n", encoding = "utf-8"
+ )
+ assert check(install_dir) is False
+
+
+def test_two_different_boxes_never_record_the_same_profile(probe):
+ """The comparison is only worth running if the cells it separates are distinguishable.
+ Every reachable cell's profile must differ from every other's, or a hardware change
+ between those two boxes would be invisible."""
+ owner_of_profile: dict[str, str] = {}
+ profile_of_cell: dict[str, str] = {}
+ for row, column in REACHABLE:
+ cell = _cell(row, column)
+ probe.set_from(cell)
+ key = json.dumps(host_profile(route_for(cell.host).host), sort_keys = True)
+ profile_of_cell[cell.cell_id] = key
+ # Rows that are the same box by construction: WSL reports itself as Linux, and
+ # HostInfo has no WSL flag, so linux-<column> and wsl-<column> are one profile.
+ if row == "wsl":
+ assert key == profile_of_cell[f"linux-{column}"], "WSL must read exactly as Linux"
+ continue
+ clash = owner_of_profile.get(key)
+ assert clash is None, f"{cell.cell_id} records the same profile as {clash}"
+ owner_of_profile[key] = cell.cell_id
diff --git a/tests/studio/install/test_pr10648_integrity_and_fs.py b/tests/studio/install/test_pr10648_integrity_and_fs.py
new file mode 100644
index 00000000000..ac559d507a1
--- /dev/null
+++ b/tests/studio/install/test_pr10648_integrity_and_fs.py
@@ -0,0 +1,1058 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""PR #10648: can the marker fast path keep a BROKEN prebuilt install?
+
+``studio update`` used to re-validate a kept prebuilt by re-reading the release and
+STARTING llama-server. The precheck replaces that with evidence recorded in the marker,
+so the question every test here asks is the same one: take a healthy install, damage
+exactly one thing, and see whether the fast path still says "already matches".
+
+Three components, three different amounts of evidence, so three different answers:
+
+ * llama (``_runtime_files_match``) records ``size`` for every allowlisted runtime file
+ and ``size + sha256`` for the three binaries a reuse decision would otherwise run.
+ Only the digest can see a corruption that preserves the byte count, which is why
+ ``test_llama_a_same_size_byte_flip_is_caught_only_by_the_digest`` is the load-bearing test
+ in this file.
+ * whisper (``installed_tree_is_intact``) now records the same two tiers in the same
+ ``runtime_files`` shape: ``size + sha256`` for ``whisper-server``, ``size`` for the
+ ggml libraries a slim bundle hardlinks. It previously recorded NO payload digests at
+ all -- it asked only that the server was a non-empty executable file and that a slim
+ bundle's wiring was still there by name, so a truncation that left bytes behind was
+ invisible to it. The three tests below that used to pin that weaker boundary now pin
+ the digest one; a marker with no record (every install predating the key) is still
+ kept and backfilled once, which ``test_pr10648_whisper_payload_digests.py`` owns.
+ * node (``_file_record_matches``) records ``sha256`` and never compares it: ``size``
+ and ``mtime_ns`` only, deliberately (see its docstring). Not exercised here beyond
+ its marker writer; ``test_install_node_prebuilt_logic.py`` owns that decision.
+
+Part 4 leaves integrity behind and tests the atomic marker rewriters the PR added
+(``prebuilt_core.write_live_marker``, ``install_node_prebuilt._write_metadata_payload``,
+``install_llama_prebuilt._write_marker``) as what they are: filesystem operations on a
+file that is already in service, which must keep its mode and its group, must never
+leave a ``.tmp-*`` sibling, and must never leave a torn marker -- ``load_prebuilt_metadata``
+reads an unparseable marker as "nothing installed".
+
+No network, no GPU. The release lookup and host detection are monkeypatched; every
+install tree is written under the test's own ``tmp_path``. POSIX-only cases (mode bits,
+``os.access(X_OK)``, symlinks, ``os.chown``) skip on Windows rather than being weakened,
+and the permission cases skip under root, which bypasses the bits they rely on.
+"""
+
+import dataclasses
+import errno
+import json
+import os
+import shutil
+import stat
+from pathlib import Path
+from typing import Any, Callable
+
+import pytest
+
+from _pr10648_helpers import (
+ IS_ROOT,
+ NEEDS_CHOWN,
+ NOT_ROOT,
+ POSIX_ONLY,
+ WINDOWS_HOST,
+ llama_host,
+ whisper_host,
+ whisper_install_is_intact,
+ whisper_selection_fields,
+)
+from _pr10648_helpers import load_studio_module as _load
+
+LLAMA = _load("studio_install_llama_prebuilt_pr10648_integrity", "install_llama_prebuilt.py")
+CORE = _load("studio_prebuilt_core_pr10648_integrity", "prebuilt_core.py")
+NODE = _load("studio_install_node_prebuilt_pr10648_integrity", "install_node_prebuilt.py")
+WHISPER = _load("studio_install_whisper_prebuilt_pr10648_integrity", "install_whisper_prebuilt.py")
+
+# build_install writes a real install tree. Imported rather than re-implemented: a private
+# copy is how a fixture drifts from the tree the installer actually makes.
+import test_keep_install_backcompat_9979 as KEEP # noqa: E402
+
+MARKER_NAME = "UNSLOTH_PREBUILT_INFO.json"
+
+
+LINUX = llama_host(LLAMA.HostInfo)
+_UPSTREAM = ("ggml-org/llama.cpp", "upstream-prebuilt")
+_SOURCE = ("ggml-org/llama.cpp", "upstream-source")
+
+
+def _asset_choice(**overrides):
+ name = overrides.pop("name", "llama-b9001-bin-ubuntu-x64.tar.gz")
+ defaults = dict(
+ repo = "unslothai/llama.cpp",
+ tag = "release-1",
+ name = name,
+ url = f"https://example.com/{name}",
+ source_label = "upstream",
+ install_kind = "linux-cpu",
+ expected_sha256 = "a" * 64,
+ )
+ defaults.update(overrides)
+ return LLAMA.AssetChoice(**defaults)
+
+
+def _artifact(asset_name: str, sha256: str, origin: "tuple[str, str]"):
+ repo, kind = origin
+ return LLAMA.ApprovedArtifactHash(asset_name = asset_name, sha256 = sha256, repo = repo, kind = kind)
+
+
+def _release_checksums(*assets: "tuple[str, str, tuple[str, str]]"):
+ logical = LLAMA.source_archive_logical_name("b9001")
+ artifacts = {logical: _artifact(logical, "b" * 64, _SOURCE)}
+ for asset_name, sha256, origin in assets:
+ artifacts[asset_name] = _artifact(asset_name, sha256, origin)
+ return LLAMA.ApprovedReleaseChecksums(
+ repo = "unslothai/llama.cpp",
+ release_tag = "release-1",
+ upstream_tag = "b9001",
+ source_commit = "deadbeef",
+ artifacts = artifacts,
+ )
+
+
+def _fill_payload(install_dir: Path, host) -> None:
+ """Give every recorded file distinct, non-empty bytes.
+
+ build_install writes the payload libraries empty, and an empty file cannot be
+ truncated to half its length or have a byte flipped in it -- the two corruptions
+ that separate the size tier from the digest tier would silently become no-ops.
+ """
+ runtime_dir = LLAMA.install_runtime_dir(install_dir, host)
+ for path in sorted(runtime_dir.iterdir()):
+ if path.is_file() and path.stat().st_size == 0:
+ path.write_bytes(b"payload:" + path.name.encode("utf-8") + b":" + b"\xa5" * 48)
+
+
+def _install(
+ tmp_path: Path,
+ monkeypatch,
+ *,
+ host = LINUX,
+) -> Path:
+ """A healthy install plus a marker written by the REAL write_prebuilt_metadata.
+
+ The fingerprint guard (step 4 of existing_install_current_without_plan) rejects any
+ marker it cannot recompute, so a hand-written one would make every "the fast path
+ accepts" baseline below pass for the wrong reason.
+ """
+ install_dir = KEEP.build_install(tmp_path, host = host, marker = None)
+ _fill_payload(install_dir, host)
+ choice = _asset_choice()
+ checksums = _release_checksums((choice.name, choice.expected_sha256, _UPSTREAM))
+ LLAMA.write_prebuilt_metadata(
+ install_dir,
+ host = host,
+ requested_tag = "latest",
+ llama_tag = "b9001",
+ release_tag = "release-1",
+ choice = choice,
+ approved_checksums = checksums,
+ prebuilt_fallback_used = False,
+ backend_request = "auto",
+ )
+ monkeypatch.setattr(LLAMA, "detect_host", lambda **_k: host)
+ # The one HEAD the precheck makes. Answered locally: this file never reaches the network.
+ monkeypatch.setattr(LLAMA, "_download_host_latest_release_tag", lambda _repo: "release-1")
+ for name in (
+ "UNSLOTH_PREBUILT_FULL_CHECK",
+ "UNSLOTH_LLAMA_DISABLE_DOWNLOAD_HOST_RESOLVE",
+ "UNSLOTH_ROCM_GFX_ARCH",
+ "UNSLOTH_ROCM_GFX_REMEMBERED",
+ ):
+ monkeypatch.delenv(name, raising = False)
+ return install_dir
+
+
+def _fast_path(install_dir: Path, **overrides) -> bool:
+ kwargs = dict(
+ llama_tag = "latest",
+ published_repo = "unslothai/llama.cpp",
+ published_release_tag = "",
+ backend_request = "auto",
+ force_cpu = False,
+ )
+ kwargs.update(overrides)
+ return LLAMA.existing_install_current_without_plan(install_dir, **kwargs)
+
+
+def _files_match(install_dir: Path, host = LINUX) -> bool:
+ return LLAMA._runtime_files_match(install_dir, host, LLAMA.load_prebuilt_metadata(install_dir))
+
+
+def _marker(install_dir: Path) -> dict:
+ return json.loads((install_dir / MARKER_NAME).read_text(encoding = "utf-8"))
+
+
+def _rewrite_marker(install_dir: Path, payload: dict) -> None:
+ (install_dir / MARKER_NAME).write_text(json.dumps(payload, indent = 2) + "\n", encoding = "utf-8")
+
+
+def _truncate_half(path: Path) -> None:
+ data = path.read_bytes()
+ assert len(data) > 2
+ path.write_bytes(data[: len(data) // 2])
+
+
+def _truncate_zero(path: Path) -> None:
+ path.write_bytes(b"")
+
+
+def _flip_one_byte(path: Path) -> None:
+ """One bit, same length: the corruption a size check cannot see."""
+ before = path.stat().st_size
+ data = bytearray(path.read_bytes())
+ data[len(data) // 2] ^= 0x01
+ path.write_bytes(bytes(data))
+ assert path.stat().st_size == before, "the flip must preserve the size or it proves nothing"
+
+
+def _delete(path: Path) -> None:
+ path.unlink()
+
+
+def _swap_for_another_binary(path: Path) -> None:
+ """A different WORKING binary of a different size, as a botched manual repair leaves."""
+ before = path.stat().st_size
+ replacement = Path("/bin/sh")
+ if not replacement.is_file():
+ pytest.skip("no /bin/sh to stand in for a different working binary")
+ path.unlink()
+ shutil.copyfile(replacement, path)
+ path.chmod(0o755)
+ assert path.stat().st_size != before
+
+
+_CORRUPTIONS = {
+ "truncate_half": _truncate_half,
+ "truncate_zero": _truncate_zero,
+ "flip_one_byte": _flip_one_byte,
+ "delete": _delete,
+}
+
+# The digest tier on a Linux install: both copies of each binary, since the root and build/bin
+# layouts can rot independently, plus the DiffusionGemma visual server.
+HASHED_TIER = (
+ "llama-server",
+ "llama-quantize",
+ "build/bin/llama-server",
+ "build/bin/llama-quantize",
+ "build/bin/llama-diffusion-gemma-visual-server",
+)
+# Allowlisted payload matched by lib*.so*: recorded size + mtime_ns, no digest.
+SIZE_TIER = ("build/bin/libggml.so", "build/bin/libllama.so")
+
+
+# ── PART 1: the corruption matrix ────────────────────────────────────────────────────
+def test_the_healthy_install_is_accepted_by_the_fast_path(tmp_path, monkeypatch):
+ """The baseline every corruption below is measured against.
+
+ Asserted again inside each corruption test before the damage is applied, so a test
+ that goes green can only have gone green because of the corruption it names.
+ """
+ install_dir = _install(tmp_path, monkeypatch)
+
+ def boom(*_a, **_k):
+ raise AssertionError("the precheck must not reach the GitHub API")
+
+ monkeypatch.setattr(LLAMA, "fetch_release_bundle", boom, raising = False)
+ monkeypatch.setattr(LLAMA, "resolve_release_tag", boom, raising = False)
+ assert _fast_path(install_dir) is True
+ assert _files_match(install_dir) is True
+ recorded = _marker(install_dir)["runtime_files"]
+ for relative in HASHED_TIER:
+ assert recorded[relative].get("sha256"), f"{relative} must carry a digest"
+ for relative in SIZE_TIER:
+ assert "sha256" not in recorded[relative], f"{relative} is the size tier"
+
+
[email protected]("relative", HASHED_TIER)
[email protected]("corruption", sorted(_CORRUPTIONS))
+def test_a_corrupted_recorded_binary_is_rejected(tmp_path, monkeypatch, relative, corruption):
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _fast_path(install_dir) is True
+ _CORRUPTIONS[corruption](install_dir / relative)
+ assert _fast_path(install_dir) is False
+ assert _files_match(install_dir) is False
+
+
[email protected]("relative", HASHED_TIER)
+def test_llama_a_same_size_byte_flip_is_caught_only_by_the_digest(tmp_path, monkeypatch, relative):
+ """The case the whole two-tier record exists for.
+
+ One bit flipped, byte count identical, mode identical, and the file still parses as
+ whatever it was: every structural check in the precheck passes it. Nothing but the
+ recorded sha256 separates this install from the one that was downloaded -- and the
+ proof is the second half, which drops the digest and watches the same flip sail
+ through on size alone.
+ """
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _fast_path(install_dir) is True
+ recorded_size = _marker(install_dir)["runtime_files"][relative]["size"]
+
+ _flip_one_byte(install_dir / relative)
+ assert (install_dir / relative).stat().st_size == recorded_size
+ assert _files_match(install_dir) is False
+ assert _fast_path(install_dir) is False
+
+ payload = _marker(install_dir)
+ payload["runtime_files"][relative].pop("sha256")
+ _rewrite_marker(install_dir, payload)
+ assert _fast_path(install_dir) is True, "size alone cannot see a same-size flip"
+
+
+@POSIX_ONLY
[email protected]("relative", HASHED_TIER)
+def test_a_recorded_binary_replaced_by_a_different_one_is_rejected(tmp_path, monkeypatch, relative):
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _fast_path(install_dir) is True
+ _swap_for_another_binary(install_dir / relative)
+ assert _files_match(install_dir) is False
+ assert _fast_path(install_dir) is False
+
+
+@POSIX_ONLY
[email protected]("relative", HASHED_TIER)
+def test_a_recorded_binary_replaced_by_a_dangling_symlink_is_rejected(
+ tmp_path, monkeypatch, relative
+):
+ """stat() follows the link, so this arrives as ENOENT rather than as a size mismatch;
+ _runtime_files_match's OSError branch is what has to catch it."""
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _fast_path(install_dir) is True
+ target = install_dir / relative
+ target.unlink()
+ target.symlink_to(target.parent / "gone-with-the-install")
+ assert target.is_symlink() and not target.exists()
+ assert _files_match(install_dir) is False
+ assert _fast_path(install_dir) is False
+
+
+@POSIX_ONLY
+@NOT_ROOT
[email protected]("relative", HASHED_TIER)
+def test_an_unreadable_recorded_binary_is_rejected(tmp_path, monkeypatch, relative):
+ """chmod 000 leaves size and mtime intact, so only the hash attempt can fail -- and
+ _runtime_files_match fails CLOSED on it, unlike the payload scans."""
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _fast_path(install_dir) is True
+ target = install_dir / relative
+ target.chmod(0o000)
+ try:
+ assert target.stat().st_size == _marker(install_dir)["runtime_files"][relative]["size"]
+ assert _files_match(install_dir) is False
+ assert _fast_path(install_dir) is False
+ finally:
+ target.chmod(0o755)
+
+
+@POSIX_ONLY
+@NOT_ROOT
[email protected]("relative", ("build/bin/llama-server", "build/bin/llama-quantize"))
+def test_a_non_executable_runtime_binary_is_rejected(tmp_path, monkeypatch, relative):
+ """The bytes are untouched, so the digests still match; os.access(X_OK) on the two
+ runtime-dir binaries is the only thing standing between this and "already matches"."""
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _fast_path(install_dir) is True
+ (install_dir / relative).chmod(0o644)
+ assert _files_match(install_dir) is True, "the record is about bytes, not mode"
+ assert _fast_path(install_dir) is False
+
+
+@POSIX_ONLY
+@NOT_ROOT
+ "relative",
+ ("llama-server", "llama-quantize", "build/bin/llama-diffusion-gemma-visual-server"),
+)
+def test_the_execute_bit_is_checked_on_the_runtime_copies_only(tmp_path, monkeypatch, relative):
+ """A gap, but a PRE-EXISTING one, and this pins both halves of that claim.
+
+ The precheck's os.access(X_OK) covers install_runtime_dir()/llama-{server,quantize}
+ and nothing else, so a root copy or the visual server can lose its execute bit and
+ still read as current. The full re-validation this PR skips did not catch it either:
+ _existing_install_runs probes the root copies, but _binary_image_runs fails OPEN on
+ EACCES (only ENOEXEC and a loader-level crash are verdicts), so it answered True too.
+ Asserting the full path here is what makes "pre-existing" evidence instead of a
+ claim -- and the blast radius is bounded, since llama_server_candidates keeps
+ scanning inside the same layout and finds the executable build/bin copy.
+ """
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _fast_path(install_dir) is True
+ assert LLAMA._existing_install_runs(install_dir, LINUX) is True
+ (install_dir / relative).chmod(0o644)
+ assert _fast_path(install_dir) is True
+ assert LLAMA._existing_install_runs(install_dir, LINUX) is True
+
+
[email protected]("relative", SIZE_TIER)
[email protected]("corruption", ("truncate_half", "truncate_zero", "delete"))
+def test_a_truncated_payload_library_is_rejected(tmp_path, monkeypatch, relative, corruption):
+ """The size tier's whole purpose: _runtime_payload_has globs for EXISTENCE, so before
+ the record a half-written libggml.so -- what a full disk leaves -- passed the payload
+ scan. A stat is enough to catch it."""
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _fast_path(install_dir) is True
+ _CORRUPTIONS[corruption](install_dir / relative)
+ assert _files_match(install_dir) is False
+ assert _fast_path(install_dir) is False
+
+
[email protected]("relative", SIZE_TIER)
+def test_a_same_size_payload_rewrite_is_not_detected(tmp_path, monkeypatch, relative):
+ """The documented edge of the size tier, asserted so it is a decision and not a surprise.
+
+ runtime_file_records hashes three binaries and stats everything else, because hashing
+ 300 MB of CUDA kernels on every update is not worth it. So a same-size rewrite of a
+ shared library is invisible here. Pre-existing in effect: the full re-validation only
+ ever globbed for existence and started llama-server, and a corrupt libggml this file
+ can write does not stop a stub from exiting 0 either -- asserted below.
+ """
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _fast_path(install_dir) is True
+ target = install_dir / relative
+ target.write_bytes(b"\x00" * target.stat().st_size)
+ assert _files_match(install_dir) is True
+ assert _fast_path(install_dir) is True
+ assert LLAMA._existing_install_runs(install_dir, LINUX) is True
+
+
+# ── PART 2: what the fingerprint does and does not cover ─────────────────────────────
+def test_runtime_files_is_not_an_input_to_the_marker_fingerprint(tmp_path, monkeypatch):
+ """Establishes the boundary the rest of Part 2 explores.
+
+ _marker_install_fingerprint reads twelve release-identity keys plus the upstream tag.
+ runtime_files is not among them, so the self-consistency guard that proves the marker
+ "was written whole by this installer" says nothing about the integrity record inside
+ it. Not a defect on its own -- the marker is a local unsigned file, and anyone who can
+ rewrite runtime_files can delete the marker outright, which costs a re-download rather
+ than keeping a broken install -- but it is what makes (a) and (c) below possible.
+ """
+ install_dir = _install(tmp_path, monkeypatch)
+ marker = _marker(install_dir)
+ before = LLAMA._marker_install_fingerprint(marker)
+ assert before == marker["install_fingerprint"]
+
+ marker["runtime_files"] = {}
+ assert LLAMA._marker_install_fingerprint(marker) == before
+ marker["runtime_files"] = {"llama-server": {"size": 1}}
+ assert LLAMA._marker_install_fingerprint(marker) == before
+
+ # A release-identity key, by contrast, is covered.
+ marker["release_tag"] = "release-2"
+ assert LLAMA._marker_install_fingerprint(marker) != before
+
+
+def test_dropping_the_digest_from_one_entry_downgrades_it_to_the_size_tier(tmp_path, monkeypatch):
+ """(a) An entry with size but no sha256 is accepted, and checked on size alone.
+
+ _runtime_files_match treats a missing digest as "this file has none to check" --
+ the same shape the payload tier legitimately writes -- so the tiers are not carried
+ in the marker as a policy, only as the presence of a key. The install is genuinely
+ broken afterwards and the fast path keeps it; what makes that a boundary statement
+ rather than a bug report is that reaching it needs a write to the marker, and the
+ same write could have said anything at all.
+ """
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _fast_path(install_dir) is True
+
+ payload = _marker(install_dir)
+ entry = payload["runtime_files"]["llama-server"]
+ assert entry.pop("sha256")
+ assert entry["size"] > 0
+ _rewrite_marker(install_dir, payload)
+ assert _fast_path(install_dir) is True, "an entry without a digest is still a valid record"
+
+ _flip_one_byte(install_dir / "llama-server")
+ assert _files_match(install_dir) is True
+ assert _fast_path(install_dir) is True
+
+ # ... and the size half of that entry still bites.
+ _truncate_half(install_dir / "llama-server")
+ assert _fast_path(install_dir) is False
+
+
+def test_an_empty_runtime_files_record_fails_closed(tmp_path, monkeypatch):
+ """(b) The record is the evidence that replaces starting the binaries, so no record
+ is not proof of anything."""
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _fast_path(install_dir) is True
+ payload = _marker(install_dir)
+ payload["runtime_files"] = {}
+ _rewrite_marker(install_dir, payload)
+ assert _files_match(install_dir) is False
+ assert _fast_path(install_dir) is False
+
+
[email protected]("record", (None, "", 0, [], "abc", ["size"], 17))
+def test_a_runtime_files_record_that_is_not_a_dict_fails_closed(tmp_path, monkeypatch, record):
+ install_dir = _install(tmp_path, monkeypatch)
+ payload = _marker(install_dir)
+ payload["runtime_files"] = record
+ _rewrite_marker(install_dir, payload)
+ assert _files_match(install_dir) is False
+ assert _fast_path(install_dir) is False
+
+
+def test_a_single_non_dict_entry_fails_the_whole_record_closed(tmp_path, monkeypatch):
+ install_dir = _install(tmp_path, monkeypatch)
+ payload = _marker(install_dir)
+ payload["runtime_files"]["llama-server"] = "nonsense"
+ _rewrite_marker(install_dir, payload)
+ assert _files_match(install_dir) is False
+ assert _fast_path(install_dir) is False
+
+
+def test_a_marker_with_no_runtime_files_key_fails_closed(tmp_path, monkeypatch):
+ """Every marker written before this PR is this shape: it takes the full path once,
+ which re-records it."""
+ install_dir = _install(tmp_path, monkeypatch)
+ payload = _marker(install_dir)
+ payload.pop("runtime_files")
+ _rewrite_marker(install_dir, payload)
+ assert _files_match(install_dir) is False
+ assert _fast_path(install_dir) is False
+
+
[email protected]("relative", ("llama-server", "build/bin/llama-server"))
+def test_deleting_one_entry_leaves_only_that_file_unchecked(tmp_path, monkeypatch, relative):
+ """(c) Removing an entry is not the same as emptying the record.
+
+ The loop iterates what is recorded, and the docstring is explicit that a binary
+ "never recorded is not evidence against the install" -- so a dropped entry leaves
+ that one file unchecked while every other entry still bites. The second half proves
+ the remaining entries are live, which is what makes the first half a scoped statement
+ rather than "the record can be switched off one key at a time".
+ """
+ install_dir = _install(tmp_path, monkeypatch)
+ payload = _marker(install_dir)
+ payload["runtime_files"].pop(relative)
+ _rewrite_marker(install_dir, payload)
+ assert _fast_path(install_dir) is True
+
+ _flip_one_byte(install_dir / relative)
+ assert _files_match(install_dir) is True
+ assert _fast_path(install_dir) is True
+
+ sibling = "llama-quantize" if relative == "llama-server" else "build/bin/llama-quantize"
+ _flip_one_byte(install_dir / sibling)
+ assert _files_match(install_dir) is False
+ assert _fast_path(install_dir) is False
+
+
+# ── PART 3: whisper, which records no payload digests ────────────────────────────────
+WHISPER_LINUX = whisper_host(WHISPER.HostInfo)
+_GGML_TREE = "ggml-tree-aaaa"
+
+
+def _whisper_selection(**overrides):
+ # CORE, not WHISPER.InstallSelection: this file loads its own prebuilt_core instance and
+ # the marker writer accepting it is part of what Part 3 says.
+ return CORE.InstallSelection(**whisper_selection_fields(WHISPER, **overrides))
+
+
+def _whisper_install(
+ tmp_path: Path,
+ monkeypatch,
+ *,
+ slim: bool = False,
+) -> Path:
+ """A whisper install tree plus a marker written by the REAL writer, so the
+ fingerprint the keep path recomputes is genuinely self-consistent."""
+ install_dir = tmp_path / "whisper.cpp"
+ bin_dir = WHISPER.runtime_bin_dir(install_dir, WHISPER_LINUX)
+ bin_dir.mkdir(parents = True)
+ server = WHISPER.installed_server_path(install_dir, WHISPER_LINUX)
+ server.write_bytes(b"#!/bin/sh\necho whisper\nexit 0\n")
+ server.chmod(0o755)
+ (bin_dir / "libwhisper.so.1").write_bytes(b"dummy-libwhisper")
+
+ linked = ("libggml.so.0", "libggml-base.so.0")
+ if slim:
+ for name in linked:
+ (bin_dir / name).write_bytes(b"ggml-" + name.encode("utf-8"))
+ # The live llama marker a slim bundle is wired against; never the real one on this box.
+ monkeypatch.setattr(WHISPER, "installed_llama_ggml_tree", lambda *_a, **_k: _GGML_TREE)
+
+ selection = _whisper_selection(
+ **(
+ dict(
+ install_kind = "slim",
+ paired_llama_tag = "b9001",
+ linked_from = str(tmp_path / "llama.cpp" / "build" / "bin"),
+ linked_libraries = linked,
+ runtime_wiring_version = WHISPER.SLIM_RUNTIME_WIRING_VERSION,
+ linked_runtime_directories = (),
+ )
+ if slim
+ else {}
+ )
+ )
+ WHISPER.write_prebuilt_metadata(install_dir, selection)
+ if slim:
+ assert _whisper_marker(install_dir)["paired_llama_ggml_tree"] == _GGML_TREE
+ return install_dir
+
+
+def _whisper_marker(install_dir: Path) -> dict:
+ return json.loads((install_dir / WHISPER.METADATA_FILENAME).read_text(encoding = "utf-8"))
+
+
+def _whisper_keep(install_dir: Path) -> bool:
+ return whisper_install_is_intact(WHISPER, install_dir, WHISPER_LINUX)
+
+
+def test_whisper_a_healthy_install_is_kept(tmp_path, monkeypatch):
+ install_dir = _whisper_install(tmp_path, monkeypatch)
+ assert WHISPER.installed_tree_is_intact(install_dir, WHISPER_LINUX) is True
+ assert _whisper_keep(install_dir) is True
+
+
+def test_whisper_records_the_digest_of_the_server_it_installed(tmp_path, monkeypatch):
+ """The premise of the next test, asserted rather than assumed.
+
+ asset_sha256 is the digest of the ARCHIVE, checked once at download time against the
+ release's checksum index; it says nothing about what is on disk a month later. The
+ marker now also records the extracted payload -- size + sha256 for whisper-server --
+ so a later no-network re-check has something to ask.
+ """
+ install_dir = _whisper_install(tmp_path, monkeypatch)
+ marker = _whisper_marker(install_dir)
+ assert marker["asset_sha256"] == "c" * 64
+ server = WHISPER.installed_server_path(install_dir, WHISPER_LINUX)
+ relative = server.relative_to(install_dir).as_posix()
+ assert marker["runtime_files"][relative]["sha256"] == CORE.sha256_file(server)
+ assert marker["runtime_files"][relative]["size"] == server.stat().st_size
+
+
+def test_whisper_a_truncated_server_with_bytes_left_is_rejected(tmp_path, monkeypatch):
+ """A whisper-server left half-written by a full disk or an interrupted extract.
+
+ It is still a non-empty executable file, which is all the shape checks can see, so
+ before the payload record this install was KEPT and the failure surfaced when the
+ user pressed the dictation key instead of at update time. The recorded size and
+ digest are what turn it into a re-download, which is the repair the user wanted.
+ """
+ install_dir = _whisper_install(tmp_path, monkeypatch)
+ server = WHISPER.installed_server_path(install_dir, WHISPER_LINUX)
+ assert _whisper_keep(install_dir) is True
+
+ data = server.read_bytes()
+ server.write_bytes(data[: len(data) // 2])
+ assert server.stat().st_size > 0
+ assert os.name == "nt" or os.access(server, os.X_OK)
+ assert WHISPER.installed_tree_is_intact(install_dir, WHISPER_LINUX) is False
+ assert _whisper_keep(install_dir) is False
+
+
+def test_whisper_a_zero_byte_server_is_rejected(tmp_path, monkeypatch):
+ """The one byte-level fact the whisper record can establish: size > 0."""
+ install_dir = _whisper_install(tmp_path, monkeypatch)
+ assert _whisper_keep(install_dir) is True
+ WHISPER.installed_server_path(install_dir, WHISPER_LINUX).write_bytes(b"")
+ assert WHISPER.installed_tree_is_intact(install_dir, WHISPER_LINUX) is False
+ assert _whisper_keep(install_dir) is False
+
+
+def test_whisper_a_missing_server_is_rejected(tmp_path, monkeypatch):
+ install_dir = _whisper_install(tmp_path, monkeypatch)
+ assert _whisper_keep(install_dir) is True
+ WHISPER.installed_server_path(install_dir, WHISPER_LINUX).unlink()
+ assert WHISPER.installed_tree_is_intact(install_dir, WHISPER_LINUX) is False
+ assert _whisper_keep(install_dir) is False
+
+
+@POSIX_ONLY
+@NOT_ROOT
+def test_whisper_a_non_executable_server_is_rejected(tmp_path, monkeypatch):
+ install_dir = _whisper_install(tmp_path, monkeypatch)
+ server = WHISPER.installed_server_path(install_dir, WHISPER_LINUX)
+ assert _whisper_keep(install_dir) is True
+ server.chmod(0o644)
+ assert WHISPER.installed_tree_is_intact(install_dir, WHISPER_LINUX) is False
+ assert _whisper_keep(install_dir) is False
+ server.chmod(0o755)
+ assert _whisper_keep(install_dir) is True
+
+
+def test_whisper_a_slim_install_is_kept_only_while_its_paired_ggml_tree_stands(
+ tmp_path, monkeypatch
+):
+ """A slim bundle ships no ggml of its own: it hardlinks llama's. So "intact" here is
+ a statement about ANOTHER install, and a llama update that moved ggml retires it."""
+ install_dir = _whisper_install(tmp_path, monkeypatch, slim = True)
+ assert _whisper_keep(install_dir) is True
+
+ monkeypatch.setattr(WHISPER, "installed_llama_ggml_tree", lambda *_a, **_k: "ggml-tree-bbbb")
+ assert _whisper_keep(install_dir) is False
+ # The tree on disk is untouched; only the pairing moved.
+ assert WHISPER.installed_tree_is_intact(install_dir, WHISPER_LINUX) is True
+
+ # A llama install that cannot say (predating ggml_tree) is not a licence to keep it.
+ monkeypatch.setattr(WHISPER, "installed_llama_ggml_tree", lambda *_a, **_k: None)
+ assert _whisper_keep(install_dir) is False
+
+
[email protected]("missing", ("libggml.so.0", "libggml-base.so.0"))
+def test_whisper_a_slim_install_missing_a_wired_library_is_rejected(tmp_path, monkeypatch, missing):
+ install_dir = _whisper_install(tmp_path, monkeypatch, slim = True)
+ assert _whisper_keep(install_dir) is True
+ bin_dir = WHISPER.runtime_bin_dir(install_dir, WHISPER_LINUX)
+ (bin_dir / missing).unlink()
+ assert WHISPER.installed_tree_is_intact(install_dir, WHISPER_LINUX) is False
+ assert _whisper_keep(install_dir) is False
+
+
+def test_whisper_a_wired_library_truncated_to_one_byte_is_rejected(tmp_path, monkeypatch):
+ """A hardlinked ggml library left as a stub -- most of a CUDA or ROCm pairing's bytes.
+
+ linked_libraries is checked for PRESENCE by name, so before the payload record this
+ was "intact" and dictation failed at load time with a dynamic linker error. The
+ recorded size catches it for the price of a stat; the paired ggml tree above answers
+ the different question of whether llama's runtime moved out from under it.
+ """
+ install_dir = _whisper_install(tmp_path, monkeypatch, slim = True)
+ bin_dir = WHISPER.runtime_bin_dir(install_dir, WHISPER_LINUX)
+ (bin_dir / "libggml.so.0").write_bytes(b"\x00")
+ assert WHISPER.installed_tree_is_intact(install_dir, WHISPER_LINUX) is False
+ assert _whisper_keep(install_dir) is False
+
+
+# ── PART 4: the marker rewrite as a filesystem operation ─────────────────────────────
[email protected](frozen = True)
+class _Writer:
+ name: str
+ module: Any
+ filename: str
+ rewrite: Callable[[Path, dict], Any]
+ raises_on_failure: bool
+
+
+WRITERS = (
+ _Writer(
+ name = "llama._write_marker",
+ module = LLAMA,
+ filename = MARKER_NAME,
+ rewrite = lambda directory, payload: LLAMA._write_marker(directory / MARKER_NAME, payload),
+ raises_on_failure = False,
+ ),
+ _Writer(
+ name = "prebuilt_core.write_live_marker",
+ module = CORE,
+ filename = MARKER_NAME,
+ rewrite = lambda directory, payload: CORE.write_live_marker(directory / MARKER_NAME, payload),
+ raises_on_failure = True,
+ ),
+ _Writer(
+ name = "node._write_metadata_payload",
+ module = NODE,
+ filename = NODE.METADATA_FILENAME,
+ rewrite = lambda directory, payload: NODE._write_metadata_payload(directory, payload),
+ raises_on_failure = True,
+ ),
+)
+_WRITER_IDS = [writer.name for writer in WRITERS]
+
+# release_tag and tag are rendered in the Studio About tab through /api/system/hardware,
+# so a rewrite that drops them changes what the UI shows.
+_LIVE_PAYLOAD = {
+ "release_tag": "release-1",
+ "tag": "b9001",
+ "version": "24.9.0",
+ "install_fingerprint": "ab" * 32,
+ "nested": {"runtime_files": {"llama-server": {"size": 17}}},
+}
+
+
+def _live_marker(
+ tmp_path: Path,
+ writer: _Writer,
+ *,
+ mode: int = 0o644,
+) -> Path:
+ path = tmp_path / writer.filename
+ path.write_text(
+ json.dumps({"release_tag": "release-0", "tag": "b9000"}) + "\n", encoding = "utf-8"
+ )
+ path.chmod(mode)
+ return path
+
+
+def _temp_siblings(directory: Path) -> list:
+ return sorted(p.name for p in directory.iterdir() if ".tmp-" in p.name)
+
+
[email protected]("writer", WRITERS, ids = _WRITER_IDS)
+def test_a_rewritten_marker_is_valid_json_and_keeps_the_keys_the_ui_reads(tmp_path, writer):
+ _live_marker(tmp_path, writer)
+ writer.rewrite(tmp_path, _LIVE_PAYLOAD)
+ written = json.loads((tmp_path / writer.filename).read_text(encoding = "utf-8"))
+ assert written == _LIVE_PAYLOAD
+ assert written["release_tag"] == "release-1" and written["tag"] == "b9001"
+ assert _temp_siblings(tmp_path) == []
+
+
+@POSIX_ONLY
[email protected]("mode", (0o600, 0o644, 0o664, 0o444))
[email protected]("writer", WRITERS, ids = _WRITER_IDS)
+def test_a_rewritten_marker_keeps_its_mode(tmp_path, writer, mode):
+ """NamedTemporaryFile is 0600 and os.replace keeps the SOURCE file's mode, so without
+ the restore a refresh silently makes a group-shared install's marker private. 0o444
+ is the read-only case the temp-and-replace shape exists to support: a plain write
+ would need the file writable, not the directory."""
+ path = _live_marker(tmp_path, writer, mode = mode)
+ writer.rewrite(tmp_path, _LIVE_PAYLOAD)
+ assert stat.S_IMODE(path.stat().st_mode) == mode
+ assert json.loads(path.read_text(encoding = "utf-8"))["tag"] == "b9001"
+ assert _temp_siblings(tmp_path) == []
+
+
+@POSIX_ONLY
+@NEEDS_CHOWN
+@NOT_ROOT
[email protected]("writer", WRITERS, ids = _WRITER_IDS)
+def test_a_group_shared_marker_keeps_its_group(tmp_path, writer):
+ """A real chown, not a recorded call: the marker is put into a secondary group of
+ this user and has to come back out of the rewrite in that group, because os.replace
+ installs the TEMP file's ownership and NamedTemporaryFile's is the primary group."""
+ groups = [gid for gid in os.getgroups() if gid != os.getegid()]
+ if not groups:
+ pytest.skip("this user is in no secondary group to share a marker with")
+ path = _live_marker(tmp_path, writer)
+ shared = groups[0]
+ try:
+ os.chown(path, -1, shared)
+ except OSError as exc: # pragma: no cover - depends on the mount
+ pytest.skip(f"cannot regroup a file here ({exc})")
+ assert path.stat().st_gid == shared
+
+ writer.rewrite(tmp_path, _LIVE_PAYLOAD)
+ assert path.stat().st_gid == shared
+ assert json.loads(path.read_text(encoding = "utf-8"))["tag"] == "b9001"
+ assert _temp_siblings(tmp_path) == []
+
+
+@NEEDS_CHOWN
[email protected]("writer", WRITERS, ids = _WRITER_IDS)
+def test_every_marker_writer_asks_for_the_owner_then_falls_back_to_the_group(
+ tmp_path, writer, monkeypatch
+):
+ """All three ask for owner AND group first, then group alone, and that uniformity is
+ the point.
+
+ Neither half is sufficient on its own. Owner+group alone is EPERM for a non-root member
+ of a group-shared install -- chown is all-or-nothing -- so the group is silently lost,
+ which is what e8d128d24 fixed in core and node. Group alone is wrong under root, which
+ is exactly when the owner CAN be restored: it leaves the marker owned by root, and an
+ 0600 marker stops being readable by the user who owns the install. The two calls in
+ this order give each caller the best it is permitted. This pins the call shape so the
+ three writers cannot drift apart again.
+ """
+ path = _live_marker(tmp_path, writer)
+ original = path.stat()
+ calls: list = []
+
+ def refusing_chown(target, uid, gid):
+ calls.append((Path(target).name, uid, gid))
+ if uid != -1:
+ raise PermissionError("a non-root member may not give a file away")
+
+ monkeypatch.setattr(writer.module.os, "chown", refusing_chown)
+ writer.rewrite(tmp_path, _LIVE_PAYLOAD)
+ monkeypatch.undo()
+
+ assert [(uid, gid) for _, uid, gid in calls] == [
+ (original.st_uid, original.st_gid),
+ (-1, original.st_gid),
+ ], calls
+ assert all(
+ ".tmp-" in name for name, _, _ in calls
+ ), "ownership must be set on the temp file, before the swap"
+
+
+@NEEDS_CHOWN
[email protected]("writer", WRITERS, ids = _WRITER_IDS)
+def test_a_permitted_writer_restores_the_owner_and_asks_no_further(tmp_path, writer, monkeypatch):
+ """The root case: when the combined call is allowed, the fallback must not run, or the
+ owner just restored would be left in place by luck rather than by intent."""
+ path = _live_marker(tmp_path, writer)
+ original = path.stat()
+ calls: list = []
+
+ monkeypatch.setattr(
+ writer.module.os,
+ "chown",
+ lambda target, uid, gid: calls.append((Path(target).name, uid, gid)),
+ )
+ writer.rewrite(tmp_path, _LIVE_PAYLOAD)
+ monkeypatch.undo()
+
+ assert [(uid, gid) for _, uid, gid in calls] == [(original.st_uid, original.st_gid)], calls
+
+
+@NEEDS_CHOWN
+@NOT_ROOT
[email protected]("writer", WRITERS, ids = _WRITER_IDS)
+def test_a_marker_owned_by_another_user_still_keeps_its_group(tmp_path, writer, monkeypatch):
+ """The case uid -1 exists for, on the install it matters for.
+
+ A group-shared install whose marker is owned by the admin who ran setup, refreshed by
+ a member of the group: chown(uid, gid) is EPERM outright there (you may not give a
+ file away) while chown(-1, gid) succeeds, and since the failure is swallowed the
+ difference is not an error but a marker that quietly changes group -- on an 0640
+ marker, the mode restore beside it undone. The kernel rule is simulated, since a test
+ cannot own a file as another user; the simulation refuses exactly what POSIX refuses.
+ """
+ path = _live_marker(tmp_path, writer, mode = 0o640)
+ other_uid = os.geteuid() + 1
+ shared_gid = os.getegid() + 1
+ applied: list = []
+
+ def kernel_chown(target, uid, gid):
+ # POSIX: only root may change a file's owner; -1 means "leave it".
+ if uid not in (-1, os.geteuid()):
+ raise PermissionError(errno.EPERM, "Operation not permitted")
+ applied.append((uid, gid))
+
+ real_stat = os.stat
+ monkeypatch.setattr(writer.module.os, "chown", kernel_chown)
+ monkeypatch.setattr(
+ writer.module.os,
+ "stat",
+ lambda *a, **k: _StatWithOwner(real_stat(*a, **k), other_uid, shared_gid),
+ )
+ writer.rewrite(tmp_path, _LIVE_PAYLOAD)
+ # Narrow window: os.stat is the whole interpreter's, so it is restored immediately.
+ monkeypatch.undo()
+ assert json.loads(path.read_text(encoding = "utf-8"))["tag"] == "b9001"
+ assert applied == [(-1, shared_gid)], "the group must survive a marker owned by someone else"
+
+
+class _StatWithOwner:
+ """An os.stat_result with st_uid/st_gid overridden, so a marker can stand in for one
+ owned by another user without needing root to create it."""
+
+ def __init__(self, base, uid: int, gid: int) -> None:
+ self._base = base
+ self.st_uid = uid
+ self.st_gid = gid
+
+ def __getattr__(self, name):
+ return getattr(self._base, name)
+
+
+@NEEDS_CHOWN
[email protected]("writer", WRITERS, ids = _WRITER_IDS)
+def test_a_chown_that_is_refused_does_not_abort_the_write(tmp_path, writer, monkeypatch):
+ """Ownership is best effort; the refreshed marker is not. Declining to write because
+ the group could not be restored would leave the field the refresh exists to record
+ (a deliberate --force-cpu, a re-probed version) unrecorded."""
+ path = _live_marker(tmp_path, writer, mode = 0o640)
+
+ def refuse(*_a, **_k):
+ raise PermissionError(errno.EPERM, "Operation not permitted")
+
+ monkeypatch.setattr(writer.module.os, "chown", refuse)
+ writer.rewrite(tmp_path, _LIVE_PAYLOAD)
+ assert json.loads(path.read_text(encoding = "utf-8")) == _LIVE_PAYLOAD
+ if not WINDOWS_HOST:
+ assert stat.S_IMODE(path.stat().st_mode) == 0o640
+ assert _temp_siblings(tmp_path) == []
+
+
[email protected]("writer", WRITERS, ids = _WRITER_IDS)
+def test_a_platform_with_no_os_chown_at_all_still_writes(tmp_path, writer, monkeypatch):
+ """Windows: os.chown does not exist. Each writer catches AttributeError beside OSError
+ for exactly this, and the rewrite has to complete anyway."""
+ path = _live_marker(tmp_path, writer)
+ monkeypatch.delattr(writer.module.os, "chown", raising = False)
+ assert not hasattr(writer.module.os, "chown")
+ writer.rewrite(tmp_path, _LIVE_PAYLOAD)
+ assert json.loads(path.read_text(encoding = "utf-8")) == _LIVE_PAYLOAD
+ assert _temp_siblings(tmp_path) == []
+
+
[email protected]("writer", WRITERS, ids = _WRITER_IDS)
+def test_a_replace_that_fails_leaves_the_previous_marker_whole(tmp_path, writer, monkeypatch):
+ """The reason for temp-and-replace at all.
+
+ An in-place rewrite truncates first, so an ENOSPC or an I/O error mid-write strands a
+ partial UNSLOTH_*_INFO.json -- and load_prebuilt_metadata reads an unparseable marker
+ as "nothing installed", retiring an install that is perfectly fine. Here the swap
+ itself fails: the previous marker must be byte-identical afterwards, and no .tmp-*
+ may survive inside the install directory, where _swap_into_place would carry it into
+ the live tree.
+ """
+ path = _live_marker(tmp_path, writer)
+ before = path.read_bytes()
+
+ def boom(*_a, **_k):
+ raise OSError(errno.EIO, "the disk went away mid-swap")
+
+ monkeypatch.setattr(writer.module, "atomic_replace_from_tempfile", boom)
+ if writer.raises_on_failure:
+ with pytest.raises(OSError):
+ writer.rewrite(tmp_path, _LIVE_PAYLOAD)
+ else:
+ assert writer.rewrite(tmp_path, _LIVE_PAYLOAD) is False
+
+ assert path.read_bytes() == before
+ assert json.loads(path.read_text(encoding = "utf-8"))["tag"] == "b9000"
+ assert _temp_siblings(tmp_path) == []
+
+
[email protected]("writer", WRITERS, ids = _WRITER_IDS)
+def test_a_write_that_fails_before_the_swap_strands_no_temp_file(tmp_path, writer, monkeypatch):
+ """The ENOSPC this shape is built to tolerate, raised from the write itself: the temp
+ path is tracked outside the try precisely so it can still be removed."""
+ path = _live_marker(tmp_path, writer)
+ before = path.read_bytes()
+
+ def boom(_fd):
+ raise OSError(errno.ENOSPC, "No space left on device")
+
+ monkeypatch.setattr(writer.module.os, "fsync", boom)
+ if writer.raises_on_failure:
+ with pytest.raises(OSError):
+ writer.rewrite(tmp_path, _LIVE_PAYLOAD)
+ else:
+ assert writer.rewrite(tmp_path, _LIVE_PAYLOAD) is False
+ monkeypatch.undo()
+
+ assert path.read_bytes() == before
+ assert _temp_siblings(tmp_path) == []
+
+
+def test_the_llama_marker_survives_a_rewrite_and_the_fast_path_still_accepts_it(
+ tmp_path, monkeypatch
+):
+ """End to end: the real marker of a real install, through the real rewriter.
+
+ The reuse path rewrites a marker that is already in service, so the rewrite must
+ preserve every field the precheck reads back -- the fingerprint inputs, the
+ runtime_files record, and the release_tag/tag pair the About tab renders.
+ """
+ install_dir = _install(tmp_path, monkeypatch)
+ marker_path = install_dir / MARKER_NAME
+ marker_path.chmod(0o640)
+ before = _marker(install_dir)
+ assert _fast_path(install_dir) is True
+
+ marker = dict(before)
+ marker["force_cpu"] = False
+ assert LLAMA._write_marker(marker_path, marker) is True
+
+ after = _marker(install_dir)
+ assert after == marker
+ assert after["release_tag"] == before["release_tag"] == "release-1"
+ assert after["tag"] == before["tag"] == "b9001"
+ assert after["runtime_files"] == before["runtime_files"]
+ assert LLAMA._marker_install_fingerprint(after) == after["install_fingerprint"]
+ if not WINDOWS_HOST:
+ assert stat.S_IMODE(marker_path.stat().st_mode) == 0o640
+ assert _temp_siblings(install_dir) == []
+ assert _fast_path(install_dir) is True
diff --git a/tests/studio/install/test_pr10648_legacy_marker_compat.py b/tests/studio/install/test_pr10648_legacy_marker_compat.py
new file mode 100644
index 00000000000..f67f3dd90e3
--- /dev/null
+++ b/tests/studio/install/test_pr10648_legacy_marker_compat.py
@@ -0,0 +1,948 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Old installs meeting the marker fast paths, and new markers meeting old installers.
+
+``studio update`` now skips re-validating a prebuilt llama.cpp / whisper.cpp / Node tree
+when the marker on disk carries enough evidence to say the install is already the one this
+run would produce. Every user upgrading into that code has a marker written by an Unsloth
+that recorded none of it, so the two directions this pins are:
+
+ * BACKWARDS. A marker written by a released, pre-fast-path Unsloth must ALWAYS fall
+ through to the full path -- the fast path may not read an absent key as agreement --
+ and it must fall through exactly ONCE, because the full path backfills the missing
+ evidence onto the marker it kept. An old install that never backfills pays the full
+ re-validation (13-63 s on macOS) on every update forever, which is the bug the fast
+ path exists to fix; an old install that is WRONGLY trusted keeps a tree this run
+ would have replaced, which is worse.
+ * FORWARDS. A user who downgrades, or who runs an older Studio out of another
+ checkout, hands a released installer a marker full of keys it has never heard of.
+ The new keys are additive, so the old reader must ignore them and still keep the
+ install rather than refuse and re-download it.
+
+The old markers here are not hand-written shapes. The released tags in this repository
+are checked out read-only with ``git show``, loaded in a subprocess whose ``PYTHONPATH``
+holds nothing but that tag's own modules, and asked to write a marker with their OWN
+writer -- so what the current readers are fed is the exact bytes that shipped. The
+hand-written corpus is still covered: ``ALL_SHAPES`` from
+``test_keep_install_backcompat_9979`` is imported rather than re-derived, so the twelve
+historical shapes and one real captured install are asserted here too.
+
+Linux-only in what it touches: platforms are simulated through ``HostInfo`` (never
+through ``os.name``, which changes pathlib underneath the install trees), no network,
+no GPU. The subprocesses are read-only against the repository and write only under
+``tmp_path``.
+"""
+
+import dataclasses
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+_TEST_DIR = Path(__file__).resolve().parent
+if str(_TEST_DIR) not in sys.path:
+ # The canonical corpus lives beside this file; pytest's prepend import mode puts this
+ # directory on sys.path too, but not necessarily before this module body runs.
+ sys.path.insert(0, str(_TEST_DIR))
+
+from _pr10648_helpers import PACKAGE_ROOT, git, llama_host # noqa: E402
+from _pr10648_helpers import load_studio_module as _load # noqa: E402
+
+import test_keep_install_backcompat_9979 as CORPUS # noqa: E402
+
+ILP = _load("studio_install_llama_prebuilt_pr10648_legacy", "install_llama_prebuilt.py")
+WSP = _load("studio_install_whisper_prebuilt_pr10648_legacy", "install_whisper_prebuilt.py")
+NDP = _load("studio_install_node_prebuilt_pr10648_legacy", "install_node_prebuilt.py")
+# The core instance whisper actually calls into, not a second copy of it.
+CORE = WSP.core
+
+
+# ── The released tags whose markers are replayed here ──
+LEGACY_TAGS = (
+ "v0.1.800-beta",
+ "v0.1.802-beta",
+ "v0.1.804-beta",
+ "v0.1.806-beta",
+ "v0.1.808-beta",
+)
+
+_LEGACY_MODULES = (
+ "install_llama_prebuilt.py",
+ "install_whisper_prebuilt.py",
+ "install_node_prebuilt.py",
+ "prebuilt_core.py",
+)
+
+_SENTINEL = "@@PR10648-RESULT@@"
+
+# Runs inside the legacy subprocess. Everything it needs is introspected from the tag's own
+# dataclasses, so a tag whose fields differ still gets a marker its own writer produced.
+_LEGACY_DRIVER = r'''
+import dataclasses
+import json
+import sys
+import tempfile
+from pathlib import Path
+
+SENTINEL = "@@PR10648-RESULT@@"
+
+
+def fill(cls, wanted):
+ """Build *cls* from the subset of *wanted* this tag's dataclass actually declares."""
+ kwargs = {}
+ for field in dataclasses.fields(cls):
+ if field.name in wanted:
+ kwargs[field.name] = wanted[field.name]
+ elif (
+ field.default is dataclasses.MISSING
+ and field.default_factory is dataclasses.MISSING
+ ):
+ kwargs[field.name] = None
+ return cls(**kwargs)
+
+
+def read(path):
+ text = Path(path).read_text(encoding="utf-8")
+ return {"text": text, "marker": json.loads(text)}
+
+
+def selection_class(whisper, core):
+ return getattr(whisper, "InstallSelection", None) or core.InstallSelection
+
+
+def write_markers(spec):
+ import install_llama_prebuilt as llama
+ import install_node_prebuilt as node
+ import install_whisper_prebuilt as whisper
+ import prebuilt_core as core
+
+ out = {}
+ root = Path(tempfile.mkdtemp())
+
+ llama_dir = root / "llama.cpp"
+ llama_dir.mkdir()
+ llama.write_prebuilt_metadata(
+ llama_dir,
+ requested_tag=spec["llama"]["requested_tag"],
+ llama_tag=spec["llama"]["upstream_tag"],
+ release_tag=spec["llama"]["release_tag"],
+ choice=fill(llama.AssetChoice, spec["llama"]["choice"]),
+ approved_checksums=fill(llama.ApprovedReleaseChecksums, spec["llama"]["checksums"]),
+ prebuilt_fallback_used=False,
+ backend_request=spec["llama"]["backend_request"],
+ )
+ out["llama"] = read(llama_dir / "UNSLOTH_PREBUILT_INFO.json")
+
+ whisper_dir = root / "whisper.cpp"
+ whisper_dir.mkdir()
+ whisper.write_prebuilt_metadata(
+ whisper_dir, fill(selection_class(whisper, core), spec["whisper"]["selection"])
+ )
+ out["whisper"] = read(whisper.metadata_path(whisper_dir))
+
+ node_dir = root / "node"
+ node_dir.mkdir()
+ node.write_metadata(node_dir, **spec["node"]["metadata"])
+ out["node"] = read(node.metadata_path(node_dir))
+ return out
+
+
+def read_markers(spec):
+ import install_llama_prebuilt as llama
+ import install_node_prebuilt as node
+ import install_whisper_prebuilt as whisper
+ import prebuilt_core as core
+
+ out = {"readers": []}
+
+ llama_dir = Path(spec["llama"]["install_dir"])
+ llama_host = fill(llama.HostInfo, spec["llama"]["host"])
+ llama_marker = llama.load_prebuilt_metadata(llama_dir)
+ out["llama_marker_keys"] = sorted(llama_marker)
+ out["llama_backend_request"] = llama.persisted_backend_request(llama_dir)
+ out["readers"].append("llama.persisted_backend_request")
+ for name in ("marker_backend",):
+ reader = getattr(llama, name, None)
+ if reader is not None:
+ out["llama_backend"] = reader(llama_marker)
+ out["readers"].append("llama." + name)
+ for name in ("_install_tree_is_usable", "_kept_install_payload_is_healthy"):
+ reader = getattr(llama, name, None)
+ if reader is not None:
+ out["llama" + name] = reader(llama_dir, llama_host)
+ out["readers"].append("llama." + name)
+
+ whisper_dir = Path(spec["whisper"]["install_dir"])
+ whisper_host = fill(whisper.HostInfo, spec["whisper"]["host"])
+ out["whisper_marker_keys"] = sorted(whisper.load_prebuilt_metadata(whisper_dir))
+ out["whisper_matches"] = whisper.existing_install_matches(
+ whisper_dir,
+ whisper_host,
+ fill(selection_class(whisper, core), spec["whisper"]["selection"]),
+ )
+ out["readers"].append("whisper.existing_install_matches")
+
+ node_dir = Path(spec["node"]["install_dir"])
+ node_host = fill(node.HostInfo, spec["node"]["host"])
+ # The two spawns an old installer would do; stubbed so this stays offline and
+ # hardware-free. What is under test is the marker reading either side of them.
+ node.installed_node_version = lambda *a, **k: spec["node"]["version"]
+ node.installed_npm_major = lambda *a, **k: spec["node"]["npm_major"]
+ out["node_marker_keys"] = sorted(node.load_metadata(node_dir))
+ out["node_matches"] = node.existing_install_matches(
+ node_dir,
+ node_host,
+ version=spec["node"]["version"],
+ expected_sha=spec["node"]["sha256"],
+ )
+ out["readers"].append("node.existing_install_matches")
+ return out
+
+
+def main():
+ request = json.loads(sys.argv[1])
+ handler = {"write_markers": write_markers, "read_markers": read_markers}[request["op"]]
+ sys.stdout.write(SENTINEL + json.dumps(handler(request["spec"]), default=str) + "\n")
+
+
+main()
+'''
+
+
+def _extract_legacy_tree(tag: str, destination: Path) -> "Path | None":
+ """``git show`` a released tag's installers into *destination*, or None if it is absent.
+
+ The four installer modules plus the ``backend.utils.prebuilt`` package they import at
+ module scope -- roughly 600 KB, against 46 MB for the whole ``studio`` tree.
+ """
+ destination.mkdir(parents = True, exist_ok = True)
+ wanted = [f"studio/{name}" for name in _LEGACY_MODULES]
+ wanted += ["studio/backend/__init__.py", "studio/backend/utils/__init__.py"]
+ listing = git("ls-tree", "-r", "--name-only", tag, "studio/backend/utils/prebuilt")
+ if listing.returncode != 0:
+ return None
+ wanted += [
+ line for line in listing.stdout.decode("utf-8", "replace").split() if line.endswith(".py")
+ ]
+ for path in wanted:
+ blob = git("show", f"{tag}:{path}")
+ if blob.returncode != 0:
+ return None
+ target = destination / Path(path).relative_to("studio")
+ target.parent.mkdir(parents = True, exist_ok = True)
+ target.write_bytes(blob.stdout)
+ return destination
+
+
+class LegacyRunFailed(RuntimeError):
+ pass
+
+
+def _run_legacy(tree: Path, op: str, spec: dict) -> dict:
+ """Run the driver against one legacy tree, with only that tree importable."""
+ environment = dict(os.environ)
+ # Replaced, not prepended: the caller's PYTHONPATH points at the CURRENT studio/, and
+ # inheriting it would let the legacy modules import today's prebuilt_core.
+ environment["PYTHONPATH"] = str(tree)
+ environment["PYTHONDONTWRITEBYTECODE"] = "1"
+ proc = subprocess.run(
+ [sys.executable, "-c", _LEGACY_DRIVER, json.dumps({"op": op, "spec": spec})],
+ cwd = str(tree),
+ env = environment,
+ capture_output = True,
+ text = True,
+ timeout = 600,
+ check = False,
+ )
+ for line in proc.stdout.splitlines():
+ if line.startswith(_SENTINEL):
+ return json.loads(line[len(_SENTINEL) :])
+ raise LegacyRunFailed(
+ f"legacy {op} failed (exit {proc.returncode})\n"
+ f"--- stdout ---\n{proc.stdout[-2000:]}\n--- stderr ---\n{proc.stderr[-2000:]}"
+ )
+
+
+# ── The install this scenario describes, in each component's dialect ──
+LLAMA_REPO = "unslothai/llama.cpp"
+LLAMA_UPSTREAM_TAG = "b10698"
+LLAMA_RELEASE_TAG = "b10698-mix-67dfc8b"
+LLAMA_ASSET = "app-b10698-mix-67dfc8b-linux-x64-vulkan.tar.gz"
+LLAMA_GGML_TREE = "0034c6eb"
+# Vulkan on a GPU-less Linux box: no CUDA runtime scan, no ROCm probe and no torch
+# preference read, so host_profile is a pure function of the HostInfo constructed here.
+LLAMA_CHOICE = {
+ "repo": LLAMA_REPO,
+ "tag": LLAMA_UPSTREAM_TAG,
+ "name": LLAMA_ASSET,
+ "url": f"https://example.invalid/{LLAMA_ASSET}",
+ "source_label": "published",
+ "expected_sha256": "d4" * 32,
+ "install_kind": "linux-vulkan",
+ "bundle_profile": "vulkan",
+ "runtime_line": None,
+ "coverage_class": None,
+ "is_ready_bundle": True,
+ "supported_sms": [],
+ "mapped_targets": [],
+ "runtime_name": None,
+ "runtime_url": None,
+ "runtime_sha256": None,
+ "gfx_target": None,
+}
+LLAMA_CHECKSUMS = {
+ "repo": LLAMA_REPO,
+ "release_tag": LLAMA_RELEASE_TAG,
+ "upstream_tag": LLAMA_UPSTREAM_TAG,
+ "source_commit": "0" * 40,
+ "ggml_tree": LLAMA_GGML_TREE,
+ "artifacts": {},
+}
+
+WHISPER_REPO = "unslothai/whisper.cpp"
+WHISPER_RELEASE_TAG = "v1.9.1-unsloth.17"
+WHISPER_UPSTREAM_TAG = "v1.9.1"
+WHISPER_ASSET = "whisper-v1.9.1-linux-x64-cpu.tar.gz"
+WHISPER_SELECTION = {
+ "published_repo": WHISPER_REPO,
+ "release_tag": WHISPER_RELEASE_TAG,
+ "upstream_tag": WHISPER_UPSTREAM_TAG,
+ "source_commit": "1" * 40,
+ "asset": WHISPER_ASSET,
+ "asset_sha256": "ab" * 32,
+ "backend": "cpu",
+ "runtime_line": None,
+ "coverage": {},
+ "studio_protocol": "inference/multipart-v1",
+}
+# platform_os/platform_arch are new here; a legacy tag's InstallSelection drops them.
+CURRENT_WHISPER_SELECTION = {**WHISPER_SELECTION, "platform_os": "linux", "platform_arch": "x64"}
+
+NODE_VERSION = "22.20.0"
+NODE_ASSET = "node-v22.20.0-linux-x64.tar.xz"
+NODE_SHA256 = "ef" * 32
+
+
+def _fill(cls, wanted: dict):
+ """Construct *cls* from the fields it declares, as the legacy driver does."""
+ kwargs = {}
+ for field in dataclasses.fields(cls):
+ if field.name in wanted:
+ kwargs[field.name] = wanted[field.name]
+ elif field.default is dataclasses.MISSING and field.default_factory is dataclasses.MISSING:
+ kwargs[field.name] = None
+ return cls(**kwargs)
+
+
+LINUX = llama_host(ILP.HostInfo)
+# Spelled out again as a plain dict: the legacy subprocess builds its own tag's HostInfo by
+# field name, so this is data crossing a process boundary rather than a second factory.
+LLAMA_HOST_KWARGS = {
+ "system": "Linux",
+ "machine": "x86_64",
+ "is_windows": False,
+ "is_linux": True,
+ "is_macos": False,
+ "is_x86_64": True,
+ "is_arm64": False,
+ "nvidia_smi": None,
+ "driver_cuda_version": None,
+ "compute_caps": [],
+ "visible_cuda_devices": None,
+ "has_physical_nvidia": False,
+ "has_usable_nvidia": False,
+}
+WHISPER_HOST_KWARGS = {
+ "system": "Linux",
+ "machine": "x86_64",
+ "whisper_os": "linux",
+ "whisper_arch": "x64",
+ "archive_ext": ".tar.gz",
+ "is_windows": False,
+ "is_macos": False,
+ "is_apple_silicon": False,
+}
+NODE_HOST_KWARGS = {
+ "system": "Linux",
+ "machine": "x86_64",
+ "node_os": "linux",
+ "node_arch": "x64",
+ "archive_ext": ".tar.gz",
+ "is_windows": False,
+}
+WHISPER_HOST = _fill(WSP.HostInfo, WHISPER_HOST_KWARGS)
+NODE_HOST = _fill(NDP.HostInfo, NODE_HOST_KWARGS)
+
+MARKER_SPEC = {
+ "llama": {
+ "requested_tag": "latest",
+ "upstream_tag": LLAMA_UPSTREAM_TAG,
+ "release_tag": LLAMA_RELEASE_TAG,
+ "backend_request": "auto",
+ "choice": LLAMA_CHOICE,
+ "checksums": LLAMA_CHECKSUMS,
+ },
+ "whisper": {"selection": WHISPER_SELECTION},
+ "node": {"metadata": {"version": NODE_VERSION, "asset": NODE_ASSET, "sha256": NODE_SHA256}},
+}
+
+
[email protected](scope = "session")
+def legacy_markers(tmp_path_factory) -> dict:
+ """One marker per component, per released tag, written by that tag's own code."""
+ if git("rev-parse", "--git-dir").returncode != 0:
+ pytest.skip("not a git checkout, so the released tags cannot be read")
+ root = tmp_path_factory.mktemp("pr10648-legacy")
+ produced: dict = {}
+ for tag in LEGACY_TAGS:
+ tree = _extract_legacy_tree(tag, root / tag)
+ if tree is None:
+ produced[tag] = {"error": f"{tag} does not carry the installer modules"}
+ continue
+ try:
+ produced[tag] = {
+ "tree": tree,
+ "markers": _run_legacy(tree, "write_markers", MARKER_SPEC),
+ }
+ except (LegacyRunFailed, subprocess.SubprocessError) as exc:
+ produced[tag] = {"error": str(exc)}
+ if not any("markers" in entry for entry in produced.values()):
+ pytest.skip("no released tag could be loaded standalone; see the per-tag skips")
+ return produced
+
+
+def _legacy(legacy_markers: dict, tag: str) -> dict:
+ entry = legacy_markers[tag]
+ if "markers" not in entry:
+ pytest.skip(f"{tag} could not be loaded standalone: {entry['error']}")
+ return entry
+
+
[email protected](autouse = True)
+def _offline(monkeypatch):
+ """No fast-path assertion here may be bought with a network call.
+
+ Every scenario pins the release tag, which is the shape of the check that resolves
+ without any lookup; anything reaching a resolver is a test that stopped testing the
+ fast path. Also clears the full-check escape hatch, which would turn every fast path
+ below into an unconditional False and make the suite pass vacuously.
+ """
+
+ def refuse(*args, **kwargs):
+ raise AssertionError("the marker fast path must not reach the network")
+
+ monkeypatch.delenv("UNSLOTH_PREBUILT_FULL_CHECK", raising = False)
+ for module in (ILP, WSP.llama, CORE):
+ for name in (
+ "_download_host_latest_release_tag",
+ "_api_newest_release_tag",
+ "github_releases",
+ "fetch_json",
+ "urlopen",
+ ):
+ if hasattr(module, name):
+ monkeypatch.setattr(module, name, refuse)
+
+
+def _llama_install(root: Path, marker):
+ """A healthy published Vulkan tree, with *marker* written verbatim."""
+ return CORPUS.build_install(root, host = LINUX, marker = marker, payload_backend = "vulkan")
+
+
+def _whisper_install(root: Path, marker_text: "str | None"):
+ install_dir = root / "whisper.cpp"
+ bin_dir = install_dir / "build" / "bin"
+ bin_dir.mkdir(parents = True)
+ server = bin_dir / "whisper-server"
+ server.write_text("#!/bin/sh\nexit 0\n", encoding = "utf-8")
+ os.chmod(server, 0o755)
+ if marker_text is not None:
+ WSP.metadata_path(install_dir).write_text(marker_text, encoding = "utf-8")
+ return install_dir
+
+
+def _node_install(root: Path, marker_text: "str | None"):
+ install_dir = root / "node"
+ node_binary = NDP.node_binary_path(install_dir, NODE_HOST)
+ node_binary.parent.mkdir(parents = True)
+ node_binary.write_text("#!/bin/sh\nexit 0\n", encoding = "utf-8")
+ os.chmod(node_binary, 0o755)
+ npm_cli = NDP.npm_cli_path(install_dir, NODE_HOST)
+ npm_cli.parent.mkdir(parents = True)
+ npm_cli.write_text("// npm-cli\n", encoding = "utf-8")
+ if marker_text is not None:
+ NDP.metadata_path(install_dir).write_text(marker_text, encoding = "utf-8")
+ return install_dir
+
+
+def _llama_route(
+ host = LINUX,
+ published_repo = LLAMA_REPO,
+ release_tag = LLAMA_RELEASE_TAG,
+):
+ return ILP.BackendRoute(
+ backend = "auto",
+ host = host,
+ published_repo = published_repo,
+ published_release_tag = release_tag,
+ persist_llama_backend = None,
+ persist_rocm_gfx = None,
+ )
+
+
+def _llama_fast_path(
+ install_dir: Path,
+ host = LINUX,
+ backend_request = "auto",
+ published_repo = LLAMA_REPO,
+ release_tag = LLAMA_RELEASE_TAG,
+) -> bool:
+ """The pre-check as ``install_prebuilt`` calls it, with the release pinned.
+
+ A pinned release is what ``_expected_release_tag_without_plan`` answers outright, so
+ nothing here resolves anything: the verdict is decided purely by the marker and the
+ tree, which is what these tests are about.
+ """
+ return ILP.existing_install_current_without_plan(
+ install_dir,
+ llama_tag = "latest",
+ published_repo = published_repo,
+ published_release_tag = release_tag,
+ backend_request = backend_request,
+ force_cpu = False,
+ route = _llama_route(host, published_repo, release_tag),
+ )
+
+
+def _current_llama_install(root: Path):
+ """A tree whose marker this very code wrote: the fast path's own best case."""
+ install_dir = _llama_install(root, marker = None)
+ ILP.write_prebuilt_metadata(
+ install_dir,
+ host = LINUX,
+ requested_tag = "latest",
+ llama_tag = LLAMA_UPSTREAM_TAG,
+ release_tag = LLAMA_RELEASE_TAG,
+ choice = _llama_choice(),
+ approved_checksums = _fill(ILP.ApprovedReleaseChecksums, LLAMA_CHECKSUMS),
+ prebuilt_fallback_used = False,
+ backend_request = "auto",
+ )
+ return install_dir
+
+
+def _rewrite_marker(marker_path: Path, marker: dict) -> None:
+ marker_path.write_text(json.dumps(marker, indent = 2) + "\n", encoding = "utf-8")
+
+
+def _whisper_fast_path(install_dir: Path) -> bool:
+ return WSP.existing_install_current_without_plan(
+ install_dir,
+ WHISPER_HOST,
+ whisper_tag = "latest",
+ published_repo = WHISPER_REPO,
+ published_release_tag = WHISPER_RELEASE_TAG,
+ requested_backend = "cpu",
+ )
+
+
+def _llama_choice():
+ return _fill(ILP.AssetChoice, LLAMA_CHOICE)
+
+
+def _whisper_selection():
+ return _fill(CORE.InstallSelection, CURRENT_WHISPER_SELECTION)
+
+
+# ── Backwards compatibility: an old marker is never trusted ──
[email protected]("tag", LEGACY_TAGS)
+def test_a_llama_marker_from_a_released_unsloth_never_takes_the_fast_path(
+ tmp_path, legacy_markers, tag
+):
+ """Upgrading into this code with a llama.cpp install from any shipped release.
+
+ The marker was written by that release's own ``write_prebuilt_metadata`` and records
+ no ``host_profile``, no ``runtime_files`` and no ``runtime_sha256``. Reading any of
+ those absences as agreement would keep a tree this run never verified.
+ """
+ marker = _legacy(legacy_markers, tag)["markers"]["llama"]
+ assert "host_profile" not in marker["marker"], tag
+ assert "runtime_files" not in marker["marker"], tag
+ install_dir = _llama_install(tmp_path, marker["text"])
+ assert _llama_fast_path(install_dir) is False
+
+
+ ("name", "marker", "backend"),
+ CORPUS.ALL_SHAPES,
+ ids = [shape[0] for shape in CORPUS.ALL_SHAPES],
+)
+def test_no_shipped_llama_marker_shape_reaches_the_fast_path(tmp_path, name, marker, backend):
+ """The twelve marker shapes that have shipped, plus one captured from a real install.
+
+ ``UNSLOTH_PREBUILT_INFO.json`` is append-only with no version field, so "the key is
+ missing" is the only signal an old shape gives. Each of these describes a tree that is
+ perfectly healthy -- the full path keeps it -- so the fast path refusing them is the
+ ONLY thing forcing the one re-validation that backfills the new evidence.
+ """
+ install_dir = CORPUS.build_install(tmp_path, host = LINUX, marker = marker, payload_backend = backend)
+ assert ILP._kept_install_payload_is_healthy(install_dir, LINUX) is True, name
+ # The shape's own release and repo, so the tag comparison AGREES and the verdict turns
+ # on the evidence the shape does not carry rather than on a release mismatch.
+ assert (
+ _llama_fast_path(
+ install_dir,
+ published_repo = marker.get("published_repo") or LLAMA_REPO,
+ release_tag = marker.get("release_tag") or LLAMA_RELEASE_TAG,
+ )
+ is False
+ ), name
+
+
[email protected]("tag", LEGACY_TAGS)
+def test_a_whisper_marker_from_a_released_unsloth_never_takes_the_fast_path(
+ tmp_path, legacy_markers, tag
+):
+ """Upgrading into this code with a whisper.cpp install from any shipped release.
+
+ ``fingerprint_coverage`` is what lets a no-network check recompute the fingerprint and
+ tell a whole marker from a ``release_tag`` edited over an old binary. No released
+ marker carries it, so all of them must recompute to None and take the full path.
+ """
+ marker = _legacy(legacy_markers, tag)["markers"]["whisper"]
+ assert "fingerprint_coverage" not in marker["marker"], tag
+ assert CORE.marker_install_fingerprint(marker["marker"]) is None, tag
+ install_dir = _whisper_install(tmp_path, marker["text"])
+ assert _whisper_fast_path(install_dir) is False
+
+
[email protected]("tag", LEGACY_TAGS)
+def test_a_node_marker_from_a_released_unsloth_never_skips_the_version_probe(
+ tmp_path, legacy_markers, tag
+):
+ """Upgrading into this code with a managed Node runtime from any shipped release.
+
+ ``node_version_checked`` is the record that stands in for spawning a 110 MB
+ interpreter. No released marker has one, so the recorded-runtime fast path must
+ decline every one of them and let the spawn happen.
+ """
+ marker = _legacy(legacy_markers, tag)["markers"]["node"]
+ assert "node_version_checked" not in marker["marker"], tag
+ install_dir = _node_install(tmp_path, marker["text"])
+ meta = NDP.load_metadata(install_dir)
+ assert NDP._recorded_runtime_matches(install_dir, NODE_HOST, meta, NODE_VERSION) is False
+
+
+# ── Each new key on its own is what an old marker is missing ──
[email protected]("key", ["host_profile", "runtime_files", "runtime_sha256"])
+def test_a_llama_marker_missing_one_new_key_is_not_read_as_agreement(tmp_path, key):
+ """Isolates the guard the released markers above rely on, one key at a time.
+
+ Those markers are missing all three at once, so on their own they cannot show WHICH
+ absence is doing the work -- and a fast path that only refused them because of, say,
+ the fingerprint would silently start trusting a hand-edited marker the day a backfill
+ added that one key. Start from a marker this code wrote, which the fast path accepts,
+ and remove exactly one key: each must be enough on its own to send the run back to
+ the full path.
+ """
+ install_dir = _current_llama_install(tmp_path)
+ assert _llama_fast_path(install_dir) is True
+
+ marker = ILP.load_prebuilt_metadata(install_dir)
+ assert key in marker
+ marker.pop(key)
+ _rewrite_marker(install_dir / "UNSLOTH_PREBUILT_INFO.json", marker)
+ assert _llama_fast_path(install_dir) is False
+
+
+def test_a_whisper_marker_missing_fingerprint_coverage_is_not_read_as_agreement(tmp_path):
+ """The one key that decides whisper's fast path, isolated the same way.
+
+ Without it the fingerprint cannot be recomputed, so a ``release_tag`` edited over an
+ old binary would read as current. Every pre-PR whisper marker lacks it.
+ """
+ install_dir = _whisper_install(tmp_path, marker_text = None)
+ WSP.write_prebuilt_metadata(install_dir, _whisper_selection())
+ assert _whisper_fast_path(install_dir) is True
+
+ marker = WSP.load_prebuilt_metadata(install_dir)
+ assert isinstance(marker.pop("fingerprint_coverage"), dict)
+ _rewrite_marker(WSP.metadata_path(install_dir), marker)
+ assert _whisper_fast_path(install_dir) is False
+
+
+def test_a_node_marker_missing_node_version_checked_is_not_read_as_agreement(tmp_path):
+ """The record that stands in for the node spawn must not be inferable from anything else.
+
+ ``node_binary`` and ``npm_cli`` can be present (an install this code wrote) while the
+ version record is not, and the spawn still has to happen.
+ """
+ install_dir = _node_install(tmp_path, marker_text = None)
+ NDP.write_metadata(install_dir, version = NODE_VERSION, asset = NODE_ASSET, sha256 = NODE_SHA256)
+ NDP.record_runtime_verification(
+ install_dir, NODE_HOST, version = NODE_VERSION, npm_major = NDP.NPM_MIN_MAJOR
+ )
+ marker = NDP.load_metadata(install_dir)
+ assert NDP._recorded_runtime_matches(install_dir, NODE_HOST, marker, NODE_VERSION) is True
+
+ marker.pop("node_version_checked")
+ assert NDP._recorded_runtime_matches(install_dir, NODE_HOST, marker, NODE_VERSION) is False
+
+
+def test_the_full_check_env_var_puts_a_current_install_back_on_the_slow_path(tmp_path, monkeypatch):
+ """The workaround a user needs when a skip is wrong for reasons nothing on disk shows.
+
+ Both components gate on the same variable, so a support answer is one instruction and
+ not two. Asserted for llama and whisper together for that reason.
+ """
+ llama_dir = _current_llama_install(tmp_path / "llama")
+ whisper_dir = _whisper_install(tmp_path / "whisper", marker_text = None)
+ WSP.write_prebuilt_metadata(whisper_dir, _whisper_selection())
+ assert _llama_fast_path(llama_dir) is True
+ assert _whisper_fast_path(whisper_dir) is True
+
+ monkeypatch.setenv("UNSLOTH_PREBUILT_FULL_CHECK", "1")
+ assert _llama_fast_path(llama_dir) is False
+ assert _whisper_fast_path(whisper_dir) is False
+
+
+# ── Backwards compatibility: the full path is paid exactly once ──
+def _newest_loadable_tag(legacy_markers: dict) -> str:
+ for tag in reversed(LEGACY_TAGS):
+ if "markers" in legacy_markers[tag]:
+ return tag
+ pytest.skip("no released tag could be loaded standalone")
+
+
+def test_an_old_llama_install_pays_the_full_path_once_and_is_fast_afterwards(
+ tmp_path, legacy_markers
+):
+ """The property every existing user gets: update once slowly, then quickly forever.
+
+ The full path reuses the bundle it found and calls ``sync_marker_selection``, which is
+ the only place that catches an old marker up to the evidence the no-network check
+ needs. Without the backfill an old install would re-validate on every single update --
+ the very cost the fast path was added to remove -- and nothing else in the tree would
+ look wrong.
+ """
+ tag = _newest_loadable_tag(legacy_markers)
+ marker = _legacy(legacy_markers, tag)["markers"]["llama"]
+ install_dir = _llama_install(tmp_path, marker["text"])
+
+ assert _llama_fast_path(install_dir) is False
+
+ ILP.sync_marker_selection(
+ install_dir,
+ choice = _llama_choice(),
+ backend_request = "auto",
+ ggml_tree = LLAMA_GGML_TREE,
+ host = LINUX,
+ prebuilt_fallback_used = False,
+ )
+ backfilled = ILP.load_prebuilt_metadata(install_dir)
+ assert isinstance(backfilled.get("host_profile"), dict)
+ assert backfilled.get("runtime_files")
+ # The fingerprint the released installer wrote must still be the one this code
+ # recomputes from the marker's own fields, or the backfill has rewritten history.
+ assert backfilled["install_fingerprint"] == marker["marker"]["install_fingerprint"]
+ assert ILP._marker_install_fingerprint(backfilled) == backfilled["install_fingerprint"]
+
+ assert _llama_fast_path(install_dir) is True
+
+
+def test_the_backfilled_llama_marker_still_refuses_a_tree_whose_bytes_moved(
+ tmp_path, legacy_markers
+):
+ """The fast path that an old install earns must still be a real check.
+
+ ``runtime_files`` replaces actually starting llama-server, so a backfilled marker that
+ answered True for any tree would have swapped a 5 s probe for no probe at all. Rewrite
+ one recorded binary and the same call has to fail closed.
+ """
+ tag = _newest_loadable_tag(legacy_markers)
+ marker = _legacy(legacy_markers, tag)["markers"]["llama"]
+ install_dir = _llama_install(tmp_path, marker["text"])
+ ILP.sync_marker_selection(
+ install_dir,
+ choice = _llama_choice(),
+ backend_request = "auto",
+ ggml_tree = LLAMA_GGML_TREE,
+ host = LINUX,
+ prebuilt_fallback_used = False,
+ )
+ assert _llama_fast_path(install_dir) is True
+
+ server = install_dir / "build" / "bin" / "llama-server"
+ server.write_text("#!/bin/sh\nexit 0\n# a different build\n", encoding = "utf-8")
+ os.chmod(server, 0o755)
+ assert _llama_fast_path(install_dir) is False
+
+
+def test_an_old_whisper_install_pays_the_full_path_once_and_is_fast_afterwards(
+ tmp_path, legacy_markers
+):
+ """The same once-only cost for dictation: whisper.cpp installs from any shipped release.
+
+ The full path keeps the bundle and settles the marker, which is where
+ ``fingerprint_coverage`` (and the os/arch tokens) are written onto a marker that
+ predates them.
+ """
+ tag = _newest_loadable_tag(legacy_markers)
+ marker = _legacy(legacy_markers, tag)["markers"]["whisper"]
+ install_dir = _whisper_install(tmp_path, marker["text"])
+ selection = _whisper_selection()
+ # If this ever fails the fingerprint formula itself moved, and the settle below would
+ # silently decline -- leaving every old install on the full path forever.
+ assert selection.fingerprint() == marker["marker"]["install_fingerprint"]
+
+ assert _whisper_fast_path(install_dir) is False
+
+ CORE._settle_kept_install(WSP._OPS, install_dir, WHISPER_HOST, selection, locked = True)
+ settled = WSP.load_prebuilt_metadata(install_dir)
+ assert isinstance(settled.get("fingerprint_coverage"), dict)
+ assert settled["install_fingerprint"] == marker["marker"]["install_fingerprint"]
+
+ assert _whisper_fast_path(install_dir) is True
+
+
+def test_an_old_node_install_spawns_node_once_and_never_again(
+ tmp_path, legacy_markers, monkeypatch
+):
+ """``node -v`` on a 110 MB runtime, on every update, to re-derive a constant.
+
+ The first call after upgrading has to pay it, because the released marker records
+ nothing about the binary; the second must not. The npm probe is deliberately still
+ paid each time -- npm-cli.js only bootstraps thousands of other files -- so only the
+ node spawn is counted here.
+ """
+ tag = _newest_loadable_tag(legacy_markers)
+ marker = _legacy(legacy_markers, tag)["markers"]["node"]
+ install_dir = _node_install(tmp_path, marker["text"])
+
+ spawns: list[str] = []
+
+ def counted_version(*args, **kwargs):
+ spawns.append("node -v")
+ return NODE_VERSION
+
+ monkeypatch.setattr(NDP, "installed_node_version", counted_version)
+ monkeypatch.setattr(NDP, "installed_npm_major", lambda *a, **k: NDP.NPM_MIN_MAJOR)
+
+ assert (
+ NDP.existing_install_matches(
+ install_dir, NODE_HOST, version = NODE_VERSION, expected_sha = NODE_SHA256
+ )
+ is True
+ )
+ assert spawns == ["node -v"]
+
+ recorded = NDP.load_metadata(install_dir)
+ assert recorded.get("node_version_checked") == NODE_VERSION
+
+ assert (
+ NDP.existing_install_matches(
+ install_dir, NODE_HOST, version = NODE_VERSION, expected_sha = NODE_SHA256
+ )
+ is True
+ )
+ assert spawns == ["node -v"]
+
+
+def test_a_replaced_node_binary_is_probed_again_after_the_record_was_written(
+ tmp_path, legacy_markers, monkeypatch
+):
+ """The record earned above must not outlive the bytes it describes.
+
+ A repaired or hand-swapped node in an existing install has to be re-probed, or the
+ fast path would keep reporting a version nothing on disk still reports.
+ """
+ tag = _newest_loadable_tag(legacy_markers)
+ marker = _legacy(legacy_markers, tag)["markers"]["node"]
+ install_dir = _node_install(tmp_path, marker["text"])
+ monkeypatch.setattr(NDP, "installed_node_version", lambda *a, **k: NODE_VERSION)
+ monkeypatch.setattr(NDP, "installed_npm_major", lambda *a, **k: NDP.NPM_MIN_MAJOR)
+ assert (
+ NDP.existing_install_matches(
+ install_dir, NODE_HOST, version = NODE_VERSION, expected_sha = NODE_SHA256
+ )
+ is True
+ )
+
+ node_binary = NDP.node_binary_path(install_dir, NODE_HOST)
+ node_binary.write_text("#!/bin/sh\nexit 0\n# another build\n", encoding = "utf-8")
+ meta = NDP.load_metadata(install_dir)
+ assert NDP._recorded_runtime_matches(install_dir, NODE_HOST, meta, NODE_VERSION) is False
+
+
+# ── Forwards compatibility: a new marker read by a released installer ──
[email protected]("tag", LEGACY_TAGS)
+def test_a_marker_written_today_is_still_read_by_every_released_unsloth(
+ tmp_path, legacy_markers, tag
+):
+ """Downgrading, or running an older Studio from a second checkout, over one install.
+
+ The fast path's keys are additive, so a released installer must ignore them and still
+ keep the install. If an old reader instead refuses, a user who ever opens an older
+ Studio gets a full re-download of llama.cpp, whisper.cpp and Node -- and gets it back
+ again the next time they open the new one.
+ """
+ entry = _legacy(legacy_markers, tag)
+
+ llama_dir = _llama_install(tmp_path / "current", marker = None)
+ ILP.write_prebuilt_metadata(
+ llama_dir,
+ host = LINUX,
+ requested_tag = "latest",
+ llama_tag = LLAMA_UPSTREAM_TAG,
+ release_tag = LLAMA_RELEASE_TAG,
+ choice = _llama_choice(),
+ approved_checksums = _fill(ILP.ApprovedReleaseChecksums, LLAMA_CHECKSUMS),
+ prebuilt_fallback_used = False,
+ backend_request = "auto",
+ )
+
+ whisper_dir = _whisper_install(tmp_path / "current", marker_text = None)
+ WSP.write_prebuilt_metadata(whisper_dir, _whisper_selection())
+
+ node_dir = _node_install(tmp_path / "current", marker_text = None)
+ NDP.write_metadata(node_dir, version = NODE_VERSION, asset = NODE_ASSET, sha256 = NODE_SHA256)
+ NDP.record_runtime_verification(
+ node_dir, NODE_HOST, version = NODE_VERSION, npm_major = NDP.NPM_MIN_MAJOR
+ )
+
+ # The keys that did not exist when these tags shipped, so the run below is a real test.
+ assert "host_profile" in ILP.load_prebuilt_metadata(llama_dir)
+ assert "fingerprint_coverage" in WSP.load_prebuilt_metadata(whisper_dir)
+ assert "node_version_checked" in NDP.load_metadata(node_dir)
+
+ result = _run_legacy(
+ entry["tree"],
+ "read_markers",
+ {
+ "llama": {"install_dir": str(llama_dir), "host": LLAMA_HOST_KWARGS},
+ "whisper": {
+ "install_dir": str(whisper_dir),
+ "host": WHISPER_HOST_KWARGS,
+ "selection": WHISPER_SELECTION,
+ },
+ "node": {
+ "install_dir": str(node_dir),
+ "host": NODE_HOST_KWARGS,
+ "version": NODE_VERSION,
+ "sha256": NODE_SHA256,
+ "npm_major": NDP.NPM_MIN_MAJOR,
+ },
+ },
+ )
+
+ assert result["llama_backend_request"] == "auto"
+ assert result.get("llama_backend", "vulkan") == "vulkan"
+ # Only the tags that HAVE these readers are asked; the older ones report neither.
+ assert result.get("llama_kept_install_payload_is_healthy", True) is True
+ assert result.get("llama_install_tree_is_usable", True) is True
+ assert result["whisper_matches"] is True
+ assert result["node_matches"] is True
diff --git a/tests/studio/install/test_pr10648_offline_and_status.py b/tests/studio/install/test_pr10648_offline_and_status.py
new file mode 100644
index 00000000000..a54d1f748cf
--- /dev/null
+++ b/tests/studio/install/test_pr10648_offline_and_status.py
@@ -0,0 +1,1196 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""PR #10648: a release lookup that could not answer must keep an intact install.
+
+Two halves, because the feature is a contract between two programs.
+
+PART 1 drives ``install_whisper_prebuilt.main()`` -- the same entrypoint setup.sh
+runs -- with every network failure injected at the module's own primitives, and
+asserts the exit code, the token in the log, and that the tree on disk did not
+move. The wrap this PR added (``fetch_release_for_install``, ``_release_plan_for_host``)
+is what turns a ``URLError`` / ``RuntimeError`` into the ``PrebuiltFallback`` the
+keep path reads, so the assertions that matter most are the negative ones: with no
+install, or a broken one, every single failure mode must still exit non-zero. An
+offline update reporting success over nothing installed is the outcome this file
+exists to rule out.
+
+PART 2 executes the real ``setup.sh`` and ``setup.ps1`` status blocks, extracted
+from the shipped scripts, and checks the label each exit-code/output pair produces
+in BOTH shells. ``pwsh`` is present on Linux CI, so the Windows arm is measured
+rather than asserted from source text.
+
+No test here touches the network: every exit is stubbed, and each stub records the
+URLs it was asked for so a test that intercepted nothing fails instead of passing.
+"""
+
+from __future__ import annotations
+
+import email.message
+import io
+import json
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import tarfile
+import urllib.error
+from pathlib import Path
+
+import pytest
+
+from _pr10648_helpers import PACKAGE_ROOT, git, load_studio_module
+from unsloth_pwsh_runner import run_pwsh
+
+
+MODULE_PATH = PACKAGE_ROOT / "studio" / "install_whisper_prebuilt.py"
+
+# A DISTINCT sys.modules name: test_install_whisper_prebuilt_logic.py owns
+# "studio_install_whisper_prebuilt", and -n 4's per-worker cache would share the monkeypatches.
+M = load_studio_module(
+ "studio_install_whisper_prebuilt_pr10648_offline", "install_whisper_prebuilt.py"
+)
+
+HostInfo = M.HostInfo
+
+# The merge base of studio-prebuilt-precheck, for the pre-PR comparison below.
+MERGE_BASE = "191b69c12b4434b5247f1fd7a455b4a760b169ae"
+
+RELEASE_TAG = "v1.9.1-unsloth.1"
+UPSTREAM_TAG = "v1.9.1"
+SOURCE_COMMIT = "0" * 40
+STUDIO_PROTOCOL = "inference/multipart-v1"
+
+# The two substrings the setup scripts grep. Spelled out here so a test that stops matching the
+# installer is a test that stops matching the shipped scripts too.
+KEEP_TOKEN = "keeping the existing complete install"
+MATCH_TOKEN = "already matches"
+FAIL_TOKEN = "prebuilt install failed"
+
+
+def _host(
+ whisper_os: str,
+ whisper_arch: str,
+ *,
+ macos_version = None,
+) -> HostInfo:
+ return HostInfo(
+ system = {"linux": "Linux", "macos": "Darwin", "windows": "Windows"}[whisper_os],
+ machine = whisper_arch,
+ whisper_os = whisper_os,
+ whisper_arch = whisper_arch,
+ archive_ext = ".zip" if whisper_os == "windows" else ".tar.gz",
+ is_windows = whisper_os == "windows",
+ is_macos = whisper_os == "macos",
+ is_apple_silicon = whisper_os == "macos" and whisper_arch == "arm64",
+ macos_version = macos_version,
+ )
+
+
+def _manifest(artifacts: list[dict]) -> dict:
+ return {
+ "schema_version": 1,
+ "component": "whisper.cpp",
+ "studio_protocol": STUDIO_PROTOCOL,
+ "upstream_tag": UPSTREAM_TAG,
+ "source_commit": SOURCE_COMMIT,
+ "artifacts": artifacts,
+ }
+
+
+def _artifact(host: HostInfo, asset: str, sha256: str, **extra) -> dict:
+ artifact = {
+ "os": host.whisper_os,
+ "arch": host.whisper_arch,
+ "backend": "cpu",
+ "asset": asset,
+ "sha256": sha256,
+ "runtime_line": None,
+ }
+ artifact.update(extra)
+ return artifact
+
+
+def _build_cpu_bundle(tmp_path: Path, host: HostInfo) -> tuple[Path, str, str]:
+ """A real archive on disk: the install path extracts and hashes it for real."""
+ asset = M.whisper_asset_name(RELEASE_TAG, host, "cpu")
+ archive = tmp_path / asset
+ with tarfile.open(archive, "w:gz") as tar:
+ for name, data, mode in (
+ (M.server_binary_name(host), b"#!/bin/sh\necho whisper\n", 0o755),
+ ("libwhisper.so", b"dummy-libwhisper", 0o644),
+ ("libggml-base.so", b"dummy-libggml", 0o644),
+ ):
+ info = tarfile.TarInfo(name)
+ info.size = len(data)
+ info.mode = mode
+ tar.addfile(info, io.BytesIO(data))
+ return archive, asset, M.sha256_file(archive)
+
+
+def _bundle(host: HostInfo, asset: str, sha256: str, **artifact_extra):
+ manifest = M.parse_manifest(_manifest([_artifact(host, asset, sha256, **artifact_extra)]))
+ return M.ReleaseBundle(
+ repo = M.DEFAULT_PUBLISHED_REPO,
+ release_tag = RELEASE_TAG,
+ manifest = manifest,
+ asset_urls = {asset: f"https://example.invalid/{asset}"},
+ )
+
+
+def _seed_install(tmp_path: Path, host: HostInfo, **artifact_extra) -> Path:
+ """Install a real prebuilt tree + marker, with every patch undone afterwards.
+
+ The seeding patches live in their own MonkeyPatch context so the failure injection the
+ test then installs is the ONLY thing standing between the installer and the network.
+ """
+ archive, asset, sha256 = _build_cpu_bundle(tmp_path, host)
+ install_dir = tmp_path / "whisper.cpp"
+ bundle = _bundle(host, asset, sha256, **artifact_extra)
+ checksums = {asset: sha256}
+
+ def fake_download(url, destination):
+ destination.parent.mkdir(parents = True, exist_ok = True)
+ destination.write_bytes(archive.read_bytes())
+
+ with pytest.MonkeyPatch.context() as seeding:
+ seeding.delenv("UNSLOTH_PREBUILT_FULL_CHECK", raising = False)
+ seeding.setattr(M, "detect_host", lambda: host)
+ seeding.setattr(M, "download_file", fake_download)
+ seeding.setattr(
+ M,
+ "fetch_release_for_install",
+ lambda repo, *, published_release_tag = None: (bundle, checksums),
+ )
+ assert M.install_prebuilt(install_dir, backend = "cpu") == M.EXIT_SUCCESS
+ assert M.metadata_path(install_dir).is_file()
+ return install_dir
+
+
+def _http_error(code: int, url: str) -> urllib.error.HTTPError:
+ return urllib.error.HTTPError(url, code, f"injected {code}", email.message.Message(), None)
+
+
+# Every failure mode, named once. The value is what the injected primitives do.
+FAILURE_MODES = (
+ "offline", # urllib.error.URLError: no network at all
+ "timeout", # socket.timeout on both the CDN HEAD and every GET
+ "http_403", # api.github.com rate limit -> fetch_json raises RuntimeError
+ "http_429", # api.github.com secondary rate limit -> same RuntimeError
+ "http_404", # the repo/release does not answer
+ "http_500", # GitHub 5xx
+ "truncated_json", # a body that stops mid-object
+ "malformed_json", # a captive-portal/proxy HTML page where JSON was expected
+ "empty_releases", # a well-formed answer listing no releases at all
+)
+
+
+class _InjectedNetwork:
+ """Every network exit this installer has, replaced by one injected failure.
+
+ There are exactly two primitives, and both are resolved through ``_OPS`` against the
+ installer module's own globals, so setting them here is the real seam rather than an
+ unused alias:
+
+ * ``download_bytes`` -- every JSON GET, api.github.com and the release CDN alike
+ (``core.fetch_json`` and ``core.fetch_download_host_json`` both route through it).
+ * ``_URL_OPENER`` -- the bare HEAD ``core.download_host_latest_release_tag`` uses to
+ read /releases/latest without spending API quota.
+
+ ``calls`` is the proof the injection fired. A test whose patch intercepted nothing would
+ otherwise be green while the real code took some other path.
+ """
+
+ def __init__(self, mode: str) -> None:
+ self.mode = mode
+ self.calls: list[tuple[str, str]] = []
+ self.downloads: list[str] = []
+
+ def _raise_transport(self, url: str) -> None:
+ if self.mode == "offline":
+ raise urllib.error.URLError(OSError("injected: network is unreachable"))
+ if self.mode == "timeout":
+ raise socket.timeout("injected: timed out")
+
+ # ── the /releases/latest HEAD ──
+ def open(
+ self,
+ request,
+ timeout = None,
+ ):
+ url = getattr(request, "full_url", str(request))
+ self.calls.append(("head", url))
+ self._raise_transport(url)
+ # Every remaining mode is about api.github.com, so the CDN shortcut declines first. A 404
+ # is how download_host_latest_release_tag says "cannot name the release": None, not a raise.
+ raise _http_error(404, url)
+
+ # ── every JSON GET ──
+ def download_bytes(self, url, **_kwargs) -> bytes:
+ self.calls.append(("get", url))
+ self._raise_transport(url)
+ if self.mode.startswith("http_"):
+ raise _http_error(int(self.mode.split("_")[1]), url)
+ if self.mode == "truncated_json":
+ return b'{"tag_name": "v1.9.1-unslo'
+ if self.mode == "malformed_json":
+ return b"<html><head><title>502 Bad Gateway</title></head></html>"
+ if self.mode == "empty_releases":
+ return b"[]"
+ raise AssertionError(f"unhandled injection mode {self.mode!r}")
+
+ # ── the pre-check's own HEAD, which lives on the llama module ──
+ def llama_latest_tag(self, repo: str) -> str | None:
+ self.calls.append(("precheck", repo))
+ self._raise_transport(repo)
+ raise _http_error(404, f"https://github.com/{repo}/releases/latest")
+
+ def download_file(self, url, destination): # pragma: no cover - asserted never to run
+ self.downloads.append(str(url))
+ raise AssertionError(f"a keep/fail path must not download anything, but fetched {url}")
+
+ @property
+ def install_calls(self) -> list[tuple[str, str]]:
+ """Calls made by the INSTALL path, i.e. not the marker-only pre-check.
+
+ A fully offline run never reaches ``download_bytes`` at all -- the CDN HEAD that
+ resolves /releases/latest raises first -- so this, not a GET count, is what proves
+ the injection fired where the PR's wrap had to catch it."""
+ return [(kind, url) for kind, url in self.calls if kind != "precheck"]
+
+
+def _inject(monkeypatch, mode: str) -> _InjectedNetwork:
+ """Close every network exit, then prove the one that mattered was used.
+
+ Patching only ``M.download_bytes`` is the trap this helper exists to avoid: the
+ installer does ``fetch_json = llama.fetch_json`` and ``download_bytes =
+ llama.download_bytes`` at import, so ``core.fetch_json`` resolves through the LLAMA
+ module's globals and an injection on the whisper alias intercepts nothing -- the first
+ version of this file reached api.github.com for real and still looked green until
+ ``calls`` was read back and showed a live release tag in the URL.
+
+ So: both modules' primitives, plus ``prebuilt_core._URL_OPENER`` underneath them, which
+ every HTTP call in all three modules ultimately goes through.
+ """
+ net = _InjectedNetwork(mode)
+ for module in (M, M.llama):
+ monkeypatch.setattr(module, "download_bytes", net.download_bytes)
+ monkeypatch.setattr(module, "download_file", net.download_file)
+ monkeypatch.setattr(module, "_URL_OPENER", net, raising = False)
+ # One attempt: the JSON decode path otherwise retries with real sleeps.
+ monkeypatch.setattr(module, "JSON_FETCH_ATTEMPTS", 1, raising = False)
+ monkeypatch.setattr(module, "HTTP_FETCH_ATTEMPTS", 1, raising = False)
+ monkeypatch.setattr(M.core, "_URL_OPENER", net)
+ monkeypatch.setattr(M.llama, "_download_host_latest_release_tag", net.llama_latest_tag)
+ monkeypatch.delenv("UNSLOTH_PREBUILT_FULL_CHECK", raising = False)
+ monkeypatch.delenv("UNSLOTH_WHISPER_FORCE_COMPILE", raising = False)
+ return net
+
+
+def _run_cli(monkeypatch, capsys, host: HostInfo, install_dir: Path, *extra: str):
+ """Drive main(), the entrypoint setup.sh runs, and return (exit code, log text)."""
+ monkeypatch.setattr(M, "detect_host", lambda: host)
+ # main() flips this module global; register the current value so it is restored.
+ monkeypatch.setattr(M, "_LOG_TO_STDOUT", M._LOG_TO_STDOUT)
+ capsys.readouterr()
+ code = M.main(["--install-dir", str(install_dir), "--backend", "cpu", *extra])
+ captured = capsys.readouterr()
+ return code, captured.out + captured.err
+
+
+def _tree_snapshot(install_dir: Path) -> dict[str, bytes]:
+ if not install_dir.exists():
+ return {}
+ return {
+ str(path.relative_to(install_dir)): path.read_bytes()
+ for path in sorted(install_dir.rglob("*"))
+ if path.is_file()
+ }
+
+
+def _partial_artifacts(install_dir: Path) -> list[str]:
+ """Anything a half-finished install would leave: staging trees and atomic temps."""
+ if not install_dir.exists():
+ return []
+ return sorted(
+ str(path.relative_to(install_dir))
+ for path in install_dir.rglob("*")
+ if ".tmp-" in path.name or path.name.startswith(".tmp-")
+ )
+
+
+# ══ Part 1: network failure modes ════════════════════════════════════════════════════
+
+
[email protected]("mode", FAILURE_MODES)
+def test_every_lookup_failure_keeps_an_intact_install(tmp_path, monkeypatch, capsys, mode):
+ """An unpinned update whose lookup could not answer keeps the tree and exits 0.
+
+ This is the whole point of the PR: before it, ``URLError`` and ``fetch_json``'s
+ ``RuntimeError`` escaped ``install_prebuilt`` uncaught and setup printed
+ "prebuilt install failed" over a healthy install.
+
+ Note which modes are in this list. 404 and 500 are here too, because with no release
+ pin they are also "the lookup could not answer": nothing named a release, so nothing
+ established that the install on disk is stale. The pinned case below is where a 404
+ stops being an unavailability and starts being an answer.
+ """
+ host = _host("linux", "x64")
+ install_dir = _seed_install(tmp_path, host)
+ before = _tree_snapshot(install_dir)
+ marker_before = M.metadata_path(install_dir).read_bytes()
+
+ net = _inject(monkeypatch, mode)
+ code, log = _run_cli(monkeypatch, capsys, host, install_dir)
+
+ assert net.install_calls, f"{mode}: the injection never fired; the test intercepted nothing"
+ assert code == M.EXIT_SUCCESS, f"{mode}: exit {code}\n{log}"
+ # The exact substring studio/setup.sh:3636 and studio/setup.ps1:6381 grep for.
+ assert KEEP_TOKEN in log, f"{mode}: {log}"
+ assert FAIL_TOKEN not in log, f"{mode}: {log}"
+ # Not "already matches": that arm names a release this run never fetched.
+ assert MATCH_TOKEN not in log, f"{mode}: {log}"
+
+ assert net.downloads == [], f"{mode}: a kept install downloaded {net.downloads}"
+ assert _tree_snapshot(install_dir) == before, f"{mode}: the kept tree moved"
+ assert M.metadata_path(install_dir).read_bytes() == marker_before, f"{mode}: marker rewritten"
+ assert _partial_artifacts(install_dir) == [], f"{mode}: half-installed leftovers"
+
+
[email protected]("mode", FAILURE_MODES)
+ "damage",
+ ["absent", "no_marker", "zero_byte_server", "tampered_marker", "server_deleted"],
+)
+def test_no_failure_mode_reports_success_without_a_working_install(
+ tmp_path, monkeypatch, capsys, mode, damage
+):
+ """The worst possible outcome, ruled out for every mode x every damage shape.
+
+ An update that cannot reach the network and reports success while nothing usable is
+ installed would leave setup.sh printing "prebuilt installed" over an empty directory
+ and dictation silently broken. Every one of these must exit non-zero and say
+ "prebuilt install failed" -- the token setup.sh's else-arm reports.
+ """
+ host = _host("linux", "x64")
+ if damage == "absent":
+ install_dir = tmp_path / "whisper.cpp"
+ else:
+ install_dir = _seed_install(tmp_path, host)
+ marker = M.metadata_path(install_dir)
+ server = M.installed_server_path(install_dir, host)
+ if damage == "no_marker":
+ marker.unlink()
+ elif damage == "zero_byte_server":
+ # Keeps its mode bits, so neither the execute-bit nor the existence check sees it.
+ server.write_bytes(b"")
+ elif damage == "server_deleted":
+ server.unlink()
+ elif damage == "tampered_marker":
+ payload = json.loads(marker.read_text(encoding = "utf-8"))
+ payload["release_tag"] = "v9.9.9-unsloth.99"
+ marker.write_text(json.dumps(payload), encoding = "utf-8")
+
+ net = _inject(monkeypatch, mode)
+ code, log = _run_cli(monkeypatch, capsys, host, install_dir)
+
+ assert net.install_calls, f"{mode}/{damage}: the injection never fired"
+ assert (
+ code != M.EXIT_SUCCESS
+ ), f"{mode}/{damage}: reported success with no working install\n{log}"
+ assert code == M.EXIT_ERROR, f"{mode}/{damage}: exit {code}\n{log}"
+ assert KEEP_TOKEN not in log, f"{mode}/{damage}: claimed to keep a broken install\n{log}"
+ assert MATCH_TOKEN not in log, f"{mode}/{damage}: {log}"
+ assert FAIL_TOKEN in log, f"{mode}/{damage}: {log}"
+ assert net.downloads == [], f"{mode}/{damage}: downloaded {net.downloads}"
+ assert _partial_artifacts(install_dir) == [], f"{mode}/{damage}: half-installed leftovers"
+ if damage == "absent":
+ assert (
+ not install_dir.exists() or _tree_snapshot(install_dir) == {}
+ ), f"{mode}: a failed install left a tree behind"
+
+
[email protected]("mode", FAILURE_MODES)
+def test_an_explicitly_pinned_release_is_never_silently_kept(tmp_path, monkeypatch, capsys, mode):
+ """A run that NAMED a release must not be answered with a different one.
+
+ Keeping ignores the pin, so the pin has to fail instead. This is also the honest
+ answer to the 404 question: a 404 on ``--published-release-tag v9.9.9-unsloth.99``
+ is a deleted or misspelled tag -- an answer -- and it exits 1 rather than reporting
+ the installed release as a success.
+ """
+ host = _host("linux", "x64")
+ install_dir = _seed_install(tmp_path, host)
+ before = _tree_snapshot(install_dir)
+
+ net = _inject(monkeypatch, mode)
+ code, log = _run_cli(
+ monkeypatch, capsys, host, install_dir, "--published-release-tag", "v9.9.9-unsloth.99"
+ )
+
+ assert net.calls, f"{mode}: the injection never fired"
+ assert code == M.EXIT_ERROR, f"{mode}: exit {code}\n{log}"
+ assert KEEP_TOKEN not in log, f"{mode}: a pinned release was papered over\n{log}"
+ assert FAIL_TOKEN in log, f"{mode}: {log}"
+ # The intact tree is still not damaged -- refusing to keep is not the same as deleting.
+ assert _tree_snapshot(install_dir) == before, f"{mode}: a failed pinned run moved the tree"
+ assert _partial_artifacts(install_dir) == [], f"{mode}: half-installed leftovers"
+
+
+ "extra_argv, env",
+ [
+ (("--force",), {}),
+ (("--whisper-tag", "v1.9.7"), {}),
+ ((), {"UNSLOTH_WHISPER_FORCE_COMPILE": "1"}),
+ ],
+ ids = ["force", "upstream-tag-pin", "force-compile"],
+)
+def test_an_explicit_request_never_keeps(tmp_path, monkeypatch, capsys, extra_argv, env):
+ """Everything this RUN asked for that keeping would ignore fails instead.
+
+ ``--force`` asked for a reinstall, ``--whisper-tag`` asked for a specific upstream
+ version, and ``UNSLOTH_WHISPER_FORCE_COMPILE=1`` asked setup.sh to try a source build,
+ which it only does after a non-zero exit. Reporting 0 would swallow all three.
+ """
+ host = _host("linux", "x64")
+ install_dir = _seed_install(tmp_path, host)
+
+ net = _inject(monkeypatch, "offline")
+ for key, value in env.items():
+ monkeypatch.setenv(key, value)
+ code, log = _run_cli(monkeypatch, capsys, host, install_dir, *extra_argv)
+
+ assert net.calls, "the injection never fired"
+ assert code == M.EXIT_ERROR, f"exit {code}\n{log}"
+ assert KEEP_TOKEN not in log, log
+ assert FAIL_TOKEN in log, log
+
+
+def test_the_keep_arm_names_the_reason_and_the_release_it_kept(tmp_path, monkeypatch, capsys):
+ """The kept line has to be readable on its own: setup.sh prints one status line and
+ throws the installer log away, so the release and the reason both belong in it."""
+ host = _host("linux", "x64")
+ install_dir = _seed_install(tmp_path, host)
+ _inject(monkeypatch, "http_403")
+ code, log = _run_cli(monkeypatch, capsys, host, install_dir)
+
+ assert code == M.EXIT_SUCCESS, log
+ assert KEEP_TOKEN in log
+ assert RELEASE_TAG in log, log
+ assert "update unavailable, existing prebuilt kept" in log, log
+ assert "prebuilt update reason:" in log, log
+ # The 403 arrives as fetch_json's RuntimeError, not an OSError; the PR's wrap is the only
+ # thing that turns it into the PrebuiltFallback the keep path reads.
+ assert "403" in log, log
+
+
+def test_a_release_compatibility_error_is_never_papered_over(tmp_path, monkeypatch, capsys):
+ """Exit 2 is a lookup that ANSWERED: no published bundle pairs with this runtime.
+
+ setup.sh names both tags from exit 2, so converting it into a kept success would hide
+ real release skew behind a warning about the network.
+ """
+ host = _host("linux", "x64")
+ install_dir = _seed_install(tmp_path, host)
+ _inject(monkeypatch, "offline")
+
+ def incompatible(*_args, **_kwargs):
+ raise M.ReleaseCompatibilityError("slim bundle requires llama.cpp b9999; installed b1000")
+
+ monkeypatch.setattr(M, "_release_plan_for_host", incompatible)
+ code, log = _run_cli(monkeypatch, capsys, host, install_dir)
+
+ assert code == M.EXIT_INCOMPATIBLE, f"exit {code}\n{log}"
+ assert KEEP_TOKEN not in log, log
+
+
+def _macos_listing_failure(tmp_path, monkeypatch, host):
+ """Newest published release incompatible with this Mac, release LISTING unavailable.
+
+ ``_release_plan_for_host`` records the newest release's incompatibility in
+ ``first_error``, then walks older releases to look for one this Mac can run. When the
+ listing that drives the walk cannot be fetched, line 1424-1427 raises a new
+ PrebuiltFallback and ``first_error`` is discarded.
+ """
+ newer_dir = tmp_path / "newer"
+ newer_dir.mkdir(parents = True, exist_ok = True)
+ archive, asset, sha256 = _build_cpu_bundle(newer_dir, host)
+ # A newest release whose only artifact needs a macOS this host does not have.
+ newest = _bundle(host, asset, sha256, min_os = "26.0")
+ # Control: the SAME bundle is selectable on a new enough Mac, so the rejection below is the
+ # min_os floor, not a manifest built wrong. Without it the scenario passes exercising nothing.
+ M.select_artifact_with_cpu_fallback(
+ newest.manifest, _host("macos", host.whisper_arch, macos_version = (26, 0)), "cpu"
+ )
+ with pytest.raises(M.PrebuiltFallback):
+ M.select_artifact_with_cpu_fallback(newest.manifest, host, "cpu")
+ monkeypatch.setattr(
+ M,
+ "fetch_release_for_install",
+ lambda repo, *, published_release_tag = None: (newest, {asset: sha256}),
+ )
+ return _inject(monkeypatch, "offline")
+
+
+def test_macos_incompatible_newest_plus_unreachable_listing_keeps_a_runnable_install(
+ tmp_path, monkeypatch, capsys
+):
+ """The open question, settled: this is an IMPROVEMENT, not a false success.
+
+ ``first_error`` describes the NEWEST release only. The walk-back loop exists precisely
+ because an older release may still run here, so with the listing unreachable the
+ installer has NOT established "your macOS is too old for any available build" -- it has
+ established that it cannot tell. Discarding first_error for "could not list releases"
+ is therefore the accurate verdict, not a downgrade of a real one.
+
+ And the install that is kept is known to run on this host: ``_existing_install_is_intact``
+ re-checks the marker's ``min_os`` against ``host.macos_version`` before keeping anything
+ (test_macos_keep_refuses_an_install_below_this_hosts_floor below is the negative case).
+ """
+ host = _host("macos", "arm64", macos_version = (13, 0))
+ install_dir = _seed_install(tmp_path, host, min_os = "13.0")
+ before = _tree_snapshot(install_dir)
+
+ net = _macos_listing_failure(tmp_path, monkeypatch, host)
+ code, log = _run_cli(monkeypatch, capsys, host, install_dir)
+
+ # The listing endpoint _published_release_tags walks, and nothing else: this is the call
+ # whose failure discards first_error.
+ assert [url for _kind, url in net.install_calls] == [
+ "https://api.github.com/repos/unslothai/whisper.cpp/releases?per_page=100"
+ ], net.install_calls
+ assert code == M.EXIT_SUCCESS, f"exit {code}\n{log}"
+ assert KEEP_TOKEN in log, log
+ # The reason names the unavailability, so nobody reads it as a compatibility verdict.
+ assert "could not list" in log, log
+ # And the discarded first_error is NOT what the user is told; a kept install is.
+ assert "no whisper.cpp prebuilt asset" not in log, log
+ assert _tree_snapshot(install_dir) == before
+
+
+def test_macos_incompatible_newest_plus_unreachable_listing_fails_without_an_install(
+ tmp_path, monkeypatch, capsys
+):
+ """The other half of the same verdict: nothing on disk, nothing invented.
+
+ If the discarded first_error had been converted into a false success, THIS is where it
+ would show -- a Mac with no whisper install reporting exit 0 after a lookup that never
+ found a compatible bundle. It exits 1.
+ """
+ host = _host("macos", "arm64", macos_version = (13, 0))
+ install_dir = tmp_path / "whisper.cpp"
+
+ net = _macos_listing_failure(tmp_path, monkeypatch, host)
+ code, log = _run_cli(monkeypatch, capsys, host, install_dir)
+
+ assert net.install_calls
+ assert code == M.EXIT_ERROR, f"exit {code}\n{log}"
+ assert KEEP_TOKEN not in log, log
+ assert FAIL_TOKEN in log, log
+
+
+def test_macos_keep_refuses_an_install_below_this_hosts_floor(tmp_path, monkeypatch, capsys):
+ """A tree carried onto an older Mac is not "intact" and must not be kept offline.
+
+ Without this, the keep path would hold an install that cannot load, which is the shape
+ a genuine false success would take.
+ """
+ build_host = _host("macos", "arm64", macos_version = (15, 0))
+ install_dir = _seed_install(tmp_path, build_host, min_os = "15.0")
+ older_mac = _host("macos", "arm64", macos_version = (13, 0))
+
+ # Only the floor changed: the same tree is intact on the Mac it was installed on. Asserting the
+ # predicate keeps the exit-1 below from passing for an unrelated reason.
+ intact = dict(
+ published_repo = M.DEFAULT_PUBLISHED_REPO,
+ requested_backend = "cpu",
+ )
+ assert M._existing_install_is_intact(install_dir, build_host, **intact) is not None
+ assert M._existing_install_is_intact(install_dir, older_mac, **intact) is None
+
+ net = _inject(monkeypatch, "offline")
+ code, log = _run_cli(monkeypatch, capsys, older_mac, install_dir)
+
+ assert net.calls
+ assert code == M.EXIT_ERROR, f"exit {code}\n{log}"
+ assert KEEP_TOKEN not in log, log
+
+
+def _git(*args: str) -> subprocess.CompletedProcess:
+ return git(*args, text = True, timeout = 60)
+
+
+requires_merge_base = pytest.mark.skipif(
+ shutil.which("git") is None or _git("cat-file", "-e", f"{MERGE_BASE}^{{commit}}").returncode,
+ reason = f"the PR merge base {MERGE_BASE} is not in this checkout",
+)
+
+
+@requires_merge_base
+def test_pre_pr_an_unreachable_release_listing_escaped_as_an_uncaught_oserror():
+ """Evidence for the verdict above, from the code this PR replaced.
+
+ Pre-PR, ``_release_plan_for_host`` iterated ``_published_release_tags(...)`` directly
+ and ``install_prebuilt`` had no except arm at all, so an unreachable listing left the
+ OSError to main()'s catch-all: "unexpected error", exit 1, over an intact install --
+ which setup.sh then reported as "prebuilt install failed".
+
+ So the macOS case is not a real verdict being converted into a false success. Pre-PR
+ there was no verdict to lose: first_error was discarded then too (the OSError simply
+ replaced it by propagating), and the user got a failure instead of a working runtime.
+ """
+ pre = _git("show", f"{MERGE_BASE}:studio/install_whisper_prebuilt.py")
+ assert pre.returncode == 0, pre.stderr
+ source = pre.stdout
+
+ # The loop was unguarded: no try/except between the planner and the listing fetch.
+ assert (
+ " for release_tag in _published_release_tags(published_repo):" in source
+ ), "the pre-PR planner did not iterate the listing directly; re-read the comparison"
+ assert "could not list" not in source, "pre-PR already had the listing guard"
+ # install_prebuilt had no keep arm, so nothing downgraded a lookup failure to exit 0.
+ pre_install = source[source.index("def install_prebuilt(") :]
+ pre_install = pre_install[: pre_install.index("\ndef ")]
+ assert "except PrebuiltFallback" not in pre_install, pre_install
+ assert KEEP_TOKEN not in source, "pre-PR already logged the keep token"
+
+ # And post-PR it is guarded, in the same function.
+ post = MODULE_PATH.read_text(encoding = "utf-8")
+ assert "except (OSError, RuntimeError) as exc:" in post
+ guard = 'raise PrebuiltFallback(f"could not list {published_repo} releases: {exc}") from exc'
+ assert guard in post
+
+
+@requires_merge_base
+def test_the_pr_only_added_new_arms_to_the_setup_status_blocks():
+ """The pre-existing tokens must mean exactly what they meant before.
+
+ ``already matches`` and llama's two arms carry other behaviour (node greps the same
+ token at setup.sh:1261), so the new keep arm is only safe if it was inserted beside
+ them rather than rewritten over them. Checked on the diff, not on the current text: a
+ reworded arm would still satisfy a substring assertion on the file.
+ """
+ diff = _git("diff", "-U0", MERGE_BASE, "--", "studio/setup.sh", "studio/setup.ps1")
+ assert diff.returncode == 0, diff.stderr
+ removed = [
+ line[1:]
+ for line in diff.stdout.splitlines()
+ if line.startswith("-") and not line.startswith("---")
+ ]
+ added = [
+ line[1:]
+ for line in diff.stdout.splitlines()
+ if line.startswith("+") and not line.startswith("+++")
+ ]
+ # Nothing that carries a status decision was removed or rewritten.
+ for line in removed:
+ assert MATCH_TOKEN not in line, f"the PR rewrote an 'already matches' arm: {line!r}"
+ assert "step " not in line, f"the PR rewrote a status line: {line!r}"
+ assert "grep -Fq" not in line, f"the PR rewrote a status grep: {line!r}"
+ assert "-match" not in line, f"the PR rewrote a status match: {line!r}"
+ # What WAS added is the keep arm, once per script.
+ keep_arms = [line for line in added if KEEP_TOKEN in line]
+ assert len(keep_arms) == 2, keep_arms
+ assert any("grep -Fq" in line for line in keep_arms), keep_arms
+ assert any("-match" in line for line in keep_arms), keep_arms
+
+ # The llama arms and node's grep are still exactly where they were.
+ setup_sh = (PACKAGE_ROOT / "studio" / "setup.sh").read_text(encoding = "utf-8")
+ setup_ps1 = (PACKAGE_ROOT / "studio" / "setup.ps1").read_text(encoding = "utf-8")
+ assert 'grep -Fq "already matches" "$_PREBUILT_LOG"' in setup_sh
+ assert 'grep -Fq "already matches" "$_NODE_LOG"' in setup_sh
+ assert '$prebuiltOutput -match "already matches"' in setup_ps1
+ for script in (setup_sh, setup_ps1):
+ assert (
+ script.count("update unavailable, existing prebuilt kept") == 2
+ ), "llama and whisper each have exactly one keep arm"
+
+
+# ══ Part 2: the status contract, executed in both shells ═════════════════════════════
+
+_SH_WHISPER_START = 'if [ "$_WHISPER_STATUS" -eq 0 ]; then'
+_PS1_WHISPER_START = "if ($whisperExit -eq 0) {"
+_SH_LLAMA_START = 'if [ "$_PREBUILT_STATUS" -eq 0 ]; then'
+_PS1_LLAMA_START = "if ($prebuiltExit -eq 0) {"
+
+# Stand-ins for the setup helpers the blocks call. `step` echoes both arguments, which is what
+# distinguishes "llama reported the keep" from "whisper reported it".
+_SH_HARNESS = """
+set -u
+C_OK=""; C_WARN=""; C_ERR=""
+_NEED_LLAMA_SOURCE_BUILD=false
+_LLAMA_CPP_NO_SPACE=false
+_LLAMA_CPP_DEGRADED=false
+_explicit_llama_backend=""
+_STUDIO_HOME_IS_CUSTOM=false
+_STUDIO_OWNED_MARKER=".unsloth-owned"
+_WHISPER_RECOVERED=false
+step() { echo "step|$1|$2"; }
+substep() { echo "substep|$1"; }
+verbose_substep() { :; }
+run_quiet_no_exit() { return 1; }
+print_llama_error_log() { :; }
+print_installed_llama_prebuilt_release() { :; }
+_has_local_llama_server() { return 1; }
+setup_fail() { echo "setup_fail|$1"; exit "$1"; }
+"""
+
+_PS1_HARNESS = """
+$ErrorActionPreference = "Stop"
+$NeedLlamaSourceBuild = $false
+$script:LlamaCppDegraded = $false
+$StudioHomeIsCustom = $false
+function step { param($a, $b, $c) Write-Output "step|$a|$b" }
+function substep { param($a, $b) Write-Output "substep|$a" }
+function Write-LlamaFailureLog { param($Output) }
+function Mark-StudioOwned { param($Path) }
+function Get-InstalledLlamaPrebuiltRelease { param($InstallDir) return $null }
+function Test-PathQuiet { param($p, $t) return $false }
+function Exit-SetupFailure {
+ param($Message, $Code = 1)
+ Write-Output "setup_fail|$Code"
+ exit $Code
+}
+"""
+
+
+def _extract_block(text: str, start_marker: str, end_marker: str) -> str:
+ start = text.index(start_marker)
+ end = text.index(end_marker, start)
+ return text[start : end + len(end_marker)]
+
+
+def _sh_block(name: str) -> str:
+ setup_sh = (PACKAGE_ROOT / "studio" / "setup.sh").read_text(encoding = "utf-8")
+ start = setup_sh.index(_SH_WHISPER_START if name == "whisper" else _SH_LLAMA_START)
+ # Both chains end at the first dedented " fi"; every nested fi inside is indented deeper.
+ end = setup_sh.index("\n fi\n", start)
+ return setup_sh[start : end + len("\n fi\n")]
+
+
+def _ps1_block(name: str) -> str:
+ setup_ps1 = (PACKAGE_ROOT / "studio" / "setup.ps1").read_text(encoding = "utf-8")
+ if name == "whisper":
+ # The exit-2 arm ends with the same words followed by "} else {", so the trailing
+ # newline is what pins this to the final closing brace of the chain.
+ return _extract_block(setup_ps1, _PS1_WHISPER_START, 'remain available" "Yellow"\n }\n')
+ return _extract_block(setup_ps1, _PS1_LLAMA_START, 'retry setup."\n }')
+
+
+def _run_bash(script: str) -> subprocess.CompletedProcess:
+ return subprocess.run(["bash", "-c", script], capture_output = True, text = True, timeout = 60)
+
+
+def _steps(stdout: str) -> dict[str, str]:
+ """The component -> label decision each block reached."""
+ found = {}
+ for line in stdout.splitlines():
+ if line.startswith("step|"):
+ _, component, label = line.split("|", 2)
+ found[component] = label
+ return found
+
+
+def _sh_whisper_script(tmp_path: Path, status: int, output: str) -> str:
+ log_path = tmp_path / "whisper.log"
+ log_path.write_text(output, encoding = "utf-8")
+ return "\n".join(
+ [
+ _SH_HARNESS,
+ f'SCRIPT_DIR="{tmp_path}"',
+ f'UNSLOTH_HOME="{tmp_path}"',
+ f'WHISPER_CPP_DIR="{tmp_path / "whisper.cpp"}"',
+ f'_WHISPER_LOG="{log_path}"',
+ f"_WHISPER_STATUS={status}",
+ _sh_block("whisper"),
+ ]
+ )
+
+
+def _ps1_whisper_script(tmp_path: Path, status: int, output: str) -> str:
+ return "\n".join(
+ [
+ _PS1_HARNESS,
+ f'$WhisperCppDir = "{tmp_path / "whisper.cpp"}"',
+ f'$LlamaCppDir = "{tmp_path / "llama.cpp"}"',
+ f"$whisperExit = {status}",
+ "$whisperOutput = @'",
+ output,
+ "'@",
+ _ps1_block("whisper"),
+ ]
+ )
+
+
+def _run_ps1(
+ tmp_path: Path,
+ script: str,
+ name: str = "status.ps1",
+):
+ script_path = tmp_path / name
+ script_path.write_text(script, encoding = "utf-8")
+ # run_pwsh, not subprocess.run: a pwsh killed at startup returns rc -6 with empty stdout,
+ # which this file would otherwise report as setup.ps1 choosing the wrong label.
+ return run_pwsh(
+ [
+ shutil.which("pwsh") or "pwsh",
+ "-NoProfile",
+ "-NonInteractive",
+ "-File",
+ str(script_path),
+ ],
+ capture_output = True,
+ text = True,
+ timeout = 120,
+ )
+
+
+requires_pwsh = pytest.mark.skipif(
+ shutil.which("pwsh") is None,
+ reason = "pwsh is required to execute the setup.ps1 status block",
+)
+requires_bash = pytest.mark.skipif(
+ shutil.which("bash") is None or os.name == "nt",
+ reason = "setup.sh is the POSIX installer",
+)
+
+# One table, run in both shells: `output` is what the installer printed, `label` what the user
+# must be told. The last three rows are the traps, where the two disagree.
+STATUS_CASES = [
+ (
+ "up-to-date",
+ 0,
+ "[whisper-prebuilt] existing whisper.cpp install already matches v1.9.1-unsloth.1 (cpu)",
+ "prebuilt up to date",
+ ),
+ (
+ "kept",
+ 0,
+ "[whisper-prebuilt] whisper.cpp update unavailable, existing prebuilt kept; keeping the "
+ "existing complete install of v1.9.1-unsloth.1",
+ "update unavailable, existing prebuilt kept",
+ ),
+ (
+ "installed",
+ 0,
+ "[whisper-prebuilt] installed whisper.cpp v1.9.2-unsloth.3 (cpu)",
+ "prebuilt installed",
+ ),
+ (
+ "kept-wins-over-installed-text",
+ 0,
+ "[whisper-prebuilt] downloading whisper-v1.9.1-unsloth.1-linux-x64-cpu.tar.gz\n"
+ "[whisper-prebuilt] whisper.cpp update unavailable, existing prebuilt kept; keeping the "
+ "existing complete install of v1.9.1-unsloth.1",
+ "update unavailable, existing prebuilt kept",
+ ),
+]
+
+# Non-zero exits: the label must never claim success, whatever the output says.
+FAILURE_CASES = [
+ ("plain-failure", 1, "[whisper-prebuilt] prebuilt install failed: could not fetch release"),
+ (
+ "failure-carrying-the-match-token",
+ 1,
+ "[whisper-prebuilt] checking whether the install already matches\n"
+ "[whisper-prebuilt] prebuilt install failed: checksum mismatch",
+ ),
+ (
+ "failure-carrying-the-keep-token",
+ 1,
+ "[whisper-prebuilt] considered keeping the existing complete install\n"
+ "[whisper-prebuilt] prebuilt install failed: archive member escaped destination",
+ ),
+ (
+ "unexpected-exit-carrying-both",
+ 137,
+ "[whisper-prebuilt] already matches\n"
+ "[whisper-prebuilt] keeping the existing complete install",
+ ),
+]
+
+_FAILURE_LABEL_PREFIX = "prebuilt install failed"
+
+
+@requires_bash
+ "name, status, output, label", STATUS_CASES, ids = [case[0] for case in STATUS_CASES]
+)
+def test_setup_sh_status_labels(tmp_path, name, status, output, label):
+ result = _run_bash(_sh_whisper_script(tmp_path, status, output))
+ assert result.returncode == 0, result
+ assert _steps(result.stdout).get("whisper.cpp") == label, result.stdout
+
+
+@requires_pwsh
+ "name, status, output, label", STATUS_CASES, ids = [case[0] for case in STATUS_CASES]
+)
+def test_setup_ps1_status_labels(tmp_path, name, status, output, label):
+ result = _run_ps1(tmp_path, _ps1_whisper_script(tmp_path, status, output))
+ assert result.returncode == 0, result
+ assert _steps(result.stdout).get("whisper.cpp") == label, (result.stdout, result.stderr)
+
+
+@requires_bash
+ "name, status, output", FAILURE_CASES, ids = [case[0] for case in FAILURE_CASES]
+)
+def test_setup_sh_never_reports_success_for_a_non_zero_exit(tmp_path, name, status, output):
+ """Success TEXT on a failed run must not become a success LABEL.
+
+ The installer writes its progress log and its error to the same stream, so a run that
+ got as far as printing either token and then died is exactly the shape that would turn
+ a broken dictation runtime into a green line if the grep ran before the exit check.
+ """
+ result = _run_bash(_sh_whisper_script(tmp_path, status, output))
+ assert result.returncode == 0, result
+ label = _steps(result.stdout).get("whisper.cpp")
+ assert label is not None, result.stdout
+ assert label.startswith(_FAILURE_LABEL_PREFIX), label
+ assert "up to date" not in label, label
+ assert "existing prebuilt kept" not in label, label
+
+
+@requires_pwsh
+ "name, status, output", FAILURE_CASES, ids = [case[0] for case in FAILURE_CASES]
+)
+def test_setup_ps1_never_reports_success_for_a_non_zero_exit(tmp_path, name, status, output):
+ result = _run_ps1(tmp_path, _ps1_whisper_script(tmp_path, status, output))
+ assert result.returncode == 0, (result.stdout, result.stderr)
+ label = _steps(result.stdout).get("whisper.cpp")
+ assert label is not None, (result.stdout, result.stderr)
+ assert label.startswith(_FAILURE_LABEL_PREFIX), label
+ assert "up to date" not in label, label
+ assert "existing prebuilt kept" not in label, label
+
+
+@requires_bash
+@requires_pwsh
+ "name, status, output, label", STATUS_CASES, ids = [case[0] for case in STATUS_CASES]
+)
+def test_both_shells_agree_on_the_label(tmp_path, name, status, output, label):
+ """Windows and POSIX must tell the user the same thing.
+
+ Asserting each side against a constant leaves them free to drift apart via the
+ constant; comparing the two measured labels is what makes "mirrored" a measurement.
+ """
+ sh = _run_bash(_sh_whisper_script(tmp_path, status, output))
+ ps = _run_ps1(tmp_path, _ps1_whisper_script(tmp_path, status, output))
+ assert sh.returncode == 0 and ps.returncode == 0, (sh, ps.stdout, ps.stderr)
+ assert _steps(sh.stdout).get("whisper.cpp") == _steps(ps.stdout).get("whisper.cpp") == label
+
+
+@requires_bash
+@requires_pwsh
+def test_a_success_token_inside_an_unrelated_line_still_decides_the_label(tmp_path):
+ """Characterisation, and a real limit of the contract, in both shells.
+
+ The grep is a plain substring on the WHOLE log, so any line that happens to contain
+ "already matches" wins the first arm even when the run actually installed something.
+ No shipped installer line does this today (the only producers are
+ existing_install_current_without_plan and the keep arm), so it is a latent precision
+ limit rather than a live bug -- and it can only ever choose between two SUCCESS labels,
+ never turn a failure into one, which the FAILURE_CASES above pin down.
+
+ This is here so that a future log line containing the token fails loudly instead of
+ silently relabelling a fresh install as "up to date".
+ """
+ output = (
+ "[whisper-prebuilt] no published release already matches this host's runtime; "
+ "downloading\n[whisper-prebuilt] installed whisper.cpp v1.9.2-unsloth.3 (cpu)"
+ )
+ sh = _run_bash(_sh_whisper_script(tmp_path, 0, output))
+ ps = _run_ps1(tmp_path, _ps1_whisper_script(tmp_path, 0, output))
+ assert _steps(sh.stdout).get("whisper.cpp") == "prebuilt up to date"
+ assert _steps(ps.stdout).get("whisper.cpp") == "prebuilt up to date"
+
+
+@requires_bash
+@requires_pwsh
+def test_the_two_shells_differ_on_token_case(tmp_path):
+ """Characterisation of a real divergence with no live trigger.
+
+ ``grep -Fq`` is case sensitive; PowerShell's ``-match`` is case INSENSITIVE by default.
+ So a log saying "Already Matches" is reported as a fresh install on Linux and as up to
+ date on Windows. Every line the installer actually emits is lowercase, so nothing
+ triggers it today -- but it is a divergence in the contract, recorded here rather than
+ left to be discovered by the first capitalised log line.
+ """
+ output = "[whisper-prebuilt] existing whisper.cpp install Already Matches v1.9.1-unsloth.1"
+ sh = _run_bash(_sh_whisper_script(tmp_path, 0, output))
+ ps = _run_ps1(tmp_path, _ps1_whisper_script(tmp_path, 0, output))
+ assert _steps(sh.stdout).get("whisper.cpp") == "prebuilt installed", sh.stdout
+ assert _steps(ps.stdout).get("whisper.cpp") == "prebuilt up to date", ps.stdout
+
+
+@requires_bash
+def test_setup_sh_routes_the_remaining_whisper_exit_codes(tmp_path):
+ """Exit 2 and exit 3 keep their own labels; neither is the new keep arm."""
+ busy = _run_bash(_sh_whisper_script(tmp_path, 3, "[whisper-prebuilt] install busy"))
+ assert _steps(busy.stdout)["whisper.cpp"] == "install busy; keeping existing runtime"
+
+ skew = _run_bash(
+ _sh_whisper_script(tmp_path, 2, "[whisper-prebuilt] slim bundle requires llama.cpp b9999;")
+ )
+ label = _steps(skew.stdout)["whisper.cpp"]
+ assert label.startswith("no compatible prebuilt ("), label
+ assert "b9999" in label, label
+ assert "existing prebuilt kept" not in label, label
+
+
+@requires_pwsh
+def test_setup_ps1_routes_the_remaining_whisper_exit_codes(tmp_path):
+ busy = _run_ps1(tmp_path, _ps1_whisper_script(tmp_path, 3, "[whisper-prebuilt] install busy"))
+ assert _steps(busy.stdout)["whisper.cpp"] == "install busy; keeping existing runtime"
+
+ skew = _run_ps1(
+ tmp_path,
+ _ps1_whisper_script(
+ tmp_path, 2, "[whisper-prebuilt] slim bundle requires llama.cpp b9999;"
+ ),
+ name = "skew.ps1",
+ )
+ label = _steps(skew.stdout)["whisper.cpp"]
+ assert label.startswith("no compatible prebuilt ("), label
+ assert "b9999" in label, label
+ assert "existing prebuilt kept" not in label, label
+
+
+_LLAMA_KEEP_LOG = (
+ "[llama-prebuilt] llama.cpp update unavailable, existing prebuilt kept; keeping the "
+ "existing complete install of b1000"
+)
+
+
+@requires_bash
+def test_setup_sh_reports_each_component_separately(tmp_path):
+ """llama keeps its install while whisper's update fails: two labels, no cross-talk.
+
+ Both blocks read from their own log file and their own status variable, and llama's
+ keep arm must not leak into whisper's line (or the reverse) just because the same
+ token appears in the same setup run.
+ """
+ llama_log = tmp_path / "prebuilt.log"
+ llama_log.write_text(_LLAMA_KEEP_LOG + "\n", encoding = "utf-8")
+ whisper_log = tmp_path / "whisper.log"
+ whisper_log.write_text(
+ "[whisper-prebuilt] prebuilt install failed: could not fetch release\n", encoding = "utf-8"
+ )
+ script = "\n".join(
+ [
+ _SH_HARNESS,
+ f'SCRIPT_DIR="{tmp_path}"',
+ f'UNSLOTH_HOME="{tmp_path}"',
+ f'LLAMA_CPP_DIR="{tmp_path / "llama.cpp"}"',
+ f'WHISPER_CPP_DIR="{tmp_path / "whisper.cpp"}"',
+ f'_PREBUILT_LOG="{llama_log}"',
+ "_PREBUILT_STATUS=0",
+ '_explicit_llama_source_backend=""',
+ _sh_block("llama"),
+ f'_WHISPER_LOG="{whisper_log}"',
+ "_WHISPER_STATUS=1",
+ _sh_block("whisper"),
+ ]
+ )
+ result = _run_bash(script)
+ assert result.returncode == 0, result
+ steps = _steps(result.stdout)
+ assert steps["llama.cpp"] == "update unavailable, existing prebuilt kept", steps
+ assert steps["whisper.cpp"].startswith(_FAILURE_LABEL_PREFIX), steps
+ # A failed whisper never aborts setup: dictation is fail-open.
+ assert "setup_fail|" not in result.stdout, result.stdout
+
+
+@requires_pwsh
+def test_setup_ps1_reports_each_component_separately(tmp_path):
+ script = "\n".join(
+ [
+ _PS1_HARNESS,
+ f'$LlamaCppDir = "{tmp_path / "llama.cpp"}"',
+ f'$WhisperCppDir = "{tmp_path / "whisper.cpp"}"',
+ "$prebuiltExit = 0",
+ "$prebuiltOutput = @'",
+ _LLAMA_KEEP_LOG,
+ "'@",
+ _ps1_block("llama"),
+ "$whisperExit = 1",
+ "$whisperOutput = @'",
+ "[whisper-prebuilt] prebuilt install failed: could not fetch release",
+ "'@",
+ _ps1_block("whisper"),
+ ]
+ )
+ result = _run_ps1(tmp_path, script, name = "both.ps1")
+ assert result.returncode == 0, (result.stdout, result.stderr)
+ steps = _steps(result.stdout)
+ assert steps["llama.cpp"] == "update unavailable, existing prebuilt kept", steps
+ assert steps["whisper.cpp"].startswith(_FAILURE_LABEL_PREFIX), steps
+ assert "setup_fail|" not in result.stdout, result.stdout
+
+
+@requires_bash
+def test_the_installer_emits_exactly_the_substrings_the_scripts_grep(tmp_path, monkeypatch, capsys):
+ """End to end: the real installer log, fed to the real setup.sh block.
+
+ Every other test in this file asserts one half. This one runs the installer offline over
+ an intact install, takes whatever it printed, and hands it to the shipped shell block --
+ so a reworded log line shows up as the wrong user-facing label, not as a passing test.
+ """
+ host = _host("linux", "x64")
+ install_dir = _seed_install(tmp_path, host)
+ _inject(monkeypatch, "offline")
+ code, log = _run_cli(monkeypatch, capsys, host, install_dir)
+ assert code == M.EXIT_SUCCESS, log
+
+ result = _run_bash(_sh_whisper_script(tmp_path, code, log))
+ assert (
+ _steps(result.stdout)["whisper.cpp"] == "update unavailable, existing prebuilt kept"
+ ), result.stdout
+
+
+# An untrustworthy release is an ANSWER, not an unavailability
+# The keep arm exists because a lookup that could not answer says nothing about the tree on disk.
+# A manifest digest disagreeing with the checksum index is the opposite: the release was fetched
+# and found untrustworthy. Reporting "update unavailable, existing prebuilt kept" over it turns a
+# tamper signal into a routine offline notice, and setup then paints it yellow rather than red.
+# Unit coverage of the exception TYPE is not enough here: with the re-raise removed the whole
+# install suite stayed green and only an end-to-end run noticed, which is why this drives the CLI.
+def test_a_tampered_release_is_not_reported_as_an_unavailable_update(tmp_path, monkeypatch, capsys):
+ host = _host("linux", "x64")
+ install_dir = _seed_install(tmp_path, host)
+ before = _tree_snapshot(install_dir)
+
+ def tampered(*_a, **_k):
+ raise M.core.ReleaseIntegrityError(
+ "manifest sha256 for whisper-x.tar.gz disagrees with whisper-prebuilt-sha256.json; "
+ "refusing a possibly tampered release"
+ )
+
+ monkeypatch.setattr(M, "_release_plan_for_host", tampered)
+ code, log = _run_cli(monkeypatch, capsys, host, install_dir)
+
+ assert code != M.EXIT_SUCCESS, f"a tampered release reported success\n{log}"
+ assert KEEP_TOKEN not in log, f"the keep arm swallowed an integrity failure\n{log}"
+ assert "tampered" in log, f"the reason never reached the user\n{log}"
+ # Refusing to bless it must not damage the install that is already there.
+ assert _tree_snapshot(install_dir) == before, "the intact tree moved"
+
+
+def test_an_unavailable_lookup_still_keeps_the_install(tmp_path, monkeypatch, capsys):
+ """The other half of the same fork, so the fix above cannot be 'refuse everything'."""
+ host = _host("linux", "x64")
+ install_dir = _seed_install(tmp_path, host)
+
+ def unavailable(*_a, **_k):
+ raise M.PrebuiltFallback("could not fetch release: network is unreachable")
+
+ monkeypatch.setattr(M, "_release_plan_for_host", unavailable)
+ code, log = _run_cli(monkeypatch, capsys, host, install_dir)
+
+ assert code == M.EXIT_SUCCESS, f"an unavailable lookup should keep the install\n{log}"
+ assert KEEP_TOKEN in log, log
diff --git a/tests/studio/install/test_pr10648_review_fixes.py b/tests/studio/install/test_pr10648_review_fixes.py
new file mode 100644
index 00000000000..5b4072ec358
--- /dev/null
+++ b/tests/studio/install/test_pr10648_review_fixes.py
@@ -0,0 +1,616 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Review follow-ups to the prebuilt marker fast path.
+
+Two defects the marker work exposed, each with the user situation it costs:
+
+ * the llama marker rewriter asked os.chown for the owner as well as the group, which a
+ non-root member of a group-shared install cannot grant. chown is all-or-nothing, so the
+ group was not applied either and os.replace installed the member's primary group,
+ leaving the marker unreadable to everyone else who shares the install. prebuilt_core
+ was fixed for this; the llama writer was not, and this work gave it a new caller
+ (whisper's slim pairing backfill).
+
+ * UNSLOTH_PREBUILT_FULL_CHECK turns the no-network shortcut off for llama and whisper.
+ Node never read it, so the one variable documented as "force a full revalidation" left
+ a third of the runtime answered from its marker.
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+from _pr10648_helpers import NEEDS_CHOWN as requires_chown
+from _pr10648_helpers import load_studio_module as _load
+
+LLAMA = _load("studio_install_llama_prebuilt_pr10648_fixes", "install_llama_prebuilt.py")
+CORE = _load("studio_prebuilt_core_pr10648_fixes", "prebuilt_core.py")
+NODE = _load("studio_install_node_prebuilt_pr10648_fixes", "install_node_prebuilt.py")
+
+
+def _node_host():
+ return NODE.HostInfo(
+ system = "Linux",
+ machine = "x64",
+ node_os = "linux",
+ node_arch = "x64",
+ archive_ext = ".tar.gz",
+ is_windows = False,
+ )
+
+
+def _node_tree(root: Path, host) -> None:
+ node = NODE.node_binary_path(root, host)
+ npm = NODE.npm_cli_path(root, host)
+ node.parent.mkdir(parents = True, exist_ok = True)
+ npm.parent.mkdir(parents = True, exist_ok = True)
+ node.write_bytes(b"node" * 64)
+ npm.write_bytes(b"npm" * 64)
+ node.chmod(0o755)
+
+
+@requires_chown
+ "writer_name",
+ ["llama._write_marker", "core.write_live_marker"],
+)
+def test_a_live_marker_rewrite_falls_back_to_the_group_when_the_owner_is_refused(
+ tmp_path, monkeypatch, writer_name
+):
+ """Both marker rewriters must ask for owner AND group, then for the group alone.
+
+ Neither half is sufficient. A shared install's marker is owned by whoever installed it,
+ and chown(2) refuses the WHOLE call when an unprivileged caller names another owner, so
+ asking only for the owner loses the group too and the next reader in that group cannot
+ read the marker. Asking only for the group is wrong the other way: under root, the one
+ caller that CAN restore the owner, it leaves the marker owned by root, and an 0600
+ marker stops being readable by the user who owns the install.
+ """
+ marker = tmp_path / "MARKER.json"
+ marker.write_text('{"release_tag": "b10840"}\n', encoding = "utf-8")
+ original = marker.stat()
+
+ calls = []
+ if writer_name == "llama._write_marker":
+ module, write = LLAMA, lambda: LLAMA._write_marker(marker, {"release_tag": "b10841"})
+ else:
+ module, write = CORE, lambda: CORE.write_live_marker(marker, {"release_tag": "b10841"})
+
+ def refusing(path, uid, gid):
+ calls.append((uid, gid))
+ if uid != -1:
+ raise PermissionError("a non-root member may not give a file away")
+
+ monkeypatch.setattr(module.os, "chown", refusing)
+ write()
+
+ assert calls == [(original.st_uid, original.st_gid), (-1, original.st_gid)], (
+ f"{writer_name} asked for {calls}; it must try the owner first and fall back to the "
+ "group, so a non-root member keeps the group and root restores the owner"
+ )
+
+
+@requires_chown
+def test_the_group_survives_a_rewrite_the_owner_cannot_be_granted(tmp_path, monkeypatch):
+ """The end state, not just the argument: a refused owner must not cost the group.
+
+ os.chown is emulated with the kernel's own rule -- an unprivileged caller may not change
+ the owner, and chown is all-or-nothing -- so this fails for any writer that names one.
+ """
+ marker = tmp_path / "MARKER.json"
+ marker.write_text('{"release_tag": "b10840"}\n', encoding = "utf-8")
+ shared_gid = marker.stat().st_gid
+ applied = []
+
+ def unprivileged_chown(path, uid, gid):
+ if uid != -1:
+ raise PermissionError(1, "Operation not permitted")
+ applied.append(gid)
+
+ monkeypatch.setattr(LLAMA.os, "chown", unprivileged_chown)
+ assert LLAMA._write_marker(marker, {"release_tag": "b10841"}) is True
+ assert applied == [shared_gid], "the group was never applied to the replacement"
+
+
+@requires_chown
+def test_a_marker_rewrite_keeps_the_keys_the_about_tab_renders(tmp_path, monkeypatch):
+ """release_tag and tag reach the user through /api/system/hardware and the About tab.
+
+ A rewrite that drops them changes what Studio displays, so pin that they survive.
+ """
+ marker = tmp_path / "MARKER.json"
+ marker.write_text('{"release_tag": "b10840", "tag": "b10840"}\n', encoding = "utf-8")
+ monkeypatch.setattr(LLAMA.os, "chown", lambda path, uid, gid: None)
+
+ assert LLAMA._write_marker(marker, {"release_tag": "b10841", "tag": "b10841"}) is True
+
+ import json
+
+ payload = json.loads(marker.read_text(encoding = "utf-8"))
+ assert payload["release_tag"] == "b10841"
+ assert payload["tag"] == "b10841"
+ assert not list(tmp_path.glob("MARKER.json.tmp-*")), "a temp marker was left behind"
+
+
+def test_node_honours_the_full_check_escape_hatch(tmp_path, monkeypatch):
+ """UNSLOTH_PREBUILT_FULL_CHECK is the documented way to force a full revalidation.
+
+ llama and whisper both turn their shortcut off for it. Node did not, so the variable
+ silently covered two of the three runtimes.
+ """
+ host = _node_host()
+ _node_tree(tmp_path, host)
+ NODE.write_metadata(tmp_path, version = "24.17.0", asset = "x", sha256 = "y")
+ NODE.record_runtime_verification(tmp_path, host, version = "24.17.0", npm_major = 11)
+
+ meta = NODE.load_metadata(tmp_path)
+ monkeypatch.delenv("UNSLOTH_PREBUILT_FULL_CHECK", raising = False)
+ assert (
+ NODE._recorded_runtime_matches(tmp_path, host, meta, "24.17.0") is True
+ ), "the recorded runtime should answer when the bypass is unset"
+
+ monkeypatch.setenv("UNSLOTH_PREBUILT_FULL_CHECK", "1")
+ assert (
+ NODE._recorded_runtime_matches(tmp_path, host, meta, "24.17.0") is False
+ ), "UNSLOTH_PREBUILT_FULL_CHECK=1 must send Node down the spawning path"
+
+
[email protected]("value", ["1", "true", "TRUE", "yes", "on", " on "])
+def test_the_node_bypass_spells_it_the_way_llama_does(monkeypatch, value):
+ """One variable, one spelling. A hatch that answers differently per component is a hatch
+ nobody can rely on."""
+ monkeypatch.setenv("UNSLOTH_PREBUILT_FULL_CHECK", value)
+ assert NODE.prebuilt_full_check_requested() is True
+ assert LLAMA.prebuilt_full_check_requested() is True
+
+
[email protected]("value", ["", "0", "false", "no", "off", "maybe"])
+def test_the_node_bypass_stays_off_for_anything_else(monkeypatch, value):
+ """An unset or negative value must leave the fast path in place, or every update pays
+ the spawns forever."""
+ monkeypatch.setenv("UNSLOTH_PREBUILT_FULL_CHECK", value)
+ assert NODE.prebuilt_full_check_requested() is False
+ assert LLAMA.prebuilt_full_check_requested() is False
+
+
+# The migration run: a pre-record marker has no digest and, on Windows, no loader preflight, so
+# this run decides whether the bytes on disk become the reference every later run is held to.
+_LOGIC = _load("studio_install_llama_prebuilt_pr10648_logic", "install_llama_prebuilt.py")
+
+
+def _legacy_marker_install(tmp_path, helpers, windows: bool):
+ """A healthy tree under a marker with no runtime_files, i.e. every install made before
+ this record existed."""
+ install_dir = tmp_path / "llama.cpp"
+ install_dir.mkdir()
+ if windows:
+ helpers.write_windows_install_shape(install_dir, include_llama_dll = True)
+ host = helpers.windows_host()
+ choice = helpers.asset_choice(
+ name = "llama-b9001-bin-win-cpu-x64.zip",
+ url = "https://example.com/x.zip",
+ source_label = "published",
+ install_kind = "windows-cpu",
+ )
+ repo = helpers.PREBUILT
+ else:
+ helpers.write_linux_install_shape(install_dir)
+ host = helpers.linux_host()
+ choice = helpers.asset_choice()
+ repo = helpers.UPSTREAM
+ checksums = helpers.release_checksums((choice.name, choice.expected_sha256, repo))
+ plan = helpers.release_plan([choice], checksums)
+ helpers.write_metadata(install_dir, choice, checksums)
+ # Strip the keys that postdate this work, which is what an older Studio's marker looks like.
+ # runtime_files is not a fingerprint input, so the stripped marker stays self-consistent.
+ import json as _json
+
+ marker_path = install_dir / "UNSLOTH_PREBUILT_INFO.json"
+ payload = _json.loads(marker_path.read_text(encoding = "utf-8"))
+ payload.pop("runtime_files", None)
+ payload.pop("host_profile", None)
+ marker_path.write_text(_json.dumps(payload, indent = 2) + "\n", encoding = "utf-8")
+ assert "runtime_files" not in (LLAMA.load_prebuilt_metadata(install_dir) or {})
+ return install_dir, host, plan
+
+
[email protected](scope = "module")
+def helpers():
+ return _load_logic_helpers()
+
+
+def _load_logic_helpers():
+ import importlib.util as _iu
+
+ path = Path(__file__).with_name("test_install_llama_prebuilt_logic.py")
+ spec = _iu.spec_from_file_location("pr10648_logic_helpers", path)
+ assert spec is not None and spec.loader is not None
+ module = _iu.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+def test_a_legacy_marker_is_not_blessed_without_asking_the_loader(tmp_path, monkeypatch, helpers):
+ """A Windows install damaged under a pre-record marker must not have its damage recorded.
+
+ Windows gets neither of the loader preflights below, and a marker with no runtime_files has
+ no digest to fail, so before this the only integrity test was "the file is not zero bytes".
+ The next run would then be compared against the damaged bytes for ever.
+ """
+ install_dir, host, plan = _legacy_marker_install(tmp_path, helpers, windows = True)
+
+ probed = []
+ monkeypatch.setattr(
+ helpers.INSTALL_LLAMA_PREBUILT,
+ "_binary_image_runs",
+ lambda path, d, h, line = None: (probed.append(Path(path).name), False)[1],
+ )
+ assert (
+ helpers.existing_install_matches_plan(install_dir, host, plan) is False
+ ), "an image the OS refuses to start was accepted as a current install"
+ assert probed, "the loader was never asked, so nothing checked the bytes"
+
+
+def test_a_healthy_legacy_windows_install_still_migrates(tmp_path, monkeypatch, helpers):
+ """The other half: the guard must not cost a working install its one migration run."""
+ install_dir, host, plan = _legacy_marker_install(tmp_path, helpers, windows = True)
+
+ probed = []
+ monkeypatch.setattr(
+ helpers.INSTALL_LLAMA_PREBUILT,
+ "_binary_image_runs",
+ lambda path, d, h, line = None: (probed.append(Path(path).name), True)[1],
+ )
+ assert helpers.existing_install_matches_plan(install_dir, host, plan) is True
+ assert probed, "the migration run should have asked the loader once"
+
+
+def test_a_linux_legacy_install_is_covered_by_its_preflight_instead(tmp_path, monkeypatch, helpers):
+ """Linux and macOS already run a real preflight below, so they must not pay a spawn too."""
+ install_dir, host, plan = _legacy_marker_install(tmp_path, helpers, windows = False)
+
+ probed = []
+ monkeypatch.setattr(
+ helpers.INSTALL_LLAMA_PREBUILT,
+ "_binary_image_runs",
+ lambda path, d, h, line = None: (probed.append(Path(path).name), True)[1],
+ )
+ helpers.existing_install_matches_plan(install_dir, host, plan)
+ assert probed == [], "Linux paid a --version spawn its ldd preflight already covers"
+
+
+def test_a_marker_that_vanishes_between_the_two_reads_does_not_crash(tmp_path, monkeypatch):
+ """_backfill_fingerprint_inputs reads the marker a second time.
+
+ Another installer swapping the tree in between leaves None, and the old code called
+ .get on it. The backfill has nothing to catch up at that point, so falling out is the
+ answer -- the run that replaced the tree wrote its own marker.
+ """
+ calls = []
+
+ class Ops:
+ def load_prebuilt_metadata(self, install_dir):
+ # Present for _kept_marker_patch's read, gone for the backfill's.
+ calls.append(1)
+ return {"install_fingerprint": "abc"} if len(calls) == 1 else None
+
+ def metadata_path(self, install_dir):
+ return tmp_path / "MARKER.json"
+
+ selection = type(
+ "Sel",
+ (),
+ {
+ "coverage": {"min_os": None},
+ "walk_back": None,
+ "platform_os": "linux",
+ "platform_arch": "x64",
+ "fingerprint": lambda self: "abc",
+ },
+ )()
+
+ monkeypatch.setattr(CORE, "_kept_marker_patch", lambda ops, d, s: {"fingerprint_coverage": {}})
+ # The bug was an AttributeError here, not a wrong answer.
+ CORE._backfill_fingerprint_inputs(Ops(), tmp_path, selection)
+
+
+def test_a_blocked_marker_swap_is_retried_then_reported(tmp_path, monkeypatch):
+ """Rename-over needs DELETE access on the destination, so a scanner holding the marker
+ open fails the swap where the in-place write this replaced would have succeeded."""
+ source = tmp_path / "tmp-marker"
+ source.write_text("{}", encoding = "utf-8")
+ destination = tmp_path / "MARKER.json"
+
+ blocked = OSError(13, "in use")
+ blocked.winerror = 32
+ attempts = []
+
+ def blocked_then_ok(src, dst):
+ attempts.append(1)
+ if len(attempts) < 3:
+ raise blocked
+ Path(src).rename(dst)
+
+ # Swap the MODULE's os reference, never os.name itself: pathlib reads os.name at runtime, so
+ # setting it globally makes every path in the process a backslash one.
+ class FakeOs:
+ name = "nt"
+ replace = staticmethod(blocked_then_ok)
+
+ def __getattr__(self, item):
+ return getattr(os, item)
+
+ monkeypatch.setattr(CORE, "os", FakeOs())
+ monkeypatch.setattr(CORE.time, "sleep", lambda _s: None)
+
+ CORE.atomic_replace_from_tempfile(source, destination)
+ assert len(attempts) == 3, "the blocked swap was not retried"
+ assert destination.is_file()
+
+
+def test_a_swap_blocked_for_any_other_reason_raises_at_once(tmp_path, monkeypatch):
+ """A real problem must not be turned into an eight-step stall."""
+ source = tmp_path / "tmp-marker"
+ source.write_text("{}", encoding = "utf-8")
+ attempts = []
+
+ def replace(src, dst):
+ attempts.append(1)
+ raise OSError(28, "No space left on device")
+
+ monkeypatch.setattr(CORE.os, "replace", replace)
+ with pytest.raises(OSError):
+ CORE.atomic_replace_from_tempfile(source, tmp_path / "MARKER.json")
+ assert len(attempts) == 1, "a non-transient failure was retried"
+
+
+# The walk-back records why AUTOMATIC selection took an older release, so reusing it for a run
+# that NAMED one answers "already matches N" while N is not installed. The full path turns off
+# older-release fallback as soon as published_release_tag is supplied.
+def _mac(macos_version = (14, 7)):
+ return LLAMA.HostInfo(
+ system = "Darwin",
+ machine = "arm64",
+ is_linux = False,
+ is_windows = False,
+ is_macos = True,
+ is_x86_64 = False,
+ is_arm64 = True,
+ nvidia_smi = None,
+ driver_cuda_version = None,
+ compute_caps = [],
+ visible_cuda_devices = None,
+ has_physical_nvidia = False,
+ has_usable_nvidia = False,
+ macos_version = macos_version,
+ )
+
+
+def _walked_back_marker(
+ host,
+ *,
+ installed = "b9998",
+ skipped = "b9999",
+):
+ return {
+ "release_tag": installed,
+ "tag": installed,
+ "walked_back_from": skipped,
+ "walked_back_on_macos": CORE.macos_version_label(host),
+ }
+
+
+def test_a_pinned_release_is_not_satisfied_by_a_walked_back_install():
+ host = _mac()
+ marker = _walked_back_marker(host)
+ assert LLAMA._release_expectation_met(marker, "b9999", host, pinned = True) is False
+
+
+def test_an_automatic_request_still_accepts_the_walk_back():
+ """The optimisation this PR exists for must survive the fix above: without a pin, a Mac
+ holding the older release because the newest needs a newer OS is still current."""
+ host = _mac()
+ marker = _walked_back_marker(host)
+ assert LLAMA._release_expectation_met(marker, "b9999", host, pinned = False) is True
+
+
+def test_a_pinned_release_that_is_actually_installed_is_still_current():
+ """The fix must not send every pinned request down the full path."""
+ host = _mac()
+ marker = _walked_back_marker(host, installed = "b9999", skipped = "b9999")
+ assert LLAMA._release_expectation_met(marker, "b9999", host, pinned = True) is True
+
+
+def test_the_pin_reaches_the_expectation_check(tmp_path, monkeypatch):
+ """The guard is only worth anything if the caller actually passes the pin, so drive the
+ real entry point and record what it asked."""
+ seen = []
+ monkeypatch.setattr(
+ LLAMA,
+ "_release_expectation_met",
+ lambda marker, expected, host, *, pinned = False: seen.append(pinned) or False,
+ )
+ host = _mac()
+ # Everything the guards AHEAD of the release check demand: failing an earlier one would
+ # make this pass vacuously, with the pin never computed at all.
+ monkeypatch.setattr(
+ LLAMA,
+ "load_prebuilt_metadata",
+ lambda *_a, **_k: {
+ "install_fingerprint": "x" * 64,
+ "backend_request": "auto",
+ "force_cpu": False,
+ "prebuilt_fallback_used": False,
+ "published_repo": LLAMA.DEFAULT_PUBLISHED_REPO,
+ "backend": "metal",
+ "host_profile": LLAMA.host_profile(host),
+ },
+ )
+ monkeypatch.setattr(LLAMA, "_marker_backend_fits_host", lambda *_a, **_k: True)
+ monkeypatch.setattr(LLAMA, "_runtime_preference_moved", lambda *_a, **_k: False)
+ route = LLAMA.BackendRoute(
+ backend = "auto",
+ host = host,
+ published_repo = LLAMA.DEFAULT_PUBLISHED_REPO,
+ published_release_tag = "b9999",
+ persist_llama_backend = None,
+ persist_rocm_gfx = None,
+ )
+ LLAMA.existing_install_current_without_plan(
+ tmp_path,
+ llama_tag = "latest",
+ published_repo = LLAMA.DEFAULT_PUBLISHED_REPO,
+ published_release_tag = "b9999",
+ backend_request = "auto",
+ force_cpu = False,
+ route = route,
+ )
+ assert seen == [True], f"the pin never reached the expectation check: {seen}"
+
+
+# The mode restore protects a REFRESH, which has another reader. A first write has neither a
+# mode to preserve nor a reader, and raising there aborts a whole Node install over a cosmetic
+# call, reachable on Windows through the sharing violation the swap already retries.
+def _refuse_mode_change(monkeypatch, module):
+ def refuse(path, mode):
+ raise PermissionError(13, "Permission denied")
+
+ monkeypatch.setattr(module.os, "chmod", refuse)
+
+
+def test_a_first_node_marker_survives_a_refused_mode_restore(tmp_path, monkeypatch):
+ install_dir = tmp_path / "node"
+ install_dir.mkdir()
+ _refuse_mode_change(monkeypatch, NODE)
+ NODE.write_metadata(install_dir, version = "24.17.0", asset = "a.tar.gz", sha256 = "b" * 64)
+ assert NODE.load_metadata(install_dir)["version"] == "24.17.0"
+ assert not list(install_dir.glob("*.tmp-*")), "a temp marker was left behind"
+
+
+def test_a_node_marker_refresh_abandons_on_a_refused_mode_restore(tmp_path, monkeypatch):
+ """The refresh is where the harm is: the existing marker must be left exactly as it was,
+ rather than replaced by one only its writer can read."""
+ install_dir = tmp_path / "node"
+ install_dir.mkdir()
+ NODE.write_metadata(install_dir, version = "24.17.0", asset = "a.tar.gz", sha256 = "b" * 64)
+ marker = NODE.metadata_path(install_dir)
+ before = marker.read_bytes()
+
+ _refuse_mode_change(monkeypatch, NODE)
+ with pytest.raises(OSError):
+ NODE.write_metadata(install_dir, version = "25.0.0", asset = "c.tar.gz", sha256 = "d" * 64)
+ assert marker.read_bytes() == before, "the good marker was replaced anyway"
+ assert not list(install_dir.glob("*.tmp-*")), "a temp marker was left behind"
+
+
+def test_the_runtime_record_refresh_still_never_raises(tmp_path, monkeypatch):
+ """record_runtime_verification is best effort by contract: an unrefreshable marker costs
+ the two spawns again, and must never take the caller down with it."""
+ host = _node_host()
+ _node_tree(tmp_path, host)
+ NODE.write_metadata(tmp_path, version = "24.17.0", asset = "a", sha256 = "b" * 64)
+ _refuse_mode_change(monkeypatch, NODE)
+ NODE.record_runtime_verification(tmp_path, host, version = "24.17.0", npm_major = 11)
+
+
+# A release found untrustworthy must not read as "update unavailable"
+# The keep paths exist because a lookup that could not ANSWER says nothing about the tree on disk.
+# An asset outside the checksum index, or a manifest digest disagreeing with it, is the opposite:
+# the release was fetched and found untrustworthy. Reporting "update unavailable, existing prebuilt
+# kept" over that turns a tamper signal into a routine offline notice.
+def _ops():
+ """The one name expected_sha256_for reads, over prebuilt_core's own defaults. A component
+ module supplies SHA256_ASSET_NAME; neither core nor llama defines it, and without it the
+ call fails for a reason that has nothing to do with integrity."""
+ return CORE.ModuleOps({**vars(CORE), "SHA256_ASSET_NAME": "whisper-prebuilt-sha256.json"})
+
+
+def test_an_unverifiable_asset_raises_the_integrity_type():
+ with pytest.raises(CORE.ReleaseIntegrityError):
+ CORE.expected_sha256_for(_ops(), {}, "whisper-x.tar.gz")
+
+
+def test_a_manifest_digest_disagreeing_with_the_index_raises_the_integrity_type():
+ with pytest.raises(CORE.ReleaseIntegrityError):
+ CORE.expected_sha256_for(
+ _ops(),
+ {"whisper-x.tar.gz": "a" * 64},
+ "whisper-x.tar.gz",
+ manifest_sha256 = "b" * 64,
+ )
+
+
+def test_the_integrity_type_is_still_a_prebuilt_fallback():
+ """A subclass, so every existing `except PrebuiltFallback` keeps catching it and only the
+ keep paths single it out. A sibling type would silently escape those handlers."""
+ assert issubclass(CORE.ReleaseIntegrityError, CORE.PrebuiltFallback)
+
+
+def test_a_plain_lookup_failure_is_not_an_integrity_error():
+ """The distinction has to cut both ways, or the keep path stops working at all."""
+ assert not isinstance(CORE.PrebuiltFallback("offline"), CORE.ReleaseIntegrityError)
+
+
+# llama's keep arm must refuse an untrustworthy release too
+# llama has kept an install on a failed lookup since before this branch, and that arm caught every
+# PrebuiltFallback including the integrity refusals. Closed here so the two installers cannot
+# disagree about what a keep is allowed to hide. The condition is inline, so this drives the real
+# install_prebuilt: with the guard removed the whole install suite stayed green.
+def _llama_keep_probe(monkeypatch, tmp_path, raised):
+ """Run llama's install_prebuilt with the planner raising *raised*, over a tree its own
+ _existing_install_runs accepts, and report whether it took the keep arm."""
+ install_dir = tmp_path / "llama.cpp"
+ (install_dir / "build" / "bin").mkdir(parents = True)
+ for name in ("llama-server", "llama-quantize"):
+ for target in (install_dir / name, install_dir / "build" / "bin" / name):
+ target.write_bytes(b"#!/bin/sh\nexit 0\n")
+ target.chmod(0o755)
+
+ monkeypatch.setattr(LLAMA, "_existing_install_runs", lambda *_a, **_k: True)
+ monkeypatch.setattr(
+ LLAMA,
+ "resolve_simple_install_release_plans",
+ lambda *_a, **_k: (_ for _ in ()).throw(raised),
+ )
+ monkeypatch.setattr(LLAMA, "collect_system_report", lambda *_a, **_k: "")
+
+ logs: list[str] = []
+ monkeypatch.setattr(LLAMA, "log", lambda msg, *a, **k: logs.append(str(msg)))
+ monkeypatch.setattr(LLAMA, "log_lines", lambda msg, *a, **k: logs.append(str(msg)))
+ outcome = "returned"
+ try:
+ LLAMA.install_prebuilt(
+ install_dir = install_dir,
+ llama_tag = "latest",
+ published_repo = LLAMA.DEFAULT_PUBLISHED_REPO,
+ published_release_tag = "",
+ )
+ except SystemExit as exc:
+ outcome = f"SystemExit({exc.code})"
+ except BaseException as exc:
+ outcome = f"{type(exc).__name__}"
+ text = "\n".join(logs)
+ return outcome, ("keeping the existing complete install" in text), text
+
+
+def test_llama_keeps_the_install_when_the_lookup_could_not_answer(tmp_path, monkeypatch):
+ outcome, kept, text = _llama_keep_probe(
+ monkeypatch,
+ tmp_path,
+ LLAMA.PrebuiltFallback("network is unreachable"),
+ )
+ assert kept, f"an unavailable lookup should keep the install; got {outcome}\n{text[:400]}"
+
+
+def test_llama_refuses_to_keep_over_an_untrustworthy_release(tmp_path, monkeypatch):
+ outcome, kept, text = _llama_keep_probe(
+ monkeypatch,
+ tmp_path,
+ LLAMA._core.ReleaseIntegrityError(
+ "manifest sha256 for app-x.tar.gz disagrees with llama-prebuilt-sha256.json; "
+ "refusing a possibly tampered release"
+ ),
+ )
+ assert not kept, f"the keep arm swallowed an integrity failure; got {outcome}\n{text[:400]}"
+ assert "tampered" in text, f"the reason never reached the user\n{text[:400]}"
diff --git a/tests/studio/install/test_pr10648_whisper_payload_digests.py b/tests/studio/install/test_pr10648_whisper_payload_digests.py
new file mode 100644
index 00000000000..a410e8e72d3
--- /dev/null
+++ b/tests/studio/install/test_pr10648_whisper_payload_digests.py
@@ -0,0 +1,873 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The whisper.cpp payload record: can an update keep an install whose bytes rotted?
+
+Before this, a whisper marker recorded nothing about the files it installed. The reuse
+predicate (``existing_install_matches``) and the no-network fast path
+(``_existing_install_is_intact``) both ask ``installed_tree_is_intact``, and all it could
+establish was that ``whisper-server`` was a non-empty file and (off Windows) executable.
+So a server truncated by a full disk, or a hardlinked ``libggml-base.so`` left as a stub,
+was reported "already matches" and the user met the failure when they pressed the
+dictation key. llama.cpp had no such gap: it records ``runtime_files`` (size + sha256 per
+runtime file) and compares them in ``_runtime_files_match``.
+
+whisper now records the same key in the same shape. The tests here are the boundary:
+
+ * PART 1 -- what is recorded, and that it is legible beside the llama marker.
+ * PART 2 -- the four corruptions that must now be REJECTED, each asserted healthy
+ first so a rejection cannot pass for the wrong reason.
+ * PART 3 -- backwards compatibility, which is the hard requirement. Every marker
+ already on a user's disk lacks the key. It must be KEPT (a re-download is 200-400 MB
+ for a tree that is fine), backfilled once under the install lock, and fast after.
+ * PART 4 -- the deliberate non-rejections: ``mtime_ns`` moves on a restore from backup
+ or a container layer without a byte changing, so it is recorded and never compared.
+ * PART 5 -- forward compatibility, proved against a real released module
+ (``v0.1.808-beta``), which must ignore a key it has never heard of.
+
+No network, no GPU: the install trees are written under ``tmp_path`` and the only
+subprocess reads files extracted from a local git tag. POSIX-only cases skip on Windows
+rather than being weakened, and the permission case skips under root.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from _pr10648_helpers import (
+ NOT_ROOT,
+ POSIX_ONLY,
+ STUDIO_DIR,
+ WHISPER_RELEASE_TAG as RELEASE_TAG,
+ WINDOWS_HOST,
+ git,
+ whisper_host,
+ whisper_install_is_intact,
+ whisper_selection_fields,
+)
+from _pr10648_helpers import load_studio_module as _load
+
+WHISPER = _load("studio_install_whisper_prebuilt_pr10648_digests", "install_whisper_prebuilt.py")
+LLAMA = _load("studio_install_llama_prebuilt_pr10648_digests", "install_llama_prebuilt.py")
+
+GGML_TREE = "ggml-tree-aaaa"
+SERVER_BYTES = b"#!/bin/sh\necho whisper-server\nexit 0\n"
+
+LINUX = whisper_host(WHISPER.HostInfo)
+
+
+def _selection(**overrides):
+ return WHISPER.InstallSelection(**whisper_selection_fields(WHISPER, **overrides))
+
+
+SLIM_LIBRARIES = ("libggml.so.0", "libggml-base.so.0")
+
+
+def _install(
+ tmp_path: Path,
+ monkeypatch,
+ *,
+ slim: bool = False,
+) -> Path:
+ """A whisper install tree plus a marker written by the REAL writer, so the record
+ under test is the one an install actually produces -- not a fixture's idea of it."""
+ install_dir = tmp_path / "whisper.cpp"
+ bin_dir = WHISPER.runtime_bin_dir(install_dir, LINUX)
+ bin_dir.mkdir(parents = True)
+ server = WHISPER.installed_server_path(install_dir, LINUX)
+ server.write_bytes(SERVER_BYTES)
+ server.chmod(0o755)
+ (bin_dir / "libwhisper.so.1").write_bytes(b"dummy-libwhisper")
+ if slim:
+ for name in SLIM_LIBRARIES:
+ (bin_dir / name).write_bytes(b"ggml-payload-" + name.encode("utf-8"))
+ # The live llama marker a slim bundle is wired against; never the real one on this box.
+ monkeypatch.setattr(WHISPER, "installed_llama_ggml_tree", lambda *_a, **_k: GGML_TREE)
+ WHISPER.write_prebuilt_metadata(install_dir, _selection(**_slim_fields(tmp_path, slim)))
+ return install_dir
+
+
+def _slim_fields(tmp_path: Path, slim: bool) -> dict:
+ if not slim:
+ return {}
+ return dict(
+ install_kind = "slim",
+ paired_llama_tag = "b9001",
+ linked_from = str(tmp_path / "llama.cpp" / "build" / "bin"),
+ linked_libraries = SLIM_LIBRARIES,
+ runtime_wiring_version = WHISPER.SLIM_RUNTIME_WIRING_VERSION,
+ linked_runtime_directories = (),
+ )
+
+
+def _marker_path(install_dir: Path) -> Path:
+ return install_dir / WHISPER.METADATA_FILENAME
+
+
+def _marker(install_dir: Path) -> dict:
+ return json.loads(_marker_path(install_dir).read_text(encoding = "utf-8"))
+
+
+def _rewrite_marker(install_dir: Path, marker: dict) -> None:
+ _marker_path(install_dir).write_text(json.dumps(marker, indent = 2), encoding = "utf-8")
+
+
+def _intact(install_dir: Path) -> bool:
+ """The shared on-disk half, which both decision paths below delegate to."""
+ return WHISPER.installed_tree_is_intact(install_dir, LINUX)
+
+
+def _reuse(
+ install_dir: Path,
+ *,
+ slim: bool = False,
+ tmp_path: Path | None = None,
+) -> bool:
+ """The ordinary reuse predicate: "this release is already installed, skip the download"."""
+ return WHISPER.existing_install_matches(
+ install_dir, LINUX, _selection(**_slim_fields(tmp_path or install_dir.parent, slim))
+ )
+
+
+def _fast_path(install_dir: Path) -> bool:
+ """The no-network fast path: "do not even ask which release is newest"."""
+ return whisper_install_is_intact(WHISPER, install_dir, LINUX)
+
+
+def _server_key(install_dir: Path) -> str:
+ return WHISPER.installed_server_path(install_dir, LINUX).relative_to(install_dir).as_posix()
+
+
+# ── PART 1: what the marker records ──────────────────────────────────────────────────
+def test_the_marker_records_the_server_it_installed(tmp_path, monkeypatch):
+ """The update path decides "already matches" without ever starting whisper-server,
+ so the recorded digest is what stands in for "it launched once"."""
+ install_dir = _install(tmp_path, monkeypatch)
+ server = WHISPER.installed_server_path(install_dir, LINUX)
+ record = _marker(install_dir)["runtime_files"][_server_key(install_dir)]
+ assert record["size"] == len(SERVER_BYTES)
+ assert record["sha256"] == WHISPER.sha256_file(server)
+ assert isinstance(record["mtime_ns"], int)
+ # Relative POSIX keys, so a marker stays valid when the install dir is moved or read
+ # on a host that spells separators the other way.
+ assert all(not Path(key).is_absolute() for key in _marker(install_dir)["runtime_files"])
+
+
+def test_a_slim_marker_records_the_ggml_libraries_it_hardlinked(tmp_path, monkeypatch):
+ """A slim bundle ships no ggml of its own: those hardlinks ARE most of its bytes on a
+ CUDA or ROCm pairing, and nothing else in the marker describes them."""
+ install_dir = _install(tmp_path, monkeypatch, slim = True)
+ records = _marker(install_dir)["runtime_files"]
+ for name in SLIM_LIBRARIES:
+ library = WHISPER.runtime_bin_dir(install_dir, LINUX) / name
+ record = records[library.relative_to(install_dir).as_posix()]
+ assert record["size"] == library.stat().st_size
+ # Size tier only: hashing hundreds of MB of kernels on every update is not worth it.
+ assert "sha256" not in record
+
+
+def test_the_record_has_the_same_shape_as_the_llama_one(tmp_path, monkeypatch):
+ """The two markers sit side by side in one Unsloth home and are read by the same
+ people; a second spelling of the same fact is how one of them comes to drift."""
+ install_dir = _install(tmp_path, monkeypatch)
+ server = WHISPER.installed_server_path(install_dir, LINUX)
+ assert WHISPER._stat_record(server).keys() == LLAMA._stat_record(server).keys()
+ record = _marker(install_dir)["runtime_files"][_server_key(install_dir)]
+ assert set(record) == {"size", "mtime_ns", "sha256"}
+
+
+# ── PART 2: the corruptions that must now be rejected ────────────────────────────────
+def test_a_truncated_server_is_rejected(tmp_path, monkeypatch):
+ """A whisper-server left half-written by a full disk or an interrupted extract.
+
+ It is still a non-empty executable file, so every shape check passes it; before the
+ record this install was kept and dictation failed at the user's next keypress.
+ """
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _intact(install_dir) is True and _reuse(install_dir) is True
+ server = WHISPER.installed_server_path(install_dir, LINUX)
+ server.write_bytes(SERVER_BYTES[: len(SERVER_BYTES) // 2])
+ assert server.stat().st_size > 0
+ assert WINDOWS_HOST or os.access(server, os.X_OK)
+ assert _intact(install_dir) is False
+ assert _reuse(install_dir) is False
+ assert _fast_path(install_dir) is False
+
+
+def test_whisper_a_same_size_byte_flip_is_caught_only_by_the_digest(tmp_path, monkeypatch):
+ """Bit rot, a bad cable or a partial overwrite: the file is exactly as long as it was.
+
+ This is the one corruption the size tier cannot see, and the reason whisper-server is
+ hashed rather than statted like the ggml libraries beside it.
+ """
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _intact(install_dir) is True
+ server = WHISPER.installed_server_path(install_dir, LINUX)
+ before = server.stat().st_size
+ data = bytearray(server.read_bytes())
+ data[len(data) // 2] ^= 0xFF
+ server.write_bytes(bytes(data))
+ assert server.stat().st_size == before, "the flip must preserve the size or it proves nothing"
+ recorded = _marker(install_dir)["runtime_files"][_server_key(install_dir)]
+ assert recorded["size"] == server.stat().st_size
+ assert _intact(install_dir) is False
+ assert _reuse(install_dir) is False
+ assert _fast_path(install_dir) is False
+
+
+def test_a_deleted_server_is_rejected(tmp_path, monkeypatch):
+ """A recorded file that vanished is evidence against the install; the shape check
+ caught this one already, and the record must not soften it."""
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _intact(install_dir) is True
+ WHISPER.installed_server_path(install_dir, LINUX).unlink()
+ assert _intact(install_dir) is False
+ assert _reuse(install_dir) is False
+ assert _fast_path(install_dir) is False
+
+
+def test_a_corrupt_wired_library_is_rejected(tmp_path, monkeypatch):
+ """A hardlinked ggml library left as a stub. linked_libraries is checked for PRESENCE
+ by name, so a one-byte libggml.so.0 used to read as intact wiring and the sidecar
+ failed with a dynamic linker error the user could do nothing with."""
+ install_dir = _install(tmp_path, monkeypatch, slim = True)
+ assert _intact(install_dir) is True
+ assert _reuse(install_dir, slim = True, tmp_path = tmp_path) is True
+ library = WHISPER.runtime_bin_dir(install_dir, LINUX) / "libggml.so.0"
+ library.write_bytes(b"\x00")
+ assert library.is_file()
+ assert _intact(install_dir) is False
+ assert _reuse(install_dir, slim = True, tmp_path = tmp_path) is False
+ assert _fast_path(install_dir) is False
+
+
+def test_a_same_size_flip_in_a_ggml_library_is_deliberately_not_caught(tmp_path, monkeypatch):
+ """The honest limit of the size tier, asserted so nobody reads the test above as more
+ than it is: the libraries are statted, not hashed, because on a CUDA pairing they are
+ hundreds of MB and this check runs on every update. Truncation and deletion -- what a
+ full disk, an interrupted extract or a half-removed llama install actually produce --
+ are what it is built to catch."""
+ install_dir = _install(tmp_path, monkeypatch, slim = True)
+ library = WHISPER.runtime_bin_dir(install_dir, LINUX) / "libggml.so.0"
+ data = bytearray(library.read_bytes())
+ data[0] ^= 0xFF
+ library.write_bytes(bytes(data))
+ assert _intact(install_dir) is True
+
+
+def test_a_marker_whose_record_is_unusable_fails_closed(tmp_path, monkeypatch):
+ """Present but unreadable is not the same as absent. The record is the evidence that
+ replaces starting the binary, so an empty or malformed one proves nothing -- and it
+ cannot have come from the writer, which omits the key rather than writing {}."""
+ install_dir = _install(tmp_path, monkeypatch)
+ for damaged in ({}, [], "runtime", {_server_key(install_dir): "not-a-record"}):
+ marker = _marker(install_dir)
+ marker["runtime_files"] = damaged
+ _rewrite_marker(install_dir, marker)
+ assert _intact(install_dir) is False, damaged
+ assert _fast_path(install_dir) is False, damaged
+
+
+@POSIX_ONLY
+@NOT_ROOT
+def test_an_unreadable_server_is_rejected_rather_than_skipped(tmp_path, monkeypatch):
+ """A tree half-owned by a previous sudo install: the digest cannot be computed, and
+ "could not check" must not read as "checked and fine"."""
+ install_dir = _install(tmp_path, monkeypatch)
+ assert _intact(install_dir) is True
+ server = WHISPER.installed_server_path(install_dir, LINUX)
+ server.chmod(0o000)
+ try:
+ assert _intact(install_dir) is False
+ finally:
+ server.chmod(0o755)
+ assert _intact(install_dir) is True
+
+
+# ── PART 3: backwards compatibility, the hard requirement ────────────────────────────
+def _legacy(install_dir: Path) -> dict:
+ """The marker every existing user has: byte-identical to what this installer writes,
+ minus the key that did not exist when it was written."""
+ marker = _marker(install_dir)
+ marker.pop("runtime_files", None)
+ _rewrite_marker(install_dir, marker)
+ assert "runtime_files" not in _marker(install_dir)
+ return marker
+
+
+def test_a_marker_written_before_the_record_is_kept_not_re_downloaded(tmp_path, monkeypatch):
+ """A user upgrading Unsloth Studio over an install made by an older one.
+
+ Nothing is wrong with their tree; it simply predates the key. The reuse predicate is
+ what decides whether the update downloads 200-400 MB again, so it must say yes. The
+ no-network fast path is allowed to decline -- that costs one release lookup, once --
+ and that is what settles the record.
+ """
+ install_dir = _install(tmp_path, monkeypatch)
+ _legacy(install_dir)
+ assert _intact(install_dir) is True
+ assert _reuse(install_dir) is True
+ assert _fast_path(install_dir) is False
+ assert WHISPER.kept_install_needs_settling(install_dir) is True
+
+
+def test_the_legacy_marker_is_backfilled_once_and_fast_after(tmp_path, monkeypatch):
+ """The other half: "takes the full path once" is only true if something writes the
+ record. Otherwise every single update re-fetches the release, its manifest and its
+ checksum index forever."""
+ install_dir = _install(tmp_path, monkeypatch)
+ _legacy(install_dir)
+ WHISPER.settle_kept_install(install_dir)
+
+ server = WHISPER.installed_server_path(install_dir, LINUX)
+ record = _marker(install_dir)["runtime_files"][_server_key(install_dir)]
+ assert record["sha256"] == WHISPER.sha256_file(server)
+ assert record["size"] == server.stat().st_size
+ assert _fast_path(install_dir) is True
+ assert WHISPER.kept_install_needs_settling(install_dir) is False
+ # And the record it just wrote is load-bearing from here on.
+ server.write_bytes(SERVER_BYTES[:4])
+ assert _intact(install_dir) is False
+
+
+def test_the_backfill_records_the_live_bytes_of_a_slim_tree(tmp_path, monkeypatch):
+ """A legacy SLIM install: the wiring it hardlinked has to be recorded too, or the
+ backfill would leave the half of the payload that is most of the bytes unprotected."""
+ install_dir = _install(tmp_path, monkeypatch, slim = True)
+ _legacy(install_dir)
+ WHISPER.settle_kept_install(install_dir)
+ records = _marker(install_dir)["runtime_files"]
+ for name in SLIM_LIBRARIES:
+ library = WHISPER.runtime_bin_dir(install_dir, LINUX) / name
+ assert records[library.relative_to(install_dir).as_posix()]["size"] == (
+ library.stat().st_size
+ )
+ (WHISPER.runtime_bin_dir(install_dir, LINUX) / SLIM_LIBRARIES[0]).write_bytes(b"\x00")
+ assert _intact(install_dir) is False
+
+
+def test_the_backfill_never_rewrites_a_record_it_did_not_take(tmp_path, monkeypatch):
+ """Added, never corrected. Overwriting a present record with whatever is on disk now
+ would re-bless bytes no run ever hashed -- it would turn the guard into a rubber
+ stamp the first time an update ran over a corrupt tree."""
+ install_dir = _install(tmp_path, monkeypatch)
+ recorded = _marker(install_dir)["runtime_files"]
+ WHISPER.installed_server_path(install_dir, LINUX).write_bytes(b"replaced-behind-our-back")
+ WHISPER.settle_kept_install(install_dir)
+ assert _marker(install_dir)["runtime_files"] == recorded
+ assert _intact(install_dir) is False
+
+
+@POSIX_ONLY
+def test_the_backfill_replaces_the_marker_atomically(tmp_path, monkeypatch):
+ """The marker is already in service: a torn write reads as "nothing installed" and
+ costs the user the whole install. Temp-and-replace, so a failure part-way leaves the
+ marker that was there, the mode survives for a group-shared install, and no .tmp-*
+ sibling is left behind."""
+ install_dir = _install(tmp_path, monkeypatch)
+ _legacy(install_dir)
+ marker_path = _marker_path(install_dir)
+ marker_path.chmod(0o644)
+ before = marker_path.stat()
+
+ WHISPER.settle_kept_install(install_dir)
+
+ after = marker_path.stat()
+ # A different inode is the observable signature of temp-and-replace: the live marker
+ # was swapped, never truncated in place.
+ assert after.st_ino != before.st_ino
+ assert after.st_mode == before.st_mode
+ assert [p.name for p in install_dir.iterdir() if ".tmp-" in p.name] == []
+ assert _marker(install_dir)["runtime_files"]
+
+
+def test_an_upgrade_does_not_re_download_and_settles_under_the_lock(tmp_path, monkeypatch):
+ """End to end over the real keep path: an older Studio's install, one `studio update`.
+
+ install_selected_prebuilt is where a keep becomes a download, so this is the test
+ that actually proves the upgrade is free. The backfill is a read-modify-write of a
+ live marker, so it must happen under the install lock -- outside it, a concurrent
+ installer swapping in a new release has its fresh marker overwritten with the old
+ release's fields.
+ """
+ install_dir = _install(tmp_path, monkeypatch)
+ _legacy(install_dir)
+ selection = _selection()
+ bundle = WHISPER.ReleaseBundle(
+ repo = WHISPER.DEFAULT_PUBLISHED_REPO,
+ release_tag = RELEASE_TAG,
+ manifest = {},
+ asset_urls = {},
+ )
+
+ downloads = {"n": 0}
+
+ def no_downloads(*_args, **_kwargs):
+ downloads["n"] += 1
+ raise AssertionError("the upgrade re-installed instead of keeping the existing tree")
+
+ depth = {"now": 0, "backfills_under_lock": 0, "backfills_outside": 0}
+ real_lock = WHISPER.install_lock
+ real_backfill = WHISPER._backfill_runtime_file_records
+
+ import contextlib
+
+ @contextlib.contextmanager
+ def counting_lock(path, **kwargs):
+ with real_lock(path, **kwargs):
+ depth["now"] += 1
+ try:
+ yield
+ finally:
+ depth["now"] -= 1
+
+ def counting_backfill(directory):
+ key = "backfills_under_lock" if depth["now"] else "backfills_outside"
+ depth[key] += 1
+ real_backfill(directory)
+
+ monkeypatch.setattr(WHISPER, "_install_from_bundle", no_downloads)
+ monkeypatch.setattr(WHISPER, "install_lock", counting_lock)
+ monkeypatch.setattr(WHISPER, "_backfill_runtime_file_records", counting_backfill)
+
+ result = WHISPER.core.install_selected_prebuilt(
+ WHISPER._OPS,
+ install_dir,
+ host = LINUX,
+ bundle = bundle,
+ selection = selection,
+ force = False,
+ )
+ assert result == WHISPER.EXIT_SUCCESS
+ assert downloads["n"] == 0
+ # The injection fired: the settle really went through the patched lock and backfill.
+ assert depth == {"now": 0, "backfills_under_lock": 1, "backfills_outside": 0}
+ assert _marker(install_dir)["runtime_files"]
+
+ # Nothing left to settle: the next update takes no lock to write.
+ depth["backfills_under_lock"] = 0
+ assert (
+ WHISPER.core.install_selected_prebuilt(
+ WHISPER._OPS,
+ install_dir,
+ host = LINUX,
+ bundle = bundle,
+ selection = selection,
+ force = False,
+ )
+ == WHISPER.EXIT_SUCCESS
+ )
+ assert depth["backfills_under_lock"] == 0 and depth["backfills_outside"] == 0
+ assert downloads["n"] == 0
+
+
+def test_a_backfill_that_cannot_hash_writes_nothing(tmp_path, monkeypatch):
+ """A scanner holding whisper-server open, or a tree half-owned by root. An empty
+ record is not evidence, and writing one would fail closed on every later update --
+ which is the re-download loop this whole part exists to prevent."""
+ install_dir = _install(tmp_path, monkeypatch)
+ _legacy(install_dir)
+ fired = {"n": 0}
+
+ def unreadable(_path):
+ fired["n"] += 1
+ raise OSError("Input/output error")
+
+ monkeypatch.setattr(WHISPER, "sha256_file", unreadable)
+ WHISPER.settle_kept_install(install_dir)
+ assert fired["n"] > 0, "the injected hasher never ran; the test proves nothing"
+ assert "runtime_files" not in _marker(install_dir)
+ # Still kept, still no download: it simply stays on the full path.
+ assert _intact(install_dir) is True
+ assert _reuse(install_dir) is True
+
+
+# ── PART 4: what must deliberately NOT reject ────────────────────────────────────────
+def test_a_changed_mtime_alone_does_not_reject(tmp_path, monkeypatch):
+ """A restore from backup, an rsync, a container layer or a `tar -x` without
+ --touch: the timestamps move, not a byte changes. mtime_ns is recorded (it is what
+ makes the size tier cheap to reason about) and never compared, because the answer to
+ a mismatch here is a 200-400 MB re-download."""
+ install_dir = _install(tmp_path, monkeypatch, slim = True)
+ recorded = _marker(install_dir)["runtime_files"]
+ for relative in recorded:
+ target = install_dir / relative
+ os.utime(target, ns = (recorded[relative]["mtime_ns"] + 10**9,) * 2)
+ assert target.stat().st_mtime_ns != recorded[relative]["mtime_ns"]
+ assert _intact(install_dir) is True
+ assert _reuse(install_dir, slim = True, tmp_path = tmp_path) is True
+ assert _fast_path(install_dir) is True
+
+
+def test_an_unrecorded_file_appearing_beside_the_payload_does_not_reject(tmp_path, monkeypatch):
+ """A user's own notes, an antivirus quarantine stub, a .bak from a support session.
+ A file that appeared since is not evidence against the install; only a RECORDED file
+ that moved is."""
+ install_dir = _install(tmp_path, monkeypatch)
+ (WHISPER.runtime_bin_dir(install_dir, LINUX) / "notes.txt").write_text("hello")
+ assert _intact(install_dir) is True
+ assert _fast_path(install_dir) is True
+
+
+# ── PART 5: forward compatibility against a really released module ───────────────────
+_OLD_TAG = "v0.1.808-beta"
+_OLD_MODULES = ("install_whisper_prebuilt.py", "prebuilt_core.py", "install_llama_prebuilt.py")
+
+_OLD_READER = """
+import dataclasses, json, pathlib, sys
+import install_whisper_prebuilt as M
+
+spec = json.loads(sys.argv[1])
+install_dir = pathlib.Path(spec["install_dir"])
+host = M.HostInfo(**{
+ key: value
+ for key, value in spec["host"].items()
+ if key in {f.name for f in dataclasses.fields(M.HostInfo)}
+})
+selection = M.InstallSelection(**{
+ key: value
+ for key, value in spec["selection"].items()
+ if key in {f.name for f in dataclasses.fields(M.InstallSelection)}
+})
+source = pathlib.Path(M.__file__).read_text(encoding = "utf-8")
+print(json.dumps({
+ "module_file": M.__file__,
+ "core_file": M.core.__file__,
+ "knows_the_key": "runtime_files" in source,
+ "matches": bool(M.existing_install_matches(install_dir, host, selection)),
+ "marker_keys": sorted(M.load_prebuilt_metadata(install_dir) or {}),
+}))
+"""
+
+
+def _extract_released_studio(tmp_path: Path) -> Path:
+ """The three installer modules exactly as `{tag}` shipped them, from the local git
+ tag. Read-only, no network, and skipped rather than faked where git cannot answer."""
+ old_dir = tmp_path / "released_studio"
+ old_dir.mkdir()
+ for name in _OLD_MODULES:
+ try:
+ blob = git("show", f"{_OLD_TAG}:studio/{name}", timeout = 60)
+ except (OSError, subprocess.SubprocessError) as exc: # pragma: no cover - CI without git
+ pytest.skip(f"git is unavailable here: {exc}")
+ if blob.returncode != 0 or not blob.stdout: # pragma: no cover - shallow checkout
+ pytest.skip(f"{_OLD_TAG}:studio/{name} is not in this checkout")
+ (old_dir / name).write_bytes(blob.stdout)
+ return old_dir
+
+
+def test_a_released_older_studio_ignores_the_new_key(tmp_path, monkeypatch):
+ """Two Unsloth Studios can share one UNSLOTH_HOME -- an older one pinned in a venv, a
+ rollback, a second checkout. The older one must read a marker carrying a key it has
+ never heard of and keep the install, not refuse it and re-download 200-400 MB.
+
+ Only ADDING a key makes this true, so it is asserted against the module v0.1.808-beta
+ actually shipped rather than argued from the diff. A subprocess, so the old module's
+ sys.path insert and its instance of prebuilt_core cannot leak into this process.
+ """
+ install_dir = _install(tmp_path, monkeypatch)
+ assert "runtime_files" in _marker(install_dir), "nothing forward-compatible to test"
+ old_dir = _extract_released_studio(tmp_path)
+
+ payload = json.dumps(
+ {
+ "install_dir": str(install_dir),
+ "host": dataclasses.asdict(LINUX),
+ "selection": {
+ key: (list(value) if isinstance(value, tuple) else value)
+ for key, value in dataclasses.asdict(_selection()).items()
+ },
+ }
+ )
+ env = dict(os.environ)
+ # The old copies first, the live studio/ behind them for the support packages those
+ # modules import (backend.utils.prebuilt.*), which are not part of this marker's story.
+ env["PYTHONPATH"] = os.pathsep.join([str(old_dir), str(STUDIO_DIR)])
+ completed = subprocess.run(
+ [sys.executable, "-c", _OLD_READER, payload],
+ cwd = str(old_dir),
+ capture_output = True,
+ text = True,
+ timeout = 300,
+ env = env,
+ )
+ assert completed.returncode == 0, completed.stderr
+ result = json.loads(completed.stdout.strip().splitlines()[-1])
+
+ # The injection fired: this really is the released module, not the one under test.
+ assert result["module_file"].startswith(str(old_dir))
+ assert result["core_file"].startswith(str(old_dir))
+ assert result["knows_the_key"] is False
+ # It read the whole marker, new key included, and kept the install.
+ assert "runtime_files" in result["marker_keys"]
+ assert result["matches"] is True
+
+
+# The ROCm kernel catalogs. installed_tree_is_intact asks only that each linked directory hold
+# ANY file, so the offline keep blessed an install whose catalog no longer loads.
+ROCM_DIRS = ("rocblas", "hipblaslt")
+
+
+def _rocm_install(tmp_path: Path, monkeypatch) -> Path:
+ """A slim ROCm install with a populated kernel catalog, marker by the real writer."""
+ install_dir = tmp_path / "whisper.cpp"
+ bin_dir = WHISPER.runtime_bin_dir(install_dir, LINUX)
+ bin_dir.mkdir(parents = True)
+ server = WHISPER.installed_server_path(install_dir, LINUX)
+ server.write_bytes(SERVER_BYTES)
+ server.chmod(0o755)
+ (bin_dir / "libwhisper.so.1").write_bytes(b"dummy-libwhisper")
+ for name in SLIM_LIBRARIES:
+ (bin_dir / name).write_bytes(b"ggml-payload-" + name.encode("utf-8"))
+ for directory in ROCM_DIRS:
+ library = bin_dir / directory / "library"
+ library.mkdir(parents = True)
+ # Two blobs, so removing one leaves the directory non-empty and the old
+ # "contains at least one file" check satisfied.
+ (library / "TensileLibrary_gfx1100.dat").write_bytes(b"kernel-catalog-gfx1100" * 8)
+ (library / "TensileLibrary_gfx90a.dat").write_bytes(b"kernel-catalog-gfx90a" * 8)
+ monkeypatch.setattr(WHISPER, "installed_llama_ggml_tree", lambda *_a, **_k: GGML_TREE)
+ WHISPER.write_prebuilt_metadata(
+ install_dir,
+ _selection(
+ install_kind = "slim",
+ paired_llama_tag = "b9001",
+ linked_from = str(tmp_path / "llama.cpp" / "build" / "bin"),
+ linked_libraries = SLIM_LIBRARIES,
+ runtime_wiring_version = WHISPER.SLIM_RUNTIME_WIRING_VERSION,
+ linked_runtime_directories = ROCM_DIRS,
+ ),
+ )
+ return install_dir
+
+
+def test_the_rocm_catalog_files_are_recorded(tmp_path: Path, monkeypatch):
+ """A record that stops at the top-level libraries cannot see inside rocblas/."""
+ install_dir = _rocm_install(tmp_path, monkeypatch)
+ recorded = _marker(install_dir)["runtime_files"]
+ catalog = [name for name in recorded if "/library/TensileLibrary" in name]
+ assert len(catalog) == 4, f"the kernel catalogs were not recorded: {sorted(recorded)}"
+ assert all(recorded[name]["size"] > 0 for name in catalog)
+
+
+def test_a_catalog_blob_removed_beside_a_sibling_is_rejected(tmp_path: Path, monkeypatch):
+ """A full disk or an interrupted extract takes SOME of rocblas/, not all of it.
+
+ The directory stays non-empty, so the existence check passed and an offline update
+ kept a tree whose kernel catalog no longer loads.
+ """
+ install_dir = _rocm_install(tmp_path, monkeypatch)
+ assert _intact(install_dir) is True
+ bin_dir = WHISPER.runtime_bin_dir(install_dir, LINUX)
+ victim = bin_dir / "rocblas" / "library" / "TensileLibrary_gfx1100.dat"
+ victim.unlink()
+ assert list((bin_dir / "rocblas" / "library").iterdir()), "the sibling must remain"
+ assert _intact(install_dir) is False
+ assert _fast_path(install_dir) is False
+
+
+def test_a_truncated_catalog_blob_is_rejected(tmp_path: Path, monkeypatch):
+ """Truncation leaves the file in place, so only a recorded size catches it."""
+ install_dir = _rocm_install(tmp_path, monkeypatch)
+ assert _intact(install_dir) is True
+ victim = WHISPER.runtime_bin_dir(install_dir, LINUX) / "rocblas" / "library"
+ (victim / "TensileLibrary_gfx90a.dat").write_bytes(b"x")
+ assert _intact(install_dir) is False
+
+
+def test_a_marker_naming_a_traversing_runtime_directory_records_nothing_outside(
+ tmp_path: Path, monkeypatch
+):
+ """rglob on a marker-supplied path would walk out of the install entirely."""
+ install_dir = _rocm_install(tmp_path, monkeypatch)
+ records = WHISPER._runtime_file_records(
+ install_dir, SLIM_LIBRARIES, ("../../../etc", "rocblas")
+ )
+ assert records, "the legitimate directory should still be recorded"
+ assert all(not name.startswith("..") for name in records), sorted(records)
+
+
+# ── PART 5: which llama INSTALL the hardlinks point into ─────────────────────────────
+# The gap these close: a release publishes one llama bundle per gfx target, so re-selecting ROCm
+# for another GPU swaps the asset while ggml_tree, a SOURCE-tree identity, stays put. The
+# hardlinks survive llama's directory swap on purpose, so the old GPU's kernels stay wired.
+LLAMA_ID_GFX1100 = "f" * 64
+LLAMA_ID_GFX1151 = "e" * 64
+
+
+def _paired(monkeypatch, runtime_id: "str | None") -> None:
+ monkeypatch.setattr(WHISPER, "installed_paired_runtime_id", lambda *_a, **_k: runtime_id)
+
+
+def test_a_slim_marker_records_which_llama_install_it_was_wired_against(tmp_path, monkeypatch):
+ _paired(monkeypatch, LLAMA_ID_GFX1100)
+ install_dir = _install(tmp_path, monkeypatch, slim = True)
+ assert _marker(install_dir)["paired_llama_runtime_id"] == LLAMA_ID_GFX1100
+
+
+def test_a_fat_marker_records_no_pairing_at_all(tmp_path, monkeypatch):
+ _paired(monkeypatch, LLAMA_ID_GFX1100)
+ install_dir = _install(tmp_path, monkeypatch, slim = False)
+ assert "paired_llama_runtime_id" not in _marker(install_dir)
+
+
+def test_a_gfx_reselection_within_one_release_is_caught(tmp_path, monkeypatch):
+ """The reported case. llama's tag, its ggml tree and whisper's own bytes are all
+ unchanged; only the asset behind the hardlinks moved."""
+ _paired(monkeypatch, LLAMA_ID_GFX1100)
+ install_dir = _install(tmp_path, monkeypatch, slim = True)
+ assert _intact(install_dir) is True
+ assert _reuse(install_dir, slim = True, tmp_path = tmp_path) is True
+
+ _paired(monkeypatch, LLAMA_ID_GFX1151)
+ assert _marker(install_dir)["paired_llama_ggml_tree"] == GGML_TREE, "tree still matches"
+ assert _intact(install_dir) is False
+ assert _reuse(install_dir, slim = True, tmp_path = tmp_path) is False
+ assert _fast_path(install_dir) is False
+
+
+def test_the_ggml_tree_alone_does_not_catch_it(tmp_path, monkeypatch):
+ """Why a second key was needed: with the runtime id stripped, the same reselection is
+ invisible. This is the pre-fix behaviour, pinned so the guard cannot be quietly dropped."""
+ _paired(monkeypatch, LLAMA_ID_GFX1100)
+ install_dir = _install(tmp_path, monkeypatch, slim = True)
+ marker = _marker(install_dir)
+ marker.pop("paired_llama_runtime_id")
+ _rewrite_marker(install_dir, marker)
+
+ _paired(monkeypatch, LLAMA_ID_GFX1151)
+ assert _intact(install_dir) is True
+
+
+def test_an_unchanged_llama_install_still_takes_the_fast_path(tmp_path, monkeypatch):
+ _paired(monkeypatch, LLAMA_ID_GFX1100)
+ install_dir = _install(tmp_path, monkeypatch, slim = True)
+ assert _fast_path(install_dir) is True
+ assert _fast_path(install_dir) is True
+
+
+def test_a_marker_predating_the_key_is_backfilled_not_re_downloaded(tmp_path, monkeypatch):
+ """An older marker cannot say which bundle it was paired against, so a swap that ALREADY
+ happened is unanswerable from it. Reinstalling to find out would re-download the bundle
+ for every existing user; recording the current pairing makes the NEXT swap detectable."""
+ install_dir, _ = _slim_with_live_llama(tmp_path, monkeypatch)
+ assert WHISPER.kept_install_needs_settling(install_dir) is True
+ assert _reuse(install_dir, slim = True, tmp_path = tmp_path) is True
+ WHISPER.settle_kept_install(install_dir)
+ assert _marker(install_dir)["paired_llama_runtime_id"] == LLAMA_ID_GFX1100
+ assert WHISPER.kept_install_needs_settling(install_dir) is False
+
+ _paired(monkeypatch, LLAMA_ID_GFX1151)
+ assert _intact(install_dir) is False
+
+
+def test_a_backfill_with_no_llama_install_to_read_writes_nothing(tmp_path, monkeypatch):
+ """Fails open, not closed: an unreadable llama marker must not stamp None as a pairing,
+ and must not fail setup over a metadata refresh either."""
+ _paired(monkeypatch, LLAMA_ID_GFX1100)
+ install_dir = _install(tmp_path, monkeypatch, slim = True)
+ marker = _marker(install_dir)
+ marker.pop("paired_llama_runtime_id")
+ _rewrite_marker(install_dir, marker)
+
+ _paired(monkeypatch, None)
+ WHISPER.settle_kept_install(install_dir)
+ assert "paired_llama_runtime_id" not in _marker(install_dir)
+ assert _intact(install_dir) is True
+
+
+def test_a_recorded_pairing_against_a_vanished_llama_install_is_rejected(tmp_path, monkeypatch):
+ """llama removed entirely, whisper's hardlinks still holding its last bytes: the recorded
+ id no longer matches anything, which is a reinstall rather than a keep."""
+ _paired(monkeypatch, LLAMA_ID_GFX1100)
+ install_dir = _install(tmp_path, monkeypatch, slim = True)
+ _paired(monkeypatch, None)
+ assert _intact(install_dir) is False
+
+
+# The pairing backfill must not record a pairing it has not verified
+# A legacy slim marker carries no pairing identity, so a llama replaced before whisper's first
+# migration -- or by another installer holding llama's own per-directory lock during this one --
+# leaves the hardlinks pointing at the previous libraries. Recording the LIVE llama identity there
+# writes a FALSE pairing, and the runtime-id check then believes it forever.
+def _slim_with_live_llama(
+ tmp_path,
+ monkeypatch,
+ *,
+ hardlink = True,
+):
+ """A slim whisper install whose wiring really does come from a llama bin dir on disk."""
+ llama_bin = tmp_path / "llama.cpp" / "build" / "bin"
+ llama_bin.mkdir(parents = True)
+ for name in SLIM_LIBRARIES:
+ (llama_bin / name).write_bytes(b"ggml-payload-" + name.encode("utf-8"))
+
+ _paired(monkeypatch, LLAMA_ID_GFX1100)
+ install_dir = _install(tmp_path, monkeypatch, slim = True)
+ bin_dir = WHISPER.runtime_bin_dir(install_dir, LINUX)
+ for name in SLIM_LIBRARIES:
+ target = bin_dir / name
+ target.unlink(missing_ok = True)
+ if hardlink:
+ os.link(llama_bin / name, target)
+ else:
+ shutil.copy2(llama_bin / name, target)
+
+ marker = _marker(install_dir)
+ marker["linked_from"] = str(llama_bin)
+ marker["linked_libraries"] = list(SLIM_LIBRARIES)
+ marker.pop("paired_llama_runtime_id", None)
+ marker.pop("paired_llama_ggml_tree", None)
+ _rewrite_marker(install_dir, marker)
+ return install_dir, llama_bin
+
+
+def test_the_backfill_records_a_pairing_the_wiring_proves(tmp_path, monkeypatch):
+ install_dir, _ = _slim_with_live_llama(tmp_path, monkeypatch)
+ WHISPER.settle_kept_install(install_dir)
+ assert _marker(install_dir).get("paired_llama_runtime_id") == LLAMA_ID_GFX1100
+
+
+def test_the_backfill_records_nothing_when_llama_was_swapped_underneath(tmp_path, monkeypatch):
+ """The reported case. llama is replaced with different bytes at a NEW inode, exactly as a
+ directory swap leaves it, while whisper keeps the old ones."""
+ install_dir, llama_bin = _slim_with_live_llama(tmp_path, monkeypatch)
+ for name in SLIM_LIBRARIES:
+ (llama_bin / name).unlink()
+ (llama_bin / name).write_bytes(b"REPLACED-" + name.encode("utf-8"))
+ _paired(monkeypatch, LLAMA_ID_GFX1151)
+
+ WHISPER.settle_kept_install(install_dir)
+ marker = _marker(install_dir)
+ assert (
+ "paired_llama_runtime_id" not in marker
+ ), "a pairing was recorded for a llama install this wiring does not point at"
+ assert (
+ WHISPER.kept_install_needs_settling(install_dir) is True
+ ), "the install stopped asking to be settled without ever having been settled"
+
+
+def test_a_copied_wiring_is_judged_by_its_bytes_not_its_inode(tmp_path, monkeypatch):
+ """_link_or_copy falls back to shutil.copy2 across filesystems, where the inodes differ
+ legitimately. Refusing on the inode alone would deny those installs the fast path forever."""
+ install_dir, _ = _slim_with_live_llama(tmp_path, monkeypatch, hardlink = False)
+ WHISPER.settle_kept_install(install_dir)
+ assert _marker(install_dir).get("paired_llama_runtime_id") == LLAMA_ID_GFX1100
+
+
+def test_a_marker_naming_no_source_directory_records_nothing(tmp_path, monkeypatch):
+ """Fails closed: a marker that cannot say where its wiring came from cannot prove anything."""
+ install_dir, _ = _slim_with_live_llama(tmp_path, monkeypatch)
+ marker = _marker(install_dir)
+ marker["linked_from"] = str(tmp_path / "gone")
+ _rewrite_marker(install_dir, marker)
+ WHISPER.settle_kept_install(install_dir)
+ assert "paired_llama_runtime_id" not in _marker(install_dir)
diff --git a/tests/studio/install/test_prebuilt_core.py b/tests/studio/install/test_prebuilt_core.py
index b26a5e8f5f3..2c07addf497 100644
--- a/tests/studio/install/test_prebuilt_core.py
+++ b/tests/studio/install/test_prebuilt_core.py
@@ -16,6 +16,7 @@ llama descriptor here exercises the canonical dialect a future migration
would use, including the "no fallback backend -> report no prebuilt" policy.
"""
+import contextlib
import importlib.util
import io
import json
@@ -964,3 +965,140 @@ def test_github_api_403_without_a_reachable_reset_makes_one_request():
)
assert len(requests) == 1
+
+
+def test_a_settle_does_not_hold_a_finished_launch_for_the_install_timeout(tmp_path):
+ """The catch-up runs on the FIRST update after an upgrade, for every existing user, and
+ it writes fields whose only effect is to spare the next run some work. A box where
+ another installer is running must not wait five minutes for that: it asks briefly, gives
+ up, logs, and reports the install it already validated."""
+ install_dir = tmp_path / "component"
+ install_dir.mkdir()
+ server = install_dir / "server"
+ server.write_text("", encoding = "utf-8")
+ events = []
+ asked = []
+
+ @contextlib.contextmanager
+ def busy_lock(path, *, timeout = None):
+ asked.append(timeout)
+ raise core.BusyInstallConflict("held elsewhere")
+ yield # pragma: no cover - unreachable, keeps this a generator
+
+ ops = SimpleNamespace(
+ COMPONENT = "test",
+ existing_install_matches = lambda d, h, s: True,
+ install_lock = busy_lock,
+ install_lock_path = lambda d: d / "lock",
+ kept_install_needs_settling = lambda d: True,
+ settle_kept_install = lambda d: events.append("settled"),
+ _install_from_bundle = lambda d, h, b, s: events.append("installed"),
+ installed_server_path = lambda d, h: server,
+ log = lambda message: events.append(message),
+ )
+ bundle = SimpleNamespace(release_tag = "b1")
+ selection = SimpleNamespace(backend = "cpu")
+ rc = core.install_selected_prebuilt(
+ ops, install_dir, host = None, bundle = bundle, selection = selection, force = False
+ )
+ assert rc == 0
+ assert "installed" not in events, "a kept install was reinstalled over a busy lock"
+ assert "settled" not in events
+ assert asked == [core.SETTLE_LOCK_TIMEOUT_SECONDS]
+ assert core.SETTLE_LOCK_TIMEOUT_SECONDS < core.INSTALL_LOCK_TIMEOUT_SECONDS
+
+
+def test_a_kept_install_that_changes_under_the_lock_is_re_validated(tmp_path):
+ """The pre-lock keep re-checks the install under the lock before settling its marker.
+ A concurrent installer that swapped the tree in between makes that re-check fail, and
+ the keep then falls through to the locked path instead of reporting the release it
+ just saw replaced as installed."""
+ install_dir = tmp_path / "component"
+ install_dir.mkdir()
+ server = install_dir / "server"
+ server.write_text("", encoding = "utf-8")
+ # pre-lock keep, the settle's re-check, the locked path's re-check
+ answers = iter([True, False, False])
+ events = []
+ ops = SimpleNamespace(
+ COMPONENT = "test",
+ existing_install_matches = lambda d, h, s: next(answers),
+ install_lock = lambda path, **kwargs: contextlib.nullcontext(),
+ install_lock_path = lambda d: d / "lock",
+ kept_install_needs_settling = lambda d: True,
+ settle_kept_install = lambda d: events.append("settled"),
+ _install_from_bundle = lambda d, h, b, s: events.append("installed"),
+ installed_server_path = lambda d, h: server,
+ log = lambda message: events.append(message),
+ )
+ bundle = SimpleNamespace(release_tag = "b1")
+ selection = SimpleNamespace(backend = "cpu")
+ rc = core.install_selected_prebuilt(
+ ops, install_dir, host = None, bundle = bundle, selection = selection, force = False
+ )
+ assert rc == 0
+ assert "settled" not in events
+ assert "installed" in events
+
+
[email protected](not hasattr(core.os, "chown"), reason = "os.chown is POSIX only")
+def test_a_live_marker_rewrite_keeps_the_group_when_the_owner_is_refused(tmp_path, monkeypatch):
+ """A non-root member of a group-shared install can hand the temp file to the
+ marker's group, but not to its owner; asking for both refuses the call before the
+ group is applied and os.replace installs the member's primary group instead. So the
+ combined call is tried first, for the root case that can honour it, and the group-only
+ call is the fallback."""
+ marker = tmp_path / "MARKER.json"
+ marker.write_text('{"a": 1}', encoding = "utf-8")
+ original = marker.stat()
+ calls = []
+
+ def refusing(path, uid, gid):
+ calls.append((uid, gid))
+ if uid != -1:
+ raise PermissionError("a non-root member may not give a file away")
+
+ monkeypatch.setattr(core.os, "chown", refusing)
+ core.write_live_marker(marker, {"a": 1, "b": 2})
+ assert json.loads(marker.read_text(encoding = "utf-8")) == {"a": 1, "b": 2}
+ assert calls == [(original.st_uid, original.st_gid), (-1, original.st_gid)]
+ assert not list(tmp_path.glob("MARKER.json.tmp-*"))
+
+
+def test_a_walk_back_is_recorded_with_the_host_version_that_decided_it():
+ """prebuilt_core.WalkBack: the marker-only re-check may hold an install on a Mac
+ below the newest release's floor current only while the newest release is the one
+ skipped AND the host is the macOS version that skipped it."""
+ mac = SimpleNamespace(is_macos = True, macos_version = (14, 7))
+ assert core.macos_version_label(mac) == "14.7"
+ assert core.macos_version_label(SimpleNamespace(is_macos = True, macos_version = None)) is None
+ assert core.macos_version_label(SimpleNamespace(is_macos = False, macos_version = (14, 7))) is None
+ walk_back = core.walk_back_for(mac, "r2")
+ assert walk_back == core.WalkBack(release_tag = "r2", macos_version = "14.7")
+ assert core.walk_back_for(mac, None) is None
+ assert core.walk_back_for(SimpleNamespace(is_macos = False, macos_version = None), "r2") is None
+ marker = {"release_tag": "r1", **walk_back.marker_fields()}
+ assert core.marker_walk_back(marker) == walk_back
+ assert core.marker_walk_back({"release_tag": "r1", "walked_back_from": "r2"}) is None
+ assert core.walk_back_stands(marker, mac, "r2") is True
+ assert core.walk_back_stands(marker, mac, "r3") is False
+ assert core.walk_back_stands(marker, mac, None) is False
+ assert (
+ core.walk_back_stands(marker, SimpleNamespace(is_macos = True, macos_version = (15, 0)), "r2")
+ is False
+ )
+ assert (
+ core.walk_back_stands(marker, SimpleNamespace(is_macos = False, macos_version = None), "r2")
+ is False
+ )
+ # What a kept marker owes this run's plan.
+ assert core.walk_back_patch({}, walk_back) == walk_back.marker_fields()
+ assert core.walk_back_patch(marker, walk_back) == {}
+ assert core.walk_back_patch({"walked_back_from": "r2"}, walk_back) == {
+ "walked_back_on_macos": "14.7"
+ }
+ assert core.walk_back_patch(marker, None) == {
+ "walked_back_from": None,
+ "walked_back_on_macos": None,
+ }
+ assert core.walk_back_patch({}, None) == {}