Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Install-completeness marker: install_python_stack.py drops it before the dependency pass and
writes it after the last step, so presence means the install finished. Read by verify-install,
desktop-capabilities and setup.sh/setup.ps1; stdlib only, packaging optional."""
from __future__ import annotations
import contextlib
import contextvars
import errno
import functools
import hashlib
import json
import os
import platform
import re
import sys
import time
from pathlib import Path
from urllib.parse import urlsplit
from typing import Dict, List, Optional, Sequence, Tuple
MANIFEST_NAME = "unsloth_install_manifest.json"
# Only the dependency pass reads it; verify_install and the fast path never do.
PREVIOUS_MANIFEST_NAME = "unsloth_install_manifest.previous.json"
# Serialises writers across processes: the MLX probe rewrites the manifest minutes after the pass,
# and an unserialised replace could recreate a completion marker over a half-built venv.
LOCK_NAME = "unsloth_install_manifest.lock"
PASS_LOCK_NAME = "unsloth_install_pass.lock"
# A peer killed holding the lock must not wedge every later update.
LOCK_WAIT_SECONDS = 10.0
MANIFEST_SCHEMA = 1
# Canonical truthy set for UNSLOTH_NO_TORCH, matching install.ps1 / install.sh.
NO_TORCH_TRUTHY: Tuple[str, ...] = ("1", "true", "yes", "on")
# Outlives the manifest (dropped before every pass), so an interrupted GGUF-only install is not stale.
NO_TORCH_MARKER = ".unsloth-no-torch"
# Fingerprinted into the manifest, relative to studio/backend/requirements/.
TRACKED_REQUIREMENT_FILES: Tuple[str, ...] = (
"studio.txt",
"base.txt",
"extras.txt",
"extras-no-deps.txt",
"no-torch-runtime.txt",
"single-env/data-designer-deps.txt",
"single-env/data-designer.txt",
)
BOOT_REQUIREMENT_FILE = "studio.txt"
# NOT in TRACKED_REQUIREMENT_FILES: verify_install compares that dict, so a new name would
# mark every install changed.
PASS_INPUT_FILES: Tuple[str, ...] = TRACKED_REQUIREMENT_FILES + (
"diffusers-pin.txt",
"triton-kernels.txt",
"overrides.txt",
"single-env/constraints.txt",
"single-env/overrides-darwin-arm64.txt",
)
# Absent means not ours to delete, so not current either.
SIDECAR_OWNED_MARKER = ".unsloth-studio-owned"
def venv_root() -> Path:
"""Directory holding pyvenv.cfg for the interpreter running this code."""
return Path(sys.prefix)
def manifest_path(root: Optional[Path] = None) -> Path:
return (root or venv_root()) / MANIFEST_NAME
def previous_manifest_path(root: Optional[Path] = None) -> Path:
return (root or venv_root()) / PREVIOUS_MANIFEST_NAME
def requirements_root(script_dir: Optional[Path] = None) -> Path:
"""studio/backend/requirements/ next to this module (or a given studio/ dir)."""
return (script_dir or Path(__file__).resolve().parent) / "backend" / "requirements"
def _sha256(path: Path) -> Optional[str]:
try:
return hashlib.sha256(path.read_bytes()).hexdigest()
except OSError:
return None
def installed_requirements_root(root: Optional[Path] = None) -> Optional[Path]:
"""The requirements the venv's *installed* package ships, if it has them.
The digests must describe the files `verify_install` will later read, and
that is always the installed package's copy: at verify time this module is
imported out of the venv, so `requirements_root()` resolves there, and
unsloth_cli/_studio_deps.py looks in the same place for a foreign venv.
The installer is a different tree. A desktop bundle carries its own
`studio/install_python_stack.py`, and its requirements are whatever they were
when that bundle was cut -- so recording the installer's digests makes every
install stale the moment a tracked requirement file changes upstream. That is
not hypothetical: v0.1.800-beta (2026-08-14) installed unsloth 2026.8.18,
#9148 had pinned openai in extras.txt in between, and every fresh Linux and
macOS desktop install came up `studio_install_requirements_changed` and paid
an immediate repair pass before it would run.
"""
prefix = root or venv_root()
for pattern in ("lib/python*/site-packages", "Lib/site-packages"):
for site in sorted(prefix.glob(pattern)):
reqs = site / "studio" / "backend" / "requirements"
if reqs.is_dir():
return reqs
return None
def requirement_digests(req_root: Optional[Path] = None) -> Dict[str, str]:
"""sha256 of every tracked requirement file that exists."""
root = req_root or requirements_root()
digests: Dict[str, str] = {}
for name in TRACKED_REQUIREMENT_FILES:
digest = _sha256(root / name)
if digest is not None:
digests[name] = digest
return digests
def digest_file(path) -> Optional[str]:
"""sha256 of one file, or None when absent or unreadable; None never equals a recorded digest,
so an unreadable input forces the consuming step to run."""
try:
return _sha256(Path(path))
except (TypeError, ValueError):
return None
def pass_input_digests(req_root: Optional[Path] = None) -> Dict[str, str]:
"""sha256 of every PASS_INPUT_FILES entry that exists, relpath -> digest."""
root = Path(req_root) if req_root is not None else requirements_root()
digests: Dict[str, str] = {}
for name in PASS_INPUT_FILES:
digest = digest_file(root / name)
if digest is not None:
digests[name] = digest
return digests
def _canonical(name: str) -> str:
"""PEP 503 normalisation, so PyJWT / pyjwt / py_jwt compare equal."""
return re.sub(r"[-_.]+", "-", name).lower()
def _metadata_scan_paths() -> List[str]:
"""Site-packages roots minus inherited sys.path entries, deduplicated by real path: purelib
hardcodes lib while platlib follows sys.platlibdir, so lib64 builds would list one directory
twice."""
import sysconfig
paths: List[str] = []
seen: set = set()
try:
configured = sysconfig.get_paths()
except Exception:
return paths
for key in ("purelib", "platlib"):
path = configured.get(key)
if not path or not os.path.isdir(path):
continue
try:
key_path = os.path.realpath(path)
except OSError:
key_path = path
if key_path in seen:
continue
seen.add(key_path)
paths.append(path)
return paths
_METADATA_TEXT_FILES = frozenset(("METADATA", "PKG-INFO", ""))
_SINGLE_READ_TYPE = None
_METADATA_SCAN: "contextvars.ContextVar[Optional[dict]]" = contextvars.ContextVar(
"unsloth_install_manifest_metadata_scan", default = None
)
def _header_block(text: str) -> str:
"""*text* through its first blank line, where the email parser stops (the README is unused)."""
cut = -1
for separator in ("\n\n", "\r\n\r\n"):
at = text.find(separator)
if at != -1 and (cut == -1 or at + len(separator) < cut):
cut = at + len(separator)
return text if cut == -1 else text[:cut]
def _single_read(dist):
"""*dist* parsing its metadata headers once (stdlib re-parses per access); errors uncached."""
global _SINGLE_READ_TYPE
from importlib.metadata import PathDistribution
if type(dist) is not PathDistribution:
return dist
if _SINGLE_READ_TYPE is None:
class _SingleReadDistribution(PathDistribution):
def read_text(self, filename):
text = super().read_text(filename)
if text and filename in _METADATA_TEXT_FILES:
return _header_block(text)
return text
@property
def metadata(self):
cached = self.__dict__.get("_unsloth_metadata")
if cached is None:
cached = super().metadata
self.__dict__["_unsloth_metadata"] = cached
return cached
_SINGLE_READ_TYPE = _SingleReadDistribution
return _SINGLE_READ_TYPE(dist._path)
@contextlib.contextmanager
def _one_metadata_scan():
"""One record scan for the block, never across calls: the installer changes site-packages."""
if _METADATA_SCAN.get() is not None:
yield
return
token = _METADATA_SCAN.set({})
try:
yield
finally:
_METADATA_SCAN.reset(token)
def _one_metadata_scan_per_call(fn):
@functools.wraps(fn)
def wrapper(*args, **kwargs):
with _one_metadata_scan():
return fn(*args, **kwargs)
return wrapper
def _metadata_record_index(backups_only: bool = False) -> Dict[str, list]:
"""Canonical name -> [(dist, dir, dir-name key)]; nameless records file by dir name, no dist."""
from importlib.metadata import MetadataPathFinder, distributions
paths = _metadata_scan_paths()
# The listing cache is keyed on directory mtime, so a new dist-info can stay invisible.
# Via an INSTANCE: invalidate_caches is a classmethod only from 3.11.9 / 3.12.3.
if getattr(MetadataPathFinder, "invalidate_caches", None) is not None:
MetadataPathFinder().invalidate_caches()
kwargs = {"path": paths} if paths else {}
index: Dict[str, list] = {}
for dist in distributions(**kwargs):
path = getattr(dist, "_path", None)
try:
record_path = Path(os.fspath(path)) if path is not None else None
except (TypeError, ValueError):
record_path = None
if backups_only and (record_path is None or not record_path.name.startswith("~")):
continue
stem = record_path.name if record_path is not None else ""
path_name, separator, _version = stem.removesuffix(".dist-info").rpartition("-")
path_key = _canonical(path_name) if stem.endswith(".dist-info") and separator else None
dist = _single_read(dist)
try:
name = dist.metadata.get("Name")
if name:
index.setdefault(_canonical(name), []).append((dist, record_path, path_key))
continue
except Exception:
pass
if path_key is not None:
index.setdefault(path_key, []).append((None, record_path, None))
return index
def _records_named(index: Dict[str, list], key: str) -> List[Tuple[str, Optional[Path]]]:
found: List[Tuple[str, Optional[Path]]] = []
for dist, record_path, path_key in index.get(key, ()):
if dist is None:
found.append(("", record_path))
continue
try:
found.append((dist.version or "", record_path))
except Exception:
# An unreadable version leaves only the directory name to match on.
if path_key == key:
found.append(("", record_path))
return _sorted_records(found)
def _installed_metadata_records(dist_name: str) -> List[Tuple[str, Optional[Path]]]:
"""Every matching metadata version and its directory, when available."""
scan = _METADATA_SCAN.get()
if scan is None:
index = _metadata_record_index()
else:
index = scan.get("records")
if index is None:
index = scan["records"] = _metadata_record_index()
return _records_named(index, _canonical(dist_name))
def _sorted_records(found) -> List[Tuple[str, Optional[Path]]]:
return sorted(found, key = lambda record: (record[0], os.fspath(record[1] or "")))
def installed_versions(dist_name: str) -> List[str]:
"""Every metadata version for one canonical distribution name.
More than one answer is an inconsistent environment, not a choice between
equivalent records: importlib.metadata.version() returns whichever record
the finder yields first, which can be a dist-info left by a failed uninstall.
"""
return [version for version, _path in _installed_metadata_records(dist_name)]
def invalid_metadata_paths(dist_name: str) -> List[Path]:
"""Matching metadata directories that pip cannot safely identify."""
return [
path
for version, path in _installed_metadata_records(dist_name)
if not version and path is not None
]
def pip_backup_metadata_paths(dist_name: str) -> List[Path]:
"""Matching records left behind by an interrupted pip upgrade.
pip renames the outgoing distribution to a `~` prefixed sibling while it
installs the replacement, so a kill mid-operation keeps both. The METADATA
still names the real project, so it counts as a duplicate here, but pip
calls the directory invalid: `pip uninstall <name>` can never consume it.
"""
if _METADATA_SCAN.get() is not None:
records = _installed_metadata_records(dist_name)
else:
records = _records_named(_metadata_record_index(backups_only = True), _canonical(dist_name))
return [path for _version, path in records if path is not None and path.name.startswith("~")]
def metadata_conflict(versions: Sequence[str]) -> bool:
"""Whether matching metadata records are duplicated or unreadable."""
return len(versions) > 1 or any(not version for version in versions)
def _metadata_is_inconsistent(dist_name: str, versions: Optional[List[str]] = None) -> bool:
"""Flags duplicated, unreadable or pip-ignored metadata; a sole ~ backup is invisible to a
version count yet pip refuses it, and unflagged the fast path would skip the reinstall that
repairs it."""
if versions is None:
versions = installed_versions(dist_name)
return bool(metadata_conflict(versions) or pip_backup_metadata_paths(dist_name))
@_one_metadata_scan_per_call
def installed_version_probe(
dist_name: str, companion_names: Sequence[str] = ()
) -> Tuple[str, bool]:
"""One unambiguous version and whether any requested metadata conflicts."""
versions = installed_versions(dist_name)
conflict = _metadata_is_inconsistent(dist_name, versions) or any(
_metadata_is_inconsistent(name) for name in companion_names
)
version = versions[0] if len(versions) == 1 and versions[0] else ""
return version, conflict
def _installed_version(dist_name: str, installed: Optional[Dict[str, str]] = None) -> Optional[str]:
if installed is not None:
return installed.get(_canonical(dist_name))
return installed_version_probe(dist_name)[0] or None
def _publish_json(
path: Path,
payload: dict,
*,
only_if_present: bool = False,
) -> bool:
"""Write *payload* to *path* through this writer's own temp file, then replace.
A shared temp name is a second writer's file as much as this one's: either could clobber
the other between the write and the replace and publish the wrong payload.
*only_if_present* declines to recreate a manifest another updater removed meanwhile.
Callers hold the lock; this raises nothing they do not already catch.
"""
import tempfile # noqa: PLC0415 - stdlib, and not needed to read a manifest
text = json.dumps(payload, indent = 2, sort_keys = True)
descriptor, name = tempfile.mkstemp(dir = str(path.parent), prefix = path.name + ".", suffix = ".tmp")
tmp = Path(name)
try:
with os.fdopen(descriptor, "w", encoding = "utf-8") as handle:
handle.write(text)
# mkstemp creates 0600; keep the manifest's existing mode or the umask default.
try:
mode = path.stat().st_mode & 0o777
except OSError:
umask = os.umask(0)
os.umask(umask)
mode = 0o666 & ~umask
try:
os.chmod(tmp, mode)
except OSError:
pass # Windows, or a filesystem without modes: the content is what matters
if only_if_present and not path.exists():
return False
os.replace(tmp, path)
except BaseException:
try:
tmp.unlink()
except OSError:
pass
raise
finally:
try:
tmp.unlink()
except OSError:
pass
return True
@contextlib.contextmanager
def _manifest_lock(root: Optional[Path] = None):
"""Holds an exclusive lock on LOCK_NAME, best effort and never raising: a filesystem that cannot
lock (network mount, read-only prefix, no fcntl or msvcrt) runs unserialised rather than
failing; the deadline bounds retries, not one stalled syscall."""
handle = None
locked = False
try:
# O_NOFOLLOW so a symlink on the reserved name is not followed.
descriptor = os.open(
(root or venv_root()) / LOCK_NAME,
os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0),
0o644,
)
handle = os.fdopen(descriptor, "a+b")
except OSError:
handle = None
if handle is not None:
try:
import fcntl # noqa: PLC0415 - POSIX only, and absent on Windows
# Non-blocking with a deadline: LOCK_EX waits forever on a suspended peer.
deadline = time.monotonic() + LOCK_WAIT_SECONDS
while True:
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
locked = True
break
except OSError as exc:
# Mounts without locking answer at once and must not cost the deadline on every write.
if exc.errno not in (
errno.EWOULDBLOCK,
errno.EAGAIN,
errno.EACCES,
errno.EINTR,
):
break
if time.monotonic() >= deadline:
break
time.sleep(0.02)
except ImportError:
try:
import msvcrt # noqa: PLC0415 - the Windows half of the same thing
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
locked = True
except (ImportError, OSError):
locked = False
except (OSError, ValueError):
locked = False
try:
yield locked
finally:
_release_lock(handle, locked)
def _release_lock(handle, locked: bool) -> None:
"""Drop the lock and the handle. Never raises."""
if handle is None:
return
if locked:
try:
try:
import fcntl # noqa: PLC0415
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
except ImportError:
import msvcrt # noqa: PLC0415
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
except (OSError, ValueError):
pass
try:
handle.close()
except OSError:
pass
@contextlib.contextmanager
def pass_lock(root: Optional[Path] = None):
"""Held for a whole dependency pass, and never waited on. Never raises.
Yields False only when a peer is already inside a pass on this venv. Its packages are the
ones the recorded evidence describes, so a second pass has to run every step rather than
skip on an answer the peer is invalidating as it reads it. Not serialisation: the pass can
take tens of minutes and a waiter would be worse than the concurrency.
Best effort, as _manifest_lock: a filesystem that cannot lock reads as uncontended, which
is what every release before this did.
"""
handle = None
locked = False
contended = False
try:
descriptor = os.open(
(root or venv_root()) / PASS_LOCK_NAME,
os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0),
0o644,
)
handle = os.fdopen(descriptor, "a+b")
except OSError:
handle = None
if handle is not None:
try:
import fcntl # noqa: PLC0415 - POSIX only, and absent on Windows
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
locked = True
except OSError as exc:
contended = exc.errno in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES)
except ImportError:
try:
import msvcrt # noqa: PLC0415 - the Windows half of the same thing
handle.seek(0)
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
locked = True
except ImportError:
pass
except OSError as exc:
contended = exc.errno in (errno.EACCES, errno.EDEADLK)
except (OSError, ValueError):
pass
try:
yield not contended
finally:
_release_lock(handle, locked)
def remove_manifest(root: Optional[Path] = None) -> bool:
"""Called before the dependency pass so an aborted run cannot leave a valid one.
True when no manifest remains. A surviving marker still names this version and these
digests, so a pass killed afterwards would verify as complete.
Parked under PREVIOUS_MANIFEST_NAME rather than deleted: setup.ps1 calls this before pip,
torch and triton are replaced, so without the parked copy every Windows update re-ran the
whole pass. It is evidence only (see read_previous_manifest) and the next write_manifest
drops it. Deleting it stays the fallback when the rename is refused.
"""
path = manifest_path(root)
parked = previous_manifest_path(root)
with _manifest_lock(root):
return _remove_manifest_locked(root, path, parked)
_ABSENT_ERRNOS = frozenset({errno.ENOENT, errno.ENOTDIR, errno.EBADF, errno.ELOOP})
def manifest_is_present(path: Path) -> bool:
"""Whether *path* is there, answering True when the filesystem refuses to say.
stat, not Path.exists(): 3.13 raises EACCES out of exists() and 3.14 returns False from it
(gh-101357), so exists() means "absent" on one and "unknown" on the other. Every caller here
is deciding whether a marker still blocks the pass, and a marker that cannot be read is
still a marker. Never raises: remove_manifest reported a refusal before this existed.
"""
try:
path.stat()
except OSError as exc:
return exc.errno not in _ABSENT_ERRNOS
except ValueError:
return False
return True
def _remove_manifest_locked(root: Optional[Path], path: Path, parked: Path) -> bool:
try:
os.replace(path, parked)
except FileNotFoundError:
# A copy on the reserved name belongs to a dead run; remove it or the pass refuses behind it.
consume_previous_manifest(root)
return not manifest_is_present(parked)
except OSError:
# Clear the reserved name and retry before unlinking: setup.ps1 reads True as permission to mutate.
consume_previous_manifest(root)
if manifest_is_present(parked):
return False
try:
os.replace(path, parked)
return True
except FileNotFoundError:
return True
except OSError:
pass
try:
path.unlink()
except FileNotFoundError:
return True
except OSError:
return False
return True
def consume_previous_manifest(root: Optional[Path] = None) -> None:
"""Drop the parked copy once a pass has read it.
The parked copy stands in for the live manifest remove_manifest took away, and it
has to go the same way: a pass killed after reading it must not leave it for the
next run to read as evidence of a completed pass. Best effort, as remove_manifest's
own fallback is.
"""
try:
previous_manifest_path(root).unlink()
except OSError:
pass
def read_previous_manifest(root: Optional[Path] = None) -> Optional[dict]:
"""The manifest remove_manifest parked, or None.
Evidence of the last COMPLETED pass, for the dependency pass alone: every skip it
permits is still re-verified on disk, and verify_install never reads it, so this
can make an update faster but never make a half-built venv look finished.
"""
try:
raw = previous_manifest_path(root).read_text(encoding = "utf-8")
except (OSError, ValueError):
return None
try:
data = json.loads(raw)
except ValueError:
return None
return data if isinstance(data, dict) else None
# Additive evidence may never shadow these keys; one constant so both writers agree.
PROTECTED_MANIFEST_KEYS: Tuple[str, ...] = (
"schema",
"completed_at_ms",
"package",
"package_version",
"python",
"platform",
"prefix",
"steps_total",
"requirement_files",
"no_torch",
"expected_torch_tag",
"expected_torch_tag_pinned",
"woa_torch_index",
)
def write_manifest(
root: Optional[Path] = None,
req_root: Optional[Path] = None,
steps_total: int = 0,
package_name: str = "unsloth",
no_torch: Optional[bool] = None,
expected_torch_tag: Optional[str] = None,
expected_torch_tag_pinned: Optional[bool] = None,
woa_torch_index: Optional[str] = None,
extra: Optional[Dict[str, object]] = None,
) -> Optional[Path]:
"""Record a completed install. Never raises: no manifest reads as incomplete,
which is the safe answer."""
payload = {
"schema": MANIFEST_SCHEMA,
"completed_at_ms": int(time.time() * 1000),
"package": package_name,
"package_version": _installed_version(package_name),
"python": platform.python_version(),
"platform": f"{sys.platform}-{platform.machine()}",
"prefix": str(venv_root()),
"steps_total": steps_total,
# verify_install reads the installed package's requirements, so record the venv's copy.
"requirement_files": requirement_digests(installed_requirements_root(root) or req_root),
}
# Absent means "unknown", NOT False. Recorded because UNSLOTH_NO_TORCH is exported for one run only,
# so a later update would reinstall torch into a GGUF-only venv.
if no_torch is not None:
payload["no_torch"] = bool(no_torch)
# The FLAVOR, never the index URL: a pinned index can carry a token.
if expected_torch_tag:
payload["expected_torch_tag"] = str(expected_torch_tag).strip().lower()
# setup.ps1 picks /cpu automatically on GPU-less hosts; only a named flavor is deliberate.
if expected_torch_tag_pinned is not None:
payload["expected_torch_tag_pinned"] = bool(expected_torch_tag_pinned)
# Windows on ARM has no CUDA wheels on download.pytorch.org. Only NVIDIA's channels are persisted.
if woa_torch_index:
candidate = str(woa_torch_index).strip()
# urlsplit raises on a malformed authority, breaking the never-raises contract.
try:
parsed = urlsplit(candidate)
_woa_ok = (
parsed.scheme.lower() == "https"
and parsed.hostname == "pypi.nvidia.com"
and parsed.netloc.lower() == parsed.hostname
and not parsed.query
and not parsed.fragment
)
except ValueError:
_woa_ok = False
# netloc, not hostname: hostname strips ":443", so a value with a port is refused.
if _woa_ok:
payload["woa_torch_index"] = "https://pypi.nvidia.com" + parsed.path.rstrip("/")
for key, value in (extra or {}).items():
if value is None or key in payload or key in PROTECTED_MANIFEST_KEYS:
continue
payload[key] = value
path = manifest_path(root)
try:
with _manifest_lock(root):
_publish_json(path, payload)
# `extra` is caller-composed; raising here would leave a finished venv with no manifest.
except (OSError, TypeError, ValueError):
return None
try:
previous_manifest_path(root).unlink()
except OSError:
pass
return path
def update_manifest(root: Optional[Path] = None, **extra: object) -> bool:
"""Merge additive keys into an existing manifest. Never raises.
For evidence only available AFTER the manifest is written: the MLX probe runs there so a
kill during its 180 s timeout cannot lose a finished install. False means "record
nothing", never a failed install.
PROTECTED_MANIFEST_KEYS are dropped as write_manifest drops them: this merges into a file
that already claims the install finished, so rewriting the fields behind that claim could
leave it valid-looking and wrong.
"""
values = {
key: value
for key, value in extra.items()
if value is not None and key not in PROTECTED_MANIFEST_KEYS
}
if not values:
return False
path = manifest_path(root)
try:
# Read, merge and replace inside the lock: the caller's data may be minutes old.
with _manifest_lock(root) as locked:
if not locked:
# A peer that timed out this lock is mid-pass; publishing could restore its removed marker.
return False
data = read_manifest(root)
if data is None:
return False
data.update(values)
# Older peers remove without the lock, so check presence as late as possible.
if not _publish_json(path, data, only_if_present = True):
return False
except (OSError, TypeError, ValueError):
return False
return True
def read_manifest(root: Optional[Path] = None) -> Optional[dict]:
try:
raw = manifest_path(root).read_text(encoding = "utf-8")
except (OSError, ValueError):
return None
try:
data = json.loads(raw)
except ValueError:
return None
return data if isinstance(data, dict) else None
def no_torch_marker_path(root: Optional[Path] = None) -> Path:
return (root or venv_root()) / NO_TORCH_MARKER
def set_no_torch_marker(no_torch: bool, root: Optional[Path] = None) -> None:
"""Records the no-torch mode outside the completion manifest before the dependency pass, so an
interrupted install still knows its mode; removed when torch is wanted so no stale marker
remains. Never raises."""
path = no_torch_marker_path(root)
try:
if no_torch:
path.write_text("", encoding = "utf-8")
else:
path.unlink(missing_ok = True)
except OSError:
pass
def recorded_no_torch(root: Optional[Path] = None) -> Optional[bool]:
"""Recorded no-torch mode, or None when no manifest key or marker exists; callers must detect
for themselves on None and never assume False, so older installs are not silently switched
out of no-torch mode."""
manifest = read_manifest(root)
if manifest is not None:
value = manifest.get("no_torch")
if isinstance(value, bool):
return value
if isinstance(value, str):
return value.strip().lower() in NO_TORCH_TRUTHY
try:
if no_torch_marker_path(root).exists():
return True
except OSError:
pass
return None
def recorded_torch_flavor(root: Optional[Path] = None) -> Optional[str]:
"""Torch flavor recorded by the previous install, or None when unknown; never read None as cpu,
or a repair reinstalls over a deliberate build; no marker companion, as the manifest is
dropped each pass; a run whose setup script exported the flavor never reaches this."""
manifest = read_manifest(root)
if manifest is None:
return None
value = manifest.get("expected_torch_tag")
if not isinstance(value, str):
return None
value = value.strip().lower()
return value or None
def recorded_torch_flavor_was_pinned(root: Optional[Path] = None) -> bool:
"""Whether the recorded flavor was named rather than auto-selected; False when unrecorded, since
an unproven CPU record read as deliberate would leave a host that later gained a GPU with no
repair offered."""
manifest = read_manifest(root)
if manifest is None:
return False
# An ACTUAL boolean: bool("false") is True.
return manifest.get("expected_torch_tag_pinned") is True
def _parse_requirement_line(line: str) -> Optional[Tuple[str, str, str]]:
"""Returns (distribution name, marker, specifier) for a requirement line, or None for blanks and
pip flags; covers the names, specifiers, inline comments and markers that studio.txt uses."""
text = line.split("#", 1)[0].strip()
if not text or text.startswith("-"):
return None
try:
from packaging.requirements import Requirement
requirement = Requirement(text)
return (
requirement.name,
str(requirement.marker or ""),
str(requirement.specifier),
)
except Exception:
pass
marker = ""
if ";" in text:
text, marker = text.split(";", 1)
marker = marker.strip()
name = text.strip()
for sep in ("===", "==", ">=", "<=", "~=", "!=", ">", "<", "[", " "):
idx = name.find(sep)
if idx > 0:
name = name[:idx]
name = name.strip()
return (name, marker, "") if name else None
def _marker_applies(marker: str) -> bool:
"""True when the environment marker matches (or cannot be evaluated)."""
if not marker:
return True
try:
from packaging.markers import Marker
except Exception:
return True
try:
return bool(Marker(marker).evaluate())
except Exception:
return True
def _version_satisfies(version: str, specifier: str) -> bool:
if not specifier:
return True
try:
from packaging.specifiers import SpecifierSet
return SpecifierSet(specifier).contains(version)
except Exception:
return False
def missing_requirements(
req_file: Optional[Path] = None, installed: Optional[Dict[str, str]] = None
) -> List[str]:
"""Distribution names missing or outside their required versions, checked via importlib.metadata
since import names (jwt, docx, fitz) differ; the installed mapping checks another venv, which
importlib.metadata cannot see."""
from importlib.metadata import PackageNotFoundError, distribution
path = req_file or (requirements_root() / BOOT_REQUIREMENT_FILE)
try:
lines = path.read_text(encoding = "utf-8").splitlines()
except OSError:
return []
missing: List[str] = []
for line in lines:
parsed = _parse_requirement_line(line)
if parsed is None:
continue
name, marker, specifier = parsed
if not _marker_applies(marker):
continue
if installed is not None:
version = installed.get(_canonical(name))
if version is None or not _version_satisfies(version, specifier):
missing.append(name)
continue
try:
dist = distribution(name)
except PackageNotFoundError:
missing.append(name)
except Exception:
missing.append(name)
else:
if not _version_satisfies(dist.version, specifier):
missing.append(name)
return missing
CLOSURE_SCAN_BUDGET_SECONDS = 10.0
_CLOSURE_MAX_VISITS = 20000
def _is_pip_backup(dist) -> bool:
"""True for the ~ame-1.0.dist-info an interrupted pip upgrade leaves behind."""
return str(getattr(getattr(dist, "_path", None), "name", "")).startswith("~")
def installed_dependency_index() -> Optional[Dict[str, Tuple[str, List[str]]]]:
"""canonical name -> (version, raw Requires-Dist lines) for this interpreter.
One pass over the metadata, because every gated step asks the same question about
the same site-packages and `distribution(name)` re-reads a dist-info per lookup.
None when the metadata cannot be enumerated at all, which the caller reads as
"cannot audit" rather than "satisfied".
"""
from importlib.metadata import distributions
try:
index: Dict[str, Tuple[str, List[str]]] = {}
for dist in distributions(path = _metadata_scan_paths()):
# An interrupted pip upgrade leaves ~ame-1.0.dist-info with no payload; pip ignores those.
if _is_pip_backup(dist):
continue
dist = _single_read(dist)
try:
name = dist.metadata["Name"]
version = dist.version
except Exception:
continue
if not name or not version:
continue
key = _canonical(str(name))
if key in index:
continue
try:
requires = list(dist.requires or [])
except Exception:
# Fails closed, or a step would skip a closure it never walked.
return None
index[key] = (str(version), requires)
return index
except Exception:
return None
def unsatisfied_closure_requirement(
req_file: Path,
index: Optional[Dict[str, Tuple[str, List[str]]]] = None,
budget_seconds: float = CLOSURE_SCAN_BUDGET_SECONDS,
) -> Optional[str]:
"""The first requirement in *req_file*'s INSTALLED closure that is not met, or None."""
unmet = closure_unmet_requirements(req_file, index, budget_seconds)
return unmet[0] if unmet else None
def closure_unmet_requirements(
req_file: Path,
index: Optional[Dict[str, Tuple[str, List[str]]]] = None,
budget_seconds: float = CLOSURE_SCAN_BUDGET_SECONDS,
) -> List[str]:
"""Every requirement in *req_file*'s INSTALLED closure that is not met; [] if none.
A missing distribution is reported by name, a version outside its specifier as
"name version", and a reason the audit could not run as one "<...>" entry that stops the
walk, so a caller reads any "<" entry as "cannot audit".
missing_requirements() reads the file's own lines, which stay true after a transitive
dependency is uninstalled: `mammoth>=1.8.0` is satisfied by a mammoth whose `cobble` is
gone, and importing it raises. Repairing that is the step being considered for a skip, so
this follows Requires-Dist down from each line.
Only for steps installed WITH dependencies: a `--no-deps` step leaves its requirements'
own dependencies unresolved on purpose, and auditing one would run it forever.
Fails CLOSED, unlike missing_requirements() and violated_constraints(). Absent
`packaging`, a direct URL, an unreadable file or a budget overrun are not evidence that
the closure holds, and being wrong costs one dependency pass rather than a broken import.
"""
try:
from packaging.requirements import Requirement
except Exception:
return ["<packaging unavailable>"]
if index is None:
index = installed_dependency_index()
if index is None:
return ["<metadata unreadable>"]
try:
lines = Path(req_file).read_text(encoding = "utf-8-sig").splitlines()
except (OSError, ValueError):
return ["<requirements unreadable>"]
deadline = time.monotonic() + budget_seconds if budget_seconds > 0 else None
pending: List[Tuple[str, Tuple[str, ...]]] = []
for line in lines:
text = line.split("#", 1)[0].strip()
if not text:
continue
if text.startswith("-"):
return [f"<flag line: {text}>"]
pending.append((text, ("",)))
unmet: List[str] = []
seen: set = set()
visits = 0
while pending:
visits += 1
if visits > _CLOSURE_MAX_VISITS:
return ["<closure too large>"]
if deadline is not None and time.monotonic() > deadline:
return ["<closure audit timed out>"]
raw, contexts = pending.pop()
try:
requirement = Requirement(raw)
except Exception:
return [f"<unparseable: {raw}>"]
marker = requirement.marker
if marker is not None:
try:
applies = any(marker.evaluate({"extra": extra}) for extra in contexts)
except Exception:
return [f"<unevaluable marker: {raw}>"]
if not applies:
continue
if requirement.url:
return [f"{requirement.name} (direct reference)"]
key = _canonical(requirement.name)
record = index.get(key)
if record is None:
if requirement.name not in unmet:
unmet.append(requirement.name)
continue
version, requires = record
if requirement.specifier and not requirement.specifier.contains(version, prereleases = True):
entry = f"{requirement.name} {version}"
if entry not in unmet:
unmet.append(entry)
extras = tuple(sorted(_canonical(extra) for extra in requirement.extras))
visit_key = (key, extras)
if visit_key in seen:
continue
seen.add(visit_key)
# Both spellings: PEP 685 marker normalisation is newer than many wheels.
child_contexts = ("", *extras, *requirement.extras)
pending.extend((child, child_contexts) for child in requires)
return unmet
@_one_metadata_scan_per_call
def violated_constraints(
req_file: Optional[Path] = None, installed: Optional[Dict[str, str]] = None
) -> List[str]:
"""Constrained distributions whose INSTALLED version sits outside the pin.
A constraints file never asks for an install, so only a resident distribution outside its
window is a violation. `-c constraints.txt` reaches every constrained step, so a
constraint that moved under an unchanged requirements file is the one input digest
equality cannot see.
Empty on an unreadable file, matching missing_requirements: the caller's other evidence
still has to pass, and a violation nobody can name would force a full pass every run.
"""
path = req_file or (requirements_root() / "single-env" / "constraints.txt")
try:
lines = path.read_text(encoding = "utf-8-sig").splitlines()
except (OSError, ValueError):
return []
violated: List[str] = []
for line in lines:
parsed = _parse_requirement_line(line)
if parsed is None:
continue
name, marker, specifier = parsed
if not specifier or not _marker_applies(marker):
continue
if metadata_conflict(installed_versions(name)):
violated.append(name)
continue
version = _installed_version(name, installed)
if version and not _version_satisfies(version, specifier):
violated.append(name)
return violated
# Mirrors _SHARED_NON_RUNTIME_ROOTS in unsloth_cli/_studio_deps.py.
_SHARED_NON_RUNTIME_ROOTS = frozenset(
(
"test",
"tests",
"doc",
"docs",
"example",
"examples",
"benchmark",
"benchmarks",
"sample",
"samples",
"scripts",
)
)
# `_move_launcher_aside` renames this before setup; nothing else is excused.
_STAGED_LAUNCHER_NAME = "unsloth.exe"
_STAGED_LAUNCHER_SUFFIXES = (".update-stale", ".update-backup", ".deleteme")
_INSTALLER_REWRITTEN_NAMES = frozenset(("package-lock.json",))
def _restored_launcher(parts: Sequence[str]) -> bool:
"""The managed Windows launcher: an update may restore an older, smaller uv trampoline (#16897)."""
return (
len(parts) >= 2
and ".." in parts
and parts[-2].lower() == "scripts"
and parts[-1].lower() == _STAGED_LAUNCHER_NAME
)
# `npm run build` rehashes every asset, so RECORD names files our setup deleted.
_INSTALLER_REGENERATED_TREES = (("studio", "frontend", "dist"),)
def _staged_beside(target) -> bool:
"""Whether an absent launcher is one an update moved aside a moment ago.
Usable, not merely present: `_recover_missing_launcher` reads these through
`_is_valid_pe`, so a copy it would reject is no excuse. Same two-byte test.
"""
try:
path = Path(target)
if path.name != _STAGED_LAUNCHER_NAME:
return False
for suffix in _STAGED_LAUNCHER_SUFFIXES:
staged = path.with_name(path.name + suffix)
try:
if staged.stat().st_size < 2:
continue
with staged.open("rb") as handle:
if handle.read(2) == b"MZ":
return True
except OSError:
continue
return False
except (OSError, ValueError):
return False
def _within(target: Path, anchor: Path) -> bool:
"""Whether a parent-relative row lands inside the environment; one outside it belongs to
something else, which reinstalling ours cannot fix."""
try:
resolved = target.resolve()
except OSError:
return False
try:
resolved.relative_to(anchor)
except ValueError:
return False
return True
def _venv_anchor(site_packages: Path) -> Optional[Path]:
"""The venv a site-packages belongs to, or None and the caller skips them."""
try:
current = site_packages.resolve()
except OSError:
return None
for _ in range(4):
if (current / "pyvenv.cfg").is_file():
return current
if current == current.parent:
break
current = current.parent
return None
# Neither installer wraps the scan in a timeout, so a stalled mount would wedge setup.
PAYLOAD_SCAN_BUDGET_SECONDS = 5.0
def damaged_payload_files(
package_name: str = "unsloth",
limit: int = 3,
budget_seconds: float = PAYLOAD_SCAN_BUDGET_SECONDS,
companion_names: Sequence[str] = (),
scan_paths: Optional[Sequence[str]] = None,
) -> List[str]:
"""Recorded files of the managed distribution that are gone or truncated.
Every check above reads metadata, which a quarantine of the payload leaves
intact. Only the named package and companions, unlike `damaged_installed_files`:
this runs on the fast path to decide whether to repair ours. `scan_paths`
aims it at another venv. Never raises, and an environment it cannot read or
finish reading is reported undamaged, since guessing the other way would
repair a healthy venv on every run.
The walk's own deadline bounds many slow stats but not one that never
returns, which a wedged mount produces and no installer wraps in a timeout.
So it runs on a daemon thread and is abandoned; the interpreter does not
wait for one at exit (1.04s measured, thread parked in a syscall).
`budget_seconds = 0` is unbounded, for the installer already committed to a
full pass.
"""
if budget_seconds <= 0:
return _scan_payload_files(package_name, limit, 0.0, companion_names, scan_paths)
import threading
done: List[List[str]] = []
def scan() -> None:
done.append(
_scan_payload_files(package_name, limit, budget_seconds, companion_names, scan_paths)
)
worker = threading.Thread(target = scan, daemon = True)
worker.start()
worker.join(budget_seconds + 1.0)
return done[0] if done else []
def _scan_payload_files(
package_name: str,
limit: int,
budget_seconds: float,
companion_names: Sequence[str],
scan_paths: Optional[Sequence[str]],
) -> List[str]:
"""The walk itself. Bounded between calls only; see `damaged_payload_files`."""
import csv
import io
import stat
from importlib.metadata import distributions
found: List[str] = []
deadline = time.monotonic() + budget_seconds if budget_seconds > 0 else None
try:
wanted = {_canonical(name) for name in (package_name, *companion_names) if name}
paths = list(scan_paths) if scan_paths is not None else _metadata_scan_paths()
if not paths:
return found
seen: set = set()
for dist in distributions(path = paths):
dist = _single_read(dist)
try:
name = _canonical(dist.metadata["Name"] or "")
if name not in wanted or name in seen:
continue
seen.add(name)
record = dist.read_text("RECORD")
except Exception:
continue
if not record:
# uv and pip write RECORD last, so a .dist-info without one was interrupted.
dist_path = str(getattr(dist, "_path", "") or "")
if dist_path.endswith(".dist-info"):
found.append(f"{name}: RECORD is missing")
if len(found) >= limit:
return found
continue
try:
anchor = _venv_anchor(Path(dist.locate_file("")))
except Exception:
anchor = None
# csv, not splitlines: a quoted field may hold a newline
for row in csv.reader(io.StringIO(record, newline = "")):
if deadline is not None and time.monotonic() > deadline:
return found
rel = row[0] if row else ""
if not rel or rel.endswith("/"):
continue
norm = rel.replace("\\", "/")
if ".dist-info/" in norm or ".egg-info/" in norm or norm.endswith(".pyc"):
continue
parts = tuple(p for p in norm.split("/") if p and p != ".")
if not parts or norm.startswith("/") or ":" in parts[0]:
continue
if len(parts) > 1 and parts[0] in _SHARED_NON_RUNTIME_ROOTS:
continue
if any(parts[: len(tree)] == tree for tree in _INSTALLER_REGENERATED_TREES):
continue
try:
target = dist.locate_file(rel)
# `..` is ordinary for console scripts; bounded so a quarantined `bin/unsloth` is still caught.
if ".." in parts and (anchor is None or not _within(Path(target), anchor)):
continue
info = target.stat()
except FileNotFoundError:
if not _staged_beside(target):
found.append(f"{rel} is missing")
except NotADirectoryError:
found.append(f"{rel} is not reachable")
except OSError:
continue
else:
if not stat.S_ISREG(info.st_mode):
found.append(f"{rel} is not a regular file")
elif (
len(row) >= 3
and row[2]
and row[2].isdigit()
and parts[-1] not in _INSTALLER_REWRITTEN_NAMES
and not _restored_launcher(parts)
and info.st_size < int(row[2])
):
found.append(f"{rel} is {info.st_size} bytes, expected {row[2]}")
if len(found) >= limit:
return found
except Exception:
return found[:limit]
return found
@_one_metadata_scan_per_call
def verify_install(
root: Optional[Path] = None,
req_root: Optional[Path] = None,
package_name: str = "unsloth",
installed: Optional[Dict[str, str]] = None,
installed_conflicts: Optional[Sequence[str]] = None,
deep: bool = False,
scan_paths: Optional[Sequence[str]] = None,
manifest: Optional[dict] = None,
) -> dict:
"""Report whether the managed install finished and can still boot.
`manifest` verifies against a manifest the caller already holds (the dependency pass
checking a parked one) instead of the live file; the verdict is otherwise unchanged.
Reason strings are surfaced verbatim by the desktop preflight, so keep them stable.
Pass `installed`, `installed_conflicts` and the matching `root` / `req_root` to describe
a venv other than this interpreter's, or the version and dependency checks answer for
whichever venv the caller runs in.
`deep` adds the payload scan, off by default because an external CLI loads this module
out of the venv it drives and opt-out would spend the preflight's 10 second budget with
no way for an old caller to decline. `scan_paths` names that venv's site-packages,
without which RECORD rows resolve against the wrong tree.
"""
reqs = req_root or requirements_root()
missing = missing_requirements(reqs / BOOT_REQUIREMENT_FILE, installed = installed)
deps_ok = not missing
if manifest is None:
manifest = read_manifest(root)
manifest_ok = False
reason: Optional[str] = None
vanished = False
if manifest is None:
reason = "studio_install_incomplete"
elif manifest.get("schema") != MANIFEST_SCHEMA:
reason = "studio_install_manifest_schema"
else:
manifest_package = manifest.get("package") or package_name
if installed is None:
companions = () if _canonical(manifest_package) == "unsloth-zoo" else ("unsloth-zoo",)
current, local_conflict = installed_version_probe(manifest_package, companions)
else:
current = _installed_version(manifest_package, installed)
local_conflict = False
foreign_conflicts = {_canonical(name) for name in (installed_conflicts or ())}
core_conflict = _canonical(manifest_package) in foreign_conflicts or (
_canonical(manifest_package) != "unsloth-zoo" and "unsloth-zoo" in foreign_conflicts
)
recorded = manifest.get("package_version")
vanished = not current
if core_conflict or local_conflict:
reason = "studio_install_metadata_conflict"
elif current and recorded and current != recorded:
reason = "studio_install_version_changed"
elif manifest.get("requirement_files") != requirement_digests(reqs):
reason = "studio_install_requirements_changed"
else:
manifest_ok = True
if manifest_ok and not deps_ok:
reason = "studio_deps_missing"
if manifest_ok and deps_ok and deep and (installed is None or scan_paths):
scan_package = (manifest or {}).get("package") or package_name
companions = ("unsloth-zoo",) if _canonical(scan_package) == "unsloth" else ()
if not vanished:
for companion in companions:
present = (
_installed_version(companion, installed)
if installed is not None
else installed_version_probe(companion)[0]
)
if not present:
vanished = True
break
if vanished or damaged_payload_files(
scan_package, companion_names = companions, scan_paths = scan_paths
):
manifest_ok = False
reason = "studio_install_damaged"
return {
"ok": manifest_ok and deps_ok,
"manifest_ok": manifest_ok,
"deps_ok": deps_ok,
"missing": missing,
"reason": None if (manifest_ok and deps_ok) else (reason or "studio_deps_missing"),
}
# MIRRORED from `_venv_dir_is_valid` + `_sidecar_scan_impl` in
# studio/backend/utils/transformers_version.py, which the stdlib-only installer cannot import. Keep in sync.
_EXT_VERSION_TAG_RE = re.compile(r"\.(?:cpython-|cp)(\d{2,}t?)\b")
# Stable-ABI binaries: fine on a GIL build, a SIGSEGV on a free-threaded one.
_ABI3_EXT_RE = re.compile(r"\.abi3\.(?:so|pyd)$")
SIDECAR_SCAN_BUDGET_SECONDS = 5.0
def _current_ext_tag() -> str:
import sysconfig
return "{}{}{}".format(
sys.version_info.major,
sys.version_info.minor,
"t" if sysconfig.get_config_var("Py_GIL_DISABLED") else "",
)
def _sidecar_payload_present(root: Path, dist) -> bool:
"""Whether anything *dist* records as installed is on disk under *root*.
The fallback for distributions a directory name cannot reach: one shipping top-level
MODULES (`six.py`) has no directory, and an import name matching neither spelling of the
project (pillow -> PIL) has one this cannot guess. Reported stale, they make
`sidecar_is_current` rebuild a healthy several-hundred-MB tree on every update.
Deliberately weaker than _sidecar_damaged_files, which reads every RECORD row: this only
answers "did the payload arrive at all". Anything unreadable answers no.
"""
try:
recorded = list(dist.files or [])
except Exception:
return False
for entry in recorded:
parts = tuple(part for part in str(entry).replace("\\", "/").split("/") if part)
if (
not parts
or ".." in parts
or parts[0] in ("bin", "Scripts")
or parts[0].endswith((".dist-info", ".egg-info"))
):
continue
try:
if (root / parts[0]).exists():
return True
except OSError:
continue
return False
def _sidecar_pin_ok(root: Path, spec: str) -> Optional[str]:
"""None when the pin is satisfied in *root*, else why it is not."""
from importlib.metadata import distributions
name, _, wanted = spec.partition("==")
name = name.strip()
wanted = wanted.strip()
if not name:
return None
canonical = _canonical(name)
module = canonical.replace("-", "_")
directory_present = any((root / candidate).is_dir() for candidate in (module, canonical))
found: List[str] = []
payload_present = False
try:
for dist in distributions(path = [str(root)]):
dist = _single_read(dist)
try:
dist_name = dist.metadata.get("Name") or ""
except Exception:
continue
if _canonical(dist_name) != canonical:
continue
found.append(dist.version or "")
if not directory_present and not payload_present:
payload_present = _sidecar_payload_present(root, dist)
except Exception:
return f"{name} metadata unreadable"
if canonical in OPTIONAL_SIDECAR_PACKAGES and (
not found or (not directory_present and not payload_present)
):
return None
if not found:
return f"{name} not installed"
if not directory_present and not payload_present:
return f"{name} directory missing"
if len(found) > 1:
return f"{name} has {len(found)} metadata records"
if wanted and found[0] != wanted:
return f"{name}=={found[0] or 'unknown'}, want {wanted}"
return None
def _sidecar_damaged_files(
root: Path,
limit: int = 3,
budget_seconds: float = SIDECAR_SCAN_BUDGET_SECONDS,
required: Sequence[str] = (),
) -> List[str]:
"""Finds RECORD rows under a sidecar that are gone, truncated or built for another CPython;
fails open since a finding triggers a large delete, and is bounded by budget_seconds since a
stalled mount would wedge setup."""
import csv
import io
import stat
deadline = time.monotonic() + budget_seconds if budget_seconds > 0 else None
ext_tag = _current_ext_tag()
entries: List[Tuple[str, str, Optional[int], Path, str]] = []
owners: Dict[str, int] = {}
required_names = {_canonical(name) for name in required if name}
recordless: List[str] = []
try:
dist_infos = sorted(root.glob("*.dist-info"))
except OSError:
return []
for dist_info in dist_infos:
name = dist_info.name.split("-")[0]
try:
record = (dist_info / "RECORD").read_text(encoding = "utf-8", errors = "replace")
except FileNotFoundError:
if _canonical(name) in required_names:
recordless.append(f"{name}: RECORD is missing")
continue
except OSError:
continue
try:
rows = list(csv.reader(io.StringIO(record)))
except csv.Error:
continue
for row in rows:
rel = row[0] if row else ""
if not rel or rel.endswith("/"):
continue
if ".dist-info/" in rel or ".egg-info/" in rel or rel.endswith(".pyc"):
continue
parts = tuple(part for part in rel.replace("\\", "/").split("/") if part)
if (
rel.startswith("/")
or (len(rel) > 1 and rel[1] == ":")
or ".." in parts
or (parts and parts[0] in ("bin", "Scripts"))
):
continue
target = root / rel
key = os.path.normcase(str(target))
owners[key] = owners.get(key, 0) + 1
if len(parts) > 1 and parts[0] in _SHARED_NON_RUNTIME_ROOTS:
continue
recorded: Optional[int] = None
if len(row) >= 3 and row[2] and parts[-1] not in _INSTALLER_REWRITTEN_NAMES:
try:
recorded = int(row[2])
except ValueError:
recorded = None
entries.append((name, rel, recorded, target, key))
found: List[str] = list(recordless[:limit])
if len(found) >= limit:
return found
for name, rel, recorded, target, key in entries:
if deadline is not None and time.monotonic() > deadline:
return found
try:
info = target.stat()
except (FileNotFoundError, NotADirectoryError):
found.append(f"{name}: {rel} is missing")
except OSError:
continue
else:
if not stat.S_ISREG(info.st_mode):
found.append(f"{name}: {rel} is not a regular file")
# A path two distributions claim makes sizes ambiguous: a collision, not damage.
elif owners[key] == 1 and recorded is not None and info.st_size < recorded:
found.append(f"{name}: {rel} is {info.st_size} bytes, expected {recorded}")
elif rel.endswith((".so", ".pyd")):
base = rel.replace("\\", "/").rsplit("/", 1)[-1]
match = _EXT_VERSION_TAG_RE.search(base)
if match and match.group(1) != ext_tag:
found.append(
f"{name}: {rel} targets cp{match.group(1)}, interpreter is cp{ext_tag}"
)
elif match is None and ext_tag.endswith("t") and _ABI3_EXT_RE.search(base):
found.append(
f"{name}: {rel} is a stable-ABI build, which free-threaded "
f"cp{ext_tag} cannot load"
)
if len(found) >= limit:
return found
return found
# Mirrors transformers_version: the false-positive escape hatch must hold on the setup side too.
OPTIONAL_SIDECAR_PACKAGES = frozenset({"tiktoken"})
SIDECAR_FILE_CHECK_ENV = "UNSLOTH_SKIP_SIDECAR_FILE_CHECK"
def _sidecar_file_check_disabled() -> bool:
return os.environ.get(SIDECAR_FILE_CHECK_ENV, "").strip().lower() in ("1", "true", "yes", "on")
def sidecar_is_current(
venv_dir,
pins: Sequence[str],
budget_seconds: float = SIDECAR_SCAN_BUDGET_SECONDS,
) -> Tuple[bool, str]:
"""Whether a transformers sidecar directory already holds exactly *pins*, intact.
`(True, "")` or `(False, reason)`. The reason is logged by the setup scripts, so
a rebuild always says what it is repairing.
*pins* are `name==version` or a bare `name` (present at any version). Every check
is on-disk evidence: nothing here trusts a previous run's record, because the
directory is what the next `import transformers` will read.
"""
root = Path(venv_dir)
try:
if not root.is_dir():
return False, "missing"
if not os.listdir(root):
return False, "empty"
except OSError as exc:
return False, f"unreadable ({exc.__class__.__name__})"
# Rebuilding means `rm -rf`: an unowned directory must not be called current.
if not (root / SIDECAR_OWNED_MARKER).is_file():
return False, f"no {SIDECAR_OWNED_MARKER} marker"
for spec in pins:
problem = _sidecar_pin_ok(root, spec)
if problem is not None:
return False, problem
if _sidecar_file_check_disabled():
return True, ""
damaged = _sidecar_damaged_files(
root,
budget_seconds = budget_seconds,
required = [spec.split("==")[0] for spec in pins if "==" in spec],
)
if damaged:
return False, "; ".join(damaged)
return True, ""
# Exit 0 current, 1 not, 2 unimplemented. Shells also require the marker line: an older
# install_manifest.py exits 0 with no output.
_SIDECAR_CLI_MARKER = "sidecar:"
def _sidecar_cli(argv: Sequence[str]) -> int:
if len(argv) < 2:
print("usage: install_manifest.py sidecar <dir> <pin>...", file = sys.stderr)
return 2
current, reason = sidecar_is_current(argv[0], tuple(argv[1:]))
print(f"{_SIDECAR_CLI_MARKER} {'current' if current else reason}")
return 0 if current else 1
if __name__ == "__main__":
if sys.argv[1:2] == ["sidecar"]:
sys.exit(_sidecar_cli(sys.argv[2:]))
print(f"usage: {os.path.basename(__file__)} sidecar <dir> <pin>...", file = sys.stderr)
sys.exit(2)