Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: read each dist-info METADATA once per install check (#15690)
Daniel Hanafd008e1b5d4
diff --git a/studio/install_manifest.py b/studio/install_manifest.pyindex 5397c8de6a9..4ed5d00ddc8 100644--- a/studio/install_manifest.py+++ b/studio/install_manifest.py@@ -8,7 +8,9 @@ desktop-capabilities and setup.sh/setup.ps1; stdlib only, packaging optional."""from __future__ import annotationsimport contextlib+import contextvarsimport errno+import functoolsimport hashlibimport jsonimport os@@ -185,36 +187,136 @@ def _metadata_scan_paths() -> List[str]:return paths-def _installed_metadata_records(dist_name: str) -> List[Tuple[str, Optional[Path]]]:- """Every matching metadata version and its directory, when available."""+_METADATA_TEXT_FILES = frozenset(("METADATA", "PKG-INFO", ""))+_SINGLE_READ_TYPE = None+_METADATA_SCAN: "contextvars.ContextVar[Optional[dict]]" = contextvars.ContextVar(+ "unsloth_install_manifest_metadata_scan", default = None+)+++def _header_block(text: str) -> str:+ """*text* through its first blank line, where the email parser stops (the README is unused)."""+ cut = -1+ for separator in ("\n\n", "\r\n\r\n"):+ at = text.find(separator)+ if at != -1 and (cut == -1 or at + len(separator) < cut):+ cut = at + len(separator)+ return text if cut == -1 else text[:cut]+++def _single_read(dist):+ """*dist* parsing its metadata headers once (stdlib re-parses per access); errors uncached."""+ global _SINGLE_READ_TYPE+ from importlib.metadata import PathDistribution++ if type(dist) is not PathDistribution:+ return dist+ if _SINGLE_READ_TYPE is None:++ class _SingleReadDistribution(PathDistribution):+ def read_text(self, filename):+ text = super().read_text(filename)+ if text and filename in _METADATA_TEXT_FILES:+ return _header_block(text)+ return text++ @property+ def metadata(self):+ cached = self.__dict__.get("_unsloth_metadata")+ if cached is None:+ cached = super().metadata+ self.__dict__["_unsloth_metadata"] = cached+ return cached++ _SINGLE_READ_TYPE = _SingleReadDistribution+ return _SINGLE_READ_TYPE(dist._path)+++def _one_metadata_scan():+ """One record scan for the block, never across calls: the installer changes site-packages."""+ if _METADATA_SCAN.get() is not None:+ yield+ return+ token = _METADATA_SCAN.set({})+ try:+ yield+ finally:+ _METADATA_SCAN.reset(token)+++def _one_metadata_scan_per_call(fn):+ @functools.wraps(fn)+ def wrapper(*args, **kwargs):+ with _one_metadata_scan():+ return fn(*args, **kwargs)++ return wrapper+++def _metadata_record_index(backups_only: bool = False) -> Dict[str, list]:+ """Canonical name -> [(dist, dir, dir-name key)]; nameless records file by dir name, no dist."""from importlib.metadata import MetadataPathFinder, distributions- wanted = _canonical(dist_name)paths = _metadata_scan_paths()# The listing cache is keyed on directory mtime, so a new dist-info can stay invisible.# Via an INSTANCE: invalidate_caches is a classmethod only from 3.11.9 / 3.12.3.if getattr(MetadataPathFinder, "invalidate_caches", None) is not None:MetadataPathFinder().invalidate_caches()kwargs = {"path": paths} if paths else {}- found: List[Tuple[str, Optional[Path]]] = []+ index: Dict[str, list] = {}for dist in distributions(**kwargs):path = getattr(dist, "_path", None)try:record_path = Path(os.fspath(path)) if path is not None else Noneexcept (TypeError, ValueError):record_path = None+ if backups_only and (record_path is None or not record_path.name.startswith("~")):+ continue+ stem = record_path.name if record_path is not None else ""+ path_name, separator, _version = stem.removesuffix(".dist-info").rpartition("-")+ path_key = _canonical(path_name) if stem.endswith(".dist-info") and separator else None+ dist = _single_read(dist)try:name = dist.metadata.get("Name")if name:- if _canonical(name) == wanted:- found.append((dist.version or "", record_path))+ index.setdefault(_canonical(name), []).append((dist, record_path, path_key))continueexcept Exception:pass- stem = record_path.name if record_path is not None else ""- path_name, separator, _version = stem.removesuffix(".dist-info").rpartition("-")- if stem.endswith(".dist-info") and separator and _canonical(path_name) == wanted:+ if path_key is not None:+ index.setdefault(path_key, []).append((None, record_path, None))+ return index+++def _records_named(index: Dict[str, list], key: str) -> List[Tuple[str, Optional[Path]]]:+ found: List[Tuple[str, Optional[Path]]] = []+ for dist, record_path, path_key in index.get(key, ()):+ if dist is None:found.append(("", record_path))+ continue+ try:+ found.append((dist.version or "", record_path))+ except Exception:+ # An unreadable version leaves only the directory name to match on.+ if path_key == key:+ found.append(("", record_path))+ return _sorted_records(found)+++def _installed_metadata_records(dist_name: str) -> List[Tuple[str, Optional[Path]]]:+ """Every matching metadata version and its directory, when available."""+ scan = _METADATA_SCAN.get()+ if scan is None:+ index = _metadata_record_index()+ else:+ index = scan.get("records")+ if index is None:+ index = scan["records"] = _metadata_record_index()+ return _records_named(index, _canonical(dist_name))+++def _sorted_records(found) -> List[Tuple[str, Optional[Path]]]:return sorted(found, key = lambda record: (record[0], os.fspath(record[1] or "")))@@ -245,11 +347,11 @@ def pip_backup_metadata_paths(dist_name: str) -> List[Path]:still names the real project, so it counts as a duplicate here, but pipcalls the directory invalid: `pip uninstall <name>` can never consume it."""- return [- path- for _version, path in _installed_metadata_records(dist_name)- if path is not None and path.name.startswith("~")- ]+ if _METADATA_SCAN.get() is not None:+ records = _installed_metadata_records(dist_name)+ else:+ records = _records_named(_metadata_record_index(backups_only = True), _canonical(dist_name))+ return [path for _version, path in records if path is not None and path.name.startswith("~")]def metadata_conflict(versions: Sequence[str]) -> bool:@@ -270,6 +372,7 @@ def _metadata_is_inconsistent(dist_name: str, versions: Optional[List[str]] = Noreturn bool(metadata_conflict(versions) or pip_backup_metadata_paths(dist_name))+@_one_metadata_scan_per_calldef installed_version_probe(dist_name: str, companion_names: Sequence[str] = ()) -> Tuple[str, bool]:@@ -912,6 +1015,7 @@ def installed_dependency_index() -> Optional[Dict[str, Tuple[str, List[str]]]]:# An interrupted pip upgrade leaves ~ame-1.0.dist-info with no payload; pip ignores those.if _is_pip_backup(dist):continue+ dist = _single_read(dist)try:name = dist.metadata["Name"]version = dist.version@@ -1035,6 +1139,7 @@ def closure_unmet_requirements(return unmet+@_one_metadata_scan_per_calldef violated_constraints(req_file: Optional[Path] = None, installed: Optional[Dict[str, str]] = None) -> List[str]:@@ -1221,6 +1326,7 @@ def _scan_payload_files(return foundseen: set = set()for dist in distributions(path = paths):+ dist = _single_read(dist)try:name = _canonical(dist.metadata["Name"] or "")if name not in wanted or name in seen:@@ -1289,6 +1395,7 @@ def _scan_payload_files(return found+@_one_metadata_scan_per_calldef verify_install(root: Optional[Path] = None,req_root: Optional[Path] = None,@@ -1450,6 +1557,7 @@ def _sidecar_pin_ok(root: Path, spec: str) -> Optional[str]:payload_present = Falsetry:for dist in distributions(path = [str(root)]):+ dist = _single_read(dist)try:dist_name = dist.metadata.get("Name") or ""except Exception:diff --git a/tests/studio/install/test_install_metadata_single_read.py b/tests/studio/install/test_install_metadata_single_read.pynew file mode 100644index 00000000000..94b6097d910--- /dev/null+++ b/tests/studio/install/test_install_metadata_single_read.py@@ -0,0 +1,251 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Install health checks read each dist-info METADATA once per call, headers only, same answers."""++from __future__ import annotations++import email+import importlib.metadata+import importlib.util+import pathlib+import sys++import pytest++REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]+MODULE_PATH = REPO_ROOT / "studio" / "install_manifest.py"+++def _load_module():+ spec = importlib.util.spec_from_file_location(+ "studio_install_manifest_single_read", MODULE_PATH+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++im = _load_module()++README = "".join(f"Line {i}: a README line that looks like a header\n" for i in range(200))++# Shapes a parser could split differently once the body is gone.+HEADER_SHAPES = {+ "lf": "Metadata-Version: 2.1\nName: lf\nVersion: 1.0\nRequires-Dist: a>=1\n\n" + README,+ "crlf": "Name: crlf\r\nVersion: 1.0\r\nRequires-Dist: a\r\n\r\nName: body\r\n",+ "lf_after_crlf": "Name: m\r\nVersion: 1.0\r\n\nRequires-Dist: hidden\r\n\r\nbody",+ "crlf_after_lf": "Name: m\nVersion: 1.0\n\r\nRequires-Dist: hidden\n\nbody",+ "cr_only": "Name: cr\rVersion: 1.0\r\rRequires-Dist: hidden\n\nbody",+ "folded": "Name: f\nVersion: 1.0\nRequires-Dist: a;\n python_version >= '3'\n\nbody\n\nmore",+ "no_separator": "Name: n\nVersion: 1.0\nnot a header\nVersion: 7\n\nbody",+ "leading_blank": "\nName: lb\nVersion: 1.0\n\nbody",+ "headers_only": "Name: h\nVersion: 1.0\n",+ "unterminated": "Name: u\nVersion: 1.0",+ "multipart": "Name: mp\nVersion: 1.0\nContent-Type: multipart/mixed; boundary=X\n\n--X\nName: in\n\nx\n--X--\n",+ "from_line": "From nobody\nName: fl\nVersion: 1.0\n\nbody",+ "duplicate_headers": "Name: d\nName: e\nVersion: 1.0\nVersion: 2.0\n\nbody",+ "empty": "",+}+++def _write(+ site: pathlib.Path,+ dirname: str,+ metadata,+ raw: bool = False,+) -> pathlib.Path:+ info = site / dirname+ info.mkdir(parents = True)+ if metadata is not None:+ if raw:+ (info / "METADATA").write_bytes(metadata)+ else:+ # write_bytes, not write_text(newline=""): that argument is 3.10+.+ (info / "METADATA").write_bytes(metadata.encode("utf-8"))+ return info+++def _md(+ name: str,+ version: str,+ extra: str = "",+) -> str:+ return f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n{extra}\n{README}"+++def site(tmp_path, monkeypatch):+ site = tmp_path / "site-packages"+ site.mkdir()+ _write(site, "unsloth-2026.10.1.dist-info", _md("unsloth", "2026.10.1"))+ _write(site, "unsloth_zoo-2026.10.1.dist-info", _md("unsloth_zoo", "2026.10.1"))+ _write(site, "demo-1.0.dist-info", _md("demo", "1.0", "Requires-Dist: dep>=1\n"))+ _write(site, "dep-2.0.dist-info", _md("dep", "2.0"))+ for index in range(20):+ _write(site, f"filler{index}-1.0.dist-info", _md(f"filler{index}", "1.0"))+ monkeypatch.setattr(im, "_metadata_scan_paths", lambda: [str(site)])+ return site+++def metadata_reads(monkeypatch):+ """Counts METADATA reads by directory, through the stdlib reader every path uses."""+ reads = []+ original = importlib.metadata.PathDistribution.read_text++ def counting(self, filename):+ if filename == "METADATA":+ reads.append(pathlib.Path(str(self._path)).name)+ return original(self, filename)++ monkeypatch.setattr(importlib.metadata.PathDistribution, "read_text", counting)+ return reads++[email protected]("shape", sorted(HEADER_SHAPES))+def test_header_block_parses_to_the_same_headers(shape):+ text = HEADER_SHAPES[shape]+ full = email.message_from_string(text)+ cut = email.message_from_string(im._header_block(text))+ assert cut.items() == full.items()+ assert im._header_block(im._header_block(text)) == im._header_block(text)++[email protected]("shape", sorted(HEADER_SHAPES))+def test_single_read_distribution_answers_like_the_stdlib(tmp_path, shape):+ site = tmp_path / "site"+ _write(site, "pkg-1.0.dist-info", HEADER_SHAPES[shape])+ (dist,) = list(importlib.metadata.distributions(path = [str(site)]))+ fresh = importlib.metadata.PathDistribution(dist._path)+ single = im._single_read(importlib.metadata.PathDistribution(dist._path))++ def answers(d):+ out = []+ for read in (+ lambda: d.metadata.get("Name"),+ lambda: d.metadata.get_all("Requires-Dist"),+ lambda: d.version,+ lambda: d.requires,+ ):+ try:+ out.append(("ok", read()))+ except Exception as exc:+ out.append(("raise", type(exc)))+ return out++ assert answers(single) == answers(fresh)+ assert answers(single) == answers(fresh)+++def test_undecodable_body_still_reads_as_unreadable(tmp_path, monkeypatch):+ """The whole file is decoded as before: a bad byte in the README is not skipped."""+ site = tmp_path / "site"+ _write(site, "latin-1.0.dist-info", b"Name: latin\nVersion: 1.0\n\nZo\xeb\n", raw = True)+ monkeypatch.setattr(im, "_metadata_scan_paths", lambda: [str(site)])+ assert im.installed_versions("latin") == [""]+ assert im.installed_version_probe("latin") == ("", True)+++def test_probe_reads_each_metadata_once(site, metadata_reads):+ assert im.installed_version_probe("unsloth", ("unsloth-zoo",)) == ("2026.10.1", False)+ assert metadata_reads, "the probe must read metadata"+ assert len(metadata_reads) == len(set(metadata_reads)), sorted(metadata_reads)+++def test_violated_constraints_reads_each_metadata_once(site, tmp_path, metadata_reads):+ constraints = tmp_path / "constraints.txt"+ constraints.write_text(+ "".join(f"filler{index}<2\n" for index in range(20)) + "demo>=2\nabsent>=1\n",+ encoding = "utf-8",+ )+ assert im.violated_constraints(constraints) == ["demo"]+ assert len(metadata_reads) == len(set(metadata_reads)), sorted(metadata_reads)+++def test_dependency_index_reads_each_metadata_once(site, metadata_reads):+ index = im.installed_dependency_index()+ assert index["demo"] == ("1.0", ["dep>=1"])+ assert len(metadata_reads) == len(set(metadata_reads)), sorted(metadata_reads)+++def test_pip_backup_lookup_reads_only_backup_directories(site, metadata_reads):+ _write(site, "~nsloth_zoo-2026.9.1.dist-info", _md("unsloth_zoo", "2026.9.1"))+ backups = im.pip_backup_metadata_paths("unsloth-zoo")+ assert [path.name for path in backups] == ["~nsloth_zoo-2026.9.1.dist-info"]+ assert set(metadata_reads) == {"~nsloth_zoo-2026.9.1.dist-info"}+++def test_the_shared_scan_never_outlives_one_call(site):+ """The installer changes site-packages between calls: each call must see the tree afresh."""+ assert im.installed_version_probe("late") == ("", False)+ _write(site, "late-1.0.dist-info", _md("late", "1.0"))+ assert im.installed_version_probe("late") == ("1.0", False)+ _write(site, "~ate-0.9.dist-info", _md("late", "0.9"))+ assert im.installed_version_probe("late") == ("", True)+ assert im._METADATA_SCAN.get() is None+++def test_monkeypatched_installed_versions_still_drives_violated_constraints(tmp_path, monkeypatch):+ constraints = tmp_path / "constraints.txt"+ constraints.write_text("torch<3\n", encoding = "utf-8")+ monkeypatch.setattr(im, "installed_versions", lambda name: ["2.0", "2.1"])+ assert im.violated_constraints(constraints) == ["torch"]+++def test_studio_deps_mirror_matches_install_manifest(tmp_path):+ sys.path.insert(0, str(REPO_ROOT))+ try:+ from unsloth_cli import _studio_deps+ finally:+ sys.path.remove(str(REPO_ROOT))+ for text in HEADER_SHAPES.values():+ assert _studio_deps._header_block(text) == im._header_block(text)+ site = tmp_path / "site"+ _write(site, "pkg-1.0.dist-info", HEADER_SHAPES["crlf"])+ (dist,) = list(importlib.metadata.distributions(path = [str(site)]))+ mirrored = _studio_deps._single_read(dist)+ own = im._single_read(dist)+ assert type(mirrored).__name__ == type(own).__name__ == "_SingleReadDistribution"+ assert (mirrored.version, mirrored.requires) == (own.version, own.requires)+++class _FakeDist:+ """A finder result whose Version cannot be read although its Name can."""++ def __init__(self, path: pathlib.Path, name: str):+ self._path = path+ self.metadata = {"Name": name}++ @property+ def version(self):+ raise KeyError("Version")+++def test_an_unreadable_version_counts_only_under_its_matching_directory(tmp_path, monkeypatch):+ dists = [+ _FakeDist(tmp_path / "foo-1.0.dist-info", "foo"),+ _FakeDist(tmp_path / "bar-1.0.dist-info", "baz"),+ ]+ monkeypatch.setattr(im, "_metadata_scan_paths", lambda: [str(tmp_path)])+ monkeypatch.setattr(importlib.metadata, "distributions", lambda **_kw: iter(dists))+ assert im.installed_versions("foo") == [""]+ assert im.installed_versions("bar") == []+ assert im.installed_versions("baz") == []+ assert im.installed_version_probe("foo") == ("", True)+ assert im.installed_version_probe("baz") == ("", False)+++def test_a_lookup_reads_versions_only_for_the_name_it_asks_about(site, monkeypatch):+ """A Version-less record elsewhere stays silent: 3.12 / 3.13 warn when its version is read."""+ _write(site, "versionless-1.0.dist-info", "Metadata-Version: 2.1\nName: versionless\n\nbody")+ asked = []+ original = importlib.metadata.Distribution.version++ def recording(self):+ asked.append(pathlib.Path(str(self._path)).name)+ return original.fget(self)++ monkeypatch.setattr(importlib.metadata.Distribution, "version", property(recording))+ assert im.installed_version_probe("demo", ("unsloth-zoo",)) == ("1.0", False)+ assert sorted(set(asked)) == ["demo-1.0.dist-info", "unsloth_zoo-2026.10.1.dist-info"]diff --git a/unsloth_cli/_studio_deps.py b/unsloth_cli/_studio_deps.pyindex 9c3af6b7cd5..ce491977449 100644--- a/unsloth_cli/_studio_deps.py+++ b/unsloth_cli/_studio_deps.py@@ -205,6 +205,47 @@ def _managed_root(extra_roots: Sequence[Path]) -> Optional[Path]:return None+# Mirror of studio/install_manifest.py (loaded by path, not importable here); a test pins them.+_METADATA_TEXT_FILES = frozenset(("METADATA", "PKG-INFO", ""))+_SINGLE_READ_TYPE = None+++def _header_block(text: str) -> str:+ cut = -1+ for separator in ("\n\n", "\r\n\r\n"):+ at = text.find(separator)+ if at != -1 and (cut == -1 or at + len(separator) < cut):+ cut = at + len(separator)+ return text if cut == -1 else text[:cut]+++def _single_read(dist):+ global _SINGLE_READ_TYPE+ from importlib.metadata import PathDistribution++ if type(dist) is not PathDistribution:+ return dist+ if _SINGLE_READ_TYPE is None:++ class _SingleReadDistribution(PathDistribution):+ def read_text(self, filename):+ text = super().read_text(filename)+ if text and filename in _METADATA_TEXT_FILES:+ return _header_block(text)+ return text++ @property+ def metadata(self):+ cached = self.__dict__.get("_unsloth_metadata")+ if cached is None:+ cached = super().metadata+ self.__dict__["_unsloth_metadata"] = cached+ return cached++ _SINGLE_READ_TYPE = _SingleReadDistribution+ return _SINGLE_READ_TYPE(dist._path)++def _distributions_in(root: Path) -> Optional[tuple[Dict[str, str], set[str]]]:"""Installed versions and metadata conflicts inside another venv.@@ -223,6 +264,7 @@ def _distributions_in(root: Path) -> Optional[tuple[Dict[str, str], set[str]]]:for dist in Distribution.discover(context = DistributionFinder.Context(path = paths)):path = getattr(dist, "_path", None)stem = os.path.basename(os.fspath(path)) if path is not None else ""+ dist = _single_read(dist)try:name = dist.metadata.get("Name")if name:@@ -452,6 +494,7 @@ def _installed_distribution_groups():groups: Dict[str, list] = {}for dist in distributions(**_scan_paths()):+ dist = _single_read(dist)try:name = dist.metadata.get("Name")except Exception: