Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: skip the startup manifest migration scan when nothing changed since a clean pass (#16270)
Daniel Hanf6f550d4bd0f
diff --git a/studio/backend/hub/utils/download_manifest.py b/studio/backend/hub/utils/download_manifest.pyindex 41f0ccc094d..983651c830b 100644--- a/studio/backend/hub/utils/download_manifest.py+++ b/studio/backend/hub/utils/download_manifest.py@@ -9,6 +9,8 @@ import hashlibimport jsonimport osimport re+import stat+import timeimport unicodedataimport uuidfrom dataclasses import dataclass@@ -28,6 +30,7 @@ from hub.utils.state_dir import (marker_path,normalize_hub_cache,state_filename_is_ambiguous,+ state_root,variant_filename_prefix,variant_key_fragments,)@@ -47,6 +50,11 @@ _MANIFEST_MIGRATION_MAX_FILES = 10_000_MANIFEST_MIGRATION_MAX_BYTES = 32 * 1024 * 1024_MANIFEST_MIGRATION_PREFIX_BYTES = 256_V2_MANIFEST_PREFIX = re.compile(rb'^\s*\{\s*"version"\s*:\s*2\s*,')+# Bump when the migration's selection rules change, so a recorded clean pass no longer applies.+_MANIFEST_MIGRATION_ID = "ordinary-v2-to-v1:1"+_MANIFEST_MIGRATION_MARKER = "manifest_migration.json"+# A dir changed this recently may sit inside the mtime granularity: not recorded (git's "racy" index rule).+_MANIFEST_MIGRATION_SETTLE_NS = 2_000_000_000# Verbatim phrase the worker emits on a degraded completion; shared so emit and match stay coupled.MANIFEST_DEGRADED_MARKER = "completed without a manifest so partial detection is degraded"@@ -635,6 +643,7 @@ def _is_reserved_dataset_completion_payload(data: dict) -> bool:def _read_migration_payload(path: Path) -> Optional[dict]:+ # OSError propagates: an unreadable file is skipped by the caller and keeps the pass unclean.try:with path.open("rb") as handle:prefix = handle.read(_MANIFEST_MIGRATION_PREFIX_BYTES)@@ -646,30 +655,140 @@ def _read_migration_payload(path: Path) -> Optional[dict]:if len(raw) > _MANIFEST_MIGRATION_MAX_BYTES:return Nonedata = json.loads(raw)- except (OSError, ValueError):+ except ValueError:return Nonereturn data if isinstance(data, dict) else None-def _migration_manifest_paths(parent: Path) -> Iterator[Path]:- yield from parent.glob("*.json")- for scoped in parent.glob("cache-*"):+def _listed(directory: Path) -> list[tuple[str, str]]:+ # Eager, and folded on Windows, as pathlib's glob is.+ with os.scandir(directory) as entries:+ return [+ (entry.name, entry.name.casefold() if os.name == "nt" else entry.name)+ for entry in entries+ ]+++def _migration_manifest_paths(parent: Path, failed: list) -> Iterator[Path]:+ """Like globbing ``*.json`` and ``cache-*/*.json``, but a scope that cannot be listed is appended to+ ``failed`` instead of silently yielding nothing (``Path.glob`` swallows ``OSError``)."""+ try:+ top = _listed(parent)+ except FileNotFoundError:+ return+ for name, folded in top:+ if folded.endswith(".json"):+ yield parent / name+ for name, folded in top:+ if not folded.startswith("cache-"):+ continue+ scoped = parent / nameif scoped.is_symlink() or not scoped.is_dir():continue- yield from scoped.glob("*.json")+ try:+ listed = _listed(scoped)+ except OSError as exc:+ failed.append(scoped)+ logger.debug("Could not list Hub manifest scope %s: %s", scoped, exc)+ continue+ for child, child_folded in listed:+ if child_folded.endswith(".json"):+ yield scoped / child+++def _migration_fingerprint(parent: Path, scopes: Optional[Iterable[str]] = None) -> Optional[dict]:+ """Dir mtimes + active cache. Manifest writes are ``tmp + os.replace``, so any write moves its dir's+ mtime and a new ``cache-*`` scope moves the parent's. ``None`` when anything cannot be read."""+ try:+ active, raw = _hub_cache_spellings(None)+ dirs = {".": os.stat(parent).st_mtime_ns}+ if scopes is None:+ scopes = [+ entry.name+ for entry in os.scandir(parent)+ if entry.name.startswith("cache-") and entry.is_dir(follow_symlinks = False)+ ]+ for name in scopes:+ st = os.lstat(parent / name)+ if not stat.S_ISDIR(st.st_mode):+ return None+ dirs[name] = st.st_mtime_ns+ except (OSError, TypeError, ValueError):+ return None+ return {+ "migration": _MANIFEST_MIGRATION_ID,+ "manifest_version": _MANIFEST_VERSION,+ "legacy_version": _LEGACY_MANIFEST_VERSION,+ "active_hub_cache": active,+ "active_hub_cache_raw": None if raw is None else str(raw),+ "dirs": dirs,+ }+++def _migration_marker_path() -> Optional[Path]:+ # Beside manifests/, not inside it: writing the marker must not move the mtime it records.+ root = state_root()+ return None if root is None else root / _MANIFEST_MIGRATION_MARKER+++def _migration_already_clean(parent: Path) -> bool:+ marker = _migration_marker_path()+ if marker is None:+ return False+ try:+ with marker.open("rb") as handle:+ raw = handle.read(64 * 1024 + 1)+ if len(raw) > 64 * 1024:+ return False+ recorded = json.loads(raw)+ if not isinstance(recorded, dict) or not isinstance(recorded.get("dirs"), dict):+ return False+ scopes = [name for name in recorded["dirs"] if name != "."]+ # RecursionError: deeply nested corrupt marker.+ except (OSError, ValueError, RecursionError):+ return False+ current = _migration_fingerprint(parent, scopes)+ return current is not None and current == recorded+++def _record_clean_migration(parent: Path, before: Optional[dict], started_ns: int) -> None:+ if before is None:+ return+ after = _migration_fingerprint(parent)+ if after != before:+ return+ if any(mtime > started_ns - _MANIFEST_MIGRATION_SETTLE_NS for mtime in after["dirs"].values()):+ return+ marker = _migration_marker_path()+ if marker is not None:+ _atomic_write_json(marker, after)def migrate_ordinary_v2_manifests_for_downgrade() -> int:+ """Rewrite ordinary v2 manifests as v1 for older Studios. A clean pass is recorded and later starts+ skip the scan until a manifest dir, the active cache or the migration id changes."""parent = manifests_dir()if parent is None:return 0+ # Optimisation only: any failure here falls back to the full scan.+ try:+ if _migration_already_clean(parent):+ return 0+ started_ns = time.time_ns()+ before = _migration_fingerprint(parent)+ except Exception as exc:+ logger.debug("Could not check the Hub manifest migration marker: %s", exc)+ started_ns, before = time.time_ns(), Nonemigrated = 0+ clean = Truetry:- candidates = _migration_manifest_paths(parent)+ unlisted: list = []+ candidates = _migration_manifest_paths(parent, unlisted)for index, path in enumerate(candidates):if index >= _MANIFEST_MIGRATION_MAX_FILES:logger.debug("Stopped Hub manifest migration at the file limit")+ clean = Falsebreaktry:if path.is_symlink() or not path.is_file():@@ -679,6 +798,8 @@ def migrate_ordinary_v2_manifests_for_downgrade() -> int:continueif _is_reserved_dataset_completion_payload(data):continue+ # Any ordinary v2 record makes the pass unclean, migrated or not.+ clean = Falserepo_type = data.get("repo_type")repo_id = data.get("repo_id")@@ -731,9 +852,16 @@ def migrate_ordinary_v2_manifests_for_downgrade() -> int:if _atomic_write_json(path, migrated_payload):migrated += 1except Exception as exc:+ clean = Falselogger.debug("Could not migrate Hub manifest %s: %s", path, exc)except Exception as exc:logger.debug("Could not enumerate Hub manifests for migration: %s", exc)+ return migrated+ if clean and not unlisted:+ try:+ _record_clean_migration(parent, before, started_ns)+ except Exception as exc:+ logger.debug("Could not record the Hub manifest migration pass: %s", exc)return migrateddiff --git a/studio/backend/tests/test_manifest_migration_marker.py b/studio/backend/tests/test_manifest_migration_marker.pynew file mode 100644index 00000000000..db13c63c947--- /dev/null+++ b/studio/backend/tests/test_manifest_migration_marker.py@@ -0,0 +1,333 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The startup manifest downgrade migration records a clean pass and skips the scan on later starts."""++import json+import os+import shutil+import time+import types+from pathlib import Path++import pytest++from hub.utils import download_manifest, hf_cache_state, state_dir+++def home(monkeypatch, tmp_path):+ active = tmp_path / "active-hub"+ active.mkdir()+ monkeypatch.setattr(hf_cache_state, "hf_cache_roots", lambda **kw: [tmp_path])+ monkeypatch.setattr("utils.paths.storage_roots.cache_root", lambda: tmp_path / "app-cache")+ monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")+ settings = types.SimpleNamespace(hub_cache = active)+ monkeypatch.setattr("utils.hf_cache_settings.get_hf_cache_paths", lambda: settings)+ return types.SimpleNamespace(root = tmp_path, active = active, settings = settings)+++def _write(+ repo_id,+ version,+ *,+ hub_cache,+ repo_type = "model",+ variant = None,+ **kw,+):+ assert download_manifest.write_manifest(+ repo_type,+ repo_id,+ variant,+ [download_manifest.ExpectedFile("model.safetensors", 8)],+ "http",+ hub_cache = hub_cache,+ _schema_version = version,+ **kw,+ )+++def _populate(home):+ for i in range(6):+ _write(f"Org/V1-{i}", 1, hub_cache = home.active)+ _write("Org/Ordinary", 2, hub_cache = home.active, variant = "Q4_K_M")+ _write("Org/Unscoped", 2, hub_cache = None)+ other = home.root / "other-hub"+ other.mkdir(exist_ok = True)+ _write(+ "Org/Data",+ 2,+ hub_cache = other,+ repo_type = "dataset",+ commit_hash = "commit-a",+ metadata_derived = True,+ )+ assert download_manifest.write_dataset_completion(+ "Org/Done",+ "c" * 40,+ [download_manifest.ExpectedFile("train.parquet", 4)],+ "http",+ hub_cache = home.active,+ )+++def _settle(path: Path, age_s: float = 30.0) -> None:+ """Age every manifest directory past the racy window, as a restart a while after the last write sees it."""+ old = time.time_ns() - int(age_s * 1e9)+ for dirpath, _dirs, _files in os.walk(path):+ os.utime(dirpath, ns = (old, old))+++def _snapshot(path: Path) -> dict:+ return {+ str(p.relative_to(path)): p.read_bytes() for p in sorted(path.rglob("*")) if p.is_file()+ }+++def _count_reads(monkeypatch):+ calls = []+ real = download_manifest._read_migration_payload++ def counted(path):+ calls.append(path)+ return real(path)++ monkeypatch.setattr(download_manifest, "_read_migration_payload", counted)+ return calls+++def _marker():+ return state_dir.state_root() / download_manifest._MANIFEST_MIGRATION_MARKER+++def test_result_matches_the_unmarked_scan(home, monkeypatch, tmp_path):+ _populate(home)+ manifests = state_dir.manifests_dir()+ reference = tmp_path / "reference"+ shutil.copytree(manifests, reference, symlinks = True)++ migrated = download_manifest.migrate_ordinary_v2_manifests_for_downgrade()+ head_state = _snapshot(manifests)++ # The same pass with the marker hooks disabled is the pre-change algorithm.+ shutil.rmtree(manifests)+ shutil.copytree(reference, manifests, symlinks = True)+ with monkeypatch.context() as m:+ m.setattr(download_manifest, "_migration_already_clean", lambda parent: False)+ m.setattr(download_manifest, "_record_clean_migration", lambda *a: None)+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == migrated+ assert _snapshot(manifests) == head_state+ assert migrated >= 3+ for path in manifests.rglob("*.json"):+ payload = json.loads(path.read_text(encoding = "utf-8"))+ if not str(payload.get("variant") or "").startswith("_studio-dataset-complete-"):+ assert payload["version"] == 1, path+++def test_fresh_home_without_manifests_dir(home):+ assert state_dir.manifests_dir() is not None+ assert not state_dir.manifests_dir().exists()+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0+ assert not _marker().exists()+++def test_second_start_skips_the_scan(home, monkeypatch):+ _populate(home)+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 3+ manifests = state_dir.manifests_dir()+ _settle(manifests)+ before = _snapshot(manifests)++ calls = _count_reads(monkeypatch)+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0+ assert len(calls) >= 8++ calls.clear()+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0+ assert calls == []+ assert _snapshot(manifests) == before+ assert _marker().is_file()+++def _clean_marked(home, monkeypatch):+ _populate(home)+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()+ _settle(state_dir.manifests_dir())+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()+ assert _marker().is_file()+ return _count_reads(monkeypatch)+++def test_new_ordinary_v2_manifest_reruns_the_migration(home, monkeypatch):+ calls = _clean_marked(home, monkeypatch)+ _write("Org/Later", 2, hub_cache = home.active)+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 1+ assert calls+ path = download_manifest.manifest_path("model", "Org/Later", None, hub_cache = home.active)+ assert json.loads(path.read_text(encoding = "utf-8"))["version"] == 1+++def test_replaced_manifest_in_scope_dir_reruns(home, monkeypatch):+ calls = _clean_marked(home, monkeypatch)+ # Overwrite an existing name: the set of names is unchanged, only the scope dir mtime moves.+ _write("Org/V1-0", 2, hub_cache = home.active)+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 1+ assert calls+++def test_new_cache_scope_reruns(home, monkeypatch):+ calls = _clean_marked(home, monkeypatch)+ fresh = home.root / "fresh-hub"+ fresh.mkdir()+ _write("Org/Fresh", 2, hub_cache = fresh)+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 1+ assert calls+++def test_migration_id_change_reruns(home, monkeypatch):+ calls = _clean_marked(home, monkeypatch)+ monkeypatch.setattr(download_manifest, "_MANIFEST_MIGRATION_ID", "ordinary-v2-to-v1:2")+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0+ assert calls+++def test_manifest_version_change_reruns(home, monkeypatch):+ calls = _clean_marked(home, monkeypatch)+ monkeypatch.setattr(download_manifest, "_MANIFEST_VERSION", 3)+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()+ assert calls+++def test_active_cache_change_reruns(home, monkeypatch):+ calls = _clean_marked(home, monkeypatch)+ moved = home.root / "moved-hub"+ moved.mkdir()+ home.settings.hub_cache = moved+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()+ assert calls+++def test_corrupt_marker_reruns(home, monkeypatch):+ calls = _clean_marked(home, monkeypatch)+ _marker().write_text("{not json", encoding = "utf-8")+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0+ assert calls+++def test_recently_changed_dirs_are_not_recorded(home):+ _populate(home)+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()+ # No settling: the dirs were written within the racy window.+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0+ assert not _marker().exists()+++def test_pass_that_migrates_is_not_recorded(home):+ _populate(home)+ _settle(state_dir.manifests_dir())+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 3+ assert not _marker().exists()+++def test_failed_write_is_not_recorded_and_retries(home, monkeypatch):+ _populate(home)+ _settle(state_dir.manifests_dir())+ with monkeypatch.context() as m:+ m.setattr(download_manifest, "_atomic_write_json", lambda *a, **k: False)+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0+ assert not _marker().exists()+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 3+++def test_unreadable_manifest_is_not_recorded_and_retries(home, monkeypatch):+ _populate(home)+ _settle(state_dir.manifests_dir())+ real_open = Path.open++ def locked(self, *a, **k):+ if self.name.endswith(".json") and self.parent.name != "hub-state":+ raise PermissionError(13, "sharing violation", str(self))+ return real_open(self, *a, **k)++ with monkeypatch.context() as m:+ m.setattr(Path, "open", locked)+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0+ assert not _marker().exists()+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 3+++def test_unlistable_scope_is_not_recorded_and_retries(home, monkeypatch):+ _populate(home)+ _settle(state_dir.manifests_dir())+ real = download_manifest._listed++ def locked(directory):+ if directory.name.startswith("cache-"):+ raise PermissionError(13, "access denied", str(directory))+ return real(directory)++ with monkeypatch.context() as m:+ m.setattr(download_manifest, "_listed", locked)+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()+ assert not _marker().exists()+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 2++[email protected](os.name == "nt" or os.geteuid() == 0, reason = "POSIX permission bits")+def test_unreadable_scope_dir_is_not_recorded(home):+ _populate(home)+ manifests = state_dir.manifests_dir()+ _settle(manifests)+ scopes = [p for p in manifests.iterdir() if p.name.startswith("cache-")]+ for scope in scopes:+ scope.chmod(0)+ try:+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()+ finally:+ for scope in scopes:+ scope.chmod(0o700)+ assert not _marker().exists()+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 1+++def test_file_cap_is_not_recorded(home, monkeypatch):+ _populate(home)+ _settle(state_dir.manifests_dir())+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()+ _settle(state_dir.manifests_dir())+ monkeypatch.setattr(download_manifest, "_MANIFEST_MIGRATION_MAX_FILES", 2)+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()+ assert not _marker().exists()+++def test_marker_lives_outside_the_manifest_dirs(home):+ _populate(home)+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()+ _settle(state_dir.manifests_dir())+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()+ manifests = state_dir.manifests_dir()+ assert _marker().parent != manifests+ assert manifests not in _marker().parents+++def test_deeply_nested_marker_still_migrates(home):+ # json.loads raises RecursionError (not ValueError) on this; it must not abort the migration.+ _populate(home)+ _settle(state_dir.manifests_dir())+ _marker().parent.mkdir(parents = True, exist_ok = True)+ _marker().write_text("[" * 60_000, encoding = "utf-8")+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 3+++def test_marker_check_error_falls_back_to_the_full_scan(home, monkeypatch):+ _populate(home)+ _settle(state_dir.manifests_dir())++ def boom(*a, **k):+ raise RuntimeError("unexpected")++ monkeypatch.setattr(download_manifest, "_migration_already_clean", boom)+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 3+ assert not _marker().exists()