Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: skip the startup manifest migration scan when nothing changed since a clean pass (#16270)

Daniel Hanf6f550d4bd0f
diff --git a/studio/backend/hub/utils/download_manifest.py b/studio/backend/hub/utils/download_manifest.py
index 41f0ccc094d..983651c830b 100644
--- a/studio/backend/hub/utils/download_manifest.py
+++ b/studio/backend/hub/utils/download_manifest.py
@@ -9,6 +9,8 @@ import hashlib
import json
import os
import re
+import stat
+import time
import unicodedata
import uuid
from dataclasses import dataclass
@@ -28,6 +30,7 @@ from hub.utils.state_dir import (
marker_path,
normalize_hub_cache,
state_filename_is_ambiguous,
+ state_root,
variant_filename_prefix,
variant_key_fragments,
)
@@ -47,6 +50,11 @@ _MANIFEST_MIGRATION_MAX_FILES = 10_000
_MANIFEST_MIGRATION_MAX_BYTES = 32 * 1024 * 1024
_MANIFEST_MIGRATION_PREFIX_BYTES = 256
_V2_MANIFEST_PREFIX = re.compile(rb'^\s*\{\s*"version"\s*:\s*2\s*,')
+# Bump when the migration's selection rules change, so a recorded clean pass no longer applies.
+_MANIFEST_MIGRATION_ID = "ordinary-v2-to-v1:1"
+_MANIFEST_MIGRATION_MARKER = "manifest_migration.json"
+# A dir changed this recently may sit inside the mtime granularity: not recorded (git's "racy" index rule).
+_MANIFEST_MIGRATION_SETTLE_NS = 2_000_000_000
# Verbatim phrase the worker emits on a degraded completion; shared so emit and match stay coupled.
MANIFEST_DEGRADED_MARKER = "completed without a manifest so partial detection is degraded"
@@ -635,6 +643,7 @@ def _is_reserved_dataset_completion_payload(data: dict) -> bool:
def _read_migration_payload(path: Path) -> Optional[dict]:
+ # OSError propagates: an unreadable file is skipped by the caller and keeps the pass unclean.
try:
with path.open("rb") as handle:
prefix = handle.read(_MANIFEST_MIGRATION_PREFIX_BYTES)
@@ -646,30 +655,140 @@ def _read_migration_payload(path: Path) -> Optional[dict]:
if len(raw) > _MANIFEST_MIGRATION_MAX_BYTES:
return None
data = json.loads(raw)
- except (OSError, ValueError):
+ except ValueError:
return None
return data if isinstance(data, dict) else None
-def _migration_manifest_paths(parent: Path) -> Iterator[Path]:
- yield from parent.glob("*.json")
- for scoped in parent.glob("cache-*"):
+def _listed(directory: Path) -> list[tuple[str, str]]:
+ # Eager, and folded on Windows, as pathlib's glob is.
+ with os.scandir(directory) as entries:
+ return [
+ (entry.name, entry.name.casefold() if os.name == "nt" else entry.name)
+ for entry in entries
+ ]
+
+
+def _migration_manifest_paths(parent: Path, failed: list) -> Iterator[Path]:
+ """Like globbing ``*.json`` and ``cache-*/*.json``, but a scope that cannot be listed is appended to
+ ``failed`` instead of silently yielding nothing (``Path.glob`` swallows ``OSError``)."""
+ try:
+ top = _listed(parent)
+ except FileNotFoundError:
+ return
+ for name, folded in top:
+ if folded.endswith(".json"):
+ yield parent / name
+ for name, folded in top:
+ if not folded.startswith("cache-"):
+ continue
+ scoped = parent / name
if scoped.is_symlink() or not scoped.is_dir():
continue
- yield from scoped.glob("*.json")
+ try:
+ listed = _listed(scoped)
+ except OSError as exc:
+ failed.append(scoped)
+ logger.debug("Could not list Hub manifest scope %s: %s", scoped, exc)
+ continue
+ for child, child_folded in listed:
+ if child_folded.endswith(".json"):
+ yield scoped / child
+
+
+def _migration_fingerprint(parent: Path, scopes: Optional[Iterable[str]] = None) -> Optional[dict]:
+ """Dir mtimes + active cache. Manifest writes are ``tmp + os.replace``, so any write moves its dir's
+ mtime and a new ``cache-*`` scope moves the parent's. ``None`` when anything cannot be read."""
+ try:
+ active, raw = _hub_cache_spellings(None)
+ dirs = {".": os.stat(parent).st_mtime_ns}
+ if scopes is None:
+ scopes = [
+ entry.name
+ for entry in os.scandir(parent)
+ if entry.name.startswith("cache-") and entry.is_dir(follow_symlinks = False)
+ ]
+ for name in scopes:
+ st = os.lstat(parent / name)
+ if not stat.S_ISDIR(st.st_mode):
+ return None
+ dirs[name] = st.st_mtime_ns
+ except (OSError, TypeError, ValueError):
+ return None
+ return {
+ "migration": _MANIFEST_MIGRATION_ID,
+ "manifest_version": _MANIFEST_VERSION,
+ "legacy_version": _LEGACY_MANIFEST_VERSION,
+ "active_hub_cache": active,
+ "active_hub_cache_raw": None if raw is None else str(raw),
+ "dirs": dirs,
+ }
+
+
+def _migration_marker_path() -> Optional[Path]:
+ # Beside manifests/, not inside it: writing the marker must not move the mtime it records.
+ root = state_root()
+ return None if root is None else root / _MANIFEST_MIGRATION_MARKER
+
+
+def _migration_already_clean(parent: Path) -> bool:
+ marker = _migration_marker_path()
+ if marker is None:
+ return False
+ try:
+ with marker.open("rb") as handle:
+ raw = handle.read(64 * 1024 + 1)
+ if len(raw) > 64 * 1024:
+ return False
+ recorded = json.loads(raw)
+ if not isinstance(recorded, dict) or not isinstance(recorded.get("dirs"), dict):
+ return False
+ scopes = [name for name in recorded["dirs"] if name != "."]
+ # RecursionError: deeply nested corrupt marker.
+ except (OSError, ValueError, RecursionError):
+ return False
+ current = _migration_fingerprint(parent, scopes)
+ return current is not None and current == recorded
+
+
+def _record_clean_migration(parent: Path, before: Optional[dict], started_ns: int) -> None:
+ if before is None:
+ return
+ after = _migration_fingerprint(parent)
+ if after != before:
+ return
+ if any(mtime > started_ns - _MANIFEST_MIGRATION_SETTLE_NS for mtime in after["dirs"].values()):
+ return
+ marker = _migration_marker_path()
+ if marker is not None:
+ _atomic_write_json(marker, after)
def migrate_ordinary_v2_manifests_for_downgrade() -> int:
+ """Rewrite ordinary v2 manifests as v1 for older Studios. A clean pass is recorded and later starts
+ skip the scan until a manifest dir, the active cache or the migration id changes."""
parent = manifests_dir()
if parent is None:
return 0
+ # Optimisation only: any failure here falls back to the full scan.
+ try:
+ if _migration_already_clean(parent):
+ return 0
+ started_ns = time.time_ns()
+ before = _migration_fingerprint(parent)
+ except Exception as exc:
+ logger.debug("Could not check the Hub manifest migration marker: %s", exc)
+ started_ns, before = time.time_ns(), None
migrated = 0
+ clean = True
try:
- candidates = _migration_manifest_paths(parent)
+ unlisted: list = []
+ candidates = _migration_manifest_paths(parent, unlisted)
for index, path in enumerate(candidates):
if index >= _MANIFEST_MIGRATION_MAX_FILES:
logger.debug("Stopped Hub manifest migration at the file limit")
+ clean = False
break
try:
if path.is_symlink() or not path.is_file():
@@ -679,6 +798,8 @@ def migrate_ordinary_v2_manifests_for_downgrade() -> int:
continue
if _is_reserved_dataset_completion_payload(data):
continue
+ # Any ordinary v2 record makes the pass unclean, migrated or not.
+ clean = False
repo_type = data.get("repo_type")
repo_id = data.get("repo_id")
@@ -731,9 +852,16 @@ def migrate_ordinary_v2_manifests_for_downgrade() -> int:
if _atomic_write_json(path, migrated_payload):
migrated += 1
except Exception as exc:
+ clean = False
logger.debug("Could not migrate Hub manifest %s: %s", path, exc)
except Exception as exc:
logger.debug("Could not enumerate Hub manifests for migration: %s", exc)
+ return migrated
+ if clean and not unlisted:
+ try:
+ _record_clean_migration(parent, before, started_ns)
+ except Exception as exc:
+ logger.debug("Could not record the Hub manifest migration pass: %s", exc)
return migrated
diff --git a/studio/backend/tests/test_manifest_migration_marker.py b/studio/backend/tests/test_manifest_migration_marker.py
new file mode 100644
index 00000000000..db13c63c947
--- /dev/null
+++ b/studio/backend/tests/test_manifest_migration_marker.py
@@ -0,0 +1,333 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The startup manifest downgrade migration records a clean pass and skips the scan on later starts."""
+
+import json
+import os
+import shutil
+import time
+import types
+from pathlib import Path
+
+import pytest
+
+from hub.utils import download_manifest, hf_cache_state, state_dir
+
+
+def home(monkeypatch, tmp_path):
+ active = tmp_path / "active-hub"
+ active.mkdir()
+ monkeypatch.setattr(hf_cache_state, "hf_cache_roots", lambda **kw: [tmp_path])
+ monkeypatch.setattr("utils.paths.storage_roots.cache_root", lambda: tmp_path / "app-cache")
+ monkeypatch.setattr(state_dir, "cache_root", lambda: tmp_path / "state")
+ settings = types.SimpleNamespace(hub_cache = active)
+ monkeypatch.setattr("utils.hf_cache_settings.get_hf_cache_paths", lambda: settings)
+ return types.SimpleNamespace(root = tmp_path, active = active, settings = settings)
+
+
+def _write(
+ repo_id,
+ version,
+ *,
+ hub_cache,
+ repo_type = "model",
+ variant = None,
+ **kw,
+):
+ assert download_manifest.write_manifest(
+ repo_type,
+ repo_id,
+ variant,
+ [download_manifest.ExpectedFile("model.safetensors", 8)],
+ "http",
+ hub_cache = hub_cache,
+ _schema_version = version,
+ **kw,
+ )
+
+
+def _populate(home):
+ for i in range(6):
+ _write(f"Org/V1-{i}", 1, hub_cache = home.active)
+ _write("Org/Ordinary", 2, hub_cache = home.active, variant = "Q4_K_M")
+ _write("Org/Unscoped", 2, hub_cache = None)
+ other = home.root / "other-hub"
+ other.mkdir(exist_ok = True)
+ _write(
+ "Org/Data",
+ 2,
+ hub_cache = other,
+ repo_type = "dataset",
+ commit_hash = "commit-a",
+ metadata_derived = True,
+ )
+ assert download_manifest.write_dataset_completion(
+ "Org/Done",
+ "c" * 40,
+ [download_manifest.ExpectedFile("train.parquet", 4)],
+ "http",
+ hub_cache = home.active,
+ )
+
+
+def _settle(path: Path, age_s: float = 30.0) -> None:
+ """Age every manifest directory past the racy window, as a restart a while after the last write sees it."""
+ old = time.time_ns() - int(age_s * 1e9)
+ for dirpath, _dirs, _files in os.walk(path):
+ os.utime(dirpath, ns = (old, old))
+
+
+def _snapshot(path: Path) -> dict:
+ return {
+ str(p.relative_to(path)): p.read_bytes() for p in sorted(path.rglob("*")) if p.is_file()
+ }
+
+
+def _count_reads(monkeypatch):
+ calls = []
+ real = download_manifest._read_migration_payload
+
+ def counted(path):
+ calls.append(path)
+ return real(path)
+
+ monkeypatch.setattr(download_manifest, "_read_migration_payload", counted)
+ return calls
+
+
+def _marker():
+ return state_dir.state_root() / download_manifest._MANIFEST_MIGRATION_MARKER
+
+
+def test_result_matches_the_unmarked_scan(home, monkeypatch, tmp_path):
+ _populate(home)
+ manifests = state_dir.manifests_dir()
+ reference = tmp_path / "reference"
+ shutil.copytree(manifests, reference, symlinks = True)
+
+ migrated = download_manifest.migrate_ordinary_v2_manifests_for_downgrade()
+ head_state = _snapshot(manifests)
+
+ # The same pass with the marker hooks disabled is the pre-change algorithm.
+ shutil.rmtree(manifests)
+ shutil.copytree(reference, manifests, symlinks = True)
+ with monkeypatch.context() as m:
+ m.setattr(download_manifest, "_migration_already_clean", lambda parent: False)
+ m.setattr(download_manifest, "_record_clean_migration", lambda *a: None)
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == migrated
+ assert _snapshot(manifests) == head_state
+ assert migrated >= 3
+ for path in manifests.rglob("*.json"):
+ payload = json.loads(path.read_text(encoding = "utf-8"))
+ if not str(payload.get("variant") or "").startswith("_studio-dataset-complete-"):
+ assert payload["version"] == 1, path
+
+
+def test_fresh_home_without_manifests_dir(home):
+ assert state_dir.manifests_dir() is not None
+ assert not state_dir.manifests_dir().exists()
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0
+ assert not _marker().exists()
+
+
+def test_second_start_skips_the_scan(home, monkeypatch):
+ _populate(home)
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 3
+ manifests = state_dir.manifests_dir()
+ _settle(manifests)
+ before = _snapshot(manifests)
+
+ calls = _count_reads(monkeypatch)
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0
+ assert len(calls) >= 8
+
+ calls.clear()
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0
+ assert calls == []
+ assert _snapshot(manifests) == before
+ assert _marker().is_file()
+
+
+def _clean_marked(home, monkeypatch):
+ _populate(home)
+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()
+ _settle(state_dir.manifests_dir())
+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()
+ assert _marker().is_file()
+ return _count_reads(monkeypatch)
+
+
+def test_new_ordinary_v2_manifest_reruns_the_migration(home, monkeypatch):
+ calls = _clean_marked(home, monkeypatch)
+ _write("Org/Later", 2, hub_cache = home.active)
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 1
+ assert calls
+ path = download_manifest.manifest_path("model", "Org/Later", None, hub_cache = home.active)
+ assert json.loads(path.read_text(encoding = "utf-8"))["version"] == 1
+
+
+def test_replaced_manifest_in_scope_dir_reruns(home, monkeypatch):
+ calls = _clean_marked(home, monkeypatch)
+ # Overwrite an existing name: the set of names is unchanged, only the scope dir mtime moves.
+ _write("Org/V1-0", 2, hub_cache = home.active)
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 1
+ assert calls
+
+
+def test_new_cache_scope_reruns(home, monkeypatch):
+ calls = _clean_marked(home, monkeypatch)
+ fresh = home.root / "fresh-hub"
+ fresh.mkdir()
+ _write("Org/Fresh", 2, hub_cache = fresh)
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 1
+ assert calls
+
+
+def test_migration_id_change_reruns(home, monkeypatch):
+ calls = _clean_marked(home, monkeypatch)
+ monkeypatch.setattr(download_manifest, "_MANIFEST_MIGRATION_ID", "ordinary-v2-to-v1:2")
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0
+ assert calls
+
+
+def test_manifest_version_change_reruns(home, monkeypatch):
+ calls = _clean_marked(home, monkeypatch)
+ monkeypatch.setattr(download_manifest, "_MANIFEST_VERSION", 3)
+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()
+ assert calls
+
+
+def test_active_cache_change_reruns(home, monkeypatch):
+ calls = _clean_marked(home, monkeypatch)
+ moved = home.root / "moved-hub"
+ moved.mkdir()
+ home.settings.hub_cache = moved
+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()
+ assert calls
+
+
+def test_corrupt_marker_reruns(home, monkeypatch):
+ calls = _clean_marked(home, monkeypatch)
+ _marker().write_text("{not json", encoding = "utf-8")
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0
+ assert calls
+
+
+def test_recently_changed_dirs_are_not_recorded(home):
+ _populate(home)
+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()
+ # No settling: the dirs were written within the racy window.
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0
+ assert not _marker().exists()
+
+
+def test_pass_that_migrates_is_not_recorded(home):
+ _populate(home)
+ _settle(state_dir.manifests_dir())
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 3
+ assert not _marker().exists()
+
+
+def test_failed_write_is_not_recorded_and_retries(home, monkeypatch):
+ _populate(home)
+ _settle(state_dir.manifests_dir())
+ with monkeypatch.context() as m:
+ m.setattr(download_manifest, "_atomic_write_json", lambda *a, **k: False)
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0
+ assert not _marker().exists()
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 3
+
+
+def test_unreadable_manifest_is_not_recorded_and_retries(home, monkeypatch):
+ _populate(home)
+ _settle(state_dir.manifests_dir())
+ real_open = Path.open
+
+ def locked(self, *a, **k):
+ if self.name.endswith(".json") and self.parent.name != "hub-state":
+ raise PermissionError(13, "sharing violation", str(self))
+ return real_open(self, *a, **k)
+
+ with monkeypatch.context() as m:
+ m.setattr(Path, "open", locked)
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() == 0
+ assert not _marker().exists()
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 3
+
+
+def test_unlistable_scope_is_not_recorded_and_retries(home, monkeypatch):
+ _populate(home)
+ _settle(state_dir.manifests_dir())
+ real = download_manifest._listed
+
+ def locked(directory):
+ if directory.name.startswith("cache-"):
+ raise PermissionError(13, "access denied", str(directory))
+ return real(directory)
+
+ with monkeypatch.context() as m:
+ m.setattr(download_manifest, "_listed", locked)
+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()
+ assert not _marker().exists()
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 2
+
+
[email protected](os.name == "nt" or os.geteuid() == 0, reason = "POSIX permission bits")
+def test_unreadable_scope_dir_is_not_recorded(home):
+ _populate(home)
+ manifests = state_dir.manifests_dir()
+ _settle(manifests)
+ scopes = [p for p in manifests.iterdir() if p.name.startswith("cache-")]
+ for scope in scopes:
+ scope.chmod(0)
+ try:
+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()
+ finally:
+ for scope in scopes:
+ scope.chmod(0o700)
+ assert not _marker().exists()
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 1
+
+
+def test_file_cap_is_not_recorded(home, monkeypatch):
+ _populate(home)
+ _settle(state_dir.manifests_dir())
+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()
+ _settle(state_dir.manifests_dir())
+ monkeypatch.setattr(download_manifest, "_MANIFEST_MIGRATION_MAX_FILES", 2)
+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()
+ assert not _marker().exists()
+
+
+def test_marker_lives_outside_the_manifest_dirs(home):
+ _populate(home)
+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()
+ _settle(state_dir.manifests_dir())
+ download_manifest.migrate_ordinary_v2_manifests_for_downgrade()
+ manifests = state_dir.manifests_dir()
+ assert _marker().parent != manifests
+ assert manifests not in _marker().parents
+
+
+def test_deeply_nested_marker_still_migrates(home):
+ # json.loads raises RecursionError (not ValueError) on this; it must not abort the migration.
+ _populate(home)
+ _settle(state_dir.manifests_dir())
+ _marker().parent.mkdir(parents = True, exist_ok = True)
+ _marker().write_text("[" * 60_000, encoding = "utf-8")
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 3
+
+
+def test_marker_check_error_falls_back_to_the_full_scan(home, monkeypatch):
+ _populate(home)
+ _settle(state_dir.manifests_dir())
+
+ def boom(*a, **k):
+ raise RuntimeError("unexpected")
+
+ monkeypatch.setattr(download_manifest, "_migration_already_clean", boom)
+ assert download_manifest.migrate_ordinary_v2_manifests_for_downgrade() >= 3
+ assert not _marker().exists()