Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
preserve local model inventory after cache filesystem errors
Lee Jacksone15867eb40b3
diff --git a/studio/backend/hub/utils/inventory_scan.py b/studio/backend/hub/utils/inventory_scan.pyindex 750c39dc714..b32ee7da6b4 100644--- a/studio/backend/hub/utils/inventory_scan.py+++ b/studio/backend/hub/utils/inventory_scan.py@@ -12,7 +12,7 @@ import reimport statimport threadingimport time-from dataclasses import dataclass, replace+from dataclasses import dataclass, fields, replacefrom pathlib import Path, PurePosixPath, PureWindowsPathfrom typing import Awaitable, Callable, Hashable, NamedTuple, Optional, TypeVar@@ -239,8 +239,10 @@ def _read_refs_by_commit(refs_dir: Path) -> Optional[dict[str, set[str]]]:return refs_by_commit-def _recover_repo_dropped_by_scan(repo_dir: Path) -> Optional[_RecoveredRepoInfo]:- """Recover readable revisions from a repo omitted by ``scan_cache_dir``. Dangling refs, broken snapshot links and stray snapshot files can hide an otherwise usable repo, so skip those bad entries and rebuild a read-only row from the intact files. None when nothing remains or the on-disk state does not explain the omission."""+def _recover_repo_dropped_by_scan(+ repo_dir: Path, *, scan_failed: bool = False+) -> Optional[_RecoveredRepoInfo]:+ """Recover readable revisions omitted by ``scan_cache_dir``, skipping bad entries. When the entire scan failed, include intact repos because upstream returned no inventory."""identity = _hf_repo_identity(repo_dir.name)if identity is None:return None@@ -314,7 +316,7 @@ def _recover_repo_dropped_by_scan(repo_dir: Path) -> Optional[_RecoveredRepoInfoif not revisions:return None# Nothing here explains why upstream omitted the repo.- if not dangling and not skipped:+ if not scan_failed and not dangling and not skipped:return Nonetry:repo_stats = repo_dir.stat()@@ -336,7 +338,12 @@ def _recover_repo_dropped_by_scan(repo_dir: Path) -> Optional[_RecoveredRepoInfo)-def _with_repos_dropped_by_scan(scan, cache_root: Path):+def _with_repos_dropped_by_scan(+ scan,+ cache_root: Path,+ *,+ scan_failed: bool = False,+):"""Add back the repos ``scan_cache_dir`` dropped over one bad entry."""try:repo_dirs = sorted(entry for entry in cache_root.iterdir() if "--" in entry.name)@@ -354,7 +361,7 @@ def _with_repos_dropped_by_scan(scan, cache_root: Path):try:if str(repo_dir.resolve(strict = False)) in scanned:continue- entry = _recover_repo_dropped_by_scan(repo_dir)+ entry = _recover_repo_dropped_by_scan(repo_dir, scan_failed = scan_failed)except (OSError, RuntimeError, ValueError):continueif entry is None:@@ -362,7 +369,11 @@ def _with_repos_dropped_by_scan(scan, cache_root: Path):logger.info("Recovered HF cache repo %s hidden by %s (%d revision(s) on disk)",entry.repo_id,- "a dangling ref" if _repo_has_a_dangling_ref(repo_dir) else "an unreadable entry",+ "a failed cache scan"+ if scan_failed+ else (+ "a dangling ref" if _repo_has_a_dangling_ref(repo_dir) else "an unreadable entry"+ ),len(entry.revisions),)recovered.append(entry)@@ -382,7 +393,7 @@ def _with_repos_dropped_by_scan(scan, cache_root: Path):def _compute_all_hf_cache_scans() -> list:- from huggingface_hub import scan_cache_dir+ from huggingface_hub import HFCacheInfo, scan_cache_dirscans: list = []for cache_root in hf_cache_roots():@@ -392,6 +403,22 @@ def _compute_all_hf_cache_scans() -> list:if getattr(scan, "warnings", None):scan = _with_repos_dropped_by_scan(scan, cache_root)scans.append(scan)+ except OSError as exc:+ # HF catches CorruptedCacheException per repo, but filesystem errors can abort+ # the entire root (e.g. Windows cannot stat a Linux-created reparse point).+ # Reuse our read-only recovery walk, including intact repos that HF never reached.+ logger.warning("Could not scan HF cache %s: %s", cache_root, exc)+ empty_fields = dict(size_on_disk = 0, repos = frozenset(), warnings = [])+ # huggingface_hub 1.x added this required field; older versions lack it.+ if any(field.name == "incomplete_files" for field in fields(HFCacheInfo)):+ empty_fields["incomplete_files"] = frozenset()+ recovered = _with_repos_dropped_by_scan(+ HFCacheInfo(**empty_fields),+ cache_root,+ scan_failed = True,+ )+ if recovered.repos:+ scans.append(recovered)except Exception as exc:logger.warning("Could not scan HF cache %s: %s", cache_root, exc)return scansdiff --git a/studio/backend/tests/test_hf_cache_dangling_refs.py b/studio/backend/tests/test_hf_cache_dangling_refs.pyindex 2bd75fe7197..4266c56fa15 100644--- a/studio/backend/tests/test_hf_cache_dangling_refs.py+++ b/studio/backend/tests/test_hf_cache_dangling_refs.py@@ -4731,3 +4731,34 @@ def test_a_whole_split_quant_is_still_complete(tmp_path):(snapshot / f"Model-Q4_K_M-0000{index}-of-00002.gguf").write_bytes(b"GGUF" + b"\0" * 252)assert inventory_scan._completed_gguf_variants(snapshot) == {"Q4_K_M"}++[email protected]("bad_repo_name", ["datasets--Org--Broken", "models--Org--Broken"])+def test_oserror_in_one_repo_does_not_hide_healthy_models(tmp_path, monkeypatch, bad_repo_name):+ from huggingface_hub import scan_cache_dir++ healthy = _build_repo(tmp_path, ref = SNAPSHOT)+ broken = _build_repo(tmp_path, ref = SNAPSHOT, name = bad_repo_name)+ bad_file = broken / "snapshots" / SNAPSHOT / "README.md"+ bad_file.write_text("unreadable", encoding = "utf-8")+ original_stat = Path.stat++ def stat_with_windows_error(path, *args, **kwargs):+ if path == bad_file:+ raise OSError(1920, "The file cannot be accessed by the system", str(path))+ return original_stat(path, *args, **kwargs)++ monkeypatch.setattr(Path, "stat", stat_with_windows_error)+ with pytest.raises(OSError):+ scan_cache_dir(tmp_path)++ scans = _scan(tmp_path, monkeypatch)+ repos = {repo.repo_id: repo for scan in scans for repo in scan.repos}+ assert "Org/Model" in repos+ assert repos["Org/Model"].size_on_disk == 11+ assert repos["Org/Model"].repo_path == healthy+ assert (healthy / "refs" / "main").read_text() == SNAPSHOT+ assert bad_file.read_text() == "unreadable"+ assert all(+ f.file_path != bad_file for r in repos.values() for rev in r.revisions for f in rev.files+ )