Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: opt-in recursive scanning for custom model folders (#6989)
Aditya Aggarwal201c28f59b07
diff --git a/studio/backend/core/inference/local_model_resolver.py b/studio/backend/core/inference/local_model_resolver.pyindex a11b8a9ea3b..5c91ad6cd85 100644--- a/studio/backend/core/inference/local_model_resolver.py+++ b/studio/backend/core/inference/local_model_resolver.py@@ -888,12 +888,16 @@ def _build_index() -> dict[str, _LocalGgufEntry]:for folder in list_scan_folders():try:fp = Path(folder["path"])- custom_found += dedupe_custom_gguf_rows(+ rows = (_scan_models_dir(fp, limit = 200)+ [row for hub in scan_folder_hf_caches(fp) for row in _scan_hf_once(hub)]+ _scan_lmstudio_dir(fp)+ _scan_ollama_dir(fp, limit = 200, materialize_links = False))+ if folder.get("recursive"):+ from routes.models import _scan_nested_compat_rows+ rows += _scan_nested_compat_rows(fp, rows, limit = 200)+ custom_found += dedupe_custom_gguf_rows(rows)except Exception as exc:logger.debug("auto-switch: scan folder %r failed: %s", folder, exc)found += suppress_grouped_gguf_file_rows(custom_found)diff --git a/studio/backend/hub/routes/inventory.py b/studio/backend/hub/routes/inventory.pyindex a6815519adb..bf0893ec0b1 100644--- a/studio/backend/hub/routes/inventory.py+++ b/studio/backend/hub/routes/inventory.py@@ -108,7 +108,7 @@ def add_scan_folder_endpoint(# a relative one anchored) cannot answer for the host. Inside the try, since an exception# raised while evaluating an argument never reaches the function it was passed to.try:- payload = local_inventory.add_scan_folder_response(body.path)+ payload = local_inventory.add_scan_folder_response(body.path, body.recursive)except HTTPException as error:raise HTTPException(status_code = error.status_code,diff --git a/studio/backend/hub/schemas/inventory.py b/studio/backend/hub/schemas/inventory.pyindex 686f88eb4f1..1518dec0bc0 100644--- a/studio/backend/hub/schemas/inventory.py+++ b/studio/backend/hub/schemas/inventory.py@@ -376,6 +376,10 @@ class AddScanFolderRequest(BaseModel):...,description = "Absolute or relative folder path, or a model weight file path",)+ recursive: Optional[bool] = Field(+ None,+ description = "Also scan sub-folders. Omitted keeps the stored setting of an already registered folder.",+ )class ScanFolderInfo(BaseModel):@@ -384,6 +388,7 @@ class ScanFolderInfo(BaseModel):id: int = Field(..., description = "Database row ID")path: str = Field(..., description = "Normalized absolute path")created_at: str = Field(..., description = "ISO 8601 creation timestamp")+ recursive: bool = Field(False, description = "Sub-folders are scanned too")status: str = Field(default = "ok",description = "Last scan result: ok, permission_denied, missing, or unreadable",diff --git a/studio/backend/hub/services/models/local_inventory.py b/studio/backend/hub/services/models/local_inventory.pyindex 27d45ac1faa..2b90522bb36 100644--- a/studio/backend/hub/services/models/local_inventory.py+++ b/studio/backend/hub/services/models/local_inventory.py@@ -159,6 +159,71 @@ def _is_model_directory_for_scan(path: Path, *, entry_limit: int | None) -> boolreturn has_config and _has_immediate_model_weight(path)+_MAX_NESTED_SCAN_DEPTH = 8+_MAX_NESTED_SCAN_DIRS = 500+_MAX_NESTED_SCAN_ENTRIES = 20000+# blobs: an Ollama store's content-addressed files, never a model folder.+_NESTED_SCAN_SKIP_NAMES = frozenset({"ollama_links", "node_modules", "__pycache__", "blobs"})+++def _is_plain_dir_entry(entry: os.DirEntry) -> bool:+ if not entry.is_dir(follow_symlinks = False):+ return False+ # A Windows junction is a directory to is_dir(follow_symlinks=False) before 3.12; other reparse+ # points (OneDrive placeholders) are real folders and stay walkable.+ tag = getattr(entry.stat(follow_symlinks = False), "st_reparse_tag", 0)+ return tag != _IO_REPARSE_TAG_MOUNT_POINT+++def is_loadable_model_dir(path: Path) -> bool:+ """A folder the scan lists as a loadable model. A config with no weights beside it is not one: its model may sit a level down."""+ return _is_diffusers_pipeline_dir(path) or _has_immediate_model_weight(path)+++def nested_scan_roots(folder_path: Path) -> list[Path]:+ """Sub-folders of a recursive scan folder to scan like the folder itself (#6371). Skips listed models, HF cache+ repos, hidden folders, symlinks and junctions, so each model is reached once and the walk stays inside."""+ # Same test _scan_models_dir uses to list the folder as one model; loose GGUFs beside sub-folders are not.+ if _is_model_directory_for_scan(folder_path, entry_limit = _MAX_CUSTOM_FOLDER_ENTRIES):+ return []+ roots: list[Path] = []+ visited = 0+ stack: list[tuple[Path, int]] = [(folder_path, 0)]+ while stack and len(roots) < _MAX_NESTED_SCAN_DIRS:+ current, depth = stack.pop()+ if depth >= _MAX_NESTED_SCAN_DEPTH:+ continue+ children: list[Path] = []+ try:+ with os.scandir(current) as entries:+ for entry in entries:+ visited += 1+ if visited > _MAX_NESTED_SCAN_ENTRIES:+ break+ name = entry.name+ if (+ name.startswith((".", "models--", "datasets--", "spaces--"))+ or name in _NESTED_SCAN_SKIP_NAMES+ ):+ continue+ try:+ if not _is_plain_dir_entry(entry):+ continue+ except OSError:+ continue+ child = Path(entry.path)+ if not is_loadable_model_dir(child):+ children.append(child)+ except OSError:+ continue+ children.sort()+ roots.extend(children[: _MAX_NESTED_SCAN_DIRS - len(roots)])+ stack.extend((child, depth + 1) for child in reversed(children))+ if visited > _MAX_NESTED_SCAN_ENTRIES:+ break+ return roots++def _resolve_hf_cache_dir() -> Path:from utils.hf_cache_settings import get_hf_cache_pathsreturn get_hf_cache_paths().hub_cache@@ -838,7 +903,9 @@ async def _collect_models_from_default_sources(("model", model_id, cache_dir) for _repo, model_id, _updated in discovered)# Carry the registered path: the status registry is keyed on the row, not on the normalized Path this scan walks.- custom_sources.append((folder_path, hf_caches, str(folder["path"])))+ custom_sources.append(+ (folder_path, hf_caches, str(folder["path"]), bool(folder.get("recursive")))+ )try:variant_states = await asyncio.to_thread(download_manifest.build_variant_state_index,@@ -871,14 +938,18 @@ async def _collect_models_from_default_sources(local_models += await _scan_source("Hermes", scan_hermes_dir, hermes_dir)hermes_identities = {_inventory_physical_identity(str(d)) for d in hermes_dirs}- for folder_path, hf_caches, row_path in custom_sources:+ for folder_path, hf_caches, row_path, recursive in custom_sources:try:+ nested_roots = (+ tuple(await asyncio.to_thread(nested_scan_roots, folder_path)) if recursive else ()+ )custom_models = await asyncio.to_thread(_scan_custom_folder,folder_path,hf_caches = hf_caches,variant_states = variant_states,active_hub_cache = hf_cache_dir,+ nested_roots = nested_roots,)if _inventory_physical_identity(str(folder_path)) in hermes_identities:# Registering ~/.hermes/models was how Hermes downloads were listed before this scan;@@ -913,12 +984,46 @@ async def _collect_models_from_default_sources(return local_models+def _scan_row_key(row: LocalModelInfo) -> tuple[str, str, Optional[str]]:+ return (row.path, row.model_format, row.format_variant)+++def _scan_nested_roots(+ nested_roots: tuple[Path, ...],+ *,+ seen: set[tuple[str, str, Optional[str]]],+ active_hub_cache: Optional[Path],+) -> List[LocalModelInfo]:+ """Rows under a recursive folder's nested roots, minus any the folder's own scan already listed."""+ found: List[LocalModelInfo] = []+ for root in nested_roots:+ if len(found) >= _MAX_MODELS_PER_CUSTOM_FOLDER:+ break+ rows = _scan_models_dir(+ root,+ limit = _MAX_MODELS_PER_CUSTOM_FOLDER - len(found),+ entry_limit = _MAX_CUSTOM_FOLDER_ENTRIES,+ ) + _scan_hf_cache(+ root,+ entry_limit = _MAX_CUSTOM_FOLDER_ENTRIES,+ active_cache = False,+ active_hub_cache = active_hub_cache,+ )+ for row in rows:+ key = _scan_row_key(row)+ if key not in seen:+ seen.add(key)+ found.append(row)+ return found++def _scan_custom_folder(folder_path: Path,*,hf_caches: Optional[list[tuple[Path, Optional[list]]]] = None,variant_states: Optional[download_manifest.VariantStateIndex] = None,active_hub_cache: Optional[Path] = None,+ nested_roots: tuple[Path, ...] = (),) -> List[LocalModelInfo]:from utils.models.model_config import detect_gguf_model@@ -957,6 +1062,16 @@ def _scan_custom_folder(if _is_supported(m)if not any(p in (".studio_links", "ollama_links") for p in Path(m.path).parts)]+ if nested_roots:+ generic += [+ m+ for m in _scan_nested_roots(+ nested_roots,+ seen = {_scan_row_key(m) for m in generic},+ active_hub_cache = active_hub_cache,+ )+ if _is_supported(m)+ ]selectable = []for model in generic:if model.model_format != "gguf" or model.partial:@@ -1206,7 +1321,9 @@ async def list_local_models_response(models_dir: str = "./models") -> LocalModel_inventory_path_identity(models_dir),sources,tuple(- _inventory_path_identity(str(folder.get("path", ""))) for folder in custom_folders+ _inventory_path_identity(str(folder.get("path", "")))+ + ("\x00r" if folder.get("recursive") else "")+ for folder in custom_folders),epoch,)@@ -1253,10 +1370,10 @@ def get_scan_folders_response() -> dict:return {"folders": annotate_scan_folders(folders)}-def add_scan_folder_response(path: str) -> dict:+def add_scan_folder_response(path: str, recursive: Optional[bool] = None) -> dict:path = _account_access().private_directory(path, "")try:- folder, inserted = add_scan_folder_with_status(_coerce_scan_folder_path(path))+ folder, inserted = add_scan_folder_with_status(_coerce_scan_folder_path(path), recursive)except ValueError as e:logger.warning("Scan folder rejected: %s (path=%s)", scrub_paths(e), short_path_for_log(path)diff --git a/studio/backend/hub/storage/scan_folders.py b/studio/backend/hub/storage/scan_folders.pyindex 55d61c5cf0d..667f1d5ee0d 100644--- a/studio/backend/hub/storage/scan_folders.py+++ b/studio/backend/hub/storage/scan_folders.py@@ -129,14 +129,14 @@ def list_scan_folders() -> list[dict]:conn = get_connection()try:rows = conn.execute(- "SELECT id, path, created_at FROM scan_folders ORDER BY created_at"+ "SELECT id, path, created_at, recursive FROM scan_folders ORDER BY created_at").fetchall()- return [dict(row) for row in rows]+ return [{**dict(row), "recursive": bool(row["recursive"])} for row in rows]finally:conn.close()-def add_scan_folder_with_status(path: str) -> tuple[dict, bool]:+def add_scan_folder_with_status(path: str, recursive: bool | None = None) -> tuple[dict, bool]:"""Add a readable scan folder and return its row plus whether it was inserted."""if not path or not path.strip():raise ValueError("Path cannot be empty")@@ -171,30 +171,38 @@ def add_scan_folder_with_status(path: str) -> tuple[dict, bool]:now = datetime.now(timezone.utc).isoformat()if is_win:existing = conn.execute(- "SELECT id, path, created_at FROM scan_folders WHERE path = ? COLLATE NOCASE",+ "SELECT id, path, created_at, recursive FROM scan_folders WHERE path = ? COLLATE NOCASE",(normalized,),).fetchone()else:existing = conn.execute(- "SELECT id, path, created_at FROM scan_folders WHERE path = ?",+ "SELECT id, path, created_at, recursive FROM scan_folders WHERE path = ?",(normalized,),).fetchone()if existing is not None:- return dict(existing), False+ # None keeps the stored flag, so a re-add from export registration never resets it.+ if recursive is None or bool(existing["recursive"]) == recursive:+ return dict(existing), False+ conn.execute(+ "UPDATE scan_folders SET recursive = ? WHERE id = ?",+ (int(recursive), existing["id"]),+ )+ conn.commit()+ return {**dict(existing), "recursive": int(recursive)}, Trueinserted = Falsetry:conn.execute(- "INSERT INTO scan_folders (path, created_at) VALUES (?, ?)",- (normalized, now),+ "INSERT INTO scan_folders (path, created_at, recursive) VALUES (?, ?, ?)",+ (normalized, now, int(bool(recursive))),)conn.commit()inserted = Trueexcept sqlite3.IntegrityError:passfallback_sql = (- "SELECT id, path, created_at FROM scan_folders WHERE path = ? COLLATE NOCASE"+ "SELECT id, path, created_at, recursive FROM scan_folders WHERE path = ? COLLATE NOCASE"if is_win- else "SELECT id, path, created_at FROM scan_folders WHERE path = ?"+ else "SELECT id, path, created_at, recursive FROM scan_folders WHERE path = ?")row = conn.execute(fallback_sql, (normalized,)).fetchone()if row is None:@@ -204,9 +212,9 @@ def add_scan_folder_with_status(path: str) -> tuple[dict, bool]:conn.close()-def add_scan_folder(path: str) -> dict:+def add_scan_folder(path: str, recursive: bool | None = None) -> dict:"""Add a readable directory for the local OS user; not a multi-user sandbox."""- row, _ = add_scan_folder_with_status(path)+ row, _ = add_scan_folder_with_status(path, recursive)return rowdiff --git a/studio/backend/models/models.py b/studio/backend/models/models.pyindex 05513fc6ab9..071b319baf6 100644--- a/studio/backend/models/models.py+++ b/studio/backend/models/models.py@@ -320,6 +320,10 @@ class AddScanFolderRequest(BaseModel):"""Request body for adding a custom scan folder."""path: str = Field(..., description = "Absolute or relative directory path to scan for models")+ recursive: Optional[bool] = Field(+ None,+ description = "Also scan sub-folders. Omitted keeps the stored setting of an already registered folder.",+ )class ScanFolderInfo(BaseModel):@@ -328,6 +332,7 @@ class ScanFolderInfo(BaseModel):id: int = Field(..., description = "Database row ID")path: str = Field(..., description = "Normalized absolute path")created_at: str = Field(..., description = "ISO 8601 creation timestamp")+ recursive: bool = Field(False, description = "Sub-folders are scanned too")status: str = Field(default = "ok",description = "Last scan result: ok, permission_denied, missing, or unreadable",diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.pyindex c138c046100..f6fa076f5df 100644--- a/studio/backend/routes/models.py+++ b/studio/backend/routes/models.py@@ -826,6 +826,38 @@ def _compat_local_inventory_sources() -> _CompatLocalInventorySources:)+def _scan_nested_compat_rows(+ folder_path: Path, existing: List[LocalModelInfo], *, limit: int+) -> List[LocalModelInfo]:+ """Rows from a recursive scan folder's sub-folders that its own scan did not already list (#6371)."""+ from hub.services.models.local_inventory import is_loadable_model_dir, nested_scan_roots++ seen = {(m.path, m.model_format) for m in existing}+ found: List[LocalModelInfo] = []+ for root in nested_scan_roots(folder_path):+ if len(existing) + len(found) >= limit:+ break+ # No shared variant index: it was built for the registered caches, so a nested one reads its own state.+ rows = _scan_models_dir(root, limit = limit - len(existing) - len(found)) + _scan_hf_cache(+ root, active_cache = False+ )+ for row in rows:+ key = (row.path, row.model_format)+ path = Path(row.path)+ # This scanner also lists config-only folders; across a whole tree those are mostly other apps' configs.+ if row.source != "hf_cache" and not (path.is_file() or is_loadable_model_dir(path)):+ continue+ if key in seen or any(+ p in (".studio_links", "ollama_links") for p in Path(row.path).parts+ ):+ continue+ seen.add(key)+ found.append(row)+ if len(existing) + len(found) >= limit:+ return found+ return found++def collect_local_models(models_root: Path,*,@@ -949,6 +981,10 @@ def collect_local_models()if not any(p in (".studio_links", "ollama_links") for p in Path(m.path).parts)]+ if folder.get("recursive"):+ _generic += _scan_nested_compat_rows(+ folder_path, _generic, limit = _MAX_MODELS_PER_FOLDER+ )custom_models = []for model in _generic:path = Path(model.path)@@ -1137,7 +1173,9 @@ async def _shared_compat_local_inventory_scan(Path(_compat_inventory_path_identity(models_root)),scan_sources,tuple(- _compat_inventory_path_identity(folder.get("path", "")) for folder in custom_folders+ _compat_inventory_path_identity(folder.get("path", ""))+ + ("\x00r" if folder.get("recursive") else "")+ for folder in custom_folders),epoch,)@@ -1271,7 +1309,9 @@ async def add_scan_folder_endpoint(from storage.studio_db import add_scan_folder_with_statustry:- folder, inserted = await asyncio.to_thread(add_scan_folder_with_status, body.path)+ folder, inserted = await asyncio.to_thread(+ add_scan_folder_with_status, body.path, body.recursive+ )except ValueError as e:logger.warning("Scan folder rejected: %s (path=%s)", e, body.path)rejection_message = str(e)diff --git a/studio/backend/storage/studio_db.py b/studio/backend/storage/studio_db.pyindex 87a8c0a2478..742e4f75724 100644--- a/studio/backend/storage/studio_db.py+++ b/studio/backend/storage/studio_db.py@@ -450,10 +450,15 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:CREATE TABLE IF NOT EXISTS scan_folders (id INTEGER PRIMARY KEY AUTOINCREMENT,path TEXT NOT NULL UNIQUE {collation},- created_at TEXT NOT NULL+ created_at TEXT NOT NULL,+ recursive INTEGER NOT NULL DEFAULT 0)""")+ if "recursive" not in {+ row[1] for row in conn.execute("PRAGMA table_info(scan_folders)").fetchall()+ }:+ conn.execute("ALTER TABLE scan_folders ADD COLUMN recursive INTEGER NOT NULL DEFAULT 0")conn.execute("""CREATE TABLE IF NOT EXISTS chat_projects (@@ -1953,14 +1958,14 @@ def list_scan_folders() -> list[dict]:conn = get_connection()try:rows = conn.execute(- "SELECT id, path, created_at FROM scan_folders ORDER BY created_at"+ "SELECT id, path, created_at, recursive FROM scan_folders ORDER BY created_at").fetchall()- return [dict(row) for row in rows]+ return [{**dict(row), "recursive": bool(row["recursive"])} for row in rows]finally:conn.close()-def add_scan_folder_with_status(path: str) -> tuple[dict, bool]:+def add_scan_folder_with_status(path: str, recursive: bool | None = None) -> tuple[dict, bool]:if not path or not path.strip():raise ValueError("Path cannot be empty")normalized = os.path.realpath(os.path.expanduser(path.strip()))@@ -2003,21 +2008,29 @@ def add_scan_folder_with_status(path: str) -> tuple[dict, bool]:# Windows: case-insensitive lookup so C:\Models and c:\models dedup.if is_win:existing = conn.execute(- "SELECT id, path, created_at FROM scan_folders WHERE path = ? COLLATE NOCASE",+ "SELECT id, path, created_at, recursive FROM scan_folders WHERE path = ? COLLATE NOCASE",(normalized,),).fetchone()else:existing = conn.execute(- "SELECT id, path, created_at FROM scan_folders WHERE path = ?",+ "SELECT id, path, created_at, recursive FROM scan_folders WHERE path = ?",(normalized,),).fetchone()if existing is not None:- return dict(existing), False+ # None keeps the stored flag, so a re-add from export registration never resets it.+ if recursive is None or bool(existing["recursive"]) == recursive:+ return dict(existing), False+ conn.execute(+ "UPDATE scan_folders SET recursive = ? WHERE id = ?",+ (int(recursive), existing["id"]),+ )+ conn.commit()+ return {**dict(existing), "recursive": int(recursive)}, Trueinserted = Falsetry:conn.execute(- "INSERT INTO scan_folders (path, created_at) VALUES (?, ?)",- (normalized, now),+ "INSERT INTO scan_folders (path, created_at, recursive) VALUES (?, ?, ?)",+ (normalized, now, int(bool(recursive))),)conn.commit()inserted = True@@ -2025,9 +2038,9 @@ def add_scan_folder_with_status(path: str) -> tuple[dict, bool]:pass # duplicate; fall through to SELECT# Same collation as the pre-check to catch concurrent writes (Windows).fallback_sql = (- "SELECT id, path, created_at FROM scan_folders WHERE path = ? COLLATE NOCASE"+ "SELECT id, path, created_at, recursive FROM scan_folders WHERE path = ? COLLATE NOCASE"if is_win- else "SELECT id, path, created_at FROM scan_folders WHERE path = ?"+ else "SELECT id, path, created_at, recursive FROM scan_folders WHERE path = ?")row = conn.execute(fallback_sql, (normalized,)).fetchone()if row is None:@@ -2037,8 +2050,8 @@ def add_scan_folder_with_status(path: str) -> tuple[dict, bool]:conn.close()-def add_scan_folder(path: str) -> dict:- row, _ = add_scan_folder_with_status(path)+def add_scan_folder(path: str, recursive: bool | None = None) -> dict:+ row, _ = add_scan_folder_with_status(path, recursive)return rowdiff --git a/studio/backend/tests/test_cache_inventory_path_boundary.py b/studio/backend/tests/test_cache_inventory_path_boundary.pyindex b5be89f3dfa..cfa2d75f68d 100644--- a/studio/backend/tests/test_cache_inventory_path_boundary.py+++ b/studio/backend/tests/test_cache_inventory_path_boundary.py@@ -740,7 +740,7 @@ def test_a_filesystem_backed_row_is_not_named_by_its_path(_local_inventory):def _stub_rejected_scan_folder(monkeypatch):- def _raises(path):+ def _raises(path, recursive = None):raise HTTPException(status_code = 400,detail = f"Path is not readable: [Errno 36] File name too long: '{REPO_DIR}'",@@ -995,7 +995,8 @@ def test_a_cache_row_pinned_to_a_snapshot_is_referenced_too(monkeypatch):def test_the_compat_scan_folder_add_does_not_answer_with_the_path(monkeypatch):created = {"id": 7, "path": f"{HOST_ROOT}/extra", "created_at": "2026-09-01"}monkeypatch.setattr(- "storage.studio_db.add_scan_folder_with_status", lambda path: (created, False)+ "storage.studio_db.add_scan_folder_with_status",+ lambda path, recursive = None: (created, False),)payload = _models(via_api_key = True).post("/api/models/scan-folders", json = {"path": "."})diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 274ad2bbd69..a3a9b487b32 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -8497,7 +8497,7 @@ def test_async_scan_folder_routes_offload_storage_and_invalidation(monkeypatch):event_loop_thread = threading.get_ident()calls = []- def _add(path):+ def _add(path, recursive = None):calls.append(("add", threading.get_ident()))return {"id": 7, "path": path, "created_at": "fake"}, True@@ -8515,7 +8515,7 @@ def test_async_scan_folder_routes_offload_storage_and_invalidation(monkeypatch):async def _run():folder = await model_routes.add_scan_folder_endpoint(- SimpleNamespace(path = "/models/custom"), current_subject = "tester"+ SimpleNamespace(path = "/models/custom", recursive = None), current_subject = "tester")removed = await model_routes.remove_scan_folder_endpoint(7, current_subject = "tester")return folder, removeddiff --git a/studio/backend/tests/test_recursive_scan_folders.py b/studio/backend/tests/test_recursive_scan_folders.pynew file mode 100644index 00000000000..4eaa8e89cf1--- /dev/null+++ b/studio/backend/tests/test_recursive_scan_folders.py@@ -0,0 +1,214 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Opt-in recursive scan folders (#6371)."""++import os+import sqlite3+from pathlib import Path++import pytest++from hub.services.models import local_inventory+from storage import studio_db++[email protected](autouse = True)+def _registrable_tmp(monkeypatch):+ # macOS denies /private/var, where pytest keeps tmp_path.+ from hub.storage import scan_folders+ monkeypatch.setattr(studio_db, "_denied_path_prefixes", lambda: [])+ monkeypatch.setattr(scan_folders, "_denied_path_prefixes", lambda: [])+++def _gguf(path: Path) -> Path:+ path.parent.mkdir(parents = True, exist_ok = True)+ path.write_bytes(b"GGUF" + b"\0" * 28)+ return path+++def _library(root: Path) -> dict[str, Path]:+ """vendor/family/model nesting like the issue reporter's E:\\AI tree."""+ return {+ "flat": _gguf(root / "flat-Q4_K_M.gguf"),+ "deep": _gguf(root / "vendor" / "family" / "deep-model" / "deep-Q4_K_M.gguf"),+ "deeper": _gguf(root / "a" / "b" / "c" / "d" / "deeper-Q8_0.gguf"),+ }+++def _paths(rows) -> set[str]:+ return {str(Path(row.path).resolve()) for row in rows}+++def _scan(root: Path, recursive: bool):+ nested = tuple(local_inventory.nested_scan_roots(root)) if recursive else ()+ return local_inventory._scan_custom_folder(root, nested_roots = nested)+++def test_walker_lists_plain_folders_and_stops_at_models(tmp_path):+ lib = _library(tmp_path)+ roots = {+ p.relative_to(tmp_path).as_posix() for p in local_inventory.nested_scan_roots(tmp_path)+ }+ assert roots == {"vendor", "vendor/family", "a", "a/b", "a/b/c"}+ # A folder that is itself a model is listed by its parent's scan, so the walk never enters it.+ assert lib["deep"].parent.relative_to(tmp_path).as_posix() not in roots+++def test_walker_skips_hidden_hf_cache_repos_and_symlinks(tmp_path):+ _gguf(tmp_path / ".hidden" / "x" / "m.gguf")+ (tmp_path / "models--org--repo" / "snapshots" / "abc").mkdir(parents = True)+ (tmp_path / "ollama_links" / "x").mkdir(parents = True)+ real = tmp_path / "real"+ real.mkdir()+ try:+ (tmp_path / "loop").symlink_to(tmp_path, target_is_directory = True)+ except (OSError, NotImplementedError):+ pytest.skip("symlinks unavailable")+ names = {p.relative_to(tmp_path).parts[0] for p in local_inventory.nested_scan_roots(tmp_path)}+ assert names == {"real"}+++def test_walker_depth_cap_bounds_the_walk(tmp_path, monkeypatch):+ monkeypatch.setattr(local_inventory, "_MAX_NESTED_SCAN_DEPTH", 2)+ _gguf(tmp_path / "a" / "b" / "c" / "m.gguf")+ roots = {+ p.relative_to(tmp_path).as_posix() for p in local_inventory.nested_scan_roots(tmp_path)+ }+ assert roots == {"a", "a/b"}+++def test_walker_on_a_model_folder_adds_nothing(tmp_path):+ (tmp_path / "config.json").write_text("{}", encoding = "utf-8")+ (tmp_path / "model.safetensors").write_bytes(b"\0" * 8)+ (tmp_path / "sub").mkdir()+ assert local_inventory.nested_scan_roots(tmp_path) == []+++def test_recursive_folder_lists_nested_models_and_flat_stays_flat(tmp_path):+ lib = _library(tmp_path)+ flat = _paths(_scan(tmp_path, recursive = False))+ deep = _paths(_scan(tmp_path, recursive = True))+ assert str(lib["flat"].resolve()) in flat+ assert str(lib["deep"].parent.resolve()) not in flat+ assert {str(lib["flat"].resolve()), str(lib["deep"].parent.resolve())} <= deep+ assert str(lib["deeper"].parent.resolve()) in deep+ # Turning the flag on only adds rows.+ assert flat <= deep+++def test_recursive_scan_does_not_relist_an_lmstudio_layout(tmp_path):+ _gguf(tmp_path / "publisher" / "repo-GGUF" / "repo-Q4_K_M.gguf")+ rows = _scan(tmp_path, recursive = True)+ target = str((tmp_path / "publisher" / "repo-GGUF").resolve())+ assert sum(1 for row in rows if str(Path(row.path).resolve()) == target) == 1+++def test_recursive_scan_finds_a_nested_hf_cache(tmp_path):+ repo = tmp_path / "team" / "hub" / "models--org--tiny"+ snapshot = repo / "snapshots" / "abc123"+ _gguf(snapshot / "tiny-Q4_K_M.gguf")+ (repo / "refs").mkdir()+ (repo / "refs" / "main").write_text("abc123", encoding = "utf-8")+ ids = {row.model_id for row in _scan(tmp_path, recursive = True)}+ assert "org/tiny" in ids+ assert "org/tiny" not in {row.model_id for row in _scan(tmp_path, recursive = False)}+++def test_compat_nested_rows_skip_rows_already_listed(tmp_path):+ from routes import models as models_routes++ lib = _library(tmp_path)+ existing = models_routes._scan_models_dir(tmp_path, limit = 200)+ nested = models_routes._scan_nested_compat_rows(tmp_path, existing, limit = 200)+ assert str(lib["deep"].parent.resolve()) in _paths(nested)+ assert not _paths(existing) & _paths(nested)+ assert models_routes._scan_nested_compat_rows(tmp_path, existing, limit = len(existing)) == []+++def test_storage_round_trips_and_updates_the_flag(tmp_path):+ folder = tmp_path / "models"+ folder.mkdir()+ row, changed = studio_db.add_scan_folder_with_status(str(folder), True)+ assert changed and row["recursive"] == 1+ assert studio_db.list_scan_folders()[0]["recursive"] is True+ # Omitted keeps the stored flag: export re-registration must not reset it.+ row, changed = studio_db.add_scan_folder_with_status(str(folder))+ assert not changed and row["recursive"] == 1+ row, changed = studio_db.add_scan_folder_with_status(str(folder), False)+ assert changed and row["recursive"] == 0+ assert studio_db.list_scan_folders()[0]["recursive"] is False+++def test_hub_storage_shares_the_flag(tmp_path):+ from hub.storage import scan_folders++ folder = tmp_path / "models"+ folder.mkdir()+ row, _ = scan_folders.add_scan_folder_with_status(str(folder), True)+ assert scan_folders.list_scan_folders()[0]["recursive"] is True+ assert studio_db.list_scan_folders()[0]["id"] == row["id"]+++def test_an_existing_table_gains_the_column(tmp_path):+ db_path = studio_db.studio_db_path()+ db_path.parent.mkdir(parents = True, exist_ok = True)+ conn = sqlite3.connect(db_path)+ conn.execute(+ "CREATE TABLE scan_folders (id INTEGER PRIMARY KEY AUTOINCREMENT, "+ "path TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL)"+ )+ conn.execute(+ "INSERT INTO scan_folders (path, created_at) VALUES (?, ?)", (str(tmp_path), "2026-01-01")+ )+ conn.commit()+ conn.close()+ rows = studio_db.list_scan_folders()+ assert rows == [+ {"id": 1, "path": str(tmp_path), "created_at": "2026-01-01", "recursive": False}+ ]+++def test_add_endpoint_threads_the_flag(tmp_path):+ folder = tmp_path / "models"+ folder.mkdir()+ row = local_inventory.add_scan_folder_response(str(folder), True)+ assert row["recursive"] == 1+ assert os.path.samefile(row["path"], folder)+++def test_a_config_only_folder_does_not_hide_the_model_below_it(tmp_path):+ from routes import models as models_routes++ family = tmp_path / "family"+ family.mkdir()+ (family / "config.json").write_text("{}", encoding = "utf-8")+ model = family / "variant" / "model"+ model.mkdir(parents = True)+ (model / "config.json").write_text("{}", encoding = "utf-8")+ (model / "model.safetensors").write_bytes(b"\0" * 8)+ assert str(model.resolve()) in _paths(_scan(tmp_path, recursive = True))+ nested = models_routes._scan_nested_compat_rows(tmp_path, [], limit = 200)+ assert str(model.resolve()) in _paths(nested)+++def test_compat_nested_rows_skip_folders_with_nothing_to_load(tmp_path):+ from routes import models as models_routes++ app = tmp_path / "tools" / "some-app"+ app.mkdir(parents = True)+ (app / "config.json").write_text("{}", encoding = "utf-8")+ assert models_routes._scan_nested_compat_rows(tmp_path, [], limit = 200) == []+++def test_compat_nested_rows_keep_the_cap_with_a_nested_hf_cache(tmp_path):+ from routes import models as models_routes++ hub = tmp_path / "team" / "hub"+ for i in range(5):+ repo = hub / f"models--org--tiny{i}"+ _gguf(repo / "snapshots" / "abc" / f"tiny{i}-Q4_K_M.gguf")+ (repo / "refs").mkdir()+ (repo / "refs" / "main").write_text("abc", encoding = "utf-8")+ assert len(models_routes._scan_nested_compat_rows(tmp_path, [], limit = 5)) == 5+ assert len(models_routes._scan_nested_compat_rows(tmp_path, [], limit = 3)) == 3diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.tsindex 5534e2c4c68..f2529b054e0 100644--- a/studio/frontend/src/features/chat/api/chat-api.ts+++ b/studio/frontend/src/features/chat/api/chat-api.ts@@ -798,6 +798,8 @@ export interface ScanFolderInfo {id: number;path: string;created_at: string;+ /** Sub-folders are scanned too. Absent on older backends. */+ recursive?: boolean;/** Result of the last scan. Absent on older backends, which means "ok". */status?: "ok" | "permission_denied" | "missing" | "unreadable" | "partial";}@@ -808,11 +810,14 @@ export async function listScanFolders(): Promise<ScanFolderInfo[]> {return data.folders;}-export async function addScanFolder(path: string): Promise<ScanFolderInfo> {+export async function addScanFolder(+ path: string,+ recursive?: boolean,+): Promise<ScanFolderInfo> {const response = await authFetch("/api/models/scan-folders", {method: "POST",headers: { "Content-Type": "application/json" },- body: JSON.stringify({ path }),+ body: JSON.stringify({ path, recursive }),});return parseJsonOrThrow<ScanFolderInfo>(response);}diff --git a/studio/frontend/src/features/hub/catalog/on-device-folders-dialog.tsx b/studio/frontend/src/features/hub/catalog/on-device-folders-dialog.tsxindex fbb134a9b3e..9ec440dd7f6 100644--- a/studio/frontend/src/features/hub/catalog/on-device-folders-dialog.tsx+++ b/studio/frontend/src/features/hub/catalog/on-device-folders-dialog.tsx@@ -2,6 +2,7 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import { Button } from "@/components/ui/button";+import { Checkbox } from "@/components/ui/checkbox";import {Dialog,DialogContent,@@ -76,6 +77,7 @@ export function OnDeviceFoldersDialog({}) {const [folders, setFolders] = useState<ScanFolderInfo[]>([]);const [path, setPath] = useState("");+ const [recursive, setRecursive] = useState(false);const [browserOpen, setBrowserOpen] = useState(false);const [loading, setLoading] = useState(false);const [error, setError] = useState<string | null>(null);@@ -195,7 +197,7 @@ export function OnDeviceFoldersDialog({setPending("add");setError(null);try {- const folder = await addScanFolder(nextPath);+ const folder = await addScanFolder(nextPath, recursive);setPath("");mutationVersionRef.current += 1;setFolders((current) => {@@ -216,7 +218,7 @@ export function OnDeviceFoldersDialog({setPending(null);}},- [handleInventoryChanged, pending],+ [handleInventoryChanged, pending, recursive],);// Scan folders are arbitrary paths that may be moved or deleted after they@@ -415,6 +417,19 @@ export function OnDeviceFoldersDialog({</Button></div></div>+ <div className="mt-2 flex items-center gap-2">+ <Checkbox+ id="scan-folder-recursive"+ checked={recursive}+ onCheckedChange={(checked) => setRecursive(checked === true)}+ />+ <label+ htmlFor="scan-folder-recursive"+ className="text-ui-12 text-muted-foreground"+ >+ Also scan sub-folders+ </label>+ </div></div>{error ? (@@ -506,6 +521,11 @@ export function OnDeviceFoldersDialog({{folder.path}</TooltipContent></Tooltip>+ {folder.recursive ? (+ <p className="text-ui-10p5 text-muted-foreground">+ Includes sub-folders+ </p>+ ) : null}{problem ? (<pdata-testid={`scan-folder-problem-${folder.id}`}diff --git a/studio/frontend/src/features/hub/inventory/api.ts b/studio/frontend/src/features/hub/inventory/api.tsindex d71b6fbcbc5..140c5d48661 100644--- a/studio/frontend/src/features/hub/inventory/api.ts+++ b/studio/frontend/src/features/hub/inventory/api.ts@@ -186,6 +186,8 @@ export interface ScanFolderInfo {id: number;path: string;created_at: string;+ /** Sub-folders are scanned too. Absent on older backends. */+ recursive?: boolean;/** Result of the last scan. Absent on older backends, which means "ok". */status?: ScanFolderStatus;}@@ -436,11 +438,14 @@ export async function listScanFolders(): Promise<ScanFolderInfo[]> {return data.folders;}-export async function addScanFolder(path: string): Promise<ScanFolderInfo> {+export async function addScanFolder(+ path: string,+ recursive?: boolean,+): Promise<ScanFolderInfo> {const response = await authFetch("/api/hub/scan-folders", {method: "POST",headers: { "Content-Type": "application/json" },- body: JSON.stringify({ path }),+ body: JSON.stringify({ path, recursive }),});const folder = await parseJsonOrThrow<ScanFolderInfo>(response);bumpInventoryVersion();