Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep linked folders syncing when files become ignored (#15441)
Nilay42802279dfc2
diff --git a/studio/backend/core/rag/folder_sync.py b/studio/backend/core/rag/folder_sync.pyindex bff1963b82d..72fce2258c6 100644--- a/studio/backend/core/rag/folder_sync.py+++ b/studio/backend/core/rag/folder_sync.py@@ -1070,6 +1070,7 @@ def _scan(root: str,expected_identity: tuple[int, int] | None = None,report: dict[str, int] | None = None,+ excluded: set[str] | None = None,) -> tuple[dict[str, dict], tuple[int, int]]:"""Map relative path -> metadata; `report` receives counts of deliberately skipped entries."""from hub.storage.scan_folders import contains_sensitive_path_component, is_denied_system_path@@ -1079,6 +1080,7 @@ def _scan(raise RuntimeError("Linked folder root identity changed")skipped = {"gitignored": 0, "build_dirs": 0, "too_large": 0, "over_limit": 0}+ excluded = set() if excluded is None else excludedfound: dict[str, dict] = {}mount_points = _mount_points()root_identities = frozenset({identity}) if identity[1] not in (None, 0) else frozenset()@@ -1098,12 +1100,15 @@ def _scan(rel = os.path.relpath(full, root).replace(os.sep, "/")if entry.is_dir(follow_symlinks = False):if _is_ignored_scan_dir(entry.name, full):+ excluded.add(rel)continueif rules and gitignore.is_ignored(rel, True, rules):skipped["gitignored"] += 1+ excluded.add(rel)continueif _is_build_output_dir(entry.name, directory):skipped["build_dirs"] += 1+ excluded.add(rel)continueresolved = os.path.realpath(full)if (@@ -1150,6 +1155,7 @@ def _scan(continueif rules and gitignore.is_ignored(rel, False, rules):skipped["gitignored"] += 1+ excluded.add(rel)continue# Compare only fstat-reproducible ids: scandir has none on Windows; lstat differs on unstable inodes.if is_appledouble_metadata(Path(full)):@@ -1172,6 +1178,7 @@ def _scan(and st.st_size > config.FOLDER_MAX_TEXT_BYTES):skipped["too_large"] += 1+ excluded.add(rel)continuefound[rel] = {"path": full,@@ -1192,6 +1199,7 @@ def _scan(if config.FOLDER_MAX_FILES and len(found) > config.FOLDER_MAX_FILES:kept = sorted(found)[: config.FOLDER_MAX_FILES]skipped["over_limit"] = len(found) - len(kept)+ excluded.update(set(found) - set(kept))found = {rel: found[rel] for rel in kept}if report is not None:report.update(skipped)@@ -1294,7 +1302,7 @@ def _check_root_identity(root: str, expected: tuple[int, int]) -> None:raise _FolderChanged("Linked folder root identity changed during reconciliation")-def _source_reappeared(root: str, relative_path: str) -> bool:+def _source_reappeared(root: str, relative_path: str, excluded: set[str]) -> bool:"""check scanner eligibility without following a replaced path component."""path = PurePosixPath(relative_path)parts = path.parts@@ -1310,6 +1318,8 @@ def _source_reappeared(root: str, relative_path: str) -> bool:# previously indexed dot directories are excluded even if they still existif index < len(parts) - 1 and part.startswith("."):return False+ if "/".join(parts[: index + 1]) in excluded:+ return Falsecurrent = os.path.join(current, part)try:current_stat = os.lstat(current)@@ -1323,7 +1333,8 @@ def _source_reappeared(root: str, relative_path: str) -> bool:if not stat.S_ISDIR(current_stat.st_mode):return Falseelse:- return stat.S_ISREG(current_stat.st_mode)+ # Empty files are skipped by the scan; one refilled since then has reappeared.+ return stat.S_ISREG(current_stat.st_mode) and current_stat.st_size > 0return False@@ -1610,7 +1621,8 @@ def _reconcile_folder(job_id: str) -> None:_check_running()expected_identity = _load_identity(folder["root_device"], folder["root_inode"])skipped: dict[str, int] = {}- current, scanned_identity = _scan(folder["path"], expected_identity, skipped)+ excluded: set[str] = set()+ current, scanned_identity = _scan(folder["path"], expected_identity, skipped, excluded)_check_running()except (_SyncStopped, _SyncCancelled, _LeaseLost):raise@@ -1889,10 +1901,20 @@ def _reconcile_folder(job_id: str) -> None:raisedeleted = 0- for rel in sorted(missing - withheld):+ removable = sorted(missing - withheld)+ if removable:+ # Exclusions can lapse during ingestion (.gitignore edited, file shrunk): judge removals on a fresh scan.+ excluded = set()+ try:+ rescanned, _ = _scan(folder["path"], scanned_identity, None, excluded)+ except RuntimeError as exc:+ raise _FolderChanged(str(exc)) from exc+ if not rescanned.keys().isdisjoint(removable):+ raise _FolderChanged("Linked source reappeared during reconciliation")+ for rel in removable:_check_running()_check_root_identity(folder["path"], scanned_identity)- if _source_reappeared(folder["path"], rel):+ if _source_reappeared(folder["path"], rel, excluded):raise _FolderChanged("Linked source reappeared during reconciliation")_check_root_identity(folder["path"], scanned_identity)_delete_mapping(folder["id"], rel)diff --git a/studio/backend/tests/test_rag_folder_scan_filters.py b/studio/backend/tests/test_rag_folder_scan_filters.pyindex 0c005166784..a05285f6cec 100644--- a/studio/backend/tests/test_rag_folder_scan_filters.py+++ b/studio/backend/tests/test_rag_folder_scan_filters.py@@ -259,3 +259,132 @@ def test_every_failed_file_is_listed_with_its_reason(rag_home, stub_embeddings,view = _folder_view(_folder_row(folder["id"]))assert view["failureCount"] == 5assert view["failures"][0] == {"path": "bad0.txt", "error": "embed unavailable"}+++def _mapped(folder_id: str) -> set[str]:+ with closing(rag_db.get_connection()) as conn:+ return {+ row["relative_path"]+ for row in conn.execute(+ "SELECT relative_path FROM linked_folder_files WHERE folder_id=?", (folder_id,)+ )+ }+++def _gitignore_the_folder(source, monkeypatch):+ _write(source, ".gitignore", "build/\n")+++def _empty_the_file(source, monkeypatch):+ _write(source, "build/plan.txt", "")+++def _add_a_project_manifest(source, monkeypatch):+ _write(source, "package.json", "{}")+++def _shrink_the_text_limit(source, monkeypatch):+ monkeypatch.setattr(config, "FOLDER_MAX_TEXT_BYTES", 10)+++def _lower_the_file_cap(source, monkeypatch):+ monkeypatch.setattr(config, "FOLDER_MAX_FILES", 1)+++def _make_it_a_venv(source, monkeypatch):+ _write(source, "build/pyvenv.cfg", "home = /usr/bin")+++@requires_sqlite_vec+ "exclude",+ [+ _gitignore_the_folder,+ _empty_the_file,+ _add_a_project_manifest,+ _shrink_the_text_limit,+ _lower_the_file_cap,+ _make_it_a_venv,+ ],+)+def test_a_file_the_scan_now_skips_is_removed_from_the_index(+ rag_home, stub_embeddings, monkeypatch, exclude+):+ source = rag_home / "source"+ _write(source, "a-notes.md", "notes")+ _write(source, "build/plan.txt", "draft plan " * 10)+ folder = folder_sync.create_folder(+ scope_type = "project", scope_id = "p1", path = str(source), name = "Docs"+ )+ assert _run(folder["id"])["status"] == "completed"+ assert _mapped(folder["id"]) == {"a-notes.md", "build/plan.txt"}++ exclude(source, monkeypatch)+ job = _run(folder["id"])+ assert job["status"] == "completed", job["error"]+ assert job["deleted"] == 1+ assert _mapped(folder["id"]) - {"package.json"} == {"a-notes.md"}+ assert (source / "build" / "plan.txt").is_file()+++@requires_sqlite_vec+def test_a_file_refilled_after_an_empty_scan_keeps_its_index_entry(+ rag_home, stub_embeddings, monkeypatch+):+ source = rag_home / "source"+ _write(source, "notes.md", "durable words")+ note = source / "notes.md"+ folder = folder_sync.create_folder(+ scope_type = "project", scope_id = "p1", path = str(source), name = "Docs"+ )+ assert _run(folder["id"])["status"] == "completed"++ note.write_text("", encoding = "utf-8")+ real_scan = folder_sync._scan++ def refill_after_scan(*args, **kwargs):+ result = real_scan(*args, **kwargs)+ note.write_text("durable words", encoding = "utf-8")+ return result++ monkeypatch.setattr(folder_sync, "_scan", refill_after_scan)+ assert _run(folder["id"])["status"] == "failed"+ assert _mapped(folder["id"]) == {"notes.md"}+++def _shrink_back_under_the_limit(source, monkeypatch):+ monkeypatch.setattr(config, "FOLDER_MAX_TEXT_BYTES", 10)+ return lambda: monkeypatch.setattr(config, "FOLDER_MAX_TEXT_BYTES", 0)+++def _drop_the_gitignore_rule(source, monkeypatch):+ _write(source, ".gitignore", "notes.txt\n")+ return lambda: (source / ".gitignore").unlink()+++@requires_sqlite_vec[email protected]("exclude", [_shrink_back_under_the_limit, _drop_the_gitignore_rule])+def test_a_file_eligible_again_before_removal_keeps_its_index_entry(+ rag_home, stub_embeddings, monkeypatch, exclude+):+ source = rag_home / "source"+ _write(source, "notes.txt", "durable words")+ folder = folder_sync.create_folder(+ scope_type = "project", scope_id = "p1", path = str(source), name = "Docs"+ )+ assert _run(folder["id"])["status"] == "completed"++ restore = exclude(source, monkeypatch)+ real_scan = folder_sync._scan+ calls = []++ def restore_after_first_scan(*args, **kwargs):+ result = real_scan(*args, **kwargs)+ if not calls:+ restore()+ calls.append(1)+ return result++ monkeypatch.setattr(folder_sync, "_scan", restore_after_first_scan)+ assert _run(folder["id"])["status"] == "failed"+ assert _mapped(folder["id"]) == {"notes.txt"}diff --git a/studio/backend/tests/test_rag_linked_folders.py b/studio/backend/tests/test_rag_linked_folders.pyindex 7dc803e7599..64598b443d8 100644--- a/studio/backend/tests/test_rag_linked_folders.py+++ b/studio/backend/tests/test_rag_linked_folders.py@@ -1014,8 +1014,9 @@ def test_large_windows_file_identity_round_trips_through_sqlite(root,expected_identity = None,report = None,+ excluded = None,):- found, root_identity = real_scan(root, expected_identity, report)+ found, root_identity = real_scan(root, expected_identity, report, excluded)for metadata in found.values():metadata["scanned"], (metadata["device"], metadata["inode"]) = ((metadata["device"], metadata["inode"]),