Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep linked folders syncing when files become ignored (#15441)

Nilay42802279dfc2
diff --git a/studio/backend/core/rag/folder_sync.py b/studio/backend/core/rag/folder_sync.py
index bff1963b82d..72fce2258c6 100644
--- a/studio/backend/core/rag/folder_sync.py
+++ b/studio/backend/core/rag/folder_sync.py
@@ -1070,6 +1070,7 @@ def _scan(
root: str,
expected_identity: tuple[int, int] | None = None,
report: dict[str, int] | None = None,
+ excluded: set[str] | None = None,
) -> tuple[dict[str, dict], tuple[int, int]]:
"""Map relative path -> metadata; `report` receives counts of deliberately skipped entries."""
from hub.storage.scan_folders import contains_sensitive_path_component, is_denied_system_path
@@ -1079,6 +1080,7 @@ def _scan(
raise RuntimeError("Linked folder root identity changed")
skipped = {"gitignored": 0, "build_dirs": 0, "too_large": 0, "over_limit": 0}
+ excluded = set() if excluded is None else excluded
found: dict[str, dict] = {}
mount_points = _mount_points()
root_identities = frozenset({identity}) if identity[1] not in (None, 0) else frozenset()
@@ -1098,12 +1100,15 @@ def _scan(
rel = os.path.relpath(full, root).replace(os.sep, "/")
if entry.is_dir(follow_symlinks = False):
if _is_ignored_scan_dir(entry.name, full):
+ excluded.add(rel)
continue
if rules and gitignore.is_ignored(rel, True, rules):
skipped["gitignored"] += 1
+ excluded.add(rel)
continue
if _is_build_output_dir(entry.name, directory):
skipped["build_dirs"] += 1
+ excluded.add(rel)
continue
resolved = os.path.realpath(full)
if (
@@ -1150,6 +1155,7 @@ def _scan(
continue
if rules and gitignore.is_ignored(rel, False, rules):
skipped["gitignored"] += 1
+ excluded.add(rel)
continue
# Compare only fstat-reproducible ids: scandir has none on Windows; lstat differs on unstable inodes.
if is_appledouble_metadata(Path(full)):
@@ -1172,6 +1178,7 @@ def _scan(
and st.st_size > config.FOLDER_MAX_TEXT_BYTES
):
skipped["too_large"] += 1
+ excluded.add(rel)
continue
found[rel] = {
"path": full,
@@ -1192,6 +1199,7 @@ def _scan(
if config.FOLDER_MAX_FILES and len(found) > config.FOLDER_MAX_FILES:
kept = sorted(found)[: config.FOLDER_MAX_FILES]
skipped["over_limit"] = len(found) - len(kept)
+ excluded.update(set(found) - set(kept))
found = {rel: found[rel] for rel in kept}
if report is not None:
report.update(skipped)
@@ -1294,7 +1302,7 @@ def _check_root_identity(root: str, expected: tuple[int, int]) -> None:
raise _FolderChanged("Linked folder root identity changed during reconciliation")
-def _source_reappeared(root: str, relative_path: str) -> bool:
+def _source_reappeared(root: str, relative_path: str, excluded: set[str]) -> bool:
"""check scanner eligibility without following a replaced path component."""
path = PurePosixPath(relative_path)
parts = path.parts
@@ -1310,6 +1318,8 @@ def _source_reappeared(root: str, relative_path: str) -> bool:
# previously indexed dot directories are excluded even if they still exist
if index < len(parts) - 1 and part.startswith("."):
return False
+ if "/".join(parts[: index + 1]) in excluded:
+ return False
current = os.path.join(current, part)
try:
current_stat = os.lstat(current)
@@ -1323,7 +1333,8 @@ def _source_reappeared(root: str, relative_path: str) -> bool:
if not stat.S_ISDIR(current_stat.st_mode):
return False
else:
- return stat.S_ISREG(current_stat.st_mode)
+ # Empty files are skipped by the scan; one refilled since then has reappeared.
+ return stat.S_ISREG(current_stat.st_mode) and current_stat.st_size > 0
return False
@@ -1610,7 +1621,8 @@ def _reconcile_folder(job_id: str) -> None:
_check_running()
expected_identity = _load_identity(folder["root_device"], folder["root_inode"])
skipped: dict[str, int] = {}
- current, scanned_identity = _scan(folder["path"], expected_identity, skipped)
+ excluded: set[str] = set()
+ current, scanned_identity = _scan(folder["path"], expected_identity, skipped, excluded)
_check_running()
except (_SyncStopped, _SyncCancelled, _LeaseLost):
raise
@@ -1889,10 +1901,20 @@ def _reconcile_folder(job_id: str) -> None:
raise
deleted = 0
- for rel in sorted(missing - withheld):
+ removable = sorted(missing - withheld)
+ if removable:
+ # Exclusions can lapse during ingestion (.gitignore edited, file shrunk): judge removals on a fresh scan.
+ excluded = set()
+ try:
+ rescanned, _ = _scan(folder["path"], scanned_identity, None, excluded)
+ except RuntimeError as exc:
+ raise _FolderChanged(str(exc)) from exc
+ if not rescanned.keys().isdisjoint(removable):
+ raise _FolderChanged("Linked source reappeared during reconciliation")
+ for rel in removable:
_check_running()
_check_root_identity(folder["path"], scanned_identity)
- if _source_reappeared(folder["path"], rel):
+ if _source_reappeared(folder["path"], rel, excluded):
raise _FolderChanged("Linked source reappeared during reconciliation")
_check_root_identity(folder["path"], scanned_identity)
_delete_mapping(folder["id"], rel)
diff --git a/studio/backend/tests/test_rag_folder_scan_filters.py b/studio/backend/tests/test_rag_folder_scan_filters.py
index 0c005166784..a05285f6cec 100644
--- a/studio/backend/tests/test_rag_folder_scan_filters.py
+++ b/studio/backend/tests/test_rag_folder_scan_filters.py
@@ -259,3 +259,132 @@ def test_every_failed_file_is_listed_with_its_reason(rag_home, stub_embeddings,
view = _folder_view(_folder_row(folder["id"]))
assert view["failureCount"] == 5
assert view["failures"][0] == {"path": "bad0.txt", "error": "embed unavailable"}
+
+
+def _mapped(folder_id: str) -> set[str]:
+ with closing(rag_db.get_connection()) as conn:
+ return {
+ row["relative_path"]
+ for row in conn.execute(
+ "SELECT relative_path FROM linked_folder_files WHERE folder_id=?", (folder_id,)
+ )
+ }
+
+
+def _gitignore_the_folder(source, monkeypatch):
+ _write(source, ".gitignore", "build/\n")
+
+
+def _empty_the_file(source, monkeypatch):
+ _write(source, "build/plan.txt", "")
+
+
+def _add_a_project_manifest(source, monkeypatch):
+ _write(source, "package.json", "{}")
+
+
+def _shrink_the_text_limit(source, monkeypatch):
+ monkeypatch.setattr(config, "FOLDER_MAX_TEXT_BYTES", 10)
+
+
+def _lower_the_file_cap(source, monkeypatch):
+ monkeypatch.setattr(config, "FOLDER_MAX_FILES", 1)
+
+
+def _make_it_a_venv(source, monkeypatch):
+ _write(source, "build/pyvenv.cfg", "home = /usr/bin")
+
+
+@requires_sqlite_vec
+ "exclude",
+ [
+ _gitignore_the_folder,
+ _empty_the_file,
+ _add_a_project_manifest,
+ _shrink_the_text_limit,
+ _lower_the_file_cap,
+ _make_it_a_venv,
+ ],
+)
+def test_a_file_the_scan_now_skips_is_removed_from_the_index(
+ rag_home, stub_embeddings, monkeypatch, exclude
+):
+ source = rag_home / "source"
+ _write(source, "a-notes.md", "notes")
+ _write(source, "build/plan.txt", "draft plan " * 10)
+ folder = folder_sync.create_folder(
+ scope_type = "project", scope_id = "p1", path = str(source), name = "Docs"
+ )
+ assert _run(folder["id"])["status"] == "completed"
+ assert _mapped(folder["id"]) == {"a-notes.md", "build/plan.txt"}
+
+ exclude(source, monkeypatch)
+ job = _run(folder["id"])
+ assert job["status"] == "completed", job["error"]
+ assert job["deleted"] == 1
+ assert _mapped(folder["id"]) - {"package.json"} == {"a-notes.md"}
+ assert (source / "build" / "plan.txt").is_file()
+
+
+@requires_sqlite_vec
+def test_a_file_refilled_after_an_empty_scan_keeps_its_index_entry(
+ rag_home, stub_embeddings, monkeypatch
+):
+ source = rag_home / "source"
+ _write(source, "notes.md", "durable words")
+ note = source / "notes.md"
+ folder = folder_sync.create_folder(
+ scope_type = "project", scope_id = "p1", path = str(source), name = "Docs"
+ )
+ assert _run(folder["id"])["status"] == "completed"
+
+ note.write_text("", encoding = "utf-8")
+ real_scan = folder_sync._scan
+
+ def refill_after_scan(*args, **kwargs):
+ result = real_scan(*args, **kwargs)
+ note.write_text("durable words", encoding = "utf-8")
+ return result
+
+ monkeypatch.setattr(folder_sync, "_scan", refill_after_scan)
+ assert _run(folder["id"])["status"] == "failed"
+ assert _mapped(folder["id"]) == {"notes.md"}
+
+
+def _shrink_back_under_the_limit(source, monkeypatch):
+ monkeypatch.setattr(config, "FOLDER_MAX_TEXT_BYTES", 10)
+ return lambda: monkeypatch.setattr(config, "FOLDER_MAX_TEXT_BYTES", 0)
+
+
+def _drop_the_gitignore_rule(source, monkeypatch):
+ _write(source, ".gitignore", "notes.txt\n")
+ return lambda: (source / ".gitignore").unlink()
+
+
+@requires_sqlite_vec
[email protected]("exclude", [_shrink_back_under_the_limit, _drop_the_gitignore_rule])
+def test_a_file_eligible_again_before_removal_keeps_its_index_entry(
+ rag_home, stub_embeddings, monkeypatch, exclude
+):
+ source = rag_home / "source"
+ _write(source, "notes.txt", "durable words")
+ folder = folder_sync.create_folder(
+ scope_type = "project", scope_id = "p1", path = str(source), name = "Docs"
+ )
+ assert _run(folder["id"])["status"] == "completed"
+
+ restore = exclude(source, monkeypatch)
+ real_scan = folder_sync._scan
+ calls = []
+
+ def restore_after_first_scan(*args, **kwargs):
+ result = real_scan(*args, **kwargs)
+ if not calls:
+ restore()
+ calls.append(1)
+ return result
+
+ monkeypatch.setattr(folder_sync, "_scan", restore_after_first_scan)
+ assert _run(folder["id"])["status"] == "failed"
+ assert _mapped(folder["id"]) == {"notes.txt"}
diff --git a/studio/backend/tests/test_rag_linked_folders.py b/studio/backend/tests/test_rag_linked_folders.py
index 7dc803e7599..64598b443d8 100644
--- a/studio/backend/tests/test_rag_linked_folders.py
+++ b/studio/backend/tests/test_rag_linked_folders.py
@@ -1014,8 +1014,9 @@ def test_large_windows_file_identity_round_trips_through_sqlite(
root,
expected_identity = None,
report = None,
+ excluded = None,
):
- found, root_identity = real_scan(root, expected_identity, report)
+ found, root_identity = real_scan(root, expected_identity, report, excluded)
for metadata in found.values():
metadata["scanned"], (metadata["device"], metadata["inode"]) = (
(metadata["device"], metadata["inode"]),