Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: linked folders honour .gitignore, folder uploads in the browser, queued drops and retry for failed sources (#13224)
Michael Han6126f3ec6805
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 5e0b55e2788..6c2b9135398 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -1100,11 +1100,15 @@ def _resolve_load_family(# Wan2.2 A14B repo, or the reverse), local or remote.if fam is not None and gguf_filename:named = detect_video_family(os.path.basename(gguf_filename))- if named is not None and named.name != fam.name and (- named.name in SAME_ARCH_VARIANTS.get(fam.name, ())- or fam.name in SAME_ARCH_VARIANTS.get(named.name, ())- # A remote pick has no header pass, so a file naming another Wan model picks the base config.- or (named.name.startswith("wan") and fam.name.startswith("wan"))+ if (+ named is not None+ and named.name != fam.name+ and (+ named.name in SAME_ARCH_VARIANTS.get(fam.name, ())+ or fam.name in SAME_ARCH_VARIANTS.get(named.name, ())+ # A remote pick has no header pass, so a file naming another Wan model picks the base config.+ or (named.name.startswith("wan") and fam.name.startswith("wan"))+ )):fam = named# A local file's header outranks its name (the name still picks 480p vs 720p).@@ -1112,7 +1116,10 @@ def _resolve_load_family(if path:file_fam = detect_video_family(os.path.basename(path))name, _ = resolve_family_with_content(- fam.name if fam else None, path, "video", file_family = file_fam.name if file_fam else None+ fam.name if fam else None,+ path,+ "video",+ file_family = file_fam.name if file_fam else None,)if name is None:return None@@ -2728,11 +2735,14 @@ class VideoBackend:# Every pick name is judged, not just an unresolved one: a T2V repo id can hold an I2V / VACE file.# A family override names the family for an opaque pick; it cannot make refused weights runnable.from .video_families import unsupported_video_checkpoint_reason+reason = unsupported_video_checkpoint_reason(display_repo_id,repo_id,gguf_filename,- wan21_checkpoint = (gguf_filename or repo_id) if fam and fam.name.startswith("wan2.1") else None,+ wan21_checkpoint = (gguf_filename or repo_id)+ if fam and fam.name.startswith("wan2.1")+ else None,)if reason:raise ValueError(reason)diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.pyindex 8babd1d03f2..783186fbb01 100644--- a/studio/backend/core/inference/video_families.py+++ b/studio/backend/core/inference/video_families.py@@ -591,7 +591,9 @@ _UNSUPPORTED_VIDEO_CHECKPOINTS: tuple[tuple[str, str], ...] = ("faithfully. Load Wan-AI/Wan2.1-T2V-1.3B-Diffusers or Wan-AI/Wan2.1-T2V-14B-Diffusers instead.",),(- r"(?:^|[/\\])(?=[^/\\]*" + _WAN21_TOKEN + r")(?=[^/\\]*(?<![a-z0-9])(?:i2v|flf2v)(?![a-z0-9]))"+ r"(?:^|[/\\])(?=[^/\\]*"+ + _WAN21_TOKEN+ + r")(?=[^/\\]*(?<![a-z0-9])(?:i2v|flf2v)(?![a-z0-9]))"r"[^/\\]*[/\\]*$","This is a Wan2.1 image-to-video checkpoint (I2V-14B 480P / 720P or FLF2V). It generates ""from a start frame through its own image-conditioned transformer and a CLIP vision "@@ -601,7 +603,9 @@ _UNSUPPORTED_VIDEO_CHECKPOINTS: tuple[tuple[str, str], ...] = (),(r"(?:^|[/\\])(?=[^/\\]*" + _WAN21_TOKEN + r")(?=[^/\\]*(?:(?<![a-z0-9])"- r"(?:vace|magref|phantom|multitalk|infinitetalk|controlnet|recammaster)|" + _WAN21_TOKEN + r"-fun)"+ r"(?:vace|magref|phantom|multitalk|infinitetalk|controlnet|recammaster)|"+ + _WAN21_TOKEN+ + r"-fun)"r"(?![a-z0-9]))[^/\\]*[/\\]*$","This is a Wan2.1 conditioning variant (VACE, Fun, MAGREF, Phantom, talking-head, ""ControlNet or ReCamMaster). Its transformer takes extra control inputs the text-to-video "@@ -624,7 +628,9 @@ def unsupported_video_checkpoint_reason(name = re.split(r"[/\\]", needle.rstrip("/\\"))[-1]# A leading "fun" is too generic to mark a Fun release unless it names one (version, size, InP, Control); the# other markers are still judged.- generic_fun = re.match(r"fun(?![a-z0-9])", name) and not re.match(r"fun-(?:v\d|1-3b|14b|inp|control)", name)+ generic_fun = re.match(r"fun(?![a-z0-9])", name) and not re.match(+ r"fun-(?:v\d|1-3b|14b|inp|control)", name+ )needles.append(("wan2-1-t2v-" if generic_fun else "wan2-1-") + name)for needle in needles:for pattern, message in _UNSUPPORTED_VIDEO_CHECKPOINTS:diff --git a/studio/backend/core/rag/config.py b/studio/backend/core/rag/config.pyindex 87dba7bf85d..3f1832e8084 100644--- a/studio/backend/core/rag/config.py+++ b/studio/backend/core/rag/config.py@@ -88,6 +88,8 @@ MAX_UPLOAD_BYTES = int(os.environ.get("RAG_MAX_UPLOAD_BYTES", str(200 * 1024 * 1# Caps prevent an accidentally broad linked folder from becoming an unbounded ingestion queue.FOLDER_SYNC_INTERVAL_S = float(os.environ.get("RAG_FOLDER_SYNC_INTERVAL_S", "30"))FOLDER_MAX_FILES = int(os.environ.get("RAG_FOLDER_MAX_FILES", "10000"))+# Linked-folder plain-text files above this are logs/data dumps, not documents; 0 disables.+FOLDER_MAX_TEXT_BYTES = int(os.environ.get("RAG_FOLDER_MAX_TEXT_BYTES", str(20 * 1024 * 1024)))FOLDER_JOB_HISTORY_LIMIT = int(os.environ.get("RAG_FOLDER_JOB_HISTORY_LIMIT", "200"))# Linked-folder documents ingested concurrently, clamped to 1..4.FOLDER_INGEST_WORKERS = int(os.environ.get("RAG_FOLDER_INGEST_WORKERS", "2"))diff --git a/studio/backend/core/rag/folder_sync.py b/studio/backend/core/rag/folder_sync.pyindex da4db383569..1bdabc402ca 100644--- a/studio/backend/core/rag/folder_sync.py+++ b/studio/backend/core/rag/folder_sync.py@@ -31,7 +31,7 @@ from pathlib import Path, PurePosixPathfrom core.rag import account_db as rag_dbfrom utils.paths import ensure_dir, rag_uploads_root-from . import config, ingestion, job_leases, store+from . import config, gitignore, ingestion, job_leases, storefrom utils.paths.path_utils import is_appledouble_metadatalogger = logging.getLogger(__name__)@@ -90,6 +90,53 @@ def _is_ignored_scan_dir(name: str, path: str) -> bool:)+# Build output is skipped only beside a project manifest: a bare "build" folder in a vault is content.+_BUILD_OUTPUT_DIRS = frozenset({"dist", "build", "out", "target", "coverage", "htmlcov", "vendor"})+_PROJECT_MANIFESTS = (+ "package.json",+ "pyproject.toml",+ "setup.py",+ "setup.cfg",+ "Cargo.toml",+ "pom.xml",+ "build.gradle",+ "build.gradle.kts",+ "go.mod",+ "composer.json",+ "CMakeLists.txt",+ "Gemfile",+ "mix.exs",+)+_MAX_REPORTED_FAILURES = 500+++def _is_build_output_dir(name: str, parent: str) -> bool:+ return name.lower() in _BUILD_OUTPUT_DIRS and any(+ os.path.isfile(os.path.join(parent, manifest)) for manifest in _PROJECT_MANIFESTS+ )+++def _read_gitignore(directory: str, base: str) -> list[gitignore.Rule]:+ """An unreadable .gitignore fails the scan: indexing without it would take in excluded files."""+ path = os.path.join(directory, ".gitignore")+ if not os.path.isfile(path) or os.path.islink(path):+ return []+ shown = f"{base}/.gitignore" if base else ".gitignore"+ try:+ with open(path, "rb") as handle:+ data = handle.read(gitignore.MAX_GITIGNORE_BYTES + 1)+ except OSError as exc:+ raise RuntimeError(f"Couldn't read {shown}: {exc.strerror or exc}") from exc+ if len(data) > gitignore.MAX_GITIGNORE_BYTES:+ raise RuntimeError(f"{shown} is larger than {gitignore.MAX_GITIGNORE_BYTES // 1024} KB")+ # utf-8-sig drops a leading BOM, as git does (dir.c add_patterns_from_buffer, skip_utf8_bom).+ return gitignore.parse(data.decode("utf-8-sig", "replace"), base)+++def _is_plain_text_ext(ext: str) -> bool:+ return ext == ".txt" or ext in config.SOURCE_TEXT_EXTS++def _is_ignored_scan_file(name: str) -> bool:# .env and Terraform state can expose plaintext secrets in retrieval; lockfiles add noise.lower = name.lower()@@ -1052,28 +1099,44 @@ def job_events(job_id: str):def _scan(- root: str, expected_identity: tuple[int, int] | None = None+ root: str,+ expected_identity: tuple[int, int] | None = None,+ report: dict[str, int] | None = None,) -> tuple[dict[str, dict], tuple[int, int]]:+ """Map relative path -> metadata; `report` receives counts of deliberately skipped entries."""from hub.storage.scan_folders import contains_sensitive_path_component, is_denied_system_pathidentity = _root_identity(root)if expected_identity is not None and identity != expected_identity:raise RuntimeError("Linked folder root identity changed")+ skipped = {"gitignored": 0, "build_dirs": 0, "too_large": 0, "over_limit": 0}found: dict[str, dict] = {}mount_points = _mount_points()root_identities = frozenset({identity}) if identity[1] not in (None, 0) else frozenset()- pending = [(root, frozenset({_path_key(root)}), root_identities, 0)]+ pending = [(root, frozenset({_path_key(root)}), root_identities, 0, ())]+ # Collect past the cap so the kept set is the sorted first FOLDER_MAX_FILES, stable across scans.+ hard_cap = config.FOLDER_MAX_FILES * 2 if config.FOLDER_MAX_FILES else 0while pending:- directory, ancestor_paths, ancestor_identities, depth = pending.pop()+ directory, ancestor_paths, ancestor_identities, depth, rules = pending.pop()+ directory_rel = os.path.relpath(directory, root).replace(os.sep, "/")+ directory_rel = "" if directory_rel == "." else directory_rel+ rules = (*rules, *_read_gitignore(directory, directory_rel))with os.scandir(directory) as entries:for entry in entries:full = entry.pathif entry.is_symlink():continue+ rel = os.path.relpath(full, root).replace(os.sep, "/")if entry.is_dir(follow_symlinks = False):if _is_ignored_scan_dir(entry.name, full):continue+ if rules and gitignore.is_ignored(rel, True, rules):+ skipped["gitignored"] += 1+ continue+ if _is_build_output_dir(entry.name, directory):+ skipped["build_dirs"] += 1+ continueresolved = os.path.realpath(full)if (not _is_within(root, resolved)@@ -1106,6 +1169,7 @@ def _scan(ancestor_paths | {resolved_key},child_identities,depth + 1,+ rules,))continue@@ -1113,7 +1177,11 @@ def _scan(continueif _is_ignored_scan_file(entry.name):continue- if os.path.splitext(entry.name)[1].lower() not in config.UPLOAD_EXTS:+ ext = os.path.splitext(entry.name)[1].lower()+ if ext not in config.UPLOAD_EXTS:+ continue+ if rules and gitignore.is_ignored(rel, False, rules):+ skipped["gitignored"] += 1continue# Finder metadata carries the document's extension, so a text parser would embed and cite it as a# real chunk.@@ -1132,7 +1200,15 @@ def _scan(# authoritative for# deletion.pass- rel = os.path.relpath(full, root).replace(os.sep, "/")+ if st.st_size == 0:+ continue+ if (+ config.FOLDER_MAX_TEXT_BYTES+ and _is_plain_text_ext(ext)+ and st.st_size > config.FOLDER_MAX_TEXT_BYTES+ ):+ skipped["too_large"] += 1+ continuefound[rel] = {"path": full,"size_bytes": st.st_size,@@ -1143,12 +1219,19 @@ def _scan(# WebDAV drivers report different ids per call path."identity_from_path": from_path,}- if config.FOLDER_MAX_FILES and len(found) > config.FOLDER_MAX_FILES:+ if hard_cap and len(found) > hard_cap:raise RuntimeError(- f"Folder contains more than the {config.FOLDER_MAX_FILES} supported files limit"+ f"Folder contains more than twice the {config.FOLDER_MAX_FILES} supported "+ "files limit. Link a smaller folder, or add a .gitignore for what to skip.")if _root_identity(root) != identity:raise RuntimeError("Linked folder root identity changed during scan")+ if config.FOLDER_MAX_FILES and len(found) > config.FOLDER_MAX_FILES:+ kept = sorted(found)[: config.FOLDER_MAX_FILES]+ skipped["over_limit"] = len(found) - len(kept)+ found = {rel: found[rel] for rel in kept}+ if report is not None:+ report.update(skipped)return found, identity@@ -1566,7 +1649,8 @@ def _reconcile_folder(job_id: str) -> None:try:_check_running()expected_identity = _load_identity(folder["root_device"], folder["root_inode"])- current, scanned_identity = _scan(folder["path"], expected_identity)+ skipped: dict[str, int] = {}+ current, scanned_identity = _scan(folder["path"], expected_identity, skipped)_check_running()except (_SyncStopped, _SyncCancelled, _LeaseLost):raise@@ -1621,6 +1705,7 @@ def _reconcile_folder(job_id: str) -> None:_set_job(job_id, stage = "ingesting", discovered = len(current), renamed = renamed)added = changed_count = 0failures: list[str] = []+ failure_errors: dict[str, str] = {}withheld: set[str] = set()processed = 0stop_event = getattr(_worker_state, "stop_event", _stop)@@ -1666,9 +1751,12 @@ def _reconcile_folder(job_id: str) -> None:_cleanup_pending_ingestion(pending, result)raise result.errorfailures.append(pending.relative_path)+ reason = _error_text(result.error, folder["path"])+ failure_errors[pending.relative_path] = reasoningest_failed[pending.relative_path] = {"sig": _failure_signature(pending.metadata),"at": time.time(),+ "error": reason,}# A failed file may be a rename or copy of a vanished path, so grant one pass.withheld.update(missing - already_withheld)@@ -1694,6 +1782,7 @@ def _reconcile_folder(job_id: str) -> None:raiseexcept Exception as exc: # noqa: BLE001failures.append(pending.relative_path)+ failure_errors[pending.relative_path] = _error_text(exc, folder["path"])withheld.update(missing - already_withheld)logger.warning("linked-folder ingestion failed for %s",@@ -1754,6 +1843,7 @@ def _reconcile_folder(job_id: str) -> None:raise _SyncCancelledfailures.append(rel)ingest_failed[rel] = known_failed[rel]+ failure_errors[rel] = known_failed[rel].get("error") or "Indexing failed"if not cached_grace:# missing only shrinks during the pass, so once covers every later failurewithheld.update(missing - already_withheld)@@ -1821,8 +1911,9 @@ def _reconcile_folder(job_id: str) -> None:except (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged):_remove_snapshot(snapshot)raise- except Exception:+ except Exception as exc: # noqa: BLE001failures.append(rel)+ failure_errors[rel] = _error_text(exc, folder["path"])withheld.update(missing - already_withheld)logger.warning("linked-folder ingestion failed for %s", rel, exc_info = True)_remove_snapshot(snapshot)@@ -1873,11 +1964,33 @@ def _reconcile_folder(job_id: str) -> None:error = error,completed_at = _now(),)+ scan_report = {+ "skipped": {key: count for key, count in skipped.items() if count},+ "failures": [+ {"path": rel, "error": failure_errors.get(rel, "Indexing failed")}+ for rel in sorted(set(failures))[:_MAX_REPORTED_FAILURES]+ ],+ "failureCount": len(set(failures)),+ }+ # last_change_at moves only when the indexed set did, so idle passes don't make clients refetch.+ changed_sources = added + changed_count + deleted + renamed > 0+ now = _now()with closing(rag_db.get_connection()) as conn:conn.execute(- "UPDATE linked_folders SET status=?, last_error=?, last_scan_at=?, updated_at=? "+ "UPDATE linked_folders SET status=?, last_error=?, last_scan_at=?, updated_at=?, "+ "scan_report=?, last_change_at=CASE WHEN ? THEN ? ELSE COALESCE(last_change_at, ?) END ""WHERE id=? AND delete_remove_index IS NULL",- ("ready" if error is None else "error", error, _now(), _now(), folder["id"]),+ (+ "ready" if error is None else "error",+ error,+ now,+ now,+ json.dumps(scan_report),+ changed_sources,+ now,+ now,+ folder["id"],+ ),)conn.commit()_prune_terminal_jobs(conn)diff --git a/studio/backend/core/rag/gitignore.py b/studio/backend/core/rag/gitignore.pynew file mode 100644index 00000000000..26eac250d12--- /dev/null+++ b/studio/backend/core/rag/gitignore.py@@ -0,0 +1,148 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The .gitignore rules a linked-folder scan honours, per https://git-scm.com/docs/gitignore."""++from __future__ import annotations++import re+from dataclasses import dataclass++MAX_GITIGNORE_BYTES = 256 * 1024+_MAX_GLOBSTARS = 3+++@dataclass(frozen = True)+class Rule:+ base: str # posix directory of the .gitignore, relative to the scan root; "" for the root+ regex: re.Pattern[str]+ negate: bool+ dir_only: bool+++def _tokens(pattern: str) -> list[tuple[str, str]]:+ """(kind, regex) pairs: "star", "char" (one non-"/" character), or "other"."""+ out: list[tuple[str, str]] = []+ i, n = 0, len(pattern)+ while i < n:+ c = pattern[i]+ if c == "*":+ if pattern.startswith("**", i):+ before = i == 0 or pattern[i - 1] == "/"+ after = i + 2 == n or pattern[i + 2] == "/"+ if before and after:+ if i + 2 == n:+ out.append(("other", ".*"))+ else:+ out.append(("other", "(?:.*/)?"))+ i += 1+ i += 2+ continue+ out.append(("star", "[^/]*"))+ elif c == "?":+ out.append(("char", "[^/]"))+ elif c == "[":+ start = i + 1+ if pattern[start : start + 1] in ("!", "^"):+ start += 1+ if pattern[start : start + 1] == "]":+ start += 1+ end = pattern.find("]", start)+ if end == -1:+ out.append(("char", re.escape(c)))+ else:+ body = pattern[i + 1 : end].replace("\\", "\\\\").replace("[", "\\[")+ if body[:1] in ("!", "^"):+ # Like "*" and "?", a negated class never matches "/".+ body = "^/" + body[1:]+ out.append(("char", "[" + body + "]"))+ i = end+ else:+ if c == "\\" and i + 1 < n:+ i += 1+ c = pattern[i]+ out.append(("other" if c == "/" else "char", re.escape(c)))+ i += 1+ return out+++def _translate(pattern: str) -> str:+ tokens = _tokens(pattern)+ out: list[str] = []+ i, n = 0, len(tokens)+ while i < n:+ kind, regex = tokens[i]+ if kind != "star":+ out.append(regex)+ i += 1+ continue+ while i < n and tokens[i][0] == "star":+ i += 1+ end = i+ while end < n and tokens[end][0] == "char":+ end += 1+ chunk = "".join(regex for _, regex in tokens[i:end])+ if end < n and tokens[end][0] == "star":+ # Atomic group (as fnmatch.translate on Python < 3.11): plain [^/]* groups backtrack+ # exponentially on "*a*a*a...b", and the earliest fit is as good as any when "*" follows.+ group = f"g{len(out)}"+ out.append(f"(?=(?P<{group}>[^/]*?{chunk}))(?P={group})")+ else:+ out.append("[^/]*" + chunk)+ i = end+ return "".join(out)+++def parse(text: str, base: str) -> list[Rule]:+ rules: list[Rule] = []+ for raw in text.splitlines():+ line = raw.rstrip("\r")+ stripped = line.rstrip(" ")+ if stripped.endswith("\\") and len(stripped) < len(line):+ stripped += " "+ line = stripped+ if not line or line.startswith("#"):+ continue+ negate = line.startswith("!")+ if negate:+ line = line[1:]+ elif line.startswith(("\\!", "\\#")):+ line = line[1:]+ dir_only = line.endswith("/")+ line = line.rstrip("/")+ if not line:+ continue+ # A leading or middle slash anchors to the .gitignore's directory; else any depth.+ anchored = "/" in line+ line = line.lstrip("/")+ body = _translate(line)+ if not anchored:+ body = "(?:.*/)?" + body+ # Each "**/" multiplies the depths tried; real rules use one or two.+ if body.count("(?:.*/)?") > _MAX_GLOBSTARS:+ continue+ try:+ regex = re.compile(f"^{body}$", re.DOTALL)+ except re.error:+ # A malformed class like "[z-a]" matches nothing in git; it must not fail the scan.+ continue+ rules.append(Rule(base, regex, negate, dir_only))+ return rules+++def is_ignored(rel: str, is_dir: bool, rules: tuple[Rule, ...]) -> bool:+ """Whether `rel` (posix, relative to the scan root) is ignored; the last matching rule wins."""+ ignored = False+ for rule in rules:+ if rule.dir_only and not is_dir:+ continue+ if rule.base:+ prefix = rule.base + "/"+ if not rel.startswith(prefix):+ continue+ sub = rel[len(prefix) :]+ else:+ sub = rel+ if rule.regex.match(sub):+ ignored = not rule.negate+ return ignoreddiff --git a/studio/backend/routes/rag.py b/studio/backend/routes/rag.pyindex c6c8f4c22f3..aae35747bf5 100644--- a/studio/backend/routes/rag.py+++ b/studio/backend/routes/rag.py@@ -327,6 +327,12 @@ def _folder_view(row: dict) -> dict:if row["status"] in {"error", "retired"}else "idle")+ try:+ report = json.loads(row.get("scan_report") or "{}")+ except ValueError:+ report = {}+ if not isinstance(report, dict):+ report = {}return {"id": row["id"],"displayName": row["name"],@@ -335,6 +341,10 @@ def _folder_view(row: dict) -> dict:"status": status,"error": row.get("last_error"),"lastSyncedAt": row.get("last_scan_at"),+ "lastChangedAt": row.get("last_change_at"),+ "skipped": report.get("skipped") or {},+ "failures": report.get("failures") or [],+ "failureCount": report.get("failureCount") or 0,"documentCount": row.get("file_count", 0),"activeJobId": row.get("active_job_id"),"scopeName": row.get("scope_name"),@@ -894,6 +904,7 @@ def _folder_job_view(row: dict) -> dict:"processedFiles": processed,"indexedFiles": (row.get("added") or 0) + (row.get("changed") or 0),"removedFiles": row.get("deleted") or 0,+ "renamedFiles": row.get("renamed") or 0,"failedFiles": row.get("failed") or 0,"error": row.get("error"),"createdAt": row.get("created_at"),diff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.pyindex bcbdec8b89a..2367fae688d 100644--- a/studio/backend/storage/rag_db.py+++ b/studio/backend/storage/rag_db.py@@ -176,6 +176,8 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:last_scan_at TEXT,withheld_paths TEXT,failed_files TEXT,+ scan_report TEXT,+ last_change_at TEXT,created_at TEXT NOT NULL,updated_at TEXT NOT NULL,UNIQUE(scope, path)@@ -298,6 +300,10 @@ def ensure_linked_folder_columns(conn: sqlite3.Connection) -> None:conn.execute("ALTER TABLE linked_folders ADD COLUMN withheld_paths TEXT")if folder_cols and "failed_files" not in folder_cols:conn.execute("ALTER TABLE linked_folders ADD COLUMN failed_files TEXT")+ if folder_cols and "scan_report" not in folder_cols:+ conn.execute("ALTER TABLE linked_folders ADD COLUMN scan_report TEXT")+ if folder_cols and "last_change_at" not in folder_cols:+ conn.execute("ALTER TABLE linked_folders ADD COLUMN last_change_at TEXT")def reset_schema_state_for_tests() -> None:diff --git a/studio/backend/tests/test_mlx_diffusion_backend.py b/studio/backend/tests/test_mlx_diffusion_backend.pyindex bdfcad51c89..e9ee3c9c2e7 100644--- a/studio/backend/tests/test_mlx_diffusion_backend.py+++ b/studio/backend/tests/test_mlx_diffusion_backend.py@@ -1043,7 +1043,11 @@ def test_base_companions_come_from_the_repo_the_plan_staged(monkeypatch):assert fetched == [("unsloth/Z-Image-Turbo", f, "abc") for f in files]-def _companion_snapshot(root, complete = True, tokenizer_config = True):+def _companion_snapshot(+ root,+ complete = True,+ tokenizer_config = True,+):root.mkdir(parents = True, exist_ok = True)for sub in ("text_encoder", "tokenizer", "vae"):(root / sub).mkdir(exist_ok = True)@@ -1120,14 +1124,23 @@ def test_cache_only_companions_skip_an_incomplete_upstream_for_the_staged_mirror"unsloth/Z-Image-Turbo": _companion_snapshot(tmp_path / "mirror"),}- def _snapshot(repo, *, cache_dir = None, local_files_only = False, **_k):+ def _snapshot(+ repo,+ *,+ cache_dir = None,+ local_files_only = False,+ **_k,+ ):assert local_files_onlyif cache_dir != live or repo not in snapshots:raise LocalEntryNotFoundError("not cached")return snapshots[repo]monkeypatch.setattr(huggingface_hub, "snapshot_download", _snapshot)- assert mb.fetch_base_snapshot(BASE, None, local_files_only = True) == snapshots["unsloth/Z-Image-Turbo"]+ assert (+ mb.fetch_base_snapshot(BASE, None, local_files_only = True)+ == snapshots["unsloth/Z-Image-Turbo"]+ )del snapshots["unsloth/Z-Image-Turbo"]with pytest.raises(LocalEntryNotFoundError, match = "complete"):mb.fetch_base_snapshot(BASE, None, local_files_only = True)@@ -1326,13 +1339,22 @@ def test_cache_only_companions_skip_a_snapshot_without_tokenizer_config(monkeypa"unsloth/Z-Image-Turbo": _companion_snapshot(tmp_path / "mirror"),}- def _snapshot(repo, *, cache_dir = None, local_files_only = False, **_k):+ def _snapshot(+ repo,+ *,+ cache_dir = None,+ local_files_only = False,+ **_k,+ ):if cache_dir != live or repo not in snapshots:raise LocalEntryNotFoundError("not cached")return snapshots[repo]monkeypatch.setattr(huggingface_hub, "snapshot_download", _snapshot)- assert mb.fetch_base_snapshot(BASE, None, local_files_only = True) == snapshots["unsloth/Z-Image-Turbo"]+ assert (+ mb.fetch_base_snapshot(BASE, None, local_files_only = True)+ == snapshots["unsloth/Z-Image-Turbo"]+ )def test_a_cancelled_load_keeps_the_companion_mirror_guarded(env, monkeypatch):@@ -1362,7 +1384,9 @@ def test_a_cancelled_load_keeps_the_companion_mirror_guarded(env, monkeypatch):time.sleep(0.01)try:assert "unsloth/Z-Image-Turbo" in backend.draining_repo_ids()- assert "still releasing" in (deletion._diffusion_blocks_delete("unsloth/Z-Image-Turbo") or "")+ assert "still releasing" in (+ deletion._diffusion_blocks_delete("unsloth/Z-Image-Turbo") or ""+ )finally:gate.set()unloader.join(5)diff --git a/studio/backend/tests/test_rag_folder_scan_filters.py b/studio/backend/tests/test_rag_folder_scan_filters.pynew file mode 100644index 00000000000..0c005166784--- /dev/null+++ b/studio/backend/tests/test_rag_folder_scan_filters.py@@ -0,0 +1,261 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""What a linked-folder scan leaves out (.gitignore, build output, oversized text, the file cap),+and what each pass reports back."""++from __future__ import annotations++import json+import time+from contextlib import closing+from pathlib import Path++import pytest++from core.rag import config, folder_sync, gitignore+from storage import rag_db++requires_sqlite_vec = pytest.mark.skipif(+ not rag_db.RAG_AVAILABLE, reason = "sqlite-vec is not installed"+)+++def _ignored(+ rules_text: str,+ rel: str,+ is_dir: bool = False,+ base: str = "",+) -> bool:+ return gitignore.is_ignored(rel, is_dir, tuple(gitignore.parse(rules_text, base)))+++ "rules, rel, is_dir, expected",+ [+ ("*.log", "app.log", False, True),+ ("*.log", "deep/nested/app.log", False, True),+ ("*.log", "app.log.md", False, False),+ ("/root.md", "root.md", False, True),+ ("/root.md", "sub/root.md", False, False),+ ("docs/*.md", "docs/a.md", False, True),+ ("docs/*.md", "docs/sub/a.md", False, False),+ ("docs/*.md", "other/docs/a.md", False, False),+ ("generated/", "generated", True, True),+ ("generated/", "generated", False, False),+ ("**/fixtures", "a/b/fixtures", True, True),+ ("a/**/b.md", "a/b.md", False, True),+ ("a/**/b.md", "a/x/y/b.md", False, True),+ ("logs/**", "logs/today.txt", False, True),+ ("*.md\n!keep.md", "keep.md", False, False),+ ("*.md\n!keep.md", "drop.md", False, True),+ ("# comment\n\n", "anything.md", False, False),+ ("\\#hash.md", "#hash.md", False, True),+ ("file?.md", "file1.md", False, True),+ ("file[0-9].md", "file7.md", False, True),+ ("file[!0-9].md", "file7.md", False, False),+ ("a[!x]b", "a/b", False, False),+ ("*a*b", "xaab", False, True),+ ("*a*b", "xaa/b", False, False),+ ("*.min.*", "app.min.js", False, True),+ ("*.min.*", "app.js", False, False),+ ("a*b*c", "abbc", False, True),+ ("a*b*c", "acbc", False, True),+ ("a*b*c", "acb", False, False),+ ("**/x/**/y/*.md", "p/x/q/r/y/a.md", False, True),+ ],+)+def test_gitignore_patterns(rules, rel, is_dir, expected):+ assert _ignored(rules, rel, is_dir) is expected+++def test_a_malformed_class_drops_only_its_own_rule():+ rules = "[z-a].txt\n*.log"+ assert not _ignored(rules, "a.txt")+ assert _ignored(rules, "debug.log")+++def test_hostile_rules_match_in_bounded_time():+ star_rule = gitignore.parse("*a" * 15 + "b", "")+ globstar_rule = gitignore.parse("a/**/" * 7 + "b", "")+ assert len(star_rule) == 1+ # Too many "**/" to match in bounded time: the rule is left out.+ assert globstar_rule == []+ started = time.monotonic()+ assert not gitignore.is_ignored("a" * 46, False, tuple(star_rule))+ assert time.monotonic() - started < 1+++def test_nested_gitignore_applies_only_below_its_directory():+ assert _ignored("*.md", "sub/a.md", base = "sub")+ assert not _ignored("*.md", "a.md", base = "sub")+ assert not _ignored("*.md", "subway/a.md", base = "sub")+++def source(linkable_temp_base, tmp_path) -> Path:+ root = linkable_temp_base / f"scan-{tmp_path.name}"+ root.mkdir(parents = True)+ return root+++def _write(+ root: Path,+ rel: str,+ text: str = "words",+) -> None:+ path = root / rel+ path.parent.mkdir(parents = True, exist_ok = True)+ path.write_text(text, encoding = "utf-8")+++def test_scan_honours_root_and_nested_gitignore(source):+ _write(source, ".gitignore", "*.log\ngenerated/\n!important.log\n")+ _write(source, "notes.md")+ _write(source, "debug.log")+ _write(source, "important.log")+ _write(source, "generated/out.md")+ _write(source, "pkg/.gitignore", "local.md\n")+ _write(source, "pkg/local.md")+ _write(source, "pkg/kept.md")+ _write(source, "local.md")+ report: dict[str, int] = {}+ found, _ = folder_sync._scan(str(source), report = report)+ assert set(found) == {"notes.md", "important.log", "pkg/kept.md", "local.md"}+ assert report["gitignored"] == 3+++def test_an_unreadable_gitignore_fails_the_scan(source, monkeypatch):+ _write(source, ".gitignore", "secret.md\n")+ _write(source, "secret.md")+ real_open = open++ def locked_open(path, *args, **kwargs):+ if str(path).endswith(".gitignore"):+ raise PermissionError(13, "Permission denied")+ return real_open(path, *args, **kwargs)++ monkeypatch.setattr("builtins.open", locked_open)+ with pytest.raises(RuntimeError, match = "Couldn't read .gitignore"):+ folder_sync._scan(str(source))+++def test_a_bom_does_not_hide_the_first_gitignore_rule(source):+ (source / ".gitignore").write_bytes(b"\xef\xbb\xbfsecret.md\nother.md\n")+ _write(source, "secret.md")+ _write(source, "other.md")+ _write(source, "notes.md")+ found, _ = folder_sync._scan(str(source))+ assert set(found) == {"notes.md"}+++def test_scan_skips_build_output_only_beside_a_project_manifest(source):+ _write(source, "app/package.json", "{}")+ _write(source, "app/dist/bundle.js", "bundled")+ _write(source, "app/src/index.js", "source")+ _write(source, "vault/build/plan.md", "the user's own build notes")+ report: dict[str, int] = {}+ found, _ = folder_sync._scan(str(source), report = report)+ assert set(found) == {"app/package.json", "app/src/index.js", "vault/build/plan.md"}+ assert report["build_dirs"] == 1+++def test_scan_skips_oversized_plain_text_but_not_documents(source, monkeypatch):+ monkeypatch.setattr(config, "FOLDER_MAX_TEXT_BYTES", 10)+ _write(source, "dump.csv", "x" * 50)+ _write(source, "small.csv", "a,b")+ _write(source, "big.md", "x" * 50)+ report: dict[str, int] = {}+ found, _ = folder_sync._scan(str(source), report = report)+ # .md is a document type, not a SOURCE_TEXT_EXTS data dump.+ assert set(found) == {"small.csv", "big.md"}+ assert report["too_large"] == 1+++def test_scan_skips_empty_files(source):+ _write(source, "pkg/__init__.py", "")+ _write(source, "pkg/core.py", "def core(): pass")+ found, _ = folder_sync._scan(str(source))+ assert set(found) == {"pkg/core.py"}+++def test_scan_keeps_the_first_files_in_sorted_order_past_the_cap(source, monkeypatch):+ monkeypatch.setattr(config, "FOLDER_MAX_FILES", 3)+ for name in ("e.md", "a.md", "d.md", "b.md", "c.md"):+ _write(source, name)+ report: dict[str, int] = {}+ found, _ = folder_sync._scan(str(source), report = report)+ assert set(found) == {"a.md", "b.md", "c.md"}+ assert report["over_limit"] == 2+++def test_scan_still_refuses_a_folder_far_past_the_cap(source, monkeypatch):+ monkeypatch.setattr(config, "FOLDER_MAX_FILES", 2)+ for index in range(5):+ _write(source, f"{index}.md")+ with pytest.raises(RuntimeError, match = "supported files limit"):+ folder_sync._scan(str(source))+++def _folder_row(folder_id: str) -> dict:+ with closing(rag_db.get_connection()) as conn:+ return dict(+ conn.execute("SELECT * FROM linked_folders WHERE id=?", (folder_id,)).fetchone()+ )+++def _run(folder_id: str) -> dict:+ job_id = folder_sync.request_sync(folder_id)+ folder_sync.reconcile_folder(job_id)+ return folder_sync.get_job(job_id)+++@requires_sqlite_vec+def test_a_pass_that_changes_nothing_leaves_last_change_at_alone(rag_home, stub_embeddings):+ source = rag_home / "source"+ source.mkdir()+ (source / "notes.txt").write_text("alpha words", encoding = "utf-8")+ folder = folder_sync.create_folder(+ scope_type = "project", scope_id = "p1", path = str(source), name = "Docs"+ )+ assert _run(folder["id"])["status"] == "completed"+ first = _folder_row(folder["id"])+ assert first["last_change_at"]++ assert _run(folder["id"])["status"] == "completed"+ idle = _folder_row(folder["id"])+ assert idle["last_change_at"] == first["last_change_at"]++ (source / "more.txt").write_text("beta words", encoding = "utf-8")+ assert _run(folder["id"])["added"] == 1+ assert _folder_row(folder["id"])["last_change_at"] != first["last_change_at"]+++@requires_sqlite_vec+def test_every_failed_file_is_listed_with_its_reason(rag_home, stub_embeddings, monkeypatch):+ source = rag_home / "source"+ source.mkdir()+ for index in range(5):+ (source / f"bad{index}.txt").write_text("words", encoding = "utf-8")+ folder = folder_sync.create_folder(+ scope_type = "project", scope_id = "p1", path = str(source), name = "Docs"+ )+ monkeypatch.setattr(+ folder_sync.ingestion,+ "start_ingestion",+ lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("embed unavailable")),+ )+ job = _run(folder["id"])+ assert job["status"] == "failed"+ assert "and 2 more" in job["error"]++ report = json.loads(_folder_row(folder["id"])["scan_report"])+ assert report["failureCount"] == 5+ assert [entry["path"] for entry in report["failures"]] == [f"bad{i}.txt" for i in range(5)]+ assert all(entry["error"] == "embed unavailable" for entry in report["failures"])++ from routes.rag import _folder_view++ view = _folder_view(_folder_row(folder["id"]))+ assert view["failureCount"] == 5+ assert view["failures"][0] == {"path": "bad0.txt", "error": "embed unavailable"}diff --git a/studio/backend/tests/test_rag_linked_folders.py b/studio/backend/tests/test_rag_linked_folders.pyindex 8240c2d7390..efd51c310de 100644--- a/studio/backend/tests/test_rag_linked_folders.py+++ b/studio/backend/tests/test_rag_linked_folders.py@@ -1013,8 +1013,12 @@ def test_large_windows_file_identity_round_trips_through_sqlite(identity = (1 << 63, 1 << 127)real_scan, real_snapshot = folder_sync._scan, folder_sync._snapshot- def scan(root, expected_identity = None):- found, root_identity = real_scan(root, expected_identity)+ def scan(+ root,+ expected_identity = None,+ report = None,+ ):+ found, root_identity = real_scan(root, expected_identity, report)for metadata in found.values():metadata["scanned"], (metadata["device"], metadata["inode"]) = ((metadata["device"], metadata["inode"]),diff --git a/studio/backend/tests/test_video_wan21_families.py b/studio/backend/tests/test_video_wan21_families.pyindex 373801de49b..a93616a26d9 100644--- a/studio/backend/tests/test_video_wan21_families.py+++ b/studio/backend/tests/test_video_wan21_families.py@@ -183,7 +183,10 @@ def test_comfyui_template_defaults():(("wan21-t2v-1.3b-Q8_0.gguf",), (30, 6.0)),(("Wan21_T2V_14B_fp8.safetensors",), (20, 6.0)),(("someone/wan21-14b-gguf",), (20, 6.0)),- (("Wan2.1_T2V_14B_FusionX-Q4_K_M.gguf", "QuantStack/Wan2.1_T2V_14B_FusionX-GGUF"), (8, 1.0)),+ (+ ("Wan2.1_T2V_14B_FusionX-Q4_K_M.gguf", "QuantStack/Wan2.1_T2V_14B_FusionX-GGUF"),+ (8, 1.0),+ ),(("wan21-lab/HunyuanVideo-1.5-Diffusers-480p_t2v",), (20, 6.0)),(("fusionx-lab/HunyuanVideo-1.5-Diffusers-480p_t2v",), (20, 6.0)),(("/models/FusionX/wan2.1-t2v-14b-Q4_K_M.gguf",), (20, 6.0)),@@ -278,12 +281,12 @@ def test_file_name_outranks_the_folder_for_a_14b_header(tmp_path):folder.mkdir()_write_safetensors(folder / "Wan2.2-T2V-A14B-HighNoise-fp8.safetensors", shapes)_write_safetensors(folder / "model-fp8.safetensors", shapes)- assert _detect_load_family(str(folder), "Wan2.2-T2V-A14B-HighNoise-fp8.safetensors", None).name == (- "wan2.2-t2v-a14b"- )- assert _detect_load_family(str(folder / "Wan2.2-T2V-A14B-HighNoise-fp8.safetensors"), None, None).name == (- "wan2.2-t2v-a14b"- )+ assert _detect_load_family(+ str(folder), "Wan2.2-T2V-A14B-HighNoise-fp8.safetensors", None+ ).name == ("wan2.2-t2v-a14b")+ assert _detect_load_family(+ str(folder / "Wan2.2-T2V-A14B-HighNoise-fp8.safetensors"), None, None+ ).name == ("wan2.2-t2v-a14b")assert _detect_load_family(str(folder), "model-fp8.safetensors", None).name == "wan2.1-t2v-14b"@@ -300,9 +303,10 @@ def test_file_name_outranks_the_folder_for_a_14b_header(tmp_path):("org/LTX-2-GGUF", "Wan2.1-T2V-1.3B-Q8_0.gguf", "ltx-2"),],)-def test_remote_file_name_outranks_the_repo_for_same_layout_wan(repo, filename, expected, monkeypatch):+def test_remote_file_name_outranks_the_repo_for_same_layout_wan(+ repo, filename, expected, monkeypatch+):import core.inference.video as video-monkeypatch.setattr(video, "_picked_gguf_arch", lambda *a, **k: None)assert video._detect_load_family(repo, filename, None).name == expected@@ -315,10 +319,11 @@ def test_remote_file_name_outranks_the_repo_for_same_layout_wan(repo, filename,("org/Wan2.1-T2V-14B-GGUF", "model-Q8_0.gguf", "wan2.1-t2v-14b"),],)-def test_remote_file_name_naming_the_other_wan21_size_outranks_the_repo(repo, filename, expected, monkeypatch):+def test_remote_file_name_naming_the_other_wan21_size_outranks_the_repo(+ repo, filename, expected, monkeypatch+):# No header pass for a remote pick, so the file's own size token decides which base config builds the transformer.import core.inference.video as video-monkeypatch.setattr(video, "_picked_gguf_arch", lambda *a, **k: None)assert video._detect_load_family(repo, filename, None).name == expected@@ -442,12 +447,36 @@ def test_load_validation_refuses_with_the_named_reason():{"gguf_filename": "FastWan2.1-T2V-1.3B-Q8_0.gguf", "model_kind": "gguf"},_FASTWAN,),- ("city96/Wan2.1-T2V-14B-gguf", {"gguf_filename": "i2v-14b-480p-Q4_K_M.gguf", "model_kind": "gguf"}, _I2V),- ("someorg/Wan2.1-T2V-1.3B-mix", {"gguf_filename": "vace-1.3b-Q8_0.gguf", "model_kind": "gguf"}, _VARIANT),- ("someorg/Wan2.1-T2V-1.3B-mix", {"gguf_filename": "fun-inp-1.3b-Q8_0.gguf", "model_kind": "gguf"}, _VARIANT),- ("someorg/Wan2.1-T2V-14B-GGUF", {"gguf_filename": "fun-14b-control-Q4_K_M.gguf", "model_kind": "gguf"}, _VARIANT),- ("someorg/Wan2.1-T2V-1.3B-mix", {"gguf_filename": "fun-v1.1-1.3b-inp-Q8_0.gguf", "model_kind": "gguf"}, _VARIANT),- ("city96/Wan2.1-T2V-14B-gguf", {"gguf_filename": "fun-style-i2v-14b-Q4_K_M.gguf", "model_kind": "gguf"}, _I2V),+ (+ "city96/Wan2.1-T2V-14B-gguf",+ {"gguf_filename": "i2v-14b-480p-Q4_K_M.gguf", "model_kind": "gguf"},+ _I2V,+ ),+ (+ "someorg/Wan2.1-T2V-1.3B-mix",+ {"gguf_filename": "vace-1.3b-Q8_0.gguf", "model_kind": "gguf"},+ _VARIANT,+ ),+ (+ "someorg/Wan2.1-T2V-1.3B-mix",+ {"gguf_filename": "fun-inp-1.3b-Q8_0.gguf", "model_kind": "gguf"},+ _VARIANT,+ ),+ (+ "someorg/Wan2.1-T2V-14B-GGUF",+ {"gguf_filename": "fun-14b-control-Q4_K_M.gguf", "model_kind": "gguf"},+ _VARIANT,+ ),+ (+ "someorg/Wan2.1-T2V-1.3B-mix",+ {"gguf_filename": "fun-v1.1-1.3b-inp-Q8_0.gguf", "model_kind": "gguf"},+ _VARIANT,+ ),+ (+ "city96/Wan2.1-T2V-14B-gguf",+ {"gguf_filename": "fun-style-i2v-14b-Q4_K_M.gguf", "model_kind": "gguf"},+ _I2V,+ ),],)def test_unsupported_file_under_a_t2v_repo_is_refused(repo, kwargs, reason):@@ -519,6 +548,7 @@ def test_vae_untiled_tables_cover_wan21():)def test_anyflow_is_refused(identifier):from core.inference.video import VideoBackend+assert _ANYFLOW in vf.unsupported_video_checkpoint_reason(identifier)assert vf.detect_video_family(identifier) is Nonewith pytest.raises(ValueError, match = _ANYFLOW):@@ -537,7 +567,11 @@ def test_anyflow_owner_or_folder_does_not_refuse():{"gguf_filename": "wan2.1-i2v-14b-480p-Q4_K_M.gguf", "model_kind": "gguf"},_I2V,),- ("someorg/opaque-gguf", {"gguf_filename": "i2v-14b-480p-Q4_K_M.gguf", "model_kind": "gguf"}, _I2V),+ (+ "someorg/opaque-gguf",+ {"gguf_filename": "i2v-14b-480p-Q4_K_M.gguf", "model_kind": "gguf"},+ _I2V,+ ),("FastVideo/FastWan2.1-T2V-1.3B-Diffusers", {}, "FastVideo FastWan checkpoint"),("nvidia/AnyFlow-Wan2.1-T2V-14B-Diffusers", {}, _ANYFLOW),],@@ -551,8 +585,13 @@ def test_family_override_does_not_bypass_the_refusal(repo, kwargs, reason):@pytest.mark.parametrize("folder", ["i2v", "vace", "fun"])def test_wan21_context_is_added_only_to_the_picked_file(tmp_path, folder):from core.inference.video import VideoBackend+(tmp_path / folder).mkdir()- _write_gguf(tmp_path / folder / "Wan2.1-T2V-1.3B-Q8_0.gguf", _row("Wan2.1-T2V-1.3B-Q8_0.gguf")["shapes"], "wan")+ _write_gguf(+ tmp_path / folder / "Wan2.1-T2V-1.3B-Q8_0.gguf",+ _row("Wan2.1-T2V-1.3B-Q8_0.gguf")["shapes"],+ "wan",+ )fam = VideoBackend().validate_load_request(str(tmp_path / folder), gguf_filename = "Wan2.1-T2V-1.3B-Q8_0.gguf", model_kind = "gguf")diff --git a/studio/frontend/scripts/check-bundle-budget.ts b/studio/frontend/scripts/check-bundle-budget.tsindex ca73ba67b82..25f1610ca15 100644--- a/studio/frontend/scripts/check-bundle-budget.ts+++ b/studio/frontend/scripts/check-bundle-budget.ts@@ -50,8 +50,12 @@ export const BUDGET = {// The long-chat render work (#13255: deferred fences and reasoning rows) adds 33.3 KB raw / 13.1 KB// transfer to the eager chat chunk (6,477.1 -> 6,510.4 KB raw, 1,942.6 -> 1,955.7 KB transfer at// 160cb41fe, one machine and build); raised by that so main's margin stays where it was.- transferBytes: 2_005_000,- rawBytes: 6_681_000,+ // Project sources follow-ups (#13224: queued drops, retry on failed sources, folder drops) add+ // 6.6 KB raw / 2.0 KB transfer, mostly in the eager chat chunk (6,513.0 -> 6,519.6 KB raw,+ // 1,956.4 -> 1,958.4 KB transfer against d4ea5a51d, one machine and build). Lazy-loading the+ // folder walker recovered 0.5 KB raw and no transfer, so the budget is raised by that growth.+ transferBytes: 2_007_100,+ rawBytes: 6_687_800,};// The chunk count is reported but not budgeted. Splitting a page out of the entry raises it while lowering thediff --git a/studio/frontend/src/features/native-intents/use-native-file-drop.ts b/studio/frontend/src/features/native-intents/use-native-file-drop.tsindex 7150f937688..9d82fb1843f 100644--- a/studio/frontend/src/features/native-intents/use-native-file-drop.ts+++ b/studio/frontend/src/features/native-intents/use-native-file-drop.ts@@ -2,6 +2,7 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import { isTauri } from "@/lib/api-base";+import { MAX_FOLDER_FILES, filesFromDrop } from "@/lib/dropped-folders";import { toast } from "@/lib/toast";import type React from "react";import { useCallback, useRef, useState } from "react";@@ -39,12 +40,14 @@ function hasAcceptedExt(name: string, exts: string[]): boolean {function toastNothingAccepted(names: string[],accept: string | undefined,+ folderHint?: string,): void {// A folder is one extension-less name, and the native side takes files only.const looksLikeFolder = names.some((name) => !name.includes("."));if (looksLikeFolder) {toast.error("Folders can't be dropped here", {- description: "Drop the files inside it, or use the picker button.",+ description:+ folderHint ?? "Drop the files inside it, or use the picker button.",});return;}@@ -125,6 +128,7 @@ export interface NativeFileDropOptions {multiple?: boolean;/** Register under a policy other than the attachment one (datasets, models). */register?: (path: string) => Promise<NativeIntent>;+ folders?: boolean;}export interface NativeFileDrop {@@ -154,13 +158,16 @@ export function useNativeFileDrop(// dragenter/dragleave fire per child, so a raw boolean flickers on inner moves.const dragDepth = useRef(0);- const deliver = useCallback((files: File[]) => {- const current = latest.current;- if (files.length === 0) return;- void current.onFiles(- current.multiple === false ? files.slice(0, 1) : files,- );- }, []);+ // Options from drop time: `latest` after an await may name another destination.+ const deliver = useCallback(+ (files: File[], current: NativeFileDropOptions) => {+ if (files.length === 0) return;+ void current.onFiles(+ current.multiple === false ? files.slice(0, 1) : files,+ );+ },+ [],+ );const handleNativePaths = useCallback(async (paths: string[]) => {@@ -176,7 +183,13 @@ export function useNativeFileDrop(hasAcceptedExt(nativeFileName(path), exts),);if (supported.length === 0) {- toastNothingAccepted(paths.map(nativeFileName), current.accept);+ toastNothingAccepted(+ paths.map(nativeFileName),+ current.accept,+ current.folders+ ? "Use Link folder to keep a whole folder indexed and in sync."+ : undefined,+ );return;}const takeIntents = current.onNativeIntents;@@ -189,7 +202,7 @@ export function useNativeFileDrop(if (takeIntents) {void takeIntents(ready as NativeIntent[]);} else {- deliver(ready as File[]);+ deliver(ready as File[], current);}}if (failed > 0) {@@ -254,19 +267,42 @@ export function useNativeFileDrop(return;}const exts = acceptedExts(current.accept);- const dropped = Array.from(event.dataTransfer.files ?? []);- const supported = dropped.filter((file) =>- hasAcceptedExt(file.name, exts),- );- if (dropped.length > 0 && supported.length === 0) {- toastNothingAccepted(- dropped.map((file) => file.name),- current.accept,+ const take = (dropped: File[]) => {+ const supported = dropped.filter((file) =>+ hasAcceptedExt(file.name, exts),);+ if (dropped.length > 0 && supported.length === 0) {+ toastNothingAccepted(+ dropped.map((file) => file.name),+ current.accept,+ );+ return;+ }+ deliver(supported, current);+ toastPartiallySkipped(dropped.length - supported.length);+ };+ if (!current.folders) {+ take(Array.from(event.dataTransfer.files ?? []));return;}- deliver(supported);- toastPartiallySkipped(dropped.length - supported.length);+ void filesFromDrop(event.dataTransfer, (name) =>+ hasAcceptedExt(name, exts),+ ).then(+ ({ files, truncated, hadFolder }) => {+ if (hadFolder && files.length === 0) {+ toast.error("That folder has no files to add");+ return;+ }+ take(files);+ if (truncated > 0) {+ toast.info(`Added the first ${MAX_FOLDER_FILES} files`, {+ description:+ "The rest were left out. Drop a smaller folder for them.",+ });+ }+ },+ (reason) => toastReadFailures(1, reason),+ );},};diff --git a/studio/frontend/src/features/rag/components/document-status-chip.tsx b/studio/frontend/src/features/rag/components/document-status-chip.tsxindex 4cd16b95c4a..1c635ee5f67 100644--- a/studio/frontend/src/features/rag/components/document-status-chip.tsx+++ b/studio/frontend/src/features/rag/components/document-status-chip.tsx@@ -25,6 +25,7 @@ export function DocumentStatusChip({stage,error,onRemove,+ onRetry,shared = false,}: {filename: string;@@ -33,6 +34,7 @@ export function DocumentStatusChip({stage?: string | null;error?: string | null;onRemove?: () => void;+ onRetry?: () => void;/** Indexed for the whole project rather than this one chat: swap the file* glyph for a folder so the two scopes are told apart at a glance. */shared?: boolean;@@ -62,6 +64,16 @@ export function DocumentStatusChip({className="size-3 shrink-0"/><span className="truncate">{filename}</span>+ {status === "failed" && onRetry ? (+ <button+ type="button"+ onClick={onRetry}+ aria-label={`Retry ${filename}`}+ className="shrink-0 text-ui-10 font-medium underline-offset-2 hover:underline"+ >+ Retry+ </button>+ ) : null}{/* spinner while indexing, else close button */}{processing ? (<span className="flex shrink-0 items-center gap-1 text-ui-10 text-muted-foreground">diff --git a/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx b/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsxindex e7089b56cda..3a19824c838 100644--- a/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx+++ b/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx@@ -49,6 +49,7 @@ import { LinkedFoldersManager } from "./linked-folders-manager";import { RAG_SOURCE_UPLOAD_ACCEPT } from "./source-drop-policy";import { type RagUploadItem, useRagDocuments } from "./use-rag-documents";import { useSourceDrop } from "./use-source-drop";+import { useUploadQueue } from "./use-upload-queue";type View =| { kind: "list" }@@ -440,19 +441,19 @@ function KnowledgeBaseDocuments({onBack: () => void;}) {const lister = useCallback(() => listKnowledgeBaseDocuments(kb.id), [kb.id]);- const { documents, loading, uploading, refresh, upload, remove } =+ const { documents, loading, uploading, refresh, upload, remove, retry, canRetry } =useRagDocuments({ type: "kb", kbId: kb.id }, lister);const fileInputRef = useRef<HTMLInputElement>(null);const handleLinkedSourcesChanged = useCallback(() => {void refresh({ quiet: true });}, [refresh]);+ const uploadNow = useCallback(+ (items: RagUploadItem[]) => void upload(items),+ [upload],+ );+ const { enqueue } = useUploadQueue(uploadNow, uploading, kb.id);const { dragging, dropProps, nativeDropTarget } = useSourceDrop({- onItems: (items) => void upload(items),- // upload() tracks one run at a time, so a second batch would clear the- // in-flight guard the first one is still relying on.- disabledReason: uploading- ? "An upload is already running. Add these when it finishes."- : undefined,+ onItems: enqueue,});// Deferred a tick: the hook's unmount cleanup aborts an upload started on StrictMode's@@ -478,11 +479,7 @@ function KnowledgeBaseDocuments({<ChevronLeftIcon className="size-4" />All knowledge bases</Button>- <Button- size="sm"- onClick={() => fileInputRef.current?.click()}- disabled={uploading}- >+ <Button size="sm" onClick={() => fileInputRef.current?.click()}>{uploading ? <Spinner /> : <UploadIcon className="size-3.5" />}Upload</Button>@@ -493,8 +490,9 @@ function KnowledgeBaseDocuments({accept={RAG_SOURCE_UPLOAD_ACCEPT}className="hidden"onChange={(e) => {- if (e.target.files?.length) void upload(e.target.files);+ const files = Array.from(e.target.files ?? []);e.target.value = "";+ enqueue(files.map((file) => ({ kind: "file" as const, file })));}}/></div>@@ -528,10 +526,12 @@ function KnowledgeBaseDocuments({stage={doc.stage}error={doc.error}onRemove={- doc.id.startsWith("pending_") || isLinkedFolderManaged(doc)+ isLinkedFolderManaged(doc) ||+ (doc.id.startsWith("pending_") && doc.status !== "failed")? undefined: () => void remove(doc.id)}+ onRetry={canRetry(doc.id) ? () => void retry(doc.id) : undefined}/>))}</div>diff --git a/studio/frontend/src/features/rag/components/linked-folder-groups.ts b/studio/frontend/src/features/rag/components/linked-folder-groups.tsnew file mode 100644index 00000000000..17d725ddd34--- /dev/null+++ b/studio/frontend/src/features/rag/components/linked-folder-groups.ts@@ -0,0 +1,50 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import { useEffect, useState } from "react";+import { listLinkedFolders } from "../api/rag-api";+import type { RagDocument } from "../types/rag";++export function useLinkedFolderNames(+ projectId: string | null,+ folderIds: string,+): ReadonlyMap<string, string> {+ const [names, setNames] = useState<ReadonlyMap<string, string>>(new Map());+ useEffect(() => {+ if (!projectId || !folderIds) return;+ let cancelled = false;+ listLinkedFolders({ type: "project", id: projectId })+ .then((folders) => {+ if (cancelled) return;+ setNames(new Map(folders.map((f) => [f.id, f.displayName])));+ })+ .catch(() => {+ });+ return () => {+ cancelled = true;+ };+ }, [projectId, folderIds]);+ return names;+}++/** A linked folder can hold thousands of files, so it is drawn as one item. */+export function groupByLinkedFolder<T extends RagDocument>(+ docs: T[],+): {+ loose: T[];+ folders: [string, T[]][];+} {+ const loose: T[] = [];+ const folders = new Map<string, T[]>();+ for (const doc of docs) {+ const folderId = doc.linkedFolderId;+ if (!folderId) {+ loose.push(doc);+ continue;+ }+ const group = folders.get(folderId);+ if (group) group.push(doc);+ else folders.set(folderId, [doc]);+ }+ return { loose, folders: [...folders] };+}diff --git a/studio/frontend/src/features/rag/components/linked-folders-manager.tsx b/studio/frontend/src/features/rag/components/linked-folders-manager.tsxindex 1598a4ee6bd..57c12f16324 100644--- a/studio/frontend/src/features/rag/components/linked-folders-manager.tsx+++ b/studio/frontend/src/features/rag/components/linked-folders-manager.tsx@@ -27,7 +27,12 @@ import { FolderAddIcon, FolderSyncIcon } from "@hugeicons/core-free-icons";import { HugeiconsIcon } from "@hugeicons/react";import { MoreHorizontalIcon } from "lucide-react";import { useState } from "react";-import type { FolderSyncJob, LinkedFolderScope } from "../types/rag";+import type {+ FolderSyncJob,+ LinkedFolder,+ LinkedFolderScope,+ LinkedFolderSkipReason,+} from "../types/rag";import { useLinkedFolders } from "./use-linked-folders";/** Grouped folder card, shared with the create project dialog. */@@ -56,6 +61,73 @@ function jobSummary(job: FolderSyncJob): string {: `${processed} of ${discovered} files`;}+const SKIP_LABELS: Record<LinkedFolderSkipReason, (count: number) => string> = {+ gitignored: (n) => `${n} ignored by .gitignore`,+ build_dirs: (n) => `${n} build folder${n === 1 ? "" : "s"}`,+ too_large: (n) => `${n} large data file${n === 1 ? "" : "s"}`,+ over_limit: (n) => `${n} past the file limit`,+};++export function skippedSummary(folder: LinkedFolder): string | null {+ const parts = (Object.keys(SKIP_LABELS) as LinkedFolderSkipReason[]).flatMap(+ (reason) => {+ const count = folder.skipped?.[reason] ?? 0;+ return count > 0 ? [SKIP_LABELS[reason](count)] : [];+ },+ );+ return parts.length > 0 ? `Skipped ${parts.join(" · ")}` : null;+}++function FolderReport({ folder }: { folder: LinkedFolder }) {+ const [open, setOpen] = useState(false);+ const skipped = skippedSummary(folder);+ const failures = folder.failures ?? [];+ const failureCount = folder.failureCount ?? failures.length;+ if (!skipped && failureCount === 0) return null;+ return (+ <div className="mt-1 flex flex-col gap-1 text-ui-11 text-muted-foreground">+ {skipped ? (+ <p+ className="truncate"+ title={`${skipped}. Hidden folders, dependencies and secrets are never indexed.`}+ >+ {skipped}+ </p>+ ) : null}+ {failureCount > 0 ? (+ <>+ <button+ type="button"+ aria-expanded={open}+ onClick={() => setOpen((value) => !value)}+ className="self-start text-destructive underline-offset-2 hover:underline"+ >+ {open ? "Hide" : "Show"} {failureCount} file+ {failureCount === 1 ? "" : "s"} that couldn't be indexed+ </button>+ {open ? (+ <ul className="max-h-48 overflow-y-auto rounded-lg bg-muted/40 px-2.5 py-1.5">+ {failures.map((failure) => (+ <li key={failure.path} className="flex min-w-0 gap-2 py-0.5">+ <span className="min-w-0 shrink truncate font-mono" title={failure.path}>+ {failure.path}+ </span>+ <span className="min-w-0 flex-1 truncate" title={failure.error}>+ {failure.error}+ </span>+ </li>+ ))}+ {failureCount > failures.length ? (+ <li className="py-0.5">and {failureCount - failures.length} more</li>+ ) : null}+ </ul>+ ) : null}+ </>+ ) : null}+ </div>+ );+}+export function LinkedFoldersManager({scope,compact = false,@@ -233,7 +305,9 @@ export function LinkedFoldersManager({aria-label={`Sync progress for ${folder.displayName}`}className="mt-1.5 h-1"/>- ) : null}+ ) : (+ <FolderReport folder={folder} />+ )}</div>{folderMenu(folder, running)}</div>@@ -383,7 +457,9 @@ export function LinkedFoldersManager({aria-label={`Sync progress for ${folder.displayName}`}className="mt-2 h-1.5"/>- ) : null}+ ) : (+ <FolderReport folder={folder} />+ )}</div>{folderMenu(folder, running)}</li>diff --git a/studio/frontend/src/features/rag/components/project-sources-panel.tsx b/studio/frontend/src/features/rag/components/project-sources-panel.tsxindex 64279e10c2b..c99b06c1b2d 100644--- a/studio/frontend/src/features/rag/components/project-sources-panel.tsx+++ b/studio/frontend/src/features/rag/components/project-sources-panel.tsx@@ -4,10 +4,13 @@import { Button } from "@/components/ui/button";import { useNativeFileDrop } from "@/features/native-intents";import type { NativeIntent } from "@/features/native-intents";+import { isTauri } from "@/lib/api-base";+import { MAX_FOLDER_FILES, openFolderPicker } from "@/lib/dropped-folders";import { FolderPlusIcon } from "@/lib/hugeicons-derived";+import { toast } from "@/lib/toast";import { cn } from "@/lib/utils";import { HugeiconsIcon } from "@hugeicons/react";-import { useCallback, useEffect, useRef } from "react";+import { useCallback, useEffect, useMemo, useRef } from "react";import {announceProjectSourcesUpdated,invalidateProjectSources,@@ -16,13 +19,22 @@ import {} from "../api/rag-api";import { isLinkedFolderManaged } from "../types/rag";import { DocumentStatusChip } from "./document-status-chip";+import {+ groupByLinkedFolder,+ useLinkedFolderNames,+} from "./linked-folder-groups";import { LinkedFoldersManager } from "./linked-folders-manager";-import { RAG_SOURCE_UPLOAD_ACCEPT } from "./source-drop-policy";+import {+ RAG_SOURCE_UPLOAD_ACCEPT,+ SUPPORTED_SOURCES_HINT,+ isSupportedSourceName,+} from "./source-drop-policy";import {type RagUploadItem,fileItems,useRagDocuments,} from "./use-rag-documents";+import { useUploadQueue } from "./use-upload-queue";/** Project "Sources" tab: documents indexed for retrieval in every chat that* belongs to the project. */@@ -32,22 +44,70 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) {() => listProjectDocuments(projectId),[projectId],);- const { documents, loading, uploading, refresh, upload, remove } =- useRagDocuments({ type: "project", projectId }, lister);+ const {+ documents,+ loading,+ uploading,+ refresh,+ upload,+ remove,+ retry,+ canRetry,+ } = useRagDocuments({ type: "project", projectId }, lister);// Invalidate the sources probe before each mutation so a chat sent mid-upload// cannot cache "no sources" for the probe's TTL, and announce after it, which// is the half other instances and other tabs listen for. Announcing before// would refetch and resurrect the row this panel has already dropped.- const handleItems = useCallback(- async (items: RagUploadItem[]) => {+ const uploadNow = useCallback(+ (items: RagUploadItem[]) => {if (items.length === 0) return;invalidateProjectSources(projectId);- await upload(items);- announceProjectSourcesUpdated(projectId);+ void upload(items).finally(() =>+ announceProjectSourcesUpdated(projectId),+ );},[projectId, upload],);+ const { enqueue: handleItems } = useUploadQueue(+ uploadNow,+ uploading,+ projectId,+ );++ const handleRetry = useCallback(+ (documentId: string) => {+ invalidateProjectSources(projectId);+ void retry(documentId).finally(() =>+ announceProjectSourcesUpdated(projectId),+ );+ },+ [projectId, retry],+ );++ const pickFolder = useCallback(() => {+ openFolderPicker(({ files: supported, truncated }) => {+ if (supported.length === 0) {+ toast.info("No supported files in that folder", {+ description: SUPPORTED_SOURCES_HINT,+ });+ return;+ }+ if (truncated > 0) {+ toast.info(`Added the first ${MAX_FOLDER_FILES} files`, {+ description:+ "The rest were left out. Pick a smaller folder for them.",+ });+ }+ handleItems(fileItems(supported));+ }, isSupportedSourceName);+ }, [handleItems]);++ const groups = useMemo(() => groupByLinkedFolder(documents), [documents]);+ const folderNames = useLinkedFolderNames(+ projectId,+ groups.folders.map(([id]) => id).join(","),+ );const handleFiles = useCallback((files: File[]) => handleItems(fileItems(files)),@@ -106,10 +166,23 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) {onFiles: handleFiles,onNativeIntents: handleNativeIntents,accept: RAG_SOURCE_UPLOAD_ACCEPT,- disabled: uploading,- disabledReason: "Wait for the current upload to finish, then drop again.",+ folders: true,});+ const addFolderButton = isTauri ? null : (+ <Button+ type="button"+ size="sm"+ variant="ghost"+ className="text-muted-foreground"+ disabled={loading}+ onClick={pickFolder}+ title="Upload every supported file in a folder. Linking a folder that stays in sync needs the desktop app."+ >+ Add folder+ </Button>+ );+return (<div className="mt-8" ref={dropRef} {...dragHandlers}><input@@ -150,20 +223,25 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) {Give this project context</p><p className="max-w-sm text-sm text-muted-foreground">- Add documents, spreadsheets, slides, e-books, email, text or- code. Every chat in this project can use them.+ Add documents, spreadsheets, slides, e-books, email, text or code.+ Every chat in this project can use them.</p></div>- <Button- type="button"- variant="outline"- className="mt-1 border-none bg-background text-foreground shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] hover:bg-background/80 dark:bg-card dark:shadow-none dark:hover:bg-accent/50"- disabled={uploading || loading}- onClick={() => fileInputRef.current?.click()}- >- Add sources- </Button>- <p className="text-ui-11 text-muted-foreground">Or drop files here</p>+ <div className="mt-1 flex items-center gap-1">+ <Button+ type="button"+ variant="outline"+ className="border-none bg-background text-foreground shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] hover:bg-background/80 dark:bg-card dark:shadow-none dark:hover:bg-accent/50"+ disabled={loading}+ onClick={() => fileInputRef.current?.click()}+ >+ Add sources+ </Button>+ {addFolderButton}+ </div>+ <p className="text-ui-11 text-muted-foreground">+ {isTauri ? "Or drop files here" : "Or drop files or a folder here"}+ </p></div>) : (<div@@ -178,19 +256,39 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) {? "1 source": `${documents.length} sources`}</p>- <Button- type="button"- size="sm"- variant="outline"- className="border-none bg-background text-foreground shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] hover:bg-background/80 dark:bg-card dark:shadow-none dark:hover:bg-accent/50"- disabled={uploading}- onClick={() => fileInputRef.current?.click()}- >- Add sources- </Button>+ <div className="flex items-center gap-1">+ {addFolderButton}+ <Button+ type="button"+ size="sm"+ variant="outline"+ className="border-none bg-background text-foreground shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] hover:bg-background/80 dark:bg-card dark:shadow-none dark:hover:bg-accent/50"+ onClick={() => fileInputRef.current?.click()}+ >+ Add sources+ </Button>+ </div></div><div className="flex flex-row flex-wrap items-center gap-1.5">- {documents.map((doc) => (+ {/* A linked folder is one chip: it can hold thousands of files. */}+ {groups.folders.map(([folderId, docs]) => {+ const indexing = docs.filter(+ (doc) => doc.status === "pending" || doc.status === "running",+ ).length;+ const name = folderNames.get(folderId) ?? "Linked folder";+ return (+ <DocumentStatusChip+ key={`folder:${folderId}`}+ filename={`${name} · ${docs.length} file${docs.length === 1 ? "" : "s"}`}+ status={indexing > 0 ? "running" : "completed"}+ progress={+ indexing > 0 ? (docs.length - indexing) / docs.length : null+ }+ shared={true}+ />+ );+ })}+ {groups.loose.map((doc) => (<DocumentStatusChipkey={doc.id}filename={doc.filename}@@ -199,10 +297,14 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) {stage={doc.stage}error={doc.error}onRemove={- doc.id.startsWith("pending_") || isLinkedFolderManaged(doc)+ isLinkedFolderManaged(doc) ||+ (doc.id.startsWith("pending_") && doc.status !== "failed")? undefined: () => void handleRemove(doc.id)}+ onRetry={+ canRetry(doc.id) ? () => handleRetry(doc.id) : undefined+ }/>))}</div>diff --git a/studio/frontend/src/features/rag/components/thread-documents-bar.tsx b/studio/frontend/src/features/rag/components/thread-documents-bar.tsxindex 5a385a64160..905c3159928 100644--- a/studio/frontend/src/features/rag/components/thread-documents-bar.tsx+++ b/studio/frontend/src/features/rag/components/thread-documents-bar.tsx@@ -45,7 +45,7 @@ import {} from "@/components/ui/tooltip";import { Tick02Icon } from "@/lib/tick-icon";import { ChevronDownStandardIcon } from "@/lib/chevron-icons";-import { useAui } from "@assistant-ui/react";+import { useAui, useAuiState } from "@assistant-ui/react";import { cn } from "@/lib/utils";import {PENDING_CHAT_ATTACHMENT_KEY,@@ -84,7 +84,6 @@ import {getDocumentFileUrl,invalidateProjectSources,listKnowledgeBases,- listLinkedFolders,subscribeKnowledgeBasesChanged,listProjectDocuments,listThreadDocuments,@@ -120,6 +119,8 @@ import {useRagDocuments,} from "./use-rag-documents";import { useSourceDrop } from "./use-source-drop";+import { useUploadQueue } from "./use-upload-queue";+import { groupByLinkedFolder, useLinkedFolderNames } from "./linked-folder-groups";// Refetched after any KB mutation so a rename shows at once.function useKnowledgeBaseName(kbId: string | null): string | null {@@ -254,51 +255,6 @@ function isRagIndexable(file: File): boolean {return isSupportedSourceName(file.name);}-/** Display names of a project's linked folders, read once per set of folder ids. */-function useLinkedFolderNames(- projectId: string | null,- folderIds: string,-): ReadonlyMap<string, string> {- const [names, setNames] = useState<ReadonlyMap<string, string>>(new Map());- useEffect(() => {- if (!projectId || !folderIds) return;- let cancelled = false;- listLinkedFolders({ type: "project", id: projectId })- .then((folders) => {- if (cancelled) return;- setNames(new Map(folders.map((f) => [f.id, f.displayName])));- })- .catch(() => {- // The card falls back to a generic label.- });- return () => {- cancelled = true;- };- }, [projectId, folderIds]);- return names;-}--/** Splits project documents into loose files and per-folder groups. A linked folder can- * hold thousands of files, so it is drawn as one card rather than one card per file. */-function groupByLinkedFolder(docs: TrackedDocument[]): {- loose: TrackedDocument[];- folders: [string, TrackedDocument[]][];-} {- const loose: TrackedDocument[] = [];- const folders = new Map<string, TrackedDocument[]>();- for (const doc of docs) {- const folderId = doc.linkedFolderId;- if (!folderId) {- loose.push(doc);- continue;- }- const group = folders.get(folderId);- if (group) group.push(doc);- else folders.set(folderId, [doc]);- }- return { loose, folders: [...folders] };-}-function isProcessing(doc: TrackedDocument): boolean {return doc.status === "pending" || doc.status === "running";}@@ -490,10 +446,12 @@ function DocumentCard({doc,shared,onRemove,+ onRetry,}: {doc: TrackedDocument;shared: boolean;onRemove?: () => void;+ onRetry?: () => void;}) {const openPreview = useDocumentPreviewStore((s) => s.openPreview);const kind = attachmentFileKind(doc.filename, undefined);@@ -616,6 +574,16 @@ function DocumentCard({card)}{shared ? <ProjectBadge /> : null}+ {failed && onRetry ? (+ <button+ type="button"+ onClick={onRetry}+ aria-label={`Retry ${doc.filename}`}+ className="absolute bottom-9 left-1/2 -translate-x-1/2 rounded-full bg-foreground px-2.5 py-0.5 text-ui-11 text-background shadow-sm"+ >+ Retry+ </button>+ ) : null}{onRemove && !processing ? (<buttontype="button"@@ -1007,6 +975,8 @@ export function ThreadDocumentsBar({loading: threadListLoading,upload,remove,+ retry,+ canRetry,} = useRagDocuments(effectiveThreadId && ragEnabled && ragSource.type === "thread"? { type: "thread", threadId: effectiveThreadId }@@ -1028,6 +998,8 @@ export function ThreadDocumentsBar({loading: projectListLoading,upload: uploadToProject,remove: removeFromProject,+ retry: retryProject,+ canRetry: canRetryProject,} = useRagDocuments(projectId ? { type: "project", projectId } : null,projectLister,@@ -1123,6 +1095,14 @@ export function ThreadDocumentsBar({[ensureThreadId, projectId, sharesWithProject, upload, uploadToProject],);+ // Keyed by the list item id, stable while a new chat materializes, so only a real switch drops the queue.+ const threadItemId = useAuiState(({ threadListItem }) => threadListItem.id);+ const { enqueue: queueAttach, queued } = useUploadQueue<RagUploadItem>(+ attach,+ uploading || projectUploading,+ `${threadItemId}:${projectId ?? ""}`,+ );+// Only files added or dropped here are indexed; composer attachments stay with the message.const attachIndexable = useCallback((files: File[]) => {@@ -1136,9 +1116,11 @@ export function ThreadDocumentsBar({{ description: SUPPORTED_SOURCES_HINT },);}- if (indexable.length > 0) attach(indexable);+ if (indexable.length > 0) {+ queueAttach(indexable.map((file) => ({ kind: "file" as const, file })));+ }},- [attach],+ [queueAttach],);const pickFiles = useCallback(() => {openFilePicker(RAG_SOURCE_UPLOAD_ACCEPT, attachIndexable);@@ -1340,14 +1322,13 @@ export function ThreadDocumentsBar({);}- // block attachments until the chat's project is known to avoid guessing their scope.- const busy = uploading || projectUploading || projectUnresolved;+ // Block attachments until the chat's project is known; the scope stays locked until the queue drains.+ const busy = projectUnresolved;+ const scopeLocked = busy || uploading || projectUploading || queued > 0;// Still claim a drop while busy, or it falls through and attaches to the message instead.- const busyReason = !busy- ? undefined- : projectUnresolved- ? "Still loading this chat's project. Try again in a moment."- : "Still uploading. Drop the files again once it finishes.";+ const busyReason = busy+ ? "Still loading this chat's project. Try again in a moment."+ : undefined;return (<>@@ -1359,7 +1340,7 @@ export function ThreadDocumentsBar({titleSuffix="(RAG)"count={countSuffix}info="Add or drop documents, spreadsheets, slides, e-books, email or code. The model searches them as you chat and cites what it uses."- onDropItems={attach}+ onDropItems={queueAttach}dropDisabledReason={busyReason}onClose={() => setRagEnabled(false)}closeLabel="Stop chatting with files"@@ -1378,7 +1359,7 @@ export function ThreadDocumentsBar({{/* Only a project chat has two scopes to choose between. */}{projectId ? (<AttachmentTargetMenu- disabled={busy}+ disabled={scopeLocked}sharesWithProject={sharesWithProject}onSelect={(target) =>setThreadProjectAttachmentTarget(effectiveThreadId, target)@@ -1413,9 +1394,23 @@ export function ThreadDocumentsBar({doc={doc}shared={true}onRemove={- doc.id.startsWith("pending_") || isLinkedFolderManaged(doc)+ isLinkedFolderManaged(doc) ||+ (doc.id.startsWith("pending_") && doc.status !== "failed")? undefined- : () => setRemovingShared(doc)+ : doc.status === "failed"+ ? // Never indexed, so no other chat loses anything: no confirmation.+ () => void removeFromProject(doc.id)+ : () => setRemovingShared(doc)+ }+ onRetry={+ canRetryProject(doc.id)+ ? () => {+ if (projectId) invalidateProjectSources(projectId);+ void retryProject(doc.id).finally(() => {+ if (projectId) announceProjectSourcesUpdated(projectId);+ });+ }+ : undefined}/></div>@@ -1429,10 +1424,11 @@ export function ThreadDocumentsBar({doc={doc}shared={false}onRemove={- doc.id.startsWith("pending_")+ doc.id.startsWith("pending_") && doc.status !== "failed"? undefined: () => void remove(doc.id)}+ onRetry={canRetry(doc.id) ? () => void retry(doc.id) : undefined}/></div>))}diff --git a/studio/frontend/src/features/rag/components/use-linked-folders.ts b/studio/frontend/src/features/rag/components/use-linked-folders.tsindex cb8b9293e16..35423acf279 100644--- a/studio/frontend/src/features/rag/components/use-linked-folders.ts+++ b/studio/frontend/src/features/rag/components/use-linked-folders.ts@@ -27,6 +27,7 @@ import type {LinkedFolderScope,} from "../types/rag";import {+ jobChangedSources,linkedFolderSourcesChanged,retainActiveFolderJobs,} from "../types/rag";@@ -80,11 +81,14 @@ export function useLinkedFolders([projectWorkScopeId],);+ // Only syncs started here toast on failure; periodic failures show on the folder row.+ const userStartedJobs = useRef(new Set<string>());+const notifySourcesChanged = useCallback((job: FolderSyncJob) => {if (notifiedJobs.current.has(job.id)) return;notifiedJobs.current.add(job.id);- onSourcesChanged?.();+ if (jobChangedSources(job)) onSourcesChanged?.();},[onSourcesChanged],);@@ -123,7 +127,8 @@ export function useLinkedFolders(apply(job);releaseController();notifySourcesChanged(job);- if (job.status === "failed") {+ const userStarted = userStartedJobs.current.delete(job.id);+ if (job.status === "failed" && userStarted) {toast.error("Folder sync failed", {description: job.error ?? "The folder could not be indexed.",});@@ -285,6 +290,7 @@ export function useLinkedFolders(...current.filter((folder) => folder.id !== result.linkedFolder.id),result.linkedFolder,]);+ userStartedJobs.current.add(result.job.id);trackJob(result.job);onSourcesChanged?.();} catch (error) {@@ -321,6 +327,7 @@ export function useLinkedFolders(return started;});if (currentScopeKey.current !== operationScopeKey) return;+ userStartedJobs.current.add(job.id);trackJob(job);onSourcesChanged?.();} catch (error) {diff --git a/studio/frontend/src/features/rag/components/use-rag-documents.ts b/studio/frontend/src/features/rag/components/use-rag-documents.tsindex 0c289cd0b31..1f642690728 100644--- a/studio/frontend/src/features/rag/components/use-rag-documents.ts+++ b/studio/frontend/src/features/rag/components/use-rag-documents.ts@@ -111,6 +111,8 @@ export function useRagDocuments(}, [documents]);// documentId -> signature; forgotten on delete, cleared on scope change.const sigByDocId = useRef<Map<string, string>>(new Map());+ // Row id -> source File for retry. Files only: a native drop's path token is spent on first use.+ const retryItems = useRef<Map<string, RagUploadItem>>(new Map());// Skip a re-selected file only if a matching doc is healthy or still indexing. A doc// that completed with 0 chunks is re-ingestable (e.g. a scan attached before a vision// model loaded); the backend re-ingests on the same hash, so let it through.@@ -189,13 +191,17 @@ export function useRagDocuments(numChunks?: number | null,) => {if (stale()) return forget();+ if (status !== "failed") retryItems.current.delete(documentId);if (status === "cancelled") {sigByDocId.current.delete(documentId);setDocuments((rows) => rows.filter((row) => row.id !== documentId));} else if (status === "failed") {- // Drop the chip rather than show "Failed"; warn via toast.sigByDocId.current.delete(documentId);- setDocuments((rows) => rows.filter((row) => row.id !== documentId));+ patchDoc(documentId, {+ status: "failed",+ error: error ?? "Indexing failed",+ progress: null,+ });toast.error(`Couldn't index ${filename}`, {description: error ?? "Indexing failed",});@@ -293,8 +299,10 @@ export function useRagDocuments(if (!opts?.quiet) setLoading(true);try {// Merge server truth with local progress so a refresh mid-index keeps a- // live "running %" chip. Failed docs hidden (toast warned at upload).- const rows = (await lister()).filter((row) => row.status !== "failed");+ // live "running %" chip. Failed docs hidden, except ones this session can retry.+ const rows = (await lister()).filter(+ (row) => row.status !== "failed" || retryItems.current.has(row.id),+ );if (refreshSeq.current !== requestId) return true;setDocuments((prev) => {const merged = rows.map((row) => {@@ -306,10 +314,13 @@ export function useRagDocuments(: row;});// Keep optimistic chips (not yet listed) so a refresh racing an upload- // can't make them vanish.+ // can't make them vanish, and failed rows this session can still retry.const serverIds = new Set(rows.map((row) => row.id));const pendingLocal = prev.filter(- (row) => row.id.startsWith("pending_") && !serverIds.has(row.id),+ (row) =>+ !serverIds.has(row.id) &&+ (row.id.startsWith("pending_") ||+ (row.status === "failed" && retryItems.current.has(row.id))),);return [...merged, ...pendingLocal];});@@ -397,6 +408,7 @@ export function useRagDocuments(uploadGenerationRef.current += 1;activeUploadsRef.current.clear();uploadInFlightRef.current = false;+ retryItems.current.clear();setUploading(false);// Scope changes intentionally clear the old scope before fetching the new// one. Keep this synchronous so React StrictMode's setup/cleanup replay@@ -551,6 +563,7 @@ export function useRagDocuments();if (generation !== uploadGenerationRef.current) return;sigByDocId.current.set(result.documentId, itemSignature(item));+ if (item.kind === "file") retryItems.current.set(result.documentId, item);setDocuments((rows) =>rows.some((row) => row.id === result.documentId)? rows.filter((row) => row.id !== tempId)@@ -569,12 +582,16 @@ export function useRagDocuments(} catch (err) {if (generation !== uploadGenerationRef.current) return;const message = err instanceof Error ? err.message : String(err);- // Drop the chip rather than show "Failed"; warn via toast.- setDocuments((rows) => rows.filter((row) => row.id !== tempId));+ if (item.kind === "file") {+ retryItems.current.set(tempId, item);+ patchDoc(tempId, { status: "failed", error: message, progress: null });+ } else {+ setDocuments((rows) => rows.filter((row) => row.id !== tempId));+ }toast.error(`Couldn't upload ${name}`, { description: message });}},- [trackJob],+ [trackJob, patchDoc],);// `overrideScope` lets a caller pass a freshly-resolved scope (or a promise of one), since the@@ -701,9 +718,12 @@ export function useRagDocuments();const remove = useCallback(- async (documentId: string) => {+ async (documentId: string): Promise<boolean> => {const prev = documents;setDocuments((rows) => rows.filter((row) => row.id !== documentId));+ const prevRetry = retryItems.current.get(documentId);+ retryItems.current.delete(documentId);+ if (documentId.startsWith("pending_")) return true;// Forget the dedup signature so re-uploading re-indexes.const prevSig = sigByDocId.current.get(documentId);sigByDocId.current.delete(documentId);@@ -720,12 +740,15 @@ export function useRagDocuments(documentId,scope?.type === "project" ? scope.projectId : undefined,);+ return true;} catch (err) {setDocuments(prev);if (prevSig !== undefined) sigByDocId.current.set(documentId, prevSig);+ if (prevRetry !== undefined) retryItems.current.set(documentId, prevRetry);toast.error("Delete failed", {description: err instanceof Error ? err.message : String(err),});+ return false;} finally {if (removingProjectId) {noteProjectWork(removingProjectId, -1);@@ -735,6 +758,21 @@ export function useRagDocuments([documents, scope],);+ const canRetry = useCallback(+ (documentId: string) => retryItems.current.has(documentId),+ [],+ );++ const retry = useCallback(+ async (documentId: string) => {+ const item = retryItems.current.get(documentId);+ if (!item) return;+ if (!(await remove(documentId))) return;+ await upload([item]);+ },+ [remove, upload],+ );+return {documents,loading,@@ -743,5 +781,7 @@ export function useRagDocuments(refresh,upload,remove,+ retry,+ canRetry,};}diff --git a/studio/frontend/src/features/rag/components/use-source-drop.ts b/studio/frontend/src/features/rag/components/use-source-drop.tsindex 22c1d171fb2..98435600ec9 100644--- a/studio/frontend/src/features/rag/components/use-source-drop.ts+++ b/studio/frontend/src/features/rag/components/use-source-drop.ts@@ -6,6 +6,7 @@ import {registerNativeAttachmentPath,useNativeDropTarget,} from "@/features/native-intents";+import { MAX_FOLDER_FILES, filesFromDrop } from "@/lib/dropped-folders";import { toast } from "@/lib/toast";import {type DragEvent as ReactDragEvent,@@ -15,6 +16,7 @@ import {} from "react";import {SUPPORTED_SOURCES_HINT,+ isSupportedSourceName,partitionSupported,} from "./source-drop-policy";import { type RagUploadItem, uploadItemFromIntent } from "./use-rag-documents";@@ -157,14 +159,30 @@ export function useSourceDrop({toast.info(reason);return;}- const { supported, unsupported } = partitionSupported(- Array.from(event.dataTransfer.files ?? []),- (file) => file.name,+ void filesFromDrop(event.dataTransfer, isSupportedSourceName).then(+ ({ files, truncated, hadFolder }) => {+ const { supported, unsupported } = partitionSupported(+ files,+ (file) => file.name,+ );+ if (!hadFolder) reportUnsupported(unsupported);+ if (hadFolder && supported.length === 0) {+ toast.info("No supported files in that folder", {+ description: SUPPORTED_SOURCES_HINT,+ });+ }+ if (truncated > 0) {+ toast.info(`Added the first ${MAX_FOLDER_FILES} files`, {+ description:+ "The rest were left out. Drop a smaller folder for them.",+ });+ }+ if (supported.length > 0) {+ onItems(supported.map((file) => ({ kind: "file" as const, file })));+ }+ },+ () => toast.error("Couldn't read the dropped folder"),);- reportUnsupported(unsupported);- if (supported.length > 0) {- onItems(supported.map((file) => ({ kind: "file" as const, file })));- }},};diff --git a/studio/frontend/src/features/rag/components/use-upload-queue.ts b/studio/frontend/src/features/rag/components/use-upload-queue.tsnew file mode 100644index 00000000000..a45bc5a7681--- /dev/null+++ b/studio/frontend/src/features/rag/components/use-upload-queue.ts@@ -0,0 +1,69 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import { toast } from "@/lib/toast";+import { useCallback, useEffect, useRef, useState } from "react";++/** Queues items added during an upload and runs them when it ends; items queued for one `key`+ * (destination) are dropped with a toast if the surface moves to another. */+export function useUploadQueue<T>(+ run: (items: T[]) => void,+ busy: boolean,+ key: string | null,+): { enqueue: (items: T[]) => void; queued: number } {+ const queue = useRef<{ key: string | null; items: T[] }>({ key, items: [] });+ const [queued, setQueued] = useState(0);+ const latestRun = useRef(run);+ latestRun.current = run;+ const latestKey = useRef(key);+ latestKey.current = key;++ const enqueue = useCallback(+ (items: T[]) => {+ if (items.length === 0) return;+ // A folder walk resolves late: latestRun may point at another destination by now.+ if (key !== latestKey.current) {+ toast.info(+ items.length === 1+ ? "A dropped file was not added"+ : `${items.length} dropped files were not added`,+ { description: "You moved away before they finished reading." },+ );+ return;+ }+ if (!busy && queue.current.items.length === 0) {+ latestRun.current(items);+ return;+ }+ queue.current = { key, items: [...queue.current.items, ...items] };+ setQueued(queue.current.items.length);+ toast.info(+ items.length === 1 ? "Queued 1 file" : `Queued ${items.length} files`,+ { description: "They upload when the current upload finishes." },+ );+ },+ [busy, key],+ );++ useEffect(() => {+ const pending = queue.current;+ if (pending.items.length === 0) return;+ if (pending.key !== key) {+ queue.current = { key, items: [] };+ setQueued(0);+ toast.info(+ pending.items.length === 1+ ? "A queued file was not added"+ : `${pending.items.length} queued files were not added`,+ { description: "You moved away before the earlier upload finished." },+ );+ return;+ }+ if (busy) return;+ queue.current = { key, items: [] };+ setQueued(0);+ latestRun.current(pending.items);+ }, [busy, key]);++ return { enqueue, queued };+}diff --git a/studio/frontend/src/features/rag/types/rag.ts b/studio/frontend/src/features/rag/types/rag.tsindex fbabc0be642..7c5a6b03f09 100644--- a/studio/frontend/src/features/rag/types/rag.ts+++ b/studio/frontend/src/features/rag/types/rag.ts@@ -99,11 +99,21 @@ export interface LinkedFolder {status: LinkedFolderStatus;documentCount?: number;lastSyncedAt?: string | null;+ lastChangedAt?: string | null;error?: string | null;activeJobId?: string | null;createdAt?: string | null;+ skipped?: Partial<Record<LinkedFolderSkipReason, number>>;+ failures?: { path: string; error: string }[];+ failureCount?: number;}+export type LinkedFolderSkipReason =+ | "gitignored"+ | "build_dirs"+ | "too_large"+ | "over_limit";+export function linkedFolderSourcesChanged(previous: LinkedFolder[] | null,current: LinkedFolder[],@@ -121,7 +131,9 @@ export function linkedFolderSourcesChanged(return (prior !== undefined &&(prior.documentCount !== folder.documentCount ||- prior.lastSyncedAt !== folder.lastSyncedAt)+ // lastSyncedAt moves every pass; older backends without lastChangedAt fall back to it.+ (prior.lastChangedAt ?? prior.lastSyncedAt) !==+ (folder.lastChangedAt ?? folder.lastSyncedAt)));});}@@ -140,10 +152,26 @@ export interface FolderSyncJob {processedFiles?: number;indexedFiles?: number;removedFiles?: number;+ renamedFiles?: number;failedFiles?: number;error?: string | null;}+/** Missing counts (older backends) count as a change. */+export function jobChangedSources(job: FolderSyncJob): boolean {+ if (+ job.indexedFiles === undefined &&+ job.removedFiles === undefined &&+ job.renamedFiles === undefined+ ) {+ return true;+ }+ return (+ (job.indexedFiles ?? 0) + (job.removedFiles ?? 0) + (job.renamedFiles ?? 0) >+ 0+ );+}+/** One SSE frame from /linked-folder-jobs/{jobId}/events. */export interface FolderSyncJobEvent extends Partial<FolderSyncJob> {type: "progress" | "complete" | "error";diff --git a/studio/frontend/src/lib/dropped-folders.ts b/studio/frontend/src/lib/dropped-folders.tsnew file mode 100644index 00000000000..0eb5661e7fe--- /dev/null+++ b/studio/frontend/src/lib/dropped-folders.ts@@ -0,0 +1,158 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// Browser folder drops and picks, flattened to files (the desktop links folders instead).++export const MAX_FOLDER_FILES = 2000;++const SKIPPED_DIRS = new Set([+ "node_modules",+ "bower_components",+ "venv",+ "__pycache__",+]);++function skippedDir(name: string): boolean {+ return name.startsWith(".") || SKIPPED_DIRS.has(name.toLowerCase());+}++// Mirrors _is_ignored_scan_file in folder_sync.py: secrets and lockfiles.+function skippedFile(name: string): boolean {+ const lower = name.toLowerCase();+ return (+ lower.startsWith(".") ||+ lower.endsWith(".env") ||+ lower.endsWith(".lock") ||+ lower.endsWith(".lockb") ||+ lower.endsWith(".tfstate") ||+ lower === "package-lock.json" ||+ lower === "npm-shrinkwrap.json" ||+ lower === "pnpm-lock.yaml" ||+ lower === "pnpm-lock.yml"+ );+}++export interface FolderFiles {+ files: File[];+ /** Files left out past MAX_FOLDER_FILES; a dropped folder stops at the first, so reports 1. */+ truncated: number;+ hadFolder: boolean;+}++function readBatch(+ reader: FileSystemDirectoryReader,+): Promise<FileSystemEntry[]> {+ return new Promise((resolve, reject) => reader.readEntries(resolve, reject));+}++function entryFile(entry: FileSystemFileEntry): Promise<File> {+ return new Promise((resolve, reject) => entry.file(resolve, reject));+}++/** Call inside the drop handler: the browser empties `items` once the event returns. */+export function filesFromDrop(+ dataTransfer: DataTransfer,+ keep: (name: string) => boolean = () => true,+): Promise<FolderFiles> {+ const entries = Array.from(dataTransfer.items ?? [])+ .filter((item) => item.kind === "file")+ .map((item) => item.webkitGetAsEntry?.() ?? null);+ const plain = Array.from(dataTransfer.files ?? []);+ if (!entries.some((entry) => entry?.isDirectory)) {+ return Promise.resolve({ files: plain, truncated: 0, hadFolder: false });+ }+ return walkEntries(+ entries.filter((entry): entry is FileSystemEntry => entry !== null),+ keep,+ );+}++async function walkEntries(+ roots: FileSystemEntry[],+ keep: (name: string) => boolean,+): Promise<FolderFiles> {+ const files: File[] = [];+ let truncated = 0;+ const pending = [...roots];+ while (pending.length > 0) {+ const entry = pending.shift() as FileSystemEntry;+ if (entry.isFile) {+ const root = roots.indexOf(entry) >= 0;+ if (!root && (skippedFile(entry.name) || !keep(entry.name))) continue;+ if (files.length >= MAX_FOLDER_FILES) {+ truncated = 1;+ break;+ }+ try {+ const file = await entryFile(entry as FileSystemFileEntry);+ if (file.size > 0 || root) files.push(file);+ } catch {+ }+ continue;+ }+ if (+ !entry.isDirectory ||+ (roots.indexOf(entry) < 0 && skippedDir(entry.name))+ ) {+ continue;+ }+ const reader = (entry as FileSystemDirectoryEntry).createReader();+ // readEntries returns ~100 entries per call; an empty batch ends the folder.+ for (;;) {+ let batch: FileSystemEntry[];+ try {+ batch = await readBatch(reader);+ } catch {+ break;+ }+ if (batch.length === 0) break;+ pending.push(...batch);+ }+ }+ return { files, truncated, hadFolder: true };+}++export function filesFromFolderInput(+ list: FileList | File[],+ keep: (name: string) => boolean = () => true,+): FolderFiles {+ const all = Array.from(list).filter((file) => {+ const segments = (file.webkitRelativePath || file.name).split("/");+ // Segment 0 is the chosen folder itself.+ return (+ file.size > 0 &&+ !segments.slice(1, -1).some(skippedDir) &&+ !skippedFile(file.name) &&+ keep(file.name)+ );+ });+ const files = all.slice(0, MAX_FOLDER_FILES);+ return { files, truncated: all.length - files.length, hadFolder: true };+}++/** Call from a click handler (needs user activation). */+export function openFolderPicker(+ onFiles: (folder: FolderFiles) => void,+ keep?: (name: string) => boolean,+): void {+ const input = document.createElement("input");+ input.type = "file";+ input.multiple = true;+ input.webkitdirectory = true;+ input.hidden = true;++ document.body.appendChild(input);+ input.onchange = () => {+ const files = input.files;+ if (files && files.length > 0) {+ onFiles(filesFromFolderInput(files, keep));+ }+ document.body.removeChild(input);+ };+ input.oncancel = () => {+ if (!input.files || input.files.length === 0) {+ document.body.removeChild(input);+ }+ };+ input.click();+}diff --git a/studio/frontend/tests/knowledge-base-dialog-focus.test.ts b/studio/frontend/tests/knowledge-base-dialog-focus.test.tsindex 377908ba1b1..3410f1a1e02 100644--- a/studio/frontend/tests/knowledge-base-dialog-focus.test.ts+++ b/studio/frontend/tests/knowledge-base-dialog-focus.test.ts@@ -187,6 +187,12 @@ function harness(rows: Array<typeof KB> = [KB]) {nativeDropTarget: () => {},}),},+ "./use-upload-queue": {+ useUploadQueue: (run: (items: unknown[]) => void) => ({+ enqueue: run,+ queued: 0,+ }),+ },},);diff --git a/studio/frontend/tests/native-dropzone-coverage.test.ts b/studio/frontend/tests/native-dropzone-coverage.test.tsindex 6d2e056da67..6dc2178775c 100644--- a/studio/frontend/tests/native-dropzone-coverage.test.ts+++ b/studio/frontend/tests/native-dropzone-coverage.test.ts@@ -52,7 +52,7 @@ test("every file drop zone is reachable from the desktop app", async () => {if (!FILE_DROP_MARKERS.some((marker) => source.includes(marker))) continue;// The shared readers themselves (the DataTransfer walker, the hook) are not// drop zones; their callers are the ones that have to be reachable.- if (/export (async )?function filesFromDataTransfer/.test(source)) continue;+ if (/export (async )?function (filesFromDataTransfer|filesFromDrop)\b/.test(source)) continue;if (NATIVE_MARKERS.some((marker) => source.includes(marker))) continue;dead.push(path.relative(new URL(".", SRC).pathname, file.pathname));}diff --git a/studio/frontend/tests/project-attachment-target-adoption.test.ts b/studio/frontend/tests/project-attachment-target-adoption.test.tsindex d66d93396d1..d5a68436745 100644--- a/studio/frontend/tests/project-attachment-target-adoption.test.ts+++ b/studio/frontend/tests/project-attachment-target-adoption.test.ts@@ -107,9 +107,10 @@ test("attaching is held until the chat's project is known", () => {/const projectUnresolved = threadProjectId === undefined;/,"unresolved has to be distinguishable from no project",);+ // An upload in flight only queues new files; an unknown project still holds them.assert.match(THREAD_DOCUMENTS_BAR,- /uploading \|\| projectUploading \|\| projectUnresolved/,+ /const busy = projectUnresolved;/,"the attach controls hold",);assert.match(diff --git a/studio/frontend/tests/project-sources-model-without-tools.test.ts b/studio/frontend/tests/project-sources-model-without-tools.test.tsindex 11727d66efa..3e9c1cafbf8 100644--- a/studio/frontend/tests/project-sources-model-without-tools.test.ts+++ b/studio/frontend/tests/project-sources-model-without-tools.test.ts@@ -8,6 +8,7 @@ import * as jsxRuntime from "react/jsx-runtime";import { renderToStaticMarkup } from "react-dom/server";import type * as BarModule from "../src/features/rag/components/thread-documents-bar.tsx";+import type * as FolderGroupsModule from "../src/features/rag/components/linked-folder-groups.ts";import type * as GateModule from "../src/features/chat/hooks/use-rag-tool-disabled.ts";import { loadWithStubs } from "./helpers/module-stubs.ts";@@ -40,6 +41,11 @@ const Nothing = () => null;const Passthrough = ({ children }: { children?: React.ReactNode }) =>React.createElement(React.Fragment, null, children);+const folderGroups = loadWithStubs<typeof FolderGroupsModule>(+ new URL("../src/features/rag/components/linked-folder-groups.ts", import.meta.url),+ { react: React, "../api/rag-api": { listLinkedFolders: async () => [] } },+);+const { ThreadDocumentsBar } = loadWithStubs<typeof BarModule>(new URL("../src/features/rag/components/thread-documents-bar.tsx", import.meta.url),{@@ -72,7 +78,11 @@ const { ThreadDocumentsBar } = loadWithStubs<typeof BarModule>(useDocumentPreviewStore: selectorStore({ openPreview: () => undefined }),},"@/lib/chevron-icons": {},- "@assistant-ui/react": { useAui: () => ({}) },+ "@assistant-ui/react": {+ useAui: () => ({}),+ useAuiState: (select: (s: { threadListItem: { id: string } }) => unknown) =>+ select({ threadListItem: { id: "local-1" } }),+ },"@/lib/utils": {cn: (...classes: unknown[]) => classes.filter(Boolean).join(" "),},@@ -124,6 +134,13 @@ const { ThreadDocumentsBar } = loadWithStubs<typeof BarModule>(nativeDropTarget: () => undefined,}),},+ "./use-upload-queue": {+ useUploadQueue: (run: (items: unknown[]) => void) => ({+ enqueue: run,+ queued: 0,+ }),+ },+ "./linked-folder-groups": folderGroups,"@/components/ui/alert-dialog": new Proxy({}, { get: () => Passthrough }),"./document-status-chip": { STAGE_LABELS: {} },"./knowledge-base-dialog": { KnowledgeBaseDialog: Nothing },@@ -138,6 +155,8 @@ const { ThreadDocumentsBar } = loadWithStubs<typeof BarModule>(loading: false,upload: async () => undefined,remove: async () => undefined,+ retry: async () => undefined,+ canRetry: () => false,}),},},diff --git a/studio/frontend/tests/rag-refresh-sequencing.test.ts b/studio/frontend/tests/rag-refresh-sequencing.test.tsindex b7c77ebd83e..a7da39cf551 100644--- a/studio/frontend/tests/rag-refresh-sequencing.test.ts+++ b/studio/frontend/tests/rag-refresh-sequencing.test.ts@@ -729,7 +729,7 @@ test("a failed project lookup leaves the scope unresolved", () => {);// And unresolved still disables the attach controls.assert.match(THREAD_DOCUMENTS_BAR, /const projectUnresolved = threadProjectId === undefined;/);- assert.match(THREAD_DOCUMENTS_BAR, /uploading \|\| projectUploading \|\| projectUnresolved/);+ assert.match(THREAD_DOCUMENTS_BAR, /const busy = projectUnresolved;/);});// A row the probe could not read is not a chat with no project: answering nulldiff --git a/studio/frontend/tests/rag-source-followups.test.ts b/studio/frontend/tests/rag-source-followups.test.tsnew file mode 100644index 00000000000..69a50878deb--- /dev/null+++ b/studio/frontend/tests/rag-source-followups.test.ts@@ -0,0 +1,231 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";++import {+ type FolderSyncJob,+ type LinkedFolder,+ jobChangedSources,+ linkedFolderSourcesChanged,+} from "../src/features/rag/types/rag.ts";+import {+ MAX_FOLDER_FILES,+ filesFromFolderInput,+} from "../src/lib/dropped-folders.ts";+import { loadWithStubs } from "./helpers/module-stubs.ts";++const folder = (overrides: Partial<LinkedFolder> = {}): LinkedFolder => ({+ id: "f1",+ displayName: "Docs",+ scopeType: "project",+ scopeId: "p1",+ status: "idle",+ documentCount: 3,+ lastSyncedAt: "2026-10-10T00:00:00Z",+ lastChangedAt: "2026-10-09T00:00:00Z",+ ...overrides,+});++// Every 30 s pass bumps lastSyncedAt; reading that as a change refetched every document list.+test("a periodic pass that changed nothing is not a sources change", () => {+ const before = [folder()];+ const after = [folder({ lastSyncedAt: "2026-10-10T00:00:30Z" })];+ assert.equal(linkedFolderSourcesChanged(before, after), false);+ assert.equal(+ linkedFolderSourcesChanged(before, [+ folder({ lastChangedAt: "2026-10-10T00:00:30Z" }),+ ]),+ true,+ );+ assert.equal(+ linkedFolderSourcesChanged(before, [folder({ documentCount: 4 })]),+ true,+ );+});++test("an older backend without lastChangedAt still reports syncs as changes", () => {+ const legacy = (at: string) =>+ folder({ lastChangedAt: undefined, lastSyncedAt: at });+ assert.equal(linkedFolderSourcesChanged([legacy("a")], [legacy("b")]), true);+});++test("a finished job changed sources only if it added, removed or renamed something", () => {+ const job = (counts: Partial<FolderSyncJob>): FolderSyncJob => ({+ id: "j",+ linkedFolderId: "f1",+ mode: "sync",+ status: "completed",+ ...counts,+ });+ assert.equal(+ jobChangedSources(+ job({ indexedFiles: 0, removedFiles: 0, renamedFiles: 0 }),+ ),+ false,+ );+ assert.equal(+ jobChangedSources(job({ indexedFiles: 2, removedFiles: 0 })),+ true,+ );+ assert.equal(+ jobChangedSources(job({ indexedFiles: 0, renamedFiles: 1 })),+ true,+ );+ assert.equal(jobChangedSources(job({})), true);+});++function inFolder(path: string, content = "x"): File {+ const file = new File([content], path.split("/").pop() ?? path);+ Object.defineProperty(file, "webkitRelativePath", { value: path });+ return file;+}++test("a picked folder leaves out dependency, hidden and secret files", () => {+ const { files, truncated } = filesFromFolderInput([+ inFolder("repo/README.md"),+ inFolder("repo/src/app.ts"),+ inFolder("repo/node_modules/dep/index.js"),+ inFolder("repo/.git/config"),+ inFolder("repo/.venv/lib/site.py"),+ inFolder("repo/prod.env"),+ inFolder("repo/package-lock.json"),+ inFolder("repo/pnpm-lock.yml"),+ inFolder("repo/src/__init__.py", ""),+ ]);+ assert.deepEqual(+ files.map((file) => file.webkitRelativePath),+ ["repo/README.md", "repo/src/app.ts"],+ );+ assert.equal(truncated, 0);+});++// Unsupported files must not use up the cap, or a folder of images hides the documents after them.+test("only files the caller takes count toward the folder cap", () => {+ const images = Array.from({ length: MAX_FOLDER_FILES }, (_, i) =>+ inFolder(`mixed/${i}.png`),+ );+ const { files, truncated } = filesFromFolderInput(+ [...images, inFolder("mixed/notes.md")],+ (name) => name.endsWith(".md"),+ );+ assert.deepEqual(+ files.map((file) => file.name),+ ["notes.md"],+ );+ assert.equal(truncated, 0);+});++test("a picked folder past the cap keeps the first files and counts the rest", () => {+ const many = Array.from({ length: MAX_FOLDER_FILES + 5 }, (_, i) =>+ inFolder(`big/${i}.md`),+ );+ const { files, truncated } = filesFromFolderInput(many);+ assert.equal(files.length, MAX_FOLDER_FILES);+ assert.equal(truncated, 5);+});++/** Drives useUploadQueue renders with a stub React whose effects run after each render. */+function queueHarness() {+ const slots: unknown[] = [];+ const effects: Array<() => void> = [];+ const toasts: string[] = [];+ let cursor = 0;+ const react = {+ useRef(value: unknown) {+ const index = cursor++;+ slots[index] ??= { current: value };+ return slots[index];+ },+ useState(value: unknown) {+ const index = cursor++;+ if (!(index in slots)) slots[index] = value;+ return [+ slots[index],+ (next: unknown) => {+ slots[index] = next;+ },+ ];+ },+ useCallback: (fn: unknown) => fn,+ useEffect(effect: () => void, deps: unknown[]) {+ const index = cursor++;+ const previous = slots[index] as unknown[] | undefined;+ if (previous && deps.every((dep, i) => Object.is(dep, previous[i])))+ return;+ slots[index] = deps;+ effects.push(effect);+ },+ };+ const { useUploadQueue } = loadWithStubs<{+ useUploadQueue: (+ run: (items: string[]) => void,+ busy: boolean,+ key: string | null,+ ) => { enqueue: (items: string[]) => void; queued: number };+ }>(+ new URL(+ "../src/features/rag/components/use-upload-queue.ts",+ import.meta.url,+ ),+ {+ react,+ "@/lib/toast": {+ toast: { info: (message: string) => toasts.push(message) },+ },+ },+ );+ const ran: string[][] = [];+ return {+ ran,+ toasts,+ render(busy: boolean, key: string | null) {+ cursor = 0;+ const result = useUploadQueue((items) => ran.push(items), busy, key);+ for (const effect of effects.splice(0)) effect();+ return result;+ },+ };+}++test("files added during an upload wait for it, then go in one batch", () => {+ const app = queueHarness();+ app.render(false, "p1").enqueue(["a"]);+ assert.deepEqual(app.ran, [["a"]]);++ let hook = app.render(true, "p1");+ hook.enqueue(["b"]);+ hook = app.render(true, "p1");+ hook.enqueue(["c"]);+ assert.deepEqual(+ app.ran,+ [["a"]],+ "nothing starts while the first upload runs",+ );+ assert.equal(app.render(true, "p1").queued, 2);++ app.render(false, "p1");+ assert.deepEqual(app.ran, [["a"], ["b", "c"]]);+ assert.equal(app.render(false, "p1").queued, 0);+});++test("a drop that resolves after a switch is not added to the new destination", () => {+ const app = queueHarness();+ const stale = app.render(false, "p1").enqueue;+ app.render(false, "p2");+ stale(["a"]);+ assert.deepEqual(app.ran, []);+ assert.ok(app.toasts.some((message) => message.includes("not added")));+ app.render(false, "p2").enqueue(["b"]);+ assert.deepEqual(app.ran, [["b"]]);+});++test("files queued for one project are not added to the next", () => {+ const app = queueHarness();+ app.render(true, "p1").enqueue(["b"]);+ app.render(true, "p2");+ app.render(false, "p2");+ assert.deepEqual(app.ran, []);+ assert.ok(app.toasts.some((message) => message.includes("not added")));+});diff --git a/studio/frontend/tests/rag-thread-initialization.test.ts b/studio/frontend/tests/rag-thread-initialization.test.tsindex 067950f8cf2..81c7b141c6b 100644--- a/studio/frontend/tests/rag-thread-initialization.test.ts+++ b/studio/frontend/tests/rag-thread-initialization.test.ts@@ -162,7 +162,11 @@ function harness("@/components/ui/spinner": {},"./preview-store": { useDocumentPreviewStore: () => undefined },"@/lib/chevron-icons": {},- "@assistant-ui/react": { useAui: () => ({ threadListItem: () => item }) },+ "@assistant-ui/react": {+ useAui: () => ({ threadListItem: () => item }),+ useAuiState: (select: (s: { threadListItem: { id: string } }) => unknown) =>+ select({ threadListItem: { id: itemId } }),+ },"@/lib/utils": { cn: () => "" },"@/features/chat/stores/chat-runtime-store": {useChatRuntimeStore: store,@@ -226,6 +230,16 @@ function harness(isSupportedSourceName: (name: string) => name.endsWith(".docx"),},"./use-source-drop": {},+ "./use-upload-queue": {+ useUploadQueue: (run: (items: unknown[]) => void) => ({+ enqueue: run,+ queued: 0,+ }),+ },+ "./linked-folder-groups": {+ groupByLinkedFolder: (docs: unknown[]) => ({ loose: docs, folders: [] }),+ useLinkedFolderNames: () => new Map(),+ },"./document-status-chip": {},"./knowledge-base-dialog": { KnowledgeBaseDialog: "KnowledgeBaseDialog" },"./staged-source": { EXPIRY_GRACE_MS: 30_000 },diff --git a/studio/frontend/tests/rag-upload-lifecycle.test.ts b/studio/frontend/tests/rag-upload-lifecycle.test.tsindex ce1f53cc2b1..d094add9fd2 100644--- a/studio/frontend/tests/rag-upload-lifecycle.test.ts+++ b/studio/frontend/tests/rag-upload-lifecycle.test.ts@@ -21,6 +21,9 @@ type Hook = {uploading: boolean;hasIndexing: boolean;upload: (files: File[], scope?: ScopeOverride) => Promise<void>;+ retry: (documentId: string) => Promise<void>;+ canRetry: (documentId: string) => boolean;+ refresh: (opts?: { quiet?: boolean }) => Promise<boolean>;};function deferred<T>() {@@ -39,8 +42,12 @@ function harness(events?: (jobId: string, signal?: AbortSignal) => AsyncGenerator<JobEvent>;getJob?: () => Promise<IndexJob>;list?: () => Promise<RagDocument[]>;+ /** This many uploads are refused before one goes through. */+ failUploads?: number;+ failDeletes?: boolean;} = {},) {+ let refusals = options.failUploads ?? 0;const slots: unknown[] = [];const effects: Array<() => void> = [];let cursor = 0;@@ -118,9 +125,17 @@ function harness("./vision-overrides": { resolveVisionOverrides: async () => ({}) },"../api/rag-api": {uploadThreadDocument: async (threadId: string) => {+ if (refusals > 0) {+ refusals -= 1;+ throw new Error("Server busy");+ }uploads.push(threadId);return uploaded;},+ deleteDocument: async () => {+ if (options.failDeletes) throw new Error("Server busy");+ },+ noteProjectWork: () => undefined,streamJobEvents:options.events ??async function* () {@@ -274,13 +289,115 @@ test("an original job failure is preserved after its event stream ends", async (await flush();await hook.upload([report()]);await flush();- assert.deepEqual(app.render().documents, []);+ // The failed row stays, with its reason, so it can be retried or removed.+ const after = app.render();+ assert.deepEqual(+ after.documents.map(({ id, status, error }) => ({ id, status, error })),+ [{ id: "doc", status: "failed", error: "Invalid PDF" }],+ );+ assert.equal(after.canRetry("doc"), true);assert.deepEqual(app.errors, ["Couldn't index report.pdf"]);} finally {app.dispose();}});+test("a refresh that lists the failure first keeps the row retryable", async () => {+ let listed = false;+ let release = () => {};+ const polled = new Promise<void>((resolve) => {+ release = resolve;+ });+ const app = harness({+ events: async function* () {},+ // The job poll answers only after the list has already reported the failure.+ getJob: async () => {+ await polled;+ return { id: "job", documentId: "doc", status: "failed", error: "Invalid PDF" };+ },+ list: async () =>+ listed+ ? ([{ id: "doc", filename: "report.pdf", status: "failed", error: "Invalid PDF" }] as unknown as RagDocument[])+ : [],+ });+ try {+ let hook = app.render();+ await flush();+ const uploading = hook.upload([report()]);+ await flush();+ listed = true;+ await app.render().refresh({ quiet: true });+ await flush();+ release();+ await uploading;+ await flush();+ hook = app.render();+ assert.deepEqual(+ hook.documents.map(({ id, status }) => ({ id, status })),+ [{ id: "doc", status: "failed" }],+ );+ assert.equal(hook.canRetry("doc"), true);+ } finally {+ app.dispose();+ }+});++test("a refused upload stays as a failed chip and Retry sends the same file again", async () => {+ const app = harness({ failUploads: 1 });+ try {+ let hook = app.render();+ await flush();+ await hook.upload([report()]);+ await flush();+ hook = app.render();+ const failed = hook.documents[0];+ assert.equal(failed?.status, "failed");+ assert.equal(failed?.error, "Server busy");+ assert.equal(hook.canRetry(failed!.id), true);+ assert.deepEqual(app.errors, ["Couldn't upload report.pdf"]);++ await hook.retry(failed!.id);+ await flush();+ hook = app.render();+ assert.deepEqual(app.uploads, ["thread"]);+ assert.deepEqual(+ hook.documents.map(({ id, status }) => ({ id, status })),+ [{ id: "doc", status: "completed" }],+ );+ } finally {+ app.dispose();+ }+});++test("a retry whose delete fails keeps the failed row retryable and uploads nothing", async () => {+ let reads = 0;+ const app = harness({+ failDeletes: true,+ events: async function* () {},+ getJob: async () => ({+ id: "job",+ documentId: "doc",+ status: ++reads === 1 ? "running" : "failed",+ error: "Invalid PDF",+ }),+ });+ try {+ let hook = app.render();+ await flush();+ await hook.upload([report()]);+ await flush();+ hook = app.render();+ await hook.retry("doc");+ await flush();+ hook = app.render();+ assert.deepEqual(app.uploads, ["thread"], "no second upload beside the undeleted one");+ assert.equal(hook.documents[0]?.status, "failed");+ assert.equal(hook.canRetry("doc"), true);+ } finally {+ app.dispose();+ }+});+test("an upload begun without a scope stops at the chat the user left", async () => {const app = harness();try {diff --git a/tests/studio/rag_upload_browser_tests.py b/tests/studio/rag_upload_browser_tests.pyindex 5c6204b1e6b..acd85c4deb5 100644--- a/tests/studio/rag_upload_browser_tests.py+++ b/tests/studio/rag_upload_browser_tests.py@@ -132,7 +132,10 @@ def error(page):timeout = 30000,)assert not page.evaluate("window.sim.hasIndexing || window.sim.uploading")- assert page.evaluate("window.sim.documents") == []+ # The failed row stays, with its reason, so it can be retried or removed.+ documents = page.evaluate("window.sim.documents")+ assert [(d["filename"], d["status"]) for d in documents] == [("report.pdf", "failed")]+ assert documents[0]["error"].startswith("simulated ")def failed_scope(page):