Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: linked folders honour .gitignore, folder uploads in the browser, queued drops and retry for failed sources (#13224)

Michael Han6126f3ec6805
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 5e0b55e2788..6c2b9135398 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -1100,11 +1100,15 @@ def _resolve_load_family(
# Wan2.2 A14B repo, or the reverse), local or remote.
if fam is not None and gguf_filename:
named = detect_video_family(os.path.basename(gguf_filename))
- if named is not None and named.name != fam.name and (
- named.name in SAME_ARCH_VARIANTS.get(fam.name, ())
- or fam.name in SAME_ARCH_VARIANTS.get(named.name, ())
- # A remote pick has no header pass, so a file naming another Wan model picks the base config.
- or (named.name.startswith("wan") and fam.name.startswith("wan"))
+ if (
+ named is not None
+ and named.name != fam.name
+ and (
+ named.name in SAME_ARCH_VARIANTS.get(fam.name, ())
+ or fam.name in SAME_ARCH_VARIANTS.get(named.name, ())
+ # A remote pick has no header pass, so a file naming another Wan model picks the base config.
+ or (named.name.startswith("wan") and fam.name.startswith("wan"))
+ )
):
fam = named
# A local file's header outranks its name (the name still picks 480p vs 720p).
@@ -1112,7 +1116,10 @@ def _resolve_load_family(
if path:
file_fam = detect_video_family(os.path.basename(path))
name, _ = resolve_family_with_content(
- fam.name if fam else None, path, "video", file_family = file_fam.name if file_fam else None
+ fam.name if fam else None,
+ path,
+ "video",
+ file_family = file_fam.name if file_fam else None,
)
if name is None:
return None
@@ -2728,11 +2735,14 @@ class VideoBackend:
# Every pick name is judged, not just an unresolved one: a T2V repo id can hold an I2V / VACE file.
# A family override names the family for an opaque pick; it cannot make refused weights runnable.
from .video_families import unsupported_video_checkpoint_reason
+
reason = unsupported_video_checkpoint_reason(
display_repo_id,
repo_id,
gguf_filename,
- wan21_checkpoint = (gguf_filename or repo_id) if fam and fam.name.startswith("wan2.1") else None,
+ wan21_checkpoint = (gguf_filename or repo_id)
+ if fam and fam.name.startswith("wan2.1")
+ else None,
)
if reason:
raise ValueError(reason)
diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.py
index 8babd1d03f2..783186fbb01 100644
--- a/studio/backend/core/inference/video_families.py
+++ b/studio/backend/core/inference/video_families.py
@@ -591,7 +591,9 @@ _UNSUPPORTED_VIDEO_CHECKPOINTS: tuple[tuple[str, str], ...] = (
"faithfully. Load Wan-AI/Wan2.1-T2V-1.3B-Diffusers or Wan-AI/Wan2.1-T2V-14B-Diffusers instead.",
),
(
- r"(?:^|[/\\])(?=[^/\\]*" + _WAN21_TOKEN + r")(?=[^/\\]*(?<![a-z0-9])(?:i2v|flf2v)(?![a-z0-9]))"
+ r"(?:^|[/\\])(?=[^/\\]*"
+ + _WAN21_TOKEN
+ + r")(?=[^/\\]*(?<![a-z0-9])(?:i2v|flf2v)(?![a-z0-9]))"
r"[^/\\]*[/\\]*$",
"This is a Wan2.1 image-to-video checkpoint (I2V-14B 480P / 720P or FLF2V). It generates "
"from a start frame through its own image-conditioned transformer and a CLIP vision "
@@ -601,7 +603,9 @@ _UNSUPPORTED_VIDEO_CHECKPOINTS: tuple[tuple[str, str], ...] = (
),
(
r"(?:^|[/\\])(?=[^/\\]*" + _WAN21_TOKEN + r")(?=[^/\\]*(?:(?<![a-z0-9])"
- r"(?:vace|magref|phantom|multitalk|infinitetalk|controlnet|recammaster)|" + _WAN21_TOKEN + r"-fun)"
+ r"(?:vace|magref|phantom|multitalk|infinitetalk|controlnet|recammaster)|"
+ + _WAN21_TOKEN
+ + r"-fun)"
r"(?![a-z0-9]))[^/\\]*[/\\]*$",
"This is a Wan2.1 conditioning variant (VACE, Fun, MAGREF, Phantom, talking-head, "
"ControlNet or ReCamMaster). Its transformer takes extra control inputs the text-to-video "
@@ -624,7 +628,9 @@ def unsupported_video_checkpoint_reason(
name = re.split(r"[/\\]", needle.rstrip("/\\"))[-1]
# A leading "fun" is too generic to mark a Fun release unless it names one (version, size, InP, Control); the
# other markers are still judged.
- generic_fun = re.match(r"fun(?![a-z0-9])", name) and not re.match(r"fun-(?:v\d|1-3b|14b|inp|control)", name)
+ generic_fun = re.match(r"fun(?![a-z0-9])", name) and not re.match(
+ r"fun-(?:v\d|1-3b|14b|inp|control)", name
+ )
needles.append(("wan2-1-t2v-" if generic_fun else "wan2-1-") + name)
for needle in needles:
for pattern, message in _UNSUPPORTED_VIDEO_CHECKPOINTS:
diff --git a/studio/backend/core/rag/config.py b/studio/backend/core/rag/config.py
index 87dba7bf85d..3f1832e8084 100644
--- a/studio/backend/core/rag/config.py
+++ b/studio/backend/core/rag/config.py
@@ -88,6 +88,8 @@ MAX_UPLOAD_BYTES = int(os.environ.get("RAG_MAX_UPLOAD_BYTES", str(200 * 1024 * 1
# Caps prevent an accidentally broad linked folder from becoming an unbounded ingestion queue.
FOLDER_SYNC_INTERVAL_S = float(os.environ.get("RAG_FOLDER_SYNC_INTERVAL_S", "30"))
FOLDER_MAX_FILES = int(os.environ.get("RAG_FOLDER_MAX_FILES", "10000"))
+# Linked-folder plain-text files above this are logs/data dumps, not documents; 0 disables.
+FOLDER_MAX_TEXT_BYTES = int(os.environ.get("RAG_FOLDER_MAX_TEXT_BYTES", str(20 * 1024 * 1024)))
FOLDER_JOB_HISTORY_LIMIT = int(os.environ.get("RAG_FOLDER_JOB_HISTORY_LIMIT", "200"))
# Linked-folder documents ingested concurrently, clamped to 1..4.
FOLDER_INGEST_WORKERS = int(os.environ.get("RAG_FOLDER_INGEST_WORKERS", "2"))
diff --git a/studio/backend/core/rag/folder_sync.py b/studio/backend/core/rag/folder_sync.py
index da4db383569..1bdabc402ca 100644
--- a/studio/backend/core/rag/folder_sync.py
+++ b/studio/backend/core/rag/folder_sync.py
@@ -31,7 +31,7 @@ from pathlib import Path, PurePosixPath
from core.rag import account_db as rag_db
from utils.paths import ensure_dir, rag_uploads_root
-from . import config, ingestion, job_leases, store
+from . import config, gitignore, ingestion, job_leases, store
from utils.paths.path_utils import is_appledouble_metadata
logger = logging.getLogger(__name__)
@@ -90,6 +90,53 @@ def _is_ignored_scan_dir(name: str, path: str) -> bool:
)
+# Build output is skipped only beside a project manifest: a bare "build" folder in a vault is content.
+_BUILD_OUTPUT_DIRS = frozenset({"dist", "build", "out", "target", "coverage", "htmlcov", "vendor"})
+_PROJECT_MANIFESTS = (
+ "package.json",
+ "pyproject.toml",
+ "setup.py",
+ "setup.cfg",
+ "Cargo.toml",
+ "pom.xml",
+ "build.gradle",
+ "build.gradle.kts",
+ "go.mod",
+ "composer.json",
+ "CMakeLists.txt",
+ "Gemfile",
+ "mix.exs",
+)
+_MAX_REPORTED_FAILURES = 500
+
+
+def _is_build_output_dir(name: str, parent: str) -> bool:
+ return name.lower() in _BUILD_OUTPUT_DIRS and any(
+ os.path.isfile(os.path.join(parent, manifest)) for manifest in _PROJECT_MANIFESTS
+ )
+
+
+def _read_gitignore(directory: str, base: str) -> list[gitignore.Rule]:
+ """An unreadable .gitignore fails the scan: indexing without it would take in excluded files."""
+ path = os.path.join(directory, ".gitignore")
+ if not os.path.isfile(path) or os.path.islink(path):
+ return []
+ shown = f"{base}/.gitignore" if base else ".gitignore"
+ try:
+ with open(path, "rb") as handle:
+ data = handle.read(gitignore.MAX_GITIGNORE_BYTES + 1)
+ except OSError as exc:
+ raise RuntimeError(f"Couldn't read {shown}: {exc.strerror or exc}") from exc
+ if len(data) > gitignore.MAX_GITIGNORE_BYTES:
+ raise RuntimeError(f"{shown} is larger than {gitignore.MAX_GITIGNORE_BYTES // 1024} KB")
+ # utf-8-sig drops a leading BOM, as git does (dir.c add_patterns_from_buffer, skip_utf8_bom).
+ return gitignore.parse(data.decode("utf-8-sig", "replace"), base)
+
+
+def _is_plain_text_ext(ext: str) -> bool:
+ return ext == ".txt" or ext in config.SOURCE_TEXT_EXTS
+
+
def _is_ignored_scan_file(name: str) -> bool:
# .env and Terraform state can expose plaintext secrets in retrieval; lockfiles add noise.
lower = name.lower()
@@ -1052,28 +1099,44 @@ def job_events(job_id: str):
def _scan(
- root: str, expected_identity: tuple[int, int] | None = None
+ root: str,
+ expected_identity: tuple[int, int] | None = None,
+ report: dict[str, int] | None = None,
) -> tuple[dict[str, dict], tuple[int, int]]:
+ """Map relative path -> metadata; `report` receives counts of deliberately skipped entries."""
from hub.storage.scan_folders import contains_sensitive_path_component, is_denied_system_path
identity = _root_identity(root)
if expected_identity is not None and identity != expected_identity:
raise RuntimeError("Linked folder root identity changed")
+ skipped = {"gitignored": 0, "build_dirs": 0, "too_large": 0, "over_limit": 0}
found: dict[str, dict] = {}
mount_points = _mount_points()
root_identities = frozenset({identity}) if identity[1] not in (None, 0) else frozenset()
- pending = [(root, frozenset({_path_key(root)}), root_identities, 0)]
+ pending = [(root, frozenset({_path_key(root)}), root_identities, 0, ())]
+ # Collect past the cap so the kept set is the sorted first FOLDER_MAX_FILES, stable across scans.
+ hard_cap = config.FOLDER_MAX_FILES * 2 if config.FOLDER_MAX_FILES else 0
while pending:
- directory, ancestor_paths, ancestor_identities, depth = pending.pop()
+ directory, ancestor_paths, ancestor_identities, depth, rules = pending.pop()
+ directory_rel = os.path.relpath(directory, root).replace(os.sep, "/")
+ directory_rel = "" if directory_rel == "." else directory_rel
+ rules = (*rules, *_read_gitignore(directory, directory_rel))
with os.scandir(directory) as entries:
for entry in entries:
full = entry.path
if entry.is_symlink():
continue
+ rel = os.path.relpath(full, root).replace(os.sep, "/")
if entry.is_dir(follow_symlinks = False):
if _is_ignored_scan_dir(entry.name, full):
continue
+ if rules and gitignore.is_ignored(rel, True, rules):
+ skipped["gitignored"] += 1
+ continue
+ if _is_build_output_dir(entry.name, directory):
+ skipped["build_dirs"] += 1
+ continue
resolved = os.path.realpath(full)
if (
not _is_within(root, resolved)
@@ -1106,6 +1169,7 @@ def _scan(
ancestor_paths | {resolved_key},
child_identities,
depth + 1,
+ rules,
)
)
continue
@@ -1113,7 +1177,11 @@ def _scan(
continue
if _is_ignored_scan_file(entry.name):
continue
- if os.path.splitext(entry.name)[1].lower() not in config.UPLOAD_EXTS:
+ ext = os.path.splitext(entry.name)[1].lower()
+ if ext not in config.UPLOAD_EXTS:
+ continue
+ if rules and gitignore.is_ignored(rel, False, rules):
+ skipped["gitignored"] += 1
continue
# Finder metadata carries the document's extension, so a text parser would embed and cite it as a
# real chunk.
@@ -1132,7 +1200,15 @@ def _scan(
# authoritative for
# deletion.
pass
- rel = os.path.relpath(full, root).replace(os.sep, "/")
+ if st.st_size == 0:
+ continue
+ if (
+ config.FOLDER_MAX_TEXT_BYTES
+ and _is_plain_text_ext(ext)
+ and st.st_size > config.FOLDER_MAX_TEXT_BYTES
+ ):
+ skipped["too_large"] += 1
+ continue
found[rel] = {
"path": full,
"size_bytes": st.st_size,
@@ -1143,12 +1219,19 @@ def _scan(
# WebDAV drivers report different ids per call path.
"identity_from_path": from_path,
}
- if config.FOLDER_MAX_FILES and len(found) > config.FOLDER_MAX_FILES:
+ if hard_cap and len(found) > hard_cap:
raise RuntimeError(
- f"Folder contains more than the {config.FOLDER_MAX_FILES} supported files limit"
+ f"Folder contains more than twice the {config.FOLDER_MAX_FILES} supported "
+ "files limit. Link a smaller folder, or add a .gitignore for what to skip."
)
if _root_identity(root) != identity:
raise RuntimeError("Linked folder root identity changed during scan")
+ if config.FOLDER_MAX_FILES and len(found) > config.FOLDER_MAX_FILES:
+ kept = sorted(found)[: config.FOLDER_MAX_FILES]
+ skipped["over_limit"] = len(found) - len(kept)
+ found = {rel: found[rel] for rel in kept}
+ if report is not None:
+ report.update(skipped)
return found, identity
@@ -1566,7 +1649,8 @@ def _reconcile_folder(job_id: str) -> None:
try:
_check_running()
expected_identity = _load_identity(folder["root_device"], folder["root_inode"])
- current, scanned_identity = _scan(folder["path"], expected_identity)
+ skipped: dict[str, int] = {}
+ current, scanned_identity = _scan(folder["path"], expected_identity, skipped)
_check_running()
except (_SyncStopped, _SyncCancelled, _LeaseLost):
raise
@@ -1621,6 +1705,7 @@ def _reconcile_folder(job_id: str) -> None:
_set_job(job_id, stage = "ingesting", discovered = len(current), renamed = renamed)
added = changed_count = 0
failures: list[str] = []
+ failure_errors: dict[str, str] = {}
withheld: set[str] = set()
processed = 0
stop_event = getattr(_worker_state, "stop_event", _stop)
@@ -1666,9 +1751,12 @@ def _reconcile_folder(job_id: str) -> None:
_cleanup_pending_ingestion(pending, result)
raise result.error
failures.append(pending.relative_path)
+ reason = _error_text(result.error, folder["path"])
+ failure_errors[pending.relative_path] = reason
ingest_failed[pending.relative_path] = {
"sig": _failure_signature(pending.metadata),
"at": time.time(),
+ "error": reason,
}
# A failed file may be a rename or copy of a vanished path, so grant one pass.
withheld.update(missing - already_withheld)
@@ -1694,6 +1782,7 @@ def _reconcile_folder(job_id: str) -> None:
raise
except Exception as exc: # noqa: BLE001
failures.append(pending.relative_path)
+ failure_errors[pending.relative_path] = _error_text(exc, folder["path"])
withheld.update(missing - already_withheld)
logger.warning(
"linked-folder ingestion failed for %s",
@@ -1754,6 +1843,7 @@ def _reconcile_folder(job_id: str) -> None:
raise _SyncCancelled
failures.append(rel)
ingest_failed[rel] = known_failed[rel]
+ failure_errors[rel] = known_failed[rel].get("error") or "Indexing failed"
if not cached_grace:
# missing only shrinks during the pass, so once covers every later failure
withheld.update(missing - already_withheld)
@@ -1821,8 +1911,9 @@ def _reconcile_folder(job_id: str) -> None:
except (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged):
_remove_snapshot(snapshot)
raise
- except Exception:
+ except Exception as exc: # noqa: BLE001
failures.append(rel)
+ failure_errors[rel] = _error_text(exc, folder["path"])
withheld.update(missing - already_withheld)
logger.warning("linked-folder ingestion failed for %s", rel, exc_info = True)
_remove_snapshot(snapshot)
@@ -1873,11 +1964,33 @@ def _reconcile_folder(job_id: str) -> None:
error = error,
completed_at = _now(),
)
+ scan_report = {
+ "skipped": {key: count for key, count in skipped.items() if count},
+ "failures": [
+ {"path": rel, "error": failure_errors.get(rel, "Indexing failed")}
+ for rel in sorted(set(failures))[:_MAX_REPORTED_FAILURES]
+ ],
+ "failureCount": len(set(failures)),
+ }
+ # last_change_at moves only when the indexed set did, so idle passes don't make clients refetch.
+ changed_sources = added + changed_count + deleted + renamed > 0
+ now = _now()
with closing(rag_db.get_connection()) as conn:
conn.execute(
- "UPDATE linked_folders SET status=?, last_error=?, last_scan_at=?, updated_at=? "
+ "UPDATE linked_folders SET status=?, last_error=?, last_scan_at=?, updated_at=?, "
+ "scan_report=?, last_change_at=CASE WHEN ? THEN ? ELSE COALESCE(last_change_at, ?) END "
"WHERE id=? AND delete_remove_index IS NULL",
- ("ready" if error is None else "error", error, _now(), _now(), folder["id"]),
+ (
+ "ready" if error is None else "error",
+ error,
+ now,
+ now,
+ json.dumps(scan_report),
+ changed_sources,
+ now,
+ now,
+ folder["id"],
+ ),
)
conn.commit()
_prune_terminal_jobs(conn)
diff --git a/studio/backend/core/rag/gitignore.py b/studio/backend/core/rag/gitignore.py
new file mode 100644
index 00000000000..26eac250d12
--- /dev/null
+++ b/studio/backend/core/rag/gitignore.py
@@ -0,0 +1,148 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The .gitignore rules a linked-folder scan honours, per https://git-scm.com/docs/gitignore."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+MAX_GITIGNORE_BYTES = 256 * 1024
+_MAX_GLOBSTARS = 3
+
+
+@dataclass(frozen = True)
+class Rule:
+ base: str # posix directory of the .gitignore, relative to the scan root; "" for the root
+ regex: re.Pattern[str]
+ negate: bool
+ dir_only: bool
+
+
+def _tokens(pattern: str) -> list[tuple[str, str]]:
+ """(kind, regex) pairs: "star", "char" (one non-"/" character), or "other"."""
+ out: list[tuple[str, str]] = []
+ i, n = 0, len(pattern)
+ while i < n:
+ c = pattern[i]
+ if c == "*":
+ if pattern.startswith("**", i):
+ before = i == 0 or pattern[i - 1] == "/"
+ after = i + 2 == n or pattern[i + 2] == "/"
+ if before and after:
+ if i + 2 == n:
+ out.append(("other", ".*"))
+ else:
+ out.append(("other", "(?:.*/)?"))
+ i += 1
+ i += 2
+ continue
+ out.append(("star", "[^/]*"))
+ elif c == "?":
+ out.append(("char", "[^/]"))
+ elif c == "[":
+ start = i + 1
+ if pattern[start : start + 1] in ("!", "^"):
+ start += 1
+ if pattern[start : start + 1] == "]":
+ start += 1
+ end = pattern.find("]", start)
+ if end == -1:
+ out.append(("char", re.escape(c)))
+ else:
+ body = pattern[i + 1 : end].replace("\\", "\\\\").replace("[", "\\[")
+ if body[:1] in ("!", "^"):
+ # Like "*" and "?", a negated class never matches "/".
+ body = "^/" + body[1:]
+ out.append(("char", "[" + body + "]"))
+ i = end
+ else:
+ if c == "\\" and i + 1 < n:
+ i += 1
+ c = pattern[i]
+ out.append(("other" if c == "/" else "char", re.escape(c)))
+ i += 1
+ return out
+
+
+def _translate(pattern: str) -> str:
+ tokens = _tokens(pattern)
+ out: list[str] = []
+ i, n = 0, len(tokens)
+ while i < n:
+ kind, regex = tokens[i]
+ if kind != "star":
+ out.append(regex)
+ i += 1
+ continue
+ while i < n and tokens[i][0] == "star":
+ i += 1
+ end = i
+ while end < n and tokens[end][0] == "char":
+ end += 1
+ chunk = "".join(regex for _, regex in tokens[i:end])
+ if end < n and tokens[end][0] == "star":
+ # Atomic group (as fnmatch.translate on Python < 3.11): plain [^/]* groups backtrack
+ # exponentially on "*a*a*a...b", and the earliest fit is as good as any when "*" follows.
+ group = f"g{len(out)}"
+ out.append(f"(?=(?P<{group}>[^/]*?{chunk}))(?P={group})")
+ else:
+ out.append("[^/]*" + chunk)
+ i = end
+ return "".join(out)
+
+
+def parse(text: str, base: str) -> list[Rule]:
+ rules: list[Rule] = []
+ for raw in text.splitlines():
+ line = raw.rstrip("\r")
+ stripped = line.rstrip(" ")
+ if stripped.endswith("\\") and len(stripped) < len(line):
+ stripped += " "
+ line = stripped
+ if not line or line.startswith("#"):
+ continue
+ negate = line.startswith("!")
+ if negate:
+ line = line[1:]
+ elif line.startswith(("\\!", "\\#")):
+ line = line[1:]
+ dir_only = line.endswith("/")
+ line = line.rstrip("/")
+ if not line:
+ continue
+ # A leading or middle slash anchors to the .gitignore's directory; else any depth.
+ anchored = "/" in line
+ line = line.lstrip("/")
+ body = _translate(line)
+ if not anchored:
+ body = "(?:.*/)?" + body
+ # Each "**/" multiplies the depths tried; real rules use one or two.
+ if body.count("(?:.*/)?") > _MAX_GLOBSTARS:
+ continue
+ try:
+ regex = re.compile(f"^{body}$", re.DOTALL)
+ except re.error:
+ # A malformed class like "[z-a]" matches nothing in git; it must not fail the scan.
+ continue
+ rules.append(Rule(base, regex, negate, dir_only))
+ return rules
+
+
+def is_ignored(rel: str, is_dir: bool, rules: tuple[Rule, ...]) -> bool:
+ """Whether `rel` (posix, relative to the scan root) is ignored; the last matching rule wins."""
+ ignored = False
+ for rule in rules:
+ if rule.dir_only and not is_dir:
+ continue
+ if rule.base:
+ prefix = rule.base + "/"
+ if not rel.startswith(prefix):
+ continue
+ sub = rel[len(prefix) :]
+ else:
+ sub = rel
+ if rule.regex.match(sub):
+ ignored = not rule.negate
+ return ignored
diff --git a/studio/backend/routes/rag.py b/studio/backend/routes/rag.py
index c6c8f4c22f3..aae35747bf5 100644
--- a/studio/backend/routes/rag.py
+++ b/studio/backend/routes/rag.py
@@ -327,6 +327,12 @@ def _folder_view(row: dict) -> dict:
if row["status"] in {"error", "retired"}
else "idle"
)
+ try:
+ report = json.loads(row.get("scan_report") or "{}")
+ except ValueError:
+ report = {}
+ if not isinstance(report, dict):
+ report = {}
return {
"id": row["id"],
"displayName": row["name"],
@@ -335,6 +341,10 @@ def _folder_view(row: dict) -> dict:
"status": status,
"error": row.get("last_error"),
"lastSyncedAt": row.get("last_scan_at"),
+ "lastChangedAt": row.get("last_change_at"),
+ "skipped": report.get("skipped") or {},
+ "failures": report.get("failures") or [],
+ "failureCount": report.get("failureCount") or 0,
"documentCount": row.get("file_count", 0),
"activeJobId": row.get("active_job_id"),
"scopeName": row.get("scope_name"),
@@ -894,6 +904,7 @@ def _folder_job_view(row: dict) -> dict:
"processedFiles": processed,
"indexedFiles": (row.get("added") or 0) + (row.get("changed") or 0),
"removedFiles": row.get("deleted") or 0,
+ "renamedFiles": row.get("renamed") or 0,
"failedFiles": row.get("failed") or 0,
"error": row.get("error"),
"createdAt": row.get("created_at"),
diff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.py
index bcbdec8b89a..2367fae688d 100644
--- a/studio/backend/storage/rag_db.py
+++ b/studio/backend/storage/rag_db.py
@@ -176,6 +176,8 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:
last_scan_at TEXT,
withheld_paths TEXT,
failed_files TEXT,
+ scan_report TEXT,
+ last_change_at TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
UNIQUE(scope, path)
@@ -298,6 +300,10 @@ def ensure_linked_folder_columns(conn: sqlite3.Connection) -> None:
conn.execute("ALTER TABLE linked_folders ADD COLUMN withheld_paths TEXT")
if folder_cols and "failed_files" not in folder_cols:
conn.execute("ALTER TABLE linked_folders ADD COLUMN failed_files TEXT")
+ if folder_cols and "scan_report" not in folder_cols:
+ conn.execute("ALTER TABLE linked_folders ADD COLUMN scan_report TEXT")
+ if folder_cols and "last_change_at" not in folder_cols:
+ conn.execute("ALTER TABLE linked_folders ADD COLUMN last_change_at TEXT")
def reset_schema_state_for_tests() -> None:
diff --git a/studio/backend/tests/test_mlx_diffusion_backend.py b/studio/backend/tests/test_mlx_diffusion_backend.py
index bdfcad51c89..e9ee3c9c2e7 100644
--- a/studio/backend/tests/test_mlx_diffusion_backend.py
+++ b/studio/backend/tests/test_mlx_diffusion_backend.py
@@ -1043,7 +1043,11 @@ def test_base_companions_come_from_the_repo_the_plan_staged(monkeypatch):
assert fetched == [("unsloth/Z-Image-Turbo", f, "abc") for f in files]
-def _companion_snapshot(root, complete = True, tokenizer_config = True):
+def _companion_snapshot(
+ root,
+ complete = True,
+ tokenizer_config = True,
+):
root.mkdir(parents = True, exist_ok = True)
for sub in ("text_encoder", "tokenizer", "vae"):
(root / sub).mkdir(exist_ok = True)
@@ -1120,14 +1124,23 @@ def test_cache_only_companions_skip_an_incomplete_upstream_for_the_staged_mirror
"unsloth/Z-Image-Turbo": _companion_snapshot(tmp_path / "mirror"),
}
- def _snapshot(repo, *, cache_dir = None, local_files_only = False, **_k):
+ def _snapshot(
+ repo,
+ *,
+ cache_dir = None,
+ local_files_only = False,
+ **_k,
+ ):
assert local_files_only
if cache_dir != live or repo not in snapshots:
raise LocalEntryNotFoundError("not cached")
return snapshots[repo]
monkeypatch.setattr(huggingface_hub, "snapshot_download", _snapshot)
- assert mb.fetch_base_snapshot(BASE, None, local_files_only = True) == snapshots["unsloth/Z-Image-Turbo"]
+ assert (
+ mb.fetch_base_snapshot(BASE, None, local_files_only = True)
+ == snapshots["unsloth/Z-Image-Turbo"]
+ )
del snapshots["unsloth/Z-Image-Turbo"]
with pytest.raises(LocalEntryNotFoundError, match = "complete"):
mb.fetch_base_snapshot(BASE, None, local_files_only = True)
@@ -1326,13 +1339,22 @@ def test_cache_only_companions_skip_a_snapshot_without_tokenizer_config(monkeypa
"unsloth/Z-Image-Turbo": _companion_snapshot(tmp_path / "mirror"),
}
- def _snapshot(repo, *, cache_dir = None, local_files_only = False, **_k):
+ def _snapshot(
+ repo,
+ *,
+ cache_dir = None,
+ local_files_only = False,
+ **_k,
+ ):
if cache_dir != live or repo not in snapshots:
raise LocalEntryNotFoundError("not cached")
return snapshots[repo]
monkeypatch.setattr(huggingface_hub, "snapshot_download", _snapshot)
- assert mb.fetch_base_snapshot(BASE, None, local_files_only = True) == snapshots["unsloth/Z-Image-Turbo"]
+ assert (
+ mb.fetch_base_snapshot(BASE, None, local_files_only = True)
+ == snapshots["unsloth/Z-Image-Turbo"]
+ )
def test_a_cancelled_load_keeps_the_companion_mirror_guarded(env, monkeypatch):
@@ -1362,7 +1384,9 @@ def test_a_cancelled_load_keeps_the_companion_mirror_guarded(env, monkeypatch):
time.sleep(0.01)
try:
assert "unsloth/Z-Image-Turbo" in backend.draining_repo_ids()
- assert "still releasing" in (deletion._diffusion_blocks_delete("unsloth/Z-Image-Turbo") or "")
+ assert "still releasing" in (
+ deletion._diffusion_blocks_delete("unsloth/Z-Image-Turbo") or ""
+ )
finally:
gate.set()
unloader.join(5)
diff --git a/studio/backend/tests/test_rag_folder_scan_filters.py b/studio/backend/tests/test_rag_folder_scan_filters.py
new file mode 100644
index 00000000000..0c005166784
--- /dev/null
+++ b/studio/backend/tests/test_rag_folder_scan_filters.py
@@ -0,0 +1,261 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""What a linked-folder scan leaves out (.gitignore, build output, oversized text, the file cap),
+and what each pass reports back."""
+
+from __future__ import annotations
+
+import json
+import time
+from contextlib import closing
+from pathlib import Path
+
+import pytest
+
+from core.rag import config, folder_sync, gitignore
+from storage import rag_db
+
+requires_sqlite_vec = pytest.mark.skipif(
+ not rag_db.RAG_AVAILABLE, reason = "sqlite-vec is not installed"
+)
+
+
+def _ignored(
+ rules_text: str,
+ rel: str,
+ is_dir: bool = False,
+ base: str = "",
+) -> bool:
+ return gitignore.is_ignored(rel, is_dir, tuple(gitignore.parse(rules_text, base)))
+
+
+ "rules, rel, is_dir, expected",
+ [
+ ("*.log", "app.log", False, True),
+ ("*.log", "deep/nested/app.log", False, True),
+ ("*.log", "app.log.md", False, False),
+ ("/root.md", "root.md", False, True),
+ ("/root.md", "sub/root.md", False, False),
+ ("docs/*.md", "docs/a.md", False, True),
+ ("docs/*.md", "docs/sub/a.md", False, False),
+ ("docs/*.md", "other/docs/a.md", False, False),
+ ("generated/", "generated", True, True),
+ ("generated/", "generated", False, False),
+ ("**/fixtures", "a/b/fixtures", True, True),
+ ("a/**/b.md", "a/b.md", False, True),
+ ("a/**/b.md", "a/x/y/b.md", False, True),
+ ("logs/**", "logs/today.txt", False, True),
+ ("*.md\n!keep.md", "keep.md", False, False),
+ ("*.md\n!keep.md", "drop.md", False, True),
+ ("# comment\n\n", "anything.md", False, False),
+ ("\\#hash.md", "#hash.md", False, True),
+ ("file?.md", "file1.md", False, True),
+ ("file[0-9].md", "file7.md", False, True),
+ ("file[!0-9].md", "file7.md", False, False),
+ ("a[!x]b", "a/b", False, False),
+ ("*a*b", "xaab", False, True),
+ ("*a*b", "xaa/b", False, False),
+ ("*.min.*", "app.min.js", False, True),
+ ("*.min.*", "app.js", False, False),
+ ("a*b*c", "abbc", False, True),
+ ("a*b*c", "acbc", False, True),
+ ("a*b*c", "acb", False, False),
+ ("**/x/**/y/*.md", "p/x/q/r/y/a.md", False, True),
+ ],
+)
+def test_gitignore_patterns(rules, rel, is_dir, expected):
+ assert _ignored(rules, rel, is_dir) is expected
+
+
+def test_a_malformed_class_drops_only_its_own_rule():
+ rules = "[z-a].txt\n*.log"
+ assert not _ignored(rules, "a.txt")
+ assert _ignored(rules, "debug.log")
+
+
+def test_hostile_rules_match_in_bounded_time():
+ star_rule = gitignore.parse("*a" * 15 + "b", "")
+ globstar_rule = gitignore.parse("a/**/" * 7 + "b", "")
+ assert len(star_rule) == 1
+ # Too many "**/" to match in bounded time: the rule is left out.
+ assert globstar_rule == []
+ started = time.monotonic()
+ assert not gitignore.is_ignored("a" * 46, False, tuple(star_rule))
+ assert time.monotonic() - started < 1
+
+
+def test_nested_gitignore_applies_only_below_its_directory():
+ assert _ignored("*.md", "sub/a.md", base = "sub")
+ assert not _ignored("*.md", "a.md", base = "sub")
+ assert not _ignored("*.md", "subway/a.md", base = "sub")
+
+
+def source(linkable_temp_base, tmp_path) -> Path:
+ root = linkable_temp_base / f"scan-{tmp_path.name}"
+ root.mkdir(parents = True)
+ return root
+
+
+def _write(
+ root: Path,
+ rel: str,
+ text: str = "words",
+) -> None:
+ path = root / rel
+ path.parent.mkdir(parents = True, exist_ok = True)
+ path.write_text(text, encoding = "utf-8")
+
+
+def test_scan_honours_root_and_nested_gitignore(source):
+ _write(source, ".gitignore", "*.log\ngenerated/\n!important.log\n")
+ _write(source, "notes.md")
+ _write(source, "debug.log")
+ _write(source, "important.log")
+ _write(source, "generated/out.md")
+ _write(source, "pkg/.gitignore", "local.md\n")
+ _write(source, "pkg/local.md")
+ _write(source, "pkg/kept.md")
+ _write(source, "local.md")
+ report: dict[str, int] = {}
+ found, _ = folder_sync._scan(str(source), report = report)
+ assert set(found) == {"notes.md", "important.log", "pkg/kept.md", "local.md"}
+ assert report["gitignored"] == 3
+
+
+def test_an_unreadable_gitignore_fails_the_scan(source, monkeypatch):
+ _write(source, ".gitignore", "secret.md\n")
+ _write(source, "secret.md")
+ real_open = open
+
+ def locked_open(path, *args, **kwargs):
+ if str(path).endswith(".gitignore"):
+ raise PermissionError(13, "Permission denied")
+ return real_open(path, *args, **kwargs)
+
+ monkeypatch.setattr("builtins.open", locked_open)
+ with pytest.raises(RuntimeError, match = "Couldn't read .gitignore"):
+ folder_sync._scan(str(source))
+
+
+def test_a_bom_does_not_hide_the_first_gitignore_rule(source):
+ (source / ".gitignore").write_bytes(b"\xef\xbb\xbfsecret.md\nother.md\n")
+ _write(source, "secret.md")
+ _write(source, "other.md")
+ _write(source, "notes.md")
+ found, _ = folder_sync._scan(str(source))
+ assert set(found) == {"notes.md"}
+
+
+def test_scan_skips_build_output_only_beside_a_project_manifest(source):
+ _write(source, "app/package.json", "{}")
+ _write(source, "app/dist/bundle.js", "bundled")
+ _write(source, "app/src/index.js", "source")
+ _write(source, "vault/build/plan.md", "the user's own build notes")
+ report: dict[str, int] = {}
+ found, _ = folder_sync._scan(str(source), report = report)
+ assert set(found) == {"app/package.json", "app/src/index.js", "vault/build/plan.md"}
+ assert report["build_dirs"] == 1
+
+
+def test_scan_skips_oversized_plain_text_but_not_documents(source, monkeypatch):
+ monkeypatch.setattr(config, "FOLDER_MAX_TEXT_BYTES", 10)
+ _write(source, "dump.csv", "x" * 50)
+ _write(source, "small.csv", "a,b")
+ _write(source, "big.md", "x" * 50)
+ report: dict[str, int] = {}
+ found, _ = folder_sync._scan(str(source), report = report)
+ # .md is a document type, not a SOURCE_TEXT_EXTS data dump.
+ assert set(found) == {"small.csv", "big.md"}
+ assert report["too_large"] == 1
+
+
+def test_scan_skips_empty_files(source):
+ _write(source, "pkg/__init__.py", "")
+ _write(source, "pkg/core.py", "def core(): pass")
+ found, _ = folder_sync._scan(str(source))
+ assert set(found) == {"pkg/core.py"}
+
+
+def test_scan_keeps_the_first_files_in_sorted_order_past_the_cap(source, monkeypatch):
+ monkeypatch.setattr(config, "FOLDER_MAX_FILES", 3)
+ for name in ("e.md", "a.md", "d.md", "b.md", "c.md"):
+ _write(source, name)
+ report: dict[str, int] = {}
+ found, _ = folder_sync._scan(str(source), report = report)
+ assert set(found) == {"a.md", "b.md", "c.md"}
+ assert report["over_limit"] == 2
+
+
+def test_scan_still_refuses_a_folder_far_past_the_cap(source, monkeypatch):
+ monkeypatch.setattr(config, "FOLDER_MAX_FILES", 2)
+ for index in range(5):
+ _write(source, f"{index}.md")
+ with pytest.raises(RuntimeError, match = "supported files limit"):
+ folder_sync._scan(str(source))
+
+
+def _folder_row(folder_id: str) -> dict:
+ with closing(rag_db.get_connection()) as conn:
+ return dict(
+ conn.execute("SELECT * FROM linked_folders WHERE id=?", (folder_id,)).fetchone()
+ )
+
+
+def _run(folder_id: str) -> dict:
+ job_id = folder_sync.request_sync(folder_id)
+ folder_sync.reconcile_folder(job_id)
+ return folder_sync.get_job(job_id)
+
+
+@requires_sqlite_vec
+def test_a_pass_that_changes_nothing_leaves_last_change_at_alone(rag_home, stub_embeddings):
+ source = rag_home / "source"
+ source.mkdir()
+ (source / "notes.txt").write_text("alpha words", encoding = "utf-8")
+ folder = folder_sync.create_folder(
+ scope_type = "project", scope_id = "p1", path = str(source), name = "Docs"
+ )
+ assert _run(folder["id"])["status"] == "completed"
+ first = _folder_row(folder["id"])
+ assert first["last_change_at"]
+
+ assert _run(folder["id"])["status"] == "completed"
+ idle = _folder_row(folder["id"])
+ assert idle["last_change_at"] == first["last_change_at"]
+
+ (source / "more.txt").write_text("beta words", encoding = "utf-8")
+ assert _run(folder["id"])["added"] == 1
+ assert _folder_row(folder["id"])["last_change_at"] != first["last_change_at"]
+
+
+@requires_sqlite_vec
+def test_every_failed_file_is_listed_with_its_reason(rag_home, stub_embeddings, monkeypatch):
+ source = rag_home / "source"
+ source.mkdir()
+ for index in range(5):
+ (source / f"bad{index}.txt").write_text("words", encoding = "utf-8")
+ folder = folder_sync.create_folder(
+ scope_type = "project", scope_id = "p1", path = str(source), name = "Docs"
+ )
+ monkeypatch.setattr(
+ folder_sync.ingestion,
+ "start_ingestion",
+ lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("embed unavailable")),
+ )
+ job = _run(folder["id"])
+ assert job["status"] == "failed"
+ assert "and 2 more" in job["error"]
+
+ report = json.loads(_folder_row(folder["id"])["scan_report"])
+ assert report["failureCount"] == 5
+ assert [entry["path"] for entry in report["failures"]] == [f"bad{i}.txt" for i in range(5)]
+ assert all(entry["error"] == "embed unavailable" for entry in report["failures"])
+
+ from routes.rag import _folder_view
+
+ view = _folder_view(_folder_row(folder["id"]))
+ assert view["failureCount"] == 5
+ assert view["failures"][0] == {"path": "bad0.txt", "error": "embed unavailable"}
diff --git a/studio/backend/tests/test_rag_linked_folders.py b/studio/backend/tests/test_rag_linked_folders.py
index 8240c2d7390..efd51c310de 100644
--- a/studio/backend/tests/test_rag_linked_folders.py
+++ b/studio/backend/tests/test_rag_linked_folders.py
@@ -1013,8 +1013,12 @@ def test_large_windows_file_identity_round_trips_through_sqlite(
identity = (1 << 63, 1 << 127)
real_scan, real_snapshot = folder_sync._scan, folder_sync._snapshot
- def scan(root, expected_identity = None):
- found, root_identity = real_scan(root, expected_identity)
+ def scan(
+ root,
+ expected_identity = None,
+ report = None,
+ ):
+ found, root_identity = real_scan(root, expected_identity, report)
for metadata in found.values():
metadata["scanned"], (metadata["device"], metadata["inode"]) = (
(metadata["device"], metadata["inode"]),
diff --git a/studio/backend/tests/test_video_wan21_families.py b/studio/backend/tests/test_video_wan21_families.py
index 373801de49b..a93616a26d9 100644
--- a/studio/backend/tests/test_video_wan21_families.py
+++ b/studio/backend/tests/test_video_wan21_families.py
@@ -183,7 +183,10 @@ def test_comfyui_template_defaults():
(("wan21-t2v-1.3b-Q8_0.gguf",), (30, 6.0)),
(("Wan21_T2V_14B_fp8.safetensors",), (20, 6.0)),
(("someone/wan21-14b-gguf",), (20, 6.0)),
- (("Wan2.1_T2V_14B_FusionX-Q4_K_M.gguf", "QuantStack/Wan2.1_T2V_14B_FusionX-GGUF"), (8, 1.0)),
+ (
+ ("Wan2.1_T2V_14B_FusionX-Q4_K_M.gguf", "QuantStack/Wan2.1_T2V_14B_FusionX-GGUF"),
+ (8, 1.0),
+ ),
(("wan21-lab/HunyuanVideo-1.5-Diffusers-480p_t2v",), (20, 6.0)),
(("fusionx-lab/HunyuanVideo-1.5-Diffusers-480p_t2v",), (20, 6.0)),
(("/models/FusionX/wan2.1-t2v-14b-Q4_K_M.gguf",), (20, 6.0)),
@@ -278,12 +281,12 @@ def test_file_name_outranks_the_folder_for_a_14b_header(tmp_path):
folder.mkdir()
_write_safetensors(folder / "Wan2.2-T2V-A14B-HighNoise-fp8.safetensors", shapes)
_write_safetensors(folder / "model-fp8.safetensors", shapes)
- assert _detect_load_family(str(folder), "Wan2.2-T2V-A14B-HighNoise-fp8.safetensors", None).name == (
- "wan2.2-t2v-a14b"
- )
- assert _detect_load_family(str(folder / "Wan2.2-T2V-A14B-HighNoise-fp8.safetensors"), None, None).name == (
- "wan2.2-t2v-a14b"
- )
+ assert _detect_load_family(
+ str(folder), "Wan2.2-T2V-A14B-HighNoise-fp8.safetensors", None
+ ).name == ("wan2.2-t2v-a14b")
+ assert _detect_load_family(
+ str(folder / "Wan2.2-T2V-A14B-HighNoise-fp8.safetensors"), None, None
+ ).name == ("wan2.2-t2v-a14b")
assert _detect_load_family(str(folder), "model-fp8.safetensors", None).name == "wan2.1-t2v-14b"
@@ -300,9 +303,10 @@ def test_file_name_outranks_the_folder_for_a_14b_header(tmp_path):
("org/LTX-2-GGUF", "Wan2.1-T2V-1.3B-Q8_0.gguf", "ltx-2"),
],
)
-def test_remote_file_name_outranks_the_repo_for_same_layout_wan(repo, filename, expected, monkeypatch):
+def test_remote_file_name_outranks_the_repo_for_same_layout_wan(
+ repo, filename, expected, monkeypatch
+):
import core.inference.video as video
-
monkeypatch.setattr(video, "_picked_gguf_arch", lambda *a, **k: None)
assert video._detect_load_family(repo, filename, None).name == expected
@@ -315,10 +319,11 @@ def test_remote_file_name_outranks_the_repo_for_same_layout_wan(repo, filename,
("org/Wan2.1-T2V-14B-GGUF", "model-Q8_0.gguf", "wan2.1-t2v-14b"),
],
)
-def test_remote_file_name_naming_the_other_wan21_size_outranks_the_repo(repo, filename, expected, monkeypatch):
+def test_remote_file_name_naming_the_other_wan21_size_outranks_the_repo(
+ repo, filename, expected, monkeypatch
+):
# No header pass for a remote pick, so the file's own size token decides which base config builds the transformer.
import core.inference.video as video
-
monkeypatch.setattr(video, "_picked_gguf_arch", lambda *a, **k: None)
assert video._detect_load_family(repo, filename, None).name == expected
@@ -442,12 +447,36 @@ def test_load_validation_refuses_with_the_named_reason():
{"gguf_filename": "FastWan2.1-T2V-1.3B-Q8_0.gguf", "model_kind": "gguf"},
_FASTWAN,
),
- ("city96/Wan2.1-T2V-14B-gguf", {"gguf_filename": "i2v-14b-480p-Q4_K_M.gguf", "model_kind": "gguf"}, _I2V),
- ("someorg/Wan2.1-T2V-1.3B-mix", {"gguf_filename": "vace-1.3b-Q8_0.gguf", "model_kind": "gguf"}, _VARIANT),
- ("someorg/Wan2.1-T2V-1.3B-mix", {"gguf_filename": "fun-inp-1.3b-Q8_0.gguf", "model_kind": "gguf"}, _VARIANT),
- ("someorg/Wan2.1-T2V-14B-GGUF", {"gguf_filename": "fun-14b-control-Q4_K_M.gguf", "model_kind": "gguf"}, _VARIANT),
- ("someorg/Wan2.1-T2V-1.3B-mix", {"gguf_filename": "fun-v1.1-1.3b-inp-Q8_0.gguf", "model_kind": "gguf"}, _VARIANT),
- ("city96/Wan2.1-T2V-14B-gguf", {"gguf_filename": "fun-style-i2v-14b-Q4_K_M.gguf", "model_kind": "gguf"}, _I2V),
+ (
+ "city96/Wan2.1-T2V-14B-gguf",
+ {"gguf_filename": "i2v-14b-480p-Q4_K_M.gguf", "model_kind": "gguf"},
+ _I2V,
+ ),
+ (
+ "someorg/Wan2.1-T2V-1.3B-mix",
+ {"gguf_filename": "vace-1.3b-Q8_0.gguf", "model_kind": "gguf"},
+ _VARIANT,
+ ),
+ (
+ "someorg/Wan2.1-T2V-1.3B-mix",
+ {"gguf_filename": "fun-inp-1.3b-Q8_0.gguf", "model_kind": "gguf"},
+ _VARIANT,
+ ),
+ (
+ "someorg/Wan2.1-T2V-14B-GGUF",
+ {"gguf_filename": "fun-14b-control-Q4_K_M.gguf", "model_kind": "gguf"},
+ _VARIANT,
+ ),
+ (
+ "someorg/Wan2.1-T2V-1.3B-mix",
+ {"gguf_filename": "fun-v1.1-1.3b-inp-Q8_0.gguf", "model_kind": "gguf"},
+ _VARIANT,
+ ),
+ (
+ "city96/Wan2.1-T2V-14B-gguf",
+ {"gguf_filename": "fun-style-i2v-14b-Q4_K_M.gguf", "model_kind": "gguf"},
+ _I2V,
+ ),
],
)
def test_unsupported_file_under_a_t2v_repo_is_refused(repo, kwargs, reason):
@@ -519,6 +548,7 @@ def test_vae_untiled_tables_cover_wan21():
)
def test_anyflow_is_refused(identifier):
from core.inference.video import VideoBackend
+
assert _ANYFLOW in vf.unsupported_video_checkpoint_reason(identifier)
assert vf.detect_video_family(identifier) is None
with pytest.raises(ValueError, match = _ANYFLOW):
@@ -537,7 +567,11 @@ def test_anyflow_owner_or_folder_does_not_refuse():
{"gguf_filename": "wan2.1-i2v-14b-480p-Q4_K_M.gguf", "model_kind": "gguf"},
_I2V,
),
- ("someorg/opaque-gguf", {"gguf_filename": "i2v-14b-480p-Q4_K_M.gguf", "model_kind": "gguf"}, _I2V),
+ (
+ "someorg/opaque-gguf",
+ {"gguf_filename": "i2v-14b-480p-Q4_K_M.gguf", "model_kind": "gguf"},
+ _I2V,
+ ),
("FastVideo/FastWan2.1-T2V-1.3B-Diffusers", {}, "FastVideo FastWan checkpoint"),
("nvidia/AnyFlow-Wan2.1-T2V-14B-Diffusers", {}, _ANYFLOW),
],
@@ -551,8 +585,13 @@ def test_family_override_does_not_bypass_the_refusal(repo, kwargs, reason):
@pytest.mark.parametrize("folder", ["i2v", "vace", "fun"])
def test_wan21_context_is_added_only_to_the_picked_file(tmp_path, folder):
from core.inference.video import VideoBackend
+
(tmp_path / folder).mkdir()
- _write_gguf(tmp_path / folder / "Wan2.1-T2V-1.3B-Q8_0.gguf", _row("Wan2.1-T2V-1.3B-Q8_0.gguf")["shapes"], "wan")
+ _write_gguf(
+ tmp_path / folder / "Wan2.1-T2V-1.3B-Q8_0.gguf",
+ _row("Wan2.1-T2V-1.3B-Q8_0.gguf")["shapes"],
+ "wan",
+ )
fam = VideoBackend().validate_load_request(
str(tmp_path / folder), gguf_filename = "Wan2.1-T2V-1.3B-Q8_0.gguf", model_kind = "gguf"
)
diff --git a/studio/frontend/scripts/check-bundle-budget.ts b/studio/frontend/scripts/check-bundle-budget.ts
index ca73ba67b82..25f1610ca15 100644
--- a/studio/frontend/scripts/check-bundle-budget.ts
+++ b/studio/frontend/scripts/check-bundle-budget.ts
@@ -50,8 +50,12 @@ export const BUDGET = {
// The long-chat render work (#13255: deferred fences and reasoning rows) adds 33.3 KB raw / 13.1 KB
// transfer to the eager chat chunk (6,477.1 -> 6,510.4 KB raw, 1,942.6 -> 1,955.7 KB transfer at
// 160cb41fe, one machine and build); raised by that so main's margin stays where it was.
- transferBytes: 2_005_000,
- rawBytes: 6_681_000,
+ // Project sources follow-ups (#13224: queued drops, retry on failed sources, folder drops) add
+ // 6.6 KB raw / 2.0 KB transfer, mostly in the eager chat chunk (6,513.0 -> 6,519.6 KB raw,
+ // 1,956.4 -> 1,958.4 KB transfer against d4ea5a51d, one machine and build). Lazy-loading the
+ // folder walker recovered 0.5 KB raw and no transfer, so the budget is raised by that growth.
+ transferBytes: 2_007_100,
+ rawBytes: 6_687_800,
};
// The chunk count is reported but not budgeted. Splitting a page out of the entry raises it while lowering the
diff --git a/studio/frontend/src/features/native-intents/use-native-file-drop.ts b/studio/frontend/src/features/native-intents/use-native-file-drop.ts
index 7150f937688..9d82fb1843f 100644
--- a/studio/frontend/src/features/native-intents/use-native-file-drop.ts
+++ b/studio/frontend/src/features/native-intents/use-native-file-drop.ts
@@ -2,6 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { isTauri } from "@/lib/api-base";
+import { MAX_FOLDER_FILES, filesFromDrop } from "@/lib/dropped-folders";
import { toast } from "@/lib/toast";
import type React from "react";
import { useCallback, useRef, useState } from "react";
@@ -39,12 +40,14 @@ function hasAcceptedExt(name: string, exts: string[]): boolean {
function toastNothingAccepted(
names: string[],
accept: string | undefined,
+ folderHint?: string,
): void {
// A folder is one extension-less name, and the native side takes files only.
const looksLikeFolder = names.some((name) => !name.includes("."));
if (looksLikeFolder) {
toast.error("Folders can't be dropped here", {
- description: "Drop the files inside it, or use the picker button.",
+ description:
+ folderHint ?? "Drop the files inside it, or use the picker button.",
});
return;
}
@@ -125,6 +128,7 @@ export interface NativeFileDropOptions {
multiple?: boolean;
/** Register under a policy other than the attachment one (datasets, models). */
register?: (path: string) => Promise<NativeIntent>;
+ folders?: boolean;
}
export interface NativeFileDrop {
@@ -154,13 +158,16 @@ export function useNativeFileDrop(
// dragenter/dragleave fire per child, so a raw boolean flickers on inner moves.
const dragDepth = useRef(0);
- const deliver = useCallback((files: File[]) => {
- const current = latest.current;
- if (files.length === 0) return;
- void current.onFiles(
- current.multiple === false ? files.slice(0, 1) : files,
- );
- }, []);
+ // Options from drop time: `latest` after an await may name another destination.
+ const deliver = useCallback(
+ (files: File[], current: NativeFileDropOptions) => {
+ if (files.length === 0) return;
+ void current.onFiles(
+ current.multiple === false ? files.slice(0, 1) : files,
+ );
+ },
+ [],
+ );
const handleNativePaths = useCallback(
async (paths: string[]) => {
@@ -176,7 +183,13 @@ export function useNativeFileDrop(
hasAcceptedExt(nativeFileName(path), exts),
);
if (supported.length === 0) {
- toastNothingAccepted(paths.map(nativeFileName), current.accept);
+ toastNothingAccepted(
+ paths.map(nativeFileName),
+ current.accept,
+ current.folders
+ ? "Use Link folder to keep a whole folder indexed and in sync."
+ : undefined,
+ );
return;
}
const takeIntents = current.onNativeIntents;
@@ -189,7 +202,7 @@ export function useNativeFileDrop(
if (takeIntents) {
void takeIntents(ready as NativeIntent[]);
} else {
- deliver(ready as File[]);
+ deliver(ready as File[], current);
}
}
if (failed > 0) {
@@ -254,19 +267,42 @@ export function useNativeFileDrop(
return;
}
const exts = acceptedExts(current.accept);
- const dropped = Array.from(event.dataTransfer.files ?? []);
- const supported = dropped.filter((file) =>
- hasAcceptedExt(file.name, exts),
- );
- if (dropped.length > 0 && supported.length === 0) {
- toastNothingAccepted(
- dropped.map((file) => file.name),
- current.accept,
+ const take = (dropped: File[]) => {
+ const supported = dropped.filter((file) =>
+ hasAcceptedExt(file.name, exts),
);
+ if (dropped.length > 0 && supported.length === 0) {
+ toastNothingAccepted(
+ dropped.map((file) => file.name),
+ current.accept,
+ );
+ return;
+ }
+ deliver(supported, current);
+ toastPartiallySkipped(dropped.length - supported.length);
+ };
+ if (!current.folders) {
+ take(Array.from(event.dataTransfer.files ?? []));
return;
}
- deliver(supported);
- toastPartiallySkipped(dropped.length - supported.length);
+ void filesFromDrop(event.dataTransfer, (name) =>
+ hasAcceptedExt(name, exts),
+ ).then(
+ ({ files, truncated, hadFolder }) => {
+ if (hadFolder && files.length === 0) {
+ toast.error("That folder has no files to add");
+ return;
+ }
+ take(files);
+ if (truncated > 0) {
+ toast.info(`Added the first ${MAX_FOLDER_FILES} files`, {
+ description:
+ "The rest were left out. Drop a smaller folder for them.",
+ });
+ }
+ },
+ (reason) => toastReadFailures(1, reason),
+ );
},
};
diff --git a/studio/frontend/src/features/rag/components/document-status-chip.tsx b/studio/frontend/src/features/rag/components/document-status-chip.tsx
index 4cd16b95c4a..1c635ee5f67 100644
--- a/studio/frontend/src/features/rag/components/document-status-chip.tsx
+++ b/studio/frontend/src/features/rag/components/document-status-chip.tsx
@@ -25,6 +25,7 @@ export function DocumentStatusChip({
stage,
error,
onRemove,
+ onRetry,
shared = false,
}: {
filename: string;
@@ -33,6 +34,7 @@ export function DocumentStatusChip({
stage?: string | null;
error?: string | null;
onRemove?: () => void;
+ onRetry?: () => void;
/** Indexed for the whole project rather than this one chat: swap the file
* glyph for a folder so the two scopes are told apart at a glance. */
shared?: boolean;
@@ -62,6 +64,16 @@ export function DocumentStatusChip({
className="size-3 shrink-0"
/>
<span className="truncate">{filename}</span>
+ {status === "failed" && onRetry ? (
+ <button
+ type="button"
+ onClick={onRetry}
+ aria-label={`Retry ${filename}`}
+ className="shrink-0 text-ui-10 font-medium underline-offset-2 hover:underline"
+ >
+ Retry
+ </button>
+ ) : null}
{/* spinner while indexing, else close button */}
{processing ? (
<span className="flex shrink-0 items-center gap-1 text-ui-10 text-muted-foreground">
diff --git a/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx b/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx
index e7089b56cda..3a19824c838 100644
--- a/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx
+++ b/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx
@@ -49,6 +49,7 @@ import { LinkedFoldersManager } from "./linked-folders-manager";
import { RAG_SOURCE_UPLOAD_ACCEPT } from "./source-drop-policy";
import { type RagUploadItem, useRagDocuments } from "./use-rag-documents";
import { useSourceDrop } from "./use-source-drop";
+import { useUploadQueue } from "./use-upload-queue";
type View =
| { kind: "list" }
@@ -440,19 +441,19 @@ function KnowledgeBaseDocuments({
onBack: () => void;
}) {
const lister = useCallback(() => listKnowledgeBaseDocuments(kb.id), [kb.id]);
- const { documents, loading, uploading, refresh, upload, remove } =
+ const { documents, loading, uploading, refresh, upload, remove, retry, canRetry } =
useRagDocuments({ type: "kb", kbId: kb.id }, lister);
const fileInputRef = useRef<HTMLInputElement>(null);
const handleLinkedSourcesChanged = useCallback(() => {
void refresh({ quiet: true });
}, [refresh]);
+ const uploadNow = useCallback(
+ (items: RagUploadItem[]) => void upload(items),
+ [upload],
+ );
+ const { enqueue } = useUploadQueue(uploadNow, uploading, kb.id);
const { dragging, dropProps, nativeDropTarget } = useSourceDrop({
- onItems: (items) => void upload(items),
- // upload() tracks one run at a time, so a second batch would clear the
- // in-flight guard the first one is still relying on.
- disabledReason: uploading
- ? "An upload is already running. Add these when it finishes."
- : undefined,
+ onItems: enqueue,
});
// Deferred a tick: the hook's unmount cleanup aborts an upload started on StrictMode's
@@ -478,11 +479,7 @@ function KnowledgeBaseDocuments({
<ChevronLeftIcon className="size-4" />
All knowledge bases
</Button>
- <Button
- size="sm"
- onClick={() => fileInputRef.current?.click()}
- disabled={uploading}
- >
+ <Button size="sm" onClick={() => fileInputRef.current?.click()}>
{uploading ? <Spinner /> : <UploadIcon className="size-3.5" />}
Upload
</Button>
@@ -493,8 +490,9 @@ function KnowledgeBaseDocuments({
accept={RAG_SOURCE_UPLOAD_ACCEPT}
className="hidden"
onChange={(e) => {
- if (e.target.files?.length) void upload(e.target.files);
+ const files = Array.from(e.target.files ?? []);
e.target.value = "";
+ enqueue(files.map((file) => ({ kind: "file" as const, file })));
}}
/>
</div>
@@ -528,10 +526,12 @@ function KnowledgeBaseDocuments({
stage={doc.stage}
error={doc.error}
onRemove={
- doc.id.startsWith("pending_") || isLinkedFolderManaged(doc)
+ isLinkedFolderManaged(doc) ||
+ (doc.id.startsWith("pending_") && doc.status !== "failed")
? undefined
: () => void remove(doc.id)
}
+ onRetry={canRetry(doc.id) ? () => void retry(doc.id) : undefined}
/>
))}
</div>
diff --git a/studio/frontend/src/features/rag/components/linked-folder-groups.ts b/studio/frontend/src/features/rag/components/linked-folder-groups.ts
new file mode 100644
index 00000000000..17d725ddd34
--- /dev/null
+++ b/studio/frontend/src/features/rag/components/linked-folder-groups.ts
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { useEffect, useState } from "react";
+import { listLinkedFolders } from "../api/rag-api";
+import type { RagDocument } from "../types/rag";
+
+export function useLinkedFolderNames(
+ projectId: string | null,
+ folderIds: string,
+): ReadonlyMap<string, string> {
+ const [names, setNames] = useState<ReadonlyMap<string, string>>(new Map());
+ useEffect(() => {
+ if (!projectId || !folderIds) return;
+ let cancelled = false;
+ listLinkedFolders({ type: "project", id: projectId })
+ .then((folders) => {
+ if (cancelled) return;
+ setNames(new Map(folders.map((f) => [f.id, f.displayName])));
+ })
+ .catch(() => {
+ });
+ return () => {
+ cancelled = true;
+ };
+ }, [projectId, folderIds]);
+ return names;
+}
+
+/** A linked folder can hold thousands of files, so it is drawn as one item. */
+export function groupByLinkedFolder<T extends RagDocument>(
+ docs: T[],
+): {
+ loose: T[];
+ folders: [string, T[]][];
+} {
+ const loose: T[] = [];
+ const folders = new Map<string, T[]>();
+ for (const doc of docs) {
+ const folderId = doc.linkedFolderId;
+ if (!folderId) {
+ loose.push(doc);
+ continue;
+ }
+ const group = folders.get(folderId);
+ if (group) group.push(doc);
+ else folders.set(folderId, [doc]);
+ }
+ return { loose, folders: [...folders] };
+}
diff --git a/studio/frontend/src/features/rag/components/linked-folders-manager.tsx b/studio/frontend/src/features/rag/components/linked-folders-manager.tsx
index 1598a4ee6bd..57c12f16324 100644
--- a/studio/frontend/src/features/rag/components/linked-folders-manager.tsx
+++ b/studio/frontend/src/features/rag/components/linked-folders-manager.tsx
@@ -27,7 +27,12 @@ import { FolderAddIcon, FolderSyncIcon } from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
import { MoreHorizontalIcon } from "lucide-react";
import { useState } from "react";
-import type { FolderSyncJob, LinkedFolderScope } from "../types/rag";
+import type {
+ FolderSyncJob,
+ LinkedFolder,
+ LinkedFolderScope,
+ LinkedFolderSkipReason,
+} from "../types/rag";
import { useLinkedFolders } from "./use-linked-folders";
/** Grouped folder card, shared with the create project dialog. */
@@ -56,6 +61,73 @@ function jobSummary(job: FolderSyncJob): string {
: `${processed} of ${discovered} files`;
}
+const SKIP_LABELS: Record<LinkedFolderSkipReason, (count: number) => string> = {
+ gitignored: (n) => `${n} ignored by .gitignore`,
+ build_dirs: (n) => `${n} build folder${n === 1 ? "" : "s"}`,
+ too_large: (n) => `${n} large data file${n === 1 ? "" : "s"}`,
+ over_limit: (n) => `${n} past the file limit`,
+};
+
+export function skippedSummary(folder: LinkedFolder): string | null {
+ const parts = (Object.keys(SKIP_LABELS) as LinkedFolderSkipReason[]).flatMap(
+ (reason) => {
+ const count = folder.skipped?.[reason] ?? 0;
+ return count > 0 ? [SKIP_LABELS[reason](count)] : [];
+ },
+ );
+ return parts.length > 0 ? `Skipped ${parts.join(" · ")}` : null;
+}
+
+function FolderReport({ folder }: { folder: LinkedFolder }) {
+ const [open, setOpen] = useState(false);
+ const skipped = skippedSummary(folder);
+ const failures = folder.failures ?? [];
+ const failureCount = folder.failureCount ?? failures.length;
+ if (!skipped && failureCount === 0) return null;
+ return (
+ <div className="mt-1 flex flex-col gap-1 text-ui-11 text-muted-foreground">
+ {skipped ? (
+ <p
+ className="truncate"
+ title={`${skipped}. Hidden folders, dependencies and secrets are never indexed.`}
+ >
+ {skipped}
+ </p>
+ ) : null}
+ {failureCount > 0 ? (
+ <>
+ <button
+ type="button"
+ aria-expanded={open}
+ onClick={() => setOpen((value) => !value)}
+ className="self-start text-destructive underline-offset-2 hover:underline"
+ >
+ {open ? "Hide" : "Show"} {failureCount} file
+ {failureCount === 1 ? "" : "s"} that couldn't be indexed
+ </button>
+ {open ? (
+ <ul className="max-h-48 overflow-y-auto rounded-lg bg-muted/40 px-2.5 py-1.5">
+ {failures.map((failure) => (
+ <li key={failure.path} className="flex min-w-0 gap-2 py-0.5">
+ <span className="min-w-0 shrink truncate font-mono" title={failure.path}>
+ {failure.path}
+ </span>
+ <span className="min-w-0 flex-1 truncate" title={failure.error}>
+ {failure.error}
+ </span>
+ </li>
+ ))}
+ {failureCount > failures.length ? (
+ <li className="py-0.5">and {failureCount - failures.length} more</li>
+ ) : null}
+ </ul>
+ ) : null}
+ </>
+ ) : null}
+ </div>
+ );
+}
+
export function LinkedFoldersManager({
scope,
compact = false,
@@ -233,7 +305,9 @@ export function LinkedFoldersManager({
aria-label={`Sync progress for ${folder.displayName}`}
className="mt-1.5 h-1"
/>
- ) : null}
+ ) : (
+ <FolderReport folder={folder} />
+ )}
</div>
{folderMenu(folder, running)}
</div>
@@ -383,7 +457,9 @@ export function LinkedFoldersManager({
aria-label={`Sync progress for ${folder.displayName}`}
className="mt-2 h-1.5"
/>
- ) : null}
+ ) : (
+ <FolderReport folder={folder} />
+ )}
</div>
{folderMenu(folder, running)}
</li>
diff --git a/studio/frontend/src/features/rag/components/project-sources-panel.tsx b/studio/frontend/src/features/rag/components/project-sources-panel.tsx
index 64279e10c2b..c99b06c1b2d 100644
--- a/studio/frontend/src/features/rag/components/project-sources-panel.tsx
+++ b/studio/frontend/src/features/rag/components/project-sources-panel.tsx
@@ -4,10 +4,13 @@
import { Button } from "@/components/ui/button";
import { useNativeFileDrop } from "@/features/native-intents";
import type { NativeIntent } from "@/features/native-intents";
+import { isTauri } from "@/lib/api-base";
+import { MAX_FOLDER_FILES, openFolderPicker } from "@/lib/dropped-folders";
import { FolderPlusIcon } from "@/lib/hugeicons-derived";
+import { toast } from "@/lib/toast";
import { cn } from "@/lib/utils";
import { HugeiconsIcon } from "@hugeicons/react";
-import { useCallback, useEffect, useRef } from "react";
+import { useCallback, useEffect, useMemo, useRef } from "react";
import {
announceProjectSourcesUpdated,
invalidateProjectSources,
@@ -16,13 +19,22 @@ import {
} from "../api/rag-api";
import { isLinkedFolderManaged } from "../types/rag";
import { DocumentStatusChip } from "./document-status-chip";
+import {
+ groupByLinkedFolder,
+ useLinkedFolderNames,
+} from "./linked-folder-groups";
import { LinkedFoldersManager } from "./linked-folders-manager";
-import { RAG_SOURCE_UPLOAD_ACCEPT } from "./source-drop-policy";
+import {
+ RAG_SOURCE_UPLOAD_ACCEPT,
+ SUPPORTED_SOURCES_HINT,
+ isSupportedSourceName,
+} from "./source-drop-policy";
import {
type RagUploadItem,
fileItems,
useRagDocuments,
} from "./use-rag-documents";
+import { useUploadQueue } from "./use-upload-queue";
/** Project "Sources" tab: documents indexed for retrieval in every chat that
* belongs to the project. */
@@ -32,22 +44,70 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) {
() => listProjectDocuments(projectId),
[projectId],
);
- const { documents, loading, uploading, refresh, upload, remove } =
- useRagDocuments({ type: "project", projectId }, lister);
+ const {
+ documents,
+ loading,
+ uploading,
+ refresh,
+ upload,
+ remove,
+ retry,
+ canRetry,
+ } = useRagDocuments({ type: "project", projectId }, lister);
// Invalidate the sources probe before each mutation so a chat sent mid-upload
// cannot cache "no sources" for the probe's TTL, and announce after it, which
// is the half other instances and other tabs listen for. Announcing before
// would refetch and resurrect the row this panel has already dropped.
- const handleItems = useCallback(
- async (items: RagUploadItem[]) => {
+ const uploadNow = useCallback(
+ (items: RagUploadItem[]) => {
if (items.length === 0) return;
invalidateProjectSources(projectId);
- await upload(items);
- announceProjectSourcesUpdated(projectId);
+ void upload(items).finally(() =>
+ announceProjectSourcesUpdated(projectId),
+ );
},
[projectId, upload],
);
+ const { enqueue: handleItems } = useUploadQueue(
+ uploadNow,
+ uploading,
+ projectId,
+ );
+
+ const handleRetry = useCallback(
+ (documentId: string) => {
+ invalidateProjectSources(projectId);
+ void retry(documentId).finally(() =>
+ announceProjectSourcesUpdated(projectId),
+ );
+ },
+ [projectId, retry],
+ );
+
+ const pickFolder = useCallback(() => {
+ openFolderPicker(({ files: supported, truncated }) => {
+ if (supported.length === 0) {
+ toast.info("No supported files in that folder", {
+ description: SUPPORTED_SOURCES_HINT,
+ });
+ return;
+ }
+ if (truncated > 0) {
+ toast.info(`Added the first ${MAX_FOLDER_FILES} files`, {
+ description:
+ "The rest were left out. Pick a smaller folder for them.",
+ });
+ }
+ handleItems(fileItems(supported));
+ }, isSupportedSourceName);
+ }, [handleItems]);
+
+ const groups = useMemo(() => groupByLinkedFolder(documents), [documents]);
+ const folderNames = useLinkedFolderNames(
+ projectId,
+ groups.folders.map(([id]) => id).join(","),
+ );
const handleFiles = useCallback(
(files: File[]) => handleItems(fileItems(files)),
@@ -106,10 +166,23 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) {
onFiles: handleFiles,
onNativeIntents: handleNativeIntents,
accept: RAG_SOURCE_UPLOAD_ACCEPT,
- disabled: uploading,
- disabledReason: "Wait for the current upload to finish, then drop again.",
+ folders: true,
});
+ const addFolderButton = isTauri ? null : (
+ <Button
+ type="button"
+ size="sm"
+ variant="ghost"
+ className="text-muted-foreground"
+ disabled={loading}
+ onClick={pickFolder}
+ title="Upload every supported file in a folder. Linking a folder that stays in sync needs the desktop app."
+ >
+ Add folder
+ </Button>
+ );
+
return (
<div className="mt-8" ref={dropRef} {...dragHandlers}>
<input
@@ -150,20 +223,25 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) {
Give this project context
</p>
<p className="max-w-sm text-sm text-muted-foreground">
- Add documents, spreadsheets, slides, e-books, email, text or
- code. Every chat in this project can use them.
+ Add documents, spreadsheets, slides, e-books, email, text or code.
+ Every chat in this project can use them.
</p>
</div>
- <Button
- type="button"
- variant="outline"
- className="mt-1 border-none bg-background text-foreground shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] hover:bg-background/80 dark:bg-card dark:shadow-none dark:hover:bg-accent/50"
- disabled={uploading || loading}
- onClick={() => fileInputRef.current?.click()}
- >
- Add sources
- </Button>
- <p className="text-ui-11 text-muted-foreground">Or drop files here</p>
+ <div className="mt-1 flex items-center gap-1">
+ <Button
+ type="button"
+ variant="outline"
+ className="border-none bg-background text-foreground shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] hover:bg-background/80 dark:bg-card dark:shadow-none dark:hover:bg-accent/50"
+ disabled={loading}
+ onClick={() => fileInputRef.current?.click()}
+ >
+ Add sources
+ </Button>
+ {addFolderButton}
+ </div>
+ <p className="text-ui-11 text-muted-foreground">
+ {isTauri ? "Or drop files here" : "Or drop files or a folder here"}
+ </p>
</div>
) : (
<div
@@ -178,19 +256,39 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) {
? "1 source"
: `${documents.length} sources`}
</p>
- <Button
- type="button"
- size="sm"
- variant="outline"
- className="border-none bg-background text-foreground shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] hover:bg-background/80 dark:bg-card dark:shadow-none dark:hover:bg-accent/50"
- disabled={uploading}
- onClick={() => fileInputRef.current?.click()}
- >
- Add sources
- </Button>
+ <div className="flex items-center gap-1">
+ {addFolderButton}
+ <Button
+ type="button"
+ size="sm"
+ variant="outline"
+ className="border-none bg-background text-foreground shadow-[0_2px_8px_-2px_rgba(0,0,0,0.16)] hover:bg-background/80 dark:bg-card dark:shadow-none dark:hover:bg-accent/50"
+ onClick={() => fileInputRef.current?.click()}
+ >
+ Add sources
+ </Button>
+ </div>
</div>
<div className="flex flex-row flex-wrap items-center gap-1.5">
- {documents.map((doc) => (
+ {/* A linked folder is one chip: it can hold thousands of files. */}
+ {groups.folders.map(([folderId, docs]) => {
+ const indexing = docs.filter(
+ (doc) => doc.status === "pending" || doc.status === "running",
+ ).length;
+ const name = folderNames.get(folderId) ?? "Linked folder";
+ return (
+ <DocumentStatusChip
+ key={`folder:${folderId}`}
+ filename={`${name} · ${docs.length} file${docs.length === 1 ? "" : "s"}`}
+ status={indexing > 0 ? "running" : "completed"}
+ progress={
+ indexing > 0 ? (docs.length - indexing) / docs.length : null
+ }
+ shared={true}
+ />
+ );
+ })}
+ {groups.loose.map((doc) => (
<DocumentStatusChip
key={doc.id}
filename={doc.filename}
@@ -199,10 +297,14 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) {
stage={doc.stage}
error={doc.error}
onRemove={
- doc.id.startsWith("pending_") || isLinkedFolderManaged(doc)
+ isLinkedFolderManaged(doc) ||
+ (doc.id.startsWith("pending_") && doc.status !== "failed")
? undefined
: () => void handleRemove(doc.id)
}
+ onRetry={
+ canRetry(doc.id) ? () => handleRetry(doc.id) : undefined
+ }
/>
))}
</div>
diff --git a/studio/frontend/src/features/rag/components/thread-documents-bar.tsx b/studio/frontend/src/features/rag/components/thread-documents-bar.tsx
index 5a385a64160..905c3159928 100644
--- a/studio/frontend/src/features/rag/components/thread-documents-bar.tsx
+++ b/studio/frontend/src/features/rag/components/thread-documents-bar.tsx
@@ -45,7 +45,7 @@ import {
} from "@/components/ui/tooltip";
import { Tick02Icon } from "@/lib/tick-icon";
import { ChevronDownStandardIcon } from "@/lib/chevron-icons";
-import { useAui } from "@assistant-ui/react";
+import { useAui, useAuiState } from "@assistant-ui/react";
import { cn } from "@/lib/utils";
import {
PENDING_CHAT_ATTACHMENT_KEY,
@@ -84,7 +84,6 @@ import {
getDocumentFileUrl,
invalidateProjectSources,
listKnowledgeBases,
- listLinkedFolders,
subscribeKnowledgeBasesChanged,
listProjectDocuments,
listThreadDocuments,
@@ -120,6 +119,8 @@ import {
useRagDocuments,
} from "./use-rag-documents";
import { useSourceDrop } from "./use-source-drop";
+import { useUploadQueue } from "./use-upload-queue";
+import { groupByLinkedFolder, useLinkedFolderNames } from "./linked-folder-groups";
// Refetched after any KB mutation so a rename shows at once.
function useKnowledgeBaseName(kbId: string | null): string | null {
@@ -254,51 +255,6 @@ function isRagIndexable(file: File): boolean {
return isSupportedSourceName(file.name);
}
-/** Display names of a project's linked folders, read once per set of folder ids. */
-function useLinkedFolderNames(
- projectId: string | null,
- folderIds: string,
-): ReadonlyMap<string, string> {
- const [names, setNames] = useState<ReadonlyMap<string, string>>(new Map());
- useEffect(() => {
- if (!projectId || !folderIds) return;
- let cancelled = false;
- listLinkedFolders({ type: "project", id: projectId })
- .then((folders) => {
- if (cancelled) return;
- setNames(new Map(folders.map((f) => [f.id, f.displayName])));
- })
- .catch(() => {
- // The card falls back to a generic label.
- });
- return () => {
- cancelled = true;
- };
- }, [projectId, folderIds]);
- return names;
-}
-
-/** Splits project documents into loose files and per-folder groups. A linked folder can
- * hold thousands of files, so it is drawn as one card rather than one card per file. */
-function groupByLinkedFolder(docs: TrackedDocument[]): {
- loose: TrackedDocument[];
- folders: [string, TrackedDocument[]][];
-} {
- const loose: TrackedDocument[] = [];
- const folders = new Map<string, TrackedDocument[]>();
- for (const doc of docs) {
- const folderId = doc.linkedFolderId;
- if (!folderId) {
- loose.push(doc);
- continue;
- }
- const group = folders.get(folderId);
- if (group) group.push(doc);
- else folders.set(folderId, [doc]);
- }
- return { loose, folders: [...folders] };
-}
-
function isProcessing(doc: TrackedDocument): boolean {
return doc.status === "pending" || doc.status === "running";
}
@@ -490,10 +446,12 @@ function DocumentCard({
doc,
shared,
onRemove,
+ onRetry,
}: {
doc: TrackedDocument;
shared: boolean;
onRemove?: () => void;
+ onRetry?: () => void;
}) {
const openPreview = useDocumentPreviewStore((s) => s.openPreview);
const kind = attachmentFileKind(doc.filename, undefined);
@@ -616,6 +574,16 @@ function DocumentCard({
card
)}
{shared ? <ProjectBadge /> : null}
+ {failed && onRetry ? (
+ <button
+ type="button"
+ onClick={onRetry}
+ aria-label={`Retry ${doc.filename}`}
+ className="absolute bottom-9 left-1/2 -translate-x-1/2 rounded-full bg-foreground px-2.5 py-0.5 text-ui-11 text-background shadow-sm"
+ >
+ Retry
+ </button>
+ ) : null}
{onRemove && !processing ? (
<button
type="button"
@@ -1007,6 +975,8 @@ export function ThreadDocumentsBar({
loading: threadListLoading,
upload,
remove,
+ retry,
+ canRetry,
} = useRagDocuments(
effectiveThreadId && ragEnabled && ragSource.type === "thread"
? { type: "thread", threadId: effectiveThreadId }
@@ -1028,6 +998,8 @@ export function ThreadDocumentsBar({
loading: projectListLoading,
upload: uploadToProject,
remove: removeFromProject,
+ retry: retryProject,
+ canRetry: canRetryProject,
} = useRagDocuments(
projectId ? { type: "project", projectId } : null,
projectLister,
@@ -1123,6 +1095,14 @@ export function ThreadDocumentsBar({
[ensureThreadId, projectId, sharesWithProject, upload, uploadToProject],
);
+ // Keyed by the list item id, stable while a new chat materializes, so only a real switch drops the queue.
+ const threadItemId = useAuiState(({ threadListItem }) => threadListItem.id);
+ const { enqueue: queueAttach, queued } = useUploadQueue<RagUploadItem>(
+ attach,
+ uploading || projectUploading,
+ `${threadItemId}:${projectId ?? ""}`,
+ );
+
// Only files added or dropped here are indexed; composer attachments stay with the message.
const attachIndexable = useCallback(
(files: File[]) => {
@@ -1136,9 +1116,11 @@ export function ThreadDocumentsBar({
{ description: SUPPORTED_SOURCES_HINT },
);
}
- if (indexable.length > 0) attach(indexable);
+ if (indexable.length > 0) {
+ queueAttach(indexable.map((file) => ({ kind: "file" as const, file })));
+ }
},
- [attach],
+ [queueAttach],
);
const pickFiles = useCallback(() => {
openFilePicker(RAG_SOURCE_UPLOAD_ACCEPT, attachIndexable);
@@ -1340,14 +1322,13 @@ export function ThreadDocumentsBar({
);
}
- // block attachments until the chat's project is known to avoid guessing their scope.
- const busy = uploading || projectUploading || projectUnresolved;
+ // Block attachments until the chat's project is known; the scope stays locked until the queue drains.
+ const busy = projectUnresolved;
+ const scopeLocked = busy || uploading || projectUploading || queued > 0;
// Still claim a drop while busy, or it falls through and attaches to the message instead.
- const busyReason = !busy
- ? undefined
- : projectUnresolved
- ? "Still loading this chat's project. Try again in a moment."
- : "Still uploading. Drop the files again once it finishes.";
+ const busyReason = busy
+ ? "Still loading this chat's project. Try again in a moment."
+ : undefined;
return (
<>
@@ -1359,7 +1340,7 @@ export function ThreadDocumentsBar({
titleSuffix="(RAG)"
count={countSuffix}
info="Add or drop documents, spreadsheets, slides, e-books, email or code. The model searches them as you chat and cites what it uses."
- onDropItems={attach}
+ onDropItems={queueAttach}
dropDisabledReason={busyReason}
onClose={() => setRagEnabled(false)}
closeLabel="Stop chatting with files"
@@ -1378,7 +1359,7 @@ export function ThreadDocumentsBar({
{/* Only a project chat has two scopes to choose between. */}
{projectId ? (
<AttachmentTargetMenu
- disabled={busy}
+ disabled={scopeLocked}
sharesWithProject={sharesWithProject}
onSelect={(target) =>
setThreadProjectAttachmentTarget(effectiveThreadId, target)
@@ -1413,9 +1394,23 @@ export function ThreadDocumentsBar({
doc={doc}
shared={true}
onRemove={
- doc.id.startsWith("pending_") || isLinkedFolderManaged(doc)
+ isLinkedFolderManaged(doc) ||
+ (doc.id.startsWith("pending_") && doc.status !== "failed")
? undefined
- : () => setRemovingShared(doc)
+ : doc.status === "failed"
+ ? // Never indexed, so no other chat loses anything: no confirmation.
+ () => void removeFromProject(doc.id)
+ : () => setRemovingShared(doc)
+ }
+ onRetry={
+ canRetryProject(doc.id)
+ ? () => {
+ if (projectId) invalidateProjectSources(projectId);
+ void retryProject(doc.id).finally(() => {
+ if (projectId) announceProjectSourcesUpdated(projectId);
+ });
+ }
+ : undefined
}
/>
</div>
@@ -1429,10 +1424,11 @@ export function ThreadDocumentsBar({
doc={doc}
shared={false}
onRemove={
- doc.id.startsWith("pending_")
+ doc.id.startsWith("pending_") && doc.status !== "failed"
? undefined
: () => void remove(doc.id)
}
+ onRetry={canRetry(doc.id) ? () => void retry(doc.id) : undefined}
/>
</div>
))}
diff --git a/studio/frontend/src/features/rag/components/use-linked-folders.ts b/studio/frontend/src/features/rag/components/use-linked-folders.ts
index cb8b9293e16..35423acf279 100644
--- a/studio/frontend/src/features/rag/components/use-linked-folders.ts
+++ b/studio/frontend/src/features/rag/components/use-linked-folders.ts
@@ -27,6 +27,7 @@ import type {
LinkedFolderScope,
} from "../types/rag";
import {
+ jobChangedSources,
linkedFolderSourcesChanged,
retainActiveFolderJobs,
} from "../types/rag";
@@ -80,11 +81,14 @@ export function useLinkedFolders(
[projectWorkScopeId],
);
+ // Only syncs started here toast on failure; periodic failures show on the folder row.
+ const userStartedJobs = useRef(new Set<string>());
+
const notifySourcesChanged = useCallback(
(job: FolderSyncJob) => {
if (notifiedJobs.current.has(job.id)) return;
notifiedJobs.current.add(job.id);
- onSourcesChanged?.();
+ if (jobChangedSources(job)) onSourcesChanged?.();
},
[onSourcesChanged],
);
@@ -123,7 +127,8 @@ export function useLinkedFolders(
apply(job);
releaseController();
notifySourcesChanged(job);
- if (job.status === "failed") {
+ const userStarted = userStartedJobs.current.delete(job.id);
+ if (job.status === "failed" && userStarted) {
toast.error("Folder sync failed", {
description: job.error ?? "The folder could not be indexed.",
});
@@ -285,6 +290,7 @@ export function useLinkedFolders(
...current.filter((folder) => folder.id !== result.linkedFolder.id),
result.linkedFolder,
]);
+ userStartedJobs.current.add(result.job.id);
trackJob(result.job);
onSourcesChanged?.();
} catch (error) {
@@ -321,6 +327,7 @@ export function useLinkedFolders(
return started;
});
if (currentScopeKey.current !== operationScopeKey) return;
+ userStartedJobs.current.add(job.id);
trackJob(job);
onSourcesChanged?.();
} catch (error) {
diff --git a/studio/frontend/src/features/rag/components/use-rag-documents.ts b/studio/frontend/src/features/rag/components/use-rag-documents.ts
index 0c289cd0b31..1f642690728 100644
--- a/studio/frontend/src/features/rag/components/use-rag-documents.ts
+++ b/studio/frontend/src/features/rag/components/use-rag-documents.ts
@@ -111,6 +111,8 @@ export function useRagDocuments(
}, [documents]);
// documentId -> signature; forgotten on delete, cleared on scope change.
const sigByDocId = useRef<Map<string, string>>(new Map());
+ // Row id -> source File for retry. Files only: a native drop's path token is spent on first use.
+ const retryItems = useRef<Map<string, RagUploadItem>>(new Map());
// Skip a re-selected file only if a matching doc is healthy or still indexing. A doc
// that completed with 0 chunks is re-ingestable (e.g. a scan attached before a vision
// model loaded); the backend re-ingests on the same hash, so let it through.
@@ -189,13 +191,17 @@ export function useRagDocuments(
numChunks?: number | null,
) => {
if (stale()) return forget();
+ if (status !== "failed") retryItems.current.delete(documentId);
if (status === "cancelled") {
sigByDocId.current.delete(documentId);
setDocuments((rows) => rows.filter((row) => row.id !== documentId));
} else if (status === "failed") {
- // Drop the chip rather than show "Failed"; warn via toast.
sigByDocId.current.delete(documentId);
- setDocuments((rows) => rows.filter((row) => row.id !== documentId));
+ patchDoc(documentId, {
+ status: "failed",
+ error: error ?? "Indexing failed",
+ progress: null,
+ });
toast.error(`Couldn't index ${filename}`, {
description: error ?? "Indexing failed",
});
@@ -293,8 +299,10 @@ export function useRagDocuments(
if (!opts?.quiet) setLoading(true);
try {
// Merge server truth with local progress so a refresh mid-index keeps a
- // live "running %" chip. Failed docs hidden (toast warned at upload).
- const rows = (await lister()).filter((row) => row.status !== "failed");
+ // live "running %" chip. Failed docs hidden, except ones this session can retry.
+ const rows = (await lister()).filter(
+ (row) => row.status !== "failed" || retryItems.current.has(row.id),
+ );
if (refreshSeq.current !== requestId) return true;
setDocuments((prev) => {
const merged = rows.map((row) => {
@@ -306,10 +314,13 @@ export function useRagDocuments(
: row;
});
// Keep optimistic chips (not yet listed) so a refresh racing an upload
- // can't make them vanish.
+ // can't make them vanish, and failed rows this session can still retry.
const serverIds = new Set(rows.map((row) => row.id));
const pendingLocal = prev.filter(
- (row) => row.id.startsWith("pending_") && !serverIds.has(row.id),
+ (row) =>
+ !serverIds.has(row.id) &&
+ (row.id.startsWith("pending_") ||
+ (row.status === "failed" && retryItems.current.has(row.id))),
);
return [...merged, ...pendingLocal];
});
@@ -397,6 +408,7 @@ export function useRagDocuments(
uploadGenerationRef.current += 1;
activeUploadsRef.current.clear();
uploadInFlightRef.current = false;
+ retryItems.current.clear();
setUploading(false);
// Scope changes intentionally clear the old scope before fetching the new
// one. Keep this synchronous so React StrictMode's setup/cleanup replay
@@ -551,6 +563,7 @@ export function useRagDocuments(
);
if (generation !== uploadGenerationRef.current) return;
sigByDocId.current.set(result.documentId, itemSignature(item));
+ if (item.kind === "file") retryItems.current.set(result.documentId, item);
setDocuments((rows) =>
rows.some((row) => row.id === result.documentId)
? rows.filter((row) => row.id !== tempId)
@@ -569,12 +582,16 @@ export function useRagDocuments(
} catch (err) {
if (generation !== uploadGenerationRef.current) return;
const message = err instanceof Error ? err.message : String(err);
- // Drop the chip rather than show "Failed"; warn via toast.
- setDocuments((rows) => rows.filter((row) => row.id !== tempId));
+ if (item.kind === "file") {
+ retryItems.current.set(tempId, item);
+ patchDoc(tempId, { status: "failed", error: message, progress: null });
+ } else {
+ setDocuments((rows) => rows.filter((row) => row.id !== tempId));
+ }
toast.error(`Couldn't upload ${name}`, { description: message });
}
},
- [trackJob],
+ [trackJob, patchDoc],
);
// `overrideScope` lets a caller pass a freshly-resolved scope (or a promise of one), since the
@@ -701,9 +718,12 @@ export function useRagDocuments(
);
const remove = useCallback(
- async (documentId: string) => {
+ async (documentId: string): Promise<boolean> => {
const prev = documents;
setDocuments((rows) => rows.filter((row) => row.id !== documentId));
+ const prevRetry = retryItems.current.get(documentId);
+ retryItems.current.delete(documentId);
+ if (documentId.startsWith("pending_")) return true;
// Forget the dedup signature so re-uploading re-indexes.
const prevSig = sigByDocId.current.get(documentId);
sigByDocId.current.delete(documentId);
@@ -720,12 +740,15 @@ export function useRagDocuments(
documentId,
scope?.type === "project" ? scope.projectId : undefined,
);
+ return true;
} catch (err) {
setDocuments(prev);
if (prevSig !== undefined) sigByDocId.current.set(documentId, prevSig);
+ if (prevRetry !== undefined) retryItems.current.set(documentId, prevRetry);
toast.error("Delete failed", {
description: err instanceof Error ? err.message : String(err),
});
+ return false;
} finally {
if (removingProjectId) {
noteProjectWork(removingProjectId, -1);
@@ -735,6 +758,21 @@ export function useRagDocuments(
[documents, scope],
);
+ const canRetry = useCallback(
+ (documentId: string) => retryItems.current.has(documentId),
+ [],
+ );
+
+ const retry = useCallback(
+ async (documentId: string) => {
+ const item = retryItems.current.get(documentId);
+ if (!item) return;
+ if (!(await remove(documentId))) return;
+ await upload([item]);
+ },
+ [remove, upload],
+ );
+
return {
documents,
loading,
@@ -743,5 +781,7 @@ export function useRagDocuments(
refresh,
upload,
remove,
+ retry,
+ canRetry,
};
}
diff --git a/studio/frontend/src/features/rag/components/use-source-drop.ts b/studio/frontend/src/features/rag/components/use-source-drop.ts
index 22c1d171fb2..98435600ec9 100644
--- a/studio/frontend/src/features/rag/components/use-source-drop.ts
+++ b/studio/frontend/src/features/rag/components/use-source-drop.ts
@@ -6,6 +6,7 @@ import {
registerNativeAttachmentPath,
useNativeDropTarget,
} from "@/features/native-intents";
+import { MAX_FOLDER_FILES, filesFromDrop } from "@/lib/dropped-folders";
import { toast } from "@/lib/toast";
import {
type DragEvent as ReactDragEvent,
@@ -15,6 +16,7 @@ import {
} from "react";
import {
SUPPORTED_SOURCES_HINT,
+ isSupportedSourceName,
partitionSupported,
} from "./source-drop-policy";
import { type RagUploadItem, uploadItemFromIntent } from "./use-rag-documents";
@@ -157,14 +159,30 @@ export function useSourceDrop({
toast.info(reason);
return;
}
- const { supported, unsupported } = partitionSupported(
- Array.from(event.dataTransfer.files ?? []),
- (file) => file.name,
+ void filesFromDrop(event.dataTransfer, isSupportedSourceName).then(
+ ({ files, truncated, hadFolder }) => {
+ const { supported, unsupported } = partitionSupported(
+ files,
+ (file) => file.name,
+ );
+ if (!hadFolder) reportUnsupported(unsupported);
+ if (hadFolder && supported.length === 0) {
+ toast.info("No supported files in that folder", {
+ description: SUPPORTED_SOURCES_HINT,
+ });
+ }
+ if (truncated > 0) {
+ toast.info(`Added the first ${MAX_FOLDER_FILES} files`, {
+ description:
+ "The rest were left out. Drop a smaller folder for them.",
+ });
+ }
+ if (supported.length > 0) {
+ onItems(supported.map((file) => ({ kind: "file" as const, file })));
+ }
+ },
+ () => toast.error("Couldn't read the dropped folder"),
);
- reportUnsupported(unsupported);
- if (supported.length > 0) {
- onItems(supported.map((file) => ({ kind: "file" as const, file })));
- }
},
};
diff --git a/studio/frontend/src/features/rag/components/use-upload-queue.ts b/studio/frontend/src/features/rag/components/use-upload-queue.ts
new file mode 100644
index 00000000000..a45bc5a7681
--- /dev/null
+++ b/studio/frontend/src/features/rag/components/use-upload-queue.ts
@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { toast } from "@/lib/toast";
+import { useCallback, useEffect, useRef, useState } from "react";
+
+/** Queues items added during an upload and runs them when it ends; items queued for one `key`
+ * (destination) are dropped with a toast if the surface moves to another. */
+export function useUploadQueue<T>(
+ run: (items: T[]) => void,
+ busy: boolean,
+ key: string | null,
+): { enqueue: (items: T[]) => void; queued: number } {
+ const queue = useRef<{ key: string | null; items: T[] }>({ key, items: [] });
+ const [queued, setQueued] = useState(0);
+ const latestRun = useRef(run);
+ latestRun.current = run;
+ const latestKey = useRef(key);
+ latestKey.current = key;
+
+ const enqueue = useCallback(
+ (items: T[]) => {
+ if (items.length === 0) return;
+ // A folder walk resolves late: latestRun may point at another destination by now.
+ if (key !== latestKey.current) {
+ toast.info(
+ items.length === 1
+ ? "A dropped file was not added"
+ : `${items.length} dropped files were not added`,
+ { description: "You moved away before they finished reading." },
+ );
+ return;
+ }
+ if (!busy && queue.current.items.length === 0) {
+ latestRun.current(items);
+ return;
+ }
+ queue.current = { key, items: [...queue.current.items, ...items] };
+ setQueued(queue.current.items.length);
+ toast.info(
+ items.length === 1 ? "Queued 1 file" : `Queued ${items.length} files`,
+ { description: "They upload when the current upload finishes." },
+ );
+ },
+ [busy, key],
+ );
+
+ useEffect(() => {
+ const pending = queue.current;
+ if (pending.items.length === 0) return;
+ if (pending.key !== key) {
+ queue.current = { key, items: [] };
+ setQueued(0);
+ toast.info(
+ pending.items.length === 1
+ ? "A queued file was not added"
+ : `${pending.items.length} queued files were not added`,
+ { description: "You moved away before the earlier upload finished." },
+ );
+ return;
+ }
+ if (busy) return;
+ queue.current = { key, items: [] };
+ setQueued(0);
+ latestRun.current(pending.items);
+ }, [busy, key]);
+
+ return { enqueue, queued };
+}
diff --git a/studio/frontend/src/features/rag/types/rag.ts b/studio/frontend/src/features/rag/types/rag.ts
index fbabc0be642..7c5a6b03f09 100644
--- a/studio/frontend/src/features/rag/types/rag.ts
+++ b/studio/frontend/src/features/rag/types/rag.ts
@@ -99,11 +99,21 @@ export interface LinkedFolder {
status: LinkedFolderStatus;
documentCount?: number;
lastSyncedAt?: string | null;
+ lastChangedAt?: string | null;
error?: string | null;
activeJobId?: string | null;
createdAt?: string | null;
+ skipped?: Partial<Record<LinkedFolderSkipReason, number>>;
+ failures?: { path: string; error: string }[];
+ failureCount?: number;
}
+export type LinkedFolderSkipReason =
+ | "gitignored"
+ | "build_dirs"
+ | "too_large"
+ | "over_limit";
+
export function linkedFolderSourcesChanged(
previous: LinkedFolder[] | null,
current: LinkedFolder[],
@@ -121,7 +131,9 @@ export function linkedFolderSourcesChanged(
return (
prior !== undefined &&
(prior.documentCount !== folder.documentCount ||
- prior.lastSyncedAt !== folder.lastSyncedAt)
+ // lastSyncedAt moves every pass; older backends without lastChangedAt fall back to it.
+ (prior.lastChangedAt ?? prior.lastSyncedAt) !==
+ (folder.lastChangedAt ?? folder.lastSyncedAt))
);
});
}
@@ -140,10 +152,26 @@ export interface FolderSyncJob {
processedFiles?: number;
indexedFiles?: number;
removedFiles?: number;
+ renamedFiles?: number;
failedFiles?: number;
error?: string | null;
}
+/** Missing counts (older backends) count as a change. */
+export function jobChangedSources(job: FolderSyncJob): boolean {
+ if (
+ job.indexedFiles === undefined &&
+ job.removedFiles === undefined &&
+ job.renamedFiles === undefined
+ ) {
+ return true;
+ }
+ return (
+ (job.indexedFiles ?? 0) + (job.removedFiles ?? 0) + (job.renamedFiles ?? 0) >
+ 0
+ );
+}
+
/** One SSE frame from /linked-folder-jobs/{jobId}/events. */
export interface FolderSyncJobEvent extends Partial<FolderSyncJob> {
type: "progress" | "complete" | "error";
diff --git a/studio/frontend/src/lib/dropped-folders.ts b/studio/frontend/src/lib/dropped-folders.ts
new file mode 100644
index 00000000000..0eb5661e7fe
--- /dev/null
+++ b/studio/frontend/src/lib/dropped-folders.ts
@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// Browser folder drops and picks, flattened to files (the desktop links folders instead).
+
+export const MAX_FOLDER_FILES = 2000;
+
+const SKIPPED_DIRS = new Set([
+ "node_modules",
+ "bower_components",
+ "venv",
+ "__pycache__",
+]);
+
+function skippedDir(name: string): boolean {
+ return name.startsWith(".") || SKIPPED_DIRS.has(name.toLowerCase());
+}
+
+// Mirrors _is_ignored_scan_file in folder_sync.py: secrets and lockfiles.
+function skippedFile(name: string): boolean {
+ const lower = name.toLowerCase();
+ return (
+ lower.startsWith(".") ||
+ lower.endsWith(".env") ||
+ lower.endsWith(".lock") ||
+ lower.endsWith(".lockb") ||
+ lower.endsWith(".tfstate") ||
+ lower === "package-lock.json" ||
+ lower === "npm-shrinkwrap.json" ||
+ lower === "pnpm-lock.yaml" ||
+ lower === "pnpm-lock.yml"
+ );
+}
+
+export interface FolderFiles {
+ files: File[];
+ /** Files left out past MAX_FOLDER_FILES; a dropped folder stops at the first, so reports 1. */
+ truncated: number;
+ hadFolder: boolean;
+}
+
+function readBatch(
+ reader: FileSystemDirectoryReader,
+): Promise<FileSystemEntry[]> {
+ return new Promise((resolve, reject) => reader.readEntries(resolve, reject));
+}
+
+function entryFile(entry: FileSystemFileEntry): Promise<File> {
+ return new Promise((resolve, reject) => entry.file(resolve, reject));
+}
+
+/** Call inside the drop handler: the browser empties `items` once the event returns. */
+export function filesFromDrop(
+ dataTransfer: DataTransfer,
+ keep: (name: string) => boolean = () => true,
+): Promise<FolderFiles> {
+ const entries = Array.from(dataTransfer.items ?? [])
+ .filter((item) => item.kind === "file")
+ .map((item) => item.webkitGetAsEntry?.() ?? null);
+ const plain = Array.from(dataTransfer.files ?? []);
+ if (!entries.some((entry) => entry?.isDirectory)) {
+ return Promise.resolve({ files: plain, truncated: 0, hadFolder: false });
+ }
+ return walkEntries(
+ entries.filter((entry): entry is FileSystemEntry => entry !== null),
+ keep,
+ );
+}
+
+async function walkEntries(
+ roots: FileSystemEntry[],
+ keep: (name: string) => boolean,
+): Promise<FolderFiles> {
+ const files: File[] = [];
+ let truncated = 0;
+ const pending = [...roots];
+ while (pending.length > 0) {
+ const entry = pending.shift() as FileSystemEntry;
+ if (entry.isFile) {
+ const root = roots.indexOf(entry) >= 0;
+ if (!root && (skippedFile(entry.name) || !keep(entry.name))) continue;
+ if (files.length >= MAX_FOLDER_FILES) {
+ truncated = 1;
+ break;
+ }
+ try {
+ const file = await entryFile(entry as FileSystemFileEntry);
+ if (file.size > 0 || root) files.push(file);
+ } catch {
+ }
+ continue;
+ }
+ if (
+ !entry.isDirectory ||
+ (roots.indexOf(entry) < 0 && skippedDir(entry.name))
+ ) {
+ continue;
+ }
+ const reader = (entry as FileSystemDirectoryEntry).createReader();
+ // readEntries returns ~100 entries per call; an empty batch ends the folder.
+ for (;;) {
+ let batch: FileSystemEntry[];
+ try {
+ batch = await readBatch(reader);
+ } catch {
+ break;
+ }
+ if (batch.length === 0) break;
+ pending.push(...batch);
+ }
+ }
+ return { files, truncated, hadFolder: true };
+}
+
+export function filesFromFolderInput(
+ list: FileList | File[],
+ keep: (name: string) => boolean = () => true,
+): FolderFiles {
+ const all = Array.from(list).filter((file) => {
+ const segments = (file.webkitRelativePath || file.name).split("/");
+ // Segment 0 is the chosen folder itself.
+ return (
+ file.size > 0 &&
+ !segments.slice(1, -1).some(skippedDir) &&
+ !skippedFile(file.name) &&
+ keep(file.name)
+ );
+ });
+ const files = all.slice(0, MAX_FOLDER_FILES);
+ return { files, truncated: all.length - files.length, hadFolder: true };
+}
+
+/** Call from a click handler (needs user activation). */
+export function openFolderPicker(
+ onFiles: (folder: FolderFiles) => void,
+ keep?: (name: string) => boolean,
+): void {
+ const input = document.createElement("input");
+ input.type = "file";
+ input.multiple = true;
+ input.webkitdirectory = true;
+ input.hidden = true;
+
+ document.body.appendChild(input);
+ input.onchange = () => {
+ const files = input.files;
+ if (files && files.length > 0) {
+ onFiles(filesFromFolderInput(files, keep));
+ }
+ document.body.removeChild(input);
+ };
+ input.oncancel = () => {
+ if (!input.files || input.files.length === 0) {
+ document.body.removeChild(input);
+ }
+ };
+ input.click();
+}
diff --git a/studio/frontend/tests/knowledge-base-dialog-focus.test.ts b/studio/frontend/tests/knowledge-base-dialog-focus.test.ts
index 377908ba1b1..3410f1a1e02 100644
--- a/studio/frontend/tests/knowledge-base-dialog-focus.test.ts
+++ b/studio/frontend/tests/knowledge-base-dialog-focus.test.ts
@@ -187,6 +187,12 @@ function harness(rows: Array<typeof KB> = [KB]) {
nativeDropTarget: () => {},
}),
},
+ "./use-upload-queue": {
+ useUploadQueue: (run: (items: unknown[]) => void) => ({
+ enqueue: run,
+ queued: 0,
+ }),
+ },
},
);
diff --git a/studio/frontend/tests/native-dropzone-coverage.test.ts b/studio/frontend/tests/native-dropzone-coverage.test.ts
index 6d2e056da67..6dc2178775c 100644
--- a/studio/frontend/tests/native-dropzone-coverage.test.ts
+++ b/studio/frontend/tests/native-dropzone-coverage.test.ts
@@ -52,7 +52,7 @@ test("every file drop zone is reachable from the desktop app", async () => {
if (!FILE_DROP_MARKERS.some((marker) => source.includes(marker))) continue;
// The shared readers themselves (the DataTransfer walker, the hook) are not
// drop zones; their callers are the ones that have to be reachable.
- if (/export (async )?function filesFromDataTransfer/.test(source)) continue;
+ if (/export (async )?function (filesFromDataTransfer|filesFromDrop)\b/.test(source)) continue;
if (NATIVE_MARKERS.some((marker) => source.includes(marker))) continue;
dead.push(path.relative(new URL(".", SRC).pathname, file.pathname));
}
diff --git a/studio/frontend/tests/project-attachment-target-adoption.test.ts b/studio/frontend/tests/project-attachment-target-adoption.test.ts
index d66d93396d1..d5a68436745 100644
--- a/studio/frontend/tests/project-attachment-target-adoption.test.ts
+++ b/studio/frontend/tests/project-attachment-target-adoption.test.ts
@@ -107,9 +107,10 @@ test("attaching is held until the chat's project is known", () => {
/const projectUnresolved = threadProjectId === undefined;/,
"unresolved has to be distinguishable from no project",
);
+ // An upload in flight only queues new files; an unknown project still holds them.
assert.match(
THREAD_DOCUMENTS_BAR,
- /uploading \|\| projectUploading \|\| projectUnresolved/,
+ /const busy = projectUnresolved;/,
"the attach controls hold",
);
assert.match(
diff --git a/studio/frontend/tests/project-sources-model-without-tools.test.ts b/studio/frontend/tests/project-sources-model-without-tools.test.ts
index 11727d66efa..3e9c1cafbf8 100644
--- a/studio/frontend/tests/project-sources-model-without-tools.test.ts
+++ b/studio/frontend/tests/project-sources-model-without-tools.test.ts
@@ -8,6 +8,7 @@ import * as jsxRuntime from "react/jsx-runtime";
import { renderToStaticMarkup } from "react-dom/server";
import type * as BarModule from "../src/features/rag/components/thread-documents-bar.tsx";
+import type * as FolderGroupsModule from "../src/features/rag/components/linked-folder-groups.ts";
import type * as GateModule from "../src/features/chat/hooks/use-rag-tool-disabled.ts";
import { loadWithStubs } from "./helpers/module-stubs.ts";
@@ -40,6 +41,11 @@ const Nothing = () => null;
const Passthrough = ({ children }: { children?: React.ReactNode }) =>
React.createElement(React.Fragment, null, children);
+const folderGroups = loadWithStubs<typeof FolderGroupsModule>(
+ new URL("../src/features/rag/components/linked-folder-groups.ts", import.meta.url),
+ { react: React, "../api/rag-api": { listLinkedFolders: async () => [] } },
+);
+
const { ThreadDocumentsBar } = loadWithStubs<typeof BarModule>(
new URL("../src/features/rag/components/thread-documents-bar.tsx", import.meta.url),
{
@@ -72,7 +78,11 @@ const { ThreadDocumentsBar } = loadWithStubs<typeof BarModule>(
useDocumentPreviewStore: selectorStore({ openPreview: () => undefined }),
},
"@/lib/chevron-icons": {},
- "@assistant-ui/react": { useAui: () => ({}) },
+ "@assistant-ui/react": {
+ useAui: () => ({}),
+ useAuiState: (select: (s: { threadListItem: { id: string } }) => unknown) =>
+ select({ threadListItem: { id: "local-1" } }),
+ },
"@/lib/utils": {
cn: (...classes: unknown[]) => classes.filter(Boolean).join(" "),
},
@@ -124,6 +134,13 @@ const { ThreadDocumentsBar } = loadWithStubs<typeof BarModule>(
nativeDropTarget: () => undefined,
}),
},
+ "./use-upload-queue": {
+ useUploadQueue: (run: (items: unknown[]) => void) => ({
+ enqueue: run,
+ queued: 0,
+ }),
+ },
+ "./linked-folder-groups": folderGroups,
"@/components/ui/alert-dialog": new Proxy({}, { get: () => Passthrough }),
"./document-status-chip": { STAGE_LABELS: {} },
"./knowledge-base-dialog": { KnowledgeBaseDialog: Nothing },
@@ -138,6 +155,8 @@ const { ThreadDocumentsBar } = loadWithStubs<typeof BarModule>(
loading: false,
upload: async () => undefined,
remove: async () => undefined,
+ retry: async () => undefined,
+ canRetry: () => false,
}),
},
},
diff --git a/studio/frontend/tests/rag-refresh-sequencing.test.ts b/studio/frontend/tests/rag-refresh-sequencing.test.ts
index b7c77ebd83e..a7da39cf551 100644
--- a/studio/frontend/tests/rag-refresh-sequencing.test.ts
+++ b/studio/frontend/tests/rag-refresh-sequencing.test.ts
@@ -729,7 +729,7 @@ test("a failed project lookup leaves the scope unresolved", () => {
);
// And unresolved still disables the attach controls.
assert.match(THREAD_DOCUMENTS_BAR, /const projectUnresolved = threadProjectId === undefined;/);
- assert.match(THREAD_DOCUMENTS_BAR, /uploading \|\| projectUploading \|\| projectUnresolved/);
+ assert.match(THREAD_DOCUMENTS_BAR, /const busy = projectUnresolved;/);
});
// A row the probe could not read is not a chat with no project: answering null
diff --git a/studio/frontend/tests/rag-source-followups.test.ts b/studio/frontend/tests/rag-source-followups.test.ts
new file mode 100644
index 00000000000..69a50878deb
--- /dev/null
+++ b/studio/frontend/tests/rag-source-followups.test.ts
@@ -0,0 +1,231 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ type FolderSyncJob,
+ type LinkedFolder,
+ jobChangedSources,
+ linkedFolderSourcesChanged,
+} from "../src/features/rag/types/rag.ts";
+import {
+ MAX_FOLDER_FILES,
+ filesFromFolderInput,
+} from "../src/lib/dropped-folders.ts";
+import { loadWithStubs } from "./helpers/module-stubs.ts";
+
+const folder = (overrides: Partial<LinkedFolder> = {}): LinkedFolder => ({
+ id: "f1",
+ displayName: "Docs",
+ scopeType: "project",
+ scopeId: "p1",
+ status: "idle",
+ documentCount: 3,
+ lastSyncedAt: "2026-10-10T00:00:00Z",
+ lastChangedAt: "2026-10-09T00:00:00Z",
+ ...overrides,
+});
+
+// Every 30 s pass bumps lastSyncedAt; reading that as a change refetched every document list.
+test("a periodic pass that changed nothing is not a sources change", () => {
+ const before = [folder()];
+ const after = [folder({ lastSyncedAt: "2026-10-10T00:00:30Z" })];
+ assert.equal(linkedFolderSourcesChanged(before, after), false);
+ assert.equal(
+ linkedFolderSourcesChanged(before, [
+ folder({ lastChangedAt: "2026-10-10T00:00:30Z" }),
+ ]),
+ true,
+ );
+ assert.equal(
+ linkedFolderSourcesChanged(before, [folder({ documentCount: 4 })]),
+ true,
+ );
+});
+
+test("an older backend without lastChangedAt still reports syncs as changes", () => {
+ const legacy = (at: string) =>
+ folder({ lastChangedAt: undefined, lastSyncedAt: at });
+ assert.equal(linkedFolderSourcesChanged([legacy("a")], [legacy("b")]), true);
+});
+
+test("a finished job changed sources only if it added, removed or renamed something", () => {
+ const job = (counts: Partial<FolderSyncJob>): FolderSyncJob => ({
+ id: "j",
+ linkedFolderId: "f1",
+ mode: "sync",
+ status: "completed",
+ ...counts,
+ });
+ assert.equal(
+ jobChangedSources(
+ job({ indexedFiles: 0, removedFiles: 0, renamedFiles: 0 }),
+ ),
+ false,
+ );
+ assert.equal(
+ jobChangedSources(job({ indexedFiles: 2, removedFiles: 0 })),
+ true,
+ );
+ assert.equal(
+ jobChangedSources(job({ indexedFiles: 0, renamedFiles: 1 })),
+ true,
+ );
+ assert.equal(jobChangedSources(job({})), true);
+});
+
+function inFolder(path: string, content = "x"): File {
+ const file = new File([content], path.split("/").pop() ?? path);
+ Object.defineProperty(file, "webkitRelativePath", { value: path });
+ return file;
+}
+
+test("a picked folder leaves out dependency, hidden and secret files", () => {
+ const { files, truncated } = filesFromFolderInput([
+ inFolder("repo/README.md"),
+ inFolder("repo/src/app.ts"),
+ inFolder("repo/node_modules/dep/index.js"),
+ inFolder("repo/.git/config"),
+ inFolder("repo/.venv/lib/site.py"),
+ inFolder("repo/prod.env"),
+ inFolder("repo/package-lock.json"),
+ inFolder("repo/pnpm-lock.yml"),
+ inFolder("repo/src/__init__.py", ""),
+ ]);
+ assert.deepEqual(
+ files.map((file) => file.webkitRelativePath),
+ ["repo/README.md", "repo/src/app.ts"],
+ );
+ assert.equal(truncated, 0);
+});
+
+// Unsupported files must not use up the cap, or a folder of images hides the documents after them.
+test("only files the caller takes count toward the folder cap", () => {
+ const images = Array.from({ length: MAX_FOLDER_FILES }, (_, i) =>
+ inFolder(`mixed/${i}.png`),
+ );
+ const { files, truncated } = filesFromFolderInput(
+ [...images, inFolder("mixed/notes.md")],
+ (name) => name.endsWith(".md"),
+ );
+ assert.deepEqual(
+ files.map((file) => file.name),
+ ["notes.md"],
+ );
+ assert.equal(truncated, 0);
+});
+
+test("a picked folder past the cap keeps the first files and counts the rest", () => {
+ const many = Array.from({ length: MAX_FOLDER_FILES + 5 }, (_, i) =>
+ inFolder(`big/${i}.md`),
+ );
+ const { files, truncated } = filesFromFolderInput(many);
+ assert.equal(files.length, MAX_FOLDER_FILES);
+ assert.equal(truncated, 5);
+});
+
+/** Drives useUploadQueue renders with a stub React whose effects run after each render. */
+function queueHarness() {
+ const slots: unknown[] = [];
+ const effects: Array<() => void> = [];
+ const toasts: string[] = [];
+ let cursor = 0;
+ const react = {
+ useRef(value: unknown) {
+ const index = cursor++;
+ slots[index] ??= { current: value };
+ return slots[index];
+ },
+ useState(value: unknown) {
+ const index = cursor++;
+ if (!(index in slots)) slots[index] = value;
+ return [
+ slots[index],
+ (next: unknown) => {
+ slots[index] = next;
+ },
+ ];
+ },
+ useCallback: (fn: unknown) => fn,
+ useEffect(effect: () => void, deps: unknown[]) {
+ const index = cursor++;
+ const previous = slots[index] as unknown[] | undefined;
+ if (previous && deps.every((dep, i) => Object.is(dep, previous[i])))
+ return;
+ slots[index] = deps;
+ effects.push(effect);
+ },
+ };
+ const { useUploadQueue } = loadWithStubs<{
+ useUploadQueue: (
+ run: (items: string[]) => void,
+ busy: boolean,
+ key: string | null,
+ ) => { enqueue: (items: string[]) => void; queued: number };
+ }>(
+ new URL(
+ "../src/features/rag/components/use-upload-queue.ts",
+ import.meta.url,
+ ),
+ {
+ react,
+ "@/lib/toast": {
+ toast: { info: (message: string) => toasts.push(message) },
+ },
+ },
+ );
+ const ran: string[][] = [];
+ return {
+ ran,
+ toasts,
+ render(busy: boolean, key: string | null) {
+ cursor = 0;
+ const result = useUploadQueue((items) => ran.push(items), busy, key);
+ for (const effect of effects.splice(0)) effect();
+ return result;
+ },
+ };
+}
+
+test("files added during an upload wait for it, then go in one batch", () => {
+ const app = queueHarness();
+ app.render(false, "p1").enqueue(["a"]);
+ assert.deepEqual(app.ran, [["a"]]);
+
+ let hook = app.render(true, "p1");
+ hook.enqueue(["b"]);
+ hook = app.render(true, "p1");
+ hook.enqueue(["c"]);
+ assert.deepEqual(
+ app.ran,
+ [["a"]],
+ "nothing starts while the first upload runs",
+ );
+ assert.equal(app.render(true, "p1").queued, 2);
+
+ app.render(false, "p1");
+ assert.deepEqual(app.ran, [["a"], ["b", "c"]]);
+ assert.equal(app.render(false, "p1").queued, 0);
+});
+
+test("a drop that resolves after a switch is not added to the new destination", () => {
+ const app = queueHarness();
+ const stale = app.render(false, "p1").enqueue;
+ app.render(false, "p2");
+ stale(["a"]);
+ assert.deepEqual(app.ran, []);
+ assert.ok(app.toasts.some((message) => message.includes("not added")));
+ app.render(false, "p2").enqueue(["b"]);
+ assert.deepEqual(app.ran, [["b"]]);
+});
+
+test("files queued for one project are not added to the next", () => {
+ const app = queueHarness();
+ app.render(true, "p1").enqueue(["b"]);
+ app.render(true, "p2");
+ app.render(false, "p2");
+ assert.deepEqual(app.ran, []);
+ assert.ok(app.toasts.some((message) => message.includes("not added")));
+});
diff --git a/studio/frontend/tests/rag-thread-initialization.test.ts b/studio/frontend/tests/rag-thread-initialization.test.ts
index 067950f8cf2..81c7b141c6b 100644
--- a/studio/frontend/tests/rag-thread-initialization.test.ts
+++ b/studio/frontend/tests/rag-thread-initialization.test.ts
@@ -162,7 +162,11 @@ function harness(
"@/components/ui/spinner": {},
"./preview-store": { useDocumentPreviewStore: () => undefined },
"@/lib/chevron-icons": {},
- "@assistant-ui/react": { useAui: () => ({ threadListItem: () => item }) },
+ "@assistant-ui/react": {
+ useAui: () => ({ threadListItem: () => item }),
+ useAuiState: (select: (s: { threadListItem: { id: string } }) => unknown) =>
+ select({ threadListItem: { id: itemId } }),
+ },
"@/lib/utils": { cn: () => "" },
"@/features/chat/stores/chat-runtime-store": {
useChatRuntimeStore: store,
@@ -226,6 +230,16 @@ function harness(
isSupportedSourceName: (name: string) => name.endsWith(".docx"),
},
"./use-source-drop": {},
+ "./use-upload-queue": {
+ useUploadQueue: (run: (items: unknown[]) => void) => ({
+ enqueue: run,
+ queued: 0,
+ }),
+ },
+ "./linked-folder-groups": {
+ groupByLinkedFolder: (docs: unknown[]) => ({ loose: docs, folders: [] }),
+ useLinkedFolderNames: () => new Map(),
+ },
"./document-status-chip": {},
"./knowledge-base-dialog": { KnowledgeBaseDialog: "KnowledgeBaseDialog" },
"./staged-source": { EXPIRY_GRACE_MS: 30_000 },
diff --git a/studio/frontend/tests/rag-upload-lifecycle.test.ts b/studio/frontend/tests/rag-upload-lifecycle.test.ts
index ce1f53cc2b1..d094add9fd2 100644
--- a/studio/frontend/tests/rag-upload-lifecycle.test.ts
+++ b/studio/frontend/tests/rag-upload-lifecycle.test.ts
@@ -21,6 +21,9 @@ type Hook = {
uploading: boolean;
hasIndexing: boolean;
upload: (files: File[], scope?: ScopeOverride) => Promise<void>;
+ retry: (documentId: string) => Promise<void>;
+ canRetry: (documentId: string) => boolean;
+ refresh: (opts?: { quiet?: boolean }) => Promise<boolean>;
};
function deferred<T>() {
@@ -39,8 +42,12 @@ function harness(
events?: (jobId: string, signal?: AbortSignal) => AsyncGenerator<JobEvent>;
getJob?: () => Promise<IndexJob>;
list?: () => Promise<RagDocument[]>;
+ /** This many uploads are refused before one goes through. */
+ failUploads?: number;
+ failDeletes?: boolean;
} = {},
) {
+ let refusals = options.failUploads ?? 0;
const slots: unknown[] = [];
const effects: Array<() => void> = [];
let cursor = 0;
@@ -118,9 +125,17 @@ function harness(
"./vision-overrides": { resolveVisionOverrides: async () => ({}) },
"../api/rag-api": {
uploadThreadDocument: async (threadId: string) => {
+ if (refusals > 0) {
+ refusals -= 1;
+ throw new Error("Server busy");
+ }
uploads.push(threadId);
return uploaded;
},
+ deleteDocument: async () => {
+ if (options.failDeletes) throw new Error("Server busy");
+ },
+ noteProjectWork: () => undefined,
streamJobEvents:
options.events ??
async function* () {
@@ -274,13 +289,115 @@ test("an original job failure is preserved after its event stream ends", async (
await flush();
await hook.upload([report()]);
await flush();
- assert.deepEqual(app.render().documents, []);
+ // The failed row stays, with its reason, so it can be retried or removed.
+ const after = app.render();
+ assert.deepEqual(
+ after.documents.map(({ id, status, error }) => ({ id, status, error })),
+ [{ id: "doc", status: "failed", error: "Invalid PDF" }],
+ );
+ assert.equal(after.canRetry("doc"), true);
assert.deepEqual(app.errors, ["Couldn't index report.pdf"]);
} finally {
app.dispose();
}
});
+test("a refresh that lists the failure first keeps the row retryable", async () => {
+ let listed = false;
+ let release = () => {};
+ const polled = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ const app = harness({
+ events: async function* () {},
+ // The job poll answers only after the list has already reported the failure.
+ getJob: async () => {
+ await polled;
+ return { id: "job", documentId: "doc", status: "failed", error: "Invalid PDF" };
+ },
+ list: async () =>
+ listed
+ ? ([{ id: "doc", filename: "report.pdf", status: "failed", error: "Invalid PDF" }] as unknown as RagDocument[])
+ : [],
+ });
+ try {
+ let hook = app.render();
+ await flush();
+ const uploading = hook.upload([report()]);
+ await flush();
+ listed = true;
+ await app.render().refresh({ quiet: true });
+ await flush();
+ release();
+ await uploading;
+ await flush();
+ hook = app.render();
+ assert.deepEqual(
+ hook.documents.map(({ id, status }) => ({ id, status })),
+ [{ id: "doc", status: "failed" }],
+ );
+ assert.equal(hook.canRetry("doc"), true);
+ } finally {
+ app.dispose();
+ }
+});
+
+test("a refused upload stays as a failed chip and Retry sends the same file again", async () => {
+ const app = harness({ failUploads: 1 });
+ try {
+ let hook = app.render();
+ await flush();
+ await hook.upload([report()]);
+ await flush();
+ hook = app.render();
+ const failed = hook.documents[0];
+ assert.equal(failed?.status, "failed");
+ assert.equal(failed?.error, "Server busy");
+ assert.equal(hook.canRetry(failed!.id), true);
+ assert.deepEqual(app.errors, ["Couldn't upload report.pdf"]);
+
+ await hook.retry(failed!.id);
+ await flush();
+ hook = app.render();
+ assert.deepEqual(app.uploads, ["thread"]);
+ assert.deepEqual(
+ hook.documents.map(({ id, status }) => ({ id, status })),
+ [{ id: "doc", status: "completed" }],
+ );
+ } finally {
+ app.dispose();
+ }
+});
+
+test("a retry whose delete fails keeps the failed row retryable and uploads nothing", async () => {
+ let reads = 0;
+ const app = harness({
+ failDeletes: true,
+ events: async function* () {},
+ getJob: async () => ({
+ id: "job",
+ documentId: "doc",
+ status: ++reads === 1 ? "running" : "failed",
+ error: "Invalid PDF",
+ }),
+ });
+ try {
+ let hook = app.render();
+ await flush();
+ await hook.upload([report()]);
+ await flush();
+ hook = app.render();
+ await hook.retry("doc");
+ await flush();
+ hook = app.render();
+ assert.deepEqual(app.uploads, ["thread"], "no second upload beside the undeleted one");
+ assert.equal(hook.documents[0]?.status, "failed");
+ assert.equal(hook.canRetry("doc"), true);
+ } finally {
+ app.dispose();
+ }
+});
+
test("an upload begun without a scope stops at the chat the user left", async () => {
const app = harness();
try {
diff --git a/tests/studio/rag_upload_browser_tests.py b/tests/studio/rag_upload_browser_tests.py
index 5c6204b1e6b..acd85c4deb5 100644
--- a/tests/studio/rag_upload_browser_tests.py
+++ b/tests/studio/rag_upload_browser_tests.py
@@ -132,7 +132,10 @@ def error(page):
timeout = 30000,
)
assert not page.evaluate("window.sim.hasIndexing || window.sim.uploading")
- assert page.evaluate("window.sim.documents") == []
+ # The failed row stays, with its reason, so it can be retried or removed.
+ documents = page.evaluate("window.sim.documents")
+ assert [(d["filename"], d["status"]) for d in documents] == [("report.pdf", "failed")]
+ assert documents[0]["error"].startswith("simulated ")
def failed_scope(page):