Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: wait for a pending embedding model download instead of failing every linked-folder file (#16722)

Debojyoti Banerjee2ea866277bbf
diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.py
index 32078bfa714..50d5c126c27 100644
--- a/studio/backend/core/rag/embed_llama_server.py
+++ b/studio/backend/core/rag/embed_llama_server.py
@@ -534,12 +534,9 @@ class LlamaServerBackend:
return self._adopt_model_path(cached, desired)
# Typed: the same condition the ST path raises, and /v1/embeddings answers it
# with a 409. Bare, it fell into the catch-all as a 502 naming nothing.
- from .embeddings import EmbeddingModelDownloadRequiredError
+ from .embeddings import EmbeddingModelDownloadRequiredError, download_required_message
- raise EmbeddingModelDownloadRequiredError(
- f"Embedding model {model!r} is not downloaded yet. "
- "Finish its Settings download before indexing documents."
- )
+ raise EmbeddingModelDownloadRequiredError(download_required_message(model))
return self._resolve_uncached_model_path(desired, candidates)
@staticmethod
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py
index 41ff05ae622..f0d4f85e957 100644
--- a/studio/backend/core/rag/embeddings.py
+++ b/studio/backend/core/rag/embeddings.py
@@ -143,6 +143,37 @@ class EmbeddingModelDownloadRequiredError(RuntimeError):
"""The picker activated a model whose explicit transfer is still pending."""
+# Shared by the raise sites and is_download_required_error: a document job keeps only the error text.
+DOWNLOAD_REQUIRED_HINT = "Finish its Settings download before indexing documents."
+
+
+def download_required_message(name: str) -> str:
+ return f"Embedding model {name!r} is not downloaded yet. {DOWNLOAD_REQUIRED_HINT}"
+
+
+def is_download_required_error(error: str | None) -> bool:
+ """A stored error text from EmbeddingModelDownloadRequiredError: a wait, not a bad file."""
+ return bool(error) and DOWNLOAD_REQUIRED_HINT in error
+
+
+def download_required(model_name: str | None = None) -> bool:
+ """True while ``model_name``'s Settings transfer is pending and the cache-only loader still cannot load it."""
+ name = model_name or config.effective_embedding_model()
+ try:
+ from utils.embedding_model_settings import get_stored_download_pending
+ if not get_stored_download_pending(name):
+ return False
+ except Exception: # noqa: BLE001 - old/unavailable settings store
+ return False
+ try:
+ warm(name)
+ except EmbeddingModelDownloadRequiredError:
+ return True
+ except Exception: # noqa: BLE001 - any other load failure stays the per-document error it was
+ logger.debug("embedding model %s failed to warm", name, exc_info = True)
+ return False
+
+
def _ambient_hf_token() -> str | None:
"""The HF token the loader itself would use (HF_TOKEN env or the cached login), so
the scan can reach a gated/private repo instead of failing open. None if unavailable."""
@@ -604,19 +635,13 @@ def _get(model_name: str | None = None):
if download_pending and st_source is None:
# Defensive: a loadable check and snapshot lookup share no lock, so eviction between them is
# still a pending model.
- raise EmbeddingModelDownloadRequiredError(
- f"Embedding model {name!r} is not downloaded yet. "
- "Finish its Settings download before indexing documents."
- )
+ raise EmbeddingModelDownloadRequiredError(download_required_message(name))
snapshot = st_source[1] if st_source else hf_cache_snapshot_dir(name)
if snapshot is not None:
# Re-emit in finally: MISSING or MISMATCH lines matter most when the load raises.
load_target = str(snapshot)
elif download_pending:
- raise EmbeddingModelDownloadRequiredError(
- f"Embedding model {name!r} is not downloaded yet. "
- "Finish its Settings download before indexing documents."
- )
+ raise EmbeddingModelDownloadRequiredError(download_required_message(name))
elif _st_accepts_local_files_only(SentenceTransformer):
st_kwargs["local_files_only"] = True
# Scan only after load_target is settled, so the scanned commit is the loaded one.
diff --git a/studio/backend/core/rag/folder_sync.py b/studio/backend/core/rag/folder_sync.py
index fcc3f78378e..d36c049e544 100644
--- a/studio/backend/core/rag/folder_sync.py
+++ b/studio/backend/core/rag/folder_sync.py
@@ -31,7 +31,7 @@ from pathlib import Path, PurePosixPath
from core.rag import account_db as rag_db
from utils.paths import ensure_dir, rag_uploads_root
-from . import config, gitignore, ingestion, job_leases, store
+from . import config, embeddings, gitignore, ingestion, job_leases, store
from utils.paths.path_utils import is_appledouble_metadata
logger = logging.getLogger(__name__)
@@ -1524,6 +1524,8 @@ def _load_failed(folder: dict) -> dict[str, dict]:
if isinstance(entry, dict)
and isinstance(entry.get("at"), (int, float))
and entry["at"] > cutoff
+ # A pending embedder download is not the file's fault; also frees entries saved before this check.
+ and not embeddings.is_download_required_error(entry.get("error"))
}
@@ -1681,6 +1683,24 @@ def _reconcile_folder(job_id: str) -> None:
# An unchanged file whose ingestion failed fails again: retrying it every periodic sync kept a
# folder of scanned PDFs indexing forever (#13095). Sync changes and Rebuild retry at once.
known_failed = {} if rebuild else _load_failed(folder)
+ if any(
+ known_failed.get(rel, {}).get("sig") != _failure_signature(current[rel]) for rel in work
+ ) and embeddings.download_required(embedding_model):
+ # One job-level wait, not every file failing the same check; the periodic sync resumes it.
+ error = f"The embedding model {embedding_model!r} is not downloaded yet. " + (
+ "Indexing starts on its own once its Settings download finishes."
+ if folder.get("auto_sync")
+ else "Sync again once its Settings download finishes."
+ )
+ _set_job(job_id, status = "failed", stage = "error", error = error, completed_at = _now())
+ with closing(rag_db.get_connection()) as conn:
+ conn.execute(
+ "UPDATE linked_folders SET status='error', last_error=?, updated_at=? WHERE id=?",
+ (error, _now(), folder["id"]),
+ )
+ conn.commit()
+ _prune_terminal_jobs(conn)
+ return
ingest_failed: dict[str, dict] = {}
cached_grace = False
total = len(work) + len(missing)
@@ -1736,11 +1756,13 @@ def _reconcile_folder(job_id: str) -> None:
failures.append(pending.relative_path)
reason = _error_text(result.error, folder["path"])
failure_errors[pending.relative_path] = reason
- ingest_failed[pending.relative_path] = {
- "sig": _failure_signature(pending.metadata),
- "at": time.time(),
- "error": reason,
- }
+ if not embeddings.is_download_required_error(reason):
+ # A transfer that went pending mid-sync is retried by the next sync.
+ ingest_failed[pending.relative_path] = {
+ "sig": _failure_signature(pending.metadata),
+ "at": time.time(),
+ "error": reason,
+ }
# A failed file may be a rename or copy of a vanished path, so grant one pass.
withheld.update(missing - already_withheld)
logger.warning(
diff --git a/studio/backend/tests/test_rag_folder_sync_download_pending.py b/studio/backend/tests/test_rag_folder_sync_download_pending.py
new file mode 100644
index 00000000000..3d1b4ef324c
--- /dev/null
+++ b/studio/backend/tests/test_rag_folder_sync_download_pending.py
@@ -0,0 +1,270 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A linked-folder sync while the embedding model's Settings transfer is still pending."""
+
+from __future__ import annotations
+
+import json
+import time
+from contextlib import closing
+
+import pytest
+
+from core.rag import config, embeddings, folder_sync
+from storage import rag_db
+
+requires_sqlite_vec = pytest.mark.skipif(
+ not rag_db.RAG_AVAILABLE, reason = "sqlite-vec is not installed"
+)
+
+
[email protected](autouse = True)
+def _forget_pending_marker():
+ # The resolution memo is process-wide: a pending marker left here would make later llama-server tests cache-only.
+ from utils import embedding_model_settings as ems
+
+ ems._resolved_gguf_memo.clear()
+ ems._invalidate_cache()
+ yield
+ ems._resolved_gguf_memo.clear()
+ ems._invalidate_cache()
+
+
+def _run(folder_id: str, *, rebuild: bool = False) -> dict:
+ job_id = folder_sync.request_sync(folder_id, rebuild = rebuild)
+ folder_sync.reconcile_folder(job_id)
+ return folder_sync.get_job(job_id)
+
+
+def _run_periodic(folder_id: str) -> dict:
+ """The background schedule's pass, which, unlike Sync changes, keeps recently failed files out."""
+ folder_sync._enqueue_periodic()
+ with closing(rag_db.get_connection()) as conn:
+ job_id = conn.execute(
+ "SELECT id FROM linked_folder_sync_jobs WHERE folder_id=? AND status='pending'",
+ (folder_id,),
+ ).fetchone()["id"]
+ folder_sync.reconcile_folder(job_id)
+ return folder_sync.get_job(job_id)
+
+
+def _folder(rag_home, names):
+ source = rag_home / "source"
+ source.mkdir()
+ for index, name in enumerate(names):
+ (source / name).write_text(f"note {index} about lighthouses", encoding = "utf-8")
+ row = folder_sync.create_folder(
+ scope_type = "knowledge_base",
+ scope_id = "scope-1",
+ path = str(source),
+ name = "Vault",
+ )
+ return source, row
+
+
+def _folder_row(folder_id: str) -> dict:
+ with closing(rag_db.get_connection()) as conn:
+ return dict(
+ conn.execute("SELECT * FROM linked_folders WHERE id=?", (folder_id,)).fetchone()
+ )
+
+
+def _mapped(folder_id: str) -> set[str]:
+ with closing(rag_db.get_connection()) as conn:
+ return {
+ row["relative_path"]
+ for row in conn.execute(
+ "SELECT relative_path FROM linked_folder_files WHERE folder_id=?", (folder_id,)
+ )
+ }
+
+
+def pending_download(rag_home, monkeypatch):
+ """Mark the active embedding model as activated before its transfer finished. ``state["landed"]``
+ flips the transfer to complete; until then the loader raises as the real cache-only one does."""
+ from utils.embedding_model_settings import (
+ clear_stored_download_pending,
+ set_rag_embedding_model,
+ )
+
+ model = config.effective_embedding_model()
+ set_rag_embedding_model(model, backend = "sentence-transformers", download_pending = True)
+ state = {"landed": False, "warms": 0}
+
+ def warm(model_name = None):
+ state["warms"] += 1
+ name = model_name or config.effective_embedding_model()
+ if not state["landed"]:
+ raise embeddings.EmbeddingModelDownloadRequiredError(
+ embeddings.download_required_message(name)
+ )
+ clear_stored_download_pending(name)
+
+ monkeypatch.setattr(embeddings, "warm", warm)
+ return state
+
+
+def test_download_required_skips_the_loader_when_nothing_is_pending(rag_home, monkeypatch):
+ def warm(model_name = None):
+ raise AssertionError("no transfer is pending, so nothing should load")
+
+ monkeypatch.setattr(embeddings, "warm", warm)
+ assert embeddings.download_required() is False
+
+
+def test_download_required_follows_the_pending_transfer(pending_download):
+ assert embeddings.download_required() is True
+ pending_download["landed"] = True
+ assert embeddings.download_required() is False
+ # The loader retired the marker, so later checks no longer load anything.
+ warms = pending_download["warms"]
+ assert embeddings.download_required() is False
+ assert pending_download["warms"] == warms
+
+
+def test_download_required_leaves_other_load_failures_to_the_documents(rag_home, monkeypatch):
+ from utils.embedding_model_settings import set_rag_embedding_model
+
+ set_rag_embedding_model(
+ config.effective_embedding_model(), backend = "sentence-transformers", download_pending = True
+ )
+
+ def warm(model_name = None):
+ raise RuntimeError("CUDA out of memory")
+
+ monkeypatch.setattr(embeddings, "warm", warm)
+ assert embeddings.download_required() is False
+
+
+def test_download_required_error_text_is_recognized():
+ message = embeddings.download_required_message("unsloth/bge-small-en-v1.5")
+ assert message == (
+ "Embedding model 'unsloth/bge-small-en-v1.5' is not downloaded yet. "
+ "Finish its Settings download before indexing documents."
+ )
+ assert embeddings.is_download_required_error(f"RuntimeError: {message}")
+ assert not embeddings.is_download_required_error("Unsupported file type")
+ assert not embeddings.is_download_required_error(None)
+
+
+@requires_sqlite_vec
+def test_sync_waits_for_a_pending_download_instead_of_failing_every_file(
+ rag_home, stub_embeddings, pending_download, monkeypatch
+):
+ names = [f"note-{index:02d}.md" for index in range(12)]
+ _, folder = _folder(rag_home, names)
+ started = []
+ real_start = folder_sync.ingestion.start_ingestion
+
+ def start(*args, **kwargs):
+ started.append(args[3])
+ return real_start(*args, **kwargs)
+
+ monkeypatch.setattr(folder_sync.ingestion, "start_ingestion", start)
+
+ waiting = _run(folder["id"])
+
+ assert waiting["status"] == "failed"
+ assert waiting["failed"] == 0
+ assert "is not downloaded yet" in waiting["error"]
+ assert started == []
+ row = _folder_row(folder["id"])
+ assert row["status"] == "error"
+ assert "is not downloaded yet" in row["last_error"]
+ assert row["failed_files"] is None
+ assert _mapped(folder["id"]) == set()
+
+ # The periodic sync, not a Rebuild, indexes everything once the transfer lands.
+ pending_download["landed"] = True
+ synced = _run_periodic(folder["id"])
+
+ assert synced["status"] == "completed", synced
+ assert synced["added"] == len(names)
+ assert _mapped(folder["id"]) == set(names)
+ assert _folder_row(folder["id"])["failed_files"] is None
+
+
+@requires_sqlite_vec
+def test_rebuild_during_a_pending_download_keeps_the_index(
+ rag_home, stub_embeddings, pending_download
+):
+ pending_download["landed"] = True
+ _, folder = _folder(rag_home, ["a.md", "b.md"])
+ assert _run(folder["id"])["status"] == "completed"
+
+ from utils.embedding_model_settings import set_rag_embedding_model
+
+ set_rag_embedding_model(
+ config.effective_embedding_model(), backend = "sentence-transformers", download_pending = True
+ )
+ pending_download["landed"] = False
+
+ rebuild = _run(folder["id"], rebuild = True)
+
+ assert rebuild["status"] == "failed"
+ assert "is not downloaded yet" in rebuild["error"]
+ assert _mapped(folder["id"]) == {"a.md", "b.md"}
+
+
+@requires_sqlite_vec
+def test_a_download_required_failure_mid_sync_is_retried_by_the_next_sync(
+ rag_home, stub_embeddings, monkeypatch
+):
+ names = ["a.md", "b.md", "c.md"]
+ _, folder = _folder(rag_home, names)
+ real_start = folder_sync.ingestion.start_ingestion
+ model = config.effective_embedding_model()
+ calls = {"pending": True}
+
+ def start(*args, **kwargs):
+ if calls["pending"] and args[3] != "a.md":
+ # The transfer went pending after the sync's up-front check.
+ raise RuntimeError(embeddings.download_required_message(model))
+ if args[3] == "a.md" and calls["pending"]:
+ raise RuntimeError("Unsupported or corrupt file")
+ return real_start(*args, **kwargs)
+
+ monkeypatch.setattr(folder_sync.ingestion, "start_ingestion", start)
+
+ first = _run(folder["id"])
+
+ assert first["status"] == "failed"
+ assert first["failed"] == 3
+ held_back = json.loads(_folder_row(folder["id"])["failed_files"] or "{}")
+ # Only the genuinely bad file is held back from the next periodic sync.
+ assert set(held_back) == {"a.md"}
+
+ calls["pending"] = False
+ second = _run_periodic(folder["id"])
+
+ assert _mapped(folder["id"]) == {"b.md", "c.md"}
+ assert second["failed"] == 1
+
+
+@requires_sqlite_vec
+def test_failures_saved_before_the_fix_for_a_pending_download_retry(rag_home, stub_embeddings):
+ names = ["a.md", "b.md"]
+ _, folder = _folder(rag_home, names)
+ scan, _ = folder_sync._scan(folder["path"], None, {}, set())
+ message = embeddings.download_required_message(config.effective_embedding_model())
+ stored = {
+ rel: {
+ "sig": folder_sync._failure_signature(scan[rel]),
+ "at": time.time(),
+ "error": message,
+ }
+ for rel in names
+ }
+ with closing(rag_db.get_connection()) as conn:
+ conn.execute(
+ "UPDATE linked_folders SET failed_files=? WHERE id=?",
+ (json.dumps(stored), folder["id"]),
+ )
+ conn.commit()
+
+ result = _run_periodic(folder["id"])
+
+ assert result["status"] == "completed", result
+ assert _mapped(folder["id"]) == set(names)