Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio RAG: embed the documents of one ingest together instead of one call per document (#16727)
Debojyoti Banerjeea82cd339be12
diff --git a/studio/backend/core/rag/config.py b/studio/backend/core/rag/config.pyindex 9a8da07543e..86d5e72bfdb 100644--- a/studio/backend/core/rag/config.py+++ b/studio/backend/core/rag/config.py@@ -90,6 +90,11 @@ FOLDER_MAX_TEXT_BYTES = int(os.environ.get("RAG_FOLDER_MAX_TEXT_BYTES", str(20 *FOLDER_JOB_HISTORY_LIMIT = int(os.environ.get("RAG_FOLDER_JOB_HISTORY_LIMIT", "200"))# Linked-folder documents ingested concurrently, clamped to 1..4.FOLDER_INGEST_WORKERS = int(os.environ.get("RAG_FOLDER_INGEST_WORKERS", "2"))+# Small linked-folder files in flight, clamped to workers..64; past the workers they only wait to embed.+FOLDER_INGEST_IN_FLIGHT = int(os.environ.get("RAG_FOLDER_INGEST_IN_FLIGHT", "16"))+FOLDER_INGEST_BATCH_FILE_BYTES = int(+ os.environ.get("RAG_FOLDER_INGEST_BATCH_FILE_BYTES", str(1024 * 1024))+)# Falls back to plain PyMuPDF text when off, when pymupdf4llm is missing, or when extraction fails.PDF_MARKDOWN = os.environ.get("RAG_PDF_MARKDOWN", "1") == "1"diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.pyindex f0d4f85e957..558be0c507d 100644--- a/studio/backend/core/rag/embeddings.py+++ b/studio/backend/core/rag/embeddings.py@@ -1361,6 +1361,17 @@ def active_backend_is_llama(model_name: str | None = None) -> bool:return False+def batches_across_documents(model_name: str | None = None) -> bool:+ """Whether concurrent documents should share encode calls (#16715). Not for sentence-transformers on+ CPU: it pads each batch to its longest text, and on CPU that padding costs more than the calls saved."""+ if active_backend_is_llama(model_name):+ return True+ model = _model+ # A loaded model's device counts: a GPU choice can still have fallen back to CPU at load.+ device = getattr(model, "device", None) if model is not None else None+ return (str(device) if device is not None else _device()).split(":")[0] != "cpu"++def _llama_pooling(name: str, served = None) -> str | None:try:from .embed_llama_server import LlamaServerBackend, _gguf_poolingdiff --git a/studio/backend/core/rag/folder_sync.py b/studio/backend/core/rag/folder_sync.pyindex d36c049e544..43975a29d8c 100644--- a/studio/backend/core/rag/folder_sync.py+++ b/studio/backend/core/rag/folder_sync.py@@ -1549,6 +1549,7 @@ def _ingest_folder_document(content_hash: str,embedding_model: str,rebuild: bool,+ stage_slots: threading.Semaphore | None = None,) -> _IngestionResult:"""Thread-local, so the coordinator lease and stop state are copied in for _check_running."""_worker_state.job_id = job_id@@ -1573,6 +1574,7 @@ def _ingest_folder_document(linked_relative_path = relative_path,background = False,reuse_identical = not rebuild,+ stage_slots = stage_slots,)result = ingestion.get_job_status(ingestion_job)if result is None:@@ -1600,6 +1602,10 @@ def _folder_ingest_workers() -> int:return max(1, min(4, config.FOLDER_INGEST_WORKERS))+def _folder_ingest_in_flight(workers: int) -> int:+ return max(workers, min(64, config.FOLDER_INGEST_IN_FLIGHT))++def _reconcile_folder(job_id: str) -> None:"""Run one complete reconciliation; called serially by the coordinator."""_check_running()@@ -1713,6 +1719,12 @@ def _reconcile_folder(job_id: str) -> None:stop_event = getattr(_worker_state, "stop_event", _stop)account = current_account()workers = _folder_ingest_workers()+ in_flight_limit = (+ _folder_ingest_in_flight(workers)+ if ingestion.embeddings.batches_across_documents(embedding_model)+ else workers+ )+ stage_slots = threading.Semaphore(workers)in_flight: dict[Future["_IngestionResult"], _PendingIngestion] = {}def update_progress() -> None:@@ -1839,7 +1851,7 @@ def _reconcile_folder(job_id: str) -> None:prune_ingestion_job(result)with ThreadPoolExecutor(- max_workers = workers,+ max_workers = in_flight_limit,thread_name_prefix = "rag-folder-ingest",) as executor:try:@@ -1858,7 +1870,13 @@ def _reconcile_folder(job_id: str) -> None:cached_grace = Trueprocessed += 1continue- if len(in_flight) >= workers:+ # Only small files queue past the workers: each queued one keeps a snapshot.+ cap = (+ in_flight_limit+ if current[rel]["size_bytes"] <= config.FOLDER_INGEST_BATCH_FILE_BYTES+ else workers+ )+ while len(in_flight) >= cap:drain_one()snapshot = Nonetry:@@ -1909,6 +1927,7 @@ def _reconcile_folder(job_id: str) -> None:content_hash,embedding_model,rebuild,+ stage_slots,)in_flight[future] = _PendingIngestion(relative_path = rel,diff --git a/studio/backend/core/rag/ingestion.py b/studio/backend/core/rag/ingestion.pyindex a152203ad23..e73b205542c 100644--- a/studio/backend/core/rag/ingestion.py+++ b/studio/backend/core/rag/ingestion.py@@ -15,6 +15,7 @@ import queueimport reimport shutilimport threading+import timeimport uuidfrom collections.abc import Callable@@ -30,6 +31,8 @@ _workers: dict[str, threading.Thread] = {}_jobs_lock = threading.Lock()_EMBED_BATCH = 64+# Only waits while some document is still parsing.+_EMBED_LINGER_SECONDS = 0.2_SHA256_HEX_RE = re.compile(r"^[0-9a-f]{64}$")# Poll with a timeout so the generator notices a gone client or a worker that died without the None sentinel.@@ -140,6 +143,190 @@ def _abort_if_document_deleted(conn, job_id: str, document_id: str) -> bool:return True+class _EmbedRequest:+ def __init__(self, texts: list[str], model_name: str | None) -> None:+ self.texts = texts+ self.model_name = model_name+ # The encode runs as whichever caller took the batch, so batches never mix accounts or models.+ self.key = account_key(model_name or "")+ self.vectors: list = [None] * len(texts)+ self.scheduled = 0+ self.filled = 0+ self.identity: str | None = None+ self.changed = False+ self.error: BaseException | None = None++ @property+ def done(self) -> bool:+ return self.error is not None or self.filled == len(self.texts)+++class _EmbedQueue:+ """Encodes concurrent documents' chunks together in ``_EMBED_BATCH`` slices (#16715). No thread of+ its own: a waiting caller encodes the next batch for everyone. Chunking counts tokens under the+ encoder's lock, so a batch lingers briefly for documents still parsing (``expect``/``unexpect``)."""++ def __init__(self) -> None:+ self._cond = threading.Condition()+ self._pending: list[_EmbedRequest] = []+ # Keys with a batch in flight: one at a time per account and model.+ self._encoding: set = set()+ # Documents still parsing per key: only those can join that key's batch.+ self._expected: dict = {}++ def expect(self, model_name: str | None = None) -> None:+ key = account_key(model_name or "")+ with self._cond:+ self._expected[key] = self._expected.get(key, 0) + 1++ def unexpect(self, model_name: str | None = None) -> None:+ key = account_key(model_name or "")+ with self._cond:+ left = self._expected.get(key, 0) - 1+ if left > 0:+ self._expected[key] = left+ else:+ self._expected.pop(key, None)+ self._cond.notify_all()++ def _queued(self, key) -> int:+ return sum(+ len(request.texts) - request.scheduled+ for request in self._pending+ if request.key == key+ )++ def _linger(self, key) -> None:+ deadline = time.monotonic() + _EMBED_LINGER_SECONDS+ while self._expected.get(key, 0) > 0 and self._queued(key) < _EMBED_BATCH:+ left = deadline - time.monotonic()+ if left <= 0:+ return+ self._cond.wait(left)++ def _take(self, key) -> list[tuple[_EmbedRequest, int, int]]:+ # Shared out across requests, so a large document cannot hold back the others' progress and+ # cancel checks (both run in on_progress) for its whole pass.+ requests = [request for request in self._pending if request.key == key]+ room = max(1, _EMBED_BATCH)+ pieces = []+ while room and requests:+ share = max(1, room // len(requests))+ for request in list(requests):+ if not room:+ break+ start = request.scheduled+ end = min(len(request.texts), start + min(share, room))+ pieces.append((request, start, end))+ request.scheduled = end+ room -= end - start+ if end == len(request.texts):+ requests.remove(request)+ self._pending.remove(request)+ return pieces++ @staticmethod+ def _encode(pieces, model_name):+ texts = [text for request, start, end in pieces for text in request.texts[start:end]]+ return embeddings.encode_with_identity(texts, model_name = model_name, normalize = True)++ def _land(self, request: _EmbedRequest, start: int, end: int, vectors, identity) -> None:+ request.vectors[start:end] = list(vectors)+ request.filled += end - start+ request.changed = request.changed or (+ request.identity is not None and identity != request.identity+ )+ request.identity = identity++ def _fail(self, request: _EmbedRequest, exc: BaseException) -> None:+ request.error = exc+ if request in self._pending:+ self._pending.remove(request)++ def _run_batch(self, pieces) -> None:+ """A failed shared batch retries each document alone, so one bad document fails only itself."""+ model_name = pieces[0][0].model_name+ try:+ vectors, identity = self._encode(pieces, model_name)+ except Exception as exc: # noqa: BLE001 - delivered to the owning documents+ if len({id(request) for request, _, _ in pieces}) == 1:+ with self._cond:+ for request, _, _ in pieces:+ self._fail(request, exc)+ return+ for piece in pieces:+ request, start, end = piece+ try:+ vectors, identity = self._encode([piece], model_name)+ except Exception as piece_exc: # noqa: BLE001+ with self._cond:+ self._fail(request, piece_exc)+ else:+ with self._cond:+ self._land(request, start, end, vectors, identity)+ return+ offset = 0+ with self._cond:+ for request, start, end in pieces:+ width = end - start+ self._land(request, start, end, vectors[offset : offset + width], identity)+ offset += width++ def embed(+ self,+ texts: list[str],+ model_name: str | None,+ on_progress: Callable[[int, int], None] | None = None,+ ):+ """Returns ``(vectors, identity, changed)``. A caller with a full batch never lingers: it keeps+ its parse slot (see ``_run``), so waiting would stall the documents it waits for."""+ request = _EmbedRequest(texts, model_name)+ may_wait = len(texts) < _EMBED_BATCH+ reported = 0+ with self._cond:+ self._pending.append(request)+ self._cond.notify_all()+ try:+ while True:+ if on_progress is not None and request.filled > reported:+ reported = request.filled+ on_progress(reported, len(texts))+ with self._cond:+ if request.done:+ break+ if request.key in self._encoding or request not in self._pending:+ self._cond.wait()+ continue+ self._encoding.add(request.key)+ if may_wait:+ self._linger(request.key)+ pieces = self._take(request.key)+ try:+ self._run_batch(pieces)+ except BaseException as exc:+ # Never leave another document waiting on a slice that will not land.+ with self._cond:+ for owner, _, _ in pieces:+ if not owner.done:+ self._fail(owner, exc)+ raise+ finally:+ with self._cond:+ self._encoding.discard(request.key)+ self._cond.notify_all()+ finally:+ with self._cond:+ if request in self._pending:+ self._pending.remove(request)+ self._cond.notify_all()+ if request.error is not None:+ raise request.error+ return request.vectors, request.identity, request.changed+++_embed_queue = _EmbedQueue()++def _embed_pass(texts: list[str],model_name: str | None,@@ -147,20 +334,24 @@ def _embed_pass():"""One batched pass. Returns ``(vectors, identity, changed)``, ``changed`` when theembedder swapped part way and the vectors therefore span two spaces."""+ if not texts:+ return [], embeddings.embedding_identity(model_name), False+ if embeddings.batches_across_documents(model_name):+ vectors, identity, changed = _embed_queue.embed(texts, model_name, on_progress)+ return vectors, identity or embeddings.embedding_identity(model_name), changedvectors: list = []identity: str | None = Nonechanged = Falsefor i in range(0, len(texts), _EMBED_BATCH):- batch = texts[i : i + _EMBED_BATCH]out, batch_identity = embeddings.encode_with_identity(- batch, model_name = model_name, normalize = True+ texts[i : i + _EMBED_BATCH], model_name = model_name, normalize = True)changed = changed or (identity is not None and batch_identity != identity)identity = batch_identityvectors.extend(out)if on_progress is not None:on_progress(min(i + _EMBED_BATCH, len(texts)), len(texts))- return vectors, identity or embeddings.embedding_identity(model_name), changed+ return vectors, identity, changeddef _embed_all(@@ -283,6 +474,36 @@ def _retire_orphan_after_failure(logger.warning("failed to retire orphaned document %s", old_id, exc_info = True)+class _ExpectedEmbed:+ def __init__(self, model_name: str | None) -> None:+ self._model_name = model_name+ _embed_queue.expect(model_name)+ self._pending = True++ def arrive(self) -> None:+ if self._pending:+ self._pending = False+ _embed_queue.unexpect(self._model_name)+++class _StageSlot:+ """One of ``stage_slots``, held around parse and store but not the embed wait."""++ def __init__(self, slots: threading.Semaphore | None) -> None:+ self._slots = slots+ self._held = False++ def acquire(self) -> None:+ if self._slots is not None and not self._held:+ self._slots.acquire()+ self._held = True++ def release(self) -> None:+ if self._held:+ self._held = False+ self._slots.release()++def _run(job_id: str,document_id: str,@@ -292,9 +513,13 @@ def _run(ocr: bool | None = None,caption: bool | None = None,replaces: tuple[str, str | None] | None = None,+ stage_slots: threading.Semaphore | None = None,) -> None:conn = None+ slot = _StageSlot(stage_slots)+ expected = _ExpectedEmbed(model_name)try:+ slot.acquire()conn = rag_db.get_connection()_progress(conn, job_id, "parsing", 0.1)pages = parsers.parse(stored_path)@@ -394,7 +619,12 @@ def _run(embedded_progress = max(embedded_progress, 0.65 + 0.25 * done / total)_progress(conn, job_id, "embedding", embedded_progress)+ # Below a full batch, give the slot up so other documents can parse and join the batch.+ if len(chunks) < _EMBED_BATCH and embeddings.batches_across_documents(model_name):+ slot.release()+ expected.arrive()vectors, identity = _embed_all([c.text for c in chunks], model_name, report_embeddings)+ slot.acquire()store.set_document_embedding_model(conn, document_id, identity)# Locate each chunk's highlight regions (non-PDFs/failures yield none).@@ -432,6 +662,8 @@ def _run(logger.exception("failed to record ingestion failure for job %s", job_id)_emit(job_id, {"type": "error", "stage": "error", "error": str(exc)})finally:+ expected.arrive()+ slot.release()if conn is not None:# Every exit but completion retires the orphan; nothing relaunches, so a lost lease ends the work._retire_orphan_after_failure(conn, replaces, stored_path, document_id)@@ -459,8 +691,10 @@ def start_ingestion(background: bool = True,content_hash: str | None = None,reuse_identical: bool = False,+ stage_slots: threading.Semaphore | None = None,) -> tuple[str, str]:- """Duplicate hash reuses the existing document; a passed content_hash is trusted, never reverified."""+ """Duplicate hash reuses the existing document; a passed content_hash is trusted, never reverified.+ ``stage_slots`` bounds concurrent parse/store while any number wait to embed together."""account_path(stored_path)if account_is_retired():raise RuntimeError("Account is retired")@@ -623,6 +857,7 @@ def start_ingestion(ocr,caption,replaces,+ stage_slots,)if not background:_run(*args)diff --git a/studio/backend/tests/test_rag_linked_folders.py b/studio/backend/tests/test_rag_linked_folders.pyindex 64598b443d8..12c722b3223 100644--- a/studio/backend/tests/test_rag_linked_folders.py+++ b/studio/backend/tests/test_rag_linked_folders.py@@ -3321,3 +3321,80 @@ def test_an_unlink_cancel_only_reaches_its_own_account():assert not run_as(other, folder_sync.is_cancel_requested, "shared-id")finally:folder_sync.clear_cancel("shared-id")+++@requires_sqlite_vec+def test_a_folder_of_small_files_embeds_across_documents(rag_home, stub_embeddings, monkeypatch):+ """A vault of one-chunk notes paid one encode call per file before (#16715)."""+ monkeypatch.setattr(embeddings, "batches_across_documents", lambda model_name = None: True)+ source, folder = _folder(rag_home)+ for i in range(12):+ (source / f"note-{i:02d}.md").write_text(f"note {i} searchable words", encoding = "utf-8")+ monkeypatch.setattr(folder_sync.config, "FOLDER_INGEST_WORKERS", 2)+ batches = []+ real_encode = embeddings.encode+ lock = threading.Lock()++ def recording(texts, **kwargs):+ with lock:+ batches.append(len(texts))+ # An instant stub finishes each file before the coordinator snapshots the next one.+ time.sleep(0.5)+ return real_encode(texts, **kwargs)++ monkeypatch.setattr(embeddings, "encode", recording)++ assert _run(folder["id"])["status"] == "completed"++ assert sum(batches) == 12, batches+ assert len(batches) < 12, batches+ assert max(batches) > 1, batches+ with _connection() as conn:+ assert (+ conn.execute(+ "SELECT COUNT(*) AS count FROM linked_folder_files WHERE folder_id=?",+ (folder["id"],),+ ).fetchone()["count"]+ == 12+ )+ for i in range(12):+ assert store.search_lexical(conn, folder["scope"], f"note {i}", 5)+++@requires_sqlite_vec+def test_large_files_stay_within_the_parse_workers(rag_home, stub_embeddings, monkeypatch):+ """Only small files queue past the workers; big ones would hold snapshots and chunks."""+ monkeypatch.setattr(embeddings, "batches_across_documents", lambda model_name = None: True)+ source, folder = _folder(rag_home)+ for i in range(8):+ (source / f"doc-{i}.txt").write_text(f"document {i} words", encoding = "utf-8")+ monkeypatch.setattr(folder_sync.config, "FOLDER_INGEST_WORKERS", 2)+ # Every file in this folder counts as large.+ monkeypatch.setattr(folder_sync.config, "FOLDER_INGEST_BATCH_FILE_BYTES", 1)+ in_flight = {"now": 0, "peak": 0}+ lock = threading.Lock()+ real_start = ingestion.start_ingestion++ def watched(*args, **kwargs):+ with lock:+ in_flight["now"] += 1+ in_flight["peak"] = max(in_flight["peak"], in_flight["now"])+ try:+ return real_start(*args, **kwargs)+ finally:+ with lock:+ in_flight["now"] -= 1++ monkeypatch.setattr(folder_sync.ingestion, "start_ingestion", watched)++ assert _run(folder["id"])["status"] == "completed"++ assert in_flight["peak"] <= 2, in_flight+ with _connection() as conn:+ assert (+ conn.execute(+ "SELECT COUNT(*) AS count FROM linked_folder_files WHERE folder_id=?",+ (folder["id"],),+ ).fetchone()["count"]+ == 8+ )diff --git a/studio/backend/tests/test_rag_shared_embed_batch.py b/studio/backend/tests/test_rag_shared_embed_batch.pynew file mode 100644index 00000000000..7f0922f5c0b--- /dev/null+++ b/studio/backend/tests/test_rag_shared_embed_batch.py@@ -0,0 +1,466 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Documents ingesting at the same time embed in shared batches (#16715), each keeping its own+embedding identity, progress and failures.++The stub embedder returns instantly, so a document would normally finish before the next one is+parsed. These tests hold the queue's "a document is on its way" count, which is what a real sync+has while other files parse, and raise the linger so the wait is not a race.+"""++import struct+import threading+import time+from contextlib import contextmanager++import pytest++from core.rag import config, embeddings, ingestion, store+from storage import rag_db++requires_sqlite_vec = pytest.mark.skipif(+ not rag_db.RAG_AVAILABLE, reason = "sqlite-vec is not installed"+)+++_real_gate = embeddings.batches_across_documents++[email protected](autouse = True)+def _embedder_on_an_accelerator(monkeypatch):+ # The stub embedder resolves to sentence-transformers on CPU, which keeps per-document calls.+ monkeypatch.setattr(embeddings, "batches_across_documents", lambda model_name = None: True)+++def _write(tmp_path, name, text):+ path = tmp_path / name+ path.write_text(text, encoding = "utf-8")+ return str(path)+++def _wait_finished(job_id, timeout = 60.0):+ deadline = time.time() + timeout+ while time.time() < deadline:+ status = ingestion.get_job_status(job_id)+ if status and status["status"] in ("completed", "failed", "cancelled"):+ return status+ time.sleep(0.05)+ raise AssertionError("ingestion did not finish in time")+++@contextmanager+def _documents_still_parsing(monkeypatch):+ """Stands in for files a sync is still parsing, so batches wait for them."""+ monkeypatch.setattr(ingestion, "_EMBED_LINGER_SECONDS", 30.0)+ model = config.effective_embedding_model()+ ingestion._embed_queue.expect(model)+ try:+ yield+ finally:+ ingestion._embed_queue.unexpect(model)+++def _record_batches(monkeypatch):+ """Returns the list of text batches the embedder is actually called with."""+ batches: list[list[str]] = []+ real = embeddings.encode_with_identity+ lock = threading.Lock()++ def recording(texts, **kwargs):+ with lock:+ batches.append(list(texts))+ return real(texts, **kwargs)++ monkeypatch.setattr(embeddings, "encode_with_identity", recording)+ return batches+++def _record_stages(monkeypatch):+ """Returns (job_id, stage, progress) as the job rows are written; SSE queues are reaped."""+ stages: list[tuple[str, str, float]] = []+ real = ingestion._progress+ lock = threading.Lock()++ def recording(conn, job_id, stage, progress):+ with lock:+ stages.append((job_id, stage, progress))+ return real(conn, job_id, stage, progress)++ monkeypatch.setattr(ingestion, "_progress", recording)+ return stages+++def _start(+ scope,+ tmp_path,+ name,+ text,+ slots = None,+ model_name = None,+):+ return ingestion.start_ingestion(+ scope,+ "K1",+ None,+ name,+ _write(tmp_path, name, text),+ model_name = model_name,+ stage_slots = slots,+ )+++def test_two_documents_embed_in_one_call(rag_home, stub_embeddings, tmp_path, monkeypatch):+ """The batch-of-1 path: before this, a one-chunk file always paid a whole encode call."""+ batches = _record_batches(monkeypatch)+ scope = store.kb_scope("K1")+ # One slot, like a linked-folder sync with a single parse worker.+ slots = threading.Semaphore(1)+ with _documents_still_parsing(monkeypatch):+ jobs = [+ _start(scope, tmp_path, "a.txt", "alpha document words", slots),+ _start(scope, tmp_path, "b.txt", "bravo document words", slots),+ ]+ for _, job_id in jobs:+ assert _wait_finished(job_id)["status"] == "completed"++ assert len(batches) == 1, batches+ assert sorted(text.split()[0] for text in batches[0]) == ["alpha", "bravo"]+ conn = rag_db.get_connection()+ try:+ for document_id, _ in jobs:+ assert store.get_document(conn, document_id)["num_chunks"] == 1+ assert store.search_lexical(conn, scope, "alpha", 5)+ assert store.search_lexical(conn, scope, "bravo", 5)+ finally:+ conn.close()+++@requires_sqlite_vec+def test_a_shared_batch_keeps_each_document_s_own_vectors(+ rag_home, stub_embeddings, tmp_path, monkeypatch+):+ """Vectors must come back on the chunk they were made from, not in the batch's order."""+ scope = store.kb_scope("K1")+ slots = threading.Semaphore(1)+ with _documents_still_parsing(monkeypatch):+ jobs = {+ name: _start(scope, tmp_path, f"{name}.txt", " ".join([name] * 40), slots)+ for name in ("alpha", "bravo", "charlie")+ }+ for _, job_id in jobs.values():+ assert _wait_finished(job_id)["status"] == "completed"++ conn = rag_db.get_connection()+ try:+ for name, (document_id, _) in jobs.items():+ rows = conn.execute(+ "SELECT id, text FROM chunks WHERE document_id=? ORDER BY chunk_index",+ (document_id,),+ ).fetchall()+ assert rows and all(name in row["text"] for row in rows)+ for row in rows:+ stored = conn.execute(+ "SELECT embedding FROM chunks_vec WHERE chunk_id=?", (row["id"],)+ ).fetchone()+ assert stored is not None+ expected = embeddings.encode([row["text"]])[0]+ written = struct.unpack(f"<{len(expected)}f", stored["embedding"])+ assert written == pytest.approx(tuple(expected), abs = 1e-6)+ finally:+ conn.close()+++def test_one_failing_document_does_not_fail_the_others(+ rag_home, stub_embeddings, tmp_path, monkeypatch+):+ """A failed shared batch retries per document, so a text the embedder rejects fails alone."""+ real = embeddings.encode_with_identity++ def refuse_bravo(texts, **kwargs):+ if any("bravo" in text for text in texts):+ raise RuntimeError("synthetic encode failure")+ return real(texts, **kwargs)++ monkeypatch.setattr(embeddings, "encode_with_identity", refuse_bravo)+ scope = store.kb_scope("K1")+ slots = threading.Semaphore(1)+ with _documents_still_parsing(monkeypatch):+ good = _start(scope, tmp_path, "a.txt", "alpha document words", slots)+ bad = _start(scope, tmp_path, "b.txt", "bravo document words", slots)++ assert _wait_finished(good[1])["status"] == "completed"+ assert _wait_finished(bad[1])["status"] == "failed"+ conn = rag_db.get_connection()+ try:+ assert store.get_document(conn, good[0])["status"] == "completed"+ assert store.get_document(conn, bad[0])["status"] == "failed"+ assert store.search_lexical(conn, scope, "alpha", 5)+ assert not store.search_lexical(conn, scope, "bravo", 5)+ finally:+ conn.close()+++def test_documents_under_different_models_are_not_batched_together(+ rag_home, stub_embeddings, tmp_path, monkeypatch+):+ """One batch is one encode call, so it can only hold texts for one model."""+ batches = _record_batches(monkeypatch)+ models = []+ real_encode = embeddings.encode++ def note_model(+ texts,+ *,+ model_name = None,+ normalize = True,+ ):+ models.append(model_name)+ return real_encode(texts, model_name = model_name, normalize = normalize)++ monkeypatch.setattr(embeddings, "encode", note_model)+ scope = store.kb_scope("K1")+ slots = threading.Semaphore(1)+ with _documents_still_parsing(monkeypatch):+ jobs = [+ _start(scope, tmp_path, name, f"{name} document words", slots, model_name = model)+ for name, model in (("a.txt", "model/one"), ("b.txt", "model/two"))+ ]+ for _, job_id in jobs:+ assert _wait_finished(job_id)["status"] == "completed"++ assert len(batches) == 2, batches+ assert all(len(batch) == 1 for batch in batches)+ assert sorted(models) == ["model/one", "model/two"]+++def test_a_document_with_a_full_batch_of_chunks_embeds_without_waiting(+ rag_home, stub_embeddings, tmp_path, monkeypatch+):+ """It gains nothing from waiting, and holding its chunks for others would cost memory."""+ from core.rag import config++ monkeypatch.setattr(ingestion, "_EMBED_BATCH", 4)+ monkeypatch.setattr(config, "CHUNK_TOKENS", 3)+ monkeypatch.setattr(config, "CHUNK_OVERLAP", 0)+ batches = _record_batches(monkeypatch)+ scope = store.kb_scope("K1")+ slots = threading.Semaphore(1)+ started = time.monotonic()+ with _documents_still_parsing(monkeypatch):+ document_id, job_id = _start(+ scope,+ tmp_path,+ "big.txt",+ " ".join(f"word{i}" for i in range(40)),+ slots,+ )+ assert _wait_finished(job_id, timeout = 20.0)["status"] == "completed"+ # It never waited out the 30 s linger for the documents still parsing.+ assert time.monotonic() - started < 20.0++ assert batches and all(len(batch) <= 4 for batch in batches)+ conn = rag_db.get_connection()+ try:+ assert store.get_document(conn, document_id)["num_chunks"] > 4+ finally:+ conn.close()+++def test_a_waiting_document_reports_its_embedding_progress(+ rag_home, stub_embeddings, tmp_path, monkeypatch+):+ """Progress must move when its slice lands in another document's batch."""+ stages = _record_stages(monkeypatch)+ scope = store.kb_scope("K1")+ slots = threading.Semaphore(1)+ with _documents_still_parsing(monkeypatch):+ jobs = [+ _start(scope, tmp_path, f"doc{i}.txt", f"document {i} words", slots) for i in range(3)+ ]+ for _, job_id in jobs:+ assert _wait_finished(job_id)["status"] == "completed"++ for _, job_id in jobs:+ written = [(stage, progress) for job, stage, progress in stages if job == job_id]+ assert "embedding" in [stage for stage, _ in written]+ assert [progress for _, progress in written] == sorted(progress for _, progress in written)+++def test_a_backend_swap_inside_a_shared_batch_re_embeds_that_document(+ rag_home, stub_embeddings, tmp_path, monkeypatch+):+ """Vectors from two spaces under one identity are unsearchable, so the document re-embeds+ whole under the backend that took over."""+ from core.rag import config++ monkeypatch.setattr(embeddings, "active_backend_is_llama", lambda *_a, **_k: False)+ monkeypatch.setattr(ingestion, "_EMBED_BATCH", 2)+ monkeypatch.setattr(config, "CHUNK_TOKENS", 3)+ monkeypatch.setattr(config, "CHUNK_OVERLAP", 0)+ real = embeddings.encode_with_identity+ swapped = {"done": False}++ def swap_after_the_first_batch(texts, **kwargs):+ out = real(texts, **kwargs)+ if not swapped["done"]:+ swapped["done"] = True+ monkeypatch.setattr(embeddings, "active_backend_is_llama", lambda *_a, **_k: True)+ return out++ monkeypatch.setattr(embeddings, "encode_with_identity", swap_after_the_first_batch)+ scope = store.kb_scope("K1")+ document_id, job_id = _start(+ scope,+ tmp_path,+ "doc.txt",+ " ".join(f"word{i}" for i in range(12)),+ )+ assert _wait_finished(job_id)["status"] == "completed"++ conn = rag_db.get_connection()+ try:+ identity = store.get_document(conn, document_id)["embedding_model"]+ finally:+ conn.close()+ assert identity.startswith("llama-server:"), identity+++def test_the_queue_keeps_nothing_after_a_failed_document(rag_home, stub_embeddings, tmp_path):+ """A document that stops early must not leave a slice others would wait on."""+ scope = store.kb_scope("K1")+ empty = _start(scope, tmp_path, "empty.txt", " ")+ assert _wait_finished(empty[1])["status"] == "failed"+ good = _start(scope, tmp_path, "good.txt", "alpha document words")+ assert _wait_finished(good[1])["status"] == "completed"++ assert not ingestion._embed_queue._pending+ assert not ingestion._embed_queue._expected++[email protected]("documents", (2, 9))+def test_many_small_documents_need_far_fewer_encode_calls(+ rag_home, stub_embeddings, tmp_path, monkeypatch, documents+):+ batches = _record_batches(monkeypatch)+ scope = store.kb_scope("K1")+ slots = threading.Semaphore(2)+ with _documents_still_parsing(monkeypatch):+ jobs = [+ _start(scope, tmp_path, f"note{i}.txt", f"note {i} searchable words", slots)+ for i in range(documents)+ ]+ for _, job_id in jobs:+ assert _wait_finished(job_id)["status"] == "completed"++ assert sum(len(batch) for batch in batches) == documents+ assert len(batches) == 1, batches+++def test_cpu_sentence_transformers_keeps_one_call_per_document(+ rag_home, stub_embeddings, tmp_path, monkeypatch+):+ """CPU encode pads every text to the batch's longest, which costs more than the calls it saves."""+ monkeypatch.setattr(embeddings, "batches_across_documents", _real_gate)+ assert not embeddings.batches_across_documents(None)+ batches = _record_batches(monkeypatch)+ scope = store.kb_scope("K1")+ slots = threading.Semaphore(2)+ with _documents_still_parsing(monkeypatch):+ jobs = [+ _start(scope, tmp_path, f"note{i}.txt", f"note {i} searchable words", slots)+ for i in range(4)+ ]+ for _, job_id in jobs:+ assert _wait_finished(job_id)["status"] == "completed"++ assert sorted(len(batch) for batch in batches) == [1, 1, 1, 1], batches+ assert not ingestion._embed_queue._pending+++def test_two_accounts_never_share_a_batch(monkeypatch):+ """The encode runs as whichever caller took the batch, so it may only hold that account's+ texts: its token, model path and device choice are the account's."""+ from utils.account_context import AccountContext, run_as++ queue = ingestion._EmbedQueue()+ monkeypatch.setattr(ingestion, "_EMBED_LINGER_SECONDS", 30.0)+ seen: list[list[str]] = []+ lock = threading.Lock()++ def fake_encode(texts, **kwargs):+ with lock:+ seen.append(list(texts))+ return [[float(len(text))] for text in texts], "identity"++ monkeypatch.setattr(embeddings, "encode_with_identity", fake_encode)+ results: dict[str, object] = {}+ arrived = threading.Barrier(3)+ queue.expect()++ def embed_as(account):+ def go():+ arrived.wait(30)+ results[account.account_id] = queue.embed([f"{account.account_id} text"], None)++ run_as(account, go)++ threads = [+ threading.Thread(target = embed_as, args = (AccountContext(f"acct-{i}", f"user{i}"),))+ for i in (1, 2)+ ]+ for thread in threads:+ thread.start()+ arrived.wait(30)+ # Both are in the queue now; releasing the expected document lets each batch leave.+ queue.unexpect()+ for thread in threads:+ thread.join(30)+ assert not thread.is_alive()++ assert sorted(len(batch) for batch in seen) == [1, 1], seen+ assert sorted(batch[0] for batch in seen) == ["acct-1 text", "acct-2 text"]+ assert set(results) == {"acct-1", "acct-2"}+++def test_a_large_document_does_not_hold_back_a_small_one(monkeypatch):+ """The small one's progress callback, and with it the unlink check, must run within a batch."""+ monkeypatch.setattr(ingestion, "_EMBED_BATCH", 4)+ queue = ingestion._EmbedQueue()+ big = ingestion._EmbedRequest([f"big {i}" for i in range(40)], None)+ small = ingestion._EmbedRequest(["small"], None)+ queue._pending += [big, small]++ pieces = queue._take(big.key)++ assert (small, 0, 1) in pieces+ assert sum(end - start for _, start, end in pieces) == 4+ assert queue._pending == [big]+++def test_a_parse_under_another_model_does_not_hold_a_batch(monkeypatch):+ monkeypatch.setattr(ingestion, "_EMBED_LINGER_SECONDS", 30.0)+ monkeypatch.setattr(+ embeddings, "encode_with_identity", lambda texts, **kwargs: ([[1.0]] * len(texts), "id")+ )+ queue = ingestion._EmbedQueue()+ queue.expect("other-model")+ started = time.monotonic()+ vectors, _, _ = queue.embed(["a note"], None)+ assert vectors == [[1.0]]+ assert time.monotonic() - started < 5+ queue.unexpect("other-model")+ assert not queue._expected+++def test_a_gpu_choice_that_loaded_on_cpu_keeps_per_document_calls(monkeypatch):+ from types import SimpleNamespace++ monkeypatch.setattr(embeddings, "active_backend_is_llama", lambda model_name = None: False)+ monkeypatch.setattr(embeddings, "_device", lambda: "cuda")+ monkeypatch.setattr(embeddings, "_model", None)+ assert _real_gate(None)+ monkeypatch.setattr(embeddings, "_model", SimpleNamespace(device = "cpu"))+ assert not _real_gate(None)+ monkeypatch.setattr(embeddings, "_model", SimpleNamespace(device = "cuda:0"))+ assert _real_gate(None)