Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio RAG: embed the documents of one ingest together instead of one call per document (#16727)

Debojyoti Banerjeea82cd339be12
diff --git a/studio/backend/core/rag/config.py b/studio/backend/core/rag/config.py
index 9a8da07543e..86d5e72bfdb 100644
--- a/studio/backend/core/rag/config.py
+++ b/studio/backend/core/rag/config.py
@@ -90,6 +90,11 @@ FOLDER_MAX_TEXT_BYTES = int(os.environ.get("RAG_FOLDER_MAX_TEXT_BYTES", str(20 *
FOLDER_JOB_HISTORY_LIMIT = int(os.environ.get("RAG_FOLDER_JOB_HISTORY_LIMIT", "200"))
# Linked-folder documents ingested concurrently, clamped to 1..4.
FOLDER_INGEST_WORKERS = int(os.environ.get("RAG_FOLDER_INGEST_WORKERS", "2"))
+# Small linked-folder files in flight, clamped to workers..64; past the workers they only wait to embed.
+FOLDER_INGEST_IN_FLIGHT = int(os.environ.get("RAG_FOLDER_INGEST_IN_FLIGHT", "16"))
+FOLDER_INGEST_BATCH_FILE_BYTES = int(
+ os.environ.get("RAG_FOLDER_INGEST_BATCH_FILE_BYTES", str(1024 * 1024))
+)
# Falls back to plain PyMuPDF text when off, when pymupdf4llm is missing, or when extraction fails.
PDF_MARKDOWN = os.environ.get("RAG_PDF_MARKDOWN", "1") == "1"
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py
index f0d4f85e957..558be0c507d 100644
--- a/studio/backend/core/rag/embeddings.py
+++ b/studio/backend/core/rag/embeddings.py
@@ -1361,6 +1361,17 @@ def active_backend_is_llama(model_name: str | None = None) -> bool:
return False
+def batches_across_documents(model_name: str | None = None) -> bool:
+ """Whether concurrent documents should share encode calls (#16715). Not for sentence-transformers on
+ CPU: it pads each batch to its longest text, and on CPU that padding costs more than the calls saved."""
+ if active_backend_is_llama(model_name):
+ return True
+ model = _model
+ # A loaded model's device counts: a GPU choice can still have fallen back to CPU at load.
+ device = getattr(model, "device", None) if model is not None else None
+ return (str(device) if device is not None else _device()).split(":")[0] != "cpu"
+
+
def _llama_pooling(name: str, served = None) -> str | None:
try:
from .embed_llama_server import LlamaServerBackend, _gguf_pooling
diff --git a/studio/backend/core/rag/folder_sync.py b/studio/backend/core/rag/folder_sync.py
index d36c049e544..43975a29d8c 100644
--- a/studio/backend/core/rag/folder_sync.py
+++ b/studio/backend/core/rag/folder_sync.py
@@ -1549,6 +1549,7 @@ def _ingest_folder_document(
content_hash: str,
embedding_model: str,
rebuild: bool,
+ stage_slots: threading.Semaphore | None = None,
) -> _IngestionResult:
"""Thread-local, so the coordinator lease and stop state are copied in for _check_running."""
_worker_state.job_id = job_id
@@ -1573,6 +1574,7 @@ def _ingest_folder_document(
linked_relative_path = relative_path,
background = False,
reuse_identical = not rebuild,
+ stage_slots = stage_slots,
)
result = ingestion.get_job_status(ingestion_job)
if result is None:
@@ -1600,6 +1602,10 @@ def _folder_ingest_workers() -> int:
return max(1, min(4, config.FOLDER_INGEST_WORKERS))
+def _folder_ingest_in_flight(workers: int) -> int:
+ return max(workers, min(64, config.FOLDER_INGEST_IN_FLIGHT))
+
+
def _reconcile_folder(job_id: str) -> None:
"""Run one complete reconciliation; called serially by the coordinator."""
_check_running()
@@ -1713,6 +1719,12 @@ def _reconcile_folder(job_id: str) -> None:
stop_event = getattr(_worker_state, "stop_event", _stop)
account = current_account()
workers = _folder_ingest_workers()
+ in_flight_limit = (
+ _folder_ingest_in_flight(workers)
+ if ingestion.embeddings.batches_across_documents(embedding_model)
+ else workers
+ )
+ stage_slots = threading.Semaphore(workers)
in_flight: dict[Future["_IngestionResult"], _PendingIngestion] = {}
def update_progress() -> None:
@@ -1839,7 +1851,7 @@ def _reconcile_folder(job_id: str) -> None:
prune_ingestion_job(result)
with ThreadPoolExecutor(
- max_workers = workers,
+ max_workers = in_flight_limit,
thread_name_prefix = "rag-folder-ingest",
) as executor:
try:
@@ -1858,7 +1870,13 @@ def _reconcile_folder(job_id: str) -> None:
cached_grace = True
processed += 1
continue
- if len(in_flight) >= workers:
+ # Only small files queue past the workers: each queued one keeps a snapshot.
+ cap = (
+ in_flight_limit
+ if current[rel]["size_bytes"] <= config.FOLDER_INGEST_BATCH_FILE_BYTES
+ else workers
+ )
+ while len(in_flight) >= cap:
drain_one()
snapshot = None
try:
@@ -1909,6 +1927,7 @@ def _reconcile_folder(job_id: str) -> None:
content_hash,
embedding_model,
rebuild,
+ stage_slots,
)
in_flight[future] = _PendingIngestion(
relative_path = rel,
diff --git a/studio/backend/core/rag/ingestion.py b/studio/backend/core/rag/ingestion.py
index a152203ad23..e73b205542c 100644
--- a/studio/backend/core/rag/ingestion.py
+++ b/studio/backend/core/rag/ingestion.py
@@ -15,6 +15,7 @@ import queue
import re
import shutil
import threading
+import time
import uuid
from collections.abc import Callable
@@ -30,6 +31,8 @@ _workers: dict[str, threading.Thread] = {}
_jobs_lock = threading.Lock()
_EMBED_BATCH = 64
+# Only waits while some document is still parsing.
+_EMBED_LINGER_SECONDS = 0.2
_SHA256_HEX_RE = re.compile(r"^[0-9a-f]{64}$")
# Poll with a timeout so the generator notices a gone client or a worker that died without the None sentinel.
@@ -140,6 +143,190 @@ def _abort_if_document_deleted(conn, job_id: str, document_id: str) -> bool:
return True
+class _EmbedRequest:
+ def __init__(self, texts: list[str], model_name: str | None) -> None:
+ self.texts = texts
+ self.model_name = model_name
+ # The encode runs as whichever caller took the batch, so batches never mix accounts or models.
+ self.key = account_key(model_name or "")
+ self.vectors: list = [None] * len(texts)
+ self.scheduled = 0
+ self.filled = 0
+ self.identity: str | None = None
+ self.changed = False
+ self.error: BaseException | None = None
+
+ @property
+ def done(self) -> bool:
+ return self.error is not None or self.filled == len(self.texts)
+
+
+class _EmbedQueue:
+ """Encodes concurrent documents' chunks together in ``_EMBED_BATCH`` slices (#16715). No thread of
+ its own: a waiting caller encodes the next batch for everyone. Chunking counts tokens under the
+ encoder's lock, so a batch lingers briefly for documents still parsing (``expect``/``unexpect``)."""
+
+ def __init__(self) -> None:
+ self._cond = threading.Condition()
+ self._pending: list[_EmbedRequest] = []
+ # Keys with a batch in flight: one at a time per account and model.
+ self._encoding: set = set()
+ # Documents still parsing per key: only those can join that key's batch.
+ self._expected: dict = {}
+
+ def expect(self, model_name: str | None = None) -> None:
+ key = account_key(model_name or "")
+ with self._cond:
+ self._expected[key] = self._expected.get(key, 0) + 1
+
+ def unexpect(self, model_name: str | None = None) -> None:
+ key = account_key(model_name or "")
+ with self._cond:
+ left = self._expected.get(key, 0) - 1
+ if left > 0:
+ self._expected[key] = left
+ else:
+ self._expected.pop(key, None)
+ self._cond.notify_all()
+
+ def _queued(self, key) -> int:
+ return sum(
+ len(request.texts) - request.scheduled
+ for request in self._pending
+ if request.key == key
+ )
+
+ def _linger(self, key) -> None:
+ deadline = time.monotonic() + _EMBED_LINGER_SECONDS
+ while self._expected.get(key, 0) > 0 and self._queued(key) < _EMBED_BATCH:
+ left = deadline - time.monotonic()
+ if left <= 0:
+ return
+ self._cond.wait(left)
+
+ def _take(self, key) -> list[tuple[_EmbedRequest, int, int]]:
+ # Shared out across requests, so a large document cannot hold back the others' progress and
+ # cancel checks (both run in on_progress) for its whole pass.
+ requests = [request for request in self._pending if request.key == key]
+ room = max(1, _EMBED_BATCH)
+ pieces = []
+ while room and requests:
+ share = max(1, room // len(requests))
+ for request in list(requests):
+ if not room:
+ break
+ start = request.scheduled
+ end = min(len(request.texts), start + min(share, room))
+ pieces.append((request, start, end))
+ request.scheduled = end
+ room -= end - start
+ if end == len(request.texts):
+ requests.remove(request)
+ self._pending.remove(request)
+ return pieces
+
+ @staticmethod
+ def _encode(pieces, model_name):
+ texts = [text for request, start, end in pieces for text in request.texts[start:end]]
+ return embeddings.encode_with_identity(texts, model_name = model_name, normalize = True)
+
+ def _land(self, request: _EmbedRequest, start: int, end: int, vectors, identity) -> None:
+ request.vectors[start:end] = list(vectors)
+ request.filled += end - start
+ request.changed = request.changed or (
+ request.identity is not None and identity != request.identity
+ )
+ request.identity = identity
+
+ def _fail(self, request: _EmbedRequest, exc: BaseException) -> None:
+ request.error = exc
+ if request in self._pending:
+ self._pending.remove(request)
+
+ def _run_batch(self, pieces) -> None:
+ """A failed shared batch retries each document alone, so one bad document fails only itself."""
+ model_name = pieces[0][0].model_name
+ try:
+ vectors, identity = self._encode(pieces, model_name)
+ except Exception as exc: # noqa: BLE001 - delivered to the owning documents
+ if len({id(request) for request, _, _ in pieces}) == 1:
+ with self._cond:
+ for request, _, _ in pieces:
+ self._fail(request, exc)
+ return
+ for piece in pieces:
+ request, start, end = piece
+ try:
+ vectors, identity = self._encode([piece], model_name)
+ except Exception as piece_exc: # noqa: BLE001
+ with self._cond:
+ self._fail(request, piece_exc)
+ else:
+ with self._cond:
+ self._land(request, start, end, vectors, identity)
+ return
+ offset = 0
+ with self._cond:
+ for request, start, end in pieces:
+ width = end - start
+ self._land(request, start, end, vectors[offset : offset + width], identity)
+ offset += width
+
+ def embed(
+ self,
+ texts: list[str],
+ model_name: str | None,
+ on_progress: Callable[[int, int], None] | None = None,
+ ):
+ """Returns ``(vectors, identity, changed)``. A caller with a full batch never lingers: it keeps
+ its parse slot (see ``_run``), so waiting would stall the documents it waits for."""
+ request = _EmbedRequest(texts, model_name)
+ may_wait = len(texts) < _EMBED_BATCH
+ reported = 0
+ with self._cond:
+ self._pending.append(request)
+ self._cond.notify_all()
+ try:
+ while True:
+ if on_progress is not None and request.filled > reported:
+ reported = request.filled
+ on_progress(reported, len(texts))
+ with self._cond:
+ if request.done:
+ break
+ if request.key in self._encoding or request not in self._pending:
+ self._cond.wait()
+ continue
+ self._encoding.add(request.key)
+ if may_wait:
+ self._linger(request.key)
+ pieces = self._take(request.key)
+ try:
+ self._run_batch(pieces)
+ except BaseException as exc:
+ # Never leave another document waiting on a slice that will not land.
+ with self._cond:
+ for owner, _, _ in pieces:
+ if not owner.done:
+ self._fail(owner, exc)
+ raise
+ finally:
+ with self._cond:
+ self._encoding.discard(request.key)
+ self._cond.notify_all()
+ finally:
+ with self._cond:
+ if request in self._pending:
+ self._pending.remove(request)
+ self._cond.notify_all()
+ if request.error is not None:
+ raise request.error
+ return request.vectors, request.identity, request.changed
+
+
+_embed_queue = _EmbedQueue()
+
+
def _embed_pass(
texts: list[str],
model_name: str | None,
@@ -147,20 +334,24 @@ def _embed_pass(
):
"""One batched pass. Returns ``(vectors, identity, changed)``, ``changed`` when the
embedder swapped part way and the vectors therefore span two spaces."""
+ if not texts:
+ return [], embeddings.embedding_identity(model_name), False
+ if embeddings.batches_across_documents(model_name):
+ vectors, identity, changed = _embed_queue.embed(texts, model_name, on_progress)
+ return vectors, identity or embeddings.embedding_identity(model_name), changed
vectors: list = []
identity: str | None = None
changed = False
for i in range(0, len(texts), _EMBED_BATCH):
- batch = texts[i : i + _EMBED_BATCH]
out, batch_identity = embeddings.encode_with_identity(
- batch, model_name = model_name, normalize = True
+ texts[i : i + _EMBED_BATCH], model_name = model_name, normalize = True
)
changed = changed or (identity is not None and batch_identity != identity)
identity = batch_identity
vectors.extend(out)
if on_progress is not None:
on_progress(min(i + _EMBED_BATCH, len(texts)), len(texts))
- return vectors, identity or embeddings.embedding_identity(model_name), changed
+ return vectors, identity, changed
def _embed_all(
@@ -283,6 +474,36 @@ def _retire_orphan_after_failure(
logger.warning("failed to retire orphaned document %s", old_id, exc_info = True)
+class _ExpectedEmbed:
+ def __init__(self, model_name: str | None) -> None:
+ self._model_name = model_name
+ _embed_queue.expect(model_name)
+ self._pending = True
+
+ def arrive(self) -> None:
+ if self._pending:
+ self._pending = False
+ _embed_queue.unexpect(self._model_name)
+
+
+class _StageSlot:
+ """One of ``stage_slots``, held around parse and store but not the embed wait."""
+
+ def __init__(self, slots: threading.Semaphore | None) -> None:
+ self._slots = slots
+ self._held = False
+
+ def acquire(self) -> None:
+ if self._slots is not None and not self._held:
+ self._slots.acquire()
+ self._held = True
+
+ def release(self) -> None:
+ if self._held:
+ self._held = False
+ self._slots.release()
+
+
def _run(
job_id: str,
document_id: str,
@@ -292,9 +513,13 @@ def _run(
ocr: bool | None = None,
caption: bool | None = None,
replaces: tuple[str, str | None] | None = None,
+ stage_slots: threading.Semaphore | None = None,
) -> None:
conn = None
+ slot = _StageSlot(stage_slots)
+ expected = _ExpectedEmbed(model_name)
try:
+ slot.acquire()
conn = rag_db.get_connection()
_progress(conn, job_id, "parsing", 0.1)
pages = parsers.parse(stored_path)
@@ -394,7 +619,12 @@ def _run(
embedded_progress = max(embedded_progress, 0.65 + 0.25 * done / total)
_progress(conn, job_id, "embedding", embedded_progress)
+ # Below a full batch, give the slot up so other documents can parse and join the batch.
+ if len(chunks) < _EMBED_BATCH and embeddings.batches_across_documents(model_name):
+ slot.release()
+ expected.arrive()
vectors, identity = _embed_all([c.text for c in chunks], model_name, report_embeddings)
+ slot.acquire()
store.set_document_embedding_model(conn, document_id, identity)
# Locate each chunk's highlight regions (non-PDFs/failures yield none).
@@ -432,6 +662,8 @@ def _run(
logger.exception("failed to record ingestion failure for job %s", job_id)
_emit(job_id, {"type": "error", "stage": "error", "error": str(exc)})
finally:
+ expected.arrive()
+ slot.release()
if conn is not None:
# Every exit but completion retires the orphan; nothing relaunches, so a lost lease ends the work.
_retire_orphan_after_failure(conn, replaces, stored_path, document_id)
@@ -459,8 +691,10 @@ def start_ingestion(
background: bool = True,
content_hash: str | None = None,
reuse_identical: bool = False,
+ stage_slots: threading.Semaphore | None = None,
) -> tuple[str, str]:
- """Duplicate hash reuses the existing document; a passed content_hash is trusted, never reverified."""
+ """Duplicate hash reuses the existing document; a passed content_hash is trusted, never reverified.
+ ``stage_slots`` bounds concurrent parse/store while any number wait to embed together."""
account_path(stored_path)
if account_is_retired():
raise RuntimeError("Account is retired")
@@ -623,6 +857,7 @@ def start_ingestion(
ocr,
caption,
replaces,
+ stage_slots,
)
if not background:
_run(*args)
diff --git a/studio/backend/tests/test_rag_linked_folders.py b/studio/backend/tests/test_rag_linked_folders.py
index 64598b443d8..12c722b3223 100644
--- a/studio/backend/tests/test_rag_linked_folders.py
+++ b/studio/backend/tests/test_rag_linked_folders.py
@@ -3321,3 +3321,80 @@ def test_an_unlink_cancel_only_reaches_its_own_account():
assert not run_as(other, folder_sync.is_cancel_requested, "shared-id")
finally:
folder_sync.clear_cancel("shared-id")
+
+
+@requires_sqlite_vec
+def test_a_folder_of_small_files_embeds_across_documents(rag_home, stub_embeddings, monkeypatch):
+ """A vault of one-chunk notes paid one encode call per file before (#16715)."""
+ monkeypatch.setattr(embeddings, "batches_across_documents", lambda model_name = None: True)
+ source, folder = _folder(rag_home)
+ for i in range(12):
+ (source / f"note-{i:02d}.md").write_text(f"note {i} searchable words", encoding = "utf-8")
+ monkeypatch.setattr(folder_sync.config, "FOLDER_INGEST_WORKERS", 2)
+ batches = []
+ real_encode = embeddings.encode
+ lock = threading.Lock()
+
+ def recording(texts, **kwargs):
+ with lock:
+ batches.append(len(texts))
+ # An instant stub finishes each file before the coordinator snapshots the next one.
+ time.sleep(0.5)
+ return real_encode(texts, **kwargs)
+
+ monkeypatch.setattr(embeddings, "encode", recording)
+
+ assert _run(folder["id"])["status"] == "completed"
+
+ assert sum(batches) == 12, batches
+ assert len(batches) < 12, batches
+ assert max(batches) > 1, batches
+ with _connection() as conn:
+ assert (
+ conn.execute(
+ "SELECT COUNT(*) AS count FROM linked_folder_files WHERE folder_id=?",
+ (folder["id"],),
+ ).fetchone()["count"]
+ == 12
+ )
+ for i in range(12):
+ assert store.search_lexical(conn, folder["scope"], f"note {i}", 5)
+
+
+@requires_sqlite_vec
+def test_large_files_stay_within_the_parse_workers(rag_home, stub_embeddings, monkeypatch):
+ """Only small files queue past the workers; big ones would hold snapshots and chunks."""
+ monkeypatch.setattr(embeddings, "batches_across_documents", lambda model_name = None: True)
+ source, folder = _folder(rag_home)
+ for i in range(8):
+ (source / f"doc-{i}.txt").write_text(f"document {i} words", encoding = "utf-8")
+ monkeypatch.setattr(folder_sync.config, "FOLDER_INGEST_WORKERS", 2)
+ # Every file in this folder counts as large.
+ monkeypatch.setattr(folder_sync.config, "FOLDER_INGEST_BATCH_FILE_BYTES", 1)
+ in_flight = {"now": 0, "peak": 0}
+ lock = threading.Lock()
+ real_start = ingestion.start_ingestion
+
+ def watched(*args, **kwargs):
+ with lock:
+ in_flight["now"] += 1
+ in_flight["peak"] = max(in_flight["peak"], in_flight["now"])
+ try:
+ return real_start(*args, **kwargs)
+ finally:
+ with lock:
+ in_flight["now"] -= 1
+
+ monkeypatch.setattr(folder_sync.ingestion, "start_ingestion", watched)
+
+ assert _run(folder["id"])["status"] == "completed"
+
+ assert in_flight["peak"] <= 2, in_flight
+ with _connection() as conn:
+ assert (
+ conn.execute(
+ "SELECT COUNT(*) AS count FROM linked_folder_files WHERE folder_id=?",
+ (folder["id"],),
+ ).fetchone()["count"]
+ == 8
+ )
diff --git a/studio/backend/tests/test_rag_shared_embed_batch.py b/studio/backend/tests/test_rag_shared_embed_batch.py
new file mode 100644
index 00000000000..7f0922f5c0b
--- /dev/null
+++ b/studio/backend/tests/test_rag_shared_embed_batch.py
@@ -0,0 +1,466 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Documents ingesting at the same time embed in shared batches (#16715), each keeping its own
+embedding identity, progress and failures.
+
+The stub embedder returns instantly, so a document would normally finish before the next one is
+parsed. These tests hold the queue's "a document is on its way" count, which is what a real sync
+has while other files parse, and raise the linger so the wait is not a race.
+"""
+
+import struct
+import threading
+import time
+from contextlib import contextmanager
+
+import pytest
+
+from core.rag import config, embeddings, ingestion, store
+from storage import rag_db
+
+requires_sqlite_vec = pytest.mark.skipif(
+ not rag_db.RAG_AVAILABLE, reason = "sqlite-vec is not installed"
+)
+
+
+_real_gate = embeddings.batches_across_documents
+
+
[email protected](autouse = True)
+def _embedder_on_an_accelerator(monkeypatch):
+ # The stub embedder resolves to sentence-transformers on CPU, which keeps per-document calls.
+ monkeypatch.setattr(embeddings, "batches_across_documents", lambda model_name = None: True)
+
+
+def _write(tmp_path, name, text):
+ path = tmp_path / name
+ path.write_text(text, encoding = "utf-8")
+ return str(path)
+
+
+def _wait_finished(job_id, timeout = 60.0):
+ deadline = time.time() + timeout
+ while time.time() < deadline:
+ status = ingestion.get_job_status(job_id)
+ if status and status["status"] in ("completed", "failed", "cancelled"):
+ return status
+ time.sleep(0.05)
+ raise AssertionError("ingestion did not finish in time")
+
+
+@contextmanager
+def _documents_still_parsing(monkeypatch):
+ """Stands in for files a sync is still parsing, so batches wait for them."""
+ monkeypatch.setattr(ingestion, "_EMBED_LINGER_SECONDS", 30.0)
+ model = config.effective_embedding_model()
+ ingestion._embed_queue.expect(model)
+ try:
+ yield
+ finally:
+ ingestion._embed_queue.unexpect(model)
+
+
+def _record_batches(monkeypatch):
+ """Returns the list of text batches the embedder is actually called with."""
+ batches: list[list[str]] = []
+ real = embeddings.encode_with_identity
+ lock = threading.Lock()
+
+ def recording(texts, **kwargs):
+ with lock:
+ batches.append(list(texts))
+ return real(texts, **kwargs)
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", recording)
+ return batches
+
+
+def _record_stages(monkeypatch):
+ """Returns (job_id, stage, progress) as the job rows are written; SSE queues are reaped."""
+ stages: list[tuple[str, str, float]] = []
+ real = ingestion._progress
+ lock = threading.Lock()
+
+ def recording(conn, job_id, stage, progress):
+ with lock:
+ stages.append((job_id, stage, progress))
+ return real(conn, job_id, stage, progress)
+
+ monkeypatch.setattr(ingestion, "_progress", recording)
+ return stages
+
+
+def _start(
+ scope,
+ tmp_path,
+ name,
+ text,
+ slots = None,
+ model_name = None,
+):
+ return ingestion.start_ingestion(
+ scope,
+ "K1",
+ None,
+ name,
+ _write(tmp_path, name, text),
+ model_name = model_name,
+ stage_slots = slots,
+ )
+
+
+def test_two_documents_embed_in_one_call(rag_home, stub_embeddings, tmp_path, monkeypatch):
+ """The batch-of-1 path: before this, a one-chunk file always paid a whole encode call."""
+ batches = _record_batches(monkeypatch)
+ scope = store.kb_scope("K1")
+ # One slot, like a linked-folder sync with a single parse worker.
+ slots = threading.Semaphore(1)
+ with _documents_still_parsing(monkeypatch):
+ jobs = [
+ _start(scope, tmp_path, "a.txt", "alpha document words", slots),
+ _start(scope, tmp_path, "b.txt", "bravo document words", slots),
+ ]
+ for _, job_id in jobs:
+ assert _wait_finished(job_id)["status"] == "completed"
+
+ assert len(batches) == 1, batches
+ assert sorted(text.split()[0] for text in batches[0]) == ["alpha", "bravo"]
+ conn = rag_db.get_connection()
+ try:
+ for document_id, _ in jobs:
+ assert store.get_document(conn, document_id)["num_chunks"] == 1
+ assert store.search_lexical(conn, scope, "alpha", 5)
+ assert store.search_lexical(conn, scope, "bravo", 5)
+ finally:
+ conn.close()
+
+
+@requires_sqlite_vec
+def test_a_shared_batch_keeps_each_document_s_own_vectors(
+ rag_home, stub_embeddings, tmp_path, monkeypatch
+):
+ """Vectors must come back on the chunk they were made from, not in the batch's order."""
+ scope = store.kb_scope("K1")
+ slots = threading.Semaphore(1)
+ with _documents_still_parsing(monkeypatch):
+ jobs = {
+ name: _start(scope, tmp_path, f"{name}.txt", " ".join([name] * 40), slots)
+ for name in ("alpha", "bravo", "charlie")
+ }
+ for _, job_id in jobs.values():
+ assert _wait_finished(job_id)["status"] == "completed"
+
+ conn = rag_db.get_connection()
+ try:
+ for name, (document_id, _) in jobs.items():
+ rows = conn.execute(
+ "SELECT id, text FROM chunks WHERE document_id=? ORDER BY chunk_index",
+ (document_id,),
+ ).fetchall()
+ assert rows and all(name in row["text"] for row in rows)
+ for row in rows:
+ stored = conn.execute(
+ "SELECT embedding FROM chunks_vec WHERE chunk_id=?", (row["id"],)
+ ).fetchone()
+ assert stored is not None
+ expected = embeddings.encode([row["text"]])[0]
+ written = struct.unpack(f"<{len(expected)}f", stored["embedding"])
+ assert written == pytest.approx(tuple(expected), abs = 1e-6)
+ finally:
+ conn.close()
+
+
+def test_one_failing_document_does_not_fail_the_others(
+ rag_home, stub_embeddings, tmp_path, monkeypatch
+):
+ """A failed shared batch retries per document, so a text the embedder rejects fails alone."""
+ real = embeddings.encode_with_identity
+
+ def refuse_bravo(texts, **kwargs):
+ if any("bravo" in text for text in texts):
+ raise RuntimeError("synthetic encode failure")
+ return real(texts, **kwargs)
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", refuse_bravo)
+ scope = store.kb_scope("K1")
+ slots = threading.Semaphore(1)
+ with _documents_still_parsing(monkeypatch):
+ good = _start(scope, tmp_path, "a.txt", "alpha document words", slots)
+ bad = _start(scope, tmp_path, "b.txt", "bravo document words", slots)
+
+ assert _wait_finished(good[1])["status"] == "completed"
+ assert _wait_finished(bad[1])["status"] == "failed"
+ conn = rag_db.get_connection()
+ try:
+ assert store.get_document(conn, good[0])["status"] == "completed"
+ assert store.get_document(conn, bad[0])["status"] == "failed"
+ assert store.search_lexical(conn, scope, "alpha", 5)
+ assert not store.search_lexical(conn, scope, "bravo", 5)
+ finally:
+ conn.close()
+
+
+def test_documents_under_different_models_are_not_batched_together(
+ rag_home, stub_embeddings, tmp_path, monkeypatch
+):
+ """One batch is one encode call, so it can only hold texts for one model."""
+ batches = _record_batches(monkeypatch)
+ models = []
+ real_encode = embeddings.encode
+
+ def note_model(
+ texts,
+ *,
+ model_name = None,
+ normalize = True,
+ ):
+ models.append(model_name)
+ return real_encode(texts, model_name = model_name, normalize = normalize)
+
+ monkeypatch.setattr(embeddings, "encode", note_model)
+ scope = store.kb_scope("K1")
+ slots = threading.Semaphore(1)
+ with _documents_still_parsing(monkeypatch):
+ jobs = [
+ _start(scope, tmp_path, name, f"{name} document words", slots, model_name = model)
+ for name, model in (("a.txt", "model/one"), ("b.txt", "model/two"))
+ ]
+ for _, job_id in jobs:
+ assert _wait_finished(job_id)["status"] == "completed"
+
+ assert len(batches) == 2, batches
+ assert all(len(batch) == 1 for batch in batches)
+ assert sorted(models) == ["model/one", "model/two"]
+
+
+def test_a_document_with_a_full_batch_of_chunks_embeds_without_waiting(
+ rag_home, stub_embeddings, tmp_path, monkeypatch
+):
+ """It gains nothing from waiting, and holding its chunks for others would cost memory."""
+ from core.rag import config
+
+ monkeypatch.setattr(ingestion, "_EMBED_BATCH", 4)
+ monkeypatch.setattr(config, "CHUNK_TOKENS", 3)
+ monkeypatch.setattr(config, "CHUNK_OVERLAP", 0)
+ batches = _record_batches(monkeypatch)
+ scope = store.kb_scope("K1")
+ slots = threading.Semaphore(1)
+ started = time.monotonic()
+ with _documents_still_parsing(monkeypatch):
+ document_id, job_id = _start(
+ scope,
+ tmp_path,
+ "big.txt",
+ " ".join(f"word{i}" for i in range(40)),
+ slots,
+ )
+ assert _wait_finished(job_id, timeout = 20.0)["status"] == "completed"
+ # It never waited out the 30 s linger for the documents still parsing.
+ assert time.monotonic() - started < 20.0
+
+ assert batches and all(len(batch) <= 4 for batch in batches)
+ conn = rag_db.get_connection()
+ try:
+ assert store.get_document(conn, document_id)["num_chunks"] > 4
+ finally:
+ conn.close()
+
+
+def test_a_waiting_document_reports_its_embedding_progress(
+ rag_home, stub_embeddings, tmp_path, monkeypatch
+):
+ """Progress must move when its slice lands in another document's batch."""
+ stages = _record_stages(monkeypatch)
+ scope = store.kb_scope("K1")
+ slots = threading.Semaphore(1)
+ with _documents_still_parsing(monkeypatch):
+ jobs = [
+ _start(scope, tmp_path, f"doc{i}.txt", f"document {i} words", slots) for i in range(3)
+ ]
+ for _, job_id in jobs:
+ assert _wait_finished(job_id)["status"] == "completed"
+
+ for _, job_id in jobs:
+ written = [(stage, progress) for job, stage, progress in stages if job == job_id]
+ assert "embedding" in [stage for stage, _ in written]
+ assert [progress for _, progress in written] == sorted(progress for _, progress in written)
+
+
+def test_a_backend_swap_inside_a_shared_batch_re_embeds_that_document(
+ rag_home, stub_embeddings, tmp_path, monkeypatch
+):
+ """Vectors from two spaces under one identity are unsearchable, so the document re-embeds
+ whole under the backend that took over."""
+ from core.rag import config
+
+ monkeypatch.setattr(embeddings, "active_backend_is_llama", lambda *_a, **_k: False)
+ monkeypatch.setattr(ingestion, "_EMBED_BATCH", 2)
+ monkeypatch.setattr(config, "CHUNK_TOKENS", 3)
+ monkeypatch.setattr(config, "CHUNK_OVERLAP", 0)
+ real = embeddings.encode_with_identity
+ swapped = {"done": False}
+
+ def swap_after_the_first_batch(texts, **kwargs):
+ out = real(texts, **kwargs)
+ if not swapped["done"]:
+ swapped["done"] = True
+ monkeypatch.setattr(embeddings, "active_backend_is_llama", lambda *_a, **_k: True)
+ return out
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", swap_after_the_first_batch)
+ scope = store.kb_scope("K1")
+ document_id, job_id = _start(
+ scope,
+ tmp_path,
+ "doc.txt",
+ " ".join(f"word{i}" for i in range(12)),
+ )
+ assert _wait_finished(job_id)["status"] == "completed"
+
+ conn = rag_db.get_connection()
+ try:
+ identity = store.get_document(conn, document_id)["embedding_model"]
+ finally:
+ conn.close()
+ assert identity.startswith("llama-server:"), identity
+
+
+def test_the_queue_keeps_nothing_after_a_failed_document(rag_home, stub_embeddings, tmp_path):
+ """A document that stops early must not leave a slice others would wait on."""
+ scope = store.kb_scope("K1")
+ empty = _start(scope, tmp_path, "empty.txt", " ")
+ assert _wait_finished(empty[1])["status"] == "failed"
+ good = _start(scope, tmp_path, "good.txt", "alpha document words")
+ assert _wait_finished(good[1])["status"] == "completed"
+
+ assert not ingestion._embed_queue._pending
+ assert not ingestion._embed_queue._expected
+
+
[email protected]("documents", (2, 9))
+def test_many_small_documents_need_far_fewer_encode_calls(
+ rag_home, stub_embeddings, tmp_path, monkeypatch, documents
+):
+ batches = _record_batches(monkeypatch)
+ scope = store.kb_scope("K1")
+ slots = threading.Semaphore(2)
+ with _documents_still_parsing(monkeypatch):
+ jobs = [
+ _start(scope, tmp_path, f"note{i}.txt", f"note {i} searchable words", slots)
+ for i in range(documents)
+ ]
+ for _, job_id in jobs:
+ assert _wait_finished(job_id)["status"] == "completed"
+
+ assert sum(len(batch) for batch in batches) == documents
+ assert len(batches) == 1, batches
+
+
+def test_cpu_sentence_transformers_keeps_one_call_per_document(
+ rag_home, stub_embeddings, tmp_path, monkeypatch
+):
+ """CPU encode pads every text to the batch's longest, which costs more than the calls it saves."""
+ monkeypatch.setattr(embeddings, "batches_across_documents", _real_gate)
+ assert not embeddings.batches_across_documents(None)
+ batches = _record_batches(monkeypatch)
+ scope = store.kb_scope("K1")
+ slots = threading.Semaphore(2)
+ with _documents_still_parsing(monkeypatch):
+ jobs = [
+ _start(scope, tmp_path, f"note{i}.txt", f"note {i} searchable words", slots)
+ for i in range(4)
+ ]
+ for _, job_id in jobs:
+ assert _wait_finished(job_id)["status"] == "completed"
+
+ assert sorted(len(batch) for batch in batches) == [1, 1, 1, 1], batches
+ assert not ingestion._embed_queue._pending
+
+
+def test_two_accounts_never_share_a_batch(monkeypatch):
+ """The encode runs as whichever caller took the batch, so it may only hold that account's
+ texts: its token, model path and device choice are the account's."""
+ from utils.account_context import AccountContext, run_as
+
+ queue = ingestion._EmbedQueue()
+ monkeypatch.setattr(ingestion, "_EMBED_LINGER_SECONDS", 30.0)
+ seen: list[list[str]] = []
+ lock = threading.Lock()
+
+ def fake_encode(texts, **kwargs):
+ with lock:
+ seen.append(list(texts))
+ return [[float(len(text))] for text in texts], "identity"
+
+ monkeypatch.setattr(embeddings, "encode_with_identity", fake_encode)
+ results: dict[str, object] = {}
+ arrived = threading.Barrier(3)
+ queue.expect()
+
+ def embed_as(account):
+ def go():
+ arrived.wait(30)
+ results[account.account_id] = queue.embed([f"{account.account_id} text"], None)
+
+ run_as(account, go)
+
+ threads = [
+ threading.Thread(target = embed_as, args = (AccountContext(f"acct-{i}", f"user{i}"),))
+ for i in (1, 2)
+ ]
+ for thread in threads:
+ thread.start()
+ arrived.wait(30)
+ # Both are in the queue now; releasing the expected document lets each batch leave.
+ queue.unexpect()
+ for thread in threads:
+ thread.join(30)
+ assert not thread.is_alive()
+
+ assert sorted(len(batch) for batch in seen) == [1, 1], seen
+ assert sorted(batch[0] for batch in seen) == ["acct-1 text", "acct-2 text"]
+ assert set(results) == {"acct-1", "acct-2"}
+
+
+def test_a_large_document_does_not_hold_back_a_small_one(monkeypatch):
+ """The small one's progress callback, and with it the unlink check, must run within a batch."""
+ monkeypatch.setattr(ingestion, "_EMBED_BATCH", 4)
+ queue = ingestion._EmbedQueue()
+ big = ingestion._EmbedRequest([f"big {i}" for i in range(40)], None)
+ small = ingestion._EmbedRequest(["small"], None)
+ queue._pending += [big, small]
+
+ pieces = queue._take(big.key)
+
+ assert (small, 0, 1) in pieces
+ assert sum(end - start for _, start, end in pieces) == 4
+ assert queue._pending == [big]
+
+
+def test_a_parse_under_another_model_does_not_hold_a_batch(monkeypatch):
+ monkeypatch.setattr(ingestion, "_EMBED_LINGER_SECONDS", 30.0)
+ monkeypatch.setattr(
+ embeddings, "encode_with_identity", lambda texts, **kwargs: ([[1.0]] * len(texts), "id")
+ )
+ queue = ingestion._EmbedQueue()
+ queue.expect("other-model")
+ started = time.monotonic()
+ vectors, _, _ = queue.embed(["a note"], None)
+ assert vectors == [[1.0]]
+ assert time.monotonic() - started < 5
+ queue.unexpect("other-model")
+ assert not queue._expected
+
+
+def test_a_gpu_choice_that_loaded_on_cpu_keeps_per_document_calls(monkeypatch):
+ from types import SimpleNamespace
+
+ monkeypatch.setattr(embeddings, "active_backend_is_llama", lambda model_name = None: False)
+ monkeypatch.setattr(embeddings, "_device", lambda: "cuda")
+ monkeypatch.setattr(embeddings, "_model", None)
+ assert _real_gate(None)
+ monkeypatch.setattr(embeddings, "_model", SimpleNamespace(device = "cpu"))
+ assert not _real_gate(None)
+ monkeypatch.setattr(embeddings, "_model", SimpleNamespace(device = "cuda:0"))
+ assert _real_gate(None)