Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(security): bound the RAG auto-index queue to prevent OOM under burst (#3939)

LearningCircuit89fb1b31e5d6
diff --git a/changelog.d/+rag-auto-index-backpressure.security.md b/changelog.d/+rag-auto-index-backpressure.security.md
new file mode 100644
index 0000000000..878e090cbc
--- /dev/null
+++ b/changelog.d/+rag-auto-index-backpressure.security.md
@@ -0,0 +1 @@
+RAG auto-indexing now applies backpressure to its background worker pool. The `ThreadPoolExecutor` previously had an unbounded internal queue; under sustained upload bursts (possible under the configurable upload rate cap from #3935), this could queue thousands of indexing jobs and exhaust memory. The pool is now capped at 100 in-flight + queued jobs; submissions over the cap are dropped with a clear log warning ("Auto-index queue saturated… trigger a manual reindex if needed"). Documents still upload successfully — only the *automatic* indexing is skipped, and users can manually reindex via the UI.
diff --git a/src/local_deep_research/research_library/routes/rag_routes.py b/src/local_deep_research/research_library/routes/rag_routes.py
index 4bd8404e1b..cbb5a40eda 100644
--- a/src/local_deep_research/research_library/routes/rag_routes.py
+++ b/src/local_deep_research/research_library/routes/rag_routes.py
@@ -87,10 +87,26 @@ def _agent_enabled_default_on(collection) -> bool:
return getattr(collection, "agent_enabled", True) is not False
-# Global ThreadPoolExecutor for auto-indexing to prevent thread proliferation
+# Global ThreadPoolExecutor for auto-indexing to prevent thread proliferation.
+#
+# ThreadPoolExecutor's internal _work_queue is unbounded. Without
+# backpressure, a sustained burst of uploads (possible under the
+# configurable upload rate cap from #3935) could queue thousands of indexing
+# jobs in memory, exhausting RAM. We track pending submissions in a counter
+# and reject new submissions once the queue is saturated.
_auto_index_executor: ThreadPoolExecutor | None = None
_auto_index_executor_lock = threading.Lock()
+# Maximum in-flight + queued indexing jobs. Each job is one upload batch
+# (an unbounded list of document IDs), so per-job duration varies with batch
+# size; 100 is a buffer on the number of *queued batches*, not documents.
+# The OOM bound still holds because a queued job holds only a small list of
+# ID strings, not document contents. Tunable if real-world workloads need
+# more headroom.
+_MAX_PENDING_AUTO_INDEX_JOBS = 100
+_pending_auto_index_jobs = 0
+_pending_auto_index_lock = threading.Lock()
+
def _get_auto_index_executor() -> ThreadPoolExecutor:
"""Get or create the global auto-indexing executor (thread-safe)."""
@@ -104,12 +120,46 @@ def _get_auto_index_executor() -> ThreadPoolExecutor:
return _auto_index_executor
+def _try_reserve_auto_index_slot() -> bool:
+ """Reserve a queue slot if capacity allows. Returns True on success.
+
+ Caller MUST call ``_release_auto_index_slot`` when the job completes
+ (success OR failure), otherwise the counter leaks and eventually
+ blocks all submissions.
+ """
+ global _pending_auto_index_jobs
+ with _pending_auto_index_lock:
+ if _pending_auto_index_jobs >= _MAX_PENDING_AUTO_INDEX_JOBS:
+ return False
+ _pending_auto_index_jobs += 1
+ return True
+
+
+def _release_auto_index_slot() -> None:
+ """Release a previously reserved queue slot."""
+ global _pending_auto_index_jobs
+ with _pending_auto_index_lock:
+ _pending_auto_index_jobs = max(0, _pending_auto_index_jobs - 1)
+
+
def _shutdown_auto_index_executor() -> None:
- """Shutdown the auto-index executor gracefully."""
- global _auto_index_executor
- if _auto_index_executor is not None:
- _auto_index_executor.shutdown(wait=True)
+ """Shutdown the auto-index executor gracefully.
+
+ Holds ``_auto_index_executor_lock`` for the read+nullify so two
+ concurrent callers (e.g. atexit + a test teardown) can't race into an
+ ``AttributeError`` on the ``None`` reference; the blocking ``shutdown``
+ runs outside the lock. Resets the pending-jobs counter afterwards so a
+ re-created executor (tests, WSGI reload) starts from a clean count
+ instead of inheriting a stale, possibly-saturated value.
+ """
+ global _auto_index_executor, _pending_auto_index_jobs
+ with _auto_index_executor_lock:
+ executor = _auto_index_executor
_auto_index_executor = None
+ if executor is not None:
+ executor.shutdown(wait=True)
+ with _pending_auto_index_lock:
+ _pending_auto_index_jobs = 0
atexit.register(_shutdown_auto_index_executor)
@@ -2420,19 +2470,80 @@ def trigger_auto_index(
)
return
+ # Reserve a queue slot before submission. If the indexing queue is
+ # saturated (too many uploads in flight), drop this auto-index job
+ # rather than letting the executor's unbounded internal queue grow
+ # without bound. The user can reindex manually later via the UI.
+ if not _try_reserve_auto_index_slot():
+ logger.warning(
+ "Auto-index queue saturated ({}+ jobs pending); dropping "
+ "auto-index for {} document(s) in collection {}. "
+ "Documents are uploaded; trigger a manual reindex if needed.",
+ _MAX_PENDING_AUTO_INDEX_JOBS,
+ len(document_ids),
+ collection_id,
+ )
+ return
+
logger.info(
f"Auto-indexing {len(document_ids)} documents in collection {collection_id}"
)
- # Submit to thread pool (bounded concurrency, prevents thread proliferation)
- executor = _get_auto_index_executor()
- executor.submit(
- _auto_index_documents_worker,
- document_ids,
- collection_id,
- username,
- db_password,
- )
+ # Release the reserved slot EXACTLY ONCE per submission. CPython's
+ # ThreadPoolExecutor.submit() enqueues the work item BEFORE it tries to
+ # spin up a worker thread; if thread-start raises (e.g.
+ # RuntimeError("can't start new thread")), the item is already queued, so
+ # a live worker can run _wrapped_worker (releasing in its finally) AND the
+ # except block below also fires. A per-call lock + flag makes the release
+ # idempotent so that double-fire only releases once — otherwise the
+ # counter under-counts in-flight jobs and erodes the OOM bound (the
+ # max(0, ...) floor would silently hide the drift).
+ _release_lock = threading.Lock()
+ _slot_released = {"done": False}
+
+ def _release_slot_once():
+ with _release_lock:
+ if _slot_released["done"]:
+ return
+ _slot_released["done"] = True
+ _release_auto_index_slot()
+
+ def _wrapped_worker(*args, **kwargs):
+ try:
+ _auto_index_documents_worker(*args, **kwargs)
+ finally:
+ _release_slot_once()
+
+ # Submit to thread pool (bounded concurrency, prevents thread proliferation).
+ # Build the executor inside the try too: if _get_auto_index_executor()
+ # itself fails (OS thread/mutex exhaustion), the slot must still be
+ # released — otherwise a reserved slot leaks permanently.
+ try:
+ executor = _get_auto_index_executor()
+ executor.submit(
+ _wrapped_worker,
+ document_ids,
+ collection_id,
+ username,
+ db_password,
+ )
+ except Exception:
+ # If submit itself fails (executor shutting down, OOM, etc.), the
+ # wrapped worker may never run, so release the slot here. The release
+ # is idempotent: if the work item was already enqueued and a worker
+ # ran (and released) before thread-start failed, this is a no-op. Do
+ # NOT re-raise: the upload has already been committed by the caller,
+ # so propagating would turn a successful upload into a 500 (and prompt
+ # the client to retry, creating duplicates). Auto-indexing is simply
+ # skipped; the user can trigger a manual reindex later via the UI.
+ _release_slot_once()
+ logger.exception(
+ "Failed to submit auto-index job for {} document(s) in "
+ "collection {}; upload succeeded, auto-indexing skipped. "
+ "Trigger a manual reindex if needed.",
+ len(document_ids),
+ collection_id,
+ )
@thread_cleanup
diff --git a/tests/library/test_auto_indexing.py b/tests/library/test_auto_indexing.py
index 29c4804988..c5c09fb322 100644
--- a/tests/library/test_auto_indexing.py
+++ b/tests/library/test_auto_indexing.py
@@ -4,6 +4,30 @@ import threading
import time
from unittest.mock import MagicMock, patch
+import pytest
+
+
[email protected](autouse=True)
+def _reset_pending_auto_index_jobs():
+ """Save/restore the module-global pending-jobs counter around each test.
+
+ Several tests mock ``_get_auto_index_executor`` with a no-op ``submit``,
+ which reserves a slot (``_pending_auto_index_jobs += 1``) that the (never
+ run) worker never releases. Without this fixture the counter leaks +1 per
+ such test and the saturation/concurrency tests become order-dependent.
+ This restores the counter to its pre-test value after every test, keeping
+ the suite order-independent.
+ """
+ import local_deep_research.research_library.routes.rag_routes as rag_module
+
+ with rag_module._pending_auto_index_lock:
+ saved = rag_module._pending_auto_index_jobs
+ try:
+ yield
+ finally:
+ with rag_module._pending_auto_index_lock:
+ rag_module._pending_auto_index_jobs = saved
+
class TestAutoIndexingSetting:
"""Test the auto-indexing setting."""
@@ -143,10 +167,20 @@ class TestTriggerAutoIndex:
mock_get_executor.assert_not_called()
def test_trigger_auto_index_passes_correct_arguments(self):
- """Test that trigger_auto_index passes correct arguments to the worker."""
+ """Backpressure wrapper still delegates correct args to the worker.
+
+ The backpressure change submits a ``_wrapped_worker`` closure rather
+ than ``_auto_index_documents_worker`` directly. This test verifies the
+ full contract of that wrapper:
+
+ * a queue slot is reserved before the job is submitted,
+ * the submitted callable still forwards the exact arguments to the
+ underlying ``_auto_index_documents_worker``, and
+ * the reserved slot is released once the wrapped worker finishes.
+ """
+ import local_deep_research.research_library.routes.rag_routes as rag_module
from local_deep_research.research_library.routes.rag_routes import (
trigger_auto_index,
- _auto_index_documents_worker,
)
with patch(
@@ -168,21 +202,61 @@ class TestTriggerAutoIndex:
mock_executor = MagicMock()
mock_get_executor.return_value = mock_executor
- trigger_auto_index(
- document_ids=["doc1", "doc2", "doc3"],
- collection_id="my_collection",
- username="alice",
- db_password="secret123",
- )
+ with patch(
+ "local_deep_research.research_library.routes.rag_routes._auto_index_documents_worker"
+ ) as mock_worker:
+ pending_before = rag_module._pending_auto_index_jobs
- # Verify correct arguments passed to submit
- mock_executor.submit.assert_called_once()
- call_args = mock_executor.submit.call_args
- assert call_args[0][0] == _auto_index_documents_worker
- assert call_args[0][1] == ["doc1", "doc2", "doc3"]
- assert call_args[0][2] == "my_collection"
- assert call_args[0][3] == "alice"
- assert call_args[0][4] == "secret123"
+ trigger_auto_index(
+ document_ids=["doc1", "doc2", "doc3"],
+ collection_id="my_collection",
+ username="alice",
+ db_password="secret123",
+ )
+
+ # A queue slot must be reserved before submission, so
+ # the pending counter is incremented while the (not yet
+ # run) wrapped worker holds the slot.
+ assert (
+ rag_module._pending_auto_index_jobs
+ == pending_before + 1
+ )
+
+ # A wrapper closure is submitted, NOT the worker
+ # directly (that is the whole point of the wrapper).
+ mock_executor.submit.assert_called_once()
+ call_args = mock_executor.submit.call_args
+ submitted_callable = call_args[0][0]
+ assert submitted_callable is not mock_worker
+ assert callable(submitted_callable)
+
+ # The wrapper forwards the original arguments unchanged.
+ assert call_args[0][1] == ["doc1", "doc2", "doc3"]
+ assert call_args[0][2] == "my_collection"
+ assert call_args[0][3] == "alice"
+ assert call_args[0][4] == "secret123"
+
+ # The underlying worker has not run yet: the executor
+ # is mocked, so submit did not invoke the callable.
+ mock_worker.assert_not_called()
+
+ # Simulate the executor running the submitted job. The
+ # wrapper must forward the args to the real worker and
+ # then release the reserved slot.
+ submitted_callable(*call_args[0][1:])
+
+ mock_worker.assert_called_once_with(
+ ["doc1", "doc2", "doc3"],
+ "my_collection",
+ "alice",
+ "secret123",
+ )
+
+ # Slot released once the wrapped worker completed.
+ assert (
+ rag_module._pending_auto_index_jobs
+ == pending_before
+ )
def test_trigger_auto_index_with_single_document(self):
"""Test that auto-indexing works with a single document."""
@@ -256,6 +330,328 @@ class TestTriggerAutoIndex:
call_args = mock_executor.submit.call_args
assert len(call_args[0][1]) == 100
+ def test_trigger_auto_index_drops_job_when_queue_saturated(self):
+ """Saturated queue: the job is dropped, not submitted, and warned.
+
+ When ``_pending_auto_index_jobs`` is already at the bound, the
+ backpressure guard must refuse the new submission entirely: it must
+ NOT touch the executor (no ``submit``), it must emit a warning, and it
+ must leave the saturation counter untouched (no slot consumed, so the
+ upload path that called it is unaffected and still succeeded).
+ """
+ import local_deep_research.research_library.routes.rag_routes as rag_module
+ from local_deep_research.research_library.routes.rag_routes import (
+ trigger_auto_index,
+ )
+
+ with patch(
+ "local_deep_research.database.session_context.get_user_db_session"
+ ) as mock_session:
+ mock_db = MagicMock()
+ mock_session.return_value.__enter__.return_value = mock_db
+
+ mock_settings = MagicMock()
+ mock_settings.get_bool_setting.return_value = True
+
+ saturated = rag_module._MAX_PENDING_AUTO_INDEX_JOBS
+ original_pending = rag_module._pending_auto_index_jobs
+ rag_module._pending_auto_index_jobs = saturated
+ try:
+ with patch(
+ "local_deep_research.research_library.routes.rag_routes.SettingsManager",
+ return_value=mock_settings,
+ ):
+ with patch(
+ "local_deep_research.research_library.routes.rag_routes._get_auto_index_executor"
+ ) as mock_get_executor:
+ mock_executor = MagicMock()
+ mock_get_executor.return_value = mock_executor
+
+ with patch.object(
+ rag_module.logger, "warning"
+ ) as mock_warning:
+ # Must not raise; the upload already succeeded.
+ trigger_auto_index(
+ document_ids=["doc1", "doc2"],
+ collection_id="coll1",
+ username="testuser",
+ db_password="testpass",
+ )
+
+ # Job dropped: executor never touched, nothing queued.
+ mock_get_executor.assert_not_called()
+ mock_executor.submit.assert_not_called()
+
+ # A saturation warning was emitted, and it carries the
+ # diagnostic context an operator needs: the cap that was
+ # hit, the number of documents dropped, and which
+ # collection they belonged to.
+ mock_warning.assert_called_once()
+ warning_args = mock_warning.call_args.args
+ # The cap, the document count (2 docs passed), and the
+ # collection id must all be present in the positional
+ # args passed to logger.warning.
+ assert (
+ rag_module._MAX_PENDING_AUTO_INDEX_JOBS
+ in warning_args
+ )
+ assert 2 in warning_args # len(["doc1", "doc2"])
+ assert "coll1" in warning_args
+
+ # Counter unchanged: a dropped job consumes no slot.
+ assert rag_module._pending_auto_index_jobs == saturated
+ finally:
+ rag_module._pending_auto_index_jobs = original_pending
+
+ def test_trigger_auto_index_releases_slot_when_worker_raises(self):
+ """Leak prevention: a slot is released even if the worker raises.
+
+ The wrapped worker submitted to the executor wraps the real worker in
+ ``try/finally`` so that the reserved slot is always released. If this
+ did not happen, a worker that raised would permanently leak its slot
+ and eventually saturate the queue forever. This asserts the counter
+ returns to its baseline after the worker raises.
+ """
+ import local_deep_research.research_library.routes.rag_routes as rag_module
+ from local_deep_research.research_library.routes.rag_routes import (
+ trigger_auto_index,
+ )
+
+ with patch(
+ "local_deep_research.database.session_context.get_user_db_session"
+ ) as mock_session:
+ mock_db = MagicMock()
+ mock_session.return_value.__enter__.return_value = mock_db
+
+ mock_settings = MagicMock()
+ mock_settings.get_bool_setting.return_value = True
+
+ with patch(
+ "local_deep_research.research_library.routes.rag_routes.SettingsManager",
+ return_value=mock_settings,
+ ):
+ with patch(
+ "local_deep_research.research_library.routes.rag_routes._get_auto_index_executor"
+ ) as mock_get_executor:
+ mock_executor = MagicMock()
+ mock_get_executor.return_value = mock_executor
+
+ with patch(
+ "local_deep_research.research_library.routes.rag_routes._auto_index_documents_worker",
+ side_effect=RuntimeError("worker boom"),
+ ):
+ pending_before = rag_module._pending_auto_index_jobs
+
+ trigger_auto_index(
+ document_ids=["doc1"],
+ collection_id="coll1",
+ username="testuser",
+ db_password="testpass",
+ )
+
+ # Slot reserved while the (not yet run) wrapper holds it.
+ assert (
+ rag_module._pending_auto_index_jobs
+ == pending_before + 1
+ )
+
+ # Run the submitted wrapper; the underlying worker
+ # raises. The wrapper's finally must still release the
+ # slot, and the raise must not escape the wrapper in a
+ # way that prevents that release.
+ submitted_callable = mock_executor.submit.call_args[0][
+ 0
+ ]
+ submitted_args = mock_executor.submit.call_args[0][1:]
+ try:
+ submitted_callable(*submitted_args)
+ except RuntimeError:
+ # The executor would capture this in the Future;
+ # what matters is the slot was released first.
+ pass
+
+ # Slot released back to baseline despite the failure.
+ assert (
+ rag_module._pending_auto_index_jobs
+ == pending_before
+ )
+
+ def test_trigger_auto_index_releases_slot_and_swallows_submit_failure(
+ self,
+ ):
+ """Submit failure must not turn a committed upload into a 500.
+
+ If ``executor.submit`` itself fails (executor shut down, OOM, etc.)
+ the wrapped worker never runs, so ``trigger_auto_index`` must release
+ the reserved slot AND must NOT propagate the exception. Propagating
+ would bubble up to the upload handler and return a 500 even though the
+ documents were already committed, prompting the client to retry and
+ create duplicates.
+ """
+ import local_deep_research.research_library.routes.rag_routes as rag_module
+ from local_deep_research.research_library.routes.rag_routes import (
+ trigger_auto_index,
+ )
+
+ with patch(
+ "local_deep_research.database.session_context.get_user_db_session"
+ ) as mock_session:
+ mock_db = MagicMock()
+ mock_session.return_value.__enter__.return_value = mock_db
+
+ mock_settings = MagicMock()
+ mock_settings.get_bool_setting.return_value = True
+
+ with patch(
+ "local_deep_research.research_library.routes.rag_routes.SettingsManager",
+ return_value=mock_settings,
+ ):
+ with patch(
+ "local_deep_research.research_library.routes.rag_routes._get_auto_index_executor"
+ ) as mock_get_executor:
+ mock_executor = MagicMock()
+ mock_executor.submit.side_effect = RuntimeError(
+ "cannot schedule new futures after shutdown"
+ )
+ mock_get_executor.return_value = mock_executor
+
+ pending_before = rag_module._pending_auto_index_jobs
+
+ # Must return normally: the exception is swallowed so the
+ # already-committed upload is not turned into a failure.
+ trigger_auto_index(
+ document_ids=["doc1"],
+ collection_id="coll1",
+ username="testuser",
+ db_password="testpass",
+ )
+
+ # submit was attempted and failed.
+ mock_executor.submit.assert_called_once()
+
+ # Reserved slot was released despite the submit failure,
+ # so a failed submit does not leak a slot.
+ assert rag_module._pending_auto_index_jobs == pending_before
+
+ def test_trigger_auto_index_releases_slot_when_executor_build_fails(self):
+ """Slot must be released if _get_auto_index_executor() itself raises.
+
+ Building the executor (OS thread/mutex exhaustion) happens after the
+ slot is reserved. It must be inside the same try/except as submit, so
+ a failure there releases the slot and is swallowed — otherwise the
+ reserved slot leaks permanently and erodes the queue bound.
+ """
+ import local_deep_research.research_library.routes.rag_routes as rag_module
+ from local_deep_research.research_library.routes.rag_routes import (
+ trigger_auto_index,
+ )
+
+ with patch(
+ "local_deep_research.database.session_context.get_user_db_session"
+ ) as mock_session:
+ mock_db = MagicMock()
+ mock_session.return_value.__enter__.return_value = mock_db
+
+ mock_settings = MagicMock()
+ mock_settings.get_bool_setting.return_value = True
+
+ with patch(
+ "local_deep_research.research_library.routes.rag_routes.SettingsManager",
+ return_value=mock_settings,
+ ):
+ with patch(
+ "local_deep_research.research_library.routes.rag_routes._get_auto_index_executor",
+ side_effect=RuntimeError("cannot allocate thread"),
+ ):
+ pending_before = rag_module._pending_auto_index_jobs
+
+ # Must return normally: the exception is swallowed.
+ trigger_auto_index(
+ document_ids=["doc1"],
+ collection_id="coll1",
+ username="testuser",
+ db_password="testpass",
+ )
+
+ # Reserved slot was released despite the executor-build
+ # failure, so it does not leak a slot.
+ assert rag_module._pending_auto_index_jobs == pending_before
+
+ def test_trigger_auto_index_releases_slot_once_on_thread_start_race(self):
+ """Thread-start race must release the reserved slot EXACTLY once.
+
+ CPython's ThreadPoolExecutor.submit() enqueues the work item BEFORE it
+ tries to spin up a worker thread. If thread-start raises
+ RuntimeError("can't start new thread"), the item is already queued, so
+ a live worker can run the wrapped worker (which releases the slot in
+ its finally) AND the except block around submit() also fires. Without
+ an idempotent release that is a DOUBLE release of one reservation,
+ under-counting in-flight jobs and eroding the OOM bound.
+
+ We reproduce that race by making submit() RUN the passed worker (so its
+ finally releases) and THEN raise the thread-start RuntimeError (so the
+ except path also runs). The slot must be released exactly once.
+
+ Critically, we pre-set the baseline to a NON-ZERO value: at baseline 0
+ the ``max(0, ...)`` floor in ``_release_auto_index_slot`` would clamp a
+ double-release back to 0 and silently mask the bug. A non-zero baseline
+ makes the second (erroneous) release observable as baseline - 1.
+ """
+ import local_deep_research.research_library.routes.rag_routes as rag_module
+ from local_deep_research.research_library.routes.rag_routes import (
+ trigger_auto_index,
+ )
+
+ with patch(
+ "local_deep_research.database.session_context.get_user_db_session"
+ ) as mock_session:
+ mock_db = MagicMock()
+ mock_session.return_value.__enter__.return_value = mock_db
+
+ mock_settings = MagicMock()
+ mock_settings.get_bool_setting.return_value = True
+
+ # Non-zero baseline so a double-release is observable (the max(0,..)
+ # floor would otherwise mask it at baseline 0).
+ baseline = 5
+ with rag_module._pending_auto_index_lock:
+ rag_module._pending_auto_index_jobs = baseline
+
+ def submit_runs_then_raises(fn, *args, **kwargs):
+ # Mirror CPython: the work item is enqueued and a worker runs
+ # it (releasing the slot in the wrapped worker's finally)...
+ fn(*args, **kwargs)
+ # ...and THEN thread-start fails, so the except path fires too.
+ raise RuntimeError("can't start new thread")
+
+ with patch(
+ "local_deep_research.research_library.routes.rag_routes.SettingsManager",
+ return_value=mock_settings,
+ ):
+ with patch(
+ "local_deep_research.research_library.routes.rag_routes._get_auto_index_executor"
+ ) as mock_get_executor:
+ mock_executor = MagicMock()
+ mock_executor.submit.side_effect = submit_runs_then_raises
+ mock_get_executor.return_value = mock_executor
+
+ with patch(
+ "local_deep_research.research_library.routes.rag_routes._auto_index_documents_worker"
+ ):
+ # Must not raise: the submit failure is swallowed.
+ trigger_auto_index(
+ document_ids=["doc1"],
+ collection_id="coll1",
+ username="testuser",
+ db_password="testpass",
+ )
+
+ # The slot was released EXACTLY ONCE despite both the
+ # wrapped worker's finally and the except block firing, so
+ # the counter returns to exactly the baseline. If the
+ # release were not idempotent, it would be baseline - 1.
+ assert rag_module._pending_auto_index_jobs == baseline
+
class TestAutoIndexExecutor:
"""Test the ThreadPoolExecutor infrastructure for auto-indexing."""
@@ -304,6 +700,26 @@ class TestAutoIndexExecutor:
assert new_executor is not None
assert new_executor is not executor
+ def test_shutdown_resets_pending_jobs_counter(self):
+ """Shutdown must reset the pending-jobs counter.
+
+ Otherwise a re-created executor (tests / WSGI reload) inherits a
+ stale, possibly-saturated count and could refuse all future
+ submissions.
+ """
+ import local_deep_research.research_library.routes.rag_routes as rag_module
+ from local_deep_research.research_library.routes.rag_routes import (
+ _shutdown_auto_index_executor,
+ )
+
+ # Simulate slots left reserved (e.g. an interrupted shutdown).
+ with rag_module._pending_auto_index_lock:
+ rag_module._pending_auto_index_jobs = 7
+
+ _shutdown_auto_index_executor()
+
+ assert rag_module._pending_auto_index_jobs == 0
+
def test_executor_has_bounded_workers(self):
"""Test that the executor has bounded max_workers to prevent thread proliferation."""
from local_deep_research.research_library.routes.rag_routes import (
@@ -424,6 +840,69 @@ class TestAutoIndexExecutor:
# All should be the same instance
assert all(e is executors[0] for e in executors)
+ def test_reserve_slot_never_oversubscribes_under_contention(self):
+ """Concurrent reservations must never exceed the cap.
+
+ ``_try_reserve_auto_index_slot`` guards the counter with
+ ``_pending_auto_index_lock``. With the baseline set near the cap, only
+ ``cap - baseline`` reservations may succeed no matter how many threads
+ race. We fire N threads (N > the remaining capacity), each waiting on a
+ ``threading.Barrier`` so they hit the critical section as close to
+ simultaneously as possible, then assert:
+
+ * the number of successful reservations equals exactly the remaining
+ capacity (not more), and
+ * the final counter never exceeds the cap.
+
+ Mutation-awareness: if ``with _pending_auto_index_lock:`` is removed
+ from ``_try_reserve_auto_index_slot``, two threads can both read the
+ same pre-increment value and both succeed, over-subscribing past the
+ cap and failing this test. Note this is *probabilistic* — the lost
+ update only manifests on an actual interleaving — but the Barrier and a
+ tight remaining capacity (1) maximize contention to make it reliable.
+ """
+ import local_deep_research.research_library.routes.rag_routes as rag_module
+ from local_deep_research.research_library.routes.rag_routes import (
+ _try_reserve_auto_index_slot,
+ )
+
+ cap = rag_module._MAX_PENDING_AUTO_INDEX_JOBS
+ # Leave exactly ONE free slot so any lost update over-subscribes the
+ # cap and is unambiguously detectable.
+ remaining = 1
+ baseline = cap - remaining
+
+ n_threads = 16
+ with rag_module._pending_auto_index_lock:
+ rag_module._pending_auto_index_jobs = baseline
+
+ barrier = threading.Barrier(n_threads)
+ results = []
+ results_lock = threading.Lock()
+
+ def contend():
+ barrier.wait(timeout=5) # all threads collide here
+ ok = _try_reserve_auto_index_slot()
+ with results_lock:
+ results.append(ok)
+
+ threads = [threading.Thread(target=contend) for _ in range(n_threads)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join(timeout=10)
+
+ successes = sum(1 for r in results if r)
+
+ # Exactly the remaining capacity may be granted — never more.
+ assert successes == remaining, (
+ f"expected {remaining} successful reservation(s), got {successes}"
+ )
+ # The counter must never exceed the cap.
+ assert rag_module._pending_auto_index_jobs <= cap
+ # And it should land exactly at the cap (baseline + remaining grants).
+ assert rag_module._pending_auto_index_jobs == baseline + remaining
+
def test_executor_handles_worker_exceptions(self):
"""Test that worker exceptions don't break the executor."""
from local_deep_research.research_library.routes.rag_routes import (
diff --git a/tests/research_library/routes/test_rag_routes_indexing_coverage.py b/tests/research_library/routes/test_rag_routes_indexing_coverage.py
index bf56a7c207..a5aa3ec002 100644
--- a/tests/research_library/routes/test_rag_routes_indexing_coverage.py
+++ b/tests/research_library/routes/test_rag_routes_indexing_coverage.py
@@ -380,6 +380,153 @@ class TestTriggerAutoIndex:
# Executor must not be called when settings check fails
mock_executor_fn.assert_not_called()
+ def test_trigger_auto_index_drops_when_queue_saturated(self):
+ """When the pending-jobs counter hits the cap, new submissions
+ are dropped and the executor is never called."""
+ import local_deep_research.research_library.routes.rag_routes as mod
+ from local_deep_research.research_library.routes.rag_routes import (
+ trigger_auto_index,
+ )
+
+ mock_sm = Mock()
+ mock_sm.get_bool_setting.return_value = True # enabled
+ db_session = _make_db_session()
+
+ @contextmanager
+ def fake_session(*a, **kw):
+ yield db_session
+
+ # Saturate the counter to its cap so the next submission is dropped.
+ original_pending = mod._pending_auto_index_jobs
+ mod._pending_auto_index_jobs = mod._MAX_PENDING_AUTO_INDEX_JOBS
+
+ try:
+ with (
+ patch(
+ f"{_DB_CTX}.get_user_db_session", side_effect=fake_session
+ ),
+ patch(f"{MODULE}.SettingsManager", return_value=mock_sm),
+ patch(f"{MODULE}._get_auto_index_executor") as mock_executor_fn,
+ ):
+ trigger_auto_index(["doc-1"], "coll-1", "testuser", "pass")
+ mock_executor_fn.assert_not_called()
+ # Counter must NOT have been bumped (or leaked) when dropped.
+ assert (
+ mod._pending_auto_index_jobs == mod._MAX_PENDING_AUTO_INDEX_JOBS
+ )
+ finally:
+ mod._pending_auto_index_jobs = original_pending
+
+ def test_trigger_auto_index_releases_slot_after_worker(self):
+ """Wrapped worker releases its queue slot when the underlying
+ worker raises — preventing counter leaks that would eventually
+ block all auto-indexing."""
+ import local_deep_research.research_library.routes.rag_routes as mod
+ from local_deep_research.research_library.routes.rag_routes import (
+ trigger_auto_index,
+ )
+
+ mock_sm = Mock()
+ mock_sm.get_bool_setting.return_value = True
+ db_session = _make_db_session()
+
+ @contextmanager
+ def fake_session(*a, **kw):
+ yield db_session
+
+ # Synchronous "executor" that faithfully emulates a real
+ # ThreadPoolExecutor: a worker exception is captured (it would live in
+ # the returned Future) rather than escaping submit(). This is what the
+ # production executor does, so the worker exception must NOT propagate
+ # out of trigger_auto_index.
+ captured = {}
+
+ class _SyncExecutor:
+ def submit(self, fn, *args, **kwargs):
+ captured["called"] = True
+ try:
+ fn(*args, **kwargs)
+ except Exception as exc: # captured into the (mock) future
+ captured["worker_exc"] = exc
+
+ original_pending = mod._pending_auto_index_jobs
+ mod._pending_auto_index_jobs = 0
+
+ try:
+ with (
+ patch(
+ f"{_DB_CTX}.get_user_db_session", side_effect=fake_session
+ ),
+ patch(f"{MODULE}.SettingsManager", return_value=mock_sm),
+ patch(
+ f"{MODULE}._get_auto_index_executor",
+ return_value=_SyncExecutor(),
+ ),
+ patch(
+ f"{MODULE}._auto_index_documents_worker",
+ side_effect=RuntimeError("worker boom"),
+ ),
+ ):
+ # Must not raise: a worker failure stays inside the future,
+ # exactly as with the real ThreadPoolExecutor.
+ trigger_auto_index(["doc-1"], "coll-1", "testuser", "pass")
+
+ assert captured.get("called") is True
+ # The worker did raise (captured in the future).
+ assert isinstance(captured.get("worker_exc"), RuntimeError)
+ # The slot must be released by the wrapper's finally so the counter
+ # is back to zero (otherwise auto-indexing would silently
+ # block forever once 100 worker exceptions accumulated).
+ assert mod._pending_auto_index_jobs == 0
+ finally:
+ mod._pending_auto_index_jobs = original_pending
+
+ def test_trigger_auto_index_releases_slot_when_submit_fails(self):
+ """If executor.submit() raises (e.g., shutting down), the slot is
+ released rather than leaked AND the failure is swallowed.
+
+ The upload has already been committed by the caller before
+ trigger_auto_index runs, so propagating a submit failure would turn a
+ successful upload into a 500 and prompt duplicate retries. The failure
+ must instead be logged and auto-indexing simply skipped.
+ """
+ import local_deep_research.research_library.routes.rag_routes as mod
+ from local_deep_research.research_library.routes.rag_routes import (
+ trigger_auto_index,
+ )
+
+ mock_sm = Mock()
+ mock_sm.get_bool_setting.return_value = True
+ db_session = _make_db_session()
+
+ @contextmanager
+ def fake_session(*a, **kw):
+ yield db_session
+
+ broken_executor = Mock()
+ broken_executor.submit.side_effect = RuntimeError("executor shut down")
+
+ original_pending = mod._pending_auto_index_jobs
+ mod._pending_auto_index_jobs = 0
+
+ try:
+ with (
+ patch(
+ f"{_DB_CTX}.get_user_db_session", side_effect=fake_session
+ ),
+ patch(f"{MODULE}.SettingsManager", return_value=mock_sm),
+ patch(
+ f"{MODULE}._get_auto_index_executor",
+ return_value=broken_executor,
+ ),
+ ):
+ # Must not raise: the committed upload must not become a 500.
+ trigger_auto_index(["doc-1"], "coll-1", "testuser", "pass")
+ broken_executor.submit.assert_called_once()
+ assert mod._pending_auto_index_jobs == 0
+ finally:
+ mod._pending_auto_index_jobs = original_pending
+
# ---------------------------------------------------------------------------
# _background_index_worker