~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(security): bound the RAG auto-index queue to prevent OOM under burst (#3939)
LearningCircuit89fb1b31e5d6
diff --git a/changelog.d/+rag-auto-index-backpressure.security.md b/changelog.d/+rag-auto-index-backpressure.security.mdnew file mode 100644index 0000000000..878e090cbc--- /dev/null+++ b/changelog.d/+rag-auto-index-backpressure.security.md@@ -0,0 +1 @@+RAG auto-indexing now applies backpressure to its background worker pool. The `ThreadPoolExecutor` previously had an unbounded internal queue; under sustained upload bursts (possible under the configurable upload rate cap from #3935), this could queue thousands of indexing jobs and exhaust memory. The pool is now capped at 100 in-flight + queued jobs; submissions over the cap are dropped with a clear log warning ("Auto-index queue saturated… trigger a manual reindex if needed"). Documents still upload successfully — only the *automatic* indexing is skipped, and users can manually reindex via the UI.diff --git a/src/local_deep_research/research_library/routes/rag_routes.py b/src/local_deep_research/research_library/routes/rag_routes.pyindex 4bd8404e1b..cbb5a40eda 100644--- a/src/local_deep_research/research_library/routes/rag_routes.py+++ b/src/local_deep_research/research_library/routes/rag_routes.py@@ -87,10 +87,26 @@ def _agent_enabled_default_on(collection) -> bool:return getattr(collection, "agent_enabled", True) is not False-# Global ThreadPoolExecutor for auto-indexing to prevent thread proliferation+# Global ThreadPoolExecutor for auto-indexing to prevent thread proliferation.+#+# ThreadPoolExecutor's internal _work_queue is unbounded. Without+# backpressure, a sustained burst of uploads (possible under the+# configurable upload rate cap from #3935) could queue thousands of indexing+# jobs in memory, exhausting RAM. We track pending submissions in a counter+# and reject new submissions once the queue is saturated._auto_index_executor: ThreadPoolExecutor | None = None_auto_index_executor_lock = threading.Lock()+# Maximum in-flight + queued indexing jobs. Each job is one upload batch+# (an unbounded list of document IDs), so per-job duration varies with batch+# size; 100 is a buffer on the number of *queued batches*, not documents.+# The OOM bound still holds because a queued job holds only a small list of+# ID strings, not document contents. Tunable if real-world workloads need+# more headroom.+_MAX_PENDING_AUTO_INDEX_JOBS = 100+_pending_auto_index_jobs = 0+_pending_auto_index_lock = threading.Lock()+def _get_auto_index_executor() -> ThreadPoolExecutor:"""Get or create the global auto-indexing executor (thread-safe)."""@@ -104,12 +120,46 @@ def _get_auto_index_executor() -> ThreadPoolExecutor:return _auto_index_executor+def _try_reserve_auto_index_slot() -> bool:+ """Reserve a queue slot if capacity allows. Returns True on success.++ Caller MUST call ``_release_auto_index_slot`` when the job completes+ (success OR failure), otherwise the counter leaks and eventually+ blocks all submissions.+ """+ global _pending_auto_index_jobs+ with _pending_auto_index_lock:+ if _pending_auto_index_jobs >= _MAX_PENDING_AUTO_INDEX_JOBS:+ return False+ _pending_auto_index_jobs += 1+ return True+++def _release_auto_index_slot() -> None:+ """Release a previously reserved queue slot."""+ global _pending_auto_index_jobs+ with _pending_auto_index_lock:+ _pending_auto_index_jobs = max(0, _pending_auto_index_jobs - 1)++def _shutdown_auto_index_executor() -> None:- """Shutdown the auto-index executor gracefully."""- global _auto_index_executor- if _auto_index_executor is not None:- _auto_index_executor.shutdown(wait=True)+ """Shutdown the auto-index executor gracefully.++ Holds ``_auto_index_executor_lock`` for the read+nullify so two+ concurrent callers (e.g. atexit + a test teardown) can't race into an+ ``AttributeError`` on the ``None`` reference; the blocking ``shutdown``+ runs outside the lock. Resets the pending-jobs counter afterwards so a+ re-created executor (tests, WSGI reload) starts from a clean count+ instead of inheriting a stale, possibly-saturated value.+ """+ global _auto_index_executor, _pending_auto_index_jobs+ with _auto_index_executor_lock:+ executor = _auto_index_executor_auto_index_executor = None+ if executor is not None:+ executor.shutdown(wait=True)+ with _pending_auto_index_lock:+ _pending_auto_index_jobs = 0atexit.register(_shutdown_auto_index_executor)@@ -2420,19 +2470,80 @@ def trigger_auto_index()return+ # Reserve a queue slot before submission. If the indexing queue is+ # saturated (too many uploads in flight), drop this auto-index job+ # rather than letting the executor's unbounded internal queue grow+ # without bound. The user can reindex manually later via the UI.+ if not _try_reserve_auto_index_slot():+ logger.warning(+ "Auto-index queue saturated ({}+ jobs pending); dropping "+ "auto-index for {} document(s) in collection {}. "+ "Documents are uploaded; trigger a manual reindex if needed.",+ _MAX_PENDING_AUTO_INDEX_JOBS,+ len(document_ids),+ collection_id,+ )+ return+logger.info(f"Auto-indexing {len(document_ids)} documents in collection {collection_id}")- # Submit to thread pool (bounded concurrency, prevents thread proliferation)- executor = _get_auto_index_executor()- executor.submit(- _auto_index_documents_worker,- document_ids,- collection_id,- username,- db_password,- )+ # Release the reserved slot EXACTLY ONCE per submission. CPython's+ # ThreadPoolExecutor.submit() enqueues the work item BEFORE it tries to+ # spin up a worker thread; if thread-start raises (e.g.+ # RuntimeError("can't start new thread")), the item is already queued, so+ # a live worker can run _wrapped_worker (releasing in its finally) AND the+ # except block below also fires. A per-call lock + flag makes the release+ # idempotent so that double-fire only releases once — otherwise the+ # counter under-counts in-flight jobs and erodes the OOM bound (the+ # max(0, ...) floor would silently hide the drift).+ _release_lock = threading.Lock()+ _slot_released = {"done": False}++ def _release_slot_once():+ with _release_lock:+ if _slot_released["done"]:+ return+ _slot_released["done"] = True+ _release_auto_index_slot()++ def _wrapped_worker(*args, **kwargs):+ try:+ _auto_index_documents_worker(*args, **kwargs)+ finally:+ _release_slot_once()++ # Submit to thread pool (bounded concurrency, prevents thread proliferation).+ # Build the executor inside the try too: if _get_auto_index_executor()+ # itself fails (OS thread/mutex exhaustion), the slot must still be+ # released — otherwise a reserved slot leaks permanently.+ try:+ executor = _get_auto_index_executor()+ executor.submit(+ _wrapped_worker,+ document_ids,+ collection_id,+ username,+ db_password,+ )+ except Exception:+ # If submit itself fails (executor shutting down, OOM, etc.), the+ # wrapped worker may never run, so release the slot here. The release+ # is idempotent: if the work item was already enqueued and a worker+ # ran (and released) before thread-start failed, this is a no-op. Do+ # NOT re-raise: the upload has already been committed by the caller,+ # so propagating would turn a successful upload into a 500 (and prompt+ # the client to retry, creating duplicates). Auto-indexing is simply+ # skipped; the user can trigger a manual reindex later via the UI.+ _release_slot_once()+ logger.exception(+ "Failed to submit auto-index job for {} document(s) in "+ "collection {}; upload succeeded, auto-indexing skipped. "+ "Trigger a manual reindex if needed.",+ len(document_ids),+ collection_id,+ )@thread_cleanupdiff --git a/tests/library/test_auto_indexing.py b/tests/library/test_auto_indexing.pyindex 29c4804988..c5c09fb322 100644--- a/tests/library/test_auto_indexing.py+++ b/tests/library/test_auto_indexing.py@@ -4,6 +4,30 @@ import threadingimport timefrom unittest.mock import MagicMock, patch+import pytest++[email protected](autouse=True)+def _reset_pending_auto_index_jobs():+ """Save/restore the module-global pending-jobs counter around each test.++ Several tests mock ``_get_auto_index_executor`` with a no-op ``submit``,+ which reserves a slot (``_pending_auto_index_jobs += 1``) that the (never+ run) worker never releases. Without this fixture the counter leaks +1 per+ such test and the saturation/concurrency tests become order-dependent.+ This restores the counter to its pre-test value after every test, keeping+ the suite order-independent.+ """+ import local_deep_research.research_library.routes.rag_routes as rag_module++ with rag_module._pending_auto_index_lock:+ saved = rag_module._pending_auto_index_jobs+ try:+ yield+ finally:+ with rag_module._pending_auto_index_lock:+ rag_module._pending_auto_index_jobs = saved+class TestAutoIndexingSetting:"""Test the auto-indexing setting."""@@ -143,10 +167,20 @@ class TestTriggerAutoIndex:mock_get_executor.assert_not_called()def test_trigger_auto_index_passes_correct_arguments(self):- """Test that trigger_auto_index passes correct arguments to the worker."""+ """Backpressure wrapper still delegates correct args to the worker.++ The backpressure change submits a ``_wrapped_worker`` closure rather+ than ``_auto_index_documents_worker`` directly. This test verifies the+ full contract of that wrapper:++ * a queue slot is reserved before the job is submitted,+ * the submitted callable still forwards the exact arguments to the+ underlying ``_auto_index_documents_worker``, and+ * the reserved slot is released once the wrapped worker finishes.+ """+ import local_deep_research.research_library.routes.rag_routes as rag_modulefrom local_deep_research.research_library.routes.rag_routes import (trigger_auto_index,- _auto_index_documents_worker,)with patch(@@ -168,21 +202,61 @@ class TestTriggerAutoIndex:mock_executor = MagicMock()mock_get_executor.return_value = mock_executor- trigger_auto_index(- document_ids=["doc1", "doc2", "doc3"],- collection_id="my_collection",- username="alice",- db_password="secret123",- )+ with patch(+ "local_deep_research.research_library.routes.rag_routes._auto_index_documents_worker"+ ) as mock_worker:+ pending_before = rag_module._pending_auto_index_jobs- # Verify correct arguments passed to submit- mock_executor.submit.assert_called_once()- call_args = mock_executor.submit.call_args- assert call_args[0][0] == _auto_index_documents_worker- assert call_args[0][1] == ["doc1", "doc2", "doc3"]- assert call_args[0][2] == "my_collection"- assert call_args[0][3] == "alice"- assert call_args[0][4] == "secret123"+ trigger_auto_index(+ document_ids=["doc1", "doc2", "doc3"],+ collection_id="my_collection",+ username="alice",+ db_password="secret123",+ )++ # A queue slot must be reserved before submission, so+ # the pending counter is incremented while the (not yet+ # run) wrapped worker holds the slot.+ assert (+ rag_module._pending_auto_index_jobs+ == pending_before + 1+ )++ # A wrapper closure is submitted, NOT the worker+ # directly (that is the whole point of the wrapper).+ mock_executor.submit.assert_called_once()+ call_args = mock_executor.submit.call_args+ submitted_callable = call_args[0][0]+ assert submitted_callable is not mock_worker+ assert callable(submitted_callable)++ # The wrapper forwards the original arguments unchanged.+ assert call_args[0][1] == ["doc1", "doc2", "doc3"]+ assert call_args[0][2] == "my_collection"+ assert call_args[0][3] == "alice"+ assert call_args[0][4] == "secret123"++ # The underlying worker has not run yet: the executor+ # is mocked, so submit did not invoke the callable.+ mock_worker.assert_not_called()++ # Simulate the executor running the submitted job. The+ # wrapper must forward the args to the real worker and+ # then release the reserved slot.+ submitted_callable(*call_args[0][1:])++ mock_worker.assert_called_once_with(+ ["doc1", "doc2", "doc3"],+ "my_collection",+ "alice",+ "secret123",+ )++ # Slot released once the wrapped worker completed.+ assert (+ rag_module._pending_auto_index_jobs+ == pending_before+ )def test_trigger_auto_index_with_single_document(self):"""Test that auto-indexing works with a single document."""@@ -256,6 +330,328 @@ class TestTriggerAutoIndex:call_args = mock_executor.submit.call_argsassert len(call_args[0][1]) == 100+ def test_trigger_auto_index_drops_job_when_queue_saturated(self):+ """Saturated queue: the job is dropped, not submitted, and warned.++ When ``_pending_auto_index_jobs`` is already at the bound, the+ backpressure guard must refuse the new submission entirely: it must+ NOT touch the executor (no ``submit``), it must emit a warning, and it+ must leave the saturation counter untouched (no slot consumed, so the+ upload path that called it is unaffected and still succeeded).+ """+ import local_deep_research.research_library.routes.rag_routes as rag_module+ from local_deep_research.research_library.routes.rag_routes import (+ trigger_auto_index,+ )++ with patch(+ "local_deep_research.database.session_context.get_user_db_session"+ ) as mock_session:+ mock_db = MagicMock()+ mock_session.return_value.__enter__.return_value = mock_db++ mock_settings = MagicMock()+ mock_settings.get_bool_setting.return_value = True++ saturated = rag_module._MAX_PENDING_AUTO_INDEX_JOBS+ original_pending = rag_module._pending_auto_index_jobs+ rag_module._pending_auto_index_jobs = saturated+ try:+ with patch(+ "local_deep_research.research_library.routes.rag_routes.SettingsManager",+ return_value=mock_settings,+ ):+ with patch(+ "local_deep_research.research_library.routes.rag_routes._get_auto_index_executor"+ ) as mock_get_executor:+ mock_executor = MagicMock()+ mock_get_executor.return_value = mock_executor++ with patch.object(+ rag_module.logger, "warning"+ ) as mock_warning:+ # Must not raise; the upload already succeeded.+ trigger_auto_index(+ document_ids=["doc1", "doc2"],+ collection_id="coll1",+ username="testuser",+ db_password="testpass",+ )++ # Job dropped: executor never touched, nothing queued.+ mock_get_executor.assert_not_called()+ mock_executor.submit.assert_not_called()++ # A saturation warning was emitted, and it carries the+ # diagnostic context an operator needs: the cap that was+ # hit, the number of documents dropped, and which+ # collection they belonged to.+ mock_warning.assert_called_once()+ warning_args = mock_warning.call_args.args+ # The cap, the document count (2 docs passed), and the+ # collection id must all be present in the positional+ # args passed to logger.warning.+ assert (+ rag_module._MAX_PENDING_AUTO_INDEX_JOBS+ in warning_args+ )+ assert 2 in warning_args # len(["doc1", "doc2"])+ assert "coll1" in warning_args++ # Counter unchanged: a dropped job consumes no slot.+ assert rag_module._pending_auto_index_jobs == saturated+ finally:+ rag_module._pending_auto_index_jobs = original_pending++ def test_trigger_auto_index_releases_slot_when_worker_raises(self):+ """Leak prevention: a slot is released even if the worker raises.++ The wrapped worker submitted to the executor wraps the real worker in+ ``try/finally`` so that the reserved slot is always released. If this+ did not happen, a worker that raised would permanently leak its slot+ and eventually saturate the queue forever. This asserts the counter+ returns to its baseline after the worker raises.+ """+ import local_deep_research.research_library.routes.rag_routes as rag_module+ from local_deep_research.research_library.routes.rag_routes import (+ trigger_auto_index,+ )++ with patch(+ "local_deep_research.database.session_context.get_user_db_session"+ ) as mock_session:+ mock_db = MagicMock()+ mock_session.return_value.__enter__.return_value = mock_db++ mock_settings = MagicMock()+ mock_settings.get_bool_setting.return_value = True++ with patch(+ "local_deep_research.research_library.routes.rag_routes.SettingsManager",+ return_value=mock_settings,+ ):+ with patch(+ "local_deep_research.research_library.routes.rag_routes._get_auto_index_executor"+ ) as mock_get_executor:+ mock_executor = MagicMock()+ mock_get_executor.return_value = mock_executor++ with patch(+ "local_deep_research.research_library.routes.rag_routes._auto_index_documents_worker",+ side_effect=RuntimeError("worker boom"),+ ):+ pending_before = rag_module._pending_auto_index_jobs++ trigger_auto_index(+ document_ids=["doc1"],+ collection_id="coll1",+ username="testuser",+ db_password="testpass",+ )++ # Slot reserved while the (not yet run) wrapper holds it.+ assert (+ rag_module._pending_auto_index_jobs+ == pending_before + 1+ )++ # Run the submitted wrapper; the underlying worker+ # raises. The wrapper's finally must still release the+ # slot, and the raise must not escape the wrapper in a+ # way that prevents that release.+ submitted_callable = mock_executor.submit.call_args[0][+ 0+ ]+ submitted_args = mock_executor.submit.call_args[0][1:]+ try:+ submitted_callable(*submitted_args)+ except RuntimeError:+ # The executor would capture this in the Future;+ # what matters is the slot was released first.+ pass++ # Slot released back to baseline despite the failure.+ assert (+ rag_module._pending_auto_index_jobs+ == pending_before+ )++ def test_trigger_auto_index_releases_slot_and_swallows_submit_failure(+ self,+ ):+ """Submit failure must not turn a committed upload into a 500.++ If ``executor.submit`` itself fails (executor shut down, OOM, etc.)+ the wrapped worker never runs, so ``trigger_auto_index`` must release+ the reserved slot AND must NOT propagate the exception. Propagating+ would bubble up to the upload handler and return a 500 even though the+ documents were already committed, prompting the client to retry and+ create duplicates.+ """+ import local_deep_research.research_library.routes.rag_routes as rag_module+ from local_deep_research.research_library.routes.rag_routes import (+ trigger_auto_index,+ )++ with patch(+ "local_deep_research.database.session_context.get_user_db_session"+ ) as mock_session:+ mock_db = MagicMock()+ mock_session.return_value.__enter__.return_value = mock_db++ mock_settings = MagicMock()+ mock_settings.get_bool_setting.return_value = True++ with patch(+ "local_deep_research.research_library.routes.rag_routes.SettingsManager",+ return_value=mock_settings,+ ):+ with patch(+ "local_deep_research.research_library.routes.rag_routes._get_auto_index_executor"+ ) as mock_get_executor:+ mock_executor = MagicMock()+ mock_executor.submit.side_effect = RuntimeError(+ "cannot schedule new futures after shutdown"+ )+ mock_get_executor.return_value = mock_executor++ pending_before = rag_module._pending_auto_index_jobs++ # Must return normally: the exception is swallowed so the+ # already-committed upload is not turned into a failure.+ trigger_auto_index(+ document_ids=["doc1"],+ collection_id="coll1",+ username="testuser",+ db_password="testpass",+ )++ # submit was attempted and failed.+ mock_executor.submit.assert_called_once()++ # Reserved slot was released despite the submit failure,+ # so a failed submit does not leak a slot.+ assert rag_module._pending_auto_index_jobs == pending_before++ def test_trigger_auto_index_releases_slot_when_executor_build_fails(self):+ """Slot must be released if _get_auto_index_executor() itself raises.++ Building the executor (OS thread/mutex exhaustion) happens after the+ slot is reserved. It must be inside the same try/except as submit, so+ a failure there releases the slot and is swallowed — otherwise the+ reserved slot leaks permanently and erodes the queue bound.+ """+ import local_deep_research.research_library.routes.rag_routes as rag_module+ from local_deep_research.research_library.routes.rag_routes import (+ trigger_auto_index,+ )++ with patch(+ "local_deep_research.database.session_context.get_user_db_session"+ ) as mock_session:+ mock_db = MagicMock()+ mock_session.return_value.__enter__.return_value = mock_db++ mock_settings = MagicMock()+ mock_settings.get_bool_setting.return_value = True++ with patch(+ "local_deep_research.research_library.routes.rag_routes.SettingsManager",+ return_value=mock_settings,+ ):+ with patch(+ "local_deep_research.research_library.routes.rag_routes._get_auto_index_executor",+ side_effect=RuntimeError("cannot allocate thread"),+ ):+ pending_before = rag_module._pending_auto_index_jobs++ # Must return normally: the exception is swallowed.+ trigger_auto_index(+ document_ids=["doc1"],+ collection_id="coll1",+ username="testuser",+ db_password="testpass",+ )++ # Reserved slot was released despite the executor-build+ # failure, so it does not leak a slot.+ assert rag_module._pending_auto_index_jobs == pending_before++ def test_trigger_auto_index_releases_slot_once_on_thread_start_race(self):+ """Thread-start race must release the reserved slot EXACTLY once.++ CPython's ThreadPoolExecutor.submit() enqueues the work item BEFORE it+ tries to spin up a worker thread. If thread-start raises+ RuntimeError("can't start new thread"), the item is already queued, so+ a live worker can run the wrapped worker (which releases the slot in+ its finally) AND the except block around submit() also fires. Without+ an idempotent release that is a DOUBLE release of one reservation,+ under-counting in-flight jobs and eroding the OOM bound.++ We reproduce that race by making submit() RUN the passed worker (so its+ finally releases) and THEN raise the thread-start RuntimeError (so the+ except path also runs). The slot must be released exactly once.++ Critically, we pre-set the baseline to a NON-ZERO value: at baseline 0+ the ``max(0, ...)`` floor in ``_release_auto_index_slot`` would clamp a+ double-release back to 0 and silently mask the bug. A non-zero baseline+ makes the second (erroneous) release observable as baseline - 1.+ """+ import local_deep_research.research_library.routes.rag_routes as rag_module+ from local_deep_research.research_library.routes.rag_routes import (+ trigger_auto_index,+ )++ with patch(+ "local_deep_research.database.session_context.get_user_db_session"+ ) as mock_session:+ mock_db = MagicMock()+ mock_session.return_value.__enter__.return_value = mock_db++ mock_settings = MagicMock()+ mock_settings.get_bool_setting.return_value = True++ # Non-zero baseline so a double-release is observable (the max(0,..)+ # floor would otherwise mask it at baseline 0).+ baseline = 5+ with rag_module._pending_auto_index_lock:+ rag_module._pending_auto_index_jobs = baseline++ def submit_runs_then_raises(fn, *args, **kwargs):+ # Mirror CPython: the work item is enqueued and a worker runs+ # it (releasing the slot in the wrapped worker's finally)...+ fn(*args, **kwargs)+ # ...and THEN thread-start fails, so the except path fires too.+ raise RuntimeError("can't start new thread")++ with patch(+ "local_deep_research.research_library.routes.rag_routes.SettingsManager",+ return_value=mock_settings,+ ):+ with patch(+ "local_deep_research.research_library.routes.rag_routes._get_auto_index_executor"+ ) as mock_get_executor:+ mock_executor = MagicMock()+ mock_executor.submit.side_effect = submit_runs_then_raises+ mock_get_executor.return_value = mock_executor++ with patch(+ "local_deep_research.research_library.routes.rag_routes._auto_index_documents_worker"+ ):+ # Must not raise: the submit failure is swallowed.+ trigger_auto_index(+ document_ids=["doc1"],+ collection_id="coll1",+ username="testuser",+ db_password="testpass",+ )++ # The slot was released EXACTLY ONCE despite both the+ # wrapped worker's finally and the except block firing, so+ # the counter returns to exactly the baseline. If the+ # release were not idempotent, it would be baseline - 1.+ assert rag_module._pending_auto_index_jobs == baseline+class TestAutoIndexExecutor:"""Test the ThreadPoolExecutor infrastructure for auto-indexing."""@@ -304,6 +700,26 @@ class TestAutoIndexExecutor:assert new_executor is not Noneassert new_executor is not executor+ def test_shutdown_resets_pending_jobs_counter(self):+ """Shutdown must reset the pending-jobs counter.++ Otherwise a re-created executor (tests / WSGI reload) inherits a+ stale, possibly-saturated count and could refuse all future+ submissions.+ """+ import local_deep_research.research_library.routes.rag_routes as rag_module+ from local_deep_research.research_library.routes.rag_routes import (+ _shutdown_auto_index_executor,+ )++ # Simulate slots left reserved (e.g. an interrupted shutdown).+ with rag_module._pending_auto_index_lock:+ rag_module._pending_auto_index_jobs = 7++ _shutdown_auto_index_executor()++ assert rag_module._pending_auto_index_jobs == 0+def test_executor_has_bounded_workers(self):"""Test that the executor has bounded max_workers to prevent thread proliferation."""from local_deep_research.research_library.routes.rag_routes import (@@ -424,6 +840,69 @@ class TestAutoIndexExecutor:# All should be the same instanceassert all(e is executors[0] for e in executors)+ def test_reserve_slot_never_oversubscribes_under_contention(self):+ """Concurrent reservations must never exceed the cap.++ ``_try_reserve_auto_index_slot`` guards the counter with+ ``_pending_auto_index_lock``. With the baseline set near the cap, only+ ``cap - baseline`` reservations may succeed no matter how many threads+ race. We fire N threads (N > the remaining capacity), each waiting on a+ ``threading.Barrier`` so they hit the critical section as close to+ simultaneously as possible, then assert:++ * the number of successful reservations equals exactly the remaining+ capacity (not more), and+ * the final counter never exceeds the cap.++ Mutation-awareness: if ``with _pending_auto_index_lock:`` is removed+ from ``_try_reserve_auto_index_slot``, two threads can both read the+ same pre-increment value and both succeed, over-subscribing past the+ cap and failing this test. Note this is *probabilistic* — the lost+ update only manifests on an actual interleaving — but the Barrier and a+ tight remaining capacity (1) maximize contention to make it reliable.+ """+ import local_deep_research.research_library.routes.rag_routes as rag_module+ from local_deep_research.research_library.routes.rag_routes import (+ _try_reserve_auto_index_slot,+ )++ cap = rag_module._MAX_PENDING_AUTO_INDEX_JOBS+ # Leave exactly ONE free slot so any lost update over-subscribes the+ # cap and is unambiguously detectable.+ remaining = 1+ baseline = cap - remaining++ n_threads = 16+ with rag_module._pending_auto_index_lock:+ rag_module._pending_auto_index_jobs = baseline++ barrier = threading.Barrier(n_threads)+ results = []+ results_lock = threading.Lock()++ def contend():+ barrier.wait(timeout=5) # all threads collide here+ ok = _try_reserve_auto_index_slot()+ with results_lock:+ results.append(ok)++ threads = [threading.Thread(target=contend) for _ in range(n_threads)]+ for t in threads:+ t.start()+ for t in threads:+ t.join(timeout=10)++ successes = sum(1 for r in results if r)++ # Exactly the remaining capacity may be granted — never more.+ assert successes == remaining, (+ f"expected {remaining} successful reservation(s), got {successes}"+ )+ # The counter must never exceed the cap.+ assert rag_module._pending_auto_index_jobs <= cap+ # And it should land exactly at the cap (baseline + remaining grants).+ assert rag_module._pending_auto_index_jobs == baseline + remaining+def test_executor_handles_worker_exceptions(self):"""Test that worker exceptions don't break the executor."""from local_deep_research.research_library.routes.rag_routes import (diff --git a/tests/research_library/routes/test_rag_routes_indexing_coverage.py b/tests/research_library/routes/test_rag_routes_indexing_coverage.pyindex bf56a7c207..a5aa3ec002 100644--- a/tests/research_library/routes/test_rag_routes_indexing_coverage.py+++ b/tests/research_library/routes/test_rag_routes_indexing_coverage.py@@ -380,6 +380,153 @@ class TestTriggerAutoIndex:# Executor must not be called when settings check failsmock_executor_fn.assert_not_called()+ def test_trigger_auto_index_drops_when_queue_saturated(self):+ """When the pending-jobs counter hits the cap, new submissions+ are dropped and the executor is never called."""+ import local_deep_research.research_library.routes.rag_routes as mod+ from local_deep_research.research_library.routes.rag_routes import (+ trigger_auto_index,+ )++ mock_sm = Mock()+ mock_sm.get_bool_setting.return_value = True # enabled+ db_session = _make_db_session()++ @contextmanager+ def fake_session(*a, **kw):+ yield db_session++ # Saturate the counter to its cap so the next submission is dropped.+ original_pending = mod._pending_auto_index_jobs+ mod._pending_auto_index_jobs = mod._MAX_PENDING_AUTO_INDEX_JOBS++ try:+ with (+ patch(+ f"{_DB_CTX}.get_user_db_session", side_effect=fake_session+ ),+ patch(f"{MODULE}.SettingsManager", return_value=mock_sm),+ patch(f"{MODULE}._get_auto_index_executor") as mock_executor_fn,+ ):+ trigger_auto_index(["doc-1"], "coll-1", "testuser", "pass")+ mock_executor_fn.assert_not_called()+ # Counter must NOT have been bumped (or leaked) when dropped.+ assert (+ mod._pending_auto_index_jobs == mod._MAX_PENDING_AUTO_INDEX_JOBS+ )+ finally:+ mod._pending_auto_index_jobs = original_pending++ def test_trigger_auto_index_releases_slot_after_worker(self):+ """Wrapped worker releases its queue slot when the underlying+ worker raises — preventing counter leaks that would eventually+ block all auto-indexing."""+ import local_deep_research.research_library.routes.rag_routes as mod+ from local_deep_research.research_library.routes.rag_routes import (+ trigger_auto_index,+ )++ mock_sm = Mock()+ mock_sm.get_bool_setting.return_value = True+ db_session = _make_db_session()++ @contextmanager+ def fake_session(*a, **kw):+ yield db_session++ # Synchronous "executor" that faithfully emulates a real+ # ThreadPoolExecutor: a worker exception is captured (it would live in+ # the returned Future) rather than escaping submit(). This is what the+ # production executor does, so the worker exception must NOT propagate+ # out of trigger_auto_index.+ captured = {}++ class _SyncExecutor:+ def submit(self, fn, *args, **kwargs):+ captured["called"] = True+ try:+ fn(*args, **kwargs)+ except Exception as exc: # captured into the (mock) future+ captured["worker_exc"] = exc++ original_pending = mod._pending_auto_index_jobs+ mod._pending_auto_index_jobs = 0++ try:+ with (+ patch(+ f"{_DB_CTX}.get_user_db_session", side_effect=fake_session+ ),+ patch(f"{MODULE}.SettingsManager", return_value=mock_sm),+ patch(+ f"{MODULE}._get_auto_index_executor",+ return_value=_SyncExecutor(),+ ),+ patch(+ f"{MODULE}._auto_index_documents_worker",+ side_effect=RuntimeError("worker boom"),+ ),+ ):+ # Must not raise: a worker failure stays inside the future,+ # exactly as with the real ThreadPoolExecutor.+ trigger_auto_index(["doc-1"], "coll-1", "testuser", "pass")++ assert captured.get("called") is True+ # The worker did raise (captured in the future).+ assert isinstance(captured.get("worker_exc"), RuntimeError)+ # The slot must be released by the wrapper's finally so the counter+ # is back to zero (otherwise auto-indexing would silently+ # block forever once 100 worker exceptions accumulated).+ assert mod._pending_auto_index_jobs == 0+ finally:+ mod._pending_auto_index_jobs = original_pending++ def test_trigger_auto_index_releases_slot_when_submit_fails(self):+ """If executor.submit() raises (e.g., shutting down), the slot is+ released rather than leaked AND the failure is swallowed.++ The upload has already been committed by the caller before+ trigger_auto_index runs, so propagating a submit failure would turn a+ successful upload into a 500 and prompt duplicate retries. The failure+ must instead be logged and auto-indexing simply skipped.+ """+ import local_deep_research.research_library.routes.rag_routes as mod+ from local_deep_research.research_library.routes.rag_routes import (+ trigger_auto_index,+ )++ mock_sm = Mock()+ mock_sm.get_bool_setting.return_value = True+ db_session = _make_db_session()++ @contextmanager+ def fake_session(*a, **kw):+ yield db_session++ broken_executor = Mock()+ broken_executor.submit.side_effect = RuntimeError("executor shut down")++ original_pending = mod._pending_auto_index_jobs+ mod._pending_auto_index_jobs = 0++ try:+ with (+ patch(+ f"{_DB_CTX}.get_user_db_session", side_effect=fake_session+ ),+ patch(f"{MODULE}.SettingsManager", return_value=mock_sm),+ patch(+ f"{MODULE}._get_auto_index_executor",+ return_value=broken_executor,+ ),+ ):+ # Must not raise: the committed upload must not become a 500.+ trigger_auto_index(["doc-1"], "coll-1", "testuser", "pass")+ broken_executor.submit.assert_called_once()+ assert mod._pending_auto_index_jobs == 0+ finally:+ mod._pending_auto_index_jobs = original_pending+# ---------------------------------------------------------------------------# _background_index_worker