~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(queue): revert queued->processing counter on capacity-reject re-queue (#4346)
LearningCircuit4e0a523003eb
diff --git a/changelog.d/+queue-counter-capacity-reject-drift.bugfix.md b/changelog.d/+queue-counter-capacity-reject-drift.bugfix.mdnew file mode 100644index 0000000000..5f349a0f00--- /dev/null+++ b/changelog.d/+queue-counter-capacity-reject-drift.bugfix.md@@ -0,0 +1 @@+The queued-research dispatch loop now reverts its queue-counter claim when a global-capacity reject re-queues a research. Previously, ``_start_queued_researches`` marked the task ``processing`` (decrementing ``queued_tasks``, incrementing ``active_tasks``) and, on ``SystemAtCapacityError``, only reset the row's ``is_processing`` flag — leaking a slot into ``active_tasks`` on every capacity-rejected retry. Under sustained capacity pressure ``queued_tasks`` would drift to 0, at which point the per-user queue processor treated the queue as empty and stopped dispatching the still-present rows. ``update_task_status`` now supports a ``processing`` → ``queued`` transition that restores the counts, and the capacity-reject path uses it.diff --git a/src/local_deep_research/database/queue_service.py b/src/local_deep_research/database/queue_service.pyindex bd939fc5ca..7654828103 100644--- a/src/local_deep_research/database/queue_service.py+++ b/src/local_deep_research/database/queue_service.py@@ -108,6 +108,14 @@ class UserQueueService:task.started_at = datetime.now(UTC)self._update_queue_counts(-1, 1) # -1 queued, +1 active+ elif status == "queued" and old_status == "processing":+ # Dispatch was rolled back (e.g. global capacity reject left+ # the research queued for the next tick). Revert the claim so+ # the counters stay balanced instead of leaking a slot into+ # active_tasks on every retry.+ task.started_at = None+ self._update_queue_counts(1, -1) # +1 queued, -1 active+elif status in ["completed", "failed"]:task.completed_at = datetime.now(UTC)self._update_queue_counts(0, -1) # 0 queued, -1 activediff --git a/src/local_deep_research/web/queue/processor_v2.py b/src/local_deep_research/web/queue/processor_v2.pyindex f034c98b2e..a089ca3cfb 100644--- a/src/local_deep_research/web/queue/processor_v2.py+++ b/src/local_deep_research/web/queue/processor_v2.py@@ -924,6 +924,12 @@ class QueueProcessorV2:f"System at capacity dispatching queued research "f"{research_id}; leaving queued for next tick")+ # Revert the queued->processing claim from+ # update_task_status("processing") above. The research stays+ # queued for the next tick, so its slot must return to+ # queued_tasks rather than leaking into active_tasks on+ # every capacity-rejected retry.+ queue_service.update_task_status(research_id, "queued")fresh_queued = (db_session.query(QueuedResearch).filter_by(username=username, research_id=research_id)diff --git a/tests/database/test_queue_service.py b/tests/database/test_queue_service.pyindex 610d1988ec..06590c0ec7 100644--- a/tests/database/test_queue_service.py+++ b/tests/database/test_queue_service.py@@ -350,3 +350,66 @@ class TestQueueCountHelpers:assert mock_status.queued_tasks == 0assert mock_status.active_tasks == 0+++class TestProcessingToQueuedRevert:+ """update_task_status('queued') after 'processing' reverts the counter.++ Regression: the queued-dispatch loop's SystemAtCapacityError branch+ re-queues a research without reverting the queued->processing counter+ claim, so each capacity-rejected retry leaked a slot into active_tasks+ and decremented queued_tasks. Under sustained capacity pressure+ queued_tasks drifts to 0, at which point _process_user_queue treats the+ queue as empty and stops dispatching the still-present rows. These use+ a real in-memory DB so the counter arithmetic is actually exercised.+ """++ @staticmethod+ def _service():+ from sqlalchemy import create_engine+ from sqlalchemy.orm import sessionmaker+ from local_deep_research.database.models.queue import (+ QueueStatus,+ TaskMetadata,+ )+ from local_deep_research.database.queue_service import (+ UserQueueService,+ )++ engine = create_engine("sqlite:///:memory:")+ QueueStatus.__table__.create(engine)+ TaskMetadata.__table__.create(engine)+ session = sessionmaker(bind=engine)()+ return UserQueueService(session)++ def test_processing_then_queued_round_trips_counts(self):+ svc = self._service()+ svc.add_task_metadata("r1", "research") # queued 1 / active 0+ assert svc.get_queue_status()["queued_tasks"] == 1+ assert svc.get_queue_status()["active_tasks"] == 0++ svc.update_task_status("r1", "processing") # queued 0 / active 1+ s = svc.get_queue_status()+ assert s["queued_tasks"] == 0+ assert s["active_tasks"] == 1++ svc.update_task_status("r1", "queued") # reverted: queued 1 / active 0+ s = svc.get_queue_status()+ assert s["queued_tasks"] == 1, (+ "queued->processing claim must be reverted on re-queue"+ )+ assert s["active_tasks"] == 0, (+ "active slot must not leak after a capacity re-queue"+ )++ def test_repeated_capacity_rejects_do_not_drift(self):+ """Three processing->queued cycles (simulating capacity retries)+ must leave the counter exactly where it started."""+ svc = self._service()+ svc.add_task_metadata("r1", "research")+ for _ in range(3):+ svc.update_task_status("r1", "processing")+ svc.update_task_status("r1", "queued")+ s = svc.get_queue_status()+ assert s["queued_tasks"] == 1+ assert s["active_tasks"] == 0