Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(queue): revert queued->processing counter on capacity-reject re-queue (#4346)

LearningCircuit4e0a523003eb
diff --git a/changelog.d/+queue-counter-capacity-reject-drift.bugfix.md b/changelog.d/+queue-counter-capacity-reject-drift.bugfix.md
new file mode 100644
index 0000000000..5f349a0f00
--- /dev/null
+++ b/changelog.d/+queue-counter-capacity-reject-drift.bugfix.md
@@ -0,0 +1 @@
+The queued-research dispatch loop now reverts its queue-counter claim when a global-capacity reject re-queues a research. Previously, ``_start_queued_researches`` marked the task ``processing`` (decrementing ``queued_tasks``, incrementing ``active_tasks``) and, on ``SystemAtCapacityError``, only reset the row's ``is_processing`` flag — leaking a slot into ``active_tasks`` on every capacity-rejected retry. Under sustained capacity pressure ``queued_tasks`` would drift to 0, at which point the per-user queue processor treated the queue as empty and stopped dispatching the still-present rows. ``update_task_status`` now supports a ``processing`` → ``queued`` transition that restores the counts, and the capacity-reject path uses it.
diff --git a/src/local_deep_research/database/queue_service.py b/src/local_deep_research/database/queue_service.py
index bd939fc5ca..7654828103 100644
--- a/src/local_deep_research/database/queue_service.py
+++ b/src/local_deep_research/database/queue_service.py
@@ -108,6 +108,14 @@ class UserQueueService:
task.started_at = datetime.now(UTC)
self._update_queue_counts(-1, 1) # -1 queued, +1 active
+ elif status == "queued" and old_status == "processing":
+ # Dispatch was rolled back (e.g. global capacity reject left
+ # the research queued for the next tick). Revert the claim so
+ # the counters stay balanced instead of leaking a slot into
+ # active_tasks on every retry.
+ task.started_at = None
+ self._update_queue_counts(1, -1) # +1 queued, -1 active
+
elif status in ["completed", "failed"]:
task.completed_at = datetime.now(UTC)
self._update_queue_counts(0, -1) # 0 queued, -1 active
diff --git a/src/local_deep_research/web/queue/processor_v2.py b/src/local_deep_research/web/queue/processor_v2.py
index f034c98b2e..a089ca3cfb 100644
--- a/src/local_deep_research/web/queue/processor_v2.py
+++ b/src/local_deep_research/web/queue/processor_v2.py
@@ -924,6 +924,12 @@ class QueueProcessorV2:
f"System at capacity dispatching queued research "
f"{research_id}; leaving queued for next tick"
)
+ # Revert the queued->processing claim from
+ # update_task_status("processing") above. The research stays
+ # queued for the next tick, so its slot must return to
+ # queued_tasks rather than leaking into active_tasks on
+ # every capacity-rejected retry.
+ queue_service.update_task_status(research_id, "queued")
fresh_queued = (
db_session.query(QueuedResearch)
.filter_by(username=username, research_id=research_id)
diff --git a/tests/database/test_queue_service.py b/tests/database/test_queue_service.py
index 610d1988ec..06590c0ec7 100644
--- a/tests/database/test_queue_service.py
+++ b/tests/database/test_queue_service.py
@@ -350,3 +350,66 @@ class TestQueueCountHelpers:
assert mock_status.queued_tasks == 0
assert mock_status.active_tasks == 0
+
+
+class TestProcessingToQueuedRevert:
+ """update_task_status('queued') after 'processing' reverts the counter.
+
+ Regression: the queued-dispatch loop's SystemAtCapacityError branch
+ re-queues a research without reverting the queued->processing counter
+ claim, so each capacity-rejected retry leaked a slot into active_tasks
+ and decremented queued_tasks. Under sustained capacity pressure
+ queued_tasks drifts to 0, at which point _process_user_queue treats the
+ queue as empty and stops dispatching the still-present rows. These use
+ a real in-memory DB so the counter arithmetic is actually exercised.
+ """
+
+ @staticmethod
+ def _service():
+ from sqlalchemy import create_engine
+ from sqlalchemy.orm import sessionmaker
+ from local_deep_research.database.models.queue import (
+ QueueStatus,
+ TaskMetadata,
+ )
+ from local_deep_research.database.queue_service import (
+ UserQueueService,
+ )
+
+ engine = create_engine("sqlite:///:memory:")
+ QueueStatus.__table__.create(engine)
+ TaskMetadata.__table__.create(engine)
+ session = sessionmaker(bind=engine)()
+ return UserQueueService(session)
+
+ def test_processing_then_queued_round_trips_counts(self):
+ svc = self._service()
+ svc.add_task_metadata("r1", "research") # queued 1 / active 0
+ assert svc.get_queue_status()["queued_tasks"] == 1
+ assert svc.get_queue_status()["active_tasks"] == 0
+
+ svc.update_task_status("r1", "processing") # queued 0 / active 1
+ s = svc.get_queue_status()
+ assert s["queued_tasks"] == 0
+ assert s["active_tasks"] == 1
+
+ svc.update_task_status("r1", "queued") # reverted: queued 1 / active 0
+ s = svc.get_queue_status()
+ assert s["queued_tasks"] == 1, (
+ "queued->processing claim must be reverted on re-queue"
+ )
+ assert s["active_tasks"] == 0, (
+ "active slot must not leak after a capacity re-queue"
+ )
+
+ def test_repeated_capacity_rejects_do_not_drift(self):
+ """Three processing->queued cycles (simulating capacity retries)
+ must leave the counter exactly where it started."""
+ svc = self._service()
+ svc.add_task_metadata("r1", "research")
+ for _ in range(3):
+ svc.update_task_status("r1", "processing")
+ svc.update_task_status("r1", "queued")
+ s = svc.get_queue_status()
+ assert s["queued_tasks"] == 1
+ assert s["active_tasks"] == 0