~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
import pytest
from sqlalchemy import MetaData, UniqueConstraint, create_engine
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session, sessionmaker
from unittest.mock import Mock
from local_deep_research.database.models.queue import QueueStatus, TaskMetadata
from local_deep_research.database.models.queued_research import QueuedResearch
from local_deep_research.database.models.research import ResearchHistory
from local_deep_research.web.queue import cleanup_queued_research_state
@pytest.fixture()
def db_session() -> Session:
engine = create_engine("sqlite:///:memory:")
QueuedResearch.__table__.create(engine)
QueueStatus.__table__.create(engine)
TaskMetadata.__table__.create(engine)
ResearchHistory.__table__.create(engine)
session = sessionmaker(bind=engine)()
yield session
session.close()
engine.dispose()
@pytest.fixture()
def legacy_db_session() -> Session:
"""A database whose queue table predates ``UNIQUE(research_id)``."""
engine = create_engine("sqlite:///:memory:")
legacy_metadata = MetaData()
legacy_queue = QueuedResearch.__table__.to_metadata(legacy_metadata)
for constraint in list(legacy_queue.constraints):
if isinstance(constraint, UniqueConstraint):
legacy_queue.constraints.discard(constraint)
legacy_metadata.create_all(engine)
QueueStatus.__table__.create(engine)
TaskMetadata.__table__.create(engine)
ResearchHistory.__table__.create(engine)
session = sessionmaker(bind=engine)()
yield session
session.close()
engine.dispose()
def _queued_research(
username: str, research_id: str, position: int, is_processing=False
) -> QueuedResearch:
return QueuedResearch(
username=username,
research_id=research_id,
query=f"query for {research_id}",
mode="quick",
position=position,
is_processing=is_processing,
)
def _task_metadata(
task_id: str, status: str, task_type: str = "research"
) -> TaskMetadata:
return TaskMetadata(task_id=task_id, status=status, task_type=task_type)
def test_cleanup_queued_research_state_removes_queue_state_and_rebalances_survivors(
db_session: Session,
) -> None:
# Given: deleted work, non-research metadata, and stale queue counters.
db_session.add_all(
[
_queued_research("alice", "remove-alice", 2),
_queued_research("alice", "keep-alice-first", 8),
_queued_research("alice", "keep-alice-second", 19),
_queued_research("bob", "remove-bob", 3),
_queued_research("bob", "keep-bob", 12),
_task_metadata("remove-alice", "queued"),
_task_metadata("keep-alice-first", "queued"),
_task_metadata("keep-alice-second", "processing"),
_task_metadata("remove-bob", "queued", task_type="benchmark"),
_task_metadata("keep-bob", "queued"),
QueueStatus(active_tasks=88, queued_tasks=99),
]
)
db_session.commit()
# When: the deleted parent IDs are cleaned in the caller's transaction.
cleanup_queued_research_state(db_session, ["remove-alice", "remove-bob"])
db_session.commit()
# Then: research state is removed, metadata counts are recomputed, and positions compact per user.
assert [
(row.username, row.research_id, row.position)
for row in db_session.query(QueuedResearch)
.order_by(QueuedResearch.username, QueuedResearch.position)
.all()
] == [
("alice", "keep-alice-first", 1),
("alice", "keep-alice-second", 2),
("bob", "keep-bob", 1),
]
assert {task.task_id for task in db_session.query(TaskMetadata).all()} == {
"keep-alice-first",
"keep-alice-second",
"keep-bob",
"remove-bob",
}
queue_status = db_session.query(QueueStatus).one()
assert queue_status.queued_tasks == 2
assert queue_status.active_tasks == 1
def test_cleanup_queued_research_state_leaves_its_transaction_uncommitted(
db_session: Session,
) -> None:
# Given: persisted queued research and stale status counters.
db_session.add_all(
[
_queued_research("alice", "remove", 3),
_queued_research("alice", "keep", 9),
_task_metadata("remove", "queued"),
_task_metadata("keep", "queued"),
QueueStatus(active_tasks=7, queued_tasks=11),
]
)
db_session.commit()
# When: the caller rolls back after lifecycle cleanup.
cleanup_queued_research_state(db_session, ["remove"])
db_session.rollback()
# Then: all cleanup changes roll back because the helper did not commit.
assert (
db_session.query(QueuedResearch).filter_by(research_id="remove").one()
)
assert db_session.query(TaskMetadata).filter_by(task_id="remove").one()
assert [
row.position
for row in db_session.query(QueuedResearch)
.filter_by(username="alice")
.order_by(QueuedResearch.position)
.all()
] == [3, 9]
queue_status = db_session.query(QueueStatus).one()
assert queue_status.queued_tasks == 11
assert queue_status.active_tasks == 7
def test_cleanup_with_an_acting_user_leaves_another_users_queue_state(
db_session: Session,
) -> None:
# Given: a caller passes another user's research ID along with its own.
db_session.add_all(
[
_queued_research("alice", "alice-done", 1),
_queued_research("alice", "alice-next", 2),
_queued_research("bob", "bob-queued", 4),
_task_metadata("alice-done", "queued"),
_task_metadata("alice-next", "queued"),
_task_metadata("bob-queued", "queued"),
]
)
db_session.commit()
# When: the cleanup runs for alice.
result = cleanup_queued_research_state(
db_session,
["alice-done", "bob-queued"],
include_claimed=True,
acting_user="alice",
)
db_session.commit()
# Then: only alice's state is removed; bob's row keeps its metadata and position.
assert [
(row.username, row.research_id, row.position)
for row in db_session.query(QueuedResearch)
.order_by(QueuedResearch.username, QueuedResearch.position)
.all()
] == [("alice", "alice-next", 1), ("bob", "bob-queued", 4)]
assert {task.task_id for task in db_session.query(TaskMetadata).all()} == {
"alice-next",
"bob-queued",
}
assert result.cleaned_ids == {"alice-done"}
assert result.protected_ids == {"bob-queued"}
def test_acting_user_keeps_another_users_row_that_shares_its_research_id(
legacy_db_session: Session,
) -> None:
# Given: without UNIQUE(research_id), alice's queue row and bob's claimed
# row carry the same research ID.
legacy_db_session.add_all(
[
_queued_research("alice", "shared", 1),
_queued_research("bob", "shared", 1, is_processing=True),
_task_metadata("shared", "processing"),
]
)
legacy_db_session.commit()
# When: alice's cleanup removes that ID.
result = cleanup_queued_research_state(
legacy_db_session, ["shared"], include_claimed=True, acting_user="alice"
)
legacy_db_session.commit()
# Then: alice's row is gone; bob's claimed row and the metadata it needs stay.
assert [
(row.username, row.research_id)
for row in legacy_db_session.query(QueuedResearch).all()
] == [("bob", "shared")]
assert [
task.task_id for task in legacy_db_session.query(TaskMetadata).all()
] == ["shared"]
assert result.cleaned_ids == {"shared"}
assert result.protected_ids == {"shared"}
def test_acting_user_still_removes_metadata_no_queue_row_points_at(
db_session: Session,
) -> None:
# Given: research metadata whose queue row is already gone.
db_session.add(_task_metadata("orphan", "queued"))
db_session.commit()
# When: the cleanup runs for alice.
result = cleanup_queued_research_state(
db_session, ["orphan"], include_claimed=True, acting_user="alice"
)
db_session.commit()
# Then: TaskMetadata has no owner column, so the orphan is removed.
assert db_session.query(TaskMetadata).count() == 0
assert result.cleaned_ids == {"orphan"}
def test_queue_rows_cannot_share_a_research_id(db_session: Session) -> None:
# The missing-parent sweep treats the queue row of a research ID as the
# owner of its metadata. That holds only while one row exists per ID.
db_session.add_all(
[
_queued_research("alice", "shared", 1),
_queued_research("bob", "shared", 1),
]
)
with pytest.raises(IntegrityError):
db_session.commit()
def test_acting_user_cleanup_preserves_claimed_and_foreign_state(
db_session: Session,
) -> None:
# Given: requested IDs span owners and claims, with an own queue survivor.
rows = [
_queued_research("alice", "own-unclaimed", 2),
_queued_research("alice", "own-claimed", 8, is_processing=True),
_queued_research("alice", "own-survivor", 19),
_queued_research("bob", "foreign-unclaimed", 7),
_queued_research("bob", "foreign-claimed", 13, is_processing=True),
]
for row in rows:
row.settings_snapshot = {"query": row.query, "iterations": 3}
db_session.add(
_task_metadata(
row.research_id,
"processing" if row.is_processing else "queued",
)
)
db_session.add_all(rows)
db_session.add(QueueStatus(active_tasks=88, queued_tasks=99))
db_session.commit()
foreign_before = db_session.execute(
QueuedResearch.__table__.select()
.where(QueuedResearch.username == "bob")
.order_by(QueuedResearch.id)
).all()
protected_ids = {"own-claimed", "foreign-unclaimed", "foreign-claimed"}
metadata_before = db_session.execute(
TaskMetadata.__table__.select()
.where(TaskMetadata.task_id.in_(protected_ids))
.order_by(TaskMetadata.task_id)
).all()
# When: ordinary actor-scoped cleanup excludes claimed work.
result = cleanup_queued_research_state(
db_session,
["own-unclaimed", *sorted(protected_ids)],
include_claimed=False,
acting_user="alice",
)
db_session.commit()
db_session.expire_all()
# Then: own unclaimed work is gone, and only the owner's positions compact.
assert [
(row.research_id, row.position, row.is_processing)
for row in db_session.query(QueuedResearch)
.filter_by(username="alice")
.order_by(QueuedResearch.position)
] == [("own-claimed", 1, True), ("own-survivor", 2, False)]
assert (
db_session.execute(
QueuedResearch.__table__.select()
.where(QueuedResearch.username == "bob")
.order_by(QueuedResearch.id)
).all()
== foreign_before
), "foreign queue rows and payloads must survive unchanged"
assert db_session.get(TaskMetadata, "own-unclaimed") is None
assert db_session.get(TaskMetadata, "own-survivor") is not None
assert (
db_session.execute(
TaskMetadata.__table__.select()
.where(TaskMetadata.task_id.in_(protected_ids))
.order_by(TaskMetadata.task_id)
).all()
== metadata_before
), "protected task metadata must survive independently of queue rows"
assert result.cleaned_ids == {"own-unclaimed"}
assert result.protected_ids == protected_ids
status = db_session.query(QueueStatus).one()
assert (status.queued_tasks, status.active_tasks) == (2, 2)
def test_missing_parent_sweep_removes_orphans_without_retry_or_failure(
db_session: Session,
) -> None:
from local_deep_research.web.queue.processor_v2 import QueueProcessorV2
# Given: legacy orphaned work and an in-progress parent in spawn grace.
db_session.add_all(
[
_queued_research("alice", "missing-parent", 3),
QueuedResearch(
username="alice",
research_id="spawn-grace",
query="still starting",
mode="quick",
position=9,
is_processing=True,
),
ResearchHistory(
id="spawn-grace",
query="still starting",
mode="quick",
status="in_progress",
created_at="2026-07-13T00:00:00+00:00",
),
_task_metadata("missing-parent", "queued"),
_task_metadata("spawn-grace", "processing"),
QueueStatus(active_tasks=11, queued_tasks=14),
]
)
db_session.commit()
processor = QueueProcessorV2()
processor._spawn_retry_counts["missing-parent"] = 2
processor.notify_research_failed = Mock()
# When: pre-dispatch cleanup sweeps rows whose parent no longer exists.
result = processor._sweep_missing_parent_queue_rows(db_session, "alice")
# Then: only the orphan disappears without retries, failure notices, or spawn-grace changes.
assert result.can_dispatch is True
assert result.cleaned_ids == frozenset({"missing-parent"})
assert (
db_session.query(QueuedResearch)
.filter_by(research_id="missing-parent")
.first()
is None
)
assert (
db_session.query(TaskMetadata)
.filter_by(task_id="missing-parent")
.first()
is None
)
assert (
db_session.query(QueuedResearch)
.filter_by(research_id="spawn-grace")
.one()
.position
== 1
)
assert (
db_session.query(ResearchHistory)
.filter_by(id="spawn-grace")
.one()
.status
== "in_progress"
)
assert "missing-parent" not in processor._spawn_retry_counts
processor.notify_research_failed.assert_not_called()
queue_status = db_session.query(QueueStatus).one()
assert queue_status.queued_tasks == 0
assert queue_status.active_tasks == 1
def test_queue_dispatch_sweeps_missing_parents_before_reclaiming_or_claiming() -> (
None
):
from local_deep_research.web.queue.processor_v2 import QueueProcessorV2
# Given: a queue processor whose cleanup steps record their ordering.
processor = QueueProcessorV2()
call_order: list[str] = []
db_session = Mock()
queued_query = Mock()
queued_query.filter_by.return_value = queued_query
queued_query.order_by.return_value = queued_query
queued_query.limit.return_value = queued_query
queued_query.all.return_value = []
db_session.query.side_effect = lambda _: (
call_order.append("query") or queued_query
)
processor._sweep_missing_parent_queue_rows = Mock(
side_effect=lambda *_: (
call_order.append("sweep") or Mock(can_dispatch=True)
)
)
processor._reclaim_stranded_queue_rows = Mock(
side_effect=lambda *_: call_order.append("reclaim") or 0
)
# When: queued work reaches the dispatch entrypoint.
processor._start_queued_researches(
db_session, Mock(), "alice", "password", 1
)
# Then: legacy orphans are removed before rows can be reclaimed or claimed.
assert call_order[:3] == ["sweep", "reclaim", "query"]
@pytest.mark.parametrize(
"interleave,rollback,acting_user",
[
(True, False, "alice"),
(True, True, "alice"),
(False, False, "alice"),
(False, False, None),
],
)
def test_cleanup_statement_time_foreign_owner(
tmp_path,
monkeypatch,
record_testsuite_property,
interleave,
rollback,
acting_user,
):
"""Legacy deferred SQLite reads do not establish a physical snapshot."""
import json
from sqlalchemy import event
from sqlalchemy.orm import Query
engine = create_engine(f"sqlite:///{tmp_path / 'ownership.sqlite'}")
with engine.connect() as connection:
assert (
connection.exec_driver_sql("PRAGMA journal_mode=WAL").scalar()
== "wal"
)
assert (
connection.exec_driver_sql("PRAGMA journal_mode").scalar() == "wal"
)
tables = MetaData()
queue = QueuedResearch.__table__.to_metadata(tables)
for constraint in list(queue.constraints):
if isinstance(constraint, UniqueConstraint) and {
c.name for c in constraint.columns
} == {"research_id"}:
queue.constraints.remove(constraint)
tables.create_all(engine)
TaskMetadata.__table__.create(engine)
QueueStatus.__table__.create(engine)
def payload(session, model):
return [
{c.name: getattr(row, c.name) for c in model.__table__.columns}
for row in session.query(model)
.order_by(list(model.__table__.primary_key)[0])
.all()
]
observations = []
try:
with Session(engine) as seed:
seed.add_all(
[
_queued_research("alice", "shared", 7),
_queued_research("alice", "keep", 19),
_task_metadata("shared", "queued"),
_task_metadata("foreign-only", "queued"),
_task_metadata("remove", "queued"),
QueueStatus(active_tasks=8, queued_tasks=9),
]
)
seed.commit()
initial_rows = payload(seed, QueuedResearch)
initial_metadata = payload(seed, TaskMetadata)
with Session(engine) as a, Session(engine) as b:
raw_a = a.connection().connection.driver_connection
raw_b = b.connection().connection.driver_connection
assert raw_a is not raw_b
observations.append(
dict(
driver=type(raw_a).__module__,
sqlite_version=engine.dialect.dbapi.sqlite_version,
isolation_level=raw_a.isolation_level,
a=id(raw_a),
b=id(raw_b),
)
)
original_all = Query.all
loaded = False
foreign_rows = []
def after_real_read(query):
nonlocal loaded, foreign_rows
rows = original_all(query)
if (
query.session is a
and not loaded
and query.column_descriptions[0]["expr"] is QueuedResearch
):
loaded = True
assert [row.research_id for row in rows] == ["shared"]
observations.append(
dict(
boundary="initial-read",
physical_transaction=raw_a.in_transaction,
logical_transaction=a.in_transaction(),
)
)
if interleave:
for research_id in ("shared", "foreign-only"):
row = _queued_research("bob", research_id, 23)
row.query = "foreign payload"
row.settings_snapshot = {"keep": [research_id, 42]}
b.add(row)
b.commit()
observations.append(
dict(
boundary="b-commit",
a_transaction=raw_a.in_transaction,
b_transaction=raw_b.in_transaction,
)
)
foreign_rows = payload(b, QueuedResearch)[-2:]
return rows
def before_sql(
conn, cursor, statement, parameters, context, executemany
):
if statement.startswith("DELETE FROM task_metadata"):
observations.append(
dict(
boundary="a-metadata-delete",
physical_transaction=raw_a.in_transaction,
sql=statement,
)
)
monkeypatch.setattr(Query, "all", after_real_read)
event.listen(engine, "before_cursor_execute", before_sql)
result = cleanup_queued_research_state(
a, ["shared", "foreign-only", "remove"], acting_user=acting_user
)
assert loaded
# Inspect real pending effects before the caller chooses rollback.
assert (
a.query(QueuedResearch)
.filter_by(username="alice", research_id="shared")
.count()
== 0
)
assert (
a.query(TaskMetadata).filter_by(task_id="remove").count() == 0
)
if rollback:
a.rollback()
else:
a.commit()
event.remove(engine, "before_cursor_execute", before_sql)
record_testsuite_property(
"statement-time-ownership", json.dumps(observations)
)
print(json.dumps(observations))
with Session(engine) as fresh:
rows = payload(fresh, QueuedResearch)
metadata = payload(fresh, TaskMetadata)
if interleave:
assert [
row for row in rows if row["username"] == "bob"
] == foreign_rows
# Original source loses this metadata even though B's committed,
# unclaimed foreign row survives. This is the persisted-state RED.
assert {row["task_id"] for row in metadata} >= {
"shared",
"foreign-only",
}, (
"committed foreign rows survived but their TaskMetadata is missing"
)
if not rollback:
assert metadata == [
row
for row in initial_metadata
if row["task_id"] != "remove"
]
assert result.protected_ids == {"shared", "foreign-only"}
assert result.cleaned_ids == {"shared", "remove"}
else:
assert metadata == []
assert result.cleaned_ids == {
"shared",
"foreign-only",
"remove",
}
assert result.protected_ids == frozenset()
if rollback:
assert [
row for row in rows if row["username"] == "alice"
] == initial_rows
assert metadata == initial_metadata
status = fresh.query(QueueStatus).one()
assert (status.active_tasks, status.queued_tasks) == (8, 9)
else:
assert [
(r["research_id"], r["position"])
for r in rows
if r["username"] == "alice"
] == [("keep", 1)]
status = fresh.query(QueueStatus).one()
assert (status.active_tasks, status.queued_tasks) == (
0,
2 if interleave else 0,
)
finally:
engine.dispose()
def _persisted_state(session: Session, model) -> list[dict]:
session.expire_all()
return [
{
column.name: getattr(row, column.name)
for column in model.__table__.columns
}
for row in session.query(model)
.order_by(list(model.__table__.primary_key)[0])
.all()
]
def _seed_legacy_shared_state(session: Session):
own = _queued_research("alice", "shared", 7)
foreign = _queued_research("bob", "shared", 12, is_processing=True)
foreign.query = "foreign payload"
foreign.mode = "detailed"
foreign.settings_snapshot = {"keep": ["foreign", 42]}
foreign_only = _queued_research("bob", "foreign-only", 19)
metadata = _task_metadata("shared", "processing")
metadata.metadata_json = {"keep": "shared metadata"}
metadata.progress_message = "foreign progress"
metadata.retry_count = 3
session.add_all(
[
own,
foreign,
foreign_only,
metadata,
_task_metadata("foreign-only", "queued"),
]
)
session.commit()
return (
_persisted_state(session, QueuedResearch),
_persisted_state(session, TaskMetadata),
)
def test_shared_cleanup_reports_deleted_rows_and_preserves_foreign_state(
legacy_db_session: Session,
) -> None:
session = legacy_db_session
rows, metadata = _seed_legacy_shared_state(session)
result = cleanup_queued_research_state(
session,
["shared", "foreign-only"],
include_claimed=True,
acting_user="alice",
)
session.commit()
assert _persisted_state(session, QueuedResearch) == [
row for row in rows if row["username"] == "bob"
]
assert _persisted_state(session, TaskMetadata) == metadata
assert result.cleaned_ids == {"shared"}
assert result.protected_ids == {"shared", "foreign-only"}
@pytest.mark.parametrize(
"fail_commit", [False, True], ids=["commit", "rollback"]
)
def test_missing_parent_shared_id_sweep_uses_committed_cleanup_result(
legacy_db_session: Session,
monkeypatch,
fail_commit: bool,
record_testsuite_property,
) -> None:
from local_deep_research.web.queue.processor_v2 import QueueProcessorV2
session = legacy_db_session
rows, metadata = _seed_legacy_shared_state(session)
assert session.query(ResearchHistory).count() == 0
processor = QueueProcessorV2()
processor._spawn_retry_counts.update({"shared": 2, "unrelated": 4})
original_commit = session.commit
observations = []
def commit_at_boundary():
session.flush()
observations.append(
{
"rows": _persisted_state(session, QueuedResearch),
"metadata": _persisted_state(session, TaskMetadata),
"retries": dict(processor._spawn_retry_counts),
}
)
if fail_commit:
raise RuntimeError(
"injected commit failure after real cleanup flush"
)
original_commit()
observations.append({"committed": True})
monkeypatch.setattr(session, "commit", commit_at_boundary)
result = processor._sweep_missing_parent_queue_rows(session, "alice")
record_testsuite_property(
"shared-id-rollback" if fail_commit else "shared-id-commit",
repr(
{
"boundary": observations,
"result": result,
"final_rows": _persisted_state(session, QueuedResearch),
"final_metadata": _persisted_state(session, TaskMetadata),
"final_retries": dict(processor._spawn_retry_counts),
}
),
)
# These assertions run outside the best-effort commit/rollback wrapper.
foreign_rows = [row for row in rows if row["username"] == "bob"]
assert observations[0] == {
"rows": foreign_rows,
"metadata": metadata,
"retries": {"shared": 2, "unrelated": 4},
}
assert len(observations) == (1 if fail_commit else 2)
if fail_commit:
assert result.can_dispatch is False
assert result.cleaned_ids == frozenset()
assert _persisted_state(session, QueuedResearch) == rows
assert processor._spawn_retry_counts == {"shared": 2, "unrelated": 4}
assert session.query(QueueStatus).count() == 0
else:
assert observations[1] == {"committed": True}
assert result.can_dispatch is True
assert result.cleaned_ids == {"shared"}
assert _persisted_state(session, QueuedResearch) == foreign_rows
assert processor._spawn_retry_counts == {"unrelated": 4}
status = session.query(QueueStatus).one()
assert (status.active_tasks, status.queued_tasks) == (1, 1)
assert _persisted_state(session, TaskMetadata) == metadata