~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Shared library content must outlive any one referencing research."""
from contextlib import contextmanager
from datetime import UTC, datetime
from hashlib import sha256
import sqlite3
import pytest
from sqlalchemy import MetaData, create_engine, event
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import Session
from local_deep_research.database import research_cascade
from local_deep_research.database.models import (
Base,
QueuedResearch,
ResearchHistory,
ResearchResource,
)
from local_deep_research.database.models.library import (
Collection,
Document,
DocumentBlob,
DocumentChunk,
DocumentCollection,
EmbeddingProvider,
RAGIndex,
RagDocumentStatus,
SourceType,
)
from local_deep_research.database.research_cascade import (
SharedDownloadIndex,
delete_research_with_dependents,
)
from local_deep_research.research_library.deletion.utils.cascade_helper import (
CascadeHelper,
)
NOW = datetime(2026, 10, 1, tzinfo=UTC)
PDF = b"%PDF-1.4 stored canonical paper"
BODY = "Complete canonical paper text."
@pytest.fixture(
params=["SET NULL", "CASCADE"], ids=["current", "legacy-resource-fk"]
)
def database(tmp_path, request):
engine = create_engine(
f"sqlite:///{tmp_path / 'shared-documents.sqlite'}",
connect_args={"isolation_level": "", "timeout": 1},
)
@event.listens_for(engine, "connect")
def enable_foreign_keys(connection, _record):
connection.execute("PRAGMA foreign_keys=ON")
# Clone metadata rather than mutating the production models or another
# test's cached cascade plan. Only the historical resource FK differs.
schema = MetaData()
for table in Base.metadata.tables.values():
table.to_metadata(schema)
for constraint in schema.tables["documents"].foreign_key_constraints:
if tuple(constraint.column_keys) == ("resource_id",):
constraint.ondelete = request.param
with engine.connect() as connection:
assert (
connection.exec_driver_sql("PRAGMA journal_mode=WAL").scalar()
== "wal"
)
# Group schema setup; legacy SQLite otherwise commits each DDL alone.
# The sessions under test still start with their normal driver behavior.
connection.exec_driver_sql("BEGIN")
schema.create_all(connection)
connection.commit()
resource_fk = next(
row
for row in connection.exec_driver_sql(
"PRAGMA foreign_key_list(documents)"
)
if row[3] == "resource_id"
)
assert resource_fk[6] == request.param
try:
yield engine
finally:
engine.dispose()
def seed_family(engine, *, references=True):
with Session(engine) as session:
session.add(
SourceType(
id="source", name="research_download", display_name="Research"
)
)
for research_id in ("owner", "sibling", "last"):
session.add(
ResearchHistory(
id=research_id,
query="q",
mode="quick",
status="completed",
created_at=NOW.isoformat(),
)
)
session.add_all(
[
Collection(id="library", name="Library"),
Collection(id="selected", name="Selected"),
]
)
session.flush()
resources = {}
for research_id in ("owner", "sibling", "last"):
resource = ResearchResource(
research_id=research_id,
title="Shared paper",
url=f"https://example.org/{research_id}.pdf",
source_type="academic",
created_at=NOW.isoformat(),
)
session.add(resource)
session.flush()
resources[research_id] = resource.id
session.add(
Document(
id="canonical",
source_type_id="source",
resource_id=resources["owner"],
research_id="owner",
document_hash=sha256(PDF).hexdigest(),
title="Shared paper",
original_url="https://example.org/owner.pdf",
file_type="pdf",
file_path="database",
storage_mode="database",
file_size=len(PDF),
text_content=BODY,
word_count=len(BODY.split()),
character_count=len(BODY),
extraction_method="html_extraction",
extraction_source="arxiv_html",
extraction_quality="high",
processed_at=NOW,
)
)
# A document with no surviving reference must retain ordinary cascade
# behavior; the shared-document exception must not preserve everything.
session.add(
Document(
id="unshared",
source_type_id="source",
research_id="owner",
document_hash="unshared",
file_size=16,
file_type="text",
text_content="Private to owner",
)
)
session.flush()
session.add(
DocumentBlob(
document_id="canonical",
pdf_binary=PDF,
blob_hash=sha256(PDF).hexdigest(),
)
)
if references:
for research_id in ("sibling", "last"):
session.get(
ResearchResource, resources[research_id]
).document_id = "canonical"
for collection_id in ("library", "selected"):
index = RAGIndex(
collection_name=f"collection_{collection_id}",
embedding_model="test",
embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,
embedding_dimension=2,
index_path=f"/{collection_id}.faiss",
index_hash=collection_id,
chunk_size=100,
chunk_overlap=0,
)
session.add(index)
session.flush()
session.add_all(
[
DocumentCollection(
document_id="canonical",
collection_id=collection_id,
indexed=True,
chunk_count=1,
last_indexed_at=NOW,
),
RagDocumentStatus(
document_id="canonical",
collection_id=collection_id,
rag_index_id=index.id,
chunk_count=1,
indexed_at=NOW,
),
DocumentChunk(
chunk_hash=collection_id,
source_type="document",
source_id="canonical",
collection_name=f"collection_{collection_id}",
chunk_text=BODY,
chunk_index=0,
start_char=0,
end_char=len(BODY),
word_count=len(BODY.split()),
embedding_id=collection_id,
embedding_model="test",
embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,
),
]
)
session.commit()
return resources
def assert_preserved(session, research_id, resource_id):
session.expire_all()
doc = session.get(Document, "canonical")
assert doc is not None
assert (doc.research_id, doc.resource_id) == (research_id, resource_id)
assert (
session.get(ResearchResource, resource_id).research_id
== doc.research_id
)
assert doc.text_content == BODY
assert doc.document_hash == sha256(PDF).hexdigest()
assert doc.extraction_source == "arxiv_html"
assert doc.extraction_quality == "high"
assert doc.original_url == "https://example.org/owner.pdf"
assert (doc.file_path, doc.storage_mode, doc.file_size) == (
"database",
"database",
len(PDF),
)
assert session.get(DocumentBlob, doc.id).pdf_binary == PDF
links = (
session.query(DocumentCollection).filter_by(document_id=doc.id).all()
)
assert {
(
link.collection_id,
link.indexed,
link.chunk_count,
link.last_indexed_at,
)
for link in links
} == {(collection, True, 1, NOW) for collection in ("library", "selected")}
assert (
session.query(RagDocumentStatus).filter_by(document_id=doc.id).count()
== 2
)
assert {
(row.embedding_id, row.chunk_text)
for row in session.query(DocumentChunk).filter_by(source_id=doc.id)
} == {("library", BODY), ("selected", BODY)}
assert (
session.connection().exec_driver_sql("PRAGMA foreign_key_check").all()
== []
)
@pytest.mark.parametrize("deleted", ["owner", "sibling"])
def test_other_research_keeps_document_and_all_stored_content(
database, deleted
):
resources = seed_family(database)
with Session(database) as session:
assert delete_research_with_dependents(session, deleted)
session.commit()
remaining_owner = "sibling" if deleted == "owner" else "owner"
assert_preserved(session, remaining_owner, resources[remaining_owner])
assert session.get(ResearchHistory, deleted) is None
assert (
session.get(ResearchResource, resources["last"]).document_id
== "canonical"
)
if deleted == "owner":
assert session.get(Document, "unshared") is None
@pytest.mark.parametrize("order", [("owner", "sibling"), ("sibling", "owner")])
def test_clear_history_transfers_to_surviving_research(database, order):
resources = seed_family(database)
with Session(database) as session:
index = SharedDownloadIndex.build(session, order)
for research_id in order:
assert delete_research_with_dependents(
session, research_id, shared_downloads=index
)
session.commit()
assert_preserved(session, "last", resources["last"])
assert (
session.get(ResearchResource, resources["last"]).document_id
== "canonical"
)
def test_last_reference_keeps_existing_document_cascade(database):
seed_family(database)
with Session(database) as session:
index = SharedDownloadIndex.build(session, ["owner", "sibling", "last"])
for research_id in ("owner", "sibling", "last"):
assert delete_research_with_dependents(
session, research_id, shared_downloads=index
)
session.commit()
for model in (
ResearchHistory,
ResearchResource,
Document,
DocumentBlob,
DocumentCollection,
RagDocumentStatus,
):
assert session.query(model).count() == 0
@pytest.mark.parametrize("guard", ["status", "queue"])
def test_guard_refusal_restores_ownership_and_content(database, guard):
resources = seed_family(database)
with Session(database) as session:
if guard == "status":
session.get(ResearchHistory, "owner").status = "in_progress"
else:
session.add(
QueuedResearch(
username="test",
research_id="owner",
query="q",
mode="quick",
position=1,
is_processing=True,
created_at=NOW,
)
)
session.commit()
assert not delete_research_with_dependents(session, "owner")
session.commit()
assert_preserved(session, "owner", resources["owner"])
assert session.get(Document, "unshared") is not None
assert session.query(ResearchHistory).count() == 3
def test_caller_rollback_restores_transferred_owner(database):
resources = seed_family(database)
with Session(database) as session:
index = SharedDownloadIndex.build(session, ["owner", "sibling"])
for research_id in ("owner", "sibling"):
assert delete_research_with_dependents(
session, research_id, shared_downloads=index
)
session.rollback()
with Session(database) as verification:
assert_preserved(verification, "owner", resources["owner"])
assert verification.query(ResearchHistory).count() == 3
assert (
verification.get(ResearchResource, resources["sibling"]).document_id
== "canonical"
)
@pytest.mark.parametrize("failure_stage", ["transfer", "parent-delete"])
def test_failed_transfer_rolls_back_deletion(database, failure_stage):
resources = seed_family(database)
def fail_owner_change(_conn, _cursor, statement, *_args):
if (
failure_stage == "transfer"
and statement.startswith("UPDATE documents SET")
and "research_id=" in statement
) or (
failure_stage == "parent-delete"
and statement.startswith("DELETE FROM research_history")
):
raise RuntimeError("controlled ownership write failure")
event.listen(database, "before_cursor_execute", fail_owner_change)
try:
with Session(database) as session:
with pytest.raises(
RuntimeError, match="controlled ownership write failure"
):
delete_research_with_dependents(session, "owner")
session.rollback()
finally:
event.remove(database, "before_cursor_execute", fail_owner_change)
with Session(database) as verification:
assert_preserved(verification, "owner", resources["owner"])
assert verification.query(ResearchHistory).count() == 3
def test_explicit_document_database_delete_remains_global(database):
resources = seed_family(database)
with Session(database) as session:
assert CascadeHelper.delete_document_completely(session, "canonical")
session.commit()
assert session.get(Document, "canonical") is None
assert session.get(DocumentBlob, "canonical") is None
assert session.query(ResearchHistory).count() == 3
for research_id in ("sibling", "last"):
assert (
session.get(
ResearchResource, resources[research_id]
).document_id
is None
)
def test_batch_retains_document_with_protected_successor(database):
resources = seed_family(database)
with Session(database) as session:
session.get(ResearchHistory, "sibling").status = "in_progress"
session.commit()
index = SharedDownloadIndex.build(session, ["owner", "sibling", "last"])
assert delete_research_with_dependents(
session, "owner", shared_downloads=index
)
assert not delete_research_with_dependents(
session, "sibling", shared_downloads=index
)
assert delete_research_with_dependents(
session, "last", shared_downloads=index
)
session.commit()
assert_preserved(session, "sibling", resources["sibling"])
assert session.query(ResearchHistory).count() == 1
@pytest.mark.parametrize("batch", [False, True], ids=["single", "batch"])
@pytest.mark.parametrize("change", ["add", "remove"])
def test_concurrent_references_require_fresh_deletion_snapshot(
database, monkeypatch, batch, change
):
resources = seed_family(database, references=change == "remove")
lookup = research_cascade._shared_document_replacements
changed = False
def change_after_lookup(*args):
nonlocal changed
result = lookup(*args)
if not changed:
changed = True
with Session(database) as writer:
for research_id in ("sibling", "last"):
writer.get(
ResearchResource, resources[research_id]
).document_id = "canonical" if change == "add" else None
writer.commit()
return result
monkeypatch.setattr(
research_cascade, "_shared_document_replacements", change_after_lookup
)
with Session(database) as deletion:
index = (
SharedDownloadIndex.build(deletion, ["owner"]) if batch else None
)
with pytest.raises(OperationalError, match="database is locked"):
delete_research_with_dependents(
deletion, "owner", shared_downloads=index
)
deletion.rollback()
assert changed
with Session(database) as verification:
assert_preserved(verification, "owner", resources["owner"])
assert verification.get(ResearchHistory, "owner") is not None
expected = "canonical" if change == "add" else None
assert (
verification.get(
ResearchResource, resources["sibling"]
).document_id
== expected
)
# The retry must follow the committed references rather than its old
# retention decision, in either direction and with a fresh batch index.
index = (
SharedDownloadIndex.build(deletion, ["owner"]) if batch else None
)
assert delete_research_with_dependents(
deletion, "owner", shared_downloads=index
)
deletion.commit()
if change == "add":
assert_preserved(deletion, "sibling", resources["sibling"])
else:
assert deletion.get(Document, "canonical") is None
assert deletion.get(DocumentBlob, "canonical") is None
assert deletion.get(ResearchHistory, "owner") is None
assert deletion.get(ResearchHistory, "sibling") is not None
@pytest.fixture
def route_session(database, monkeypatch):
from local_deep_research.web.routers import research
# Match the production context: a thread reuses its Session, but a failed
# outer scope rolls back before another operation may use it.
with Session(database) as session:
@contextmanager
def context(_username):
try:
yield session
finally:
if session.in_transaction():
session.rollback()
monkeypatch.setattr(research, "get_user_db_session", context)
monkeypatch.setattr(
research, "get_active_research_ids", lambda: {"sibling", "last"}
)
yield session
def call_delete_route(batch):
from local_deep_research.web.routers import research
if batch:
return research.clear_history(request=None, username="test-user")
return research.delete_research(
request=None, research_id="owner", username="test-user"
)
@pytest.mark.parametrize("batch", [False, True], ids=["single", "batch"])
@pytest.mark.parametrize("change", ["add", "remove"])
def test_routes_retry_changed_document_references(
database, route_session, monkeypatch, batch, change
):
resources = seed_family(database, references=change == "remove")
lookup = research_cascade._shared_document_replacements
lookups = 0
def change_after_lookup(*args):
nonlocal lookups
result = lookup(*args)
lookups += 1
if lookups == 1:
with Session(database) as writer:
for research_id in ("sibling", "last"):
writer.get(
ResearchResource, resources[research_id]
).document_id = "canonical" if change == "add" else None
writer.commit()
return result
monkeypatch.setattr(
research_cascade, "_shared_document_replacements", change_after_lookup
)
assert call_delete_route(batch) == {"status": "success"}
assert lookups == 2
assert not route_session.in_transaction()
with Session(database) as verification:
assert verification.get(ResearchHistory, "owner") is None
assert verification.get(ResearchHistory, "sibling") is not None
if change == "add":
assert_preserved(verification, "sibling", resources["sibling"])
else:
assert verification.get(Document, "canonical") is None
assert verification.get(DocumentBlob, "canonical") is None
@pytest.mark.parametrize("batch", [False, True], ids=["single", "batch"])
def test_retry_rechecks_a_research_claimed_by_another_writer(
database, route_session, monkeypatch, batch
):
resources = seed_family(database)
lookup = research_cascade._shared_document_replacements
changed = False
def claim_after_lookup(*args):
nonlocal changed
result = lookup(*args)
if not changed:
changed = True
with Session(database) as writer:
writer.get(ResearchHistory, "owner").status = "in_progress"
writer.commit()
return result
monkeypatch.setattr(
research_cascade, "_shared_document_replacements", claim_after_lookup
)
response = call_delete_route(batch)
if batch:
assert response == {"status": "success"}
else:
assert response.status_code == 400
assert changed
with Session(database) as verification:
assert (
verification.get(ResearchHistory, "owner").status == "in_progress"
)
assert_preserved(verification, "owner", resources["owner"])
@pytest.mark.parametrize("batch", [False, True], ids=["single", "batch"])
@pytest.mark.parametrize("failure", ["transient-lock", "persistent-lock", "io"])
def test_report_file_survives_until_deletion_commits(
database, route_session, monkeypatch, tmp_path, batch, failure
):
from local_deep_research.config import paths
resources = seed_family(database)
report = tmp_path / "research-report.md"
report.write_text("Retain the report when its database deletion fails.")
monkeypatch.setattr(
paths, "get_research_outputs_directory", lambda: tmp_path
)
with Session(database) as writer:
writer.get(ResearchHistory, "owner").report_path = str(report)
writer.commit()
real_commit = route_session.commit
attempts = 0
def commit():
nonlocal attempts
attempts += 1
# This is a real file: even an ultimately successful retry must not
# remove it before the successful database commit.
assert report.is_file()
if failure == "persistent-lock" or attempts == 1:
message = (
"disk I/O error" if failure == "io" else "database is locked"
)
raise OperationalError(
"COMMIT", None, sqlite3.OperationalError(message)
)
real_commit()
monkeypatch.setattr(route_session, "commit", commit)
response = call_delete_route(batch)
if failure == "transient-lock":
assert response == {"status": "success"}
assert attempts == 2
assert not report.exists()
with Session(database) as verification:
assert verification.get(ResearchHistory, "owner") is None
else:
assert response.status_code == 500
assert attempts == (3 if failure == "persistent-lock" else 1)
assert (
report.read_text()
== "Retain the report when its database deletion fails."
)
with Session(database) as verification:
assert verification.get(ResearchHistory, "owner") is not None
assert_preserved(verification, "owner", resources["owner"])