Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

test(downloads): cover dedup across production sessions (#7172)

LearningCircuitff14cf1a3ec1
diff --git a/tests/research_library/services/test_pdf_dedup_production_sessions.py b/tests/research_library/services/test_pdf_dedup_production_sessions.py
new file mode 100644
index 0000000000..5ac93b197b
--- /dev/null
+++ b/tests/research_library/services/test_pdf_dedup_production_sessions.py
@@ -0,0 +1,287 @@
+"""PDF deduplication through download_resource's real transaction boundaries."""
+
+import hashlib
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+from sqlalchemy import create_engine, event, inspect
+from sqlalchemy.orm import sessionmaker
+
+from local_deep_research.database import encrypted_db, thread_local_session
+from local_deep_research.database.models import Base
+from local_deep_research.database.models.download_tracker import DownloadTracker
+from local_deep_research.database.models.library import (
+ Collection,
+ Document,
+ DocumentBlob,
+ DocumentCollection,
+ DocumentStatus,
+ DownloadQueue,
+ SourceType,
+)
+from local_deep_research.database.models.research import (
+ ResearchHistory,
+ ResearchResource,
+)
+from local_deep_research.library.download_management import RetryManager
+from local_deep_research.research_library.services import download_service
+from local_deep_research.web.routers import rag
+
+
+def database(tmp_path, monkeypatch):
+ engine = create_engine(f"sqlite:///{tmp_path / 'downloads.sqlite'}")
+
+ @event.listens_for(engine, "connect")
+ def enable_foreign_keys(connection, _record):
+ connection.execute("PRAGMA foreign_keys=ON")
+
+ Base.metadata.create_all(engine)
+ monkeypatch.setattr(
+ encrypted_db.db_manager,
+ "open_user_database",
+ lambda username, password: engine,
+ )
+ sessions = sessionmaker(bind=engine)
+ # Keep the real get_user_db_session context manager and its shared-session
+ # commit/rollback behavior; only replace encrypted connection discovery.
+ with sessions() as shared_session:
+ monkeypatch.setattr(
+ thread_local_session,
+ "get_metrics_session",
+ lambda username, password: shared_session,
+ )
+ try:
+ yield sessions, shared_session
+ finally:
+ shared_session.rollback()
+ engine.dispose()
+
+
+ "first_mode,second_mode",
+ [
+ ("none", "none"),
+ ("none", "database"),
+ ("database", "none"),
+ ("database", "database"),
+ ],
+)
[email protected]("requested_collection", [False, True])
+def test_pdf_dedup_commits_storage_links_and_detached_tracker_state(
+ database,
+ tmp_path,
+ monkeypatch,
+ first_mode,
+ second_mode,
+ requested_collection,
+):
+ sessions, shared_session = database
+ with sessions() as seed:
+ source = SourceType(
+ id="source", name="research_download", display_name="Research"
+ )
+ research = ResearchHistory(
+ id="research",
+ query="PDF deduplication",
+ mode="quick",
+ status="completed",
+ created_at="2026-10-05T00:00:00",
+ )
+ seed.add_all(
+ [
+ source,
+ research,
+ Collection(
+ id="library",
+ name="Library",
+ is_default=True,
+ collection_type="default_library",
+ ),
+ Collection(id="selected", name="Selected papers"),
+ ]
+ )
+ seed.flush()
+ resources = [
+ ResearchResource(
+ research_id=research.id,
+ title="Identical paper",
+ url=url,
+ source_type="academic",
+ created_at="2026-10-05T00:00:00",
+ )
+ for url in (
+ "https://example.org/paper.pdf",
+ "https://mirror.example.org/paper.pdf",
+ )
+ ]
+ seed.add_all(resources)
+ seed.flush()
+ resource_ids = [resource.id for resource in resources]
+ if requested_collection:
+ seed.add(
+ DownloadQueue(
+ resource_id=resource_ids[1],
+ research_id=research.id,
+ collection_id="selected",
+ status=DocumentStatus.PENDING,
+ )
+ )
+ seed.commit()
+
+ service = download_service.DownloadService.__new__(
+ download_service.DownloadService
+ )
+ service.username = "dedup-user"
+ service.password = "test-password"
+ service.library_root = tmp_path / "library"
+ service.legacy_library_root = None
+ service._closed = False
+ settings = {"research_library.pdf_storage_mode": first_mode}
+ service.settings = SimpleNamespace(get_setting=settings.get)
+ service.retry_manager = RetryManager(service.username, service.password)
+
+ pdf_bytes = b"%PDF-1.4 identical PDF bytes from separate URLs"
+ pdf_hash = hashlib.sha256(pdf_bytes).hexdigest()
+
+ def fetch_pdf(_url, content_type):
+ assert content_type == download_service.ContentType.PDF
+ assert not shared_session.in_transaction()
+ return SimpleNamespace(
+ is_success=True,
+ content=pdf_bytes,
+ status_code=200,
+ skip_reason=None,
+ )
+
+ downloader = SimpleNamespace(
+ can_handle=lambda url: True,
+ download_with_result=Mock(side_effect=fetch_pdf),
+ )
+ service.downloaders = [downloader]
+ monkeypatch.setattr(
+ service,
+ "_extract_text_from_pdf",
+ lambda content: "The same extracted paper text.",
+ )
+ monkeypatch.setattr(
+ download_service, "get_source_type_id", lambda *args: "source"
+ )
+ monkeypatch.setattr(
+ download_service, "get_default_library_id", lambda *args: "library"
+ )
+ from local_deep_research.database import library_init
+
+ monkeypatch.setattr(
+ library_init, "get_default_library_id", lambda *args: "library"
+ )
+ index = Mock()
+ monkeypatch.setattr(rag, "trigger_auto_index", index)
+
+ # Observe, but execute, the production no-session branch. A direct call
+ # passing a Session would use the same ORM objects and miss mirror bugs.
+ original_download = service._download_pdf
+ observed_trackers = []
+
+ def observe_download(resource, tracker, session, collection_id):
+ assert session is None
+ assert inspect(resource).detached
+ assert inspect(tracker).detached
+ result = original_download(resource, tracker, session, collection_id)
+ assert result == (True, None, 200)
+ with sessions() as verification:
+ stored = (
+ verification.query(DownloadTracker)
+ .filter_by(url_hash=tracker.url_hash)
+ .one()
+ )
+ assert stored is not tracker
+ for field in (
+ "file_hash",
+ "file_size",
+ "is_downloaded",
+ "downloaded_at",
+ "file_path",
+ "file_name",
+ ):
+ assert getattr(tracker, field) == getattr(stored, field), field
+ observed_trackers.append(tracker)
+ return result
+
+ monkeypatch.setattr(service, "_download_pdf", observe_download)
+ assert service.download_resource(resource_ids[0]) == (True, None)
+
+ # Check the first committed state before a second transaction can repair
+ # it, including the lack of a blob in intentionally text-only mode.
+ with sessions() as verification:
+ canonical = verification.query(Document).one()
+ canonical_id = canonical.id
+ assert canonical.storage_mode == first_mode
+ blob = verification.get(DocumentBlob, canonical_id)
+ if first_mode == "database":
+ assert blob.pdf_binary == pdf_bytes
+ else:
+ assert blob is None
+
+ settings["research_library.pdf_storage_mode"] = second_mode
+ assert service.download_resource(resource_ids[1]) == (True, None)
+
+ with sessions() as verification:
+ canonical = verification.query(Document).one()
+ assert canonical.id == canonical_id
+ assert canonical.document_hash == pdf_hash
+ assert canonical.text_content == "The same extracted paper text."
+ assert canonical.status == DocumentStatus.COMPLETED
+ assert (
+ verification.get(ResearchResource, resource_ids[1]).document_id
+ == canonical_id
+ )
+ links = verification.query(DocumentCollection).all()
+ expected_collections = {"library"}
+ if requested_collection:
+ expected_collections.add("selected")
+ assert (
+ verification.query(DownloadQueue).one().status
+ == DocumentStatus.COMPLETED
+ )
+ assert {(link.document_id, link.collection_id) for link in links} == {
+ (canonical_id, collection) for collection in expected_collections
+ }
+ assert len(links) == len(expected_collections)
+
+ stored_pdf = "database" in (first_mode, second_mode)
+ assert canonical.storage_mode == ("database" if stored_pdf else "none")
+ blob = verification.get(DocumentBlob, canonical_id)
+ if stored_pdf:
+ assert blob.pdf_binary == pdf_bytes
+ assert blob.blob_hash == pdf_hash
+ assert verification.query(DocumentBlob).count() == 1
+ else:
+ assert blob is None
+ trackers = verification.query(DownloadTracker).all()
+ assert len(trackers) == 2
+ assert all(tracker.is_downloaded for tracker in trackers)
+ assert all(tracker.file_hash == pdf_hash for tracker in trackers)
+ assert all(
+ tracker.file_path == ("database" if stored_pdf else None)
+ for tracker in trackers
+ )
+
+ assert len(observed_trackers) == 2
+ assert observed_trackers[0] is not observed_trackers[1]
+ assert downloader.download_with_result.call_count == 2
+ assert index.call_count == 2
+ assert index.call_args_list[0].args == (
+ [canonical_id],
+ "library",
+ service.username,
+ service.password,
+ )
+ assert index.call_args_list[1].args == (
+ [canonical_id],
+ "selected" if requested_collection else "library",
+ service.username,
+ service.password,
+ )
+ assert not shared_session.in_transaction()