~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
test(downloads): cover dedup across production sessions (#7172)
LearningCircuitff14cf1a3ec1
diff --git a/tests/research_library/services/test_pdf_dedup_production_sessions.py b/tests/research_library/services/test_pdf_dedup_production_sessions.pynew file mode 100644index 0000000000..5ac93b197b--- /dev/null+++ b/tests/research_library/services/test_pdf_dedup_production_sessions.py@@ -0,0 +1,287 @@+"""PDF deduplication through download_resource's real transaction boundaries."""++import hashlib+from types import SimpleNamespace+from unittest.mock import Mock++import pytest+from sqlalchemy import create_engine, event, inspect+from sqlalchemy.orm import sessionmaker++from local_deep_research.database import encrypted_db, thread_local_session+from local_deep_research.database.models import Base+from local_deep_research.database.models.download_tracker import DownloadTracker+from local_deep_research.database.models.library import (+ Collection,+ Document,+ DocumentBlob,+ DocumentCollection,+ DocumentStatus,+ DownloadQueue,+ SourceType,+)+from local_deep_research.database.models.research import (+ ResearchHistory,+ ResearchResource,+)+from local_deep_research.library.download_management import RetryManager+from local_deep_research.research_library.services import download_service+from local_deep_research.web.routers import rag+++def database(tmp_path, monkeypatch):+ engine = create_engine(f"sqlite:///{tmp_path / 'downloads.sqlite'}")++ @event.listens_for(engine, "connect")+ def enable_foreign_keys(connection, _record):+ connection.execute("PRAGMA foreign_keys=ON")++ Base.metadata.create_all(engine)+ monkeypatch.setattr(+ encrypted_db.db_manager,+ "open_user_database",+ lambda username, password: engine,+ )+ sessions = sessionmaker(bind=engine)+ # Keep the real get_user_db_session context manager and its shared-session+ # commit/rollback behavior; only replace encrypted connection discovery.+ with sessions() as shared_session:+ monkeypatch.setattr(+ thread_local_session,+ "get_metrics_session",+ lambda username, password: shared_session,+ )+ try:+ yield sessions, shared_session+ finally:+ shared_session.rollback()+ engine.dispose()+++ "first_mode,second_mode",+ [+ ("none", "none"),+ ("none", "database"),+ ("database", "none"),+ ("database", "database"),+ ],+)[email protected]("requested_collection", [False, True])+def test_pdf_dedup_commits_storage_links_and_detached_tracker_state(+ database,+ tmp_path,+ monkeypatch,+ first_mode,+ second_mode,+ requested_collection,+):+ sessions, shared_session = database+ with sessions() as seed:+ source = SourceType(+ id="source", name="research_download", display_name="Research"+ )+ research = ResearchHistory(+ id="research",+ query="PDF deduplication",+ mode="quick",+ status="completed",+ created_at="2026-10-05T00:00:00",+ )+ seed.add_all(+ [+ source,+ research,+ Collection(+ id="library",+ name="Library",+ is_default=True,+ collection_type="default_library",+ ),+ Collection(id="selected", name="Selected papers"),+ ]+ )+ seed.flush()+ resources = [+ ResearchResource(+ research_id=research.id,+ title="Identical paper",+ url=url,+ source_type="academic",+ created_at="2026-10-05T00:00:00",+ )+ for url in (+ "https://example.org/paper.pdf",+ "https://mirror.example.org/paper.pdf",+ )+ ]+ seed.add_all(resources)+ seed.flush()+ resource_ids = [resource.id for resource in resources]+ if requested_collection:+ seed.add(+ DownloadQueue(+ resource_id=resource_ids[1],+ research_id=research.id,+ collection_id="selected",+ status=DocumentStatus.PENDING,+ )+ )+ seed.commit()++ service = download_service.DownloadService.__new__(+ download_service.DownloadService+ )+ service.username = "dedup-user"+ service.password = "test-password"+ service.library_root = tmp_path / "library"+ service.legacy_library_root = None+ service._closed = False+ settings = {"research_library.pdf_storage_mode": first_mode}+ service.settings = SimpleNamespace(get_setting=settings.get)+ service.retry_manager = RetryManager(service.username, service.password)++ pdf_bytes = b"%PDF-1.4 identical PDF bytes from separate URLs"+ pdf_hash = hashlib.sha256(pdf_bytes).hexdigest()++ def fetch_pdf(_url, content_type):+ assert content_type == download_service.ContentType.PDF+ assert not shared_session.in_transaction()+ return SimpleNamespace(+ is_success=True,+ content=pdf_bytes,+ status_code=200,+ skip_reason=None,+ )++ downloader = SimpleNamespace(+ can_handle=lambda url: True,+ download_with_result=Mock(side_effect=fetch_pdf),+ )+ service.downloaders = [downloader]+ monkeypatch.setattr(+ service,+ "_extract_text_from_pdf",+ lambda content: "The same extracted paper text.",+ )+ monkeypatch.setattr(+ download_service, "get_source_type_id", lambda *args: "source"+ )+ monkeypatch.setattr(+ download_service, "get_default_library_id", lambda *args: "library"+ )+ from local_deep_research.database import library_init++ monkeypatch.setattr(+ library_init, "get_default_library_id", lambda *args: "library"+ )+ index = Mock()+ monkeypatch.setattr(rag, "trigger_auto_index", index)++ # Observe, but execute, the production no-session branch. A direct call+ # passing a Session would use the same ORM objects and miss mirror bugs.+ original_download = service._download_pdf+ observed_trackers = []++ def observe_download(resource, tracker, session, collection_id):+ assert session is None+ assert inspect(resource).detached+ assert inspect(tracker).detached+ result = original_download(resource, tracker, session, collection_id)+ assert result == (True, None, 200)+ with sessions() as verification:+ stored = (+ verification.query(DownloadTracker)+ .filter_by(url_hash=tracker.url_hash)+ .one()+ )+ assert stored is not tracker+ for field in (+ "file_hash",+ "file_size",+ "is_downloaded",+ "downloaded_at",+ "file_path",+ "file_name",+ ):+ assert getattr(tracker, field) == getattr(stored, field), field+ observed_trackers.append(tracker)+ return result++ monkeypatch.setattr(service, "_download_pdf", observe_download)+ assert service.download_resource(resource_ids[0]) == (True, None)++ # Check the first committed state before a second transaction can repair+ # it, including the lack of a blob in intentionally text-only mode.+ with sessions() as verification:+ canonical = verification.query(Document).one()+ canonical_id = canonical.id+ assert canonical.storage_mode == first_mode+ blob = verification.get(DocumentBlob, canonical_id)+ if first_mode == "database":+ assert blob.pdf_binary == pdf_bytes+ else:+ assert blob is None++ settings["research_library.pdf_storage_mode"] = second_mode+ assert service.download_resource(resource_ids[1]) == (True, None)++ with sessions() as verification:+ canonical = verification.query(Document).one()+ assert canonical.id == canonical_id+ assert canonical.document_hash == pdf_hash+ assert canonical.text_content == "The same extracted paper text."+ assert canonical.status == DocumentStatus.COMPLETED+ assert (+ verification.get(ResearchResource, resource_ids[1]).document_id+ == canonical_id+ )+ links = verification.query(DocumentCollection).all()+ expected_collections = {"library"}+ if requested_collection:+ expected_collections.add("selected")+ assert (+ verification.query(DownloadQueue).one().status+ == DocumentStatus.COMPLETED+ )+ assert {(link.document_id, link.collection_id) for link in links} == {+ (canonical_id, collection) for collection in expected_collections+ }+ assert len(links) == len(expected_collections)++ stored_pdf = "database" in (first_mode, second_mode)+ assert canonical.storage_mode == ("database" if stored_pdf else "none")+ blob = verification.get(DocumentBlob, canonical_id)+ if stored_pdf:+ assert blob.pdf_binary == pdf_bytes+ assert blob.blob_hash == pdf_hash+ assert verification.query(DocumentBlob).count() == 1+ else:+ assert blob is None+ trackers = verification.query(DownloadTracker).all()+ assert len(trackers) == 2+ assert all(tracker.is_downloaded for tracker in trackers)+ assert all(tracker.file_hash == pdf_hash for tracker in trackers)+ assert all(+ tracker.file_path == ("database" if stored_pdf else None)+ for tracker in trackers+ )++ assert len(observed_trackers) == 2+ assert observed_trackers[0] is not observed_trackers[1]+ assert downloader.download_with_result.call_count == 2+ assert index.call_count == 2+ assert index.call_args_list[0].args == (+ [canonical_id],+ "library",+ service.username,+ service.password,+ )+ assert index.call_args_list[1].args == (+ [canonical_id],+ "selected" if requested_collection else "library",+ service.username,+ service.password,+ )+ assert not shared_session.in_transaction()