Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(library): preserve document identity in sync and deletion (#7201)

LearningCircuitf9ff45ff28a2
diff --git a/changelog.d/+library-document-identity.bugfix.md b/changelog.d/+library-document-identity.bugfix.md
new file mode 100644
index 0000000000..97138feb1f
--- /dev/null
+++ b/changelog.d/+library-document-identity.bugfix.md
@@ -0,0 +1 @@
+Library sync and document deletion now use each document's own storage and only reset matching download trackers, preserving other PDF versions whose URLs normalize to the same address. The primary deletion service and removal from a final collection also preserve other versions' trackers. Existing files and blobs remain recognized even when their storage-mode labels are outdated.
diff --git a/src/local_deep_research/research_library/deletion/utils/cascade_helper.py b/src/local_deep_research/research_library/deletion/utils/cascade_helper.py
index 234349c868..6f16df10ed 100644
--- a/src/local_deep_research/research_library/deletion/utils/cascade_helper.py
+++ b/src/local_deep_research/research_library/deletion/utils/cascade_helper.py
@@ -924,12 +924,13 @@ class CascadeHelper:
if not document.original_url:
return False
- # B2 compat (#6758): match legacy-hash rows as well as the
- # canonical hash so pre-upgrade trackers are still reset.
- from ...utils import find_tracker_by_url
+ # URL-equivalent trackers can belong to different PDF versions.
+ # Reset only a tracker whose content hash or document link matches,
+ # retaining the legacy URL-hash fallback for the deleted document.
+ from ...utils import find_tracker_for_document
try:
- tracker = find_tracker_by_url(session, str(document.original_url))
+ tracker = find_tracker_for_document(session, document)
if tracker:
tracker.is_downloaded = False # type: ignore[assignment]
diff --git a/src/local_deep_research/research_library/services/library_service.py b/src/local_deep_research/research_library/services/library_service.py
index f719ad5a91..b9c3511249 100644
--- a/src/local_deep_research/research_library/services/library_service.py
+++ b/src/local_deep_research/research_library/services/library_service.py
@@ -39,6 +39,7 @@ from ..utils import (
document_awaits_pdf_clause,
document_counts_as_downloaded,
find_tracker_by_url,
+ find_tracker_for_document,
get_absolute_path_from_settings,
open_file_location,
resource_document_clause,
@@ -75,9 +76,12 @@ def _names_a_file(file_path: Optional[str]) -> bool:
def _tracker_describes_storage(
tracker_path: Optional[str], doc: Document
) -> bool:
- """Whether a URL's tracker path names where ``doc`` stores its PDF.
+ """Whether an identifying tracker's path names where ``doc`` stores its PDF.
- Every document with the URL shares its tracker, and a download that
+ The sync asks this only for a tracker whose content hash or explicit
+ document link identifies ``doc`` (``find_tracker_for_document``):
+ URL-equivalent addresses can hold different PDF versions, so a URL
+ match alone does not say whose file a tracker names. A download that
stores a PDF writes to the tracker where it stored it:
* filesystem storage: the file's path, which
@@ -92,20 +96,17 @@ def _tracker_describes_storage(
``"database"`` for a matched document with that label too.
So the tracker describes a document when it names the document's own
- path, or when it is ``"database"`` and the document is labelled as
- stored in the database, without a path. The label alone does not say
- that a blob was stored (a failed-download placeholder keeps the
- column's ``"database"`` default), so the sync still checks the
- document's own blob. ``"database"`` can describe several documents of
- the URL at once.
+ path. An identifying tracker can still carry a stale path, and the
+ document's own path wins over it. A document without a path of its
+ own, a database-stored one or a legacy row that kept its path only in
+ the tracker, takes the path of the tracker that identifies it. Neither
+ ``"database"`` nor the document's label says that a blob was stored (a
+ failed-download placeholder keeps the column's ``"database"``
+ default), so the sync still checks the document's own blob.
"""
- if tracker_path == doc.file_path:
+ if not doc.file_path:
return True
- return (
- tracker_path == "database"
- and doc.storage_mode == "database"
- and doc.file_path is None
- )
+ return tracker_path == doc.file_path
def _missing_file_entry(
@@ -117,17 +118,20 @@ def _missing_file_entry(
``path`` is the stored path, which is relative to the user's library
root (the library listing returns the same value as ``file_path``), or
``"invalid"`` when it does not resolve inside that root
- (``resolved_path`` is ``None``). For a document whose PDF the URL's
- tracker records as stored in the database, the stored value is
- ``"database"``, which names no file. It is never ``resolved_path``
- itself: the absolute path is the server's directory layout, which
- ``/library/api/check-downloads`` and the library listing withhold from
- the client (#6462).
+ (``resolved_path`` is ``None``). For a document whose PDF its
+ identifying tracker records as stored in the database, or that is
+ itself labelled as stored there, the stored value is ``"database"``: a
+ fixed label that names no file, returned whatever ``resolved_path`` is.
+ It is never ``resolved_path`` itself: the absolute path is the server's
+ directory layout, which ``/library/api/check-downloads`` and the
+ library listing withhold from the client (#6462).
"""
return {
"id": doc.id,
"title": doc.title,
- "path": stored_path if resolved_path and stored_path else "invalid",
+ "path": stored_path
+ if stored_path == "database" or (resolved_path and stored_path)
+ else "invalid",
"url": doc.original_url,
}
@@ -251,15 +255,17 @@ class LibraryService:
longer offers a PDF that is not there, and ``tracker`` is reset
to not downloaded unless it is None (see ``_clear_missing_pdf``).
It is counted in ``files_missing`` and listed in ``missing_files``
- under ``stored_path``, the document's own path that named the
- missing file (see ``_missing_file_entry``), and
+ under ``stored_path``, the path that named the missing file: the
+ document's own path or, for a document without one, the path of the
+ tracker that identifies it (see ``_missing_file_entry``), and
``document_awaits_pdf`` holds for it, so the download manager
downloads its PDF into it again.
The label is set here because what went missing was a PDF: the
sync calls this only for a missing file that the document's own
- ``file_path`` names, and only PDF storage writes a file path into
- a document. A
+ ``file_path`` names, or that the identifying tracker of a document
+ without a path of its own names, and the file paths of documents and
+ trackers come only from PDF storage. A
failed-download placeholder that a PDF download completed before
``_download_pdf`` labelled such documents "pdf" is still labelled
"unknown", and would otherwise never be downloaded again.
@@ -278,16 +284,17 @@ class LibraryService:
It is stored the way text-only documents are (``storage_mode``
"none", ``file_path`` ``FILE_PATH_TEXT_ONLY``), and ``tracker``,
unless it is None, is reset to not downloaded, so neither names the
- missing file any more. The sync passes the URL's tracker when it
- names this document's path or no path, and None when it describes
- where other documents store their PDF (another document's path, or
- ``"database"`` for a document not stored there, see
- ``_tracker_describes_storage``), which is left as it is. The reset
- counts in ``trackers_updated`` only when it changes the tracker: a
- tracker without a path can be reset already, for example by the
- deletion of another document of the URL whose file it named, when
- the sync read that document first. Counted again, that tracker
- counted twice in that read order and once in the other.
+ missing file any more. The sync passes the tracker whose content
+ hash or explicit document link identifies this document (see
+ ``find_tracker_for_document``) when it names this document's path,
+ supplies the path of a document without one of its own, or has no
+ path. It passes None when no tracker identifies the document, or
+ when the identifying tracker names another path than the
+ document's own (see ``_tracker_describes_storage``); that tracker is
+ left as it is. The reset counts in ``trackers_updated`` only when it
+ changes the tracker, because a tracker without a path can be reset
+ already. A tracker identifies at most one document, so the sync
+ changes it at most once.
The sync calls this directly, without reporting the document, for
a download that keeps its extracted text but no longer has its
@@ -1370,21 +1377,34 @@ class LibraryService:
)
return False
- # Get file path from tracker (only if document has original_url).
- # B2 compat (#6758): match legacy-hash rows so pre-upgrade
- # documents still unlink their physical file.
- tracker = None
- if doc.original_url:
- tracker = find_tracker_by_url(session, doc.original_url)
+ # A canonical URL can cover several historical PDF versions.
+ # Use this document's storage; a tracker is only a legacy fallback
+ # when its content hash or document link proves ownership.
+ tracker = find_tracker_for_document(session, doc)
+ stored_path = doc.file_path
+ if (
+ not stored_path
+ and tracker
+ and doc.storage_mode != "none"
+ and not self._has_blob_in_db(session, doc.id)
+ ):
+ # A blob-backed or explicitly text-only document must not
+ # borrow a stale physical tracker path. A legacy row with
+ # neither a path nor a blob can still fall back to the
+ # verified tracker's path, as the library sync does.
+ stored_path = tracker.file_path
# Delete physical file. Own-root only: never resolve via the
# legacy shared root, where a colliding relative path can be
# another tenant's file (#5521), consistent with the delete
# sites in document_deletion.py / PDFStorageManager.delete_pdf.
- if tracker and tracker.file_path:
+ if stored_path and stored_path not in (
+ *FILE_PATH_SENTINELS,
+ "database",
+ ):
try:
file_path = get_absolute_path_from_settings(
- tracker.file_path,
+ stored_path,
self.username,
)
if file_path and file_path.is_file():
@@ -1512,67 +1532,74 @@ class LibraryService:
Sync library database with filesystem.
Check which PDF files exist and update database accordingly.
- Every document with the same URL shares that URL's tracker. The
- tracker is used for a document only when it names where the
- document stores its PDF (see ``_tracker_describes_storage``): the
- document's own path, or ``"database"`` for a document labelled as
- stored in the database. Each document is judged by what it stores
- itself, and a document the tracker does not describe leaves the
- tracker unchanged.
+ Every document with the same URL shares that URL's tracker, and
+ URL-equivalent addresses (``paper.pdf`` and
+ ``paper.pdf?utm_source=old``) can hold different PDF versions, each
+ in a document of its own. So a tracker supplies a path for a
+ document, or is reset for it, only when its content hash or
+ explicit document link identifies the document, with contradictory
+ identity fields rejected (see ``find_tracker_for_document``), and
+ only when it names where the document stores its PDF (see
+ ``_tracker_describes_storage``): the document's own path, which wins
+ over any other tracker path, or for a document without a path of
+ its own, the tracker's ``"database"`` or legacy path. Each document
+ is judged by what it stores itself, and a tracker that does not
+ describe the document is left unchanged.
A download whose PDF file is missing is handled by what it still
stores (see ``_remaining_content``):
* its PDF, as a database blob or a file at its own ``file_path``:
kept unchanged;
- * only its extracted text, while its own ``file_path`` names a
- file, and it still has its research resource: kept with its text
- but made text-only, the URL's tracker reset if it names that file
- or no file, and counted in ``files_missing`` and
- ``missing_files``, so that the download manager downloads its PDF
- again (see ``_keep_text_report_missing_pdf``);
- * only its extracted text, while its own ``file_path`` names a
- file, with no research resource (e.g. a download whose resource
- was deleted): kept with its text and made text-only, the URL's
- tracker reset as above, and counted in ``files_found``. No
- download path asks for its PDF, so it is not reported, but a
- later download of the same PDF bytes stores the PDF in it (see
+ * only its extracted text, while its own ``file_path`` (or, without
+ one, its identifying tracker's path) names a file, and it still
+ has its research resource: kept with its text but made text-only,
+ its identifying tracker reset if it names that file or no file,
+ and counted in ``files_missing`` and ``missing_files``, so that
+ the download manager downloads its PDF again (see
+ ``_keep_text_report_missing_pdf``);
+ * only its extracted text, while a file is named as above, with no
+ research resource (e.g. a download whose resource was deleted):
+ kept with its text and made text-only, its identifying tracker
+ reset as above, and counted in ``files_found``. No download path
+ asks for its PDF, so it is not reported, but a later download of
+ the same PDF bytes stores the PDF in it (see
``_clear_missing_pdf``);
* only its extracted text, with no file named (a "database" label
without a blob): kept unchanged;
* nothing: deleted, so that its URL can be downloaded again, and
- counted in ``files_missing``. The URL's tracker is reset if it
- names the document's file. If it is ``"database"`` and the
- document is stored there, it is reset only when, once every
- document has been checked, no other completed document under
- that tracker still holds a PDF blob: until then the URL's PDF is
- still stored in the database, as the tracker records.
+ counted in ``files_missing``. Its identifying tracker is reset if
+ it names the document's file. If that tracker is ``"database"``,
+ it is reset only when, once every document has been checked, no
+ other completed document under that tracker still holds a PDF
+ blob, whether the tracker identifies that document or not: until
+ then the URL's PDF is still stored in the database, as the
+ tracker records.
Left alone before any of these checks: a document whose own
``file_path`` is a sentinel, such as ``blob_deleted`` after "Remove
PDF to save space", and a document whose ``storage_mode`` is "none"
- or unset (a text-only document), unless the URL's tracker names its
- path.
+ (a text-only document), whatever a tracker names. A document without
+ a path of its own that no identifying tracker gives one is left
+ alone too, unless it is labelled as stored in the database: its
+ missing or ambiguous metadata cannot prove which file is absent.
``missing_files`` lists every document counted in ``files_missing``
- whose ``file_path`` names a file, by the path stored for it,
- relative to the library root, never by the absolute path on the
- server (see ``_missing_file_entry``). It also lists every document
- deleted while the URL's tracker records it as stored in the
- database, as ``"database"``.
+ by the path stored for it, relative to the library root, never by
+ the absolute path on the server (see ``_missing_file_entry``): the
+ path that named its missing file, ``"database"`` for a document
+ whose PDF was to be a database blob, or ``"invalid"`` for a path
+ outside the library root.
``trackers_updated`` counts the trackers the sync changed, each
- once, whichever document of their URL it reads first (see
- ``_clear_missing_pdf``).
+ once (see ``_clear_missing_pdf``).
Returns:
Statistics about the sync operation
"""
with get_user_db_session(self.username) as session:
# Sync only research downloads — uploads have no original_url
- # and no DownloadTracker, so they don't belong in this routine
- # (and the destructive `else` branch below would otherwise
- # silently delete every uploaded document).
+ # and no DownloadTracker, so they don't belong in this routine.
# Don't load the large text_content body: this sync loop only
# reads original_url/id/title (and may delete the row), so loading
# every completed document's full text would needlessly exhaust
@@ -1620,32 +1647,66 @@ class LibraryService:
settings_manager = None
try:
for doc in documents:
- # Get download tracker. B2 compat (#6758): match
- # legacy-hash rows so pre-upgrade documents are not
- # mistaken for tracker-less orphans and deleted below.
- tracker = find_tracker_by_url(session, doc.original_url)
- if tracker is not None and tracker.file_path == "database":
- database_tracker_documents.setdefault(
- tracker.id, []
- ).append(doc.id)
# A URL has one tracker, shared by every document with
# that URL: the documents of two researches that cite
# the same paper, or a failed-download placeholder that
# "Extract Text to DB" completed with text next to the
- # document that holds the PDF. The tracker's path is the
- # one written last. It describes this document only when
- # it names where the document stores its PDF: the
- # document's own path, which every download that stores
- # a PDF file writes to both, or "database" for a
- # document labelled as stored in the database (see
+ # document that holds the PDF. URL-equivalent addresses,
+ # such as ``paper.pdf`` and ``paper.pdf?utm_source=old``,
+ # can also hold different PDF versions, each in a
+ # document of its own. So a URL match alone does not say
+ # whose file a tracker names: a tracker supplies a path
+ # for this document, or is reset for it, only when its
+ # content hash or explicit document link identifies the
+ # document (find_tracker_for_document). The census of
+ # "database" trackers below also counts every document
+ # under the URL's tracker (``url_tracker``). B2 compat
+ # (#6758): both lookups match legacy-hash rows too.
+ url_tracker = find_tracker_by_url(session, doc.original_url)
+ tracker = find_tracker_for_document(session, doc)
+ for database_tracker in (url_tracker, tracker):
+ if (
+ database_tracker is not None
+ and database_tracker.file_path == "database"
+ ):
+ database_tracker_documents.setdefault(
+ database_tracker.id, []
+ ).append(doc.id)
+ # Explicit text-only mode and a sentinel path record a
+ # PDF left out on purpose. Sentinel paths
+ # (``blob_deleted``, ``text_only_not_stored``,
+ # ``metadata_only``) name no file by design: resolving
+ # one as a relative path always misses and would run
+ # ``delete_document_completely`` on the text/chunks/
+ # vectors the sentinel was protecting (issue #6760
+ # follow-up). "Remove PDF to save space"
+ # (delete_blob_only) writes ``blob_deleted`` to the
+ # document but leaves the tracker naming the copy it
+ # removed. That PDF was removed on purpose, so it is
+ # neither missing nor to be downloaded again. Such a
+ # document borrows no tracker path, not even from a
+ # tracker that identifies it.
+ if (
+ doc.storage_mode == "none"
+ or doc.file_path in FILE_PATH_SENTINELS
+ ):
+ stats["files_found"] += 1
+ continue
+ # The tracker's path is the one written last. It
+ # describes this document when it names the document's
+ # own path, which every download that stores a PDF file
+ # writes to both. A document without a path of its own,
+ # a database-stored one (its tracker records
+ # "database") or a legacy row, takes the path of the
+ # tracker that identifies it (see
# _tracker_describes_storage). Any other document is
# judged by what it stores itself, in the ``else``
- # branch, and leaves the tracker alone. Judged by
- # another document's file, it was counted found while
- # its own file was missing, or labelled a PDF, made
- # text-only and reported missing for a file it never
- # had, which the download manager then downloaded into
- # it as a second copy.
+ # branch, and leaves the tracker alone.
+ # Judged by another document's file, it was counted
+ # found while its own file was missing, or labelled a
+ # PDF, made text-only and reported missing for a file it
+ # never had, which the download manager then downloaded
+ # into it as a second copy.
if (
tracker is not None
and tracker.file_path
@@ -1656,17 +1717,12 @@ class LibraryService:
tracker = None
if tracker and tracker.file_path:
- # The tracker names where this document stores its
- # PDF: its own path, or the database.
- # Sentinel paths (``blob_deleted``,
- # ``text_only_not_stored``, ``metadata_only``) name
- # no file by design — e.g. a blob purge keeps the
- # document complete with ``file_path="blob_deleted"``.
- # Resolving one as a relative path always misses and
- # would run ``delete_document_completely`` on the
- # text/chunks/vectors the sentinel was protecting
- # (issue #6760 follow-up). Leave them alone like the
- # text-only branch below.
+ # The tracker identifies this document and names
+ # where it stores its PDF: its own path, the
+ # database, or, for a document without a path of its
+ # own, the tracker's legacy path. A sentinel there
+ # names no file either: leave the document alone, as
+ # above.
if tracker.file_path in FILE_PATH_SENTINELS:
stats["files_found"] += 1
continue
@@ -1690,9 +1746,6 @@ class LibraryService:
if has_blob:
stats["files_found"] += 1
continue
- # Text-only documents (storage none) have no file
- # either; leave them alone (see the no-path branch
- # below for the tracker.file_path=None shape).
# Check if the file exists in the per-user root.
if settings_manager is None:
settings_manager = get_settings_manager(
@@ -1756,10 +1809,10 @@ class LibraryService:
)
if tracker.file_path == "database":
- # "database" describes every document of
- # the URL stored there, and another one can
- # still hold its PDF blob. Decided after the
- # loop.
+ # "database" records the URL's PDF for every
+ # document of the URL stored there, and
+ # another one can still hold its PDF blob.
+ # Decided after the loop.
emptied_database_trackers[tracker.id] = tracker
else:
# Reset tracker
@@ -1777,24 +1830,19 @@ class LibraryService:
)
removed_doc_ids.append(doc.id)
else:
- # No tracker, a tracker without a path, or one that
- # names where other documents store their PDF:
- # another document's path, or "database" for a
- # document not stored there. That tracker is not
- # passed on (``tracker`` is None). Text-only
- # documents (storage_mode "none") never have a file
- # by design — deleting them here would destroy
- # text, chunks, and vectors that are recoverable
- # only by re-download (dead links = permanent loss).
- # Leave them alone. A sentinel path on the document
- # means the same: "Remove PDF to save space"
- # (delete_blob_only) writes ``blob_deleted`` to the
- # document but leaves the tracker naming the copy it
- # removed. That PDF was removed on purpose, so it is
- # neither missing nor to be downloaded again.
- if (doc.storage_mode in (None, "none")) or (
- doc.file_path in FILE_PATH_SENTINELS
- ):
+ # No tracker identifies the document, its tracker has
+ # no path, or its tracker names another path than
+ # the document's own: another file, or "database"
+ # for a document with a file path. Only the
+ # document's own tracker without a path is passed
+ # on; otherwise ``tracker`` is None. A document
+ # without a path of its own that is not labelled as
+ # stored in the database names no storage to check,
+ # e.g. a pre-upgrade row with an unset (NULL)
+ # storage_mode. That missing or ambiguous metadata
+ # cannot prove which file is absent, so the document
+ # is left alone.
+ if not doc.file_path and doc.storage_mode != "database":
stats["files_found"] += 1
continue
# Any other label can be stale, e.g. a lost tracker
@@ -1842,7 +1890,11 @@ class LibraryService:
# Nothing of it is left: delete it so that its URL
# can be downloaded again. A file its own path names
# is reported missing, as the branch above reports
- # one that the tracker names.
+ # one that the tracker names. A document whose PDF
+ # was to be a database blob (its label or its path
+ # says "database") is reported as ``"database"``, as
+ # the branch above reports one that a "database"
+ # tracker describes.
stats["files_missing"] += 1
if _names_a_file(doc.file_path):
stats["missing_files"].append(
@@ -1856,6 +1908,13 @@ class LibraryService:
),
)
)
+ elif (
+ doc.storage_mode == "database"
+ or doc.file_path == "database"
+ ):
+ stats["missing_files"].append(
+ _missing_file_entry(doc, "database", None)
+ )
from ..deletion.utils.cascade_helper import (
CascadeHelper,
)
@@ -1871,12 +1930,13 @@ class LibraryService:
# A "database" tracker records that its URL's PDF is stored in
# the database, which can hold it for several documents of the
- # URL. One of them was deleted above with nothing left. While
- # another still holds a PDF blob, the record is true and the
- # tracker is left as it is; otherwise it is reset. Every
- # document under the tracker has been checked by now, so the
- # order in which the sync read them does not change whether it
- # is reset.
+ # URL. The one it identifies was deleted above with nothing
+ # left. While another still holds a PDF blob, the record is true
+ # and the tracker is left as it is; otherwise it is reset. A
+ # document the tracker does not identify cannot reset it, but its
+ # blob still keeps the record true. Every document under the
+ # tracker has been checked by now, so the order in which the
+ # sync read them does not change whether it is reset.
removed = set(removed_doc_ids)
for tracker_id, tracker in emptied_database_trackers.items():
others = [
diff --git a/src/local_deep_research/research_library/utils/__init__.py b/src/local_deep_research/research_library/utils/__init__.py
index 643f7e88d8..97a4132b27 100644
--- a/src/local_deep_research/research_library/utils/__init__.py
+++ b/src/local_deep_research/research_library/utils/__init__.py
@@ -295,9 +295,10 @@ def document_awaits_pdf(document) -> bool:
research resource (``resource_id``). It labels the document a PDF and
stores it the way text-only documents are stored (``file_type`` "pdf",
``storage_mode`` "none", ``file_path`` ``FILE_PATH_TEXT_ONLY``), resets
- the URL's tracker unless it names where other documents store their
- PDF, and reports the file missing, so every document it reports that
- way matches here
+ the tracker that identifies the document (see
+ ``find_tracker_for_document``) if it names that file or no file, and
+ reports the file missing, so every document it reports that way matches
+ here
(see ``LibraryService._keep_text_report_missing_pdf``). Everywhere else
a completed document counts as downloaded, so ``download_resource`` and
the PDF queueing paths (``queue_research_downloads`` and the "Download
@@ -334,15 +335,16 @@ def document_awaits_pdf(document) -> bool:
``storage_mode``, so a writer that set it to "none", the value for a
document without a stored PDF, would make every extracted text ask for
a PDF. The sync can set the label itself because what it found missing
- was a PDF: only ``PDFStorageManager.save_pdf`` writes a file path into a
- document, and the sync converts a document only for a missing file
- that its own ``file_path`` names. The URL's tracker, which every
- document with that URL shares, counts for a document only when it
- names that path, or when it is ``"database"`` and the document is
- labelled as stored in the database, which names no file and so leads
- to no conversion. So a document it converts with another label, such as a
- failed-download placeholder that a PDF download completed before
- ``_download_pdf`` labelled such documents "pdf", is picked up too.
+ was a PDF: the file paths of documents and trackers come only from PDF
+ storage (``PDFStorageManager.save_pdf``), and the sync converts a document
+ only for a missing file that its own ``file_path`` names or, for a
+ document without a path of its own, that the tracker identifying it
+ names. A tracker counts for a document only when its content hash or
+ explicit document link identifies the document, and its ``"database"``
+ path names no file and so leads to no conversion. So a document it
+ converts with another label, such as a failed-download placeholder that
+ a PDF download completed before ``_download_pdf`` labelled such
+ documents "pdf", is picked up too.
No other writer leaves a research download in that shape: a
"none"-mode download keeps ``file_path=None``, "Remove PDF to save
@@ -462,6 +464,38 @@ def find_tracker_by_url(session, url: str):
return None
+def find_tracker_for_document(session, document: Document):
+ """Find a URL tracker whose content identity belongs to this document.
+
+ URL equivalence alone is insufficient for destructive operations: legacy
+ trackers can contain different PDF versions at the same canonical URL.
+ Require a matching content hash or explicit document link, reject
+ contradictory identity fields, and keep looking through legacy hashes
+ when the canonical tracker belongs to another document.
+ """
+ from ...database.models.download_tracker import DownloadTracker
+
+ if not document.original_url:
+ return None
+ for candidate in get_url_hash_candidates(document.original_url):
+ tracker = (
+ session.query(DownloadTracker).filter_by(url_hash=candidate).first()
+ )
+ if tracker is None:
+ continue
+ if tracker.library_document_id and (
+ tracker.library_document_id != document.id
+ ):
+ continue
+ if tracker.file_hash and tracker.file_hash != document.document_hash:
+ continue
+ if tracker.library_document_id == document.id or (
+ tracker.file_hash and tracker.file_hash == document.document_hash
+ ):
+ return tracker
+ return None
+
+
def ensure_in_collection(
session, document_id: str, collection_id: str
) -> "DocumentCollection":
diff --git a/tests/research_library/deletion/utils/test_cascade_helper.py b/tests/research_library/deletion/utils/test_cascade_helper.py
index e3fddca7e0..44f43e9d9c 100644
--- a/tests/research_library/deletion/utils/test_cascade_helper.py
+++ b/tests/research_library/deletion/utils/test_cascade_helper.py
@@ -438,8 +438,12 @@ class TestCascadeHelperUpdateDownloadTracker:
mock_session = MagicMock()
mock_doc = MagicMock()
mock_doc.original_url = "https://example.com/paper.pdf"
+ mock_doc.id = "document-1"
+ mock_doc.document_hash = "content-hash"
mock_tracker = MagicMock()
+ mock_tracker.library_document_id = mock_doc.id
+ mock_tracker.file_hash = mock_doc.document_hash
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_tracker
# Patch where update_download_tracker's call chain reads it:
diff --git a/tests/research_library/services/test_download_service_3827_fix.py b/tests/research_library/services/test_download_service_3827_fix.py
index f6d102b3ef..ad96ba15f9 100644
--- a/tests/research_library/services/test_download_service_3827_fix.py
+++ b/tests/research_library/services/test_download_service_3827_fix.py
@@ -1993,7 +1993,7 @@ def test_sync_leaves_sentinel_tracker_path_alone(mocker):
side_effect=_cm,
)
mocker.patch(
- "local_deep_research.research_library.services.library_service.find_tracker_by_url",
+ "local_deep_research.research_library.services.library_service.find_tracker_for_document",
return_value=mock_tracker,
)
delete_mock = mocker.patch(
diff --git a/tests/research_library/services/test_library_document_identity.py b/tests/research_library/services/test_library_document_identity.py
new file mode 100644
index 0000000000..c0b1383a8a
--- /dev/null
+++ b/tests/research_library/services/test_library_document_identity.py
@@ -0,0 +1,623 @@
+"""Destructive library operations must distinguish URL-equivalent PDF versions."""
+
+import hashlib
+import uuid
+from contextlib import contextmanager
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+from sqlalchemy import create_engine, event
+from sqlalchemy.orm import Session
+
+from local_deep_research.constants import FILE_PATH_SENTINELS
+from local_deep_research.database.models import Base
+from local_deep_research.database.models.download_tracker import DownloadTracker
+from local_deep_research.database.models.library import (
+ Collection,
+ Document,
+ DocumentBlob,
+ DocumentCollection,
+ DocumentStatus,
+ SourceType,
+)
+from local_deep_research.database.models.research import (
+ ResearchHistory,
+ ResearchResource,
+)
+from local_deep_research.research_library.services.library_service import (
+ LibraryService,
+)
+from local_deep_research.research_library.services.pdf_storage_manager import (
+ PDFStorageManager,
+)
+from local_deep_research.research_library.utils import (
+ document_awaits_pdf,
+ get_absolute_path_from_settings,
+ get_legacy_url_hashes,
+ get_url_hash,
+)
+
+LIBRARY_MODULE = "local_deep_research.research_library.services.library_service"
+OLD_URL = "https://example.org/paper.pdf?utm_source=old"
+NEW_URL = "https://example.org/paper.pdf"
+
+
+def library(tmp_path, monkeypatch):
+ engine = create_engine("sqlite:///:memory:")
+
+ @event.listens_for(engine, "connect")
+ def enable_foreign_keys(connection, _):
+ connection.execute("PRAGMA foreign_keys=ON")
+
+ Base.metadata.create_all(engine)
+ with Session(engine) as session:
+ source = SourceType(
+ id=str(uuid.uuid4()),
+ name="research_download",
+ display_name="Download",
+ )
+ collection = Collection(id=str(uuid.uuid4()), name="Library")
+ research = ResearchHistory(
+ id=str(uuid.uuid4()),
+ query="PDF versions",
+ mode="quick",
+ status="completed",
+ created_at="2026-01-01T00:00:00",
+ )
+ session.add_all([source, collection, research])
+ session.commit()
+
+ @contextmanager
+ def scoped_session(*args, **kwargs):
+ yield session
+
+ settings = Mock()
+ settings.get_setting.side_effect = lambda key, default=None: {
+ "research_library.storage_path": str(tmp_path),
+ "research_library.shared_library": False,
+ }.get(key, default)
+ monkeypatch.setattr(
+ f"{LIBRARY_MODULE}.get_user_db_session", scoped_session
+ )
+ monkeypatch.setattr(
+ f"{LIBRARY_MODULE}.get_settings_manager", lambda **kw: settings
+ )
+ monkeypatch.setattr(
+ "local_deep_research.utilities.db_utils.get_settings_manager",
+ lambda: settings,
+ )
+ # Exercise the real per-user path resolver. A colliding file in the
+ # old shared root must stay untouched now that no legacy fallback
+ # exists (#7268).
+ root = get_absolute_path_from_settings(
+ "", "identity_test", settings_manager=settings
+ )
+ root.mkdir(parents=True)
+ purge = Mock()
+ monkeypatch.setattr(
+ "local_deep_research.research_library.deletion.services.document_deletion.DocumentDeletionService._purge_document_rag",
+ purge,
+ )
+
+ def add_document(url, name, *, legacy=False, with_tracker=True):
+ content = f"%PDF-1.4 {name}".encode()
+ resource = ResearchResource(
+ research_id=research.id,
+ title=name,
+ url=url,
+ source_type="academic",
+ created_at="2026-01-01T00:00:00",
+ )
+ session.add(resource)
+ session.flush()
+ doc = Document(
+ id=str(uuid.uuid4()),
+ source_type_id=source.id,
+ resource_id=resource.id,
+ research_id=research.id,
+ document_hash=hashlib.sha256(content).hexdigest(),
+ original_url=url,
+ title=name,
+ file_path=name,
+ file_size=len(content),
+ file_type="pdf",
+ storage_mode="filesystem",
+ status=DocumentStatus.COMPLETED,
+ text_content=f"Extracted text from {name}",
+ )
+ session.add(doc)
+ session.flush()
+ session.add(
+ DocumentCollection(
+ document_id=doc.id, collection_id=collection.id
+ )
+ )
+ tracker = None
+ if with_tracker:
+ tracker = DownloadTracker(
+ url=url,
+ url_hash=get_legacy_url_hashes(url)[1]
+ if legacy
+ else get_url_hash(url),
+ first_resource_id=resource.id,
+ file_path=name,
+ file_hash=doc.document_hash,
+ is_downloaded=True,
+ )
+ session.add(tracker)
+ session.commit()
+ (root / name).write_bytes(content)
+ return doc, tracker
+
+ yield SimpleNamespace(
+ session=session,
+ service=LibraryService("identity_test"),
+ root=root,
+ shared_root=tmp_path,
+ add=add_document,
+ purge=purge,
+ )
+ engine.dispose()
+
+
[email protected]("missing", ["old.pdf", "new.pdf", None])
[email protected]("has_text", [False, True])
+def test_sync_checks_each_documents_own_file(library, missing, has_text):
+ old, old_tracker = library.add(OLD_URL, "old.pdf", legacy=True)
+ new, new_tracker = library.add(NEW_URL, "new.pdf")
+ if not has_text:
+ old.text_content = new.text_content = None
+ library.session.commit()
+ ids = {old.id: "old.pdf", new.id: "new.pdf"}
+ tracker_ids = {old_tracker.id: "old.pdf", new_tracker.id: "new.pdf"}
+ if missing:
+ (library.root / missing).unlink()
+
+ stats = library.service.sync_library_with_filesystem()
+ library.session.expunge_all()
+
+ for doc_id, name in ids.items():
+ surviving = library.session.get(Document, doc_id)
+ # A missing PDF removes only a document that holds nothing else;
+ # extracted text keeps the document, which then awaits its PDF.
+ removed = name == missing and not has_text
+ assert (surviving is None) == removed
+ membership = (
+ library.session.query(DocumentCollection)
+ .filter_by(document_id=doc_id)
+ .first()
+ )
+ assert (membership is None) == removed
+ if surviving:
+ assert surviving.text_content == (
+ f"Extracted text from {name}" if has_text else None
+ )
+ assert document_awaits_pdf(surviving) == (name == missing)
+ for tracker_id, name in tracker_ids.items():
+ actual = library.session.get(DownloadTracker, tracker_id)
+ assert actual.is_downloaded == (name != missing)
+ assert actual.file_path == (None if name == missing else name)
+ assert stats["files_found"] == (1 if missing else 2)
+ assert stats["files_missing"] == stats["trackers_updated"] == bool(missing)
+ assert [entry["path"] for entry in stats["missing_files"]] == (
+ [missing] if missing else []
+ )
+ assert library.purge.call_count == bool(missing and not has_text)
+
+
+def test_delete_only_unlinks_and_resets_requested_legacy_document(library):
+ old, old_tracker = library.add(OLD_URL, "old.pdf", legacy=True)
+ new, new_tracker = library.add(NEW_URL, "new.pdf")
+ old_id, new_id = old.id, new.id
+
+ assert library.service.delete_document(old_id)
+ library.session.expire_all()
+
+ assert library.session.get(Document, old_id) is None
+ assert (
+ library.session.get(Document, new_id).text_content
+ == "Extracted text from new.pdf"
+ )
+ assert not (library.root / "old.pdf").exists()
+ assert (library.root / "new.pdf").read_bytes() == b"%PDF-1.4 new.pdf"
+ assert old_tracker.file_path is None
+ assert old_tracker.is_downloaded is False
+ assert new_tracker.file_path == "new.pdf"
+ assert new_tracker.is_downloaded is True
+ library.purge.assert_called_once_with(old_id, [], full_delete=True)
+
+
[email protected]("operation", ["sync", "delete"])
+def test_document_path_wins_over_stale_matching_tracker_path(
+ library, operation
+):
+ doc, tracker = library.add(OLD_URL, "old.pdf", legacy=True)
+ tracker.file_path = "unrelated.pdf"
+ unrelated = library.root / "unrelated.pdf"
+ unrelated.write_bytes(b"another saved document")
+ doc_id = doc.id
+ library.session.commit()
+
+ if operation == "sync":
+ unrelated.unlink()
+ stats = library.service.sync_library_with_filesystem()
+ library.session.expire_all()
+ assert stats["files_found"] == 1
+ assert library.session.get(Document, doc_id) is not None
+ assert (library.root / "old.pdf").exists()
+ assert tracker.is_downloaded is True
+ else:
+ assert library.service.delete_document(doc_id)
+ assert not (library.root / "old.pdf").exists()
+ assert unrelated.read_bytes() == b"another saved document"
+
+
[email protected]("mode", ["database", "none", None])
+def test_delete_uses_own_file_despite_stale_storage_mode(library, mode):
+ old, old_tracker = library.add(OLD_URL, "old.pdf", legacy=True)
+ new, new_tracker = library.add(NEW_URL, "new.pdf")
+ old.storage_mode = mode
+ old_id, new_id = old.id, new.id
+ library.session.commit()
+
+ assert library.service.delete_document(old_id)
+ library.session.expire_all()
+
+ assert library.session.get(Document, old_id) is None
+ assert not (library.root / "old.pdf").exists()
+ assert old_tracker.is_downloaded is False
+ assert library.session.get(Document, new_id) is not None
+ assert (library.root / "new.pdf").read_bytes() == b"%PDF-1.4 new.pdf"
+ assert new_tracker.is_downloaded is True
+
+
+ "mode,has_blob,keep_file",
+ [
+ ("database", True, True),
+ ("database", False, False),
+ ("none", False, True),
+ ("filesystem", True, True),
+ (None, True, True),
+ ],
+)
+def test_delete_borrows_tracker_path_only_for_missing_legacy_storage(
+ library, mode, has_blob, keep_file
+):
+ doc, tracker = library.add(OLD_URL, "old.pdf", legacy=True)
+ doc.storage_mode = mode
+ doc.file_path = None
+ doc_id = doc.id
+ if has_blob:
+ library.session.add(
+ DocumentBlob(
+ document_id=doc_id,
+ pdf_binary=b"%PDF-1.4 old.pdf",
+ blob_hash=doc.document_hash,
+ )
+ )
+ library.session.commit()
+
+ assert library.service.delete_document(doc_id)
+ library.session.expire_all()
+
+ assert library.session.get(Document, doc_id) is None
+ assert library.session.get(DocumentBlob, doc_id) is None
+ assert (library.root / "old.pdf").exists() == keep_file
+ assert tracker.is_downloaded is False
+
+
[email protected]("mode", ["filesystem", "database", "none"])
+def test_sync_preserves_own_storage_without_a_tracker(library, mode):
+ doc, _ = library.add(OLD_URL, "old.pdf", with_tracker=False)
+ if mode == "database":
+ doc.storage_mode = "database"
+ doc.file_path = "database"
+ library.session.add(
+ DocumentBlob(
+ document_id=doc.id,
+ pdf_binary=b"%PDF-1.4 old.pdf",
+ blob_hash=doc.document_hash,
+ )
+ )
+ elif mode == "none":
+ doc.storage_mode = "none"
+ doc.file_path = None
+ doc_id = doc.id
+ library.session.commit()
+
+ stats = library.service.sync_library_with_filesystem()
+ library.session.expire_all()
+
+ assert (
+ library.session.get(Document, doc_id).text_content
+ == "Extracted text from old.pdf"
+ )
+ assert stats["files_found"] == 1
+ assert stats["files_missing"] == 0
+ library.purge.assert_not_called()
+ if mode == "database":
+ assert (
+ library.session.get(DocumentBlob, doc_id).pdf_binary
+ == b"%PDF-1.4 old.pdf"
+ )
+
+
[email protected]("mode", ["database", None])
[email protected]("sentinel", FILE_PATH_SENTINELS)
+def test_sync_preserves_sentinels_independently_of_storage_mode(
+ library, mode, sentinel
+):
+ doc, _ = library.add(OLD_URL, "old.pdf", with_tracker=False)
+ (library.root / "old.pdf").unlink()
+ doc.storage_mode = mode
+ doc.file_path = sentinel
+ doc_id = doc.id
+ library.session.commit()
+
+ stats = library.service.sync_library_with_filesystem()
+ library.session.expire_all()
+
+ assert (
+ library.session.get(Document, doc_id).text_content
+ == "Extracted text from old.pdf"
+ )
+ assert stats["files_found"] == 1
+ assert stats["files_missing"] == 0
+ library.purge.assert_not_called()
+
+
[email protected]("own_path", [True, False])
+def test_sync_checks_disk_when_database_label_has_no_blob(library, own_path):
+ doc, tracker = library.add(OLD_URL, "old.pdf", legacy=True)
+ doc.storage_mode = "database"
+ if not own_path:
+ doc.file_path = None
+ doc_id = doc.id
+ library.session.commit()
+ if own_path:
+ # The production reader supports this metadata/storage mismatch.
+ manager = PDFStorageManager(
+ library.root, "database", username="identity_test"
+ )
+ assert manager.load_pdf(doc, library.session) == b"%PDF-1.4 old.pdf"
+
+ stats = library.service.sync_library_with_filesystem()
+ library.session.expire_all()
+
+ assert (
+ library.session.get(Document, doc_id).text_content
+ == "Extracted text from old.pdf"
+ )
+ assert (
+ library.session.query(DocumentCollection)
+ .filter_by(document_id=doc_id)
+ .count()
+ == 1
+ )
+ assert tracker.is_downloaded is True
+ assert stats["files_found"] == 1
+ assert stats["files_missing"] == 0
+ library.purge.assert_not_called()
+
+
[email protected]("mode", ["filesystem", None])
+def test_sync_checks_blob_when_filesystem_label_has_no_file(library, mode):
+ doc, tracker = library.add(OLD_URL, "old.pdf", legacy=True)
+ doc.storage_mode = mode
+ (library.root / "old.pdf").unlink()
+ doc_id = doc.id
+ library.session.add(
+ DocumentBlob(
+ document_id=doc_id,
+ pdf_binary=b"%PDF-1.4 old.pdf",
+ blob_hash=doc.document_hash,
+ )
+ )
+ library.session.commit()
+
+ stats = library.service.sync_library_with_filesystem()
+ library.session.expire_all()
+
+ assert (
+ library.session.get(DocumentBlob, doc_id).pdf_binary
+ == b"%PDF-1.4 old.pdf"
+ )
+ assert (
+ library.session.get(Document, doc_id).text_content
+ == "Extracted text from old.pdf"
+ )
+ assert tracker.is_downloaded is True
+ assert stats["files_found"] == 1
+ assert stats["files_missing"] == 0
+ library.purge.assert_not_called()
+
+
[email protected]("has_text", [False, True])
+def test_sync_removes_document_only_when_no_content_remains(library, has_text):
+ """Both declared backends are missing: no blob, and no file on disk."""
+ doc, tracker = library.add(OLD_URL, "old.pdf", legacy=True)
+ doc.storage_mode = "database"
+ if not has_text:
+ doc.text_content = None
+ (library.root / "old.pdf").unlink()
+ doc_id = doc.id
+ library.session.commit()
+
+ stats = library.service.sync_library_with_filesystem()
+ library.session.expire_all()
+
+ retained = library.session.get(Document, doc_id)
+ assert (retained is not None) == has_text
+ assert tracker.is_downloaded is False
+ assert stats["files_missing"] == stats["trackers_updated"] == 1
+ if has_text:
+ assert retained.text_content == "Extracted text from old.pdf"
+ assert document_awaits_pdf(retained)
+ library.purge.assert_not_called()
+ else:
+ library.purge.assert_called_once_with(doc_id, [], full_delete=True)
+
+
+ "identity",
+ ["hash", "link", "unknown", "conflicting_hash", "conflicting_link"],
+)
[email protected]("has_text", [False, True])
[email protected]("missing_path", [None, ""])
+def test_legacy_tracker_fallback_requires_unambiguous_document_identity(
+ library, identity, has_text, missing_path
+):
+ old, old_tracker = library.add(OLD_URL, "old.pdf", legacy=True)
+ new, new_tracker = library.add(NEW_URL, "new.pdf")
+ old_id, new_id = old.id, new.id
+ # Pre-upgrade row with no stored Document path. A verified tracker can
+ # identify its missing file; a URL alone cannot establish that identity.
+ old.file_path = missing_path
+ old.storage_mode = None
+ if not has_text:
+ old.text_content = None
+ if identity == "link":
+ old_tracker.file_hash = None
+ old_tracker.library_document_id = old.id
+ elif identity == "unknown":
+ old_tracker.file_hash = None
+ elif identity == "conflicting_hash":
+ old_tracker.library_document_id = old.id
+ old_tracker.file_hash = new.document_hash
+ elif identity == "conflicting_link":
+ old_tracker.library_document_id = new.id
+ (library.root / "old.pdf").unlink()
+ library.session.commit()
+
+ stats = library.service.sync_library_with_filesystem()
+ library.session.expire_all()
+
+ identified = identity in ("hash", "link")
+ retained = library.session.get(Document, old_id)
+ assert (retained is None) == (identified and not has_text)
+ if retained is not None:
+ # Text keeps an identified document, which then awaits its PDF; an
+ # unidentified one is left exactly as it was.
+ assert document_awaits_pdf(retained) == identified
+ assert (
+ library.session.query(DocumentCollection)
+ .filter_by(document_id=old_id)
+ .count()
+ == 1
+ )
+ assert old_tracker.is_downloaded == (not identified)
+ assert library.session.get(Document, new_id) is not None
+ assert new_tracker.is_downloaded is True
+ assert new_tracker.file_path == "new.pdf"
+ assert stats["files_missing"] == int(identified)
+
+
[email protected]("mode", ["filesystem", None, "database"])
[email protected]("tracker", ["absent", "other_version"])
+def test_sync_judges_pathless_document_by_its_database_label(
+ library, mode, tracker
+):
+ """Without a path or an identifying tracker only a database label is storage.
+
+ Missing or ambiguous legacy metadata cannot prove which file is absent,
+ so such a document is retained. A document labelled as stored in the
+ database with no blob and nothing else is removed and reported under
+ the fixed ``"database"`` label. A tracker of another PDF version is
+ never reset for it.
+ """
+ doc, url_tracker = library.add(
+ OLD_URL, "old.pdf", with_tracker=tracker == "other_version"
+ )
+ (library.root / "old.pdf").unlink()
+ doc.file_path = None
+ doc.storage_mode = mode
+ doc.text_content = None
+ if url_tracker is not None:
+ url_tracker.file_hash = hashlib.sha256(b"another version").hexdigest()
+ doc_id = doc.id
+ library.session.commit()
+
+ stats = library.service.sync_library_with_filesystem()
+ library.session.expire_all()
+
+ removed = mode == "database"
+ assert (library.session.get(Document, doc_id) is None) == removed
+ assert (stats["files_found"], stats["files_missing"]) == (
+ (0, 1) if removed else (1, 0)
+ )
+ assert stats["missing_files"] == (
+ [{"id": doc_id, "title": "old.pdf", "path": "database", "url": OLD_URL}]
+ if removed
+ else []
+ )
+ assert stats["trackers_updated"] == 0
+ if url_tracker is not None:
+ assert (url_tracker.is_downloaded, url_tracker.file_path) == (
+ True,
+ "old.pdf",
+ )
+ assert library.purge.call_count == int(removed)
+
+
+def test_delete_never_unlinks_legacy_shared_root(library):
+ doc, tracker = library.add(OLD_URL, "old.pdf", legacy=True)
+ (library.root / "old.pdf").unlink()
+ shared_file = library.shared_root / "old.pdf"
+ shared_file.write_bytes(b"another user's colliding legacy file")
+
+ assert library.service.delete_document(doc.id)
+
+ assert shared_file.read_bytes() == b"another user's colliding legacy file"
+ assert tracker.is_downloaded is False
+
+
[email protected]("operation", ["delete", "remove_from_collection"])
[email protected]("identity", ["hash", "link", "conflicting_hash"])
+def test_primary_deletion_resets_only_matching_version_tracker(
+ library, monkeypatch, operation, identity
+):
+ """Both deletion entry points must preserve a newer canonical tracker."""
+ from local_deep_research.research_library.deletion.services import (
+ document_deletion,
+ )
+
+ old, old_tracker = library.add(OLD_URL, "old.pdf", legacy=True)
+ new, new_tracker = library.add(NEW_URL, "new.pdf")
+ old_id, new_id = old.id, new.id
+ collection_id = old.collections[0].collection_id
+ if identity == "link":
+ old_tracker.file_hash = None
+ old_tracker.library_document_id = old.id
+ elif identity == "conflicting_hash":
+ old_tracker.library_document_id = old.id
+ old_tracker.file_hash = new.document_hash
+ library.session.commit()
+
+ @contextmanager
+ def scoped_session(*args, **kwargs):
+ yield library.session
+
+ monkeypatch.setattr(
+ document_deletion, "get_user_db_session", scoped_session
+ )
+ service = document_deletion.DocumentDeletionService("identity_test")
+ if operation == "delete":
+ result = service.delete_document(old_id)
+ assert result["deleted"] is True
+ else:
+ result = service.remove_from_collection(old_id, collection_id)
+ assert result["unlinked"] is True
+ assert result["document_deleted"] is True
+
+ library.session.expire_all()
+ assert library.session.get(Document, old_id) is None
+ assert library.session.get(Document, new_id) is not None
+ assert (library.root / "new.pdf").read_bytes() == b"%PDF-1.4 new.pdf"
+ assert new_tracker.is_downloaded is True
+ assert new_tracker.file_path == "new.pdf"
+ matched = identity != "conflicting_hash"
+ assert old_tracker.is_downloaded is (not matched)
+ assert old_tracker.file_path == (None if matched else "old.pdf")
diff --git a/tests/research_library/services/test_library_service.py b/tests/research_library/services/test_library_service.py
index 0ad60e6ddc..3e5db079e8 100644
--- a/tests/research_library/services/test_library_service.py
+++ b/tests/research_library/services/test_library_service.py
@@ -1466,11 +1466,9 @@ class TestSyncLibraryUploadsIntegration:
assert library_session.get(Document, upload_id) is not None
assert stats["total_documents"] == 1 # only the download is counted
- # The download has no DownloadTracker, so it must still travel through
- # the destructive `else` branch and be cascade-deleted. Asserting this
- # explicitly proves the deletion path stays functional for legitimate
- # downloads — without it, a regression that broke the else branch
- # would still pass the upload-survival check above.
+ # The download declares database storage (the model default), but has
+ # no blob. It must still be removed, proving missing-storage cleanup
+ # stays functional while uploads remain outside the sync scope.
assert library_session.get(Document, download_id) is None
assert stats["files_missing"] == 1
diff --git a/tests/research_library/services/test_library_service_crud_deep.py b/tests/research_library/services/test_library_service_crud_deep.py
index 88732788c9..ba0bcaa21e 100644
--- a/tests/research_library/services/test_library_service_crud_deep.py
+++ b/tests/research_library/services/test_library_service_crud_deep.py
@@ -18,7 +18,10 @@ from local_deep_research.constants import (
FILE_PATH_METADATA_ONLY,
FILE_PATH_TEXT_ONLY,
)
-from local_deep_research.database.models.library import DocumentStatus
+from local_deep_research.database.models.library import (
+ DocumentBlob,
+ DocumentStatus,
+)
# ============== Helper ==============
@@ -59,10 +62,17 @@ class TestDeleteDocumentHappyPaths:
service = _make_service()
mock_session = MagicMock()
- mock_doc = Mock()
+ mock_doc = Mock(
+ id="doc-123",
+ document_hash="a" * 64,
+ file_path=None,
+ storage_mode="filesystem",
+ )
mock_doc.original_url = "https://arxiv.org/abs/2301.00001"
- mock_tracker = Mock()
+ mock_tracker = Mock(
+ library_document_id=None, file_hash=mock_doc.document_hash
+ )
mock_tracker.file_path = "pdfs/test.pdf"
# Session lookup and query routing
@@ -74,6 +84,10 @@ class TestDeleteDocumentHappyPaths:
)
def query_router(model):
+ if model is DocumentBlob.document_id:
+ blob_query = MagicMock()
+ blob_query.filter_by.return_value.first.return_value = None
+ return blob_query
name = getattr(model, "__name__", str(model))
if "DownloadTracker" in str(name) or "Tracker" in str(model):
return mock_tracker_query
@@ -106,7 +120,12 @@ class TestDeleteDocumentHappyPaths:
service = _make_service()
mock_session = MagicMock()
- mock_doc = Mock()
+ mock_doc = Mock(
+ id="doc-123",
+ document_hash="a" * 64,
+ file_path=None,
+ storage_mode="filesystem",
+ )
mock_doc.original_url = None
mock_session.get.return_value = mock_doc
@@ -124,10 +143,17 @@ class TestDeleteDocumentHappyPaths:
service = _make_service()
mock_session = MagicMock()
- mock_doc = Mock()
+ mock_doc = Mock(
+ id="doc-123",
+ document_hash="a" * 64,
+ file_path=None,
+ storage_mode="filesystem",
+ )
mock_doc.original_url = "https://example.com/doc.pdf"
- mock_tracker = Mock()
+ mock_tracker = Mock(
+ library_document_id=None, file_hash=mock_doc.document_hash
+ )
mock_tracker.file_path = "pdfs/gone.pdf"
mock_doc_query = MagicMock()
@@ -138,6 +164,10 @@ class TestDeleteDocumentHappyPaths:
)
def query_router(model):
+ if model is DocumentBlob.document_id:
+ blob_query = MagicMock()
+ blob_query.filter_by.return_value.first.return_value = None
+ return blob_query
name = getattr(model, "__name__", str(model))
if "DownloadTracker" in str(name) or "Tracker" in str(model):
return mock_tracker_query
@@ -165,10 +195,17 @@ class TestDeleteDocumentHappyPaths:
service = _make_service()
mock_session = MagicMock()
- mock_doc = Mock()
+ mock_doc = Mock(
+ id="doc-123",
+ document_hash="a" * 64,
+ file_path=None,
+ storage_mode="filesystem",
+ )
mock_doc.original_url = "https://example.com/doc.pdf"
- mock_tracker = Mock()
+ mock_tracker = Mock(
+ library_document_id=None, file_hash=mock_doc.document_hash
+ )
mock_tracker.file_path = None
mock_doc_query = MagicMock()
@@ -179,6 +216,10 @@ class TestDeleteDocumentHappyPaths:
)
def query_router(model):
+ if model is DocumentBlob.document_id:
+ blob_query = MagicMock()
+ blob_query.filter_by.return_value.first.return_value = None
+ return blob_query
name = getattr(model, "__name__", str(model))
if "DownloadTracker" in str(name) or "Tracker" in str(model):
return mock_tracker_query
@@ -200,10 +241,17 @@ class TestDeleteDocumentHappyPaths:
service = _make_service()
mock_session = MagicMock()
- mock_doc = Mock()
+ mock_doc = Mock(
+ id="doc-123",
+ document_hash="a" * 64,
+ file_path=None,
+ storage_mode="filesystem",
+ )
mock_doc.original_url = "https://example.com/doc.pdf"
- mock_tracker = Mock()
+ mock_tracker = Mock(
+ library_document_id=None, file_hash=mock_doc.document_hash
+ )
mock_tracker.file_path = "pdfs/locked.pdf"
mock_doc_query = MagicMock()
@@ -214,6 +262,10 @@ class TestDeleteDocumentHappyPaths:
)
def query_router(model):
+ if model is DocumentBlob.document_id:
+ blob_query = MagicMock()
+ blob_query.filter_by.return_value.first.return_value = None
+ return blob_query
name = getattr(model, "__name__", str(model))
if "DownloadTracker" in str(name) or "Tracker" in str(model):
return mock_tracker_query
@@ -241,10 +293,17 @@ class TestDeleteDocumentHappyPaths:
service = _make_service()
mock_session = MagicMock()
- mock_doc = Mock()
+ mock_doc = Mock(
+ id="doc-123",
+ document_hash="a" * 64,
+ file_path=None,
+ storage_mode="filesystem",
+ )
mock_doc.original_url = "https://example.com/doc.pdf"
- mock_tracker = Mock()
+ mock_tracker = Mock(
+ library_document_id=None, file_hash=mock_doc.document_hash
+ )
mock_tracker.file_path = "pdfs/test.pdf"
mock_tracker.is_downloaded = True
@@ -256,6 +315,10 @@ class TestDeleteDocumentHappyPaths:
)
def query_router(model):
+ if model is DocumentBlob.document_id:
+ blob_query = MagicMock()
+ blob_query.filter_by.return_value.first.return_value = None
+ return blob_query
name = getattr(model, "__name__", str(model))
if "DownloadTracker" in str(name) or "Tracker" in str(model):
return mock_tracker_query
@@ -542,6 +605,24 @@ _REMAINING_CONTENT = (
)
+def _patch_tracker_lookups(mocker, docs, trackers):
+ """Give each Mock document its own tracker, by URL and by identity.
+
+ The sync reads the URL's tracker for its census of "database" trackers
+ and the tracker that identifies the document for everything else.
+ """
+ by_url = {doc.original_url: tracker for doc, tracker in zip(docs, trackers)}
+ by_id = {doc.id: tracker for doc, tracker in zip(docs, trackers)}
+ mocker.patch(
+ "local_deep_research.research_library.services.library_service.find_tracker_by_url",
+ side_effect=lambda _session, url: by_url[url],
+ )
+ mocker.patch(
+ "local_deep_research.research_library.services.library_service.find_tracker_for_document",
+ side_effect=lambda _session, doc: by_id[doc.id],
+ )
+
+
class TestSyncLibraryWithFilesystem:
"""Tests for sync_library_with_filesystem logic."""
@@ -563,13 +644,17 @@ class TestSyncLibraryWithFilesystem:
service = _make_service()
mock_session = MagicMock()
- mock_doc = Mock()
+ mock_doc = Mock(
+ document_hash="a" * 64, file_path=None, storage_mode="filesystem"
+ )
mock_doc.id = "doc-1"
mock_doc.title = "Test"
mock_doc.original_url = "https://example.com/doc.pdf"
mock_doc.file_path = "pdfs/doc.pdf"
- mock_tracker = Mock()
+ mock_tracker = Mock(
+ library_document_id=None, file_hash=mock_doc.document_hash
+ )
mock_tracker.file_path = "pdfs/doc.pdf"
mock_session.query.return_value.filter_by.return_value.filter.return_value.options.return_value.all.return_value = [
@@ -583,6 +668,10 @@ class TestSyncLibraryWithFilesystem:
call_count = {"n": 0}
def query_router(model):
+ if model is DocumentBlob.document_id:
+ blob_query = MagicMock()
+ blob_query.filter_by.return_value.first.return_value = None
+ return blob_query
call_count["n"] += 1
if call_count["n"] == 1:
# Document query
@@ -612,18 +701,26 @@ class TestSyncLibraryWithFilesystem:
service = _make_service()
mock_session = MagicMock()
- mock_doc = Mock()
+ mock_doc = Mock(
+ document_hash="a" * 64, file_path=None, storage_mode="filesystem"
+ )
mock_doc.id = "doc-1"
mock_doc.title = "Test"
mock_doc.original_url = "https://example.com/doc.pdf"
mock_doc.file_path = "pdfs/gone.pdf"
- mock_tracker = Mock()
+ mock_tracker = Mock(
+ library_document_id=None, file_hash=mock_doc.document_hash
+ )
mock_tracker.file_path = "pdfs/gone.pdf"
call_count = {"n": 0}
def query_router(model):
+ if model is DocumentBlob.document_id:
+ blob_query = MagicMock()
+ blob_query.filter_by.return_value.first.return_value = None
+ return blob_query
call_count["n"] += 1
if call_count["n"] == 1:
q = MagicMock()
@@ -658,12 +755,15 @@ class TestSyncLibraryWithFilesystem:
def test_doc_without_tracker(self, mocker):
"""Doc with no tracker that holds nothing else → cascade delete.
- The missing file its own path names is reported under that path.
+ A known missing file is removed even without a tracker, and the
+ missing file its own path names is reported under that path.
"""
service = _make_service()
mock_session = MagicMock()
- mock_doc = Mock()
+ mock_doc = Mock(
+ document_hash="a" * 64, file_path=None, storage_mode="filesystem"
+ )
mock_doc.id = "doc-1"
mock_doc.title = "Test"
mock_doc.original_url = "https://example.com/doc.pdf"
@@ -672,6 +772,10 @@ class TestSyncLibraryWithFilesystem:
call_count = {"n": 0}
def query_router(model):
+ if model is DocumentBlob.document_id:
+ blob_query = MagicMock()
+ blob_query.filter_by.return_value.first.return_value = None
+ return blob_query
call_count["n"] += 1
if call_count["n"] == 1:
q = MagicMock()
@@ -712,32 +816,28 @@ class TestSyncLibraryWithFilesystem:
docs = []
for i in range(3):
- d = Mock()
+ d = Mock(
+ document_hash=f"hash-{i}",
+ file_path=f"pdfs/doc{i}.pdf",
+ storage_mode="filesystem",
+ )
d.id = f"doc-{i}"
d.title = f"Doc {i}"
d.original_url = f"https://example.com/doc{i}.pdf"
d.file_path = f"pdfs/doc{i}.pdf"
docs.append(d)
- trackers = [Mock(file_path=f"pdfs/doc{i}.pdf") for i in range(3)]
-
- call_count = {"n": 0}
-
- def query_router(model):
- call_count["n"] += 1
- if call_count["n"] == 1:
- q = MagicMock()
- q.filter_by.return_value.filter.return_value.options.return_value.all.return_value = docs
- return q
- idx = call_count["n"] - 2
- tq = MagicMock()
- if idx < 3:
- tq.filter_by.return_value.first.return_value = trackers[idx]
- else:
- tq.filter_by.return_value.first.return_value = None
- return tq
+ trackers = [
+ Mock(
+ file_path=d.file_path,
+ file_hash=d.document_hash,
+ library_document_id=None,
+ )
+ for d in docs
+ ]
- mock_session.query.side_effect = query_router
+ mock_session.query.return_value.filter_by.return_value.filter.return_value.options.return_value.all.return_value = docs
+ _patch_tracker_lookups(mocker, docs, trackers)
_mock_session_cm(mocker, mock_session)
# First 2 files exist, third missing
@@ -792,29 +892,28 @@ class TestSyncLibraryWithFilesystem:
docs = []
for i in range(3):
- d = Mock()
+ d = Mock(
+ document_hash=f"hash-{i}",
+ file_path=f"pdfs/doc{i}.pdf",
+ storage_mode="filesystem",
+ )
d.id = f"doc-{i}"
d.title = f"Doc {i}"
d.original_url = f"https://example.com/doc{i}.pdf"
d.file_path = f"pdfs/doc{i}.pdf"
docs.append(d)
- trackers = [Mock(file_path=f"pdfs/doc{i}.pdf") for i in range(3)]
-
- call_count = {"n": 0}
-
- def query_router(model):
- call_count["n"] += 1
- if call_count["n"] == 1:
- q = MagicMock()
- q.filter_by.return_value.filter.return_value.options.return_value.all.return_value = docs
- return q
- idx = call_count["n"] - 2
- tq = MagicMock()
- tq.filter_by.return_value.first.return_value = trackers[idx]
- return tq
+ trackers = [
+ Mock(
+ file_path=d.file_path,
+ file_hash=d.document_hash,
+ library_document_id=None,
+ )
+ for d in docs
+ ]
- mock_session.query.side_effect = query_router
+ mock_session.query.return_value.filter_by.return_value.filter.return_value.options.return_value.all.return_value = docs
+ _patch_tracker_lookups(mocker, docs, trackers)
_mock_session_cm(mocker, mock_session)
mock_path = MagicMock()
diff --git a/tests/research_library/test_library_sync_contracts.py b/tests/research_library/test_library_sync_contracts.py
index bfeae4c341..d06dd30476 100644
--- a/tests/research_library/test_library_sync_contracts.py
+++ b/tests/research_library/test_library_sync_contracts.py
@@ -3584,12 +3584,14 @@ def test_tracker_reset_for_two_documents_counts_once_in_either_order(
else [kept_id, deleted_id]
)
# The tracker names the file written last. In both orders it names the
- # first research's file here, as when that file was written last.
+ # first research's file here, as when that file was written last: with
+ # that file's content hash, which identifies the document it names.
db.session.query(DownloadTracker).filter_by(
url_hash=get_url_hash(url)
).update(
{
DownloadTracker.file_path: deleted_path,
+ DownloadTracker.file_hash: PDF_HASH,
DownloadTracker.is_downloaded: True,
},
synchronize_session=False,
diff --git a/tests/security/test_document_path_deletion.py b/tests/security/test_document_path_deletion.py
index 6bf6fd2e6d..4d47deb8f7 100644
--- a/tests/security/test_document_path_deletion.py
+++ b/tests/security/test_document_path_deletion.py
@@ -10,9 +10,10 @@ Surfaces verified on this branch (security/test-file-write-risk-battery):
(1) Document file deletion cascade — every unlink is guarded today:
* ``src/local_deep_research/research_library/services/library_service.py``
- lines 1041-1052 (``LibraryService.delete_document``): resolves
- ``DownloadTracker.file_path`` through
- ``get_absolute_path_from_settings(..., allow_legacy_fallback=False)``
+ lines 1041-1052 (``LibraryService.delete_document``): resolves the
+ document's own ``file_path`` or, for a document without one, the
+ ``DownloadTracker.file_path`` of a tracker that identifies it, through
+ ``get_absolute_path_from_settings(...)``
-> ``_resolve_within_root`` -> ``PathValidator.validate_safe_path``
(werkzeug ``safe_join``: absolute / ``..`` / encoded / unicode
traversal all rejected) + a leaf ``is_symlink()`` refusal, and only
@@ -217,18 +218,24 @@ def _make_document(session, *, file_path=None, original_url=None):
return doc
-def _make_tracker(session, url, file_path):
+def _make_tracker(session, doc, file_path):
"""Commit one DownloadTracker row whose file_path is the payload.
+ The tracker carries the document's content hash, as a download's
+ tracker does, so it identifies ``doc``: ``delete_document`` takes the
+ path of a tracker that identifies a document without a path of its own,
+ and never the path of another document's tracker.
+
``first_resource_id`` is nullable=False but SQLite FK enforcement is
off by default in a fresh SQLAlchemy engine, so a synthetic id
commits cleanly without seeding a ResearchResource row.
"""
tracker = DownloadTracker(
- url=url,
- url_hash=get_url_hash(url),
+ url=doc.original_url,
+ url_hash=get_url_hash(doc.original_url),
first_resource_id=1,
file_path=file_path,
+ file_hash=doc.document_hash,
is_downloaded=True,
)
session.add(tracker)
@@ -250,6 +257,15 @@ def _relative_escape(root: Path, canary: Path) -> str:
return "/".join(ups + downs)
+def _files_outside_root(tmp_path: Path, root: Path) -> list[str]:
+ """Every file or symlink under ``tmp_path`` that is not inside ``root``."""
+ return sorted(
+ str(p.relative_to(tmp_path))
+ for p in tmp_path.rglob("*")
+ if (p.is_file() or p.is_symlink()) and root not in p.parents
+ )
+
+
# ---------------------------------------------------------------------------
# LibraryService.delete_document — DownloadTracker.file_path -> unlink
# (library_service.py:1041-1052)
@@ -278,7 +294,7 @@ class TestLibraryServiceDeleteDocumentTrackerFile:
doc = _make_document(
session, original_url="https://example.com/abs.pdf"
)
- _make_tracker(session, doc.original_url, str(canary))
+ _make_tracker(session, doc, str(canary))
deleted = self._run_delete(session, doc.id)
finally:
session.close()
@@ -306,7 +322,7 @@ class TestLibraryServiceDeleteDocumentTrackerFile:
doc = _make_document(
session, original_url="https://example.com/trav.pdf"
)
- _make_tracker(session, doc.original_url, traversal)
+ _make_tracker(session, doc, traversal)
deleted = self._run_delete(session, doc.id)
finally:
session.close()
@@ -338,7 +354,7 @@ class TestLibraryServiceDeleteDocumentTrackerFile:
doc = _make_document(
session, original_url="https://example.com/link.pdf"
)
- _make_tracker(session, doc.original_url, "evil_link")
+ _make_tracker(session, doc, "evil_link")
deleted = self._run_delete(session, doc.id)
finally:
session.close()
@@ -369,7 +385,7 @@ class TestLibraryServiceDeleteDocumentTrackerFile:
doc = _make_document(
session, original_url="https://example.com/legit.pdf"
)
- _make_tracker(session, doc.original_url, "pdfs/legit.pdf")
+ _make_tracker(session, doc, "pdfs/legit.pdf")
deleted = self._run_delete(session, doc.id)
finally:
session.close()
@@ -383,6 +399,103 @@ class TestLibraryServiceDeleteDocumentTrackerFile:
)
+# ---------------------------------------------------------------------------
+# LibraryService.delete_document — Document.file_path -> unlink
+# The document's OWN path is the primary unlink input; the tracker path
+# above is only borrowed for a document that has none.
+# ---------------------------------------------------------------------------
+
+
+class TestLibraryServiceDeleteDocumentOwnFile:
+ """The unlink of ``Document.file_path`` in ``LibraryService.
+ delete_document`` must never reach outside the caller's per-user
+ library root.
+
+ The tracker cases above use pathless documents, so they only exercise
+ the path ``delete_document`` borrows from a tracker. These cases plant
+ the same hostile values in the document's own ``file_path``.
+ """
+
+ def _run_delete(self, session, document_id):
+ with _patched_user_session([_LS_SESSION, _DD_SESSION], session):
+ return LibraryService(_TEST_USER).delete_document(document_id)
+
+ @pytest.mark.parametrize("kind", ["absolute", "traversal", "symlink"])
+ def test_hostile_document_file_path_is_preserved(
+ self, tmp_path, monkeypatch, kind
+ ):
+ with _settings_root(tmp_path, monkeypatch) as (_base, per_user_root):
+ outside = tmp_path / f"outside_own_{kind}"
+ outside.mkdir()
+ canary = outside / "canary.txt"
+ canary.write_text("DO NOT DELETE")
+ link = None
+ if kind == "absolute":
+ payload = str(canary)
+ elif kind == "traversal":
+ payload = _relative_escape(per_user_root, canary)
+ assert payload.startswith(".."), "payload must traverse"
+ else:
+ link = per_user_root / "evil_own_link"
+ try:
+ link.symlink_to(canary)
+ except OSError:
+ pytest.skip("Cannot create symlinks in this environment")
+ assert link.is_symlink()
+ payload = "evil_own_link"
+ outside_before = _files_outside_root(tmp_path, per_user_root)
+
+ session, engine = _seed_library_db()
+ try:
+ doc = _make_document(session, file_path=payload)
+ deleted = self._run_delete(session, doc.id)
+ finally:
+ session.close()
+ engine.dispose()
+
+ assert deleted is True
+ assert canary.exists(), (
+ "LibraryService.delete_document unlinked the target of a "
+ f"hostile ({kind}) Document.file_path outside the library "
+ "root — no containment check on the document's own path"
+ )
+ assert canary.read_text() == "DO NOT DELETE"
+ outside_after = _files_outside_root(tmp_path, per_user_root)
+ assert outside_after == outside_before, (
+ "LibraryService.delete_document removed a file outside the "
+ f"library root through a hostile ({kind}) Document.file_path"
+ )
+ if link is not None:
+ assert link.is_symlink() or link.exists(), (
+ "LibraryService.delete_document removed a symlink planted in "
+ "the library root through Document.file_path — no "
+ "leaf-symlink refusal present"
+ )
+
+ def test_legitimate_document_file_is_deleted(self, tmp_path, monkeypatch):
+ """POSITIVE CONTROL: a real file inside the per-user root named by
+ ``Document.file_path`` is still unlinked by delete_document."""
+ with _settings_root(tmp_path, monkeypatch) as (_base, per_user_root):
+ (per_user_root / "pdfs").mkdir(exist_ok=True)
+ legit = per_user_root / "pdfs" / "legit_own.pdf"
+ legit.write_bytes(b"%PDF-1.4 legit")
+
+ session, engine = _seed_library_db()
+ try:
+ doc = _make_document(session, file_path="pdfs/legit_own.pdf")
+ deleted = self._run_delete(session, doc.id)
+ finally:
+ session.close()
+ engine.dispose()
+
+ assert deleted is True
+ assert not legit.exists(), (
+ "POSITIVE CONTROL FAILED: a legitimate in-root file named by "
+ "Document.file_path was not deleted. If a containment fix was "
+ "just added, it is overly restrictive."
+ )
+
+
# ---------------------------------------------------------------------------
# DocumentDeletionService — Document.file_path -> unlink
# (document_deletion.py:239-266; routed via delete_routes.py:80)