Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(upload): release stale WAL snapshots before extraction (#7135)

LearningCircuitb98ce5f583d2
diff --git a/changelog.d/+upload-extraction-transactions.bugfix.md b/changelog.d/+upload-extraction-transactions.bugfix.md
new file mode 100644
index 0000000000..5f08f3c50c
--- /dev/null
+++ b/changelog.d/+upload-extraction-transactions.bugfix.md
@@ -0,0 +1 @@
+Collection uploads release their database snapshot and commit earlier files before parsing each document. The upload therefore holds no snapshot or lock while a parser runs, and a background write that commits during parsing no longer makes the file fail with `database is locked`. Brief windows remain between a file's lookups and its writes, where a background commit can still fail that file. Because earlier files are committed before the next parse, a request that fails partway through (an error response, not a per-file error) keeps the files it had already saved but does not auto-index them. Re-uploading them reports them as already in the collection, so index the collection to pick them up.
diff --git a/src/local_deep_research/web/routers/rag.py b/src/local_deep_research/web/routers/rag.py
index 47443b86b3..676fcf3480 100644
--- a/src/local_deep_research/web/routers/rag.py
+++ b/src/local_deep_research/web/routers/rag.py
@@ -2770,10 +2770,20 @@ def _upload_to_collection_sync(
else file_extension
)
- # Extract text using document_loaders module
+ # Parsing can import large libraries or spend minutes
+ # extracting a document. Release this read snapshot and
+ # any earlier file's write lock before that work. A
+ # background writer may otherwise make the snapshot
+ # impossible to upgrade (SQLITE_BUSY_SNAPSHOT).
+ sp.commit()
+ sp = None
+ db_session.commit()
extracted_text = extract_text_from_bytes(
file_content, file_extension, filename
)
+ # Keep write failures isolated to this file after the
+ # parser returns, using a fresh database snapshot.
+ sp = db_session.begin_nested()
# Clean the extracted text to remove surrogate characters
if extracted_text:
diff --git a/tests/conftest.py b/tests/conftest.py
index d8a7b20769..4a27f3859f 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -30,6 +30,17 @@ from local_deep_research.settings.manager import (
SettingsManager,
)
+# The document and collection deletion services bind
+# ``database.session_context.get_user_db_session`` when they are first
+# imported. Several code paths import the deletion package lazily
+# (``get_collection_documents``, the indexing routes and workers,
+# ``LibraryService.delete_document`` ...), and many tests patch that helper
+# around them. Whichever such test imported the package first would leave its
+# mock in the services for the rest of the process, and later tests that
+# delete through the real services would fail depending on test order.
+# Import the package here, before any test can patch the helper.
+import local_deep_research.research_library.deletion # noqa: F401
+
# Import our mock fixtures
try:
from .mock_fixtures import (
diff --git a/tests/research_library/routes/test_rag_routes_deep_main_port.py b/tests/research_library/routes/test_rag_routes_deep_main_port.py
index 5b950d9d93..b63d7b0c4b 100644
--- a/tests/research_library/routes/test_rag_routes_deep_main_port.py
+++ b/tests/research_library/routes/test_rag_routes_deep_main_port.py
@@ -781,11 +781,9 @@ class TestCollectionDocumentsNoIndex:
_fake_request(), collection_id, username="testuser"
)
- def test_no_rag_index_returns_null_size(self):
- """With no ``RAGIndex`` row, both index-size fields are ``None``
- rather than 0 or absent -- the details page distinguishes "never
- indexed" from "indexed, empty"."""
- collection = Mock(
+ @staticmethod
+ def _user_collection():
+ return Mock(
id="coll-1",
name="Test Collection",
description="Desc",
@@ -801,11 +799,42 @@ class TestCollectionDocumentsNoIndex:
collection_type="user_uploads",
)
- result = self._call(collection)
+ def test_no_rag_index_returns_null_size(self):
+ """With no ``RAGIndex`` row, both index-size fields are ``None``
+ rather than 0 or absent -- the details page distinguishes "never
+ indexed" from "indexed, empty"."""
+ result = self._call(self._user_collection())
assert result["collection"]["index_file_size"] is None
assert result["collection"]["index_file_size_bytes"] is None
+ def test_lookup_leaves_deletion_services_on_the_real_session_helper(
+ self,
+ ):
+ """The protected flag lazily imports the deletion services while
+ ``_call`` patches ``get_user_db_session``. Both services bind that
+ helper when first imported, so ``tests/conftest.py`` loads them
+ before any test runs. If this lookup imported them first, they would
+ keep this test's mock, and later tests that delete through the real
+ services in the same process would use it."""
+ from local_deep_research.database import session_context
+
+ result = self._call(self._user_collection())
+
+ assert result["success"] is True
+ assert result["collection"]["is_protected"] is False
+
+ from local_deep_research.research_library.deletion.services import (
+ collection_deletion,
+ document_deletion,
+ )
+
+ for service in (collection_deletion, document_deletion):
+ assert (
+ service.get_user_db_session
+ is session_context.get_user_db_session
+ ), service.__name__
+
def test_collection_not_found_returns_404(self):
result = self._call(None, collection_id="nonexistent")
diff --git a/tests/research_library/routes/test_rag_routes_upload_main_port.py b/tests/research_library/routes/test_rag_routes_upload_main_port.py
index f84c12a7f5..9be052c5c3 100644
--- a/tests/research_library/routes/test_rag_routes_upload_main_port.py
+++ b/tests/research_library/routes/test_rag_routes_upload_main_port.py
@@ -75,6 +75,7 @@ from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock, patch
+import pytest
from starlette.datastructures import FormData, UploadFile
from starlette.responses import JSONResponse
@@ -117,12 +118,19 @@ def _build_mock_query(all_result=None, first_result=None, count_result=0):
return q
-def _make_db_session():
+def _make_db_session(fail=None):
"""Mock session whose ``begin_nested()`` hands out inspectable SAVEPOINTs.
Each savepoint records ``commit``/``rollback`` and flips ``is_active``,
so the ported tests can assert per-file SAVEPOINT wiring exactly as main
- did.
+ did. Each one also records its ``role`` and ``file_number``. A file's
+ ``read`` SAVEPOINT covers the duplicate check; a new document's
+ ``write`` SAVEPOINT opens once the parser has returned (``_upload_env``
+ counts the returns). Files are numbered by their read SAVEPOINTs.
+
+ ``fail`` maps ``(role, file_number)`` to the step that raises for that
+ SAVEPOINT: ``"open"`` (``begin_nested()`` itself), ``"commit"`` or
+ ``"rollback"``. A failed step leaves ``is_active`` unchanged.
"""
db_session = Mock()
db_session.query = Mock(return_value=_build_mock_query())
@@ -130,14 +138,43 @@ def _make_db_session():
db_session.add = Mock()
db_session.flush = Mock()
db_session.expire_all = Mock()
+ db_session._parser_returns = 0
savepoints: list = []
+ fail = fail or {}
+ seen = {"files": 0, "parser_returns": 0}
def _begin_nested():
+ if db_session._parser_returns > seen["parser_returns"]:
+ role = "write"
+ else:
+ role = "read"
+ seen["files"] += 1
+ seen["parser_returns"] = db_session._parser_returns
+ file_number = seen["files"]
+
+ def step(name):
+ if fail.get((role, file_number)) == name:
+ raise RuntimeError(
+ f"{name} failed for the {role} SAVEPOINT of file "
+ f"{file_number}"
+ )
+
+ step("open")
sp = Mock()
+ sp.role = role
+ sp.file_number = file_number
sp.is_active = True
- sp.commit = Mock(side_effect=lambda: setattr(sp, "is_active", False))
- sp.rollback = Mock(side_effect=lambda: setattr(sp, "is_active", False))
+
+ def close(name):
+ def _close():
+ step(name)
+ sp.is_active = False
+
+ return _close
+
+ sp.commit = Mock(side_effect=close("commit"))
+ sp.rollback = Mock(side_effect=close("rollback"))
savepoints.append(sp)
return sp
@@ -146,6 +183,67 @@ def _make_db_session():
return db_session
+def _roles(db_session):
+ """``(role, file_number)`` of every SAVEPOINT, in the order opened."""
+ return [(sp.role, sp.file_number) for sp in db_session._savepoints]
+
+
+def _savepoint(db_session, role, file_number):
+ """The one SAVEPOINT opened for ``file_number`` in ``role``."""
+ matches = [
+ sp
+ for sp in db_session._savepoints
+ if (sp.role, sp.file_number) == (role, file_number)
+ ]
+ assert len(matches) == 1, _roles(db_session)
+ return matches[0]
+
+
+@contextmanager
+def _count_parser_returns(db_session):
+ """Count each parser return on a ``_make_db_session()`` session.
+
+ Wraps whichever ``extract_text_from_bytes`` is installed (a test's mock
+ or the real parser), so ``begin_nested()`` can tell a write SAVEPOINT
+ from the next file's read SAVEPOINT. A parser call that raises is not
+ counted. Sessions built some other way are left alone.
+ """
+ if not isinstance(getattr(db_session, "_parser_returns", None), int):
+ yield
+ return
+
+ from local_deep_research import document_loaders
+
+ parse = document_loaders.extract_text_from_bytes
+
+ def counted(*args, **kwargs):
+ text = parse(*args, **kwargs)
+ db_session._parser_returns += 1
+ return text
+
+ with patch.object(document_loaders, "extract_text_from_bytes", counted):
+ yield
+
+
+def _fail_document_flush(db_session, filename, message):
+ """Inject a write failure while the document's SAVEPOINT is active."""
+ from local_deep_research.database.models.library import Document
+
+ failure = Mock(side_effect=RuntimeError(message))
+
+ def flush():
+ for call in reversed(db_session.add.call_args_list):
+ document = call.args[0]
+ if isinstance(document, Document):
+ if document.filename == filename:
+ assert db_session._savepoints[-1].is_active
+ failure()
+ return
+
+ db_session.flush.side_effect = flush
+ return failure
+
+
def _make_settings_mock(overrides=None):
"""Settings manager mock with the RAG defaults the upload path reads."""
from local_deep_research.constants import (
@@ -218,7 +316,8 @@ def _upload_env(
``utilities.db_utils`` (the branch calls it as
``get_settings_manager(db_session, username)``), and the session password
store (defaulting to "no password", so auto-indexing stays out of the way
- unless a test asks for it).
+ unless a test asks for it). It also counts parser returns, which is how
+ a ``_make_db_session()`` session labels its read and write SAVEPOINTs.
"""
db_session = db_session if db_session is not None else _make_db_session()
mock_sm = _make_settings_mock(settings_overrides)
@@ -243,6 +342,8 @@ def _upload_env(
with ExitStack() as stack:
for p in patches:
stack.enter_context(p)
+ # Inside the test's own parser patch, so it wraps that mock.
+ stack.enter_context(_count_parser_returns(db_session))
yield SimpleNamespace(
db_session=db_session,
settings=mock_sm,
@@ -499,7 +600,8 @@ class TestUploadToCollection:
mock_source = Mock()
mock_source.id = "src-sp"
- db_session = _make_db_session()
+ # The second file's write fails, and so does rolling it back.
+ db_session = _make_db_session(fail={("write", 2): "rollback"})
db_session.query = Mock(
side_effect=_model_query_side_effect(
collection=mock_coll,
@@ -508,29 +610,10 @@ class TestUploadToCollection:
)
)
- def fake_extract(content, ext, filename):
- if filename == "ErrorFile.txt":
- raise RuntimeError("Simulated processing error on second file")
- return f"Extracted text for {filename}"
-
- call_idx = {"n": 0}
-
- def _custom_begin_nested():
- idx = call_idx["n"]
- call_idx["n"] += 1
- sp = Mock()
- sp.is_active = True
- sp.commit = Mock()
- if idx == 1:
- sp.rollback = Mock(
- side_effect=RuntimeError("connection dead during rollback")
- )
- else:
- sp.rollback = Mock()
- db_session._savepoints.append(sp)
- return sp
+ fail_flush = _fail_document_flush(
+ db_session, "ErrorFile.txt", "Simulated write error on second file"
+ )
- db_session.begin_nested = Mock(side_effect=_custom_begin_nested)
mock_opt_logger = Mock()
mock_opt = Mock(return_value=mock_opt_logger)
@@ -540,7 +623,7 @@ class TestUploadToCollection:
db_session=db_session,
settings_overrides={"research_library.upload_pdf_storage": "none"},
extra_patches=[
- *_extraction_patches(extract_side_effect=fake_extract),
+ *_extraction_patches(text="extracted text"),
patch(f"{MODULE}.logger.opt", mock_opt),
],
):
@@ -566,15 +649,23 @@ class TestUploadToCollection:
assert "Third.txt" in by_name
assert by_name["Third.txt"]["status"] == "duplicate_in_batch"
- assert len(db_session._savepoints) == 3
- db_session._savepoints[0].commit.assert_called_once()
- db_session._savepoints[0].rollback.assert_not_called()
- db_session._savepoints[1].rollback.assert_called_once()
- db_session._savepoints[1].commit.assert_not_called()
- db_session._savepoints[2].commit.assert_called_once()
- db_session._savepoints[2].rollback.assert_not_called()
+ fail_flush.assert_called_once()
+ assert _roles(db_session) == [
+ ("read", 1),
+ ("write", 1),
+ ("read", 2),
+ ("write", 2),
+ ("read", 3),
+ ]
+ failed = _savepoint(db_session, "write", 2)
+ failed.rollback.assert_called_once()
+ failed.commit.assert_not_called()
+ for sp in db_session._savepoints:
+ if sp is not failed:
+ sp.commit.assert_called_once()
+ sp.rollback.assert_not_called()
- db_session.commit.assert_called_once()
+ assert db_session.commit.call_count == 3
assert mock_opt.called
assert any(
@@ -630,33 +721,22 @@ class TestUploadToCollection:
db_session._savepoints[0].rollback.assert_called_once()
db_session.commit.assert_called_once()
- def test_upload_begin_nested_failure_isolates_failing_file(self):
+ @pytest.mark.parametrize("role", ["read", "write"])
+ def test_upload_begin_nested_failure_isolates_failing_file(self, role):
"""Ported from ``...::test_upload_begin_nested_failure_isolates_failing_file``.
- ``begin_nested()`` itself raising for one file is caught as a per-file
- error (``sp`` is still None, so no rollback is attempted); earlier
- successes survive and the batch is still a 200.
+ Opening either the read or the write SAVEPOINT for the second file
+ can fail. Both failures are isolated to that file. ``sp`` is None
+ when the open fails, so nothing is rolled back; in the write case
+ that includes the second file's read SAVEPOINT, which was already
+ committed. The first file's upload survives and the batch stays a 200.
"""
mock_coll = Mock()
mock_coll.id = "coll-1"
mock_source = Mock()
mock_source.id = "src-bn"
- db_session = _make_db_session()
- bn_count = {"n": 0}
-
- def _conditional_begin_nested():
- bn_count["n"] += 1
- if bn_count["n"] == 1:
- sp = Mock()
- sp.is_active = True
- sp.commit = Mock()
- sp.rollback = Mock()
- db_session._savepoints.append(sp)
- return sp
- raise RuntimeError("begin_nested failed on file 2")
-
- db_session.begin_nested = Mock(side_effect=_conditional_begin_nested)
+ db_session = _make_db_session(fail={(role, 2): "open"})
db_session.query = Mock(
side_effect=_model_query_side_effect(
collection=mock_coll, source_type=mock_source
@@ -685,10 +765,21 @@ class TestUploadToCollection:
assert len(rdata["errors"]) == 1
assert rdata["errors"][0]["filename"] == "file2.txt"
assert rdata["errors"][0]["error"] == "Failed to upload file"
- db_session.commit.assert_called_once()
+ opened = [("read", 1), ("write", 1)]
+ if role == "write":
+ opened.append(("read", 2))
+ assert _roles(db_session) == opened
+ # The failed open is the one extra begin_nested() call.
+ assert db_session.begin_nested.call_count == len(opened) + 1
+ for sp in db_session._savepoints:
+ sp.commit.assert_called_once()
+ sp.rollback.assert_not_called()
+ # One commit before each parse, plus the final one.
+ assert db_session.commit.call_count == (3 if role == "write" else 2)
+ @pytest.mark.parametrize("role", ["read", "write"])
def test_upload_soft_validation_savepoint_commit_failure_does_not_double_report(
- self,
+ self, role
):
"""Ported from ``...::test_upload_soft_validation_savepoint_commit_failure_does_not_double_report``.
@@ -699,36 +790,28 @@ class TestUploadToCollection:
Trigger translated: main used an oversized file, but on this branch the
size check moved into the async wrapper and its ``errors.append``
happens BEFORE ``begin_nested()``, so it can no longer reach a
- savepoint commit. The unsupported-extension branch is the same
- ``sp.commit(); errors.append(...); continue`` shape inside the
- savepoint, so it exercises the identical ordering property.
+ savepoint commit. Two soft exits have the same
+ ``sp.commit(); errors.append(...); continue`` shape: an unsupported
+ extension exits the read SAVEPOINT, and empty extracted text exits
+ the write SAVEPOINT. Each is driven by failing that SAVEPOINT's commit.
"""
mock_coll = Mock()
mock_coll.id = "coll-1"
- db_session = _make_db_session()
+ db_session = _make_db_session(fail={(role, 1): "commit"})
db_session.query = Mock(
side_effect=_model_query_side_effect(collection=mock_coll)
)
-
- def _failing_commit_begin_nested():
- sp = Mock()
- sp.is_active = True
- sp.commit = Mock(side_effect=RuntimeError("commit release failed"))
- sp.rollback = Mock()
- db_session._savepoints.append(sp)
- return sp
-
- db_session.begin_nested = Mock(side_effect=_failing_commit_begin_nested)
-
- with _upload_env(
- db_session=db_session,
- extra_patches=[
+ if role == "read":
+ soft_exit = [
patch(
f"{_DOC_LOADERS}.is_extension_supported", return_value=False
)
- ],
- ):
+ ]
+ else:
+ soft_exit = _extraction_patches(text="")
+
+ with _upload_env(db_session=db_session, extra_patches=soft_exit):
status, rdata = _body(_run_upload([("big.txt", b"x" * 200)]))
assert status == 200
@@ -736,25 +819,29 @@ class TestUploadToCollection:
assert len(rdata["errors"]) == 1
assert rdata["errors"][0]["filename"] == "big.txt"
assert "Failed to upload file" in rdata["errors"][0]["error"]
+ _savepoint(db_session, role, 1).rollback.assert_called_once()
def test_upload_success_path_savepoint_commit_failure_suppresses_publication(
self,
):
"""Ported from ``...::test_upload_success_path_savepoint_commit_failure_suppresses_publication``.
- When ``sp.commit()`` raises on the new-document success path, the file
- lands in ``errors`` (not ``uploaded``) AND is never registered in
- ``seen_hashes`` -- so a later identical file is processed fresh instead
- of being falsely flagged ``duplicate_in_batch`` against a file that
- never made it in. Move ``seen_hashes[file_hash] = new_doc`` above the
- ``sp.commit()`` and the second assertion below goes red.
+ When the write SAVEPOINT's ``sp.commit()`` raises on the new-document
+ success path, the file lands in ``errors`` (not ``uploaded``) AND is
+ never registered in ``seen_hashes`` -- so a later identical file is
+ processed fresh instead of being falsely flagged ``duplicate_in_batch``
+ against a file that never made it in. Move
+ ``seen_hashes[file_hash] = new_doc`` above that ``sp.commit()`` and
+ the ``uploaded`` assertions below go red. Failing the read
+ SAVEPOINT's commit instead would stop the file before parsing and
+ never reach this path.
"""
mock_coll = Mock()
mock_coll.id = "coll-1"
mock_source = Mock()
mock_source.id = "src-sp"
- db_session = _make_db_session()
+ db_session = _make_db_session(fail={("write", 1): "commit"})
db_session.query = Mock(
side_effect=_model_query_side_effect(
collection=mock_coll,
@@ -762,27 +849,6 @@ class TestUploadToCollection:
once_collection=True,
)
)
-
- call_idx = {"n": 0}
-
- def _custom_begin_nested():
- idx = call_idx["n"]
- call_idx["n"] += 1
- sp = Mock()
- sp.is_active = True
- if idx == 0:
- sp.commit = Mock(
- side_effect=RuntimeError(
- "savepoint commit failed for file 1"
- )
- )
- else:
- sp.commit = Mock()
- sp.rollback = Mock()
- db_session._savepoints.append(sp)
- return sp
-
- db_session.begin_nested = Mock(side_effect=_custom_begin_nested)
shared_bytes = b"same content for file 1 and 2"
with _upload_env(
@@ -810,6 +876,16 @@ class TestUploadToCollection:
assert rdata["uploaded"][0]["filename"] == "File2.txt"
assert rdata["uploaded"][0]["status"] == "uploaded"
+ # File1 reached its write SAVEPOINT, whose failed commit was rolled
+ # back; File2 was parsed and written as a new document.
+ assert _roles(db_session) == [
+ ("read", 1),
+ ("write", 1),
+ ("read", 2),
+ ("write", 2),
+ ]
+ _savepoint(db_session, "write", 1).rollback.assert_called_once()
+
def test_upload_empty_files_list(self):
"""Ported from ``...::test_upload_empty_files_list``.
@@ -1242,9 +1318,9 @@ class TestUploadToCollection:
hash in ``seen_hashes`` before the success is published would silently
drop both copies of the user's content.
- Also pins that a soft failure still COMMITS its savepoint (main's
- ``sp.commit()`` on the soft-validation exit), so the batch never leaves
- nested SAVEPOINTs stacked open.
+ Both files close their read SAVEPOINT before parsing and their write
+ SAVEPOINT afterward, including the soft-validation exit. The batch
+ never leaves nested SAVEPOINTs stacked open.
"""
mock_coll = Mock()
mock_coll.id = "coll-1"
@@ -1285,19 +1361,27 @@ class TestUploadToCollection:
uploaded_names = {f["filename"]: f for f in rdata["uploaded"]}
assert "Second.txt" in uploaded_names
assert uploaded_names["Second.txt"]["status"] == "uploaded"
- assert db_session.begin_nested.call_count == 2
- assert len(db_session._savepoints) == 2
+ assert db_session.begin_nested.call_count == 4
+ assert _roles(db_session) == [
+ ("read", 1),
+ ("write", 1),
+ ("read", 2),
+ ("write", 2),
+ ]
for sp in db_session._savepoints:
sp.commit.assert_called_once()
sp.rollback.assert_not_called()
- def test_upload_savepoint_isolates_failing_file_new_doc(self):
+ @pytest.mark.parametrize("failure_stage", ["parsing", "writing"])
+ def test_upload_savepoint_isolates_failing_file_new_doc(
+ self, failure_stage
+ ):
"""Ported from ``...::test_upload_savepoint_isolates_failing_file_new_doc``.
A middle file failing rolls back ONLY its savepoint: the earlier
insert and its ``seen_hashes`` entry survive, so a third file with the
- first file's bytes is still ``duplicate_in_batch``. Pins the exact
- per-savepoint commit/rollback pattern (commit, rollback, commit).
+ first file's bytes is still ``duplicate_in_batch``. Parsing fails
+ between transactions; a failed flush rolls back its write SAVEPOINT.
"""
mock_coll = Mock()
mock_coll.id = "coll-1"
@@ -1313,9 +1397,16 @@ class TestUploadToCollection:
)
)
+ if failure_stage == "writing":
+ failure = _fail_document_flush(
+ db_session, "ErrorFile.txt", "Simulated DB error on second file"
+ )
+ else:
+ failure = Mock(side_effect=RuntimeError("Simulated parser error"))
+
def fake_extract(content, ext, filename):
- if filename == "ErrorFile.txt":
- raise RuntimeError("Simulated DB error on second file")
+ if failure_stage == "parsing" and filename == "ErrorFile.txt":
+ failure()
return f"Extracted text for {filename}"
shared_bytes = b"identical bytes for First and Third"
@@ -1344,16 +1435,35 @@ class TestUploadToCollection:
assert "Third.txt" in by_name
assert by_name["Third.txt"]["status"] == "duplicate_in_batch"
- assert db_session.begin_nested.call_count == 3
- assert len(db_session._savepoints) == 3
- db_session._savepoints[0].commit.assert_called_once()
- db_session._savepoints[0].rollback.assert_not_called()
- db_session._savepoints[1].rollback.assert_called_once()
- db_session._savepoints[1].commit.assert_not_called()
- db_session._savepoints[2].commit.assert_called_once()
- db_session._savepoints[2].rollback.assert_not_called()
+ failure.assert_called_once()
+ if failure_stage == "writing":
+ opened = [
+ ("read", 1),
+ ("write", 1),
+ ("read", 2),
+ ("write", 2),
+ ("read", 3),
+ ]
+ rolled_back = _savepoint(db_session, "write", 2)
+ else:
+ # The parser raised, so the second file never opened a write
+ # SAVEPOINT and nothing needs rolling back.
+ opened = [("read", 1), ("write", 1), ("read", 2), ("read", 3)]
+ rolled_back = None
+ assert _roles(db_session) == opened
+ assert db_session.begin_nested.call_count == len(opened)
+ for sp in db_session._savepoints:
+ if sp is rolled_back:
+ sp.rollback.assert_called_once()
+ sp.commit.assert_not_called()
+ else:
+ sp.commit.assert_called_once()
+ sp.rollback.assert_not_called()
- def test_upload_savepoint_isolates_failing_file_existing_doc(self):
+ @pytest.mark.parametrize("failure_stage", ["parsing", "writing"])
+ def test_upload_savepoint_isolates_failing_file_existing_doc(
+ self, failure_stage
+ ):
"""Ported from ``...::test_upload_savepoint_isolates_failing_file_existing_doc``.
Same isolation property on the EXISTING-document arm: the first file's
@@ -1402,9 +1512,16 @@ class TestUploadToCollection:
db_session.query = Mock(side_effect=query_side_effect)
+ if failure_stage == "writing":
+ failure = _fail_document_flush(
+ db_session, "ErrorFile.txt", "Simulated flush error"
+ )
+ else:
+ failure = Mock(side_effect=RuntimeError("Simulated parser error"))
+
def fake_extract(content, ext, filename):
- if filename == "ErrorFile.txt":
- raise RuntimeError("Simulated flush error")
+ if failure_stage == "parsing" and filename == "ErrorFile.txt":
+ failure()
return f"Extracted text for {filename}"
with _upload_env(
@@ -1431,11 +1548,24 @@ class TestUploadToCollection:
assert "Third.txt" in by_name
assert by_name["Third.txt"]["status"] == "duplicate_in_batch"
- assert db_session.begin_nested.call_count == 3
- assert len(db_session._savepoints) == 3
- db_session._savepoints[0].commit.assert_called_once()
- db_session._savepoints[1].rollback.assert_called_once()
- db_session._savepoints[2].commit.assert_called_once()
+ failure.assert_called_once()
+ # The first file is a pre-existing document, so it is linked inside
+ # its read SAVEPOINT and never parsed.
+ if failure_stage == "writing":
+ opened = [("read", 1), ("read", 2), ("write", 2), ("read", 3)]
+ rolled_back = _savepoint(db_session, "write", 2)
+ else:
+ opened = [("read", 1), ("read", 2), ("read", 3)]
+ rolled_back = None
+ assert _roles(db_session) == opened
+ assert db_session.begin_nested.call_count == len(opened)
+ for sp in db_session._savepoints:
+ if sp is rolled_back:
+ sp.rollback.assert_called_once()
+ sp.commit.assert_not_called()
+ else:
+ sp.commit.assert_called_once()
+ sp.rollback.assert_not_called()
def test_upload_intra_batch_pdf_upgrade_with_pre_existing_doc_twin(self):
"""Ported from ``...::test_upload_intra_batch_pdf_upgrade_with_pre_existing_doc_twin``.
diff --git a/tests/research_library/test_library_pipeline_contracts.py b/tests/research_library/test_library_pipeline_contracts.py
index ce10b59219..7fc77b9f3c 100644
--- a/tests/research_library/test_library_pipeline_contracts.py
+++ b/tests/research_library/test_library_pipeline_contracts.py
@@ -232,6 +232,73 @@ class TestUploadStoreRetrieveFidelity:
assert sizes[doc_id] == len(body), sizes
assert sizes[sib_id] == len(sibling), sizes
+ @pytest.mark.parametrize("file_count", [1, 2])
+ def test_upload_survives_another_writer_during_extraction(
+ self, upload_client, monkeypatch, file_count
+ ):
+ """A parser must not hold a stale WAL snapshot or an earlier file's lock."""
+ from sqlalchemy import text
+
+ from local_deep_research import document_loaders
+ from local_deep_research.database.encrypted_db import db_manager
+
+ collection_id = _new_collection(
+ upload_client, f"concurrent-extraction-{file_count}"
+ )
+ engine = db_manager.connections[upload_client.username]
+ with engine.connect() as connection:
+ assert (
+ connection.exec_driver_sql("PRAGMA journal_mode").scalar()
+ == "wal"
+ )
+ extract = document_loaders.extract_text_from_bytes
+ committed_during_extraction = []
+
+ def extract_with_an_independent_writer(content, extension, filename):
+ result = extract(content, extension, filename)
+ # This is a separate real SQLCipher connection, as used by
+ # post-login work and background jobs. No sleeps or mocked DB.
+ with engine.begin() as writer:
+ writer.exec_driver_sql("PRAGMA busy_timeout=1000")
+ writer.execute(
+ text(
+ "UPDATE collections SET description = :value WHERE id = :id"
+ ),
+ {"value": filename, "id": collection_id},
+ )
+ committed_during_extraction.append(filename)
+ return result
+
+ monkeypatch.setattr(
+ document_loaders,
+ "extract_text_from_bytes",
+ extract_with_an_independent_writer,
+ )
+ filenames = [f"concurrent-{index}.txt" for index in range(file_count)]
+ contents = {name: f"{name}: {uuid.uuid4().hex}" for name in filenames}
+ response = upload_client.post(
+ f"/library/api/collections/{collection_id}/upload",
+ files=[
+ ("files", (name, contents[name].encode(), "text/plain"))
+ for name in filenames
+ ],
+ )
+ assert response.status_code == 200, response.text
+ body = response.json()
+ assert committed_during_extraction == filenames
+ assert body["errors"] == [], body
+ assert body["summary"] == {
+ "total": file_count,
+ "successful": file_count,
+ "failed": 0,
+ }
+ for uploaded in body["uploaded"]:
+ assert (
+ _text_of(upload_client, uploaded["id"]).json()["text_content"]
+ == contents[uploaded["filename"]]
+ )
+ assert len(body["uploaded"]) == file_count
+
def test_markdown_is_stripped_of_markup_before_storage(self, upload_client):
"""``.md`` does NOT round-trip: the loader flattens it to prose.