~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Stress and boundary tests for Notes functionality."""
import asyncio
import re
import uuid
from unittest.mock import AsyncMock, MagicMock
import pytest
from local_deep_research.database.models import (
Document,
NoteLink,
NoteVersion,
)
from tests.notes.helpers import _generate_hash
class TestNoteSynthesisBoundary:
"""Boundary tests for note synthesis."""
@staticmethod
def _patch_empty_notes_db(monkeypatch, service):
"""Patch the DB so the note lookup succeeds but finds no notes.
synthesize_notes now re-raises infra failures (like every other AI
method) instead of swallowing them into an error dict, so these
boundary tests must mock the DB rather than rely on an unreachable
encrypted DB being converted to a generic error.
"""
from contextlib import contextmanager
monkeypatch.setattr(
service, "_get_note_source_type_id", lambda session: "note-st"
)
sess = MagicMock()
sess.query.return_value.filter.return_value.all.return_value = []
@contextmanager
def fake_session(username=None, password=None):
yield sess
monkeypatch.setattr(
"local_deep_research.research_library.notes.services.note_ai_service.get_user_db_session",
fake_session,
)
def test_synthesis_with_exactly_two_notes(self, monkeypatch):
"""Test synthesis with minimum (2) notes."""
from local_deep_research.research_library.notes.services.note_ai_service import (
NoteAIService,
)
service = NoteAIService("test_user")
self._patch_empty_notes_db(monkeypatch, service)
# Exactly 2 notes - should not fail on count
result = asyncio.run(
service.synthesize_notes(
[str(uuid.uuid4()), str(uuid.uuid4())], "merge"
)
)
# Will fail because notes don't exist, but not due to count
assert "2-5" not in str(result.get("error", ""))
def test_synthesis_with_exactly_five_notes(self, monkeypatch):
"""Test synthesis with maximum (5) notes."""
from local_deep_research.research_library.notes.services.note_ai_service import (
NoteAIService,
)
service = NoteAIService("test_user")
self._patch_empty_notes_db(monkeypatch, service)
# Exactly 5 notes - should not fail on count
result = asyncio.run(
service.synthesize_notes(
[str(uuid.uuid4()) for _ in range(5)], "merge"
)
)
# Will fail because notes don't exist, but not due to count
assert "2-5" not in str(result.get("error", ""))
def test_synthesis_with_one_note_fails(self):
"""Test synthesis with 1 note fails."""
from local_deep_research.research_library.notes.services.note_ai_service import (
NoteAIService,
)
service = NoteAIService("test_user")
result = asyncio.run(
service.synthesize_notes([str(uuid.uuid4())], "merge")
)
assert "error" in result
assert "2-5" in result["error"]
def test_synthesis_with_six_notes_fails(self):
"""Test synthesis with 6 notes fails."""
from local_deep_research.research_library.notes.services.note_ai_service import (
NoteAIService,
)
service = NoteAIService("test_user")
result = asyncio.run(
service.synthesize_notes(
[str(uuid.uuid4()) for _ in range(6)], "merge"
)
)
assert "error" in result
assert "2-5" in result["error"]
def test_synthesis_with_zero_notes_fails(self):
"""Test synthesis with 0 notes fails."""
from local_deep_research.research_library.notes.services.note_ai_service import (
NoteAIService,
)
service = NoteAIService("test_user")
result = asyncio.run(service.synthesize_notes([], "merge"))
assert "error" in result
assert "2-5" in result["error"]
def test_all_synthesis_types(self, monkeypatch):
"""Each valid synthesis type passes the type-check.
Pre-fix this used ``A or B`` where ``A`` was almost always True
(the type literal doesn't appear in the unrelated error
message), so the assertion short-circuited and never actually
checked the intended condition. The intent: for each valid
synthesis type, the service should NOT reject it as "Unknown
synthesis type" — any other failure reason (e.g. notes not
found) is fine.
"""
from local_deep_research.research_library.notes.services.note_ai_service import (
NoteAIService,
)
service = NoteAIService("test_user")
self._patch_empty_notes_db(monkeypatch, service)
note_ids = [str(uuid.uuid4()) for _ in range(2)]
for synthesis_type in ["merge", "summarize", "compare"]:
result = asyncio.run(
service.synthesize_notes(note_ids, synthesis_type)
)
error_msg = str(result.get("error", ""))
# The valid type must not be flagged as unknown.
assert "Unknown synthesis type" not in error_msg, (
f"Valid type {synthesis_type!r} reported as unknown: "
f"{error_msg!r}"
)
def test_synthesize_notes_rejects_unknown_synthesis_type_without_invoking_llm(
self, monkeypatch
):
"""The enum gate (note_ai_service.py:1145-1149) rejects an unknown
synthesis_type BEFORE any LLM/persistence work. Positive complement
to test_all_synthesis_types (which only asserts valid types aren't
flagged). Pass exactly 2 valid note ids so the 2-5 count check passes
and the synthesis_type check is the failing gate; spy on the LLM to
prove it's never invoked.
"""
from local_deep_research.research_library.notes.services.note_ai_service import (
NoteAIService,
)
service = NoteAIService("test_user")
self._patch_empty_notes_db(monkeypatch, service)
# Two valid ids → passes the count gate (line 1137); the type gate
# (line 1145) is the one that should fire.
note_ids = [str(uuid.uuid4()), str(uuid.uuid4())]
def _no_llm(*args, **kwargs):
raise AssertionError(
"LLM must not be called for unknown synthesis type"
)
monkeypatch.setattr(service, "_get_llm", _no_llm)
result = asyncio.run(service.synthesize_notes(note_ids, "translate"))
assert result["success"] is False
assert "Unknown synthesis type" in result["error"]
# The f-string includes the bad value (line 1147).
assert "translate" in result["error"]
class TestNoteServiceStress:
"""Stress tests for NoteService."""
def test_slug_generation_stress(self):
"""Test slug generation with various inputs."""
from local_deep_research.research_library.notes.services.note_service import (
NoteService,
)
# Titles that strip to empty now fall back to "note" instead of
# "" so downstream consumers always have a non-empty identifier.
test_cases = [
("", "note"),
("a", "a"),
("A B C", "a-b-c"),
("123", "123"),
("a" * 1000, "a" * 500), # Truncated
("Hello World", "hello-world"),
(" spaces ", "spaces"),
("CAPS", "caps"),
("under_score", "underscore"), # Underscores removed
("dot.dot", "dotdot"),
("slash/slash", "slashslash"),
# CJK has no NFKD ASCII form → dropped (we don't ship a copyleft
# transliterator); a CJK-only title takes the title-derived
# `note-<hex>` fallback instead of the shared constant (#6389),
# so it is checked by shape below rather than by literal.
("Mix日本語Mix", "mixmix"),
]
for input_title, expected_slug in test_cases:
result = NoteService._generate_slug(input_title)
assert result == expected_slug, f"Failed for '{input_title}'"
# Titles that leave nothing behind after the ASCII fold still get a
# slug of their own, derived from the title (#6389). "---" is in this
# set rather than the table above for the same reason: it has content,
# so it no longer shares the "note" constant with every other such
# title.
for empty_slug_title in ("日本語", "---"):
assert re.fullmatch(
r"note-[0-9a-f]{12}",
NoteService._generate_slug(empty_slug_title),
), f"Failed for '{empty_slug_title}'"
def test_content_hash_stress(self):
"""Test content hash with various inputs."""
from local_deep_research.research_library.notes.services.note_service import (
NoteService,
)
test_cases = [
"",
"a",
"a" * 10000,
"Hello\nWorld",
"Hello\r\nWorld",
"Unicode 世界",
"Emoji 🎉",
" " * 100,
]
for content in test_cases:
result = NoteService._compute_content_hash(content)
assert len(result) == 64
# Should be deterministic
assert result == NoteService._compute_content_hash(content)
def test_link_pattern_stress(self):
"""Test link pattern with various inputs."""
from local_deep_research.research_library.notes.services.note_service import (
LINK_PATTERN,
)
test_cases = [
("No links", 0),
("[[Single]]", 1),
("[[One]] [[Two]]", 2),
("[[" + "a" * 100 + "]]", 1),
("[[]]", 0), # Empty brackets don't match (require content)
("[Not a link]", 0),
("[[Nested [[inside]]]]", 1),
("[[With special !@#$%]]", 1),
("Multi\nline\n[[link]]", 1),
("[[a]]" * 100, 100),
]
for content, expected_count in test_cases:
matches = LINK_PATTERN.findall(content)
assert len(matches) == expected_count, (
f"Failed for '{content[:50]}...'"
)
class TestDatabaseStress:
"""Stress tests for database operations."""
def test_create_many_notes(self, db_session, note_source_type):
"""Test creating many notes."""
for i in range(100):
note = Document(
id=str(uuid.uuid4()),
title=f"Note {i}",
text_content=f"Content for note {i}",
file_type="note",
file_size=0,
document_hash=_generate_hash(f"many_notes_{i}"),
source_type_id=note_source_type.id,
)
db_session.add(note)
db_session.commit()
count = db_session.query(Document).count()
assert count == 100
def test_create_many_versions(self, db_session, note_source_type):
"""Test creating many versions for a single note."""
note = Document(
id=str(uuid.uuid4()),
title="Versioned Note",
text_content="Initial content",
file_type="note",
file_size=0,
document_hash=_generate_hash("versioned_note"),
source_type_id=note_source_type.id,
)
db_session.add(note)
db_session.commit()
for i in range(100):
version = NoteVersion(
id=str(uuid.uuid4()),
document_id=note.id,
title=f"Title v{i + 1}",
content=f"Content v{i + 1}",
change_type="manual_save",
content_hash=f"hash{i + 1}",
)
db_session.add(version)
db_session.commit()
count = (
db_session.query(NoteVersion).filter_by(document_id=note.id).count()
)
assert count == 100
def test_create_many_links(self, db_session, note_source_type):
"""Test creating many links between notes."""
# Create 10 notes
notes = []
for i in range(10):
note = Document(
id=str(uuid.uuid4()),
title=f"Note {i}",
text_content=f"Content {i}",
file_type="note",
file_size=0,
document_hash=_generate_hash(f"link_note_{i}"),
source_type_id=note_source_type.id,
)
db_session.add(note)
notes.append(note)
db_session.commit()
# Create links from first note to all others
source = notes[0]
for target in notes[1:]:
link = NoteLink(
source_document_id=source.id,
target_document_id=target.id,
link_text=target.title,
)
db_session.add(link)
db_session.commit()
count = (
db_session.query(NoteLink)
.filter_by(source_document_id=source.id)
.count()
)
assert count == 9
def test_query_performance_with_many_notes(
self, db_session, note_source_type
):
"""Test query performance with many notes."""
# Create 500 notes
for i in range(500):
note = Document(
id=str(uuid.uuid4()),
title=f"Note {i}",
text_content=f"Content for note {i} with some text",
tags=["tag1", "tag2"] if i % 2 == 0 else ["tag3"],
file_type="note",
file_size=0,
document_hash=_generate_hash(f"perf_note_{i}"),
source_type_id=note_source_type.id,
)
db_session.add(note)
db_session.commit()
# Test various queries
# Count
count = db_session.query(Document).count()
assert count == 500
# Filter by title
filtered = (
db_session.query(Document)
.filter(Document.title.like("Note 1%"))
.all()
)
assert len(filtered) > 0
# Limit and offset
page = db_session.query(Document).limit(10).offset(50).all()
assert len(page) == 10
class TestLLMInteractionStress:
"""Stress tests for LLM interactions."""
def test_summarize_with_very_long_content(self):
"""Test summarizing very long content."""
from local_deep_research.research_library.notes.services.note_ai_service import (
NoteAIService,
)
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.content = "Summary of changes"
mock_llm.ainvoke = AsyncMock(return_value=mock_response)
service = NoteAIService("test_user")
service._llm = mock_llm
# Very long content
old_content = "Old " * 5000
new_content = "New " * 5000
result = asyncio.run(
service.summarize_changes(old_content, new_content)
)
# Should handle without error
assert result == "Summary of changes"
mock_llm.ainvoke.assert_called_once()
def test_semantic_diff_with_json_edge_cases(self):
"""Test semantic diff with various JSON responses."""
from local_deep_research.research_library.notes.services.note_ai_service import (
NoteAIService,
)
test_cases = [
# Valid JSON
(
'{"added": ["x"], "removed": [], "modified": [], "summary": "test"}',
True,
),
# Empty JSON
("{}", True),
# Invalid JSON
("not json at all", True), # Should handle gracefully
# Partial JSON
('{"added": ["x"]', True),
# Extra fields
(
'{"added": [], "removed": [], "modified": [], "summary": "", "extra": 1}',
True,
),
]
for response_content, should_succeed in test_cases:
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.content = response_content
mock_llm.ainvoke = AsyncMock(return_value=mock_response)
service = NoteAIService("test_user")
service._llm = mock_llm
result = asyncio.run(service.semantic_diff("old", "new"))
# Should always return a valid structure
assert "added" in result
assert "removed" in result
assert "modified" in result
class TestNoteContentSizeCap:
"""Service-layer size-cap enforcement for note content."""
def test_assert_content_size_rejects_over_limit(self):
import pytest
from local_deep_research.research_library.notes.services.note_service import (
NOTE_CONTENT_MAX_BYTES,
_assert_content_size,
)
_assert_content_size("x" * (NOTE_CONTENT_MAX_BYTES)) # at limit: OK
with pytest.raises(ValueError, match="maximum size"):
_assert_content_size("x" * (NOTE_CONTENT_MAX_BYTES + 1))
def test_many_versions_of_large_note(self, db_session, sample_note):
"""Saving 10 versions of a ~5 MB note should not blow up.
Verifies version snapshots with realistic large content can be
stored and re-read. Guards against hidden regressions in the
version table under non-trivial content sizes.
"""
from local_deep_research.database.models import NoteVersion
content = "x" * (5 * 1024 * 1024) # 5 MB, well below the 50 MB cap
for i in range(10):
version = NoteVersion(
id=str(uuid.uuid4()),
document_id=sample_note.id,
title=sample_note.title,
content=content + str(i), # unique content per version
tags=["stress"],
change_type="auto_save",
content_hash=f"stress_hash_{i}",
)
db_session.add(version)
db_session.commit()
versions = (
db_session.query(NoteVersion)
.filter_by(document_id=sample_note.id)
.all()
)
assert len(versions) == 10
for v in versions:
assert len(v.content) >= 5 * 1024 * 1024
class TestNoteVersionCap:
"""The snapshot + prune sequence (as run inside ``update_note``)
caps rows at MAX_VERSIONS_PER_NOTE.
We don't assert which *specific* rows got pruned — SQLite's
CURRENT_TIMESTAMP groups rapid inserts into the same second, so within
a second the tie-breaker (NoteVersion.id, a random UUID) picks an
arbitrary-but-consistent subset. What matters for the cap is "at most
MAX_VERSIONS_PER_NOTE rows survive."
"""
@pytest.fixture
def patched_session(self, db_session, monkeypatch):
from contextlib import contextmanager
@contextmanager
def fake_session(username=None, password=None):
yield db_session
monkeypatch.setattr(
"local_deep_research.research_library.notes.services.note_service.get_user_db_session",
fake_session,
)
return db_session
def test_caps_at_max_versions_per_note(
self, patched_session, sample_note, monkeypatch
):
from local_deep_research.database.models import NoteVersion
from local_deep_research.research_library.notes.services import (
note_service as note_service_mod,
)
from local_deep_research.research_library.notes.services.note_service import (
NoteService,
)
# Use a smaller cap for test speed; the module default is 100.
monkeypatch.setattr(note_service_mod, "MAX_VERSIONS_PER_NOTE", 10)
service = NoteService("test_user")
for i in range(15):
service._create_version_snapshot_in_session(
patched_session,
sample_note.id,
sample_note.title,
f"v{i}-body", # unique → bypass content_hash dedup
["cap-test"],
"auto_save",
None,
)
service._prune_versions_in_session(patched_session, sample_note.id)
patched_session.commit()
rows = (
patched_session.query(NoteVersion)
.filter_by(document_id=sample_note.id)
.all()
)
assert len(rows) == 10
def test_prune_keeps_the_newest_versions_by_created_at(
self, patched_session, sample_note, monkeypatch
):
"""the cap keeps exactly the NEWEST MAX_VERSIONS_PER_NOTE
non-bookend rows and drops the OLDEST — the cap's whole value
proposition. test_caps_at_max_versions_per_note only asserts the count
(rapid inserts share a CURRENT_TIMESTAMP second, so survivor identity
is arbitrary there); this inserts rows with DISTINCT created_at and
pins which survive."""
from datetime import datetime, timedelta, timezone
from local_deep_research.database.models import (
NoteChangeType,
NoteVersion,
)
from local_deep_research.research_library.notes.services import (
note_service as note_service_mod,
)
from local_deep_research.research_library.notes.services.note_service import (
NoteService,
)
monkeypatch.setattr(note_service_mod, "MAX_VERSIONS_PER_NOTE", 5)
base = datetime(2024, 1, 1, tzinfo=timezone.utc)
for i in range(8): # three over the cap
patched_session.add(
NoteVersion(
id=str(uuid.uuid4()),
document_id=sample_note.id,
title=f"t{i}",
content=f"c{i}",
tags=[],
change_type=NoteChangeType.MANUAL_SAVE.value,
content_hash=f"h{i}",
created_at=base + timedelta(minutes=i),
)
)
patched_session.commit()
NoteService("test_user")._prune_versions_in_session(
patched_session, sample_note.id
)
patched_session.commit()
survivors = (
patched_session.query(NoteVersion)
.filter_by(document_id=sample_note.id)
.order_by(NoteVersion.created_at.asc())
.all()
)
# Newest 5 survive; the oldest 3 (t0, t1, t2) are pruned.
assert [v.title for v in survivors] == ["t3", "t4", "t5", "t6", "t7"]
def test_initial_version_is_prunable_not_a_bookend(
self, patched_session, sample_note, monkeypatch
):
"""an INITIAL snapshot is NOT in _AUDIT_BOOKEND_TYPES, so it is
FIFO-prunable like any other version — once newer edits exceed the cap
the oldest INITIAL is dropped, it is not protected the way a
PRE_RESTORE/RESTORE bookend is."""
from datetime import datetime, timedelta, timezone
from local_deep_research.database.models import (
NoteChangeType,
NoteVersion,
)
from local_deep_research.research_library.notes.services import (
note_service as note_service_mod,
)
from local_deep_research.research_library.notes.services.note_service import (
NoteService,
)
monkeypatch.setattr(note_service_mod, "MAX_VERSIONS_PER_NOTE", 3)
base = datetime(2024, 1, 1, tzinfo=timezone.utc)
# Oldest row is the INITIAL snapshot; four newer manual saves follow.
rows = [("INITIAL", NoteChangeType.INITIAL.value)] + [
(f"edit{i}", NoteChangeType.MANUAL_SAVE.value) for i in range(4)
]
for i, (title, change_type) in enumerate(rows):
patched_session.add(
NoteVersion(
id=str(uuid.uuid4()),
document_id=sample_note.id,
title=title,
content=f"c{i}",
tags=[],
change_type=change_type,
content_hash=f"h{i}",
created_at=base + timedelta(minutes=i),
)
)
patched_session.commit()
NoteService("test_user")._prune_versions_in_session(
patched_session, sample_note.id
)
patched_session.commit()
survivors = (
patched_session.query(NoteVersion)
.filter_by(document_id=sample_note.id)
.all()
)
titles = {v.title for v in survivors}
# The INITIAL row (oldest) was pruned, not protected.
assert "INITIAL" not in titles
assert titles == {"edit1", "edit2", "edit3"}
def test_prune_emits_info_log_with_counts(
self, patched_session, sample_note, monkeypatch
):
"""Pruning returns the ordinary and bookend counts actually pruned,
and ``_log_version_prune`` turns those counts into a single
``logger.info`` call, so silent version loss becomes observable.
``_prune_versions_in_session`` itself must NOT log: its deletes are
only flushed, not committed, so logging there would claim a prune
that a caller's later rollback could undo. The log only fires once
the caller's transaction has actually committed (see
``update_note`` / ``restore_with_bookends``, which call
``_log_version_prune`` after ``session.commit()`` succeeds).
"""
from datetime import datetime, timedelta, timezone
from local_deep_research.database.models import (
NoteChangeType,
NoteVersion,
)
from local_deep_research.research_library.notes.services import (
note_service as note_service_mod,
)
from local_deep_research.research_library.notes.services.note_service import (
NoteService,
)
# Small caps so a handful of rows exceeds both pools.
monkeypatch.setattr(note_service_mod, "MAX_VERSIONS_PER_NOTE", 3)
monkeypatch.setattr(note_service_mod, "MAX_BOOKEND_VERSIONS", 2)
base = datetime(2024, 1, 1, tzinfo=timezone.utc)
for i in range(5): # 2 over the ordinary cap
patched_session.add(
NoteVersion(
id=str(uuid.uuid4()),
document_id=sample_note.id,
title=f"o{i}",
content=f"oc{i}",
tags=[],
change_type=NoteChangeType.MANUAL_SAVE.value,
content_hash=f"oh{i}",
created_at=base + timedelta(minutes=i),
)
)
for i in range(4): # 2 over the bookend cap
patched_session.add(
NoteVersion(
id=str(uuid.uuid4()),
document_id=sample_note.id,
title=f"b{i}",
content=f"bc{i}",
tags=[],
change_type=NoteChangeType.PRE_RESTORE.value,
content_hash=f"bh{i}",
created_at=base + timedelta(minutes=i),
)
)
patched_session.commit()
mock_logger = MagicMock()
monkeypatch.setattr(note_service_mod, "logger", mock_logger)
service = NoteService("test_user")
prune_counts = service._prune_versions_in_session(
patched_session, sample_note.id
)
# No log yet — the caller hasn't committed. This is the fix for
# the false-positive-on-rollback defect: the prune function only
# hands back counts, it never logs them itself.
assert prune_counts == (2, 2)
mock_logger.info.assert_not_called()
service._log_version_prune(sample_note.id, prune_counts)
mock_logger.info.assert_called_once()
template, *rest = mock_logger.info.call_args.args
assert "note version prune" in template
assert rest == [sample_note.id, 2, 2]
def test_prune_returns_actually_deleted_count_not_stale_excess(
self, patched_session, sample_note, monkeypatch
):
"""Regression: ``_prune_versions_in_session`` must return the number
of rows its DELETE actually removed -- the ids the database hands
back through ``DELETE ... RETURNING`` -- never the ``excess``
computed from a ``count()`` taken before the id SELECT.
``count()`` and the subsequent ``SELECT ... LIMIT excess`` are
separate statements, so a version delete for the same note that
lands between them leaves the pre-delete ``count()`` stale relative
to what the SELECT finds and the DELETE removes. Simulated here by
inflating ONLY the first ``Query.count()`` call (the
ordinary-versions count) so it over-reports by 5, while leaving the
real SELECT/DELETE untouched -- the shape of a stale-snapshot race,
without needing real concurrent transactions.
"""
from datetime import datetime, timedelta, timezone
from sqlalchemy.orm import Query
from local_deep_research.database.models import (
NoteChangeType,
NoteVersion,
)
from local_deep_research.research_library.notes.services import (
note_service as note_service_mod,
)
from local_deep_research.research_library.notes.services.note_service import (
NoteService,
)
monkeypatch.setattr(note_service_mod, "MAX_VERSIONS_PER_NOTE", 2)
base = datetime(2024, 1, 1, tzinfo=timezone.utc)
# Only 3 real ordinary versions exist for this note.
for i in range(3):
patched_session.add(
NoteVersion(
id=str(uuid.uuid4()),
document_id=sample_note.id,
title=f"t{i}",
content=f"c{i}",
tags=[],
change_type=NoteChangeType.MANUAL_SAVE.value,
content_hash=f"h{i}",
created_at=base + timedelta(minutes=i),
)
)
patched_session.commit()
real_count = Query.count
call_count = {"n": 0}
def stale_count(self):
call_count["n"] += 1
result = real_count(self)
if call_count["n"] == 1:
# Simulate a stale snapshot: claims 5 more rows exist than
# actually do, as if this note had 5 additional versions a
# moment ago that a concurrent transaction has since
# deleted and committed.
return result + 5
return result
monkeypatch.setattr(Query, "count", stale_count)
service = NoteService("test_user")
prune_counts = service._prune_versions_in_session(
patched_session, sample_note.id
)
patched_session.commit()
# The stale count() claims excess = (3 + 5) - 2 = 6, but only 3
# rows exist to match LIMIT(6) against, so the SELECT (and the
# deletes) can only ever touch 3 rows -- never 6.
remaining = (
patched_session.query(NoteVersion)
.filter_by(document_id=sample_note.id)
.count()
)
assert remaining == 0
ordinary_pruned, _bookend_pruned = prune_counts
assert ordinary_pruned == 3, (
"must report the number of rows ACTUALLY deleted (3), not the "
f"stale count()-derived excess (6); got {ordinary_pruned}"
)
def test_prune_reports_delete_matched_rows_not_fetched_rows(
self, patched_session, sample_note, monkeypatch
):
"""Regression: the reported count must be the rows THIS call's
DELETE removed, not the rows its id SELECT fetched — 0 here, not 3.
The prune fetches the ids of the oldest ordinary versions, then
deletes exactly those ids. Here every fetched row is removed in
between, so the DELETE matches nothing and the honest count is 0,
while the ORM's old per-instance ``session.delete()`` path only
WARNS on that rowcount mismatch (non-versioned mapper) and still
reported the 3 rows it had fetched.
The removal is injected between the id SELECT and the ``delete()``
statement by wrapping the ``delete`` callable the service module
uses: the wrapper deletes the rows through a second Session on the
same engine and commits, then hands the real statement construction
through untouched, so the returned counts still come from the real
DELETE's RETURNING. The in-memory fixture engine uses SQLAlchemy's
``SingletonThreadPool``, so that second Session shares this one's
connection and transaction: this exercises the SELECT-then-DELETE
ordering, not isolation between genuinely concurrent transactions.
"""
from datetime import datetime, timedelta, timezone
from sqlalchemy import delete as sa_delete
from sqlalchemy.orm import sessionmaker
from local_deep_research.database.models import (
NoteChangeType,
NoteVersion,
)
from local_deep_research.research_library.notes.services import (
note_service as note_service_mod,
)
from local_deep_research.research_library.notes.services.note_service import (
NoteService,
)
# Cap 0 => excess == total => the prune's SELECT fetches every
# ordinary version of this note (3), so the concurrent delete
# below removes exactly the fetched set.
monkeypatch.setattr(note_service_mod, "MAX_VERSIONS_PER_NOTE", 0)
base = datetime(2024, 1, 1, tzinfo=timezone.utc)
for i in range(3):
patched_session.add(
NoteVersion(
id=str(uuid.uuid4()),
document_id=sample_note.id,
title=f"t{i}",
content=f"c{i}",
tags=[],
change_type=NoteChangeType.MANUAL_SAVE.value,
content_hash=f"h{i}",
created_at=base + timedelta(minutes=i),
)
)
patched_session.commit()
calls = {"n": 0, "rows_at_delete": None}
def concurrent_delete_first(*args, **kwargs):
calls["n"] += 1
calls["rows_at_delete"] = (
patched_session.query(NoteVersion)
.filter_by(document_id=sample_note.id)
.count()
)
# A competing pruner removes the fetched rows and COMMITS
# between this call's id SELECT and its DELETE.
ConcurrentSession = sessionmaker(bind=patched_session.get_bind())
concurrent = ConcurrentSession()
concurrent.query(NoteVersion).filter_by(
document_id=sample_note.id
).delete(synchronize_session=False)
concurrent.commit()
concurrent.close()
return sa_delete(*args, **kwargs)
# Pins the bare-name ``delete`` imported at note_service module scope.
monkeypatch.setattr(note_service_mod, "delete", concurrent_delete_first)
service = NoteService("test_user")
ordinary_pruned, bookend_pruned = service._prune_versions_in_session(
patched_session, sample_note.id
)
patched_session.commit()
# The prune's id SELECT ran while all 3 rows were still present,
# and the wrapper's committed delete is what removed them —
# verified against the real DB state, not a mock.
assert calls["n"] == 1
assert calls["rows_at_delete"] == 3
remaining = (
patched_session.query(NoteVersion)
.filter_by(document_id=sample_note.id)
.count()
)
assert remaining == 0
assert (ordinary_pruned, bookend_pruned) == (0, 0), (
"must report the rows THIS session's DELETE matched (0, 0), not "
"the rows its SELECT fetched (3); got "
f"{(ordinary_pruned, bookend_pruned)}"
)
def test_prune_reports_delete_matched_bookend_rows_not_fetched_rows(
self, patched_session, sample_note, monkeypatch
):
"""Same guard as the test above, for the bookend half of the prune.
The bookend block is a structural duplicate of the ordinary one and
is just as able to regress to ``len(oldest_bookend_ids)``. The test
above cannot catch that: its note has no bookends, so
``bookend_excess`` is negative, the block never runs, and its
``bookend_pruned == 0`` assertion holds vacuously.
Here the note has ONLY bookends, so the ordinary block is skipped
(``ordinary_total`` is 0, well under the cap) and the wrapped
``delete`` fires exactly once — for the bookend statement. Its rows
are removed between the id SELECT and the DELETE, so the honest
bookend count is 0, not the 3 ids the SELECT fetched.
"""
from datetime import datetime, timedelta, timezone
from sqlalchemy import delete as sa_delete
from sqlalchemy.orm import sessionmaker
from local_deep_research.database.models import (
NoteChangeType,
NoteVersion,
)
from local_deep_research.research_library.notes.services import (
note_service as note_service_mod,
)
from local_deep_research.research_library.notes.services.note_service import (
NoteService,
)
# Ordinary cap comfortably above the note's 0 ordinary versions, so
# that block is skipped; bookend cap 0 => bookend_excess == the whole
# bookend pool, so the bookend SELECT fetches all 3 of them.
monkeypatch.setattr(note_service_mod, "MAX_VERSIONS_PER_NOTE", 5)
monkeypatch.setattr(note_service_mod, "MAX_BOOKEND_VERSIONS", 0)
base = datetime(2024, 1, 1, tzinfo=timezone.utc)
for i in range(3):
patched_session.add(
NoteVersion(
id=str(uuid.uuid4()),
document_id=sample_note.id,
title=f"bookend{i}",
content=f"bookend-content{i}",
tags=[],
change_type=(
NoteChangeType.RESTORE.value
if i % 2 == 0
else NoteChangeType.PRE_RESTORE.value
),
content_hash=f"bookend-hash{i}",
created_at=base + timedelta(minutes=i),
)
)
patched_session.commit()
calls = {"n": 0, "rows_at_delete": None}
def concurrent_delete_first(*args, **kwargs):
calls["n"] += 1
calls["rows_at_delete"] = (
patched_session.query(NoteVersion)
.filter_by(document_id=sample_note.id)
.count()
)
# A competing pruner removes the fetched bookends and COMMITS
# between this call's id SELECT and its DELETE.
ConcurrentSession = sessionmaker(bind=patched_session.get_bind())
concurrent = ConcurrentSession()
concurrent.query(NoteVersion).filter_by(
document_id=sample_note.id
).delete(synchronize_session=False)
concurrent.commit()
concurrent.close()
return sa_delete(*args, **kwargs)
# Pins the bare-name ``delete`` imported at note_service module scope.
monkeypatch.setattr(note_service_mod, "delete", concurrent_delete_first)
service = NoteService("test_user")
ordinary_pruned, bookend_pruned = service._prune_versions_in_session(
patched_session, sample_note.id
)
patched_session.commit()
# Exactly one DELETE was built — the bookend one — and its id SELECT
# ran while all 3 bookends were still present.
assert calls["n"] == 1
assert calls["rows_at_delete"] == 3
remaining = (
patched_session.query(NoteVersion)
.filter_by(document_id=sample_note.id)
.count()
)
assert remaining == 0
assert bookend_pruned == 0, (
"the bookend count must be the rows THIS call's DELETE matched "
f"(0), not the ids its SELECT fetched (3); got {bookend_pruned}"
)
assert ordinary_pruned == 0
def test_prune_counts_ordinary_versions_separately_from_bookends(
self, patched_session, sample_note, monkeypatch
):
"""Bookends must not consume the ordinary-version retention budget.
Regression guard for #5438: pre-fix, ``_prune_versions_in_session``
counted ALL rows — bookends included — against
MAX_VERSIONS_PER_NOTE, so PRE_RESTORE/RESTORE audit rows could push
a note's ordinary snapshot count over the cap and get ordinary rows
evicted to compensate. The fix counts the ordinary and bookend
pools against their own independent caps (MAX_VERSIONS_PER_NOTE and
MAX_BOOKEND_VERSIONS respectively), so 3 bookends + 5 ordinary
versions — each under its own cap — must ALL survive even though
their combined total (8) exceeds MAX_VERSIONS_PER_NOTE (5).
"""
from datetime import datetime, timedelta, timezone
from local_deep_research.database.models import (
NoteChangeType,
NoteVersion,
)
from local_deep_research.research_library.notes.services import (
note_service as note_service_mod,
)
from local_deep_research.research_library.notes.services.note_service import (
NoteService,
)
monkeypatch.setattr(note_service_mod, "MAX_VERSIONS_PER_NOTE", 5)
monkeypatch.setattr(note_service_mod, "MAX_BOOKEND_VERSIONS", 3)
base = datetime(2024, 1, 1, tzinfo=timezone.utc)
bookend_ids = []
for i in range(3):
change_type = (
NoteChangeType.RESTORE.value
if i % 2 == 0
else NoteChangeType.PRE_RESTORE.value
)
v = NoteVersion(
id=str(uuid.uuid4()),
document_id=sample_note.id,
title=f"bookend{i}",
content=f"bookend-content{i}",
tags=[],
change_type=change_type,
content_hash=f"bookend-hash{i}",
created_at=base + timedelta(minutes=i),
)
patched_session.add(v)
bookend_ids.append(v.id)
ordinary_ids = []
for j in range(5):
v = NoteVersion(
id=str(uuid.uuid4()),
document_id=sample_note.id,
title=f"ordinary{j}",
content=f"ordinary-content{j}",
tags=[],
change_type=NoteChangeType.AUTO_SAVE.value,
content_hash=f"ordinary-hash{j}",
created_at=base + timedelta(minutes=10 + j),
)
patched_session.add(v)
ordinary_ids.append(v.id)
patched_session.commit()
NoteService("test_user")._prune_versions_in_session(
patched_session, sample_note.id
)
patched_session.commit()
surviving = {
v.id
for v in patched_session.query(NoteVersion)
.filter_by(document_id=sample_note.id)
.all()
}
assert set(ordinary_ids).issubset(surviving), (
"ordinary versions must not be sacrificed to make room for "
"bookends — they have independent caps."
)
assert set(bookend_ids).issubset(surviving), (
"bookends must never be pruned below their own cap."
)
assert len(surviving) == 8, (
"3 bookends + 5 ordinary versions, each under its own "
"independent cap, must all survive even though the combined "
"total (8) exceeds MAX_VERSIONS_PER_NOTE (5)."
)
class TestLinkResearchConcurrency:
"""Regression guard for the IntegrityError-retry path on
``UniqueConstraint('document_id', 'display_order')`` (migration 0021).
Pre-fix, two concurrent calls to ``link_research_to_note(note_id, X)``
and ``link_research_to_note(note_id, Y)`` could both read the same
``MAX(display_order)+1`` value before either committed, then both
try to INSERT with the same display_order. One would succeed, the
other would surface a 500 to the user. The fix is a bounded retry
loop in ``note_service.link_research_to_note`` that recomputes
``MAX(display_order)+1`` on ``IntegrityError`` and retries.
Without a concurrent test the retry path is completely unverified
even though the unique constraint and retry logic are both shipped.
"""
def test_eight_threads_race_link_research(self, tmp_path):
import threading
from datetime import datetime, timezone
from sqlalchemy import create_engine, event
from sqlalchemy.orm import scoped_session, sessionmaker
from local_deep_research.database.models import (
Base,
Document,
NoteResearch,
ResearchHistory,
SourceType,
)
from local_deep_research.research_library.notes.services import (
note_service as note_service_mod,
)
from local_deep_research.research_library.notes.services.note_service import (
NoteService,
)
db_path = tmp_path / "race.sqlite"
engine = create_engine(
f"sqlite:///{db_path}",
connect_args={"check_same_thread": False, "timeout": 30},
)
@event.listens_for(engine, "connect")
def _pragmas(dbapi_conn, _record):
cur = dbapi_conn.cursor()
cur.execute("PRAGMA foreign_keys=ON")
cur.execute("PRAGMA journal_mode=WAL")
cur.execute("PRAGMA busy_timeout=30000")
cur.close()
Base.metadata.create_all(engine)
Session = scoped_session(sessionmaker(bind=engine))
setup_session = Session()
note_source_type = SourceType(
id=str(uuid.uuid4()),
name="note",
display_name="Note",
description="User-created notes",
icon="sticky-note",
)
setup_session.add(note_source_type)
setup_session.commit()
note_id = str(uuid.uuid4())
setup_session.add(
Document(
id=note_id,
title="Race Test",
text_content="body",
file_type="note",
file_size=0,
document_hash=_generate_hash("race_note"),
source_type_id=note_source_type.id,
)
)
now = datetime.now(timezone.utc)
N = 8
research_ids = [str(uuid.uuid4()) for _ in range(N)]
for i, rid in enumerate(research_ids):
setup_session.add(
ResearchHistory(
id=rid,
query=f"q{i}",
mode="quick",
status="completed",
created_at=now.isoformat(),
)
)
setup_session.commit()
setup_session.close()
# Monkeypatch get_user_db_session via direct module attribute
# assignment so each worker thread gets its own session bound
# to the shared on-disk SQLite. We can't use a pytest
# monkeypatch fixture here without dragging it into the
# threads.
from contextlib import contextmanager
@contextmanager
def fake_session(username=None, password=None):
with Session() as sess:
yield sess
original = note_service_mod.get_user_db_session
note_service_mod.get_user_db_session = fake_session
try:
barrier = threading.Barrier(N)
errors = []
def worker(rid):
try:
# Synchronize the read of MAX(display_order)+1
# across all workers to maximise the chance of
# hitting the constraint and exercising the retry.
barrier.wait(timeout=15)
NoteService("test_user").link_research_to_note(note_id, rid)
except Exception as exc: # noqa: BLE001
errors.append(exc)
threads = [
threading.Thread(target=worker, args=(rid,))
for rid in research_ids
]
for t in threads:
t.start()
for t in threads:
t.join(timeout=30)
finally:
note_service_mod.get_user_db_session = original
assert not errors, f"unexpected errors: {errors}"
try:
with Session() as verify_session:
rows = (
verify_session.query(NoteResearch)
.filter_by(document_id=note_id)
.all()
)
display_orders = sorted(r.display_order for r in rows)
# All N research links should have landed with distinct
# display_orders. Order doesn't matter (the threads
# race); what matters is no duplicates and no missing
# rows. The retry loop in link_research_to_note is
# responsible for turning any concurrent collisions
# into a clean sequence.
assert len(rows) == N
assert display_orders == sorted(set(display_orders))
finally:
Session.remove()
engine.dispose()
class TestConcurrentIdenticalUpdateNoUniqueCollision:
"""Regression guard for the version-dedup path under concurrency.
``update_note`` writes a MANUAL_SAVE NoteVersion guarded by
UNIQUE(document_id, content_hash) (uix_note_version_content). The dedup
check in ``_create_version_snapshot_in_session`` (note_service.py:1698-1704)
queries that pair and returns the EXISTING version id instead of
inserting on a hit. Without it, N concurrent update_note calls carrying
byte-identical content all pass their pre-insert read, then race to
INSERT the same (document_id, content_hash) — one wins, the rest
surface an IntegrityError 500 to the user.
Mirrors TestLinkResearchConcurrency: real on-disk SQLite (WAL +
busy_timeout serialises writers so later threads see the committed
row), a thread barrier to maximise the collision window, and a
fresh session per worker. Reverting the dedup early-return makes the
losing threads raise IntegrityError and flips ``assert not errors``.
"""
def test_concurrent_update_note_identical_content_no_unique_constraint_500(
self, tmp_path
):
import threading
from contextlib import contextmanager
from sqlalchemy import create_engine, event
from sqlalchemy.orm import scoped_session, sessionmaker
from local_deep_research.database.models import (
Base,
Collection,
Document,
DocumentCollection,
NoteVersion,
SourceType,
)
from local_deep_research.research_library.notes.services import (
note_service as note_service_mod,
)
from local_deep_research.research_library.notes.services.note_service import (
NoteService,
)
db_path = tmp_path / "identical_update.sqlite"
engine = create_engine(
f"sqlite:///{db_path}",
connect_args={"check_same_thread": False, "timeout": 30},
)
@event.listens_for(engine, "connect")
def _pragmas(dbapi_conn, _record):
cur = dbapi_conn.cursor()
cur.execute("PRAGMA foreign_keys=ON")
cur.execute("PRAGMA journal_mode=WAL")
cur.execute("PRAGMA busy_timeout=30000")
cur.close()
Base.metadata.create_all(engine)
Session = scoped_session(sessionmaker(bind=engine))
setup_session = Session()
note_source_type = SourceType(
id=str(uuid.uuid4()),
name="note",
display_name="Note",
description="User-created notes",
icon="sticky-note",
)
setup_session.add(note_source_type)
collection_id = str(uuid.uuid4())
setup_session.add(
Collection(
id=collection_id,
name="Notes",
description="default",
collection_type="notes",
)
)
note_id = str(uuid.uuid4())
setup_session.add(
Document(
id=note_id,
title="Race Note",
text_content="original body",
file_type="note",
file_size=13,
document_hash=_generate_hash(f"{note_id}:original body"),
source_type_id=note_source_type.id,
tags=[],
)
)
setup_session.add(
DocumentCollection(
document_id=note_id,
collection_id=collection_id,
indexed=True,
)
)
setup_session.commit()
setup_session.close()
@contextmanager
def fake_session(username=None, password=None):
with Session() as sess:
yield sess
# The post-commit change-summary worker can't open the encrypted DB
# in this test harness; force the capture to None so update_note
# skips the submit (mirrors its own None-capture guard) and we
# isolate the version-insert race itself.
original_session = note_service_mod.get_user_db_session
original_capture = note_service_mod._capture_request_db_password
note_service_mod.get_user_db_session = fake_session
note_service_mod._capture_request_db_password = lambda username: None
try:
N = 8
# All threads write the SAME new content (different from the
# seeded "original body") so they all compute one shared
# content_hash and contend on the single (document_id,
# content_hash) slot.
new_content = "identical edited content for every thread"
barrier = threading.Barrier(N)
errors = []
def worker():
try:
barrier.wait(timeout=15)
NoteService("test_user").update_note(
note_id, content=new_content
)
except Exception as exc: # noqa: BLE001
errors.append(exc)
threads = [threading.Thread(target=worker) for _ in range(N)]
for t in threads:
t.start()
for t in threads:
t.join(timeout=30)
finally:
note_service_mod.get_user_db_session = original_session
note_service_mod._capture_request_db_password = original_capture
# No UNIQUE(document_id, content_hash) IntegrityError reached the
# caller — the dedup path absorbed every redundant snapshot.
assert not errors, f"unexpected errors from concurrent update: {errors}"
try:
with Session() as verify_session:
rows = (
verify_session.query(NoteVersion)
.filter_by(document_id=note_id)
.all()
)
# Exactly one MANUAL_SAVE snapshot for the shared content —
# the first writer inserts, the rest dedup-absorb.
assert len(rows) == 1, (
"concurrent identical updates must collapse to a single "
f"version row via the dedup path; got {len(rows)}"
)
assert rows[0].content == new_content
finally:
Session.remove()
engine.dispose()