~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(notes): return actual deleted-row counts from _prune_versions_in_session (#6192)
LearningCircuit48688d7e808d
diff --git a/changelog.d/+notes-prune-count-accuracy.bugfix.md b/changelog.d/+notes-prune-count-accuracy.bugfix.mdnew file mode 100644index 0000000000..337d3870a7--- /dev/null+++ b/changelog.d/+notes-prune-count-accuracy.bugfix.md@@ -0,0 +1 @@+Fixed `_prune_versions_in_session` (#5498) reporting a pruned-version count taken from what its SELECT fetched (or, before that, a pre-delete `count()`) rather than what its DELETE actually removed — under concurrent prunes of the same note these diverged and over-reported (SQLAlchemy only warns when a non-versioned ORM delete matches fewer rows than fetched). It now deletes by the fetched ids and counts the rows the database itself reports deleting (`DELETE ... RETURNING`), so the logged prune counts reflect exactly the rows that call's own deletes removed — fewer, honestly, if a concurrent transaction had already removed some of them.diff --git a/src/local_deep_research/research_library/notes/services/note_service.py b/src/local_deep_research/research_library/notes/services/note_service.pyindex a974625f92..c5103fae73 100644--- a/src/local_deep_research/research_library/notes/services/note_service.py+++ b/src/local_deep_research/research_library/notes/services/note_service.py@@ -18,7 +18,7 @@ from concurrent.futures import ThreadPoolExecutorfrom typing import Any, Dict, List, Optionalfrom loguru import logger-from sqlalchemy import func, or_+from sqlalchemy import delete, func, or_from sqlalchemy.exc import IntegrityErrorfrom sqlalchemy.orm import Session, joinedload@@ -2539,17 +2539,34 @@ class NoteService:(which adds PRE_RESTORE + RESTORE rows without consuming the ordinaryversion budget).- Returns ``(ordinary_pruned, bookend_pruned)`` — the counts of rows- deleted in *this* call. The deletes are only flushed here, not- committed: if the caller's transaction later rolls back, these- rows survive. Callers must NOT log these counts themselves until- after their own ``session.commit()`` succeeds — logging here (at- flush time) would claim a prune that a subsequent rollback undoes,- turning the one operator-facing signal for version loss into a- false positive on exactly the failure paths it exists to surface.- See ``update_note`` and ``restore_with_bookends`` for the- post-commit logging call.+ Returns ``(ordinary_pruned, bookend_pruned)`` — the number of rows+ this call's own DELETE statements actually removed, counted from+ the rows the database itself reports deleting (``DELETE ...+ RETURNING``; same SQLite/SQLCipher baseline the chat service's+ ``UPDATE ... RETURNING`` counters already rely on). The prune+ selects the ids of the oldest rows over the cap, then deletes+ exactly those ids: when a concurrently-committed transaction has+ removed some of them in the SELECT→DELETE window, this call's+ DELETE matches fewer rows and that smaller, honest number is+ returned — never the fetched count, which the previous+ per-instance ``session.delete()`` implementation reported even+ when its DELETE matched nothing (SQLAlchemy only warns on that+ mismatch for non-versioned mappers). The counts therefore never+ over-report what this call removed; they say nothing about rows+ other transactions deleted, and can be lower than the excess the+ caps imply when concurrent pruners shared the work.++ The deletes run in the caller's transaction but are NOT committed+ here: if that transaction later rolls back, these rows survive.+ Callers must NOT log these counts themselves until after their own+ ``session.commit()`` succeeds — logging here (before the commit)+ would claim a prune that a subsequent rollback undoes, turning the+ one operator-facing signal for version loss into a false positive+ on exactly the failure paths it exists to surface. See+ ``update_note`` and ``restore_with_bookends`` for the post-commit+ logging call."""+ ordinary_pruned = 0ordinary_total = (session.query(NoteVersion).filter_by(document_id=note_id)@@ -2560,22 +2577,38 @@ class NoteService:if excess > 0:# Audit bookends have their own cap below and never reduce the# number of ordinary snapshots retained for the user.- oldest = (- session.query(NoteVersion)+ oldest_ids = [+ row_id+ for (row_id,) in session.query(NoteVersion.id).filter_by(document_id=note_id).filter(NoteVersion.change_type.notin_(_AUDIT_BOOKEND_TYPES)).order_by(NoteVersion.created_at.asc(), NoteVersion.id.asc()).limit(excess).all()- )- for row in oldest:- session.delete(row)- session.flush()+ ]+ if oldest_ids:+ # Delete by the fetched primary keys and count what the+ # DELETE actually matched — the rows the DB reports via+ # RETURNING — not what the SELECT fetched. Under a+ # concurrently-committed delete of the same rows, the ORM's+ # per-instance session.delete() path only WARNS on the+ # rowcount mismatch yet would report the fetched count.+ # synchronize_session=False leaves these rows persistent in the+ # session identity map until commit — do not touch NoteVersion+ # instances after this call.+ result = session.execute(+ delete(NoteVersion)+ .where(NoteVersion.id.in_(oldest_ids))+ .returning(NoteVersion.id)+ .execution_options(synchronize_session=False)+ )+ ordinary_pruned = len(result.all())# Separately bound the bookend pool. Excluding bookends from the prune# above keeps the restore trail, but each restore writes two# un-prunable rows, so without this ceiling repeated restores grow# note_versions without limit. Keep the most recent MAX_BOOKEND_VERSIONS.+ bookend_pruned = 0bookend_total = (session.query(NoteVersion).filter_by(document_id=note_id)@@ -2584,19 +2617,24 @@ class NoteService:)bookend_excess = bookend_total - MAX_BOOKEND_VERSIONSif bookend_excess > 0:- oldest_bookends = (- session.query(NoteVersion)+ oldest_bookend_ids = [+ row_id+ for (row_id,) in session.query(NoteVersion.id).filter_by(document_id=note_id).filter(NoteVersion.change_type.in_(_AUDIT_BOOKEND_TYPES)).order_by(NoteVersion.created_at.asc(), NoteVersion.id.asc()).limit(bookend_excess).all()- )- for row in oldest_bookends:- session.delete(row)- session.flush()- ordinary_pruned = max(excess, 0)- bookend_pruned = max(bookend_excess, 0)+ ]+ if oldest_bookend_ids:+ # Same delete-matched counting as the ordinary prune above.+ result = session.execute(+ delete(NoteVersion)+ .where(NoteVersion.id.in_(oldest_bookend_ids))+ .returning(NoteVersion.id)+ .execution_options(synchronize_session=False)+ )+ bookend_pruned = len(result.all())return (ordinary_pruned, bookend_pruned)def _log_version_prune(diff --git a/tests/notes/test_note_stress.py b/tests/notes/test_note_stress.pyindex 6ac296d250..da8c3a9f84 100644--- a/tests/notes/test_note_stress.py+++ b/tests/notes/test_note_stress.py@@ -765,6 +765,304 @@ class TestNoteVersionCap:assert "note version prune" in templateassert rest == [sample_note.id, 2, 2]+ def test_prune_returns_actually_deleted_count_not_stale_excess(+ self, patched_session, sample_note, monkeypatch+ ):+ """Regression: ``_prune_versions_in_session`` must return the number+ of rows its DELETE actually removed -- the ids the database hands+ back through ``DELETE ... RETURNING`` -- never the ``excess``+ computed from a ``count()`` taken before the id SELECT.++ ``count()`` and the subsequent ``SELECT ... LIMIT excess`` are+ separate statements, so a version delete for the same note that+ lands between them leaves the pre-delete ``count()`` stale relative+ to what the SELECT finds and the DELETE removes. Simulated here by+ inflating ONLY the first ``Query.count()`` call (the+ ordinary-versions count) so it over-reports by 5, while leaving the+ real SELECT/DELETE untouched -- the shape of a stale-snapshot race,+ without needing real concurrent transactions.+ """+ from datetime import datetime, timedelta, timezone++ from sqlalchemy.orm import Query++ from local_deep_research.database.models import (+ NoteChangeType,+ NoteVersion,+ )+ from local_deep_research.research_library.notes.services import (+ note_service as note_service_mod,+ )+ from local_deep_research.research_library.notes.services.note_service import (+ NoteService,+ )++ monkeypatch.setattr(note_service_mod, "MAX_VERSIONS_PER_NOTE", 2)++ base = datetime(2024, 1, 1, tzinfo=timezone.utc)+ # Only 3 real ordinary versions exist for this note.+ for i in range(3):+ patched_session.add(+ NoteVersion(+ id=str(uuid.uuid4()),+ document_id=sample_note.id,+ title=f"t{i}",+ content=f"c{i}",+ tags=[],+ change_type=NoteChangeType.MANUAL_SAVE.value,+ content_hash=f"h{i}",+ created_at=base + timedelta(minutes=i),+ )+ )+ patched_session.commit()++ real_count = Query.count+ call_count = {"n": 0}++ def stale_count(self):+ call_count["n"] += 1+ result = real_count(self)+ if call_count["n"] == 1:+ # Simulate a stale snapshot: claims 5 more rows exist than+ # actually do, as if this note had 5 additional versions a+ # moment ago that a concurrent transaction has since+ # deleted and committed.+ return result + 5+ return result++ monkeypatch.setattr(Query, "count", stale_count)++ service = NoteService("test_user")+ prune_counts = service._prune_versions_in_session(+ patched_session, sample_note.id+ )+ patched_session.commit()++ # The stale count() claims excess = (3 + 5) - 2 = 6, but only 3+ # rows exist to match LIMIT(6) against, so the SELECT (and the+ # deletes) can only ever touch 3 rows -- never 6.+ remaining = (+ patched_session.query(NoteVersion)+ .filter_by(document_id=sample_note.id)+ .count()+ )+ assert remaining == 0+ ordinary_pruned, _bookend_pruned = prune_counts+ assert ordinary_pruned == 3, (+ "must report the number of rows ACTUALLY deleted (3), not the "+ f"stale count()-derived excess (6); got {ordinary_pruned}"+ )++ def test_prune_reports_delete_matched_rows_not_fetched_rows(+ self, patched_session, sample_note, monkeypatch+ ):+ """Regression: the reported count must be the rows THIS call's+ DELETE removed, not the rows its id SELECT fetched — 0 here, not 3.++ The prune fetches the ids of the oldest ordinary versions, then+ deletes exactly those ids. Here every fetched row is removed in+ between, so the DELETE matches nothing and the honest count is 0,+ while the ORM's old per-instance ``session.delete()`` path only+ WARNS on that rowcount mismatch (non-versioned mapper) and still+ reported the 3 rows it had fetched.++ The removal is injected between the id SELECT and the ``delete()``+ statement by wrapping the ``delete`` callable the service module+ uses: the wrapper deletes the rows through a second Session on the+ same engine and commits, then hands the real statement construction+ through untouched, so the returned counts still come from the real+ DELETE's RETURNING. The in-memory fixture engine uses SQLAlchemy's+ ``SingletonThreadPool``, so that second Session shares this one's+ connection and transaction: this exercises the SELECT-then-DELETE+ ordering, not isolation between genuinely concurrent transactions.+ """+ from datetime import datetime, timedelta, timezone++ from sqlalchemy import delete as sa_delete+ from sqlalchemy.orm import sessionmaker++ from local_deep_research.database.models import (+ NoteChangeType,+ NoteVersion,+ )+ from local_deep_research.research_library.notes.services import (+ note_service as note_service_mod,+ )+ from local_deep_research.research_library.notes.services.note_service import (+ NoteService,+ )++ # Cap 0 => excess == total => the prune's SELECT fetches every+ # ordinary version of this note (3), so the concurrent delete+ # below removes exactly the fetched set.+ monkeypatch.setattr(note_service_mod, "MAX_VERSIONS_PER_NOTE", 0)++ base = datetime(2024, 1, 1, tzinfo=timezone.utc)+ for i in range(3):+ patched_session.add(+ NoteVersion(+ id=str(uuid.uuid4()),+ document_id=sample_note.id,+ title=f"t{i}",+ content=f"c{i}",+ tags=[],+ change_type=NoteChangeType.MANUAL_SAVE.value,+ content_hash=f"h{i}",+ created_at=base + timedelta(minutes=i),+ )+ )+ patched_session.commit()++ calls = {"n": 0, "rows_at_delete": None}++ def concurrent_delete_first(*args, **kwargs):+ calls["n"] += 1+ calls["rows_at_delete"] = (+ patched_session.query(NoteVersion)+ .filter_by(document_id=sample_note.id)+ .count()+ )+ # A competing pruner removes the fetched rows and COMMITS+ # between this call's id SELECT and its DELETE.+ ConcurrentSession = sessionmaker(bind=patched_session.get_bind())+ concurrent = ConcurrentSession()+ concurrent.query(NoteVersion).filter_by(+ document_id=sample_note.id+ ).delete(synchronize_session=False)+ concurrent.commit()+ concurrent.close()+ return sa_delete(*args, **kwargs)++ # Pins the bare-name ``delete`` imported at note_service module scope.+ monkeypatch.setattr(note_service_mod, "delete", concurrent_delete_first)++ service = NoteService("test_user")+ ordinary_pruned, bookend_pruned = service._prune_versions_in_session(+ patched_session, sample_note.id+ )+ patched_session.commit()++ # The prune's id SELECT ran while all 3 rows were still present,+ # and the wrapper's committed delete is what removed them —+ # verified against the real DB state, not a mock.+ assert calls["n"] == 1+ assert calls["rows_at_delete"] == 3+ remaining = (+ patched_session.query(NoteVersion)+ .filter_by(document_id=sample_note.id)+ .count()+ )+ assert remaining == 0+ assert (ordinary_pruned, bookend_pruned) == (0, 0), (+ "must report the rows THIS session's DELETE matched (0, 0), not "+ "the rows its SELECT fetched (3); got "+ f"{(ordinary_pruned, bookend_pruned)}"+ )++ def test_prune_reports_delete_matched_bookend_rows_not_fetched_rows(+ self, patched_session, sample_note, monkeypatch+ ):+ """Same guard as the test above, for the bookend half of the prune.++ The bookend block is a structural duplicate of the ordinary one and+ is just as able to regress to ``len(oldest_bookend_ids)``. The test+ above cannot catch that: its note has no bookends, so+ ``bookend_excess`` is negative, the block never runs, and its+ ``bookend_pruned == 0`` assertion holds vacuously.++ Here the note has ONLY bookends, so the ordinary block is skipped+ (``ordinary_total`` is 0, well under the cap) and the wrapped+ ``delete`` fires exactly once — for the bookend statement. Its rows+ are removed between the id SELECT and the DELETE, so the honest+ bookend count is 0, not the 3 ids the SELECT fetched.+ """+ from datetime import datetime, timedelta, timezone++ from sqlalchemy import delete as sa_delete+ from sqlalchemy.orm import sessionmaker++ from local_deep_research.database.models import (+ NoteChangeType,+ NoteVersion,+ )+ from local_deep_research.research_library.notes.services import (+ note_service as note_service_mod,+ )+ from local_deep_research.research_library.notes.services.note_service import (+ NoteService,+ )++ # Ordinary cap comfortably above the note's 0 ordinary versions, so+ # that block is skipped; bookend cap 0 => bookend_excess == the whole+ # bookend pool, so the bookend SELECT fetches all 3 of them.+ monkeypatch.setattr(note_service_mod, "MAX_VERSIONS_PER_NOTE", 5)+ monkeypatch.setattr(note_service_mod, "MAX_BOOKEND_VERSIONS", 0)++ base = datetime(2024, 1, 1, tzinfo=timezone.utc)+ for i in range(3):+ patched_session.add(+ NoteVersion(+ id=str(uuid.uuid4()),+ document_id=sample_note.id,+ title=f"bookend{i}",+ content=f"bookend-content{i}",+ tags=[],+ change_type=(+ NoteChangeType.RESTORE.value+ if i % 2 == 0+ else NoteChangeType.PRE_RESTORE.value+ ),+ content_hash=f"bookend-hash{i}",+ created_at=base + timedelta(minutes=i),+ )+ )+ patched_session.commit()++ calls = {"n": 0, "rows_at_delete": None}++ def concurrent_delete_first(*args, **kwargs):+ calls["n"] += 1+ calls["rows_at_delete"] = (+ patched_session.query(NoteVersion)+ .filter_by(document_id=sample_note.id)+ .count()+ )+ # A competing pruner removes the fetched bookends and COMMITS+ # between this call's id SELECT and its DELETE.+ ConcurrentSession = sessionmaker(bind=patched_session.get_bind())+ concurrent = ConcurrentSession()+ concurrent.query(NoteVersion).filter_by(+ document_id=sample_note.id+ ).delete(synchronize_session=False)+ concurrent.commit()+ concurrent.close()+ return sa_delete(*args, **kwargs)++ # Pins the bare-name ``delete`` imported at note_service module scope.+ monkeypatch.setattr(note_service_mod, "delete", concurrent_delete_first)++ service = NoteService("test_user")+ ordinary_pruned, bookend_pruned = service._prune_versions_in_session(+ patched_session, sample_note.id+ )+ patched_session.commit()++ # Exactly one DELETE was built — the bookend one — and its id SELECT+ # ran while all 3 bookends were still present.+ assert calls["n"] == 1+ assert calls["rows_at_delete"] == 3+ remaining = (+ patched_session.query(NoteVersion)+ .filter_by(document_id=sample_note.id)+ .count()+ )+ assert remaining == 0+ assert bookend_pruned == 0, (+ "the bookend count must be the rows THIS call's DELETE matched "+ f"(0), not the ids its SELECT fetched (3); got {bookend_pruned}"+ )+ assert ordinary_pruned == 0+def test_prune_counts_ordinary_versions_separately_from_bookends(self, patched_session, sample_note, monkeypatch):