Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(notes): return actual deleted-row counts from _prune_versions_in_session (#6192)

LearningCircuit48688d7e808d
diff --git a/changelog.d/+notes-prune-count-accuracy.bugfix.md b/changelog.d/+notes-prune-count-accuracy.bugfix.md
new file mode 100644
index 0000000000..337d3870a7
--- /dev/null
+++ b/changelog.d/+notes-prune-count-accuracy.bugfix.md
@@ -0,0 +1 @@
+Fixed `_prune_versions_in_session` (#5498) reporting a pruned-version count taken from what its SELECT fetched (or, before that, a pre-delete `count()`) rather than what its DELETE actually removed — under concurrent prunes of the same note these diverged and over-reported (SQLAlchemy only warns when a non-versioned ORM delete matches fewer rows than fetched). It now deletes by the fetched ids and counts the rows the database itself reports deleting (`DELETE ... RETURNING`), so the logged prune counts reflect exactly the rows that call's own deletes removed — fewer, honestly, if a concurrent transaction had already removed some of them.
diff --git a/src/local_deep_research/research_library/notes/services/note_service.py b/src/local_deep_research/research_library/notes/services/note_service.py
index a974625f92..c5103fae73 100644
--- a/src/local_deep_research/research_library/notes/services/note_service.py
+++ b/src/local_deep_research/research_library/notes/services/note_service.py
@@ -18,7 +18,7 @@ from concurrent.futures import ThreadPoolExecutor
from typing import Any, Dict, List, Optional
from loguru import logger
-from sqlalchemy import func, or_
+from sqlalchemy import delete, func, or_
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session, joinedload
@@ -2539,17 +2539,34 @@ class NoteService:
(which adds PRE_RESTORE + RESTORE rows without consuming the ordinary
version budget).
- Returns ``(ordinary_pruned, bookend_pruned)`` — the counts of rows
- deleted in *this* call. The deletes are only flushed here, not
- committed: if the caller's transaction later rolls back, these
- rows survive. Callers must NOT log these counts themselves until
- after their own ``session.commit()`` succeeds — logging here (at
- flush time) would claim a prune that a subsequent rollback undoes,
- turning the one operator-facing signal for version loss into a
- false positive on exactly the failure paths it exists to surface.
- See ``update_note`` and ``restore_with_bookends`` for the
- post-commit logging call.
+ Returns ``(ordinary_pruned, bookend_pruned)`` — the number of rows
+ this call's own DELETE statements actually removed, counted from
+ the rows the database itself reports deleting (``DELETE ...
+ RETURNING``; same SQLite/SQLCipher baseline the chat service's
+ ``UPDATE ... RETURNING`` counters already rely on). The prune
+ selects the ids of the oldest rows over the cap, then deletes
+ exactly those ids: when a concurrently-committed transaction has
+ removed some of them in the SELECT→DELETE window, this call's
+ DELETE matches fewer rows and that smaller, honest number is
+ returned — never the fetched count, which the previous
+ per-instance ``session.delete()`` implementation reported even
+ when its DELETE matched nothing (SQLAlchemy only warns on that
+ mismatch for non-versioned mappers). The counts therefore never
+ over-report what this call removed; they say nothing about rows
+ other transactions deleted, and can be lower than the excess the
+ caps imply when concurrent pruners shared the work.
+
+ The deletes run in the caller's transaction but are NOT committed
+ here: if that transaction later rolls back, these rows survive.
+ Callers must NOT log these counts themselves until after their own
+ ``session.commit()`` succeeds — logging here (before the commit)
+ would claim a prune that a subsequent rollback undoes, turning the
+ one operator-facing signal for version loss into a false positive
+ on exactly the failure paths it exists to surface. See
+ ``update_note`` and ``restore_with_bookends`` for the post-commit
+ logging call.
"""
+ ordinary_pruned = 0
ordinary_total = (
session.query(NoteVersion)
.filter_by(document_id=note_id)
@@ -2560,22 +2577,38 @@ class NoteService:
if excess > 0:
# Audit bookends have their own cap below and never reduce the
# number of ordinary snapshots retained for the user.
- oldest = (
- session.query(NoteVersion)
+ oldest_ids = [
+ row_id
+ for (row_id,) in session.query(NoteVersion.id)
.filter_by(document_id=note_id)
.filter(NoteVersion.change_type.notin_(_AUDIT_BOOKEND_TYPES))
.order_by(NoteVersion.created_at.asc(), NoteVersion.id.asc())
.limit(excess)
.all()
- )
- for row in oldest:
- session.delete(row)
- session.flush()
+ ]
+ if oldest_ids:
+ # Delete by the fetched primary keys and count what the
+ # DELETE actually matched — the rows the DB reports via
+ # RETURNING — not what the SELECT fetched. Under a
+ # concurrently-committed delete of the same rows, the ORM's
+ # per-instance session.delete() path only WARNS on the
+ # rowcount mismatch yet would report the fetched count.
+ # synchronize_session=False leaves these rows persistent in the
+ # session identity map until commit — do not touch NoteVersion
+ # instances after this call.
+ result = session.execute(
+ delete(NoteVersion)
+ .where(NoteVersion.id.in_(oldest_ids))
+ .returning(NoteVersion.id)
+ .execution_options(synchronize_session=False)
+ )
+ ordinary_pruned = len(result.all())
# Separately bound the bookend pool. Excluding bookends from the prune
# above keeps the restore trail, but each restore writes two
# un-prunable rows, so without this ceiling repeated restores grow
# note_versions without limit. Keep the most recent MAX_BOOKEND_VERSIONS.
+ bookend_pruned = 0
bookend_total = (
session.query(NoteVersion)
.filter_by(document_id=note_id)
@@ -2584,19 +2617,24 @@ class NoteService:
)
bookend_excess = bookend_total - MAX_BOOKEND_VERSIONS
if bookend_excess > 0:
- oldest_bookends = (
- session.query(NoteVersion)
+ oldest_bookend_ids = [
+ row_id
+ for (row_id,) in session.query(NoteVersion.id)
.filter_by(document_id=note_id)
.filter(NoteVersion.change_type.in_(_AUDIT_BOOKEND_TYPES))
.order_by(NoteVersion.created_at.asc(), NoteVersion.id.asc())
.limit(bookend_excess)
.all()
- )
- for row in oldest_bookends:
- session.delete(row)
- session.flush()
- ordinary_pruned = max(excess, 0)
- bookend_pruned = max(bookend_excess, 0)
+ ]
+ if oldest_bookend_ids:
+ # Same delete-matched counting as the ordinary prune above.
+ result = session.execute(
+ delete(NoteVersion)
+ .where(NoteVersion.id.in_(oldest_bookend_ids))
+ .returning(NoteVersion.id)
+ .execution_options(synchronize_session=False)
+ )
+ bookend_pruned = len(result.all())
return (ordinary_pruned, bookend_pruned)
def _log_version_prune(
diff --git a/tests/notes/test_note_stress.py b/tests/notes/test_note_stress.py
index 6ac296d250..da8c3a9f84 100644
--- a/tests/notes/test_note_stress.py
+++ b/tests/notes/test_note_stress.py
@@ -765,6 +765,304 @@ class TestNoteVersionCap:
assert "note version prune" in template
assert rest == [sample_note.id, 2, 2]
+ def test_prune_returns_actually_deleted_count_not_stale_excess(
+ self, patched_session, sample_note, monkeypatch
+ ):
+ """Regression: ``_prune_versions_in_session`` must return the number
+ of rows its DELETE actually removed -- the ids the database hands
+ back through ``DELETE ... RETURNING`` -- never the ``excess``
+ computed from a ``count()`` taken before the id SELECT.
+
+ ``count()`` and the subsequent ``SELECT ... LIMIT excess`` are
+ separate statements, so a version delete for the same note that
+ lands between them leaves the pre-delete ``count()`` stale relative
+ to what the SELECT finds and the DELETE removes. Simulated here by
+ inflating ONLY the first ``Query.count()`` call (the
+ ordinary-versions count) so it over-reports by 5, while leaving the
+ real SELECT/DELETE untouched -- the shape of a stale-snapshot race,
+ without needing real concurrent transactions.
+ """
+ from datetime import datetime, timedelta, timezone
+
+ from sqlalchemy.orm import Query
+
+ from local_deep_research.database.models import (
+ NoteChangeType,
+ NoteVersion,
+ )
+ from local_deep_research.research_library.notes.services import (
+ note_service as note_service_mod,
+ )
+ from local_deep_research.research_library.notes.services.note_service import (
+ NoteService,
+ )
+
+ monkeypatch.setattr(note_service_mod, "MAX_VERSIONS_PER_NOTE", 2)
+
+ base = datetime(2024, 1, 1, tzinfo=timezone.utc)
+ # Only 3 real ordinary versions exist for this note.
+ for i in range(3):
+ patched_session.add(
+ NoteVersion(
+ id=str(uuid.uuid4()),
+ document_id=sample_note.id,
+ title=f"t{i}",
+ content=f"c{i}",
+ tags=[],
+ change_type=NoteChangeType.MANUAL_SAVE.value,
+ content_hash=f"h{i}",
+ created_at=base + timedelta(minutes=i),
+ )
+ )
+ patched_session.commit()
+
+ real_count = Query.count
+ call_count = {"n": 0}
+
+ def stale_count(self):
+ call_count["n"] += 1
+ result = real_count(self)
+ if call_count["n"] == 1:
+ # Simulate a stale snapshot: claims 5 more rows exist than
+ # actually do, as if this note had 5 additional versions a
+ # moment ago that a concurrent transaction has since
+ # deleted and committed.
+ return result + 5
+ return result
+
+ monkeypatch.setattr(Query, "count", stale_count)
+
+ service = NoteService("test_user")
+ prune_counts = service._prune_versions_in_session(
+ patched_session, sample_note.id
+ )
+ patched_session.commit()
+
+ # The stale count() claims excess = (3 + 5) - 2 = 6, but only 3
+ # rows exist to match LIMIT(6) against, so the SELECT (and the
+ # deletes) can only ever touch 3 rows -- never 6.
+ remaining = (
+ patched_session.query(NoteVersion)
+ .filter_by(document_id=sample_note.id)
+ .count()
+ )
+ assert remaining == 0
+ ordinary_pruned, _bookend_pruned = prune_counts
+ assert ordinary_pruned == 3, (
+ "must report the number of rows ACTUALLY deleted (3), not the "
+ f"stale count()-derived excess (6); got {ordinary_pruned}"
+ )
+
+ def test_prune_reports_delete_matched_rows_not_fetched_rows(
+ self, patched_session, sample_note, monkeypatch
+ ):
+ """Regression: the reported count must be the rows THIS call's
+ DELETE removed, not the rows its id SELECT fetched — 0 here, not 3.
+
+ The prune fetches the ids of the oldest ordinary versions, then
+ deletes exactly those ids. Here every fetched row is removed in
+ between, so the DELETE matches nothing and the honest count is 0,
+ while the ORM's old per-instance ``session.delete()`` path only
+ WARNS on that rowcount mismatch (non-versioned mapper) and still
+ reported the 3 rows it had fetched.
+
+ The removal is injected between the id SELECT and the ``delete()``
+ statement by wrapping the ``delete`` callable the service module
+ uses: the wrapper deletes the rows through a second Session on the
+ same engine and commits, then hands the real statement construction
+ through untouched, so the returned counts still come from the real
+ DELETE's RETURNING. The in-memory fixture engine uses SQLAlchemy's
+ ``SingletonThreadPool``, so that second Session shares this one's
+ connection and transaction: this exercises the SELECT-then-DELETE
+ ordering, not isolation between genuinely concurrent transactions.
+ """
+ from datetime import datetime, timedelta, timezone
+
+ from sqlalchemy import delete as sa_delete
+ from sqlalchemy.orm import sessionmaker
+
+ from local_deep_research.database.models import (
+ NoteChangeType,
+ NoteVersion,
+ )
+ from local_deep_research.research_library.notes.services import (
+ note_service as note_service_mod,
+ )
+ from local_deep_research.research_library.notes.services.note_service import (
+ NoteService,
+ )
+
+ # Cap 0 => excess == total => the prune's SELECT fetches every
+ # ordinary version of this note (3), so the concurrent delete
+ # below removes exactly the fetched set.
+ monkeypatch.setattr(note_service_mod, "MAX_VERSIONS_PER_NOTE", 0)
+
+ base = datetime(2024, 1, 1, tzinfo=timezone.utc)
+ for i in range(3):
+ patched_session.add(
+ NoteVersion(
+ id=str(uuid.uuid4()),
+ document_id=sample_note.id,
+ title=f"t{i}",
+ content=f"c{i}",
+ tags=[],
+ change_type=NoteChangeType.MANUAL_SAVE.value,
+ content_hash=f"h{i}",
+ created_at=base + timedelta(minutes=i),
+ )
+ )
+ patched_session.commit()
+
+ calls = {"n": 0, "rows_at_delete": None}
+
+ def concurrent_delete_first(*args, **kwargs):
+ calls["n"] += 1
+ calls["rows_at_delete"] = (
+ patched_session.query(NoteVersion)
+ .filter_by(document_id=sample_note.id)
+ .count()
+ )
+ # A competing pruner removes the fetched rows and COMMITS
+ # between this call's id SELECT and its DELETE.
+ ConcurrentSession = sessionmaker(bind=patched_session.get_bind())
+ concurrent = ConcurrentSession()
+ concurrent.query(NoteVersion).filter_by(
+ document_id=sample_note.id
+ ).delete(synchronize_session=False)
+ concurrent.commit()
+ concurrent.close()
+ return sa_delete(*args, **kwargs)
+
+ # Pins the bare-name ``delete`` imported at note_service module scope.
+ monkeypatch.setattr(note_service_mod, "delete", concurrent_delete_first)
+
+ service = NoteService("test_user")
+ ordinary_pruned, bookend_pruned = service._prune_versions_in_session(
+ patched_session, sample_note.id
+ )
+ patched_session.commit()
+
+ # The prune's id SELECT ran while all 3 rows were still present,
+ # and the wrapper's committed delete is what removed them —
+ # verified against the real DB state, not a mock.
+ assert calls["n"] == 1
+ assert calls["rows_at_delete"] == 3
+ remaining = (
+ patched_session.query(NoteVersion)
+ .filter_by(document_id=sample_note.id)
+ .count()
+ )
+ assert remaining == 0
+ assert (ordinary_pruned, bookend_pruned) == (0, 0), (
+ "must report the rows THIS session's DELETE matched (0, 0), not "
+ "the rows its SELECT fetched (3); got "
+ f"{(ordinary_pruned, bookend_pruned)}"
+ )
+
+ def test_prune_reports_delete_matched_bookend_rows_not_fetched_rows(
+ self, patched_session, sample_note, monkeypatch
+ ):
+ """Same guard as the test above, for the bookend half of the prune.
+
+ The bookend block is a structural duplicate of the ordinary one and
+ is just as able to regress to ``len(oldest_bookend_ids)``. The test
+ above cannot catch that: its note has no bookends, so
+ ``bookend_excess`` is negative, the block never runs, and its
+ ``bookend_pruned == 0`` assertion holds vacuously.
+
+ Here the note has ONLY bookends, so the ordinary block is skipped
+ (``ordinary_total`` is 0, well under the cap) and the wrapped
+ ``delete`` fires exactly once — for the bookend statement. Its rows
+ are removed between the id SELECT and the DELETE, so the honest
+ bookend count is 0, not the 3 ids the SELECT fetched.
+ """
+ from datetime import datetime, timedelta, timezone
+
+ from sqlalchemy import delete as sa_delete
+ from sqlalchemy.orm import sessionmaker
+
+ from local_deep_research.database.models import (
+ NoteChangeType,
+ NoteVersion,
+ )
+ from local_deep_research.research_library.notes.services import (
+ note_service as note_service_mod,
+ )
+ from local_deep_research.research_library.notes.services.note_service import (
+ NoteService,
+ )
+
+ # Ordinary cap comfortably above the note's 0 ordinary versions, so
+ # that block is skipped; bookend cap 0 => bookend_excess == the whole
+ # bookend pool, so the bookend SELECT fetches all 3 of them.
+ monkeypatch.setattr(note_service_mod, "MAX_VERSIONS_PER_NOTE", 5)
+ monkeypatch.setattr(note_service_mod, "MAX_BOOKEND_VERSIONS", 0)
+
+ base = datetime(2024, 1, 1, tzinfo=timezone.utc)
+ for i in range(3):
+ patched_session.add(
+ NoteVersion(
+ id=str(uuid.uuid4()),
+ document_id=sample_note.id,
+ title=f"bookend{i}",
+ content=f"bookend-content{i}",
+ tags=[],
+ change_type=(
+ NoteChangeType.RESTORE.value
+ if i % 2 == 0
+ else NoteChangeType.PRE_RESTORE.value
+ ),
+ content_hash=f"bookend-hash{i}",
+ created_at=base + timedelta(minutes=i),
+ )
+ )
+ patched_session.commit()
+
+ calls = {"n": 0, "rows_at_delete": None}
+
+ def concurrent_delete_first(*args, **kwargs):
+ calls["n"] += 1
+ calls["rows_at_delete"] = (
+ patched_session.query(NoteVersion)
+ .filter_by(document_id=sample_note.id)
+ .count()
+ )
+ # A competing pruner removes the fetched bookends and COMMITS
+ # between this call's id SELECT and its DELETE.
+ ConcurrentSession = sessionmaker(bind=patched_session.get_bind())
+ concurrent = ConcurrentSession()
+ concurrent.query(NoteVersion).filter_by(
+ document_id=sample_note.id
+ ).delete(synchronize_session=False)
+ concurrent.commit()
+ concurrent.close()
+ return sa_delete(*args, **kwargs)
+
+ # Pins the bare-name ``delete`` imported at note_service module scope.
+ monkeypatch.setattr(note_service_mod, "delete", concurrent_delete_first)
+
+ service = NoteService("test_user")
+ ordinary_pruned, bookend_pruned = service._prune_versions_in_session(
+ patched_session, sample_note.id
+ )
+ patched_session.commit()
+
+ # Exactly one DELETE was built — the bookend one — and its id SELECT
+ # ran while all 3 bookends were still present.
+ assert calls["n"] == 1
+ assert calls["rows_at_delete"] == 3
+ remaining = (
+ patched_session.query(NoteVersion)
+ .filter_by(document_id=sample_note.id)
+ .count()
+ )
+ assert remaining == 0
+ assert bookend_pruned == 0, (
+ "the bookend count must be the rows THIS call's DELETE matched "
+ f"(0), not the ids its SELECT fetched (3); got {bookend_pruned}"
+ )
+ assert ordinary_pruned == 0
+
def test_prune_counts_ordinary_versions_separately_from_bookends(
self, patched_session, sample_note, monkeypatch
):