~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Unit tests for ``VectorIndex`` (``vector_stores/facade.py``).
Needs a real DB (in-memory SQLite, ``Base.metadata.create_all``) because the
facade is the one module that bridges the vector store to ``DocumentChunk``
rows. Mirrors the DB-session mocking pattern used in
``tests/research_library/conftest.py`` (``mock_db_session_context``) /
``tests/research_library/routes/test_rag_routes.py``, but patches
``get_user_db_session`` at its ``vector_stores.facade`` import site (the
facade module's own binding), not at ``database.session_context`` — the
facade imports the name directly, so patching the origin module would not
affect calls made from inside ``facade.py``.
"""
import threading
from contextlib import contextmanager
import numpy as np
import pytest
from langchain_core.embeddings import Embeddings
from sqlalchemy import create_engine, event
from sqlalchemy.orm import sessionmaker
from local_deep_research.database.models import Base
from local_deep_research.database.models.library import (
DocumentChunk,
EmbeddingProvider,
)
from local_deep_research.vector_stores.base import VectorStoreIntegrityError
from local_deep_research.vector_stores.facade import (
ChunkInput,
VectorIndex,
_settle_vectors,
)
# --------------------------------------------------------------------------
# DB fixtures (mirrors tests/research_library/conftest.py's
# library_engine / library_session / mock_db_session_context)
# --------------------------------------------------------------------------
@pytest.fixture
def engine():
e = create_engine("sqlite:///:memory:")
Base.metadata.create_all(e)
yield e
e.dispose()
@pytest.fixture
def db_session(engine):
session_local = sessionmaker(bind=engine)
s = session_local()
yield s
s.close()
@pytest.fixture
def patched_db(db_session, mocker):
"""Patch ``get_user_db_session`` at the facade's own import site so
``VectorIndex._db()`` (which opens its own session when the caller
doesn't pass one) gets our in-memory test session instead of a real
encrypted DB connection."""
@contextmanager
def _mock_session(*_args, **_kwargs):
yield db_session
mocker.patch(
"local_deep_research.vector_stores.facade.get_user_db_session",
_mock_session,
)
return db_session
# --------------------------------------------------------------------------
# Deterministic fake Embeddings (no model download, no network)
# --------------------------------------------------------------------------
class _FakeEmbeddings(Embeddings):
"""Deterministic per-text vector: hash the text into a small int seed
so identical text always embeds identically and different text embeds
differently (good enough for exercising store add/search, not for
testing embedding *quality*)."""
def __init__(self, dim: int = 4):
self.dim = dim
def _vec(self, text: str):
seed = abs(hash(text)) % (2**32)
rng = np.random.default_rng(seed)
return rng.random(self.dim).tolist()
def embed_documents(self, texts):
return [self._vec(t) for t in texts]
def embed_query(self, text):
return self._vec(text)
def _noop(_p):
return None
def _ok(_p):
return (True, None)
def _make_index(
tmp_path,
*,
dimension=4,
collection_name="c1",
metric="cosine",
index_type="flat",
normalize=True,
):
return VectorIndex(
username="testuser",
db_password="pw",
embeddings=_FakeEmbeddings(dim=dimension),
embedding_model="fake-model",
embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,
collection_name=collection_name,
dimension=dimension,
path=tmp_path / f"{collection_name}.faiss",
lock=threading.Lock(),
integrity_record=_noop,
integrity_verify=_ok,
index_type=index_type,
metric=metric,
normalize=normalize,
)
# --------------------------------------------------------------------------
# index()
# --------------------------------------------------------------------------
class TestIndex:
def test_index_writes_rows_with_int_pks_and_vectors(
self, tmp_path, patched_db
):
vi = _make_index(tmp_path)
chunks = [
ChunkInput(text="hello world", metadata={"chunk_index": 0}),
ChunkInput(text="second chunk", metadata={"chunk_index": 1}),
]
stats = vi.index(
source_type="document", source_id="doc1", chunks=chunks
)
assert stats.chunks == 2
assert stats.added == 2
assert stats.removed == 0
rows = (
patched_db.query(DocumentChunk)
.filter_by(source_type="document", source_id="doc1")
.all()
)
assert len(rows) == 2
assert all(isinstance(r.id, int) for r in rows)
assert vi.count() == 2
assert set(vi._store.live_ids()) == {r.id for r in rows}
def test_index_empty_chunks_is_noop(self, tmp_path, patched_db):
vi = _make_index(tmp_path)
stats = vi.index(source_type="document", source_id="d1", chunks=[])
assert stats.chunks == 0
assert stats.added == 0
assert vi.count() == 0
def test_index_replace_true_purges_prior_chunks(self, tmp_path, patched_db):
vi = _make_index(tmp_path)
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="old1"), ChunkInput(text="old2")],
)
assert vi.count() == 2
stats = vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="new1")],
replace=True,
)
assert stats.removed == 2
assert stats.added == 1
assert vi.count() == 1
rows = patched_db.query(DocumentChunk).filter_by(source_id="d1").all()
assert len(rows) == 1
assert rows[0].chunk_text == "new1"
def test_index_replace_true_empty_chunks_purges_prior(
self, tmp_path, patched_db
):
# Re-indexing a source down to ZERO chunks (e.g. a note edited to
# whitespace) with replace=True must still remove its prior
# vectors/rows — otherwise the cleared content stays searchable.
vi = _make_index(tmp_path)
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="old1"), ChunkInput(text="old2")],
)
assert vi.count() == 2
stats = vi.index(
source_type="document",
source_id="d1",
chunks=[],
replace=True,
)
assert stats.removed == 2
assert stats.added == 0
assert vi.count() == 0
assert (
patched_db.query(DocumentChunk).filter_by(source_id="d1").count()
== 0
)
def test_index_replace_false_appends(self, tmp_path, patched_db):
vi = _make_index(tmp_path)
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="a")],
)
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="b")],
replace=False,
)
assert vi.count() == 2
assert (
patched_db.query(DocumentChunk).filter_by(source_id="d1").count()
== 2
)
def test_index_replace_does_not_touch_other_sources(
self, tmp_path, patched_db
):
vi = _make_index(tmp_path)
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="d1 chunk")],
)
vi.index(
source_type="document",
source_id="d2",
chunks=[ChunkInput(text="d2 chunk")],
replace=True,
)
assert vi.count() == 2
assert (
patched_db.query(DocumentChunk).filter_by(source_id="d1").count()
== 1
)
def test_index_rolls_back_db_on_apply_failure(
self, tmp_path, patched_db, mocker
):
"""If store.apply() raises, the flushed-but-uncommitted DocumentChunk
rows must be rolled back — no row can point at a missing vector."""
vi = _make_index(tmp_path)
mocker.patch.object(
vi._store, "apply", side_effect=RuntimeError("boom")
)
with pytest.raises(RuntimeError, match="boom"):
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="z")],
)
assert patched_db.query(DocumentChunk).count() == 0
def test_index_duplicate_text_chunks_produce_two_rows_and_vectors(
self, tmp_path, patched_db
):
"""0023 deliberately dropped the ``chunk_hash`` unique constraint so
identical text produces one row PER chunk (no cross-document
sharing). Guards against a reintroduced unique constraint or an
accidental hash-dedup short-circuit in ``index()``/``_build_row``."""
vi = _make_index(tmp_path)
chunks = [
ChunkInput(text="same text"),
ChunkInput(text="same text"),
]
stats = vi.index(source_type="document", source_id="d1", chunks=chunks)
assert stats.added == 2
rows = patched_db.query(DocumentChunk).filter_by(source_id="d1").all()
assert len(rows) == 2
assert rows[0].chunk_hash == rows[1].chunk_hash
assert rows[0].id != rows[1].id
assert vi.count() == 2
def test_index_uses_caller_provided_session_without_committing(
self, tmp_path, db_session, mocker
):
"""When the caller passes its own session, VectorIndex must not
commit/rollback it (the caller owns the transaction boundary)."""
vi = _make_index(tmp_path)
commit_spy = mocker.spy(db_session, "commit")
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="a")],
session=db_session,
)
commit_spy.assert_not_called()
# Row is visible via flush() even though nothing committed yet.
assert db_session.query(DocumentChunk).count() == 1
# --------------------------------------------------------------------------
# index_prepared() — bulk path used by parallel RAG indexing (PR #5235)
# --------------------------------------------------------------------------
class TestIndexPrepared:
"""Pre-embedded write path: embedding stays outside the FAISS/DB lock."""
def test_index_prepared_writes_rows_and_vectors(self, tmp_path, patched_db):
vi = _make_index(tmp_path)
chunks = [
ChunkInput(text="hello world", metadata={"chunk_index": 0}),
ChunkInput(text="second chunk", metadata={"chunk_index": 1}),
]
vectors = np.asarray(
vi.embeddings.embed_documents([c.text for c in chunks]),
dtype="float32",
)
stats = vi.index_prepared(
source_type="document",
source_id="doc1",
chunks=chunks,
vectors=vectors,
)
assert stats.chunks == 2
assert stats.added == 2
assert stats.removed == 0
rows = (
patched_db.query(DocumentChunk)
.filter_by(source_type="document", source_id="doc1")
.all()
)
assert len(rows) == 2
assert vi.count() == 2
assert set(vi._store.live_ids()) == {r.id for r in rows}
def test_index_prepared_does_not_call_embed_documents(
self, tmp_path, patched_db, mocker
):
vi = _make_index(tmp_path)
embed_spy = mocker.spy(vi.embeddings, "embed_documents")
chunks = [ChunkInput(text="pre-embedded")]
vectors = np.ones((1, 4), dtype="float32")
vi.index_prepared(
source_type="document",
source_id="d1",
chunks=chunks,
vectors=vectors,
)
embed_spy.assert_not_called()
assert vi.count() == 1
def test_index_prepared_rejects_missing_vectors(self, tmp_path, patched_db):
vi = _make_index(tmp_path)
with pytest.raises(ValueError, match="chunks given without vectors"):
vi.index_prepared(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="x")],
vectors=None,
)
def test_index_prepared_rejects_vector_count_mismatch(
self, tmp_path, patched_db
):
vi = _make_index(tmp_path)
with pytest.raises(ValueError, match="vector count must match"):
vi.index_prepared(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="a"), ChunkInput(text="b")],
vectors=np.ones((1, 4), dtype="float32"),
)
def test_index_prepared_replace_true_purges_prior(
self, tmp_path, patched_db
):
vi = _make_index(tmp_path)
old = [ChunkInput(text="old1"), ChunkInput(text="old2")]
vi.index_prepared(
source_type="document",
source_id="d1",
chunks=old,
vectors=np.ones((2, 4), dtype="float32"),
)
assert vi.count() == 2
stats = vi.index_prepared(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="new1")],
vectors=np.ones((1, 4), dtype="float32"),
replace=True,
)
assert stats.removed == 2
assert stats.added == 1
assert vi.count() == 1
rows = patched_db.query(DocumentChunk).filter_by(source_id="d1").all()
assert len(rows) == 1
assert rows[0].chunk_text == "new1"
def test_index_prepared_rolls_back_db_on_apply_failure(
self, tmp_path, patched_db, mocker
):
vi = _make_index(tmp_path)
mocker.patch.object(
vi._store, "apply", side_effect=RuntimeError("boom")
)
with pytest.raises(RuntimeError, match="boom"):
vi.index_prepared(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="z")],
vectors=np.ones((1, 4), dtype="float32"),
)
assert patched_db.query(DocumentChunk).count() == 0
def test_index_prepared_empty_chunks_replace_purges_prior(
self, tmp_path, patched_db
):
vi = _make_index(tmp_path)
vi.index_prepared(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="old")],
vectors=np.ones((1, 4), dtype="float32"),
)
stats = vi.index_prepared(
source_type="document",
source_id="d1",
chunks=[],
vectors=None,
replace=True,
)
assert stats.removed == 1
assert stats.added == 0
assert vi.count() == 0
# --------------------------------------------------------------------------
# search()
# --------------------------------------------------------------------------
class TestSearch:
def test_search_rehydrates_text_and_metadata_from_db(
self, tmp_path, patched_db
):
vi = _make_index(tmp_path)
vi.index(
source_type="document",
source_id="d1",
chunks=[
ChunkInput(text="alpha text", metadata={"k": "v"}),
ChunkInput(text="beta text"),
],
)
results = vi.search("alpha", top_k=5)
assert len(results) == 2
texts = {r.text for r in results}
assert texts == {"alpha text", "beta text"}
alpha = next(r for r in results if r.text == "alpha text")
assert alpha.metadata == {"k": "v"}
def test_search_skips_orphan_hit_missing_db_row(self, tmp_path, patched_db):
"""A vector whose DocumentChunk row was deleted out-of-band (DB
write succeeded, e.g. a caller cleaned up the row without going
through VectorIndex.delete) must be skipped, not crash or fabricate
text."""
vi = _make_index(tmp_path)
vi.index(
source_type="document",
source_id="d1",
chunks=[
ChunkInput(text="alpha text"),
ChunkInput(text="beta text"),
],
)
orphan_row = (
patched_db.query(DocumentChunk)
.filter_by(chunk_text="beta text")
.one()
)
patched_db.query(DocumentChunk).filter(
DocumentChunk.id == orphan_row.id
).delete()
patched_db.commit()
# The vector for "beta text" is still live in the store.
assert orphan_row.id in vi._store.live_ids()
results = vi.search("alpha", top_k=5)
assert len(results) == 1
assert results[0].text == "alpha text"
def test_search_does_not_leak_other_collections_text(
self, tmp_path, patched_db
):
"""A vector in THIS collection's store whose id belongs to a chunk in a
DIFFERENT collection (orphan + reused id) must NOT rehydrate that other
collection's text — rehydration is scoped to this collection."""
c1 = _make_index(tmp_path, collection_name="c1")
c2 = _make_index(tmp_path, collection_name="c2")
# A chunk indexed into c2.
c2.index(
source_type="document",
source_id="d2",
chunks=[ChunkInput(text="c2 SECRET text")],
)
c2_chunk = (
patched_db.query(DocumentChunk)
.filter_by(collection_name="c2")
.one()
)
# That SAME id sits as an orphan vector in c1's store (e.g. a
# rolled-back insert whose id was later reused by c2).
c1._store.apply(
add_ids=[c2_chunk.id],
add_vectors=np.ones((1, 4), dtype="float32"),
)
assert c2_chunk.id in c1._store.live_ids()
results = c1.search("anything", top_k=5)
# c2's text must NOT appear in c1's search results.
assert all("c2 SECRET" not in r.text for r in results)
def test_search_top_k_zero_returns_empty(self, tmp_path, patched_db):
vi = _make_index(tmp_path)
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="a")],
)
assert vi.search("a", top_k=0) == []
def test_search_result_metric_matches_collection_metric(
self, tmp_path, patched_db
):
vi = _make_index(tmp_path, metric="l2", normalize=False)
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="only")],
)
results = vi.search("only", top_k=1)
assert results[0].metric == "l2"
def test_search_result_metric_cosine(self, tmp_path, patched_db):
vi = _make_index(tmp_path, metric="cosine", normalize=True)
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="only")],
)
results = vi.search("only", top_k=1)
assert results[0].metric == "cosine"
def test_search_preserves_relevance_order_through_rehydration_with_orphan_in_middle(
self, tmp_path, patched_db
):
"""The facade must return hits in the STORE's relevance order (best
match first), not in whatever order the DB round-trip happens to
yield — a refactor that iterated ``rows.values()`` (an IN-query
keyed by id, with no ORDER BY) instead of ``hits`` would silently
scramble the result order. ``top_k`` truncates below the chunk
count, and the middle-ranked surviving hit's row is deleted
out-of-band (orphan) to prove the remaining hits still resolve in
relevance order around the gap, with no crash and no leaked text.
"""
import math
class _AngleEmbeddings(Embeddings):
"""Deterministic 2-D unit vectors at a fixed angle per text, so
cosine similarity to a theta=0 query is exactly ordered by the
angle table below — precise, reproducible ranking (no hashing).
"""
_ANGLES_DEG = {
"sim-high": 0,
"sim-2nd": 10,
"sim-3rd": 20,
"sim-low": 30,
}
def _vec(self, text):
theta = math.radians(self._ANGLES_DEG[text])
return [math.cos(theta), math.sin(theta)]
def embed_documents(self, texts):
return [self._vec(t) for t in texts]
def embed_query(self, _text):
return [1.0, 0.0]
vi = VectorIndex(
username="testuser",
db_password="pw",
embeddings=_AngleEmbeddings(),
embedding_model="fake-model",
embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,
collection_name="c-order",
dimension=2,
path=tmp_path / "c-order.faiss",
lock=threading.Lock(),
integrity_record=_noop,
integrity_verify=_ok,
index_type="flat",
metric="cosine",
normalize=True,
)
# Insertion order deliberately scrambled relative to similarity
# rank, so PK order != relevance order.
vi.index(
source_type="document",
source_id="d1",
chunks=[
ChunkInput(text="sim-low"),
ChunkInput(text="sim-3rd"),
ChunkInput(text="sim-high"),
ChunkInput(text="sim-2nd"),
],
)
mid_row = (
patched_db.query(DocumentChunk)
.filter_by(chunk_text="sim-2nd")
.one()
)
patched_db.query(DocumentChunk).filter(
DocumentChunk.id == mid_row.id
).delete()
patched_db.commit()
# The vector for "sim-2nd" is still live in the store (orphan).
assert mid_row.id in vi._store.live_ids()
results = vi.search("query", top_k=3)
# The orphan must not consume a result slot. Fetch the next valid
# result while preserving the store's relevance order across the gap.
assert [r.text for r in results] == ["sim-high", "sim-3rd", "sim-low"]
def test_search_carries_source_and_title_fields(self, tmp_path, patched_db):
vi = _make_index(tmp_path)
vi.index(
source_type="document",
source_id="doc-42",
chunks=[ChunkInput(text="titled chunk", metadata={"title": "T"})],
)
results = vi.search("titled", top_k=1)
assert results[0].source_type == "document"
assert results[0].source_id == "doc-42"
assert results[0].document_title == "T"
@pytest.mark.parametrize("top_k", [1, 3, 501])
def test_search_materializes_only_requested_rows(
self, tmp_path, patched_db, mocker, top_k
):
vi = _make_index(tmp_path, metric="l2", normalize=False)
count = 512
vi.index_prepared(
source_type="file",
source_id="sample",
chunks=[ChunkInput(text=f"sample chunk {i}") for i in range(count)],
vectors=np.ones((count, 4), dtype="float32"),
)
vi._store.apply(
add_ids=list(range(100_000, 100_000 + count)),
add_vectors=np.zeros((count, 4), dtype="float32"),
)
patched_db.expunge_all()
mocker.patch.object(
vi.embeddings, "embed_query", return_value=[0.0] * 4
)
materialized = set()
def on_load(row, _context):
materialized.add(row.id)
event.listen(DocumentChunk, "load", on_load)
try:
results = vi.search("sample", top_k=top_k)
finally:
event.remove(DocumentChunk, "load", on_load)
assert len(results) == top_k
assert all(result.source_id == "sample" for result in results)
assert materialized == {result.chunk_id for result in results}
# --------------------------------------------------------------------------
# delete()
# --------------------------------------------------------------------------
class TestDelete:
def test_delete_removes_vectors_and_rows(self, tmp_path, patched_db):
vi = _make_index(tmp_path)
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="x"), ChunkInput(text="y")],
)
assert vi.count() == 2
stats = vi.delete(source_type="document", source_id="d1")
assert stats.removed == 2
assert stats.rows_deleted == 2
assert vi.count() == 0
assert (
patched_db.query(DocumentChunk).filter_by(source_id="d1").count()
== 0
)
def test_delete_nonexistent_source_is_zero_noop(self, tmp_path, patched_db):
vi = _make_index(tmp_path)
stats = vi.delete(source_type="document", source_id="missing")
assert stats == type(stats)(removed=0, rows_deleted=0)
def test_delete_rolls_back_db_on_apply_failure(
self, tmp_path, patched_db, mocker
):
"""Mirrors ``test_index_rolls_back_db_on_apply_failure`` for
``delete()``: if ``store.apply()`` raises, the rows targeted for
removal must survive (no data loss) and the exception must
propagate. Also guards #3827: the owned session must remain usable
for later calls, not left broken by the failed rollback."""
vi = _make_index(tmp_path)
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="x"), ChunkInput(text="y")],
)
assert vi.count() == 2
mocker.patch.object(
vi._store, "apply", side_effect=RuntimeError("boom")
)
with pytest.raises(RuntimeError, match="boom"):
vi.delete(source_type="document", source_id="d1")
# No data loss: both target rows survive the failed delete.
assert (
patched_db.query(DocumentChunk).filter_by(source_id="d1").count()
== 2
)
# #3827: the (shared, thread-local) session VectorIndex owns must
# still be usable afterward — not left in a broken transaction
# state by the failed rollback. Prove it with an unrelated,
# unpatched index() call reusing the same session.
vi2 = _make_index(tmp_path, collection_name="c2")
stats = vi2.index(
source_type="document",
source_id="d2",
chunks=[ChunkInput(text="z")],
)
assert stats.added == 1
assert (
patched_db.query(DocumentChunk).filter_by(source_id="d2").count()
== 1
)
def test_delete_with_caller_session_survives_apply_failure(
self, tmp_path, db_session, mocker
):
"""delete()'s except-branch only rolls back ``if owns`` (facade
~470). When the CALLER passes its own session (``owns=False``), a
``store.apply()`` failure must NOT roll back the caller's session —
that would wipe the caller's other, unrelated uncommitted work too.
A plausible mirror-of-``index()`` refactor that dropped the ``if
owns:`` guard (unconditional rollback) would wipe both the
unrelated pending row and the still-uncommitted d1 row, failing the
assertions below.
"""
vi = _make_index(tmp_path)
# Index d1 using the CALLER's own session -> owns=False inside
# index(), so it does not commit; the row stays pending.
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="x")],
session=db_session,
)
d1_row = db_session.query(DocumentChunk).filter_by(source_id="d1").one()
# Unrelated pending row in the SAME caller transaction, simulating
# other work the caller is doing that must survive a delete()
# failure elsewhere in that same transaction.
x_row = DocumentChunk(
chunk_hash="x" * 64,
source_type="document",
source_id="unrelated-x",
collection_name=vi.collection_name,
chunk_text="unrelated pending row",
chunk_index=0,
start_char=0,
end_char=10,
word_count=2,
embedding_id="unrelated-x-embedding-id",
embedding_model=vi.embedding_model,
embedding_model_type=vi.embedding_model_type,
embedding_dimension=vi._store.dimension,
)
db_session.add(x_row)
db_session.flush()
x_id = x_row.id
mocker.patch.object(
vi._store, "apply", side_effect=RuntimeError("boom")
)
with pytest.raises(RuntimeError, match="boom"):
vi.delete(
source_type="document", source_id="d1", session=db_session
)
# DATA SURVIVAL, not a rollback spy: both the unrelated pending row
# X and the still-uncommitted d1 row must still be queryable in
# THIS session -- delete() must not have rolled it back.
assert (
db_session.query(DocumentChunk).filter_by(id=x_id).one().chunk_text
== "unrelated pending row"
)
assert (
db_session.query(DocumentChunk)
.filter_by(id=d1_row.id)
.one()
.chunk_text
== "x"
)
def test_index_then_delete_invariant_zero_state(self, tmp_path, patched_db):
"""After index()-then-delete() of the same source, both the store
and the DB must be empty — no leftover vectors or rows."""
vi = _make_index(tmp_path)
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text=f"c{i}") for i in range(5)],
)
vi.delete(source_type="document", source_id="d1")
assert vi.count() == 0
assert patched_db.query(DocumentChunk).count() == 0
class TestDeleteIds:
"""``delete_ids`` — used by the zotero sync path to remove specific
chunk ids whose rows may already be gone by the time it runs."""
def test_delete_ids_removes_specific_vectors_and_rows(
self, tmp_path, patched_db
):
vi = _make_index(tmp_path)
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="a"), ChunkInput(text="b")],
)
rows = patched_db.query(DocumentChunk).filter_by(source_id="d1").all()
ids = sorted(r.id for r in rows)
stats = vi.delete_ids([ids[0]])
assert stats.removed == 1
assert stats.rows_deleted == 1
assert vi.count() == 1
assert (
patched_db.query(DocumentChunk).filter_by(id=ids[0]).first() is None
)
def test_delete_ids_empty_is_noop(self, tmp_path, patched_db):
vi = _make_index(tmp_path)
stats = vi.delete_ids([])
assert stats.removed == 0
assert stats.rows_deleted == 0
def test_delete_ids_tolerates_rows_already_gone(self, tmp_path, patched_db):
"""The ids' DocumentChunk rows may already be deleted elsewhere by
the time this runs (see the docstring on delete_ids) — the vector
removal must still happen."""
vi = _make_index(tmp_path)
vi.index(
source_type="document",
source_id="d1",
chunks=[ChunkInput(text="a")],
)
row = patched_db.query(DocumentChunk).filter_by(source_id="d1").one()
chunk_id = row.id
patched_db.query(DocumentChunk).filter(
DocumentChunk.id == chunk_id
).delete()
patched_db.commit()
assert chunk_id in vi._store.live_ids()
stats = vi.delete_ids([chunk_id])
assert stats.removed == 1
assert stats.rows_deleted == 0 # row was already gone
assert vi.count() == 0
# --------------------------------------------------------------------------
# Settle wrapper: an unloadable-generation failure must surface, while a
# recoverable cleanup failure stays contained
# --------------------------------------------------------------------------
class _FakeSession:
def __init__(self):
self.info = {}
def in_nested_transaction(self):
return False
class _FailingStore:
def __init__(self, exc):
self._exc = exc
def end_transaction(self, *, committed):
raise self._exc
def test_settle_surfaces_integrity_failures_not_swallow_them():
"""A settle failure that indicates an unloadable vector generation must
not vanish behind the log line used for recoverable cleanup failures —
the caller must not report success over it."""
session = _FakeSession()
session.info["ldr_vector_transactions"] = {
"x": _FailingStore(VectorStoreIntegrityError("unloadable generation"))
}
with pytest.raises(VectorStoreIntegrityError):
_settle_vectors(session, True)
assert "ldr_vector_transactions" not in session.info
def test_settle_still_contains_recoverable_cleanup_failures():
"""A plain cleanup failure (the checkpoint remains recoverable against
the committed checksum) is logged, not raised."""
session = _FakeSession()
session.info["ldr_vector_transactions"] = {
"x": _FailingStore(OSError("simulated disk failure"))
}
_settle_vectors(session, True)