Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

33.3 KB
Raw Blame
"""High-level semantic-index facade: ``index`` / ``search`` / ``delete``.

This is the single simple entry point to the vector subsystem. It is the ONE
place in the ``vector_stores`` package that bridges three things:

  * the embedding model (a LangChain ``Embeddings``),
  * the encrypted DB (``DocumentChunk`` — the authoritative home of chunk text
    and metadata), and
  * a swappable :class:`BaseVectorStore` (ids + vectors only).

``base.py`` and ``implementations/`` stay DB-agnostic; only this module imports
``DocumentChunk``. The vector store holds an int64 id per chunk
(``DocumentChunk.id``); text/metadata are rehydrated from the DB by that id on
search — the store never receives text (see the "SECURITY INVARIANT" in
:mod:`.base`).

One row = one vector. Identical text in two documents produces two rows/vectors
(no cross-document sharing); this keeps provenance correct and deletes trivial.
Exact-duplicate *files* are an ingestion-time concern, handled upstream.
"""

import hashlib
import uuid
from contextlib import contextmanager
from dataclasses import dataclass, field
from pathlib import Path
from typing import (
    Iterator,
    List,
    Optional,
    Sequence,
    Tuple,
    TypedDict,
    cast,
)

import numpy as np
from langchain_core.embeddings import Embeddings
from loguru import logger
from sqlalchemy import event, exists, func, or_, select
from sqlalchemy.orm import Session

from ..database.models.library import (
    Document,
    DocumentChunk,
    DocumentCollection,
)
from ..database.session_context import get_user_db_session
from .base import (
    IntegrityRecord,
    IntegrityVerify,
    VectorStoreIntegrityError,
    WriteLock,
)
from .config import get_vector_store_class


def _settle_vectors(session: Session, committed: bool):
    if session.in_nested_transaction():
        return
    pending = session.info.pop("ldr_vector_transactions", {})
    for store in pending.values():
        try:
            store.end_transaction(committed=committed)
        except VectorStoreIntegrityError:
            # A settle failure of this class means the published vector
            # generation failed validation — surface it instead of letting it
            # vanish behind a log line while the caller reports success.
            raise
        except (OSError, ValueError):
            # The rollback checkpoint is unlinked only AFTER the replacement
            # generation validates, so a failed settle leaves the pre-image in
            # place. On the rollback path the DB still records the pre-image
            # checksum, so the checksum-gated restore (verify_file ->
            # recover_checkpoint) can still recover it. After a commit the DB
            # already records the NEW checksum: the stale checkpoint no longer
            # matches and is never restored (the next begin unlinks it), so
            # loads rely on the new root verifying. Either way, any
            # checkpoint that does not match the committed checksum is
            # refused, and a symlinked checkpoint is rejected (by
            # SegmentedFaissStore._end_transaction_locked and by
            # recover_checkpoint).
            logger.exception("Could not finish vector generation cleanup")


def _commit_vectors(session: Session):
    _settle_vectors(session, True)


def _rollback_vectors(session: Session):
    _settle_vectors(session, False)


def _close_vectors(session: Session, transaction):
    if transaction.parent is None:
        _settle_vectors(session, False)


class _BindKwargs(TypedDict):
    """Keyword arguments forwarded to ``store_cls.create()``/``.load()``.

    A plain ``dict`` would infer a uniform (``object``) value type, erasing
    each field's real type when unpacked via ``**bind`` — a ``TypedDict``
    keeps each key's type through the ``**`` unpacking so mypy can check the
    call against ``BaseVectorStore.create``/``.load``.
    """

    dimension: int
    index_type: str
    metric: str
    normalize: bool
    path: Path
    lock: WriteLock
    integrity_record: IntegrityRecord
    integrity_verify: IntegrityVerify


@dataclass
class ChunkInput:
    """One chunk to index: its text plus optional positional/title metadata.

    Kept backend- and framework-neutral (not a LangChain ``Document``) so the
    facade never couples to a specific splitter/vector library.
    """

    text: str
    metadata: dict = field(default_factory=dict)


@dataclass
class SearchResult:
    """A search hit, rehydrated from the encrypted DB by chunk id."""

    chunk_id: int
    text: str
    # Raw backend score; the relevance formula is the caller's choice. ``metric``
    # says how to read it: for cosine/dot_product (inner product) HIGHER is more
    # similar; for l2 LOWER is closer. Branch on it or scores invert.
    distance: float
    metric: str
    metadata: dict
    document_title: Optional[str]
    source_id: Optional[str]
    source_type: Optional[str]


@dataclass
class IndexStats:
    """Result of :meth:`VectorIndex.index`."""

    chunks: int
    added: int
    removed: int


@dataclass
class DeleteStats:
    """Result of :meth:`VectorIndex.delete`."""

    removed: int
    rows_deleted: int


class VectorIndex:
    """Per-collection semantic index. Construct once per (collection, user).

    Persistence resources (``path`` / ``lock`` / integrity hooks) are injected —
    the caller (service layer) owns per-user path resolution and the write-lock
    registry; this facade just wires them into the store.
    """

    def __init__(
        self,
        *,
        username: str,
        db_password: str,
        embeddings: Embeddings,
        embedding_model: str,
        embedding_model_type,  # EmbeddingProvider enum (DocumentChunk column)
        collection_name: str,
        dimension: int,
        path: Path,
        lock: WriteLock,
        integrity_record: IntegrityRecord,
        integrity_verify: IntegrityVerify,
        index_type: str = "flat",
        metric: str = "cosine",
        normalize: bool = True,
        provider: str = "faiss",
        index_key: Optional[str] = None,
        require_document_membership: bool = False,
    ) -> None:
        self.username = username
        self.db_password = db_password  # gitleaks:allow
        self.embeddings = embeddings
        self.embedding_model = embedding_model
        self.embedding_model_type = embedding_model_type
        self.collection_name = collection_name
        self._path = Path(path)
        self.index_key = (
            index_key
            or hashlib.sha256(str(Path(path).resolve()).encode()).hexdigest()
        )
        self.require_document_membership = require_document_membership
        self.membership_collection_id = collection_name.removeprefix(
            "collection_"
        )
        # Canonicalize the metric so the physical index BUILD (faiss_store:
        # inner-product iff metric in ("cosine","dot_product")) and the search
        # engines' relevance formula (which branch on ``metric == "l2"``) agree.
        # A non-canonical value like "L2"/"Cosine"/" cosine " would otherwise
        # build one index type but be scored as the other, silently corrupting
        # relevance. Both the store (via ``bind`` below) and SearchResult.metric
        # then see the same canonical string.
        metric = (metric or "cosine").strip().lower()
        self.metric = metric

        store_cls = get_vector_store_class(provider)
        path = Path(path)
        bind: _BindKwargs = {
            "dimension": dimension,
            "index_type": index_type,
            "metric": metric,
            "normalize": normalize,
            "path": path,
            "lock": lock,
            "integrity_record": integrity_record,
            "integrity_verify": integrity_verify,
        }
        # ``bind`` already carries ``path``; pass it only via kwargs (load()'s
        # first param is path) so we don't hand ``path`` twice.
        self._store = (
            store_cls.load(**bind)
            if path.exists()
            else store_cls.create(**bind)
        )

    @contextmanager
    def _db(self, session: Optional[Session]) -> Iterator[Tuple[Session, bool]]:
        """Yield ``(session, owns)``.

        ``get_user_db_session`` returns a THREAD-LOCAL, reused session, so this
        facade must not blindly ``commit()``/``rollback()`` — it would settle a
        caller's unrelated pending work. When the caller passes its own
        ``session``, we use it and yield ``owns=False`` (the caller owns the
        transaction boundary and the apply()-then-commit durability gap). Only
        when no session is passed do we open one and own commit/rollback.
        """
        if session is not None:
            yield session, False
        else:
            with get_user_db_session(self.username, self.db_password) as s:
                yield s, True

    def ownership_filter(self, *, include_legacy: bool = False):
        """Encrypted chunk ownership must match the physical index."""
        owner = DocumentChunk.vector_index_key == self.index_key
        if include_legacy:
            owner = or_(owner, DocumentChunk.vector_index_key.is_(None))
        return (
            DocumentChunk.collection_name == self.collection_name,
            DocumentChunk.embedding_model == self.embedding_model,
            DocumentChunk.embedding_model_type == self.embedding_model_type,
            owner,
        )

    def _apply(self, session: Session, *, orphan_scanned=False, **kwargs):
        """Settle vector checkpoints with the caller's outer DB transaction."""
        from .implementations.segmented_faiss import SegmentedFaissStore

        if isinstance(self._store, SegmentedFaissStore):
            kwargs["orphan_scanned"] = orphan_scanned
            if session.in_nested_transaction():
                raise RuntimeError(
                    "Vector writes require an outer database transaction"
                )
            pending = session.info.setdefault("ldr_vector_transactions", {})
            key = str(self._path.resolve())
            self._store.begin_transaction(first=key not in pending)
            pending[key] = self._store
            if not session.info.get("ldr_vector_transaction_listeners"):
                event.listen(session, "after_commit", _commit_vectors)
                event.listen(session, "after_rollback", _rollback_vectors)
                event.listen(session, "after_transaction_end", _close_vectors)
                session.info["ldr_vector_transaction_listeners"] = True
        return self._store.apply(**kwargs)

    def _membership_filter(self):
        if not self.require_document_membership:
            return True
        return or_(
            DocumentChunk.source_type != "document",
            exists().where(
                Document.id == DocumentChunk.source_id,
                DocumentCollection.document_id == Document.id,
                DocumentCollection.collection_id
                == self.membership_collection_id,
            ),
        )

    def _prepare_cleanup(
        self, session: Session, *, force: bool = False
    ) -> tuple[list[int], list[int], bool]:
        """Discover real orphans before allocating IDs that a rollback reused.

        Partial documents remain owned: completeness is a status concern, not
        a garbage-collection criterion. Legacy ownership is recovered only
        from membership in this physical index. Queries use bounded IN lists.

        A store that tracks its scans (segmented FAISS) is fully scanned only
        periodically, or when it holds IDs above the highest chunk row; the
        last value says whether this call scanned.
        """
        due = getattr(self._store, "orphan_scan_due", None)
        if not force and due is not None:
            highest = session.scalar(select(func.max(DocumentChunk.id)))
            if not due(highest):
                return [], [], False
        orphan_vectors: list[int] = []
        orphan_rows: list[int] = []
        for ids in self._store.iter_id_batches():
            session.query(DocumentChunk).filter(
                DocumentChunk.id.in_(ids),
                *self.ownership_filter(include_legacy=True),
                DocumentChunk.vector_index_key.is_(None),
            ).update(
                {DocumentChunk.vector_index_key: self.index_key},
                synchronize_session="fetch",
            )
            owned = set(
                session.scalars(
                    select(DocumentChunk.id).where(
                        DocumentChunk.id.in_(ids),
                        *self.ownership_filter(),
                    )
                )
            )
            valid = set(
                session.scalars(
                    select(DocumentChunk.id).where(
                        DocumentChunk.id.in_(ids),
                        *self.ownership_filter(),
                        self._membership_filter(),
                    )
                )
            )
            orphan_vectors.extend(set(ids) - valid)
            orphan_rows.extend(owned - valid)
        return orphan_vectors, orphan_rows, True

    def compact_orphans(self, session: Optional[Session] = None) -> int:
        """Remove only unowned vectors; retain every valid partial document."""
        with self._db(session) as (sess, owns):
            vectors, rows, scanned = self._prepare_cleanup(sess, force=True)
            if vectors:
                self._apply(
                    sess,
                    orphan_scanned=scanned,
                    add_ids=[],
                    add_vectors=None,
                    remove_ids=vectors,
                )
            self._finalize_db(sess, owns, rows, "compact orphan vectors")
        return len(vectors)

    def index_prepared(
        self,
        *,
        source_type: str,
        source_id: Optional[str],
        chunks: Sequence[ChunkInput],
        vectors: Optional[np.ndarray],
        replace: bool = True,
        session: Optional[Session] = None,
    ) -> IndexStats:
        """Persist pre-embedded chunks without invoking the embedding backend.

        Bulk indexing uses this after parallel preparation. Database row/id
        allocation, vector application, prior-row removal, and commit semantics
        remain identical to :meth:`index`; only embedding is moved outside the
        collection writer critical section.
        """
        chunks = list(chunks)
        if not chunks and (not replace or source_id is None):
            return IndexStats(chunks=0, added=0, removed=0)
        if chunks:
            if vectors is None:
                raise ValueError("chunks given without vectors")
            vectors = np.asarray(vectors, dtype="float32")
            if len(vectors) != len(chunks):
                raise ValueError("vector count must match chunk count")

        sid = str(source_id) if source_id is not None else None
        with self._db(session) as (sess, owns):
            orphan_vectors, orphan_rows, scanned = self._prepare_cleanup(sess)
            prior_ids: List[int] = []
            if replace and sid is not None:
                prior_ids = list(
                    sess.scalars(
                        select(DocumentChunk.id).filter_by(
                            source_type=source_type,
                            source_id=sid,
                            collection_name=self.collection_name,
                            embedding_model=self.embedding_model,
                            embedding_model_type=self.embedding_model_type,
                            vector_index_key=self.index_key,
                        )
                    )
                )

            rows = [self._build_row(c, source_type, sid) for c in chunks]
            if rows:
                sess.add_all(rows)
                sess.flush()
            new_ids = [row.id for row in rows]
            try:
                stats = self._apply(
                    sess,
                    orphan_scanned=scanned,
                    add_ids=new_ids,
                    add_vectors=vectors,
                    remove_ids=prior_ids + orphan_vectors,
                )
            except Exception:
                if owns:
                    sess.rollback()
                raise
            self._finalize_db(
                sess,
                owns,
                prior_ids + orphan_rows,
                f"index prepared {source_type}/{sid}",
            )

        return IndexStats(
            chunks=len(chunks),
            added=stats["added"],
            removed=stats["removed"],
        )

    # ------------------------------------------------------------------ #
    # index
    # ------------------------------------------------------------------ #
    def index(
        self,
        *,
        source_type: str,
        source_id: Optional[str],
        chunks: Sequence[ChunkInput],
        replace: bool = True,
        session: Optional[Session] = None,
    ) -> IndexStats:
        """Embed + persist ``chunks`` for one source into this collection.

        Ordering: write chunk rows and ``flush()`` for their int ids, then
        ``store.apply()`` (persists vectors atomically under its lock), then
        delete the source's prior rows, then ``commit()`` (only if we own the
        session). If ``store.apply()`` fails we roll back — the new rows never
        existed, so no DB row can point at a missing vector.

        Segmented FAISS retains a checkpoint until the database transaction
        commits; a rollback restores the prior generation. The next write or
        explicit reconciliation also removes legacy orphan vectors before
        allocating replacement IDs. When ``session`` is passed the caller
        owns the transaction boundary.

        ``replace=True`` removes this source's PRIOR chunks/vectors first;
        ``False`` appends.
        """
        chunks = list(chunks)
        # An empty chunk list with replace=True still must remove this source's
        # PRIOR rows/vectors — e.g. a note edited down to whitespace yields zero
        # chunks, and skipping here would leave the user-cleared content
        # permanently searchable. Only a genuine no-op (nothing to add AND
        # nothing to replace) returns early.
        if not chunks and (not replace or source_id is None):
            return IndexStats(chunks=0, added=0, removed=0)

        texts = [c.text for c in chunks]
        vectors = (
            np.asarray(self.embeddings.embed_documents(texts), dtype="float32")
            if texts
            else None
        )

        sid = str(source_id) if source_id is not None else None
        with self._db(session) as (sess, owns):
            orphan_vectors, orphan_rows, scanned = self._prepare_cleanup(sess)
            prior_ids: List[int] = []
            if replace and sid is not None:
                # Scope the prior-chunk lookup to THIS store's embedding-model
                # config. collection_name is identical across every model config
                # for a collection (only RAGIndex.index_hash / the physical
                # .faiss path differ per model), so a lookup by
                # (source_type, source_id, collection_name) alone would also
                # match rows indexed under a DIFFERENT model. Those ids would
                # then be handed to THIS (physically separate, model-specific)
                # store as remove_ids — where faiss silently no-ops on unknown
                # ids — and _finalize_db would still DELETE the rows. Result: a
                # reindex under model B destroys model A's chunk text (its only
                # copy) while A's .faiss keeps the now-unresolvable vector,
                # even though A's index is deliberately kept for model revert.
                prior_ids = list(
                    sess.scalars(
                        select(DocumentChunk.id).filter_by(
                            source_type=source_type,
                            source_id=sid,
                            collection_name=self.collection_name,
                            embedding_model=self.embedding_model,
                            embedding_model_type=self.embedding_model_type,
                            vector_index_key=self.index_key,
                        )
                    )
                )

            rows = [self._build_row(c, source_type, sid) for c in chunks]
            if rows:
                sess.add_all(rows)
                sess.flush()  # assigns int PKs to every row in one round-trip
            new_ids = [row.id for row in rows]

            try:
                stats = self._apply(
                    sess,
                    orphan_scanned=scanned,
                    add_ids=new_ids,
                    add_vectors=vectors,
                    remove_ids=prior_ids + orphan_vectors,
                )
            except Exception:
                # apply() failed without a durable change (fail-closed reload /
                # add error / atomic-save) — a clean DB rollback keeps them
                # consistent.
                if owns:
                    sess.rollback()
                raise
            # apply() is DURABLE from here; a DB-side failure below leaves the
            # store and DB possibly disagreeing (orphan vectors/rows).
            self._finalize_db(
                sess,
                owns,
                prior_ids + orphan_rows,
                f"index {source_type}/{sid}",
            )

        return IndexStats(
            chunks=len(chunks),
            added=stats["added"],
            removed=stats["removed"],
        )

    def _finalize_db(
        self,
        session: Session,
        owns: bool,
        delete_ids: List[int],
        what: str,
    ) -> int:
        """Delete rows + commit (if owned), AFTER a durable ``store.apply()``.

        Covers the whole post-apply DB window in one place. If the row delete
        or commit fails, roll back when we own the session, restoring the
        segmented-store checkpoint. Other providers' unowned vectors are
        removed by the next write or reconciliation.
        ``synchronize_session="fetch"`` keeps the reused thread-local session's
        identity map from serving stale, already-deleted objects. Returns the
        number of rows deleted.
        """
        deleted = 0
        try:
            if delete_ids:
                for start in range(0, len(delete_ids), 500):
                    deleted += (
                        session.query(DocumentChunk)
                        .filter(
                            DocumentChunk.id.in_(
                                delete_ids[start : start + 500]
                            ),
                            *self.ownership_filter(include_legacy=True),
                        )
                        .delete(synchronize_session="fetch")
                    )
            if owns:
                session.commit()
        except Exception:
            logger.exception(
                f"{what}: database update failed after vector persistence; "
                "the owning transaction must roll back"
            )
            if owns:
                session.rollback()
            raise
        return deleted

    def _build_row(
        self, chunk: ChunkInput, source_type: str, source_id: Optional[str]
    ) -> DocumentChunk:
        meta = chunk.metadata or {}
        return DocumentChunk(
            chunk_hash=hashlib.sha256(chunk.text.encode()).hexdigest(),
            source_type=source_type,
            source_id=source_id,
            collection_name=self.collection_name,
            chunk_text=chunk.text,
            chunk_index=meta.get("chunk_index", 0),
            start_char=meta.get("start_char", 0),
            end_char=meta.get("end_char", len(chunk.text)),
            word_count=len(chunk.text.split()),
            # embedding_id stays populated: it's the old-uuid -> new-int join key
            # the one-time index-format migration needs. (Legacy NOT NULL UNIQUE
            # column; drop it only in a later, separate migration.)
            embedding_id=uuid.uuid4().hex,
            embedding_model=self.embedding_model,
            embedding_model_type=self.embedding_model_type,
            embedding_dimension=self._store.dimension,
            vector_index_key=self.index_key,
            document_title=meta.get("document_title") or meta.get("title"),
            document_metadata=meta or None,
        )

    # ------------------------------------------------------------------ #
    # search
    # ------------------------------------------------------------------ #
    def search(
        self,
        query: str,
        top_k: int,
        session: Optional[Session] = None,
    ) -> List[SearchResult]:
        """Embed ``query``, search the store, rehydrate hits from the DB by id.

        Orphan hits (a vector whose DB row is gone) are skipped — there is no
        in-store text to fall back to, and skipping is the correct signal.
        """
        if top_k <= 0:
            return []
        query_vec = np.asarray(
            self.embeddings.embed_query(query), dtype="float32"
        )
        requested = top_k
        hits = self._store.search(query_vec, requested)
        if not hits:
            return []
        with self._db(session) as (sess, _owns):
            # Scope rehydration to THIS collection. A hit id is a DocumentChunk.id
            # from this collection's index, but an orphan/stale vector (e.g. a
            # rolled-back insert left a vector whose id was later reused by a
            # chunk in a DIFFERENT collection) could otherwise match a row that
            # belongs elsewhere and leak that other document's text into this
            # collection's results. Filtering by collection_name makes such a
            # hit resolve to no row here and be skipped as an orphan.
            while True:
                selected_hits = []
                for start in range(0, len(hits), 500):
                    batch = hits[start : start + 500]
                    eligible = set(
                        sess.scalars(
                            select(DocumentChunk.id).where(
                                DocumentChunk.id.in_([cid for cid, _ in batch]),
                                *self.ownership_filter(include_legacy=True),
                                self._membership_filter(),
                            )
                        )
                    )
                    for hit in batch:
                        if hit[0] in eligible:
                            selected_hits.append(hit)
                            if len(selected_hits) == top_k:
                                break
                    if len(selected_hits) == top_k:
                        break
                if len(selected_hits) >= top_k or len(hits) < requested:
                    break
                count = self._store.count()
                if requested >= count:
                    break
                requested = min(count, requested * 2)
                hits = self._store.search(query_vec, requested)

            # Materialize full records only after selecting their ranked IDs.
            ids = [cid for cid, _ in selected_hits]
            rows = {}
            for start in range(0, len(ids), 500):
                rows.update(
                    {
                        row.id: row
                        for row in sess.query(DocumentChunk).filter(
                            DocumentChunk.id.in_(ids[start : start + 500]),
                            *self.ownership_filter(include_legacy=True),
                            self._membership_filter(),
                        )
                    }
                )

        results: List[SearchResult] = []
        for chunk_id, distance in selected_hits:
            row = rows.get(chunk_id)
            if row is None:
                logger.debug(
                    f"Search hit {chunk_id} has no DocumentChunk row; skipping"
                )
                continue
            # SQLAlchemy's legacy Column() declarative style types instance
            # attribute access as Column[...] rather than the plain Python
            # value; cast() to the DB-verified runtime type (no conversion,
            # so a None stays None rather than becoming e.g. the "None"
            # string a str()/int() call would produce).
            results.append(
                SearchResult(
                    chunk_id=chunk_id,
                    text=cast(str, row.chunk_text),
                    distance=distance,
                    metric=self.metric,
                    metadata=cast(Optional[dict], row.document_metadata) or {},
                    document_title=cast(Optional[str], row.document_title),
                    source_id=cast(Optional[str], row.source_id),
                    source_type=cast(Optional[str], row.source_type),
                )
            )
        return results[:top_k]

    # ------------------------------------------------------------------ #
    # delete
    # ------------------------------------------------------------------ #
    def delete(
        self,
        *,
        source_type: str,
        source_id: str,
        session: Optional[Session] = None,
    ) -> DeleteStats:
        """Remove all of one source's chunks/vectors from this collection.

        ``store.apply()`` (the durable side) runs before the DB delete + commit,
        matching :meth:`index`'s ordering.

        Re-queries ``DocumentChunk`` by (source_type, source_id,
        collection_name) at call time to discover the ids to remove, so the
        rows must still exist when this is called. If a caller has already
        deleted those rows elsewhere, this finds nothing to do; use
        :meth:`delete_ids` instead with the ids captured before deletion.
        """
        return self.delete_sources(
            source_type=source_type, source_ids=[source_id], session=session
        )

    def delete_sources(
        self,
        *,
        source_type: str,
        source_ids: Sequence[str],
        session: Optional[Session] = None,
    ) -> DeleteStats:
        """Purge a batch of sources with one durable operation for this index."""
        source_ids = list({str(source_id) for source_id in source_ids})
        if not source_ids:
            return DeleteStats(removed=0, rows_deleted=0)
        with self._db(session) as (sess, owns):
            orphan_vectors, orphan_rows, scanned = self._prepare_cleanup(sess)
            ids: list[int] = []
            for start in range(0, len(source_ids), 500):
                ids.extend(
                    sess.scalars(
                        select(DocumentChunk.id).where(
                            DocumentChunk.source_type == source_type,
                            DocumentChunk.source_id.in_(
                                source_ids[start : start + 500]
                            ),
                            *self.ownership_filter(),
                        )
                    )
                )
            removed = sorted(set(ids) | set(orphan_vectors))
            if not removed:
                return DeleteStats(removed=0, rows_deleted=0)
            try:
                stats = self._apply(
                    sess,
                    orphan_scanned=scanned,
                    add_ids=[],
                    add_vectors=None,
                    remove_ids=removed,
                )
            except Exception:
                if owns:
                    sess.rollback()
                raise
            rows_deleted = self._finalize_db(
                sess,
                owns,
                sorted(set(ids) | set(orphan_rows)),
                "delete sources",
            )
        return DeleteStats(removed=stats["removed"], rows_deleted=rows_deleted)

    def delete_ids(
        self,
        ids: Sequence[int],
        *,
        session: Optional[Session] = None,
    ) -> DeleteStats:
        """Remove specific chunk ids' vectors from this collection's store.

        For callers that already captured the ``DocumentChunk`` int ids
        BEFORE the matching rows were deleted elsewhere — e.g. a caller
        whose own per-item transaction must delete the rows eagerly (for
        its own atomicity/rollback semantics) ahead of a later BATCHED
        vector-store cleanup pass. Unlike :meth:`delete`, this does NOT
        re-query ``DocumentChunk`` by (source_type, source_id,
        collection_name) to discover the ids — by the time this runs the
        rows the ids came from may already be gone, and that lookup would
        find nothing (see the "rows must still exist" requirement noted on
        :meth:`delete`). Any rows matching the given ids that do still
        exist are cleaned up too (defensive; normally none are left).
        """
        wanted = sorted({int(i) for i in ids if i is not None})
        if not wanted:
            return DeleteStats(removed=0, rows_deleted=0)

        with self._db(session) as (sess, owns):
            try:
                stats = self._apply(
                    sess, add_ids=[], add_vectors=None, remove_ids=wanted
                )
            except Exception:
                if owns:
                    sess.rollback()
                raise
            # Guard the row-delete AND commit together (mirrors index()).
            rows_deleted = self._finalize_db(
                sess, owns, wanted, f"delete_ids ({len(wanted)} ids)"
            )

        return DeleteStats(removed=stats["removed"], rows_deleted=rows_deleted)

    def live_ids(self) -> List[int]:
        """Authoritative vector ids currently present in the store."""
        return self._store.live_ids()

    def count(self) -> int:
        """Number of vectors currently in the store."""
        return self._store.count()