Enterprise agent platform for building, deploying, and operating AI teammates.
fix(kb): sweep embedding rows again after chunk rows are deleted (#3823)
蓝鱼bluefish4f21db067cdb
diff --git a/src/xagent/core/tools/core/RAG_tools/kb/collection_handle.py b/src/xagent/core/tools/core/RAG_tools/kb/collection_handle.pyindex 46dca4315..e03ffd5ca 100644--- a/src/xagent/core/tools/core/RAG_tools/kb/collection_handle.py+++ b/src/xagent/core/tools/core/RAG_tools/kb/collection_handle.py@@ -2620,16 +2620,23 @@ class LanceDBCollectionHandle(KBCollectionHandle):for (stale_parse, config_hash), ids in self.stale_chunks(doc_id, parse_hash, user_id=user_id).items():- for start in range(0, len(ids), size):- self.delete_embedding_records(- doc_id, chunk_ids=ids[start : start + size], is_admin=True- )+ # Vectors first: a failure here leaves the chunk rows a retry reads.+ self._delete_stale_vectors(doc_id, ids, size)self.delete_chunk_records(doc_id, parse_hash=stale_parse, config_hash=config_hash, is_admin=True)+ # Again after the chunk rows: a vector committed after the first+ # delete would otherwise outlive its chunk.+ self._delete_stale_vectors(doc_id, ids, size)deleted += idsreturn deleted+ def _delete_stale_vectors(self, doc_id: str, ids: list[str], size: int) -> None:+ for start in range(0, len(ids), size):+ self.delete_embedding_records(+ doc_id, chunk_ids=ids[start : start + size], is_admin=True+ )+def discard_uncommitted_embeddings(self, doc_id: str, *, user_id: int | None = None) -> int:diff --git a/src/xagent/core/tools/core/RAG_tools/storage/lancedb_cleanup_filters.py b/src/xagent/core/tools/core/RAG_tools/storage/lancedb_cleanup_filters.pyindex 73b5c07cd..58c6e4a58 100644--- a/src/xagent/core/tools/core/RAG_tools/storage/lancedb_cleanup_filters.py+++ b/src/xagent/core/tools/core/RAG_tools/storage/lancedb_cleanup_filters.py@@ -33,52 +33,56 @@ def build_embedding_cleanup_filters(scope: KBCleanupScope,) -> dict[str, list[str]]:"""Build per-embeddings-table cleanup predicates for a resolved scope."""- table_filters: dict[str, list[str]] = {}- for table_name in select_embedding_tables(conn, model_tag=scope.model_tag):- table = None- try:- table = conn.open_table(table_name)- table_filters[table_name] = _build_filters_for_table(table, scope)- except Exception:- table_filters[table_name] = _build_filters_without_schema(scope)- finally:- _safe_close_table(table)- return table_filters---def build_embedding_cleanup_filters_from_base(+ return {+ table_name: embedding_cleanup_filters_for_table(conn, table_name, scope)+ for table_name in select_embedding_tables(conn, model_tag=scope.model_tag)+ }+++def embedding_cleanup_filters_for_table(+ conn: Any, table_name: str, scope: KBCleanupScope+) -> list[str]:+ """The cleanup predicates of a resolved scope for one embeddings table."""+ table = None+ try:+ table = conn.open_table(table_name)+ return _build_filters_for_table(table, scope)+ except Exception:+ return _build_filters_without_schema(scope)+ finally:+ _safe_close_table(table)+++def embedding_cleanup_filters_from_base(conn: Any,+ table_name: str,*,base_filter: str,user_id: Optional[int],is_admin: bool,- model_tag: Optional[str] = None,-) -> dict[str, list[str]]:- """Build per-embeddings-table predicate lists from an existing base filter."""- table_filters: dict[str, list[str]] = {}- for table_name in select_embedding_tables(conn, model_tag=model_tag):- table = None- try:- table = conn.open_table(table_name)- table_filters[table_name] = [- append_user_filter_for_table(- table=table,- filter_expr=base_filter,- user_id=user_id,- is_admin=is_admin,- )- ]- except Exception:- table_filters[table_name] = [- append_user_filter_without_schema(- filter_expr=base_filter,- user_id=user_id,- is_admin=is_admin,- )- ]- finally:- _safe_close_table(table)- return table_filters+) -> list[str]:+ """One embeddings table's predicates from an existing base filter."""+ table = None+ try:+ table = conn.open_table(table_name)+ return [+ append_user_filter_for_table(+ table=table,+ filter_expr=base_filter,+ user_id=user_id,+ is_admin=is_admin,+ )+ ]+ except Exception:+ return [+ append_user_filter_without_schema(+ filter_expr=base_filter,+ user_id=user_id,+ is_admin=is_admin,+ )+ ]+ finally:+ _safe_close_table(table)def append_user_filter_for_table(diff --git a/src/xagent/core/tools/core/RAG_tools/storage/lancedb_stores.py b/src/xagent/core/tools/core/RAG_tools/storage/lancedb_stores.pyindex 93b4e83df..e9ecc93c5 100644--- a/src/xagent/core/tools/core/RAG_tools/storage/lancedb_stores.py+++ b/src/xagent/core/tools/core/RAG_tools/storage/lancedb_stores.py@@ -11,7 +11,18 @@ from collections.abc import Iterablefrom contextlib import contextmanagerfrom datetime import datetime, timedelta, timezonefrom enum import Enum-from typing import Any, Dict, Iterator, List, Literal, Optional, Sequence, Tuple, cast+from typing import (+ Any,+ Callable,+ Dict,+ Iterator,+ List,+ Literal,+ Optional,+ Sequence,+ Tuple,+ cast,+)import pyarrow as pa # type: ignorefrom filelock import FileLock, Timeout@@ -3108,7 +3119,10 @@ class LanceDBVectorIndexStore(VectorIndexStore):from ..version_management.main_pointer_manager import (_get_main_pointer_impl as get_main_pointer,)- from .lancedb_cleanup_filters import build_embedding_cleanup_filters+ from .lancedb_cleanup_filters import (+ embedding_cleanup_filters_for_table,+ embedding_cleanup_filters_from_base,+ )conn = self._get_connection()ensure_documents_table(conn)@@ -3140,6 +3154,8 @@ class LanceDBVectorIndexStore(VectorIndexStore):}predicates: Dict[str, list] = {}+ embedding_filters: Optional[Callable[[str], list]] = None+ embedding_base: Optional[str] = Noneif scope == "parse":if old_parse_hash is None:@@ -3160,14 +3176,7 @@ class LanceDBVectorIndexStore(VectorIndexStore):is_admin=is_admin,skip_user_filter=True,)- _vis_replace_embedding_predicates(- predicates=predicates,- conn=conn,- base_expr=base,- user_id=user_id,- is_admin=is_admin,- model_tag=model_tag,- )+ embedding_base = basepredicates["chunks"] = [_vis_append_user_filter_if_needed(conn=conn,@@ -3188,14 +3197,7 @@ class LanceDBVectorIndexStore(VectorIndexStore):f"doc_id == '{escaped_doc_id}' AND "f"parse_hash != '{escaped_new_parse_hash}'")- _vis_replace_embedding_predicates(- predicates=predicates,- conn=conn,- base_expr=other,- user_id=user_id,- is_admin=is_admin,- model_tag=model_tag,- )+ embedding_base = otherpredicates["chunks"] = [_vis_append_user_filter_if_needed(conn=conn,@@ -3232,14 +3234,7 @@ class LanceDBVectorIndexStore(VectorIndexStore):is_admin=is_admin,skip_user_filter=True,)- _vis_replace_embedding_predicates(- predicates=predicates,- conn=conn,- base_expr=base,- user_id=user_id,- is_admin=is_admin,- model_tag=model_tag,- )+ embedding_base = baseif new_parse_hash:from ..utils.string_utils import escape_lancedb_string@@ -3251,14 +3246,7 @@ class LanceDBVectorIndexStore(VectorIndexStore):f"doc_id == '{escaped_doc_id}' AND "f"parse_hash != '{escaped_parse_hash}'")- _vis_replace_embedding_predicates(- predicates=predicates,- conn=conn,- base_expr=other,- user_id=user_id,- is_admin=is_admin,- model_tag=model_tag,- )+ embedding_base = otherpredicates["chunks"] = [_vis_append_user_filter_if_needed(conn=conn,@@ -3276,10 +3264,11 @@ class LanceDBVectorIndexStore(VectorIndexStore):is_admin=is_admin,model_tag=model_tag,)- filters_by_table = build_embedding_cleanup_filters(conn, scope_obj)- for table_name, filter_exprs in filters_by_table.items():- if filter_exprs:- predicates.setdefault(table_name, []).extend(filter_exprs)++ def _scope_filters(table_name: str) -> list:+ return embedding_cleanup_filters_for_table(conn, table_name, scope_obj)++ embedding_filters = _scope_filterselif scope == "pointers":filt = _vis_build_document_filter(conn=conn,@@ -3293,6 +3282,20 @@ class LanceDBVectorIndexStore(VectorIndexStore):else:raise CascadeCleanupError(f"Unsupported scope: {scope}")+ if embedding_base is not None:+ base_expr = embedding_base++ def _base_filters(table_name: str) -> list:+ return embedding_cleanup_filters_from_base(+ conn,+ table_name,+ base_filter=base_expr,+ user_id=user_id,+ is_admin=is_admin,+ )++ embedding_filters = _base_filters+try:result = _vis_execute_or_plan_by_predicates(conn,@@ -3300,6 +3303,7 @@ class LanceDBVectorIndexStore(VectorIndexStore):preview_only=preview_only,confirm=confirm,model_tag=model_tag,+ embedding_filters=embedding_filters,)finally:# Earlier tables may already be deleted when a later one raises.@@ -3329,8 +3333,8 @@ class LanceDBVectorIndexStore(VectorIndexStore):ensure_main_pointers_table,ensure_parses_table,)+ from ..utils.lancedb_query_utils import list_table_namesfrom ..utils.user_scope import resolve_user_scope- from .lancedb_cleanup_filters import select_embedding_tablesuser_scope = resolve_user_scope(user_id=user_id, is_admin=is_admin)user_id = user_scope.user_id@@ -3346,7 +3350,7 @@ class LanceDBVectorIndexStore(VectorIndexStore):ensure_main_pointers_table(conn)ensure_ingestion_runs_table(conn)- table_names = _vis_get_table_names(conn)+ table_names = list(list_table_names(conn))predicates: Dict[str, list] = {}core_tables = [@@ -3378,17 +3382,19 @@ class LanceDBVectorIndexStore(VectorIndexStore):)predicates[table_name] = [filter_expr]- for table_name in select_embedding_tables(conn, model_tag=model_tag):+ def _embedding_filter(table_name: str) -> list:if target == "collection":- filter_expr = _vis_build_collection_filter(- conn=conn,- table_name=table_name,- collection=collection,- user_id=user_id,- is_admin=is_admin,- )- else:- filter_expr = _vis_build_document_filter(+ return [+ _vis_build_collection_filter(+ conn=conn,+ table_name=table_name,+ collection=collection,+ user_id=user_id,+ is_admin=is_admin,+ )+ ]+ return [+ _vis_build_document_filter(conn=conn,table_name=table_name,collection=collection,@@ -3396,7 +3402,7 @@ class LanceDBVectorIndexStore(VectorIndexStore):user_id=user_id,is_admin=is_admin,)- predicates[table_name] = [filter_expr]+ ]try:result = _vis_execute_or_plan_by_predicates(@@ -3404,7 +3410,8 @@ class LanceDBVectorIndexStore(VectorIndexStore):predicates,preview_only=preview_only,confirm=confirm,- model_tag=None,+ model_tag=model_tag,+ embedding_filters=_embedding_filter,)finally:# Earlier tables may already be deleted when a later one raises.@@ -3433,7 +3440,8 @@ def _vis_delete_rows_by_filters(table: Any, filter_exprs: list) -> int:deleted_count = 0for filter_expr in filter_exprs:- count = _safe_count_rows(table, filter_expr)+ # A count read as zero would skip this delete and report it done.+ count = _safe_count_rows(table, filter_expr, on_error="raise")if count > 0:table.delete(filter_expr)deleted_count += count@@ -3473,29 +3481,6 @@ def _vis_append_user_filter_if_needed(_safe_close_table(table)-def _vis_replace_embedding_predicates(- *,- predicates: Dict[str, list],- conn: Any,- base_expr: str,- user_id: Optional[int],- is_admin: bool,- model_tag: Optional[str] = None,-) -> None:- from .lancedb_cleanup_filters import build_embedding_cleanup_filters_from_base-- table_filters = build_embedding_cleanup_filters_from_base(- conn,- base_filter=base_expr,- user_id=user_id,- is_admin=is_admin,- model_tag=model_tag,- )- for table_name, filter_exprs in table_filters.items():- if filter_exprs:- predicates[table_name] = list(filter_exprs)--def _vis_get_table_names(conn: Any) -> list:from ..utils.lancedb_query_utils import list_table_names@@ -3506,10 +3491,9 @@ def _vis_get_table_names(conn: Any) -> list:def _vis_plan_by_predicates(- conn: Any, table_to_filter: Dict[str, list], model_tag: Optional[str] = None+ conn: Any, table_to_filter: Dict[str, list]) -> Dict[str, int]:from ..LanceDB.schema_manager import _safe_close_table- from .lancedb_cleanup_filters import select_embedding_tablescounts: Dict[str, int] = {}table_names = _vis_get_table_names(conn)@@ -3524,18 +3508,6 @@ def _vis_plan_by_predicates(_safe_close_table(table)for table_name, filter_exprs in table_to_filter.items():- if table_name == "__embeddings__":- total = 0- target_tables = select_embedding_tables(conn, model_tag=model_tag)- for t in target_tables:- table = None- try:- table = conn.open_table(t)- total += _vis_count_rows_by_filters(table, filter_exprs)- finally:- _safe_close_table(table)- counts[table_name] = total- continueif table_name.startswith("embeddings_"):continueif table_name not in table_names:@@ -3550,83 +3522,57 @@ def _vis_plan_by_predicates(return counts-def _vis_delete_by_predicates(- conn: Any, table_to_filter: Dict[str, list], model_tag: Optional[str] = None-) -> Dict[str, int]:+def _vis_embedding_pass(+ conn: Any,+ *,+ model_tag: Optional[str],+ embedding_filters: Callable[[str], list],+ deleted: Dict[str, int],+) -> None:+ """The listing raises: a swallowed failure would report a cascade complete+ with its embedding rows still searchable."""import logging as _loggingfrom ..LanceDB.schema_manager import _safe_close_tablefrom .lancedb_cleanup_filters import select_embedding_tables_logger = _logging.getLogger(__name__)- deleted: Dict[str, int] = {}- table_names = _vis_get_table_names(conn)-- for t in table_names:- if not t.startswith("embeddings_") or t not in table_to_filter:+ for t in select_embedding_tables(conn, model_tag=model_tag):+ filter_exprs = embedding_filters(t)+ if not filter_exprs:continue- filter_exprs = table_to_filter[t]table = Nonetry:table = conn.open_table(t)cnt = _vis_delete_rows_by_filters(table, filter_exprs)if cnt > 0:_logger.info("Cascade cleanup: deleted %s rows from %s", cnt, t)- deleted[t] = cnt+ deleted[t] = deleted.get(t, 0) + cntfinally:_safe_close_table(table)- order = [- "__embeddings__",- "chunks",- "parses",- "main_pointers",- "ingestion_runs",- "documents",- ]-- if "__embeddings__" in table_to_filter:- filter_exprs = table_to_filter["__embeddings__"]- total = 0- target_tables = select_embedding_tables(conn, model_tag=model_tag)- for t in target_tables:- table = None- try:- table = conn.open_table(t)- total += _vis_delete_rows_by_filters(table, filter_exprs)- finally:- _safe_close_table(table)- deleted["embeddings"] = total- if total > 0:- _logger.info(- "Cascade cleanup: deleted %s rows from embeddings tables", total- )- for name in order[1:]:- if name in table_to_filter and name in table_names:- table = None- try:- table = conn.open_table(name)- cnt = _vis_delete_rows_by_filters(table, table_to_filter[name])- if cnt > 0:- _logger.info("Cascade cleanup: deleted %s rows from %s", cnt, name)- deleted[name] = cnt- finally:- _safe_close_table(table)+def _vis_delete_by_predicates(+ conn: Any,+ table_to_filter: Dict[str, list],+ model_tag: Optional[str] = None,+ embedding_filters: Optional[Callable[[str], list]] = None,+) -> Dict[str, int]:+ import logging as _logging- for name, filter_exprs in table_to_filter.items():- if name in (- "__embeddings__",- "chunks",- "parses",- "main_pointers",- "ingestion_runs",- "documents",- ) or name.startswith("embeddings_"):- continue- if name not in table_names:- deleted[name] = 0- continue+ from ..LanceDB.schema_manager import _safe_close_table+ from ..utils.lancedb_query_utils import list_table_names++ _logger = _logging.getLogger(__name__)+ deleted: Dict[str, int] = {}+ order = ["chunks", "parses", "main_pointers", "ingestion_runs"]+ # Documents last: its row is what a retried document delete looks up, so+ # every failure before this point can still be retried whole.+ late = ["documents"]+ # A listing that fails must not shrink this delete to the embeddings.+ table_names = list(list_table_names(conn))++ def _delete(name: str, filter_exprs: list) -> None:table = Nonetry:table = conn.open_table(name)@@ -3637,6 +3583,44 @@ def _vis_delete_by_predicates(finally:_safe_close_table(table)+ # Embeddings first: a failure leaves every core row in place, so a retry+ # finds the document or the stale chunks again.+ if embedding_filters is not None:+ _vis_embedding_pass(+ conn,+ model_tag=model_tag,+ embedding_filters=embedding_filters,+ deleted=deleted,+ )++ for name in order:+ if name in table_to_filter and name in table_names:+ _delete(name, table_to_filter[name])++ for name, filter_exprs in table_to_filter.items():+ if name in ("documents", *order):+ continue+ if name not in table_names:+ deleted[name] = 0+ continue+ _delete(name, filter_exprs)++ if embedding_filters is not None and any(+ name in table_to_filter for name in (*order, *late)+ ):+ # The chunk rows are gone now: a fresh listing finds the embedding tables+ # a writer created since the first pass, each with its own predicates.+ _vis_embedding_pass(+ conn,+ model_tag=model_tag,+ embedding_filters=embedding_filters,+ deleted=deleted,+ )++ for name in late:+ if name in table_to_filter and name in table_names:+ _delete(name, table_to_filter[name])+return deleted@@ -3647,17 +3631,31 @@ def _vis_execute_or_plan_by_predicates(preview_only: bool,confirm: bool,model_tag: Optional[str] = None,+ embedding_filters: Optional[Callable[[str], list]] = None,) -> Dict[str, int]:+ """Embedding tables come only from ``embedding_filters``: a predicate built from+ an earlier listing misses a table a writer creates while the delete runs."""+ from .lancedb_cleanup_filters import select_embedding_tables++ if any(+ name == "__embeddings__" or name.startswith("embeddings_")+ for name in predicates+ ):+ raise ValueError("embedding tables are passed as embedding_filters")if not (confirm and not preview_only):- return _vis_plan_by_predicates(conn, predicates, model_tag=model_tag)- return _vis_delete_by_predicates(conn, predicates, model_tag=model_tag)+ plan = dict(predicates)+ if embedding_filters is not None:+ for table_name in select_embedding_tables(conn, model_tag=model_tag):+ plan[table_name] = embedding_filters(table_name)+ return _vis_plan_by_predicates(conn, plan)+ return _vis_delete_by_predicates(+ conn, predicates, model_tag=model_tag, embedding_filters=embedding_filters+ )def _vis_collapse_embedding_table_counts(counts: Dict[str, int]) -> Dict[str, int]:collapsed = dict(counts)- embeddings_total = int(collapsed.pop("__embeddings__", 0)) + int(- collapsed.pop("embeddings", 0)- )+ embeddings_total = 0for table_name in list(collapsed):if str(table_name).startswith("embeddings_"):embeddings_total += int(collapsed.pop(table_name, 0))@@ -3866,8 +3864,8 @@ def _vis_cascade_delete_documents(ensure_main_pointers_table,ensure_parses_table,)+ from ..utils.lancedb_query_utils import list_table_namesfrom ..utils.user_scope import resolve_user_scope- from .lancedb_cleanup_filters import select_embedding_tablesnormalized_doc_ids = sorted({str(d) for d in doc_ids if d})if not normalized_doc_ids:@@ -3885,7 +3883,7 @@ def _vis_cascade_delete_documents(ensure_main_pointers_table(conn)ensure_ingestion_runs_table(conn)- table_names = _vis_get_table_names(conn)+ table_names = list(list_table_names(conn))predicates: Dict[str, list] = {}core_tables = ["documents", "parses", "chunks", "main_pointers", "ingestion_runs"]@@ -3903,8 +3901,8 @@ def _vis_cascade_delete_documents()]- for table_name in select_embedding_tables(conn):- predicates[table_name] = [+ def _embedding_filter(table_name: str) -> list:+ return [_vis_build_documents_filter(conn=conn,table_name=table_name,@@ -3921,6 +3919,7 @@ def _vis_cascade_delete_documents(preview_only=preview_only,confirm=confirm,model_tag=None,+ embedding_filters=_embedding_filter,)diff --git a/tests/core/tools/core/RAG_tools/storage/test_lancedb_delete_sweep_order.py b/tests/core/tools/core/RAG_tools/storage/test_lancedb_delete_sweep_order.pynew file mode 100644index 000000000..72d5afac9--- /dev/null+++ b/tests/core/tools/core/RAG_tools/storage/test_lancedb_delete_sweep_order.py@@ -0,0 +1,690 @@+"""Deletes remove vectors, then chunk rows, then sweep the embedding tables in scope again."""++from __future__ import annotations++import asyncio+import json+from datetime import datetime, timezone+from typing import Any, Dict, List, Optional++import pyarrow as pa+import pytest++from xagent.core.tools.core.RAG_tools.LanceDB.model_tag_utils import (+ written_embeddings_table_name,+)+from xagent.core.tools.core.RAG_tools.storage.factory import get_metadata_store+from xagent.core.tools.core.RAG_tools.utils.lancedb_query_utils import list_table_names++OTHER_MODEL = "new-model"+SOURCE_MODEL = "BAAI/old-model"+++def _connection() -> Any:+ return get_metadata_store().get_raw_connection()+++def _save_collection() -> None:+ from xagent.core.tools.core.RAG_tools.core.schemas import CollectionInfo+ from xagent.core.tools.core.RAG_tools.management import collection_manager as cm++ asyncio.run(+ cm.collection_manager.save_collection(+ CollectionInfo(+ name="kb",+ embedding_model_id=SOURCE_MODEL,+ embedding_dimension=2,+ embeddings=4,+ )+ )+ )+++_SCHEMA = pa.schema(+ [+ ("collection", pa.string()),+ ("doc_id", pa.string()),+ ("chunk_id", pa.string()),+ ("parse_hash", pa.string()),+ ("user_id", pa.int64()),+ ("text", pa.string()),+ ("chunk_hash", pa.string()),+ ("metadata", pa.string()),+ ("model", pa.string()),+ ("created_at", pa.timestamp("us")),+ ("vector", pa.list_(pa.float32(), 2)),+ ]+)+++def _row(chunk_id: str, *, model: str = SOURCE_MODEL) -> Dict[str, Any]:+ return {+ "collection": "kb",+ "doc_id": "d1",+ "chunk_id": chunk_id,+ "parse_hash": "p",+ "user_id": 7,+ "text": f"text {chunk_id}",+ "chunk_hash": f"h-{chunk_id}",+ "metadata": None,+ "model": model,+ "created_at": datetime(2026, 1, 1),+ "vector": [0.5, 0.5],+ }+++def _table(name: str, rows: List[Dict[str, Any]]) -> None:+ _connection().create_table(name, data=pa.Table.from_pylist(rows, schema=_SCHEMA))+++def _rows_in(name: str) -> List[Dict[str, Any]]:+ conn = _connection()+ if name not in set(list_table_names(conn)):+ return []+ return conn.open_table(name).search().where("true").to_arrow().to_pylist()+++def _source(rows: List[Dict[str, Any]]) -> None:+ _table(written_embeddings_table_name(SOURCE_MODEL), rows)+++def test_the_documents_cascade_sweeps_a_table_a_writer_created_meanwhile(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from xagent.core.tools.core.RAG_tools.LanceDB import schema_manager+ from xagent.core.tools.core.RAG_tools.storage import lancedb_stores++ for name in (+ "ensure_documents_table",+ "ensure_parses_table",+ "ensure_chunks_table",+ "ensure_main_pointers_table",+ "ensure_ingestion_runs_table",+ ):+ monkeypatch.setattr(schema_manager, name, lambda *args, **kwargs: None)++ conn = _connection()+ core = pa.schema(+ [+ ("collection", pa.string()),+ ("doc_id", pa.string()),+ ("chunk_id", pa.string()),+ ("user_id", pa.int64()),+ ]+ )+ documents = pa.Table.from_pylist(+ [{"collection": "kb", "doc_id": "d1", "chunk_id": "", "user_id": 7}],+ schema=core,+ )+ conn.create_table("documents", data=documents)+ conn.create_table("chunks", data=documents)++ late = written_embeddings_table_name(OTHER_MODEL)+ real_delete = lancedb_stores._vis_delete_rows_by_filters+ created: List[str] = []++ def _delete_then_writer_commits(table: Any, filters: List[str]) -> int:+ count = real_delete(table, filters)+ if count and not created:+ # A writer that read these chunk rows before they went commits+ # into another model's table while the cascade runs.+ conn.create_table(+ late,+ data=pa.Table.from_pylist(+ [_row("c1", model=OTHER_MODEL)], schema=_SCHEMA+ ),+ )+ created.append(late)+ return count++ monkeypatch.setattr(+ lancedb_stores, "_vis_delete_rows_by_filters", _delete_then_writer_commits+ )++ lancedb_stores._vis_cascade_delete_documents(+ conn,+ collection="kb",+ doc_ids=["d1"],+ user_id=None,+ is_admin=True,+ preview_only=False,+ confirm=True,+ )++ assert created == [late]+ assert _rows_in(late) == []+++def _handle(collection: str = "kb") -> Any:+ from xagent.core.tools.core.RAG_tools.kb.collection_handle import (+ LanceDBCollectionHandle,+ )+ from xagent.core.tools.core.RAG_tools.kb.models import (+ KBAccessMode,+ KBBackendCapabilities,+ KBCollectionContext,+ KBStorageBackend,+ KBUserScope,+ )+ from xagent.core.tools.core.RAG_tools.storage.factory import (+ get_ingestion_status_store,+ get_main_pointer_store,+ get_metadata_store,+ get_vector_index_store,+ )++ context = KBCollectionContext(+ collection=collection,+ user_scope=KBUserScope(user_id=None, is_admin=True),+ access_mode=KBAccessMode.WRITE,+ allow_create=True,+ hide_missing=True,+ metadata_store=get_metadata_store(),+ vector_index_store=get_vector_index_store(),+ ingestion_status_store=get_ingestion_status_store(),+ main_pointer_store=get_main_pointer_store(),+ backend=KBStorageBackend.LANCEDB,+ capabilities=KBBackendCapabilities.lancedb(),+ collection_info=None,+ )+ return LanceDBCollectionHandle(context)+++def _seed_retirable_core() -> None:+ """A document re-ingested as C2: chunk ``c1`` is stale, ``c2`` is current."""+ from xagent.core.tools.core.RAG_tools.storage.factory import (+ get_vector_index_store,+ )+ from xagent.core.tools.core.RAG_tools.utils.metadata_utils import (+ serialize_metadata,+ )++ store = get_vector_index_store()+ store.upsert_documents(+ [+ {+ "collection": "kb",+ "doc_id": "d1",+ "file_id": None,+ "source_path": "/x.txt",+ "file_type": "txt",+ "content_hash": "C2",+ "uploaded_at": datetime.now(timezone.utc),+ "title": None,+ "language": None,+ "user_id": None,+ }+ ]+ )+ store.upsert_parses(+ [+ {+ "collection": "kb",+ "doc_id": "d1",+ "parse_hash": "p",+ "parser": "local:[email protected]",+ "created_at": datetime.now(timezone.utc),+ "params_json": json.dumps({"content_hash": "C2"}),+ "parsed_content": "[]",+ "user_id": None,+ }+ ]+ )+ store.upsert_chunks(+ [+ {+ "collection": "kb",+ "doc_id": "d1",+ "parse_hash": "p",+ "chunk_id": chunk_id,+ "index": 0,+ "text": chunk_id,+ "page_number": None,+ "section": None,+ "anchor": None,+ "json_path": None,+ "chunk_hash": "x",+ "config_hash": config_hash,+ "created_at": datetime.now(timezone.utc),+ "metadata": serialize_metadata({"content_hash": made_from}),+ "user_id": None,+ }+ for chunk_id, config_hash, made_from in (+ ("c1", "cfg-old", "C1"),+ ("c2", "cfg-new", "C2"),+ )+ ]+ )+++def _store() -> Any:+ from xagent.core.tools.core.RAG_tools.storage.factory import (+ get_vector_index_store,+ )++ return get_vector_index_store()+++def _emb(doc_id: str, chunk_id: str, model: str = "m1") -> Dict[str, Any]:+ return {+ "collection": "kb",+ "doc_id": doc_id,+ "chunk_id": chunk_id,+ "parse_hash": "p",+ "model": model,+ "vector": [0.1, 0.2],+ "vector_dimension": 2,+ "text": f"secret {chunk_id}",+ "chunk_hash": "h",+ "created_at": datetime.now(timezone.utc),+ "metadata": "{}",+ "user_id": None,+ }+++def _emb_rows(doc_id: str, table: str = "embeddings_m1") -> List[str]:+ conn = _connection()+ return sorted(+ row["chunk_id"]+ for row in conn.open_table(table)+ .search()+ .where(f"doc_id = '{doc_id}'")+ .to_arrow()+ .to_pylist()+ )+++def _doc_rows(doc_id: str) -> int:+ conn = _connection()+ return conn.open_table("documents").count_rows(f"doc_id = '{doc_id}'")+++def _chunk_rows(doc_id: str) -> int:+ conn = _connection()+ return conn.open_table("chunks").count_rows(f"doc_id = '{doc_id}'")+++def test_a_document_delete_that_fails_on_embeddings_can_be_retried(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from xagent.core.tools.core.RAG_tools.management import collections as mc+ from xagent.core.tools.core.RAG_tools.storage import lancedb_stores++ _save_collection()+ _seed_retirable_core()+ store = _store()+ store.upsert_embeddings("m1", [_emb("d1", "c1"), _emb("d1", "c2")])+ real = lancedb_stores._vis_delete_rows_by_filters+ failed = {"once": False}++ def _fail_embeddings(table: Any, filters: List[str]) -> int:+ if getattr(table, "name", "").startswith("embeddings_") and not failed["once"]:+ failed["once"] = True+ raise OSError("injected embedding delete failure")+ return real(table, filters)++ monkeypatch.setattr(lancedb_stores, "_vis_delete_rows_by_filters", _fail_embeddings)++ with pytest.raises(Exception):+ store.delete_documents_data("kb", ["d1"], None, True)++ # Vectors go first: the failed attempt left every core row, document+ # included, so nothing a retry reads is gone.+ assert _doc_rows("d1") == 1+ assert _chunk_rows("d1") == 2+ assert _emb_rows("d1") == ["c1", "c2"]+ monkeypatch.setattr(lancedb_stores, "_vis_delete_rows_by_filters", real)++ result = mc.delete_document("kb", "d1", 1, is_admin=True)++ assert result.status == "success", result.message+ assert _doc_rows("d1") == 0+ assert _emb_rows("d1") == []+++def test_a_retirement_that_fails_on_vectors_can_be_retried(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from xagent.core.tools.core.RAG_tools.kb.collection_handle import (+ LanceDBCollectionHandle,+ )++ _save_collection()+ _seed_retirable_core()+ _store().upsert_embeddings("m1", [_emb("d1", "c1"), _emb("d1", "c2")])+ real = LanceDBCollectionHandle.delete_embedding_records+ failed = {"once": False}++ def _fail_once(handle: Any, doc_id: str, **kwargs: Any) -> int:+ if not failed["once"]:+ failed["once"] = True+ raise OSError("injected")+ return real(handle, doc_id, **kwargs)++ monkeypatch.setattr(LanceDBCollectionHandle, "delete_embedding_records", _fail_once)++ with pytest.raises(OSError):+ _handle().retire_superseded_chunks("d1", "p")++ # The chunk rows are still there, so the retry can find the stale chunk.+ assert _handle().stale_chunks("d1", "p") == {("p", "cfg-old"): ["c1"]}+ assert _emb_rows("d1") == ["c1", "c2"]++ assert _handle().retire_superseded_chunks("d1", "p") == ["c1"]+ assert "c1" not in _emb_rows("d1")+++def test_a_retirement_sweeps_a_table_a_writer_created_meanwhile(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from xagent.core.tools.core.RAG_tools.kb.collection_handle import (+ LanceDBCollectionHandle,+ )++ _save_collection()+ _seed_retirable_core()+ _source([_row("c1"), _row("c2")])+ late = written_embeddings_table_name(OTHER_MODEL)+ real_core = LanceDBCollectionHandle.delete_chunk_records+ created: List[str] = []++ def _writer_commits_then_core_goes(handle: Any, doc_id: str, **kwargs: Any) -> int:+ if not created:+ _table(late, [_row("c1", model=OTHER_MODEL)])+ created.append(late)+ return real_core(handle, doc_id, **kwargs)++ monkeypatch.setattr(+ LanceDBCollectionHandle, "delete_chunk_records", _writer_commits_then_core_goes+ )++ assert _handle().retire_superseded_chunks("d1", "p") == ["c1"]++ assert created == [late]+ assert _rows_in(late) == []+ assert {+ row["chunk_id"] for row in _rows_in(written_embeddings_table_name(SOURCE_MODEL))+ } == {"c2"}+++def test_a_delete_reports_no_success_when_the_final_listing_fails(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from xagent.core.tools.core.RAG_tools.management import collections as mc+ from xagent.core.tools.core.RAG_tools.storage import lancedb_cleanup_filters++ _save_collection()+ _seed_retirable_core()+ store = _store()+ store.upsert_embeddings("m1", [_emb("d1", "c1"), _emb("d1", "c2")])+ real = lancedb_cleanup_filters.select_embedding_tables+ calls = {"n": 0}++ def _flaky(conn: Any, model_tag: Optional[str] = None) -> List[str]:+ calls["n"] += 1+ if calls["n"] == 2:+ raise OSError("listing failed")+ return real(conn, model_tag=model_tag)++ monkeypatch.setattr(lancedb_cleanup_filters, "select_embedding_tables", _flaky)++ with pytest.raises(Exception):+ store.delete_documents_data("kb", ["d1"], None, True)++ assert calls["n"] == 2+ monkeypatch.setattr(lancedb_cleanup_filters, "select_embedding_tables", real)++ # Not reported as a success, and still deletable through the document's row.+ assert _doc_rows("d1") == 1+ result = mc.delete_document("kb", "d1", 1, is_admin=True)+ assert result.status == "success", result.message+ assert _doc_rows("d1") == 0+ assert _emb_rows("d1") == []+++def test_the_generic_cascade_sweeps_a_table_a_writer_created_meanwhile(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from xagent.core.tools.core.RAG_tools.storage import lancedb_stores++ _save_collection()+ _seed_retirable_core()+ store = _store()+ store.upsert_embeddings("m1", [_emb("d1", "c1")])+ late = written_embeddings_table_name(OTHER_MODEL)+ real = lancedb_stores._vis_delete_rows_by_filters+ created: List[str] = []++ def _writer_commits_during_the_core_delete(table: Any, filters: List[str]) -> int:+ if not created and getattr(table, "name", "") == "chunks":+ # A writer that read the chunk rows before they went creates its+ # table while this delete runs.+ _connection().create_table(+ late,+ data=pa.Table.from_pylist(+ [_row("c1", model=OTHER_MODEL)], schema=_SCHEMA+ ),+ )+ created.append(late)+ return real(table, filters)++ monkeypatch.setattr(+ lancedb_stores,+ "_vis_delete_rows_by_filters",+ _writer_commits_during_the_core_delete,+ )++ result = store.cascade_delete(+ target="document",+ collection="kb",+ doc_id="d1",+ is_admin=True,+ preview_only=False,+ confirm=True,+ )++ assert created == [late]+ assert _rows_in(late) == []+ assert result["documents"] == 1++[email protected]("scope_name", ["chunk", "parse"])+def test_a_version_cleanup_sweeps_a_table_a_writer_created_meanwhile(+ monkeypatch: pytest.MonkeyPatch, scope_name: str+) -> None:+ from xagent.core.tools.core.RAG_tools.storage import lancedb_stores++ _save_collection()+ _seed_retirable_core()+ _source([_row("c1"), _row("c2")])+ late = written_embeddings_table_name(OTHER_MODEL)+ real = lancedb_stores._vis_delete_rows_by_filters+ created: List[str] = []++ def _writer_commits_during_the_core_delete(table: Any, filters: List[str]) -> int:+ if not created and getattr(table, "name", "") == "chunks":+ _connection().create_table(+ late,+ data=pa.Table.from_pylist(+ [_row("c1", model=OTHER_MODEL)], schema=_SCHEMA+ ),+ )+ created.append(late)+ return real(table, filters)++ monkeypatch.setattr(+ lancedb_stores,+ "_vis_delete_rows_by_filters",+ _writer_commits_during_the_core_delete,+ )++ _store().cleanup_cascade_by_scope(+ "kb",+ "d1",+ scope_name,+ new_parse_hash="p-new",+ old_parse_hash="p",+ preview_only=False,+ confirm=True,+ )++ assert created == [late]+ assert _chunk_rows("d1") == 0+ assert _rows_in(late) == []+++def test_a_cascade_whose_count_fails_deletes_nothing_and_raises(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from xagent.core.tools.core.RAG_tools.utils import lancedb_query_utils++ _save_collection()+ _seed_retirable_core()+ _store().upsert_embeddings("m1", [_emb("d1", "c1"), _emb("d1", "c2")])+ real = lancedb_query_utils._safe_count_rows++ class _Unreadable:+ def count_rows(self, *args: Any) -> int:+ raise OSError("count failed")++ def _embeddings_count_fails(+ table: Any, filter_expr: Optional[str] = None, **kwargs: Any+ ) -> int:+ if str(getattr(table, "name", "")).startswith("embeddings_"):+ table = _Unreadable()+ return real(table, filter_expr, **kwargs)++ monkeypatch.setattr(+ lancedb_query_utils, "_safe_count_rows", _embeddings_count_fails+ )++ with pytest.raises(OSError):+ _store().cascade_delete(+ target="document",+ collection="kb",+ doc_id="d1",+ is_admin=True,+ preview_only=False,+ confirm=True,+ )++ assert _emb_rows("d1") == ["c1", "c2"]+ assert _chunk_rows("d1") == 2+ assert _doc_rows("d1") == 1+++def test_a_failed_core_listing_fails_the_document_cascade_before_any_delete(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from xagent.core.tools.core.RAG_tools.storage import lancedb_stores+ from xagent.core.tools.core.RAG_tools.utils import lancedb_query_utils++ _save_collection()+ _seed_retirable_core()+ store = _store()+ store.upsert_embeddings("m1", [_emb("d1", "c1"), _emb("d1", "c2")])+ real_listing = lancedb_query_utils.list_table_names+ real_cascade = lancedb_stores._vis_cascade_delete_documents+ armed = {"fail": False}++ def _listing(conn: Any) -> List[str]:+ if armed["fail"]:+ armed["fail"] = False+ raise OSError("listing failed")+ return real_listing(conn)++ def _cascade(*args: Any, **kwargs: Any) -> Dict[str, int]:+ armed["fail"] = True+ return real_cascade(*args, **kwargs)++ monkeypatch.setattr(lancedb_query_utils, "list_table_names", _listing)+ monkeypatch.setattr(lancedb_stores, "_vis_cascade_delete_documents", _cascade)++ with pytest.raises(Exception):+ store.delete_documents_data("kb", ["d1"], None, True)++ assert armed["fail"] is False+ assert _emb_rows("d1") == ["c1", "c2"]+ assert _chunk_rows("d1") == 2+ assert _doc_rows("d1") == 1+++def _fail_listing_call(monkeypatch: pytest.MonkeyPatch, failing: int) -> Dict[str, int]:+ from xagent.core.tools.core.RAG_tools.utils import lancedb_query_utils++ real = lancedb_query_utils.list_table_names+ calls = {"n": 0}++ def _listing(conn: Any) -> List[str]:+ calls["n"] += 1+ if calls["n"] == failing:+ raise OSError("listing failed")+ return real(conn)++ monkeypatch.setattr(lancedb_query_utils, "list_table_names", _listing)+ return calls+++ ("target", "failing"),+ [("collection", 1), ("document", 2)],+ ids=["collection-builder-listing", "document-executor-listing"],+)+def test_a_failed_listing_fails_the_cascade_before_any_delete(+ monkeypatch: pytest.MonkeyPatch, target: str, failing: int+) -> None:+ _save_collection()+ _seed_retirable_core()+ store = _store()+ store.upsert_embeddings("m1", [_emb("d1", "c1"), _emb("d1", "c2")])+ calls = _fail_listing_call(monkeypatch, failing)++ with pytest.raises(OSError):+ store.cascade_delete(+ target=target,+ collection="kb",+ doc_id="d1",+ is_admin=True,+ preview_only=False,+ confirm=True,+ )++ assert calls["n"] == failing+ assert _emb_rows("d1") == ["c1", "c2"]+ assert _chunk_rows("d1") == 2+ assert _doc_rows("d1") == 1++[email protected]("delete", ["document-cascade", "parse-cleanup"])+def test_a_model_scoped_delete_keeps_the_other_models_vectors(delete: str) -> None:+ _save_collection()+ _seed_retirable_core()+ store = _store()+ store.upsert_embeddings("m1", [_emb("d1", "c1")])+ store.upsert_embeddings("m2", [_emb("d1", "c1", model="m2")])++ if delete == "document-cascade":+ store.cascade_delete(+ target="document",+ collection="kb",+ doc_id="d1",+ is_admin=True,+ model_tag="m1",+ preview_only=False,+ confirm=True,+ )+ else:+ store.cleanup_cascade_by_scope(+ "kb",+ "d1",+ "parse",+ new_parse_hash="p-new",+ old_parse_hash="p",+ model_tag="m1",+ preview_only=False,+ confirm=True,+ )++ assert _chunk_rows("d1") == 0+ assert _emb_rows("d1") == []+ assert _emb_rows("d1", "embeddings_m2") == ["c1"]diff --git a/tests/core/tools/core/RAG_tools/storage/test_lancedb_store_version.py b/tests/core/tools/core/RAG_tools/storage/test_lancedb_store_version.pyindex c0e7f0ffb..451921a6b 100644--- a/tests/core/tools/core/RAG_tools/storage/test_lancedb_store_version.py+++ b/tests/core/tools/core/RAG_tools/storage/test_lancedb_store_version.py@@ -407,3 +407,57 @@ class TestCleanupCascadeByScope:))assert sync_result == async_result++ def test_embedding_tables_are_swept_before_and_after_the_chunk_delete(+ self, isolated_store, monkeypatch+ ):+ """A failure keeps the chunk rows; the sweep after them catches a late table."""+ from xagent.core.tools.core.RAG_tools.storage import lancedb_stores++ store = isolated_store+ collection, doc_id, parse_hash = "cc_order", "cc_order_doc", "orderhash"+ store.upsert_parses([_make_parse_row(collection, doc_id, parse_hash)])+ store.upsert_chunks([_make_chunk_row(collection, doc_id, parse_hash, "ck1")])+ store.upsert_embeddings(+ "m1",+ [+ {+ "collection": collection,+ "doc_id": doc_id,+ "chunk_id": "ck1",+ "parse_hash": parse_hash,+ "model": "m1",+ "vector": [0.1, 0.2],+ "vector_dimension": 2,+ "text": "ck1",+ "chunk_hash": "ch_ck1",+ "created_at": datetime.now(timezone.utc),+ "metadata": "{}",+ "user_id": None,+ }+ ],+ )++ order = []+ real = lancedb_stores._vis_delete_rows_by_filters++ def _recording(table, filter_exprs):+ order.append(getattr(table, "name", ""))+ return real(table, filter_exprs)++ monkeypatch.setattr(lancedb_stores, "_vis_delete_rows_by_filters", _recording)++ result = store.cleanup_cascade_by_scope(+ collection,+ doc_id,+ "parse",+ old_parse_hash=parse_hash,+ preview_only=False,+ confirm=True,+ )++ assert result["chunks"] == 1+ sweeps = [i for i, name in enumerate(order) if name.startswith("embeddings_")]+ chunks_at = order.index("chunks")+ assert sweeps[0] < chunks_at+ assert chunks_at < sweeps[-1]diff --git a/tests/core/tools/core/RAG_tools/version_management/test_cascade_cleaner.py b/tests/core/tools/core/RAG_tools/version_management/test_cascade_cleaner.pyindex 4d3e53665..b89a5273a 100644--- a/tests/core/tools/core/RAG_tools/version_management/test_cascade_cleaner.py+++ b/tests/core/tools/core/RAG_tools/version_management/test_cascade_cleaner.py@@ -604,15 +604,14 @@ def test_cleanup_embed_cascade_preserves_multiple_predicates_per_table() -> Nonewith (patch.object(LanceDBVectorIndexStore, "_get_connection") as mock_get_conn,patch(- "xagent.core.tools.core.RAG_tools.storage.lancedb_cleanup_filters.build_embedding_cleanup_filters"+ "xagent.core.tools.core.RAG_tools.storage.lancedb_cleanup_filters.embedding_cleanup_filters_for_table") as mock_build_filters,patch(f"{_STORE}._vis_plan_by_predicates") as mock_plan,):conn = MagicMock(spec=["table_names", "open_table"])+ conn.table_names.return_value = ["embeddings_m1"]mock_get_conn.return_value = conn- mock_build_filters.return_value = {- "embeddings_m1": ["chunk_id == 'ch1'", "chunk_id == 'ch2'"],- }+ mock_build_filters.return_value = ["chunk_id == 'ch1'", "chunk_id == 'ch2'"]mock_plan.return_value = {"embeddings_m1": 2}result = get_vector_index_store().cleanup_cascade_by_scope(@@ -636,7 +635,7 @@ def test_cleanup_embed_cascade_deletes_multiple_predicates_per_table() -> None:with (patch.object(LanceDBVectorIndexStore, "_get_connection") as mock_get_conn,patch(- "xagent.core.tools.core.RAG_tools.storage.lancedb_cleanup_filters.build_embedding_cleanup_filters"+ "xagent.core.tools.core.RAG_tools.storage.lancedb_cleanup_filters.embedding_cleanup_filters_for_table") as mock_build_filters,):table = MagicMock()@@ -645,9 +644,7 @@ def test_cleanup_embed_cascade_deletes_multiple_predicates_per_table() -> None:conn.table_names.return_value = ["embeddings_m1"]conn.open_table.return_value = tablemock_get_conn.return_value = conn- mock_build_filters.return_value = {- "embeddings_m1": ["chunk_id == 'ch1'", "chunk_id == 'ch2'"],- }+ mock_build_filters.return_value = ["chunk_id == 'ch1'", "chunk_id == 'ch2'"]result = get_vector_index_store().cleanup_cascade_by_scope("c_multi",@@ -787,14 +784,15 @@ def test_cleanup_cascade_plans_unless_confirmed_outside_preview(with (patch.object(LanceDBVectorIndexStore, "_get_connection") as mock_get_conn,patch(- "xagent.core.tools.core.RAG_tools.storage.lancedb_cleanup_filters.build_embedding_cleanup_filters"+ "xagent.core.tools.core.RAG_tools.storage.lancedb_cleanup_filters.embedding_cleanup_filters_for_table") as mock_build_filters,patch(f"{_STORE}._vis_plan_by_predicates") as mock_plan,patch(f"{_STORE}._vis_delete_by_predicates") as mock_delete,):conn = MagicMock(spec=["table_names", "open_table"])+ conn.table_names.return_value = ["embeddings_m1"]mock_get_conn.return_value = conn- mock_build_filters.return_value = {"embeddings_m1": ["collection == 'c_gate'"]}+ mock_build_filters.return_value = ["collection == 'c_gate'"]mock_plan.return_value = {"embeddings_m1": 1}result = get_vector_index_store().cleanup_cascade_by_scope(@@ -810,3 +808,17 @@ def test_cleanup_cascade_plans_unless_confirmed_outside_preview(assert result == {"embeddings": 1}mock_plan.assert_called_once()mock_delete.assert_not_called()++[email protected]("key", ["embeddings_m1", "__embeddings__"])+def test_a_cascade_refuses_embedding_predicates_built_ahead_of_the_delete(+ key: str,+) -> None:+ from xagent.core.tools.core.RAG_tools.storage import lancedb_stores++ conn = MagicMock(spec=["table_names", "open_table"])+ with pytest.raises(ValueError):+ lancedb_stores._vis_execute_or_plan_by_predicates(+ conn, {key: ["true"]}, preview_only=False, confirm=True+ )+ conn.open_table.assert_not_called()