Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

fix(kb): sweep embedding rows again after chunk rows are deleted (#3823)

蓝鱼bluefish4f21db067cdb
diff --git a/src/xagent/core/tools/core/RAG_tools/kb/collection_handle.py b/src/xagent/core/tools/core/RAG_tools/kb/collection_handle.py
index 46dca4315..e03ffd5ca 100644
--- a/src/xagent/core/tools/core/RAG_tools/kb/collection_handle.py
+++ b/src/xagent/core/tools/core/RAG_tools/kb/collection_handle.py
@@ -2620,16 +2620,23 @@ class LanceDBCollectionHandle(KBCollectionHandle):
for (stale_parse, config_hash), ids in self.stale_chunks(
doc_id, parse_hash, user_id=user_id
).items():
- for start in range(0, len(ids), size):
- self.delete_embedding_records(
- doc_id, chunk_ids=ids[start : start + size], is_admin=True
- )
+ # Vectors first: a failure here leaves the chunk rows a retry reads.
+ self._delete_stale_vectors(doc_id, ids, size)
self.delete_chunk_records(
doc_id, parse_hash=stale_parse, config_hash=config_hash, is_admin=True
)
+ # Again after the chunk rows: a vector committed after the first
+ # delete would otherwise outlive its chunk.
+ self._delete_stale_vectors(doc_id, ids, size)
deleted += ids
return deleted
+ def _delete_stale_vectors(self, doc_id: str, ids: list[str], size: int) -> None:
+ for start in range(0, len(ids), size):
+ self.delete_embedding_records(
+ doc_id, chunk_ids=ids[start : start + size], is_admin=True
+ )
+
def discard_uncommitted_embeddings(
self, doc_id: str, *, user_id: int | None = None
) -> int:
diff --git a/src/xagent/core/tools/core/RAG_tools/storage/lancedb_cleanup_filters.py b/src/xagent/core/tools/core/RAG_tools/storage/lancedb_cleanup_filters.py
index 73b5c07cd..58c6e4a58 100644
--- a/src/xagent/core/tools/core/RAG_tools/storage/lancedb_cleanup_filters.py
+++ b/src/xagent/core/tools/core/RAG_tools/storage/lancedb_cleanup_filters.py
@@ -33,52 +33,56 @@ def build_embedding_cleanup_filters(
scope: KBCleanupScope,
) -> dict[str, list[str]]:
"""Build per-embeddings-table cleanup predicates for a resolved scope."""
- table_filters: dict[str, list[str]] = {}
- for table_name in select_embedding_tables(conn, model_tag=scope.model_tag):
- table = None
- try:
- table = conn.open_table(table_name)
- table_filters[table_name] = _build_filters_for_table(table, scope)
- except Exception:
- table_filters[table_name] = _build_filters_without_schema(scope)
- finally:
- _safe_close_table(table)
- return table_filters
-
-
-def build_embedding_cleanup_filters_from_base(
+ return {
+ table_name: embedding_cleanup_filters_for_table(conn, table_name, scope)
+ for table_name in select_embedding_tables(conn, model_tag=scope.model_tag)
+ }
+
+
+def embedding_cleanup_filters_for_table(
+ conn: Any, table_name: str, scope: KBCleanupScope
+) -> list[str]:
+ """The cleanup predicates of a resolved scope for one embeddings table."""
+ table = None
+ try:
+ table = conn.open_table(table_name)
+ return _build_filters_for_table(table, scope)
+ except Exception:
+ return _build_filters_without_schema(scope)
+ finally:
+ _safe_close_table(table)
+
+
+def embedding_cleanup_filters_from_base(
conn: Any,
+ table_name: str,
*,
base_filter: str,
user_id: Optional[int],
is_admin: bool,
- model_tag: Optional[str] = None,
-) -> dict[str, list[str]]:
- """Build per-embeddings-table predicate lists from an existing base filter."""
- table_filters: dict[str, list[str]] = {}
- for table_name in select_embedding_tables(conn, model_tag=model_tag):
- table = None
- try:
- table = conn.open_table(table_name)
- table_filters[table_name] = [
- append_user_filter_for_table(
- table=table,
- filter_expr=base_filter,
- user_id=user_id,
- is_admin=is_admin,
- )
- ]
- except Exception:
- table_filters[table_name] = [
- append_user_filter_without_schema(
- filter_expr=base_filter,
- user_id=user_id,
- is_admin=is_admin,
- )
- ]
- finally:
- _safe_close_table(table)
- return table_filters
+) -> list[str]:
+ """One embeddings table's predicates from an existing base filter."""
+ table = None
+ try:
+ table = conn.open_table(table_name)
+ return [
+ append_user_filter_for_table(
+ table=table,
+ filter_expr=base_filter,
+ user_id=user_id,
+ is_admin=is_admin,
+ )
+ ]
+ except Exception:
+ return [
+ append_user_filter_without_schema(
+ filter_expr=base_filter,
+ user_id=user_id,
+ is_admin=is_admin,
+ )
+ ]
+ finally:
+ _safe_close_table(table)
def append_user_filter_for_table(
diff --git a/src/xagent/core/tools/core/RAG_tools/storage/lancedb_stores.py b/src/xagent/core/tools/core/RAG_tools/storage/lancedb_stores.py
index 93b4e83df..e9ecc93c5 100644
--- a/src/xagent/core/tools/core/RAG_tools/storage/lancedb_stores.py
+++ b/src/xagent/core/tools/core/RAG_tools/storage/lancedb_stores.py
@@ -11,7 +11,18 @@ from collections.abc import Iterable
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from enum import Enum
-from typing import Any, Dict, Iterator, List, Literal, Optional, Sequence, Tuple, cast
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ Iterator,
+ List,
+ Literal,
+ Optional,
+ Sequence,
+ Tuple,
+ cast,
+)
import pyarrow as pa # type: ignore
from filelock import FileLock, Timeout
@@ -3108,7 +3119,10 @@ class LanceDBVectorIndexStore(VectorIndexStore):
from ..version_management.main_pointer_manager import (
_get_main_pointer_impl as get_main_pointer,
)
- from .lancedb_cleanup_filters import build_embedding_cleanup_filters
+ from .lancedb_cleanup_filters import (
+ embedding_cleanup_filters_for_table,
+ embedding_cleanup_filters_from_base,
+ )
conn = self._get_connection()
ensure_documents_table(conn)
@@ -3140,6 +3154,8 @@ class LanceDBVectorIndexStore(VectorIndexStore):
}
predicates: Dict[str, list] = {}
+ embedding_filters: Optional[Callable[[str], list]] = None
+ embedding_base: Optional[str] = None
if scope == "parse":
if old_parse_hash is None:
@@ -3160,14 +3176,7 @@ class LanceDBVectorIndexStore(VectorIndexStore):
is_admin=is_admin,
skip_user_filter=True,
)
- _vis_replace_embedding_predicates(
- predicates=predicates,
- conn=conn,
- base_expr=base,
- user_id=user_id,
- is_admin=is_admin,
- model_tag=model_tag,
- )
+ embedding_base = base
predicates["chunks"] = [
_vis_append_user_filter_if_needed(
conn=conn,
@@ -3188,14 +3197,7 @@ class LanceDBVectorIndexStore(VectorIndexStore):
f"doc_id == '{escaped_doc_id}' AND "
f"parse_hash != '{escaped_new_parse_hash}'"
)
- _vis_replace_embedding_predicates(
- predicates=predicates,
- conn=conn,
- base_expr=other,
- user_id=user_id,
- is_admin=is_admin,
- model_tag=model_tag,
- )
+ embedding_base = other
predicates["chunks"] = [
_vis_append_user_filter_if_needed(
conn=conn,
@@ -3232,14 +3234,7 @@ class LanceDBVectorIndexStore(VectorIndexStore):
is_admin=is_admin,
skip_user_filter=True,
)
- _vis_replace_embedding_predicates(
- predicates=predicates,
- conn=conn,
- base_expr=base,
- user_id=user_id,
- is_admin=is_admin,
- model_tag=model_tag,
- )
+ embedding_base = base
if new_parse_hash:
from ..utils.string_utils import escape_lancedb_string
@@ -3251,14 +3246,7 @@ class LanceDBVectorIndexStore(VectorIndexStore):
f"doc_id == '{escaped_doc_id}' AND "
f"parse_hash != '{escaped_parse_hash}'"
)
- _vis_replace_embedding_predicates(
- predicates=predicates,
- conn=conn,
- base_expr=other,
- user_id=user_id,
- is_admin=is_admin,
- model_tag=model_tag,
- )
+ embedding_base = other
predicates["chunks"] = [
_vis_append_user_filter_if_needed(
conn=conn,
@@ -3276,10 +3264,11 @@ class LanceDBVectorIndexStore(VectorIndexStore):
is_admin=is_admin,
model_tag=model_tag,
)
- filters_by_table = build_embedding_cleanup_filters(conn, scope_obj)
- for table_name, filter_exprs in filters_by_table.items():
- if filter_exprs:
- predicates.setdefault(table_name, []).extend(filter_exprs)
+
+ def _scope_filters(table_name: str) -> list:
+ return embedding_cleanup_filters_for_table(conn, table_name, scope_obj)
+
+ embedding_filters = _scope_filters
elif scope == "pointers":
filt = _vis_build_document_filter(
conn=conn,
@@ -3293,6 +3282,20 @@ class LanceDBVectorIndexStore(VectorIndexStore):
else:
raise CascadeCleanupError(f"Unsupported scope: {scope}")
+ if embedding_base is not None:
+ base_expr = embedding_base
+
+ def _base_filters(table_name: str) -> list:
+ return embedding_cleanup_filters_from_base(
+ conn,
+ table_name,
+ base_filter=base_expr,
+ user_id=user_id,
+ is_admin=is_admin,
+ )
+
+ embedding_filters = _base_filters
+
try:
result = _vis_execute_or_plan_by_predicates(
conn,
@@ -3300,6 +3303,7 @@ class LanceDBVectorIndexStore(VectorIndexStore):
preview_only=preview_only,
confirm=confirm,
model_tag=model_tag,
+ embedding_filters=embedding_filters,
)
finally:
# Earlier tables may already be deleted when a later one raises.
@@ -3329,8 +3333,8 @@ class LanceDBVectorIndexStore(VectorIndexStore):
ensure_main_pointers_table,
ensure_parses_table,
)
+ from ..utils.lancedb_query_utils import list_table_names
from ..utils.user_scope import resolve_user_scope
- from .lancedb_cleanup_filters import select_embedding_tables
user_scope = resolve_user_scope(user_id=user_id, is_admin=is_admin)
user_id = user_scope.user_id
@@ -3346,7 +3350,7 @@ class LanceDBVectorIndexStore(VectorIndexStore):
ensure_main_pointers_table(conn)
ensure_ingestion_runs_table(conn)
- table_names = _vis_get_table_names(conn)
+ table_names = list(list_table_names(conn))
predicates: Dict[str, list] = {}
core_tables = [
@@ -3378,17 +3382,19 @@ class LanceDBVectorIndexStore(VectorIndexStore):
)
predicates[table_name] = [filter_expr]
- for table_name in select_embedding_tables(conn, model_tag=model_tag):
+ def _embedding_filter(table_name: str) -> list:
if target == "collection":
- filter_expr = _vis_build_collection_filter(
- conn=conn,
- table_name=table_name,
- collection=collection,
- user_id=user_id,
- is_admin=is_admin,
- )
- else:
- filter_expr = _vis_build_document_filter(
+ return [
+ _vis_build_collection_filter(
+ conn=conn,
+ table_name=table_name,
+ collection=collection,
+ user_id=user_id,
+ is_admin=is_admin,
+ )
+ ]
+ return [
+ _vis_build_document_filter(
conn=conn,
table_name=table_name,
collection=collection,
@@ -3396,7 +3402,7 @@ class LanceDBVectorIndexStore(VectorIndexStore):
user_id=user_id,
is_admin=is_admin,
)
- predicates[table_name] = [filter_expr]
+ ]
try:
result = _vis_execute_or_plan_by_predicates(
@@ -3404,7 +3410,8 @@ class LanceDBVectorIndexStore(VectorIndexStore):
predicates,
preview_only=preview_only,
confirm=confirm,
- model_tag=None,
+ model_tag=model_tag,
+ embedding_filters=_embedding_filter,
)
finally:
# Earlier tables may already be deleted when a later one raises.
@@ -3433,7 +3440,8 @@ def _vis_delete_rows_by_filters(table: Any, filter_exprs: list) -> int:
deleted_count = 0
for filter_expr in filter_exprs:
- count = _safe_count_rows(table, filter_expr)
+ # A count read as zero would skip this delete and report it done.
+ count = _safe_count_rows(table, filter_expr, on_error="raise")
if count > 0:
table.delete(filter_expr)
deleted_count += count
@@ -3473,29 +3481,6 @@ def _vis_append_user_filter_if_needed(
_safe_close_table(table)
-def _vis_replace_embedding_predicates(
- *,
- predicates: Dict[str, list],
- conn: Any,
- base_expr: str,
- user_id: Optional[int],
- is_admin: bool,
- model_tag: Optional[str] = None,
-) -> None:
- from .lancedb_cleanup_filters import build_embedding_cleanup_filters_from_base
-
- table_filters = build_embedding_cleanup_filters_from_base(
- conn,
- base_filter=base_expr,
- user_id=user_id,
- is_admin=is_admin,
- model_tag=model_tag,
- )
- for table_name, filter_exprs in table_filters.items():
- if filter_exprs:
- predicates[table_name] = list(filter_exprs)
-
-
def _vis_get_table_names(conn: Any) -> list:
from ..utils.lancedb_query_utils import list_table_names
@@ -3506,10 +3491,9 @@ def _vis_get_table_names(conn: Any) -> list:
def _vis_plan_by_predicates(
- conn: Any, table_to_filter: Dict[str, list], model_tag: Optional[str] = None
+ conn: Any, table_to_filter: Dict[str, list]
) -> Dict[str, int]:
from ..LanceDB.schema_manager import _safe_close_table
- from .lancedb_cleanup_filters import select_embedding_tables
counts: Dict[str, int] = {}
table_names = _vis_get_table_names(conn)
@@ -3524,18 +3508,6 @@ def _vis_plan_by_predicates(
_safe_close_table(table)
for table_name, filter_exprs in table_to_filter.items():
- if table_name == "__embeddings__":
- total = 0
- target_tables = select_embedding_tables(conn, model_tag=model_tag)
- for t in target_tables:
- table = None
- try:
- table = conn.open_table(t)
- total += _vis_count_rows_by_filters(table, filter_exprs)
- finally:
- _safe_close_table(table)
- counts[table_name] = total
- continue
if table_name.startswith("embeddings_"):
continue
if table_name not in table_names:
@@ -3550,83 +3522,57 @@ def _vis_plan_by_predicates(
return counts
-def _vis_delete_by_predicates(
- conn: Any, table_to_filter: Dict[str, list], model_tag: Optional[str] = None
-) -> Dict[str, int]:
+def _vis_embedding_pass(
+ conn: Any,
+ *,
+ model_tag: Optional[str],
+ embedding_filters: Callable[[str], list],
+ deleted: Dict[str, int],
+) -> None:
+ """The listing raises: a swallowed failure would report a cascade complete
+ with its embedding rows still searchable."""
import logging as _logging
from ..LanceDB.schema_manager import _safe_close_table
from .lancedb_cleanup_filters import select_embedding_tables
_logger = _logging.getLogger(__name__)
- deleted: Dict[str, int] = {}
- table_names = _vis_get_table_names(conn)
-
- for t in table_names:
- if not t.startswith("embeddings_") or t not in table_to_filter:
+ for t in select_embedding_tables(conn, model_tag=model_tag):
+ filter_exprs = embedding_filters(t)
+ if not filter_exprs:
continue
- filter_exprs = table_to_filter[t]
table = None
try:
table = conn.open_table(t)
cnt = _vis_delete_rows_by_filters(table, filter_exprs)
if cnt > 0:
_logger.info("Cascade cleanup: deleted %s rows from %s", cnt, t)
- deleted[t] = cnt
+ deleted[t] = deleted.get(t, 0) + cnt
finally:
_safe_close_table(table)
- order = [
- "__embeddings__",
- "chunks",
- "parses",
- "main_pointers",
- "ingestion_runs",
- "documents",
- ]
-
- if "__embeddings__" in table_to_filter:
- filter_exprs = table_to_filter["__embeddings__"]
- total = 0
- target_tables = select_embedding_tables(conn, model_tag=model_tag)
- for t in target_tables:
- table = None
- try:
- table = conn.open_table(t)
- total += _vis_delete_rows_by_filters(table, filter_exprs)
- finally:
- _safe_close_table(table)
- deleted["embeddings"] = total
- if total > 0:
- _logger.info(
- "Cascade cleanup: deleted %s rows from embeddings tables", total
- )
- for name in order[1:]:
- if name in table_to_filter and name in table_names:
- table = None
- try:
- table = conn.open_table(name)
- cnt = _vis_delete_rows_by_filters(table, table_to_filter[name])
- if cnt > 0:
- _logger.info("Cascade cleanup: deleted %s rows from %s", cnt, name)
- deleted[name] = cnt
- finally:
- _safe_close_table(table)
+def _vis_delete_by_predicates(
+ conn: Any,
+ table_to_filter: Dict[str, list],
+ model_tag: Optional[str] = None,
+ embedding_filters: Optional[Callable[[str], list]] = None,
+) -> Dict[str, int]:
+ import logging as _logging
- for name, filter_exprs in table_to_filter.items():
- if name in (
- "__embeddings__",
- "chunks",
- "parses",
- "main_pointers",
- "ingestion_runs",
- "documents",
- ) or name.startswith("embeddings_"):
- continue
- if name not in table_names:
- deleted[name] = 0
- continue
+ from ..LanceDB.schema_manager import _safe_close_table
+ from ..utils.lancedb_query_utils import list_table_names
+
+ _logger = _logging.getLogger(__name__)
+ deleted: Dict[str, int] = {}
+ order = ["chunks", "parses", "main_pointers", "ingestion_runs"]
+ # Documents last: its row is what a retried document delete looks up, so
+ # every failure before this point can still be retried whole.
+ late = ["documents"]
+ # A listing that fails must not shrink this delete to the embeddings.
+ table_names = list(list_table_names(conn))
+
+ def _delete(name: str, filter_exprs: list) -> None:
table = None
try:
table = conn.open_table(name)
@@ -3637,6 +3583,44 @@ def _vis_delete_by_predicates(
finally:
_safe_close_table(table)
+ # Embeddings first: a failure leaves every core row in place, so a retry
+ # finds the document or the stale chunks again.
+ if embedding_filters is not None:
+ _vis_embedding_pass(
+ conn,
+ model_tag=model_tag,
+ embedding_filters=embedding_filters,
+ deleted=deleted,
+ )
+
+ for name in order:
+ if name in table_to_filter and name in table_names:
+ _delete(name, table_to_filter[name])
+
+ for name, filter_exprs in table_to_filter.items():
+ if name in ("documents", *order):
+ continue
+ if name not in table_names:
+ deleted[name] = 0
+ continue
+ _delete(name, filter_exprs)
+
+ if embedding_filters is not None and any(
+ name in table_to_filter for name in (*order, *late)
+ ):
+ # The chunk rows are gone now: a fresh listing finds the embedding tables
+ # a writer created since the first pass, each with its own predicates.
+ _vis_embedding_pass(
+ conn,
+ model_tag=model_tag,
+ embedding_filters=embedding_filters,
+ deleted=deleted,
+ )
+
+ for name in late:
+ if name in table_to_filter and name in table_names:
+ _delete(name, table_to_filter[name])
+
return deleted
@@ -3647,17 +3631,31 @@ def _vis_execute_or_plan_by_predicates(
preview_only: bool,
confirm: bool,
model_tag: Optional[str] = None,
+ embedding_filters: Optional[Callable[[str], list]] = None,
) -> Dict[str, int]:
+ """Embedding tables come only from ``embedding_filters``: a predicate built from
+ an earlier listing misses a table a writer creates while the delete runs."""
+ from .lancedb_cleanup_filters import select_embedding_tables
+
+ if any(
+ name == "__embeddings__" or name.startswith("embeddings_")
+ for name in predicates
+ ):
+ raise ValueError("embedding tables are passed as embedding_filters")
if not (confirm and not preview_only):
- return _vis_plan_by_predicates(conn, predicates, model_tag=model_tag)
- return _vis_delete_by_predicates(conn, predicates, model_tag=model_tag)
+ plan = dict(predicates)
+ if embedding_filters is not None:
+ for table_name in select_embedding_tables(conn, model_tag=model_tag):
+ plan[table_name] = embedding_filters(table_name)
+ return _vis_plan_by_predicates(conn, plan)
+ return _vis_delete_by_predicates(
+ conn, predicates, model_tag=model_tag, embedding_filters=embedding_filters
+ )
def _vis_collapse_embedding_table_counts(counts: Dict[str, int]) -> Dict[str, int]:
collapsed = dict(counts)
- embeddings_total = int(collapsed.pop("__embeddings__", 0)) + int(
- collapsed.pop("embeddings", 0)
- )
+ embeddings_total = 0
for table_name in list(collapsed):
if str(table_name).startswith("embeddings_"):
embeddings_total += int(collapsed.pop(table_name, 0))
@@ -3866,8 +3864,8 @@ def _vis_cascade_delete_documents(
ensure_main_pointers_table,
ensure_parses_table,
)
+ from ..utils.lancedb_query_utils import list_table_names
from ..utils.user_scope import resolve_user_scope
- from .lancedb_cleanup_filters import select_embedding_tables
normalized_doc_ids = sorted({str(d) for d in doc_ids if d})
if not normalized_doc_ids:
@@ -3885,7 +3883,7 @@ def _vis_cascade_delete_documents(
ensure_main_pointers_table(conn)
ensure_ingestion_runs_table(conn)
- table_names = _vis_get_table_names(conn)
+ table_names = list(list_table_names(conn))
predicates: Dict[str, list] = {}
core_tables = ["documents", "parses", "chunks", "main_pointers", "ingestion_runs"]
@@ -3903,8 +3901,8 @@ def _vis_cascade_delete_documents(
)
]
- for table_name in select_embedding_tables(conn):
- predicates[table_name] = [
+ def _embedding_filter(table_name: str) -> list:
+ return [
_vis_build_documents_filter(
conn=conn,
table_name=table_name,
@@ -3921,6 +3919,7 @@ def _vis_cascade_delete_documents(
preview_only=preview_only,
confirm=confirm,
model_tag=None,
+ embedding_filters=_embedding_filter,
)
diff --git a/tests/core/tools/core/RAG_tools/storage/test_lancedb_delete_sweep_order.py b/tests/core/tools/core/RAG_tools/storage/test_lancedb_delete_sweep_order.py
new file mode 100644
index 000000000..72d5afac9
--- /dev/null
+++ b/tests/core/tools/core/RAG_tools/storage/test_lancedb_delete_sweep_order.py
@@ -0,0 +1,690 @@
+"""Deletes remove vectors, then chunk rows, then sweep the embedding tables in scope again."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+import pyarrow as pa
+import pytest
+
+from xagent.core.tools.core.RAG_tools.LanceDB.model_tag_utils import (
+ written_embeddings_table_name,
+)
+from xagent.core.tools.core.RAG_tools.storage.factory import get_metadata_store
+from xagent.core.tools.core.RAG_tools.utils.lancedb_query_utils import list_table_names
+
+OTHER_MODEL = "new-model"
+SOURCE_MODEL = "BAAI/old-model"
+
+
+def _connection() -> Any:
+ return get_metadata_store().get_raw_connection()
+
+
+def _save_collection() -> None:
+ from xagent.core.tools.core.RAG_tools.core.schemas import CollectionInfo
+ from xagent.core.tools.core.RAG_tools.management import collection_manager as cm
+
+ asyncio.run(
+ cm.collection_manager.save_collection(
+ CollectionInfo(
+ name="kb",
+ embedding_model_id=SOURCE_MODEL,
+ embedding_dimension=2,
+ embeddings=4,
+ )
+ )
+ )
+
+
+_SCHEMA = pa.schema(
+ [
+ ("collection", pa.string()),
+ ("doc_id", pa.string()),
+ ("chunk_id", pa.string()),
+ ("parse_hash", pa.string()),
+ ("user_id", pa.int64()),
+ ("text", pa.string()),
+ ("chunk_hash", pa.string()),
+ ("metadata", pa.string()),
+ ("model", pa.string()),
+ ("created_at", pa.timestamp("us")),
+ ("vector", pa.list_(pa.float32(), 2)),
+ ]
+)
+
+
+def _row(chunk_id: str, *, model: str = SOURCE_MODEL) -> Dict[str, Any]:
+ return {
+ "collection": "kb",
+ "doc_id": "d1",
+ "chunk_id": chunk_id,
+ "parse_hash": "p",
+ "user_id": 7,
+ "text": f"text {chunk_id}",
+ "chunk_hash": f"h-{chunk_id}",
+ "metadata": None,
+ "model": model,
+ "created_at": datetime(2026, 1, 1),
+ "vector": [0.5, 0.5],
+ }
+
+
+def _table(name: str, rows: List[Dict[str, Any]]) -> None:
+ _connection().create_table(name, data=pa.Table.from_pylist(rows, schema=_SCHEMA))
+
+
+def _rows_in(name: str) -> List[Dict[str, Any]]:
+ conn = _connection()
+ if name not in set(list_table_names(conn)):
+ return []
+ return conn.open_table(name).search().where("true").to_arrow().to_pylist()
+
+
+def _source(rows: List[Dict[str, Any]]) -> None:
+ _table(written_embeddings_table_name(SOURCE_MODEL), rows)
+
+
+def test_the_documents_cascade_sweeps_a_table_a_writer_created_meanwhile(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from xagent.core.tools.core.RAG_tools.LanceDB import schema_manager
+ from xagent.core.tools.core.RAG_tools.storage import lancedb_stores
+
+ for name in (
+ "ensure_documents_table",
+ "ensure_parses_table",
+ "ensure_chunks_table",
+ "ensure_main_pointers_table",
+ "ensure_ingestion_runs_table",
+ ):
+ monkeypatch.setattr(schema_manager, name, lambda *args, **kwargs: None)
+
+ conn = _connection()
+ core = pa.schema(
+ [
+ ("collection", pa.string()),
+ ("doc_id", pa.string()),
+ ("chunk_id", pa.string()),
+ ("user_id", pa.int64()),
+ ]
+ )
+ documents = pa.Table.from_pylist(
+ [{"collection": "kb", "doc_id": "d1", "chunk_id": "", "user_id": 7}],
+ schema=core,
+ )
+ conn.create_table("documents", data=documents)
+ conn.create_table("chunks", data=documents)
+
+ late = written_embeddings_table_name(OTHER_MODEL)
+ real_delete = lancedb_stores._vis_delete_rows_by_filters
+ created: List[str] = []
+
+ def _delete_then_writer_commits(table: Any, filters: List[str]) -> int:
+ count = real_delete(table, filters)
+ if count and not created:
+ # A writer that read these chunk rows before they went commits
+ # into another model's table while the cascade runs.
+ conn.create_table(
+ late,
+ data=pa.Table.from_pylist(
+ [_row("c1", model=OTHER_MODEL)], schema=_SCHEMA
+ ),
+ )
+ created.append(late)
+ return count
+
+ monkeypatch.setattr(
+ lancedb_stores, "_vis_delete_rows_by_filters", _delete_then_writer_commits
+ )
+
+ lancedb_stores._vis_cascade_delete_documents(
+ conn,
+ collection="kb",
+ doc_ids=["d1"],
+ user_id=None,
+ is_admin=True,
+ preview_only=False,
+ confirm=True,
+ )
+
+ assert created == [late]
+ assert _rows_in(late) == []
+
+
+def _handle(collection: str = "kb") -> Any:
+ from xagent.core.tools.core.RAG_tools.kb.collection_handle import (
+ LanceDBCollectionHandle,
+ )
+ from xagent.core.tools.core.RAG_tools.kb.models import (
+ KBAccessMode,
+ KBBackendCapabilities,
+ KBCollectionContext,
+ KBStorageBackend,
+ KBUserScope,
+ )
+ from xagent.core.tools.core.RAG_tools.storage.factory import (
+ get_ingestion_status_store,
+ get_main_pointer_store,
+ get_metadata_store,
+ get_vector_index_store,
+ )
+
+ context = KBCollectionContext(
+ collection=collection,
+ user_scope=KBUserScope(user_id=None, is_admin=True),
+ access_mode=KBAccessMode.WRITE,
+ allow_create=True,
+ hide_missing=True,
+ metadata_store=get_metadata_store(),
+ vector_index_store=get_vector_index_store(),
+ ingestion_status_store=get_ingestion_status_store(),
+ main_pointer_store=get_main_pointer_store(),
+ backend=KBStorageBackend.LANCEDB,
+ capabilities=KBBackendCapabilities.lancedb(),
+ collection_info=None,
+ )
+ return LanceDBCollectionHandle(context)
+
+
+def _seed_retirable_core() -> None:
+ """A document re-ingested as C2: chunk ``c1`` is stale, ``c2`` is current."""
+ from xagent.core.tools.core.RAG_tools.storage.factory import (
+ get_vector_index_store,
+ )
+ from xagent.core.tools.core.RAG_tools.utils.metadata_utils import (
+ serialize_metadata,
+ )
+
+ store = get_vector_index_store()
+ store.upsert_documents(
+ [
+ {
+ "collection": "kb",
+ "doc_id": "d1",
+ "file_id": None,
+ "source_path": "/x.txt",
+ "file_type": "txt",
+ "content_hash": "C2",
+ "uploaded_at": datetime.now(timezone.utc),
+ "title": None,
+ "language": None,
+ "user_id": None,
+ }
+ ]
+ )
+ store.upsert_parses(
+ [
+ {
+ "collection": "kb",
+ "doc_id": "d1",
+ "parse_hash": "p",
+ "parser": "local:[email protected]",
+ "created_at": datetime.now(timezone.utc),
+ "params_json": json.dumps({"content_hash": "C2"}),
+ "parsed_content": "[]",
+ "user_id": None,
+ }
+ ]
+ )
+ store.upsert_chunks(
+ [
+ {
+ "collection": "kb",
+ "doc_id": "d1",
+ "parse_hash": "p",
+ "chunk_id": chunk_id,
+ "index": 0,
+ "text": chunk_id,
+ "page_number": None,
+ "section": None,
+ "anchor": None,
+ "json_path": None,
+ "chunk_hash": "x",
+ "config_hash": config_hash,
+ "created_at": datetime.now(timezone.utc),
+ "metadata": serialize_metadata({"content_hash": made_from}),
+ "user_id": None,
+ }
+ for chunk_id, config_hash, made_from in (
+ ("c1", "cfg-old", "C1"),
+ ("c2", "cfg-new", "C2"),
+ )
+ ]
+ )
+
+
+def _store() -> Any:
+ from xagent.core.tools.core.RAG_tools.storage.factory import (
+ get_vector_index_store,
+ )
+
+ return get_vector_index_store()
+
+
+def _emb(doc_id: str, chunk_id: str, model: str = "m1") -> Dict[str, Any]:
+ return {
+ "collection": "kb",
+ "doc_id": doc_id,
+ "chunk_id": chunk_id,
+ "parse_hash": "p",
+ "model": model,
+ "vector": [0.1, 0.2],
+ "vector_dimension": 2,
+ "text": f"secret {chunk_id}",
+ "chunk_hash": "h",
+ "created_at": datetime.now(timezone.utc),
+ "metadata": "{}",
+ "user_id": None,
+ }
+
+
+def _emb_rows(doc_id: str, table: str = "embeddings_m1") -> List[str]:
+ conn = _connection()
+ return sorted(
+ row["chunk_id"]
+ for row in conn.open_table(table)
+ .search()
+ .where(f"doc_id = '{doc_id}'")
+ .to_arrow()
+ .to_pylist()
+ )
+
+
+def _doc_rows(doc_id: str) -> int:
+ conn = _connection()
+ return conn.open_table("documents").count_rows(f"doc_id = '{doc_id}'")
+
+
+def _chunk_rows(doc_id: str) -> int:
+ conn = _connection()
+ return conn.open_table("chunks").count_rows(f"doc_id = '{doc_id}'")
+
+
+def test_a_document_delete_that_fails_on_embeddings_can_be_retried(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from xagent.core.tools.core.RAG_tools.management import collections as mc
+ from xagent.core.tools.core.RAG_tools.storage import lancedb_stores
+
+ _save_collection()
+ _seed_retirable_core()
+ store = _store()
+ store.upsert_embeddings("m1", [_emb("d1", "c1"), _emb("d1", "c2")])
+ real = lancedb_stores._vis_delete_rows_by_filters
+ failed = {"once": False}
+
+ def _fail_embeddings(table: Any, filters: List[str]) -> int:
+ if getattr(table, "name", "").startswith("embeddings_") and not failed["once"]:
+ failed["once"] = True
+ raise OSError("injected embedding delete failure")
+ return real(table, filters)
+
+ monkeypatch.setattr(lancedb_stores, "_vis_delete_rows_by_filters", _fail_embeddings)
+
+ with pytest.raises(Exception):
+ store.delete_documents_data("kb", ["d1"], None, True)
+
+ # Vectors go first: the failed attempt left every core row, document
+ # included, so nothing a retry reads is gone.
+ assert _doc_rows("d1") == 1
+ assert _chunk_rows("d1") == 2
+ assert _emb_rows("d1") == ["c1", "c2"]
+ monkeypatch.setattr(lancedb_stores, "_vis_delete_rows_by_filters", real)
+
+ result = mc.delete_document("kb", "d1", 1, is_admin=True)
+
+ assert result.status == "success", result.message
+ assert _doc_rows("d1") == 0
+ assert _emb_rows("d1") == []
+
+
+def test_a_retirement_that_fails_on_vectors_can_be_retried(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from xagent.core.tools.core.RAG_tools.kb.collection_handle import (
+ LanceDBCollectionHandle,
+ )
+
+ _save_collection()
+ _seed_retirable_core()
+ _store().upsert_embeddings("m1", [_emb("d1", "c1"), _emb("d1", "c2")])
+ real = LanceDBCollectionHandle.delete_embedding_records
+ failed = {"once": False}
+
+ def _fail_once(handle: Any, doc_id: str, **kwargs: Any) -> int:
+ if not failed["once"]:
+ failed["once"] = True
+ raise OSError("injected")
+ return real(handle, doc_id, **kwargs)
+
+ monkeypatch.setattr(LanceDBCollectionHandle, "delete_embedding_records", _fail_once)
+
+ with pytest.raises(OSError):
+ _handle().retire_superseded_chunks("d1", "p")
+
+ # The chunk rows are still there, so the retry can find the stale chunk.
+ assert _handle().stale_chunks("d1", "p") == {("p", "cfg-old"): ["c1"]}
+ assert _emb_rows("d1") == ["c1", "c2"]
+
+ assert _handle().retire_superseded_chunks("d1", "p") == ["c1"]
+ assert "c1" not in _emb_rows("d1")
+
+
+def test_a_retirement_sweeps_a_table_a_writer_created_meanwhile(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from xagent.core.tools.core.RAG_tools.kb.collection_handle import (
+ LanceDBCollectionHandle,
+ )
+
+ _save_collection()
+ _seed_retirable_core()
+ _source([_row("c1"), _row("c2")])
+ late = written_embeddings_table_name(OTHER_MODEL)
+ real_core = LanceDBCollectionHandle.delete_chunk_records
+ created: List[str] = []
+
+ def _writer_commits_then_core_goes(handle: Any, doc_id: str, **kwargs: Any) -> int:
+ if not created:
+ _table(late, [_row("c1", model=OTHER_MODEL)])
+ created.append(late)
+ return real_core(handle, doc_id, **kwargs)
+
+ monkeypatch.setattr(
+ LanceDBCollectionHandle, "delete_chunk_records", _writer_commits_then_core_goes
+ )
+
+ assert _handle().retire_superseded_chunks("d1", "p") == ["c1"]
+
+ assert created == [late]
+ assert _rows_in(late) == []
+ assert {
+ row["chunk_id"] for row in _rows_in(written_embeddings_table_name(SOURCE_MODEL))
+ } == {"c2"}
+
+
+def test_a_delete_reports_no_success_when_the_final_listing_fails(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from xagent.core.tools.core.RAG_tools.management import collections as mc
+ from xagent.core.tools.core.RAG_tools.storage import lancedb_cleanup_filters
+
+ _save_collection()
+ _seed_retirable_core()
+ store = _store()
+ store.upsert_embeddings("m1", [_emb("d1", "c1"), _emb("d1", "c2")])
+ real = lancedb_cleanup_filters.select_embedding_tables
+ calls = {"n": 0}
+
+ def _flaky(conn: Any, model_tag: Optional[str] = None) -> List[str]:
+ calls["n"] += 1
+ if calls["n"] == 2:
+ raise OSError("listing failed")
+ return real(conn, model_tag=model_tag)
+
+ monkeypatch.setattr(lancedb_cleanup_filters, "select_embedding_tables", _flaky)
+
+ with pytest.raises(Exception):
+ store.delete_documents_data("kb", ["d1"], None, True)
+
+ assert calls["n"] == 2
+ monkeypatch.setattr(lancedb_cleanup_filters, "select_embedding_tables", real)
+
+ # Not reported as a success, and still deletable through the document's row.
+ assert _doc_rows("d1") == 1
+ result = mc.delete_document("kb", "d1", 1, is_admin=True)
+ assert result.status == "success", result.message
+ assert _doc_rows("d1") == 0
+ assert _emb_rows("d1") == []
+
+
+def test_the_generic_cascade_sweeps_a_table_a_writer_created_meanwhile(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from xagent.core.tools.core.RAG_tools.storage import lancedb_stores
+
+ _save_collection()
+ _seed_retirable_core()
+ store = _store()
+ store.upsert_embeddings("m1", [_emb("d1", "c1")])
+ late = written_embeddings_table_name(OTHER_MODEL)
+ real = lancedb_stores._vis_delete_rows_by_filters
+ created: List[str] = []
+
+ def _writer_commits_during_the_core_delete(table: Any, filters: List[str]) -> int:
+ if not created and getattr(table, "name", "") == "chunks":
+ # A writer that read the chunk rows before they went creates its
+ # table while this delete runs.
+ _connection().create_table(
+ late,
+ data=pa.Table.from_pylist(
+ [_row("c1", model=OTHER_MODEL)], schema=_SCHEMA
+ ),
+ )
+ created.append(late)
+ return real(table, filters)
+
+ monkeypatch.setattr(
+ lancedb_stores,
+ "_vis_delete_rows_by_filters",
+ _writer_commits_during_the_core_delete,
+ )
+
+ result = store.cascade_delete(
+ target="document",
+ collection="kb",
+ doc_id="d1",
+ is_admin=True,
+ preview_only=False,
+ confirm=True,
+ )
+
+ assert created == [late]
+ assert _rows_in(late) == []
+ assert result["documents"] == 1
+
+
[email protected]("scope_name", ["chunk", "parse"])
+def test_a_version_cleanup_sweeps_a_table_a_writer_created_meanwhile(
+ monkeypatch: pytest.MonkeyPatch, scope_name: str
+) -> None:
+ from xagent.core.tools.core.RAG_tools.storage import lancedb_stores
+
+ _save_collection()
+ _seed_retirable_core()
+ _source([_row("c1"), _row("c2")])
+ late = written_embeddings_table_name(OTHER_MODEL)
+ real = lancedb_stores._vis_delete_rows_by_filters
+ created: List[str] = []
+
+ def _writer_commits_during_the_core_delete(table: Any, filters: List[str]) -> int:
+ if not created and getattr(table, "name", "") == "chunks":
+ _connection().create_table(
+ late,
+ data=pa.Table.from_pylist(
+ [_row("c1", model=OTHER_MODEL)], schema=_SCHEMA
+ ),
+ )
+ created.append(late)
+ return real(table, filters)
+
+ monkeypatch.setattr(
+ lancedb_stores,
+ "_vis_delete_rows_by_filters",
+ _writer_commits_during_the_core_delete,
+ )
+
+ _store().cleanup_cascade_by_scope(
+ "kb",
+ "d1",
+ scope_name,
+ new_parse_hash="p-new",
+ old_parse_hash="p",
+ preview_only=False,
+ confirm=True,
+ )
+
+ assert created == [late]
+ assert _chunk_rows("d1") == 0
+ assert _rows_in(late) == []
+
+
+def test_a_cascade_whose_count_fails_deletes_nothing_and_raises(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from xagent.core.tools.core.RAG_tools.utils import lancedb_query_utils
+
+ _save_collection()
+ _seed_retirable_core()
+ _store().upsert_embeddings("m1", [_emb("d1", "c1"), _emb("d1", "c2")])
+ real = lancedb_query_utils._safe_count_rows
+
+ class _Unreadable:
+ def count_rows(self, *args: Any) -> int:
+ raise OSError("count failed")
+
+ def _embeddings_count_fails(
+ table: Any, filter_expr: Optional[str] = None, **kwargs: Any
+ ) -> int:
+ if str(getattr(table, "name", "")).startswith("embeddings_"):
+ table = _Unreadable()
+ return real(table, filter_expr, **kwargs)
+
+ monkeypatch.setattr(
+ lancedb_query_utils, "_safe_count_rows", _embeddings_count_fails
+ )
+
+ with pytest.raises(OSError):
+ _store().cascade_delete(
+ target="document",
+ collection="kb",
+ doc_id="d1",
+ is_admin=True,
+ preview_only=False,
+ confirm=True,
+ )
+
+ assert _emb_rows("d1") == ["c1", "c2"]
+ assert _chunk_rows("d1") == 2
+ assert _doc_rows("d1") == 1
+
+
+def test_a_failed_core_listing_fails_the_document_cascade_before_any_delete(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from xagent.core.tools.core.RAG_tools.storage import lancedb_stores
+ from xagent.core.tools.core.RAG_tools.utils import lancedb_query_utils
+
+ _save_collection()
+ _seed_retirable_core()
+ store = _store()
+ store.upsert_embeddings("m1", [_emb("d1", "c1"), _emb("d1", "c2")])
+ real_listing = lancedb_query_utils.list_table_names
+ real_cascade = lancedb_stores._vis_cascade_delete_documents
+ armed = {"fail": False}
+
+ def _listing(conn: Any) -> List[str]:
+ if armed["fail"]:
+ armed["fail"] = False
+ raise OSError("listing failed")
+ return real_listing(conn)
+
+ def _cascade(*args: Any, **kwargs: Any) -> Dict[str, int]:
+ armed["fail"] = True
+ return real_cascade(*args, **kwargs)
+
+ monkeypatch.setattr(lancedb_query_utils, "list_table_names", _listing)
+ monkeypatch.setattr(lancedb_stores, "_vis_cascade_delete_documents", _cascade)
+
+ with pytest.raises(Exception):
+ store.delete_documents_data("kb", ["d1"], None, True)
+
+ assert armed["fail"] is False
+ assert _emb_rows("d1") == ["c1", "c2"]
+ assert _chunk_rows("d1") == 2
+ assert _doc_rows("d1") == 1
+
+
+def _fail_listing_call(monkeypatch: pytest.MonkeyPatch, failing: int) -> Dict[str, int]:
+ from xagent.core.tools.core.RAG_tools.utils import lancedb_query_utils
+
+ real = lancedb_query_utils.list_table_names
+ calls = {"n": 0}
+
+ def _listing(conn: Any) -> List[str]:
+ calls["n"] += 1
+ if calls["n"] == failing:
+ raise OSError("listing failed")
+ return real(conn)
+
+ monkeypatch.setattr(lancedb_query_utils, "list_table_names", _listing)
+ return calls
+
+
+ ("target", "failing"),
+ [("collection", 1), ("document", 2)],
+ ids=["collection-builder-listing", "document-executor-listing"],
+)
+def test_a_failed_listing_fails_the_cascade_before_any_delete(
+ monkeypatch: pytest.MonkeyPatch, target: str, failing: int
+) -> None:
+ _save_collection()
+ _seed_retirable_core()
+ store = _store()
+ store.upsert_embeddings("m1", [_emb("d1", "c1"), _emb("d1", "c2")])
+ calls = _fail_listing_call(monkeypatch, failing)
+
+ with pytest.raises(OSError):
+ store.cascade_delete(
+ target=target,
+ collection="kb",
+ doc_id="d1",
+ is_admin=True,
+ preview_only=False,
+ confirm=True,
+ )
+
+ assert calls["n"] == failing
+ assert _emb_rows("d1") == ["c1", "c2"]
+ assert _chunk_rows("d1") == 2
+ assert _doc_rows("d1") == 1
+
+
[email protected]("delete", ["document-cascade", "parse-cleanup"])
+def test_a_model_scoped_delete_keeps_the_other_models_vectors(delete: str) -> None:
+ _save_collection()
+ _seed_retirable_core()
+ store = _store()
+ store.upsert_embeddings("m1", [_emb("d1", "c1")])
+ store.upsert_embeddings("m2", [_emb("d1", "c1", model="m2")])
+
+ if delete == "document-cascade":
+ store.cascade_delete(
+ target="document",
+ collection="kb",
+ doc_id="d1",
+ is_admin=True,
+ model_tag="m1",
+ preview_only=False,
+ confirm=True,
+ )
+ else:
+ store.cleanup_cascade_by_scope(
+ "kb",
+ "d1",
+ "parse",
+ new_parse_hash="p-new",
+ old_parse_hash="p",
+ model_tag="m1",
+ preview_only=False,
+ confirm=True,
+ )
+
+ assert _chunk_rows("d1") == 0
+ assert _emb_rows("d1") == []
+ assert _emb_rows("d1", "embeddings_m2") == ["c1"]
diff --git a/tests/core/tools/core/RAG_tools/storage/test_lancedb_store_version.py b/tests/core/tools/core/RAG_tools/storage/test_lancedb_store_version.py
index c0e7f0ffb..451921a6b 100644
--- a/tests/core/tools/core/RAG_tools/storage/test_lancedb_store_version.py
+++ b/tests/core/tools/core/RAG_tools/storage/test_lancedb_store_version.py
@@ -407,3 +407,57 @@ class TestCleanupCascadeByScope:
)
)
assert sync_result == async_result
+
+ def test_embedding_tables_are_swept_before_and_after_the_chunk_delete(
+ self, isolated_store, monkeypatch
+ ):
+ """A failure keeps the chunk rows; the sweep after them catches a late table."""
+ from xagent.core.tools.core.RAG_tools.storage import lancedb_stores
+
+ store = isolated_store
+ collection, doc_id, parse_hash = "cc_order", "cc_order_doc", "orderhash"
+ store.upsert_parses([_make_parse_row(collection, doc_id, parse_hash)])
+ store.upsert_chunks([_make_chunk_row(collection, doc_id, parse_hash, "ck1")])
+ store.upsert_embeddings(
+ "m1",
+ [
+ {
+ "collection": collection,
+ "doc_id": doc_id,
+ "chunk_id": "ck1",
+ "parse_hash": parse_hash,
+ "model": "m1",
+ "vector": [0.1, 0.2],
+ "vector_dimension": 2,
+ "text": "ck1",
+ "chunk_hash": "ch_ck1",
+ "created_at": datetime.now(timezone.utc),
+ "metadata": "{}",
+ "user_id": None,
+ }
+ ],
+ )
+
+ order = []
+ real = lancedb_stores._vis_delete_rows_by_filters
+
+ def _recording(table, filter_exprs):
+ order.append(getattr(table, "name", ""))
+ return real(table, filter_exprs)
+
+ monkeypatch.setattr(lancedb_stores, "_vis_delete_rows_by_filters", _recording)
+
+ result = store.cleanup_cascade_by_scope(
+ collection,
+ doc_id,
+ "parse",
+ old_parse_hash=parse_hash,
+ preview_only=False,
+ confirm=True,
+ )
+
+ assert result["chunks"] == 1
+ sweeps = [i for i, name in enumerate(order) if name.startswith("embeddings_")]
+ chunks_at = order.index("chunks")
+ assert sweeps[0] < chunks_at
+ assert chunks_at < sweeps[-1]
diff --git a/tests/core/tools/core/RAG_tools/version_management/test_cascade_cleaner.py b/tests/core/tools/core/RAG_tools/version_management/test_cascade_cleaner.py
index 4d3e53665..b89a5273a 100644
--- a/tests/core/tools/core/RAG_tools/version_management/test_cascade_cleaner.py
+++ b/tests/core/tools/core/RAG_tools/version_management/test_cascade_cleaner.py
@@ -604,15 +604,14 @@ def test_cleanup_embed_cascade_preserves_multiple_predicates_per_table() -> None
with (
patch.object(LanceDBVectorIndexStore, "_get_connection") as mock_get_conn,
patch(
- "xagent.core.tools.core.RAG_tools.storage.lancedb_cleanup_filters.build_embedding_cleanup_filters"
+ "xagent.core.tools.core.RAG_tools.storage.lancedb_cleanup_filters.embedding_cleanup_filters_for_table"
) as mock_build_filters,
patch(f"{_STORE}._vis_plan_by_predicates") as mock_plan,
):
conn = MagicMock(spec=["table_names", "open_table"])
+ conn.table_names.return_value = ["embeddings_m1"]
mock_get_conn.return_value = conn
- mock_build_filters.return_value = {
- "embeddings_m1": ["chunk_id == 'ch1'", "chunk_id == 'ch2'"],
- }
+ mock_build_filters.return_value = ["chunk_id == 'ch1'", "chunk_id == 'ch2'"]
mock_plan.return_value = {"embeddings_m1": 2}
result = get_vector_index_store().cleanup_cascade_by_scope(
@@ -636,7 +635,7 @@ def test_cleanup_embed_cascade_deletes_multiple_predicates_per_table() -> None:
with (
patch.object(LanceDBVectorIndexStore, "_get_connection") as mock_get_conn,
patch(
- "xagent.core.tools.core.RAG_tools.storage.lancedb_cleanup_filters.build_embedding_cleanup_filters"
+ "xagent.core.tools.core.RAG_tools.storage.lancedb_cleanup_filters.embedding_cleanup_filters_for_table"
) as mock_build_filters,
):
table = MagicMock()
@@ -645,9 +644,7 @@ def test_cleanup_embed_cascade_deletes_multiple_predicates_per_table() -> None:
conn.table_names.return_value = ["embeddings_m1"]
conn.open_table.return_value = table
mock_get_conn.return_value = conn
- mock_build_filters.return_value = {
- "embeddings_m1": ["chunk_id == 'ch1'", "chunk_id == 'ch2'"],
- }
+ mock_build_filters.return_value = ["chunk_id == 'ch1'", "chunk_id == 'ch2'"]
result = get_vector_index_store().cleanup_cascade_by_scope(
"c_multi",
@@ -787,14 +784,15 @@ def test_cleanup_cascade_plans_unless_confirmed_outside_preview(
with (
patch.object(LanceDBVectorIndexStore, "_get_connection") as mock_get_conn,
patch(
- "xagent.core.tools.core.RAG_tools.storage.lancedb_cleanup_filters.build_embedding_cleanup_filters"
+ "xagent.core.tools.core.RAG_tools.storage.lancedb_cleanup_filters.embedding_cleanup_filters_for_table"
) as mock_build_filters,
patch(f"{_STORE}._vis_plan_by_predicates") as mock_plan,
patch(f"{_STORE}._vis_delete_by_predicates") as mock_delete,
):
conn = MagicMock(spec=["table_names", "open_table"])
+ conn.table_names.return_value = ["embeddings_m1"]
mock_get_conn.return_value = conn
- mock_build_filters.return_value = {"embeddings_m1": ["collection == 'c_gate'"]}
+ mock_build_filters.return_value = ["collection == 'c_gate'"]
mock_plan.return_value = {"embeddings_m1": 1}
result = get_vector_index_store().cleanup_cascade_by_scope(
@@ -810,3 +808,17 @@ def test_cleanup_cascade_plans_unless_confirmed_outside_preview(
assert result == {"embeddings": 1}
mock_plan.assert_called_once()
mock_delete.assert_not_called()
+
+
[email protected]("key", ["embeddings_m1", "__embeddings__"])
+def test_a_cascade_refuses_embedding_predicates_built_ahead_of_the_delete(
+ key: str,
+) -> None:
+ from xagent.core.tools.core.RAG_tools.storage import lancedb_stores
+
+ conn = MagicMock(spec=["table_names", "open_table"])
+ with pytest.raises(ValueError):
+ lancedb_stores._vis_execute_or_plan_by_predicates(
+ conn, {key: ["true"]}, preview_only=False, confirm=True
+ )
+ conn.open_table.assert_not_called()