Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

feat(kb): report an embedding migration's scope without writing (#3825)

蓝鱼bluefishe11b436fbd4e
diff --git a/src/xagent/core/tools/core/RAG_tools/kb/coordinator.py b/src/xagent/core/tools/core/RAG_tools/kb/coordinator.py
index a107d12b8..1234b30df 100644
--- a/src/xagent/core/tools/core/RAG_tools/kb/coordinator.py
+++ b/src/xagent/core/tools/core/RAG_tools/kb/coordinator.py
@@ -2755,6 +2755,11 @@ def reset_kb_coordinator_for_tests() -> None:
_coordinator = None
+def resolve_collection_backend(collection_info: object | None) -> KBStorageBackend:
+ """The engine a collection is bound to, defaulting to the deployment's."""
+ return get_kb_coordinator()._resolve_backend(collection_info)
+
+
def _run_in_separate_loop(awaitable: Coroutine[Any, Any, T]) -> T:
"""Run a coroutine from sync code, including inside an existing event loop."""
try:
diff --git a/src/xagent/migrations/embedding/__init__.py b/src/xagent/migrations/embedding/__init__.py
new file mode 100644
index 000000000..9c251d0ac
--- /dev/null
+++ b/src/xagent/migrations/embedding/__init__.py
@@ -0,0 +1 @@
+"""Operator commands for moving knowledge-base embeddings to another model."""
diff --git a/src/xagent/migrations/embedding/scope.py b/src/xagent/migrations/embedding/scope.py
new file mode 100644
index 000000000..8be64c253
--- /dev/null
+++ b/src/xagent/migrations/embedding/scope.py
@@ -0,0 +1,650 @@
+"""Read-only scope of an embedding migration, per knowledge-base collection.
+
+ python -m xagent.migrations.embedding.scope [--model TARGET] \
+ [--collection NAME ...]
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import math
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Iterator, List, Optional, Sequence, Set, Tuple
+
+import lancedb
+
+from xagent.core.tools.core.RAG_tools.LanceDB.model_tag_utils import (
+ embeddings_table_candidates,
+ written_embeddings_table_name,
+)
+from xagent.core.tools.core.RAG_tools.LanceDB.schema_manager import _safe_close_table
+from xagent.core.tools.core.RAG_tools.utils.lancedb_query_utils import list_table_names
+from xagent.core.tools.core.RAG_tools.utils.string_utils import escape_lancedb_string
+from xagent.providers.vector_store.lancedb import LanceDBConnectionManager
+
+logger = logging.getLogger(__name__)
+
+METADATA_TABLE = "collection_metadata"
+CORE_TABLE = "chunks"
+
+# Why a collection is not counted.
+SKIP_OTHER_ENGINE = "other-engine"
+SKIP_UNREADABLE_BINDING = "unreadable-binding"
+SKIP_UNBOUND = "unbound"
+SKIP_BOUND_WITHOUT_DIMENSION = "bound-without-dimension"
+SKIP_ROWS_IN_OTHER_TABLE = "rows-in-another-table"
+SKIP_ROWS_IN_MULTIPLE_TABLES = "rows-in-multiple-tables"
+SKIP_MISSING_COLUMN = "missing-required-column"
+SKIP_SHARED_TABLE = "shared-source-target-table"
+SKIP_TARGET_ROWS_IN_OTHER_TABLE = "target-rows-in-another-table"
+SKIP_TARGET_IS_SOURCE = "target-is-source"
+SKIP_NO_METADATA = "no-metadata"
+# Why a source row is not counted as missing.
+SKIP_NO_DOC_ID = "no-doc-id"
+SKIP_NO_CHUNK_ID = "no-chunk-id"
+SKIP_NO_OWNER = "no-owner"
+SKIP_NO_TEXT = "no-text"
+SKIP_NO_CORE_CHUNK = "no-core-chunk"
+
+REQUIRED_COLUMNS = ("collection", "doc_id", "chunk_id", "user_id", "text")
+
+READ_BATCH_SIZE = 1000
+EMBEDDINGS_PREFIX = "embeddings_"
+
+
+class MigrationCliError(RuntimeError):
+ """A problem the command reports: exit 2 before any collection is read,
+ otherwise an error on the collection it concerns."""
+
+
+class MissingCollectionError(MigrationCliError):
+ """The collection has no metadata row (never existed, or was just deleted)."""
+
+
+@dataclass(frozen=True)
+class SearchableChunk:
+ """One row of the source model's embeddings table."""
+
+ doc_id: str
+ chunk_id: str
+ text: str
+ user_id: Optional[int]
+
+
+@dataclass(frozen=True)
+class Eligibility:
+ """Whether one collection can be counted: ``reason`` is ``None`` only then."""
+
+ model_id: Optional[str] = None
+ reason: Optional[str] = None
+
+
+@dataclass
+class CollectionScope:
+ """One collection's report line."""
+
+ name: str
+ source_model_id: Optional[str] = None
+ skip_reason: Optional[str] = None
+ usable_documents: int = 0
+ chunks: int = 0
+ # ``None`` unless the collection was counted against a target.
+ missing_chunks: Optional[int] = None
+ skipped: Dict[str, int] = field(default_factory=dict)
+ error: Optional[str] = None
+
+ @property
+ def usable(self) -> bool:
+ return self.skip_reason is None and self.error is None
+
+
+def connection() -> Any:
+ """The application's LanceDB directory, opened only if it already exists."""
+ db_dir = LanceDBConnectionManager().resolve_dir_from_env(create=False)
+ if not Path(db_dir).is_dir():
+ raise MigrationCliError(f"LanceDB directory {db_dir} is not a directory")
+ return lancedb.connect(db_dir)
+
+
+def table_names(conn: Any) -> Set[str]:
+ return set(list_table_names(conn))
+
+
+def source_table(model_id: str) -> str:
+ return written_embeddings_table_name(model_id)
+
+
+def iter_rows(
+ conn: Any,
+ table: str,
+ where: str,
+ columns: Sequence[str],
+) -> Iterator[Dict[str, Any]]:
+ """Stream rows of ``table``; a failed read or a missing column raises."""
+ opened = conn.open_table(table)
+ try:
+ missing = [name for name in columns if name not in opened.schema.names]
+ if missing:
+ raise MigrationCliError(f"table '{table}' has no {', '.join(missing)}")
+ reader = (
+ opened.search()
+ .where(where)
+ .select(list(columns))
+ .limit(None)
+ .to_batches(READ_BATCH_SIZE)
+ )
+ for batch in reader:
+ yield from batch.to_pylist()
+ finally:
+ _safe_close_table(opened)
+
+
+def schema_names(conn: Any, table: str) -> List[str]:
+ opened = conn.open_table(table)
+ try:
+ return list(opened.schema.names)
+ finally:
+ _safe_close_table(opened)
+
+
+def collection_filter(collection: str) -> str:
+ return f"collection = '{escape_lancedb_string(collection)}'"
+
+
+def chunk_key(row: Dict[str, Any]) -> Tuple[str, str]:
+ """The ``(doc_id, chunk_id)`` every reader compares, exactly as stored."""
+ return (str(row.get("doc_id") or ""), str(row.get("chunk_id") or ""))
+
+
+def collection_info(collection: str, conn: Optional[Any] = None) -> Any:
+ """The metadata row; only an absent row is :class:`MissingCollectionError`."""
+ from xagent.core.tools.core.RAG_tools.core.schemas import CollectionInfo
+
+ conn = conn if conn is not None else connection()
+ if METADATA_TABLE not in table_names(conn):
+ raise MissingCollectionError(f"collection '{collection}' does not exist")
+ table = conn.open_table(METADATA_TABLE)
+ try:
+ rows = (
+ table.search()
+ .where(f"name = '{escape_lancedb_string(collection)}'")
+ .to_arrow()
+ .to_pylist()
+ )
+ finally:
+ _safe_close_table(table)
+ if not rows:
+ raise MissingCollectionError(f"collection '{collection}' does not exist")
+ try:
+ return CollectionInfo.from_storage(rows[0])
+ except Exception as exc: # noqa: BLE001 - an error on the collection
+ raise MigrationCliError(
+ f"collection '{collection}' metadata does not decode: {exc}"
+ ) from exc
+
+
+def collection_engine(info: Any) -> str:
+ """The engine a collection is bound to; ``""`` for a malformed binding."""
+ from xagent.core.tools.core.RAG_tools.kb.coordinator import (
+ get_kb_coordinator,
+ resolve_collection_backend,
+ )
+
+ # Built outside the try: only a malformed binding may become a reason.
+ get_kb_coordinator()
+ try:
+ return str(resolve_collection_backend(info).value)
+ except ValueError:
+ return ""
+
+
+def own_tables(model_id: str) -> List[str]:
+ """The tables search tries for ``model_id`` before asking the model hub."""
+ return embeddings_table_candidates(model_id)
+
+
+def written_by(row: Dict[str, Any], model_id: str) -> bool:
+ """Whether the row's ``model`` is ``model_id``: distinct ids can share a writer
+ table (``old-model`` and ``old_model``)."""
+ return row.get("model") == model_id
+
+
+def _count_rows(conn: Any, table: str, where: Optional[str] = None) -> int:
+ opened = conn.open_table(table)
+ try:
+ return int(opened.count_rows(where) if where else opened.count_rows())
+ finally:
+ _safe_close_table(opened)
+
+
+def holds_rows(conn: Any, table: str, collection: str) -> bool:
+ if "collection" not in schema_names(conn, table):
+ # Cannot attribute a row to a collection: any row counts.
+ return _count_rows(conn, table) > 0
+ return _count_rows(conn, table, collection_filter(collection)) > 0
+
+
+def _holds_rows_written_elsewhere(
+ conn: Any, table: str, collection: str, model: str
+) -> bool:
+ """Whether ``table`` holds rows of this collection that ``model`` did not
+ write; without a ``collection`` column any row counts."""
+ if "collection" not in schema_names(conn, table):
+ return _count_rows(conn, table) > 0
+ if "model" not in schema_names(conn, table):
+ return _count_rows(conn, table, collection_filter(collection)) > 0
+ foreign = (
+ f"{collection_filter(collection)} AND "
+ f"(model IS NULL OR model != '{escape_lancedb_string(model)}')"
+ )
+ return _count_rows(conn, table, foreign) > 0
+
+
+def _embeddings_tables(names: Set[str]) -> List[str]:
+ return sorted(name for name in names if name.startswith(EMBEDDINGS_PREFIX))
+
+
+def target_tables_hold_foreign_rows(
+ conn: Any, collection: str, model_id: str, names: Set[str], source: List[str]
+) -> bool:
+ """Whether search for ``model_id`` could serve rows of this collection other
+ than those ``model_id`` wrote in its writer table."""
+ present = [table for table in own_tables(model_id) if table in names]
+ if not present:
+ # Search would ask the model hub for a table this command cannot name.
+ return any(
+ holds_rows(conn, table, collection)
+ for table in _embeddings_tables(names)
+ if table not in source
+ )
+ writer = source_table(model_id)
+ for table in present:
+ if table != writer:
+ if holds_rows(conn, table, collection):
+ return True
+ elif _holds_rows_written_elsewhere(conn, table, collection, model_id):
+ return True
+ return False
+
+
+def missing_columns(conn: Any, table: str) -> List[str]:
+ present = set(schema_names(conn, table))
+ return [name for name in REQUIRED_COLUMNS if name not in present]
+
+
+def precompute_eligibility(
+ collection: str,
+ *,
+ conn: Optional[Any] = None,
+ target_model_id: Optional[str] = None,
+) -> Eligibility:
+ """:func:`_source_eligibility`, also refusing a target that is the bound model."""
+ result = _source_eligibility(collection, conn=conn, target_model_id=target_model_id)
+ if result.reason is None and target_model_id and target_model_id == result.model_id:
+ return Eligibility(reason=SKIP_TARGET_IS_SOURCE)
+ return result
+
+
+def _source_eligibility(
+ collection: str,
+ *,
+ conn: Optional[Any] = None,
+ target_model_id: Optional[str] = None,
+) -> Eligibility:
+ """Eligible only when bound with a dimension, with no row search could serve
+ from a table but the writer's, and (with a target) no shared or foreign table."""
+ from xagent.core.tools.core.RAG_tools.management.collection_manager import (
+ _normalize_model_id,
+ )
+
+ conn = conn if conn is not None else connection()
+ info = collection_info(collection, conn)
+ engine = collection_engine(info)
+ if engine != "lancedb":
+ return Eligibility(
+ reason=SKIP_OTHER_ENGINE if engine else SKIP_UNREADABLE_BINDING
+ )
+ model = _normalize_model_id(info.embedding_model_id)
+ if not model:
+ return Eligibility(reason=SKIP_UNBOUND)
+ dimension = info.embedding_dimension
+ if not isinstance(dimension, int) or dimension <= 0:
+ return Eligibility(reason=SKIP_BOUND_WITHOUT_DIMENSION)
+ table = source_table(model)
+ own = own_tables(model)
+ names = table_names(conn)
+ if table in names and missing_columns(conn, table):
+ return Eligibility(reason=SKIP_MISSING_COLUMN)
+ if target_model_id and target_model_id != model:
+ if set(own) & set(own_tables(target_model_id)):
+ return Eligibility(reason=SKIP_SHARED_TABLE)
+ if target_tables_hold_foreign_rows(
+ conn, collection, target_model_id, names, own
+ ):
+ return Eligibility(reason=SKIP_TARGET_ROWS_IN_OTHER_TABLE)
+ if not any(name in names for name in own):
+ # Search would ask the model hub for a table this command cannot name.
+ candidates = _embeddings_tables(names)
+ if any(holds_rows(conn, name, collection) for name in candidates):
+ return Eligibility(reason=SKIP_ROWS_IN_OTHER_TABLE)
+ return Eligibility(model_id=model)
+ holders = [
+ name for name in own if name in names and holds_rows(conn, name, collection)
+ ]
+ if table not in holders:
+ if holders:
+ return Eligibility(reason=SKIP_ROWS_IN_OTHER_TABLE)
+ return Eligibility(model_id=model)
+ if len(holders) > 1:
+ return Eligibility(reason=SKIP_ROWS_IN_MULTIPLE_TABLES)
+ return Eligibility(model_id=model)
+
+
+@dataclass
+class SourceScan:
+ """One pass over a collection's source rows; ``pending`` counts the usable rows
+ the target does not hold."""
+
+ usable_documents: Set[str] = field(default_factory=set)
+ pending: int = 0
+ skipped: Dict[str, int] = field(default_factory=dict)
+ rows: int = 0
+
+
+def scan_source(
+ collection: str,
+ model_id: str,
+ *,
+ have: Optional[Set[Tuple[str, str]]] = None,
+ core: Optional[Set[Tuple[str, str]]] = None,
+ conn: Optional[Any] = None,
+) -> SourceScan:
+ """Classify every source row once; without ``core`` (:func:`core_chunk_pairs`)
+ no row is checked against the core table."""
+ conn = conn if conn is not None else connection()
+ scan = SourceScan()
+ core_keys = core if core is not None else set()
+ core_docs = {doc_id for doc_id, _ in core_keys}
+ table = source_table(model_id)
+ if table not in table_names(conn):
+ return scan
+ for row in iter_rows(
+ conn,
+ table,
+ collection_filter(collection),
+ ["doc_id", "chunk_id", "user_id", "text"],
+ ):
+ scan.rows += 1
+ key = chunk_key(row)
+ chunk = SearchableChunk(
+ doc_id=key[0],
+ chunk_id=key[1],
+ text=str(row.get("text") or ""),
+ user_id=_owner(row.get("user_id")),
+ )
+ reason = skip_reason(chunk)
+ if reason is None and chunk.doc_id in core_docs and key not in core_keys:
+ reason = SKIP_NO_CORE_CHUNK
+ if reason:
+ scan.skipped[reason] = scan.skipped.get(reason, 0) + 1
+ continue
+ scan.usable_documents.add(chunk.doc_id)
+ if have is not None and key not in have:
+ scan.pending += 1
+ return scan
+
+
+def core_chunk_pairs(
+ collection: str, *, conn: Optional[Any] = None
+) -> Optional[Set[Tuple[str, str]]]:
+ """Every ``(doc_id, chunk_id)`` the core chunks table holds, or ``None`` without one."""
+ conn = conn if conn is not None else connection()
+ if CORE_TABLE not in table_names(conn):
+ return None
+ return {
+ chunk_key(row)
+ for row in iter_rows(
+ conn,
+ CORE_TABLE,
+ collection_filter(collection),
+ ["doc_id", "chunk_id"],
+ )
+ }
+
+
+def skip_reason(chunk: SearchableChunk) -> Optional[str]:
+ """Why this source row is excluded, or ``None`` when it is usable."""
+ if not chunk.doc_id.strip():
+ return SKIP_NO_DOC_ID
+ if not chunk.chunk_id.strip():
+ return SKIP_NO_CHUNK_ID
+ if chunk.user_id is None:
+ return SKIP_NO_OWNER
+ if not chunk.text:
+ return SKIP_NO_TEXT
+ return None
+
+
+def _owner(raw: Any) -> Optional[int]:
+ if raw is None:
+ return None
+ try:
+ return int(raw)
+ except (TypeError, ValueError):
+ return None
+
+
+def target_chunk_ids(
+ collection: str,
+ model_id: str,
+ *,
+ conn: Optional[Any] = None,
+ dimension: Optional[int] = None,
+) -> Set[Tuple[str, str]]:
+ """The ``(doc_id, chunk_id)`` of each row in the target's writer table that
+ ``model_id`` wrote with a finite, non-zero vector (of ``dimension``, if given)."""
+ conn = conn if conn is not None else connection()
+ table = source_table(model_id)
+ if table not in table_names(conn) or "collection" not in schema_names(conn, table):
+ return set()
+ held: Set[Tuple[str, str]] = set()
+ columns = ["doc_id", "chunk_id", "vector", "model"]
+ for row in iter_rows(conn, table, collection_filter(collection), columns):
+ if not written_by(row, model_id):
+ continue
+ if not _usable_vector(row.get("vector"), dimension):
+ continue
+ held.add(chunk_key(row))
+ return held
+
+
+def _usable_vector(vector: Any, dimension: Optional[int]) -> bool:
+ if not isinstance(vector, list) or not vector:
+ return False
+ if dimension is not None and len(vector) != dimension:
+ return False
+ if not all(isinstance(value, (int, float)) for value in vector):
+ return False
+ return all(math.isfinite(value) for value in vector) and any(vector)
+
+
+def resolve_collection_names(
+ *, collections: Optional[Sequence[str]] = None, conn: Optional[Any] = None
+) -> List[str]:
+ """The collections this run covers; a named one that does not exist raises."""
+ conn = conn if conn is not None else connection()
+ existing = _all_collection_names(conn)
+ if not collections:
+ return existing
+ names: List[str] = []
+ for name in collections:
+ if not name.strip():
+ raise MigrationCliError("--collection must not be empty")
+ if name not in existing:
+ raise MissingCollectionError(f"collection '{name}' does not exist")
+ if name not in names:
+ names.append(name)
+ return names
+
+
+def _all_collection_names(conn: Any) -> List[str]:
+ """Every collection name; ``list_collections`` would create missing tables."""
+ if METADATA_TABLE not in table_names(conn):
+ return []
+ return sorted(
+ {
+ str(row.get("name") or "")
+ for row in iter_rows(conn, METADATA_TABLE, "true", ["name"])
+ if row.get("name")
+ }
+ )
+
+
+def resolve_target(model_id: str) -> str:
+ """The target id, normalised like a binding; a blank id raises."""
+ from xagent.core.tools.core.RAG_tools.management.collection_manager import (
+ _normalize_model_id,
+ )
+
+ target = _normalize_model_id(model_id)
+ if not target:
+ raise MigrationCliError("--model must name a model")
+ return target
+
+
+def scope_collection(
+ name: str,
+ *,
+ conn: Optional[Any] = None,
+ target: Optional[str] = None,
+) -> CollectionScope:
+ """Count one collection's source rows and, with a target, those it lacks."""
+ target_model_id = target
+ conn = conn if conn is not None else connection()
+ eligibility = precompute_eligibility(
+ name, conn=conn, target_model_id=target_model_id
+ )
+ item = CollectionScope(name=name)
+ if eligibility.reason:
+ item.skip_reason = eligibility.reason
+ return item
+ source_model_id = eligibility.model_id
+ assert source_model_id is not None
+ item.source_model_id = source_model_id
+ have: Optional[Set[Tuple[str, str]]] = None
+ if target_model_id:
+ have = target_chunk_ids(name, target_model_id, conn=conn)
+ scan = scan_source(
+ name,
+ source_model_id,
+ have=have,
+ core=core_chunk_pairs(name, conn=conn),
+ conn=conn,
+ )
+ item.chunks = scan.rows
+ item.skipped = dict(scan.skipped)
+ item.usable_documents = len(scan.usable_documents)
+ if have is not None:
+ item.missing_chunks = scan.pending
+ return item
+
+
+def scope_report(
+ *,
+ collections: Optional[Sequence[str]] = None,
+ target_model_id: Optional[str] = None,
+) -> List[CollectionScope]:
+ """Count every collection in scope, keeping a failed read on its collection."""
+ target = resolve_target(target_model_id) if target_model_id is not None else None
+ conn = connection()
+ names = resolve_collection_names(collections=collections, conn=conn)
+ report: List[CollectionScope] = []
+ for name in names:
+ try:
+ report.append(scope_collection(name, conn=conn, target=target))
+ except MissingCollectionError:
+ # Listed, then gone before it was read: nothing to report on.
+ report.append(CollectionScope(name=name, skip_reason=SKIP_NO_METADATA))
+ except Exception as exc: # noqa: BLE001 - kept on the collection, exit 1
+ report.append(
+ CollectionScope(name=name, error=f"{type(exc).__name__}: {exc}")
+ )
+ return report
+
+
+def describe(item: CollectionScope) -> str:
+ parts = [f"{item.usable_documents} usable documents", f"{item.chunks} chunks"]
+ if item.source_model_id:
+ parts.insert(0, f"source model {item.source_model_id}")
+ if item.missing_chunks is not None:
+ parts.append(f"{item.missing_chunks} missing for the target model")
+ if item.skipped:
+ parts.append(
+ "skipped " + ", ".join(f"{k}={v}" for k, v in sorted(item.skipped.items()))
+ )
+ return f"{item.name}: " + "; ".join(parts)
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+ logging.basicConfig(
+ level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+ )
+ parser = argparse.ArgumentParser(
+ description=(
+ "Report the scope of an embedding migration.\n\n"
+ "Read-only. For each collection in scope it logs the model it is\n"
+ "bound to, its usable documents and its chunks; with --model, also\n"
+ "how many usable chunks the target model does not hold yet.\n\n"
+ "'chunks' counts every source row of the collection, including the\n"
+ "rows listed after 'skipped', which are never counted as missing.\n"
+ "A collection that cannot be counted is logged as skipped with a\n"
+ "reason and does not change the exit code.\n\n"
+ "Exit codes: 0 every collection in scope was read (not: every\n"
+ "collection can be migrated); 1 a collection failed to read, or\n"
+ "nothing is in scope; 2 the run could not start.\n\n"
+ "Not a readiness check: the --model id is not checked against the\n"
+ "configured models or their dimension, and the store is read live\n"
+ "without a lock, so the counts are not a consistent snapshot."
+ ),
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ )
+ parser.add_argument(
+ "--model",
+ default=None,
+ help="Target embedding model id; adds the missing-chunk count",
+ )
+ parser.add_argument(
+ "--collection",
+ action="append",
+ metavar="NAME",
+ help="Restrict to this collection; repeatable",
+ )
+ args = parser.parse_args(argv)
+
+ try:
+ report = scope_report(collections=args.collection, target_model_id=args.model)
+ except MigrationCliError as exc:
+ logger.error("%s", exc)
+ return 2
+ except Exception as exc: # noqa: BLE001 - report and exit non-zero
+ logger.error("Scope report failed: %s", exc, exc_info=True)
+ return 2
+
+ if not report:
+ logger.warning("No collections in scope")
+ return 1
+ for item in report:
+ if item.error:
+ logger.error("%s: failed: %s", item.name, item.error)
+ elif item.usable:
+ logger.info("%s", describe(item))
+ else:
+ logger.info("%s: skipped (%s)", item.name, item.skip_reason)
+ return 1 if any(item.error for item in report) else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/src/xagent/providers/vector_store/lancedb.py b/src/xagent/providers/vector_store/lancedb.py
index 26e9a0546..71692328d 100644
--- a/src/xagent/providers/vector_store/lancedb.py
+++ b/src/xagent/providers/vector_store/lancedb.py
@@ -130,14 +130,20 @@ class LanceDBConnectionManager:
@staticmethod
@functools.lru_cache(maxsize=None)
def get_default_lancedb_dir() -> str:
- """Get the default LanceDB directory path.
+ """The default LanceDB directory, created if missing; cached."""
+ path = LanceDBConnectionManager.locate_default_lancedb_dir()
+ path.mkdir(parents=True, exist_ok=True)
+ return str(path)
+
+ @staticmethod
+ def locate_default_lancedb_dir() -> Path:
+ """Get the default LanceDB directory path, without creating it.
Returns:
Default LanceDB directory path
Note:
Priority: legacy_location (if has data) > ~/.xagent/data/lancedb
- Result is cached after first call.
"""
# TODO: This function has confusing logic and potential issues.
#
@@ -162,15 +168,13 @@ class LanceDBConnectionManager:
)
if legacy_dir.is_dir() and list(legacy_dir.iterdir()):
logger.info(f"Using legacy LanceDB location: {legacy_dir}")
- return str(legacy_dir)
+ return legacy_dir
# Use new default location from unified config module
# Note: get_lancedb_path() returns relative path, so we combine with
# get_storage_root() to get absolute path.
# The former is: new_dir = Path.home() / ".xagent" / "data" / "lancedb"
- new_dir = get_storage_root() / get_lancedb_path()
- new_dir.mkdir(parents=True, exist_ok=True)
- return str(new_dir)
+ return get_storage_root() / get_lancedb_path()
@staticmethod
def _cleanup_expired_connections() -> None:
@@ -255,11 +259,14 @@ class LanceDBConnectionManager:
"""
return self.get_connection(self.resolve_dir_from_env(env_var))
- def resolve_dir_from_env(self, env_var: str = "LANCEDB_DIR") -> str:
+ def resolve_dir_from_env(
+ self, env_var: str = "LANCEDB_DIR", *, create: bool = True
+ ) -> str:
"""Resolve the configured directory without opening a connection.
Args:
env_var: Environment variable name containing database directory
+ create: Create the default directory when it is the one resolved
Returns:
Normalized directory path
@@ -273,7 +280,11 @@ class LanceDBConnectionManager:
if db_dir is None:
if env_var == "LANCEDB_DIR":
# Use default path only for the standard LANCEDB_DIR environment variable
- db_dir = self.get_default_lancedb_dir()
+ db_dir = (
+ self.get_default_lancedb_dir()
+ if create
+ else str(self.locate_default_lancedb_dir())
+ )
# Reached on every connection request now that the KB stores
# no longer memoise one (LanceDBVectorStore still does), so
# this must be neither INFO nor an eager f-string.
diff --git a/tests/migrations/test_embedding_migration_scope.py b/tests/migrations/test_embedding_migration_scope.py
new file mode 100644
index 000000000..9855bdf3b
--- /dev/null
+++ b/tests/migrations/test_embedding_migration_scope.py
@@ -0,0 +1,1373 @@
+"""Scope: eligibility, strict reads and per-collection isolation."""
+
+from __future__ import annotations
+
+import re
+import sqlite3
+from pathlib import Path
+from typing import Any, Dict, Iterator, List, Optional
+
+import pyarrow as pa
+import pytest
+
+from xagent.migrations.embedding import scope as mod
+
+SCHEMA = pa.schema(
+ [
+ ("collection", pa.string()),
+ ("doc_id", pa.string()),
+ ("chunk_id", pa.string()),
+ ("parse_hash", pa.string()),
+ ("user_id", pa.int64()),
+ ("text", pa.string()),
+ ("chunk_hash", pa.string()),
+ ("metadata", pa.string()),
+ ("model", pa.string()),
+ ("vector", pa.list_(pa.float32(), 2)),
+ ]
+)
+
+
+def _row(
+ chunk_id: str = "c1",
+ *,
+ doc_id: Optional[str] = "d1",
+ user_id: Optional[int] = 7,
+ text: Optional[str] = "t",
+ collection: str = "kb",
+ model: Optional[str] = None,
+) -> Dict[str, Any]:
+ return {
+ "collection": collection,
+ "doc_id": doc_id,
+ "chunk_id": chunk_id,
+ "parse_hash": "p",
+ "user_id": user_id,
+ "text": text,
+ "chunk_hash": "h",
+ "metadata": None,
+ "model": model,
+ "vector": [0.5, 0.5],
+ }
+
+
+class _Query:
+ def __init__(self, table: "_Table") -> None:
+ self.table = table
+ self.filter = ""
+ self.columns: List[str] = []
+
+ def where(self, filter: str) -> "_Query":
+ self.filter = filter
+ return self
+
+ def select(self, columns: List[str]) -> "_Query":
+ self.columns = list(columns)
+ return self
+
+ def limit(self, value: Any) -> "_Query":
+ return self
+
+ def to_batches(self, batch_size: int) -> Iterator[Any]:
+ _require_fields(self.table, self.filter)
+ rows = [row for row in self.table.rows if _matches(row, self.filter)]
+ for start in range(0, len(rows), batch_size):
+ yield pa.RecordBatch.from_pylist(
+ [
+ {key: row.get(key) for key in self.columns or row}
+ for row in rows[start : start + batch_size]
+ ]
+ )
+
+ def to_arrow(self) -> Any:
+ _require_fields(self.table, self.filter)
+ return pa.Table.from_pylist(
+ [row for row in self.table.rows if _matches(row, self.filter)]
+ )
+
+
+def _require_fields(table: "_Table", where: str) -> None:
+ # LanceDB rejects a filter on a column the table lacks; so must the double.
+ for name in re.findall(r"(\w+) (?:=|!=|IS NULL)", where or ""):
+ if name not in table.schema.names:
+ raise RuntimeError(f"No field named {name}")
+
+
+def _matches(row: Dict[str, Any], where: str) -> bool:
+ if not where or where == "true":
+ return True
+ return all(_matches_clause(row, part) for part in where.split(" AND "))
+
+
+def _matches_clause(row: Dict[str, Any], clause: str) -> bool:
+ if clause.startswith("(") and clause.endswith(")"):
+ return any(_matches_clause(row, part) for part in clause[1:-1].split(" OR "))
+ if clause.endswith(" IS NULL"):
+ return row.get(clause[: -len(" IS NULL")].strip()) is None
+ for operator in (" != ", " = "):
+ if operator in clause:
+ column, _, literal = clause.partition(operator)
+ value = row.get(column.strip())
+ if operator == " != ":
+ # SQL: NULL is not "different from", it is unknown.
+ return value is not None and str(value) != literal.strip("'")
+ return str(value) == literal.strip("'")
+ raise AssertionError(f"unsupported filter clause: {clause!r}")
+
+
+class _Table:
+ def __init__(self, rows: List[Dict[str, Any]], names: Optional[List[str]] = None):
+ self.rows = rows
+ self.schema = pa.schema(
+ [(name, pa.string()) for name in (names or list(SCHEMA.names))]
+ )
+
+ def search(self) -> _Query:
+ return _Query(self)
+
+ def count_rows(self, filter: Optional[str] = None) -> int:
+ _require_fields(self, filter or "")
+ return len([row for row in self.rows if _matches(row, filter or "")])
+
+
+class _Conn:
+ def __init__(self, tables: Dict[str, _Table]) -> None:
+ self.tables = tables
+
+ def open_table(self, name: str) -> _Table:
+ if name not in self.tables:
+ raise FileNotFoundError(name)
+ return self.tables[name]
+
+ def list_tables(self) -> List[str]:
+ return list(self.tables)
+
+
+def _bound(
+ model_id: Optional[str] = "BAAI/old-model",
+ dimension: Optional[int] = 2,
+ name: str = "kb",
+) -> Dict[str, Any]:
+ return {
+ "name": name,
+ "embedding_model_id": model_id,
+ "embedding_dimension": dimension,
+ "extra_metadata": "{}",
+ }
+
+
+def _metadata(rows: List[Dict[str, Any]]) -> _Table:
+ return _Table(rows, names=["name", "embedding_model_id", "embedding_dimension"])
+
+
+def _wire(
+ monkeypatch: pytest.MonkeyPatch, conn: _Conn, engine: str = "lancedb"
+) -> None:
+ monkeypatch.setattr(mod, "connection", lambda: conn)
+ monkeypatch.setattr(mod, "collection_engine", lambda info: engine)
+
+
+def _writer(model_id: str = "BAAI/old-model") -> str:
+ return mod.source_table(model_id)
+
+
+# --- eligibility ---
+
+
+def test_an_eligible_collection_is_counted(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): _Table([_row("c1"), _row("c2")]),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ item = mod.scope_collection("kb")
+
+ assert item.usable
+ assert item.source_model_id == "BAAI/old-model"
+ assert (item.usable_documents, item.chunks) == (1, 2)
+ assert mod.describe(item).startswith("kb: source model BAAI/old-model")
+ assert "missing" not in mod.describe(item)
+
+
+def test_an_eligible_collection_without_rows_is_counted_empty(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()])})
+ _wire(monkeypatch, conn)
+
+ item = mod.scope_collection("kb")
+
+ assert item.usable and (item.usable_documents, item.chunks) == (0, 0)
+ assert mod.precompute_eligibility("kb").model_id == "BAAI/old-model"
+
+
+def test_an_unbound_collection_is_skipped(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound(None, None)]),
+ _writer(): _Table([_row()]),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ assert mod.precompute_eligibility("kb").reason == mod.SKIP_UNBOUND
+ assert mod.scope_collection("kb").skip_reason == mod.SKIP_UNBOUND
+
+
+def test_a_binding_without_a_dimension_is_skipped(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound("BAAI/old-model", None)]),
+ _writer(): _Table([_row()]),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ assert mod.precompute_eligibility("kb").reason == mod.SKIP_BOUND_WITHOUT_DIMENSION
+
+
+def test_rows_in_another_table_are_skipped(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ "embeddings_BAAI_old_model": _Table([_row()]),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ assert _writer() == "embeddings_baai_old_model"
+ assert mod.precompute_eligibility("kb").reason == mod.SKIP_ROWS_IN_OTHER_TABLE
+
+
+def test_rows_split_across_tables_are_skipped(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): _Table([_row()]),
+ "embeddings_BAAI_old_model": _Table([_row("c2")]),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ assert mod.precompute_eligibility("kb").reason == mod.SKIP_ROWS_IN_MULTIPLE_TABLES
+
+
+def test_rows_of_another_collection_do_not_disqualify(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): _Table([_row()]),
+ "embeddings_legacy_name": _Table([_row(collection="other")]),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ assert mod.precompute_eligibility("kb").reason is None
+
+
+ "column", ["collection", "doc_id", "chunk_id", "user_id", "text"]
+)
+def test_a_missing_required_column_is_skipped(
+ monkeypatch: pytest.MonkeyPatch, column: str
+) -> None:
+ table = _Table([_row()], names=[name for name in SCHEMA.names if name != column])
+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()]), _writer(): table})
+ _wire(monkeypatch, conn)
+
+ assert mod.precompute_eligibility("kb").reason == mod.SKIP_MISSING_COLUMN
+
+
+def test_another_engine_is_skipped(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()])})
+ _wire(monkeypatch, conn, engine="milvus")
+
+ assert mod.precompute_eligibility("kb").reason == mod.SKIP_OTHER_ENGINE
+
+
+def test_an_unreadable_engine_binding_is_skipped(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()])})
+ _wire(monkeypatch, conn, engine="")
+
+ assert mod.precompute_eligibility("kb").reason == mod.SKIP_UNREADABLE_BINDING
+
+
+# --- counts ---
+
+
+def test_excluded_rows_are_counted_once(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): _Table(
+ [
+ _row("c1"),
+ _row("c2", doc_id=None),
+ _row("c3", user_id=None),
+ _row("c4", text=None),
+ _row("", doc_id="d9"),
+ _row("c5", doc_id="d9"),
+ ]
+ ),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ item = mod.scope_collection("kb")
+
+ assert item.chunks == 6
+ assert item.skipped == {
+ mod.SKIP_NO_DOC_ID: 1,
+ mod.SKIP_NO_CHUNK_ID: 1,
+ mod.SKIP_NO_OWNER: 1,
+ mod.SKIP_NO_TEXT: 1,
+ }
+ assert item.usable_documents == 2
+ assert "2 usable documents" in mod.describe(item)
+
+
+def test_the_missing_count_uses_the_target_table(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): _Table([_row("c1"), _row("c2"), _row("c3")]),
+ mod.source_table("new-model"): _Table([_row("c1", model="new-model")]),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ assert mod.scope_collection("kb", target="new-model").missing_chunks == 2
+
+
[email protected]("vector", [[0.0, 0.0], [float("nan"), 0.5]])
+def test_a_target_row_with_a_bad_vector_is_still_missing(
+ monkeypatch: pytest.MonkeyPatch, vector: List[float]
+) -> None:
+ bad = _row("c1", model="new-model")
+ bad["vector"] = vector
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): _Table([_row("c1")]),
+ mod.source_table("new-model"): _Table([bad]),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ assert mod.scope_collection("kb", target="new-model").missing_chunks == 1
+
+
+def test_a_chunk_id_held_under_another_document_is_still_missing(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): _Table([_row("c1", doc_id="d1"), _row("c1", doc_id="d2")]),
+ mod.source_table("new-model"): _Table(
+ [_row("c1", doc_id="d1", model="new-model")]
+ ),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ assert mod.scope_collection("kb", target="new-model").missing_chunks == 1
+
+
+def test_only_rows_the_target_wrote_in_its_writer_table_are_held(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.source_table("new-model"): _Table(
+ [
+ _row("c1", model="new-model"),
+ _row("c2", model="new_model"),
+ _row("c4", model="new-model "),
+ ]
+ ),
+ "embeddings_new_legacy": _Table([_row("c3", model="new-model")]),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ held = mod.target_chunk_ids("kb", "new-model", dimension=2)
+
+ assert held == {("d1", "c1")}
+
+
+ ("target_rows", "missing"), [([], 1), ([_row("c1", model="new-model")], 0)]
+)
+def test_a_coreless_row_is_excluded_not_missing(
+ monkeypatch: pytest.MonkeyPatch, target_rows: List[Dict[str, Any]], missing: int
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): _Table([_row("c1"), _row("c2")]),
+ mod.CORE_TABLE: _Table(
+ [{"collection": "kb", "doc_id": "d1", "chunk_id": "c1"}],
+ names=["collection", "doc_id", "chunk_id"],
+ ),
+ mod.source_table("new-model"): _Table(target_rows),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ item = mod.scope_collection("kb", target="new-model")
+
+ assert item.chunks == 2
+ assert item.missing_chunks == missing
+ assert f"{missing} missing for the target model" in mod.describe(item)
+ assert item.skipped == {mod.SKIP_NO_CORE_CHUNK: 1}
+
+
+def _core(*keys: Any) -> _Table:
+ return _Table(
+ [{"collection": "kb", "doc_id": d, "chunk_id": c} for d, c in keys],
+ names=["collection", "doc_id", "chunk_id"],
+ )
+
+
+def test_rows_of_a_document_the_core_table_lacks_are_missing(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): _Table([_row("c1", doc_id="d1"), _row("c2", doc_id="d2")]),
+ mod.CORE_TABLE: _core(("d1", "c1")),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ item = mod.scope_collection("kb", target="new-model")
+
+ assert (item.missing_chunks, item.skipped) == (2, {})
+
+
+def test_ids_are_compared_as_stored(monkeypatch: pytest.MonkeyPatch) -> None:
+ source = _Table([_row("c1", doc_id="d1 ")])
+ held = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): source,
+ mod.source_table("new-model"): _Table(
+ [_row("c1", doc_id="d1", model="new-model")]
+ ),
+ }
+ )
+ _wire(monkeypatch, held)
+ assert mod.scope_collection("kb", target="new-model").missing_chunks == 1
+
+ backed = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): source,
+ mod.CORE_TABLE: _core(("d1 ", "c2")),
+ }
+ )
+ _wire(monkeypatch, backed)
+ assert mod.scope_collection("kb").skipped == {mod.SKIP_NO_CORE_CHUNK: 1}
+
+
+# --- strict reads and isolation ---
+
+
+def test_an_absent_table_or_row_is_a_missing_collection() -> None:
+ with pytest.raises(mod.MissingCollectionError):
+ mod.collection_info("kb")
+ _save_collection()
+ with pytest.raises(mod.MissingCollectionError):
+ mod.collection_info("other")
+
+
+def test_a_missing_collection_named_explicitly_stops_the_run(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _wire(monkeypatch, _Conn({}))
+
+ assert mod.main(["--collection", "kb"]) == 2
+
+
+def test_an_unknown_name_stops_the_run_before_any_collection_is_read(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _wire(monkeypatch, _Conn({mod.METADATA_TABLE: _metadata([_bound()])}))
+ scoped: List[str] = []
+ monkeypatch.setattr(
+ mod, "scope_collection", lambda name, *a, **k: scoped.append(name)
+ )
+
+ assert mod.main(["--collection", "kb", "--collection", "nope"]) == 2
+ assert scoped == []
+
+
+def test_a_row_that_does_not_decode_is_an_error(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ import asyncio
+
+ from xagent.core.tools.core.RAG_tools.core.schemas import CollectionInfo
+ from xagent.core.tools.core.RAG_tools.management import collection_manager as cm
+
+ asyncio.run(
+ cm.collection_manager.save_collection(
+ CollectionInfo(
+ name="kb",
+ embedding_model_id="BAAI/old-model",
+ embedding_dimension=2,
+ )
+ )
+ )
+ conn = mod.connection()
+ conn.open_table(mod.METADATA_TABLE).update(
+ "name = 'kb'", {"extra_metadata": "{not json"}
+ )
+ # The manager reports this row as "not found"; the strict read must not.
+ with pytest.raises(ValueError, match="not found"):
+ cm.get_collection_sync("kb")
+ with pytest.raises(mod.MigrationCliError) as raised:
+ mod.collection_info("kb")
+ assert not isinstance(raised.value, mod.MissingCollectionError)
+
+ monkeypatch.setattr(mod, "resolve_collection_names", lambda **kwargs: ["kb"])
+ (item,) = mod.scope_report()
+ assert item.error and item.skip_reason is None
+ assert mod.main([]) == 1
+
+
+class _FailingWhere:
+ def __init__(self, table: Any) -> None:
+ self._table = table
+
+ def where(self, where: str) -> Any:
+ if "'bad'" in where:
+ raise OSError("metadata store unavailable")
+ return self._table.search().where(where)
+
+
+def test_a_metadata_store_failure_is_not_a_missing_collection(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ import asyncio
+
+ from xagent.core.tools.core.RAG_tools.core.schemas import CollectionInfo
+ from xagent.core.tools.core.RAG_tools.management import collection_manager as cm
+
+ for name in ("bad", "ok"):
+ asyncio.run(
+ cm.collection_manager.save_collection(
+ CollectionInfo(
+ name=name,
+ embedding_model_id="BAAI/old-model",
+ embedding_dimension=2,
+ )
+ )
+ )
+ real = mod.connection()
+
+ class _Broken:
+ def __init__(self, table: Any) -> None:
+ self._table = table
+
+ def search(self) -> Any:
+ return _FailingWhere(self._table)
+
+ def __getattr__(self, attr: str) -> Any:
+ return getattr(self._table, attr)
+
+ class _Conn:
+ def open_table(self, name: str) -> Any:
+ table = real.open_table(name)
+ return _Broken(table) if name == mod.METADATA_TABLE else table
+
+ def __getattr__(self, attr: str) -> Any:
+ return getattr(real, attr)
+
+ monkeypatch.setattr(mod, "connection", lambda: _Conn())
+ monkeypatch.setattr(mod, "resolve_collection_names", lambda **kwargs: ["bad", "ok"])
+
+ report = {item.name: item for item in mod.scope_report()}
+
+ assert "metadata store unavailable" in (report["bad"].error or "")
+ assert report["bad"].skip_reason is None
+ assert report["ok"].error is None
+ assert mod.main([]) == 1
+
+
[email protected]("setting", ["qdrant", "no-such-engine"])
+def test_an_unusable_backend_setting_is_an_error_not_a_skip(
+ monkeypatch: pytest.MonkeyPatch, setting: str
+) -> None:
+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()])})
+ monkeypatch.setattr(mod, "connection", lambda: conn)
+ monkeypatch.setenv("XAGENT_VECTOR_BACKEND", setting)
+
+ (item,) = mod.scope_report(collections=["kb"])
+
+ assert item.error and item.skip_reason is None
+
+
+def test_a_malformed_engine_binding_is_unreadable() -> None:
+ from types import SimpleNamespace
+
+ from xagent.core.tools.core.RAG_tools.kb.coordinator import (
+ KB_STORAGE_METADATA_KEY,
+ )
+
+ info = SimpleNamespace(extra_metadata={KB_STORAGE_METADATA_KEY: "bogus"})
+
+ assert mod.collection_engine(info) == ""
+
+
+def test_a_coordinator_that_fails_to_build_is_an_error(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from types import SimpleNamespace
+
+ from xagent.core.tools.core.RAG_tools.kb import coordinator
+
+ def _broken() -> Any:
+ raise ValueError("cannot build the coordinator")
+
+ monkeypatch.setattr(coordinator, "get_kb_coordinator", _broken)
+
+ with pytest.raises(ValueError):
+ mod.collection_engine(SimpleNamespace(extra_metadata={}))
+
+
+def test_a_reader_failure_does_not_stop_the_other_collections(
+ monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ monkeypatch.setattr(mod, "resolve_collection_names", lambda **kwargs: ["a", "b"])
+
+ def _scope(name: str, target: Any = None, *, conn: Any, **kwargs: Any) -> Any:
+ if name == "a":
+ raise RuntimeError("batch read failed")
+ return mod.CollectionScope(name=name)
+
+ monkeypatch.setattr(mod, "scope_collection", _scope)
+ monkeypatch.setattr(mod, "connection", lambda: object())
+
+ report = mod.scope_report()
+
+ assert [item.name for item in report] == ["a", "b"]
+ assert "batch read failed" in (report[0].error or "")
+ assert mod.main([]) == 1
+ assert caplog.text.count("batch read failed") == 1
+
+
+def test_a_discovered_collection_that_vanished_is_not_a_failure(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(
+ mod, "resolve_collection_names", lambda **kwargs: ["gone", "ok"]
+ )
+
+ def _scope(name: str, target: Any = None, *, conn: Any, **kwargs: Any) -> Any:
+ if name == "gone":
+ raise mod.MissingCollectionError("gone")
+ return mod.CollectionScope(name=name)
+
+ monkeypatch.setattr(mod, "scope_collection", _scope)
+ monkeypatch.setattr(mod, "connection", lambda: object())
+
+ report = mod.scope_report()
+
+ assert report[0].skip_reason == mod.SKIP_NO_METADATA and report[0].error is None
+ assert mod.main([]) == 0
+
+
+def test_an_empty_scope_is_reported_as_such(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(mod, "resolve_collection_names", lambda **kwargs: [])
+
+ assert mod.main([]) == 1
+
+
+# --- the store directory and the arguments ---
+
+
[email protected]("argv", [["--collection", "kb"], []])
[email protected]("configured", [True, False])
+def test_a_missing_lancedb_dir_is_left_absent(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ argv: List[str],
+ configured: bool,
+) -> None:
+ from xagent.providers.vector_store.lancedb import LanceDBConnectionManager
+
+ missing = tmp_path / "absent" / "lancedb"
+ if configured:
+ monkeypatch.setenv("LANCEDB_DIR", str(missing))
+ else:
+ monkeypatch.delenv("LANCEDB_DIR", raising=False)
+ monkeypatch.setattr(
+ LanceDBConnectionManager,
+ "locate_default_lancedb_dir",
+ staticmethod(lambda: missing),
+ )
+ models = tmp_path / "language_models"
+ models.mkdir()
+ monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(models))
+
+ assert mod.main(argv) == 2
+
+ assert not (tmp_path / "absent").exists()
+ assert list(models.iterdir()) == []
+
+
+def test_the_application_still_creates_its_default_directory(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from xagent.providers.vector_store.lancedb import LanceDBConnectionManager
+
+ default = tmp_path / "default" / "lancedb"
+ monkeypatch.delenv("LANCEDB_DIR", raising=False)
+ monkeypatch.setattr(
+ LanceDBConnectionManager,
+ "locate_default_lancedb_dir",
+ staticmethod(lambda: default),
+ )
+ cached = LanceDBConnectionManager.get_default_lancedb_dir
+ cached.cache_clear()
+ try:
+ assert LanceDBConnectionManager().resolve_dir_from_env(create=False)
+ assert not default.exists()
+ assert LanceDBConnectionManager().resolve_dir_from_env() == str(default)
+ assert default.is_dir()
+ finally:
+ cached.cache_clear()
+
+
+def _unreadable(*args: Any, **kwargs: Any) -> Any:
+ raise OSError("unreadable")
+
+
+ ("patched", "argv"),
+ [
+ ("connection", []),
+ ("_all_collection_names", []),
+ (None, ["--collection", " "]),
+ ],
+)
+def test_a_run_that_cannot_start_exits_2(
+ monkeypatch: pytest.MonkeyPatch, patched: Optional[str], argv: List[str]
+) -> None:
+ _wire(monkeypatch, _Conn({mod.METADATA_TABLE: _metadata([_bound()])}))
+ if patched:
+ monkeypatch.setattr(mod, patched, _unreadable)
+
+ assert mod.main(argv) == 2
+
+
[email protected]("model", ["", " ", "none"])
+def test_a_blank_target_stops_the_run(
+ monkeypatch: pytest.MonkeyPatch, model: str
+) -> None:
+ _wire(monkeypatch, _Conn({mod.METADATA_TABLE: _metadata([_bound()])}))
+
+ assert mod.main(["--model", model]) == 2
+
+
+def test_a_target_id_is_normalised_like_a_binding() -> None:
+ assert mod.resolve_target(" new-model ") == "new-model"
+
+
+def test_collection_names_are_matched_as_stored() -> None:
+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound(), _bound(name=" kb")])})
+
+ assert mod.resolve_collection_names(collections=["kb", " kb", "kb"], conn=conn) == [
+ "kb",
+ " kb",
+ ]
+ with pytest.raises(mod.MissingCollectionError):
+ mod.resolve_collection_names(collections=["kb "], conn=conn)
+
+
+def test_reading_is_batched(monkeypatch: pytest.MonkeyPatch) -> None:
+ rows = [_row(f"c{index}", doc_id=f"d{index}") for index in range(2500)]
+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()]), _writer(): _Table(rows)})
+ _wire(monkeypatch, conn)
+ sizes: List[int] = []
+ batches = _Query.to_batches
+
+ def _spy(query: _Query, batch_size: int) -> Iterator[Any]:
+ sizes.append(batch_size)
+ return batches(query, batch_size)
+
+ monkeypatch.setattr(_Query, "to_batches", _spy)
+
+ assert mod.scope_collection("kb").chunks == 2500
+ assert mod.READ_BATCH_SIZE in sizes
+
+
+# --- imports ---
+
+
+def test_every_module_in_the_migration_package_imports() -> None:
+ import importlib
+ import pkgutil
+
+ import xagent.migrations.embedding as package
+
+ names = {module.name for module in pkgutil.iter_modules(package.__path__)}
+ assert "scope" in names
+ for name in sorted(names):
+ importlib.import_module(f"xagent.migrations.embedding.{name}")
+
+
+# --- source and target must not share a physical table ---
+
+
+def test_a_target_that_writes_the_source_table_is_refused(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from xagent.core.tools.core.RAG_tools.LanceDB.model_tag_utils import (
+ written_embeddings_table_name,
+ )
+
+ # ``old-model`` and ``old_model`` share one writer spelling.
+ assert written_embeddings_table_name("old-model") == written_embeddings_table_name(
+ "old_model"
+ )
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound("old-model", 2)]),
+ mod.source_table("old-model"): _Table([_row("c1")]),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ assert (
+ mod.precompute_eligibility("kb", target_model_id="old_model").reason
+ == mod.SKIP_SHARED_TABLE
+ )
+ item = mod.scope_collection("kb", target="old_model")
+ assert item.skip_reason == mod.SKIP_SHARED_TABLE
+ assert item.chunks == 0
+
+
+def test_a_target_that_is_the_bound_model_is_refused(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): _Table([_row("c1")]),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ item = mod.scope_collection("kb", target="BAAI/old-model")
+
+ assert item.skip_reason == mod.SKIP_TARGET_IS_SOURCE
+ assert item.missing_chunks is None
+
+
+def test_the_refusal_happens_before_anything_is_counted(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound("old-model", 2)]),
+ mod.source_table("old-model"): _Table([_row("c1")]),
+ }
+ )
+ _wire(monkeypatch, conn)
+ monkeypatch.setattr(mod, "resolve_target", lambda model_id: "old_model")
+ monkeypatch.setattr(mod, "resolve_collection_names", lambda **kwargs: ["kb"])
+
+ report = mod.scope_report(target_model_id="old_model")
+
+ assert [item.error for item in report] == [None]
+ assert report[0].skip_reason == mod.SKIP_SHARED_TABLE
+ assert report[0].chunks == 0
+
+
+# --- a target table whose rows cannot be attributed to a collection ---
+
+_NO_COLLECTION_COLUMN = ["doc_id", "chunk_id", "user_id", "text", "model", "vector"]
+
+
+def test_a_target_table_that_cannot_name_its_collection_is_refused(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Without the column a row cannot be proven foreign.
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): _Table([_row("c1")]),
+ mod.source_table("new-model"): _Table(
+ [_row("c9")], names=_NO_COLLECTION_COLUMN
+ ),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ assert (
+ mod.precompute_eligibility("kb", target_model_id="new-model").reason
+ == mod.SKIP_TARGET_ROWS_IN_OTHER_TABLE
+ )
+
+
+def test_a_target_row_no_model_wrote_is_refused(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): _Table([_row("c1")]),
+ mod.source_table("new-model"): _Table([_row("c9", model=None)]),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ assert (
+ mod.precompute_eligibility("kb", target_model_id="new-model").reason
+ == mod.SKIP_TARGET_ROWS_IN_OTHER_TABLE
+ )
+
+
+def test_an_empty_target_table_without_the_column_does_not_refuse(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): _Table([_row("c1")]),
+ mod.source_table("new-model"): _Table([], names=_NO_COLLECTION_COLUMN),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ assert mod.precompute_eligibility("kb", target_model_id="new-model").reason is None
+ assert mod.scope_collection("kb", target="new-model").missing_chunks == 1
+
+
+def test_an_empty_collection_with_an_absent_source_table_is_counted(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()])})
+ _wire(monkeypatch, conn)
+
+ eligibility = mod.precompute_eligibility("kb", target_model_id="new-model")
+
+ assert eligibility.reason is None
+ assert mod.scope_collection("kb", target="new-model").missing_chunks == 0
+
+
+# --- one pass over the source rows ---
+
+
+def test_the_scan_classifies_every_row_once(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn(
+ {
+ mod.METADATA_TABLE: _metadata([_bound()]),
+ _writer(): _Table(
+ [
+ _row("c1", doc_id="d1"),
+ _row("c2", doc_id="d2"),
+ _row("c3", doc_id="d1"),
+ _row("c4", doc_id=None),
+ _row("c5", doc_id="d3", user_id=None),
+ ]
+ ),
+ }
+ )
+ _wire(monkeypatch, conn)
+
+ scan = mod.scan_source("kb", "BAAI/old-model", have={("d2", "c2")})
+
+ assert scan.usable_documents == {"d1", "d2"}
+ assert scan.pending == 2
+ assert scan.skipped == {mod.SKIP_NO_DOC_ID: 1, mod.SKIP_NO_OWNER: 1}
+
+
+def test_a_missing_source_table_scans_empty(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()])})
+ _wire(monkeypatch, conn)
+
+ scan = mod.scan_source("kb", "BAAI/old-model", have=set())
+
+ assert (scan.rows, scan.pending) == (0, 0)
+
+
+# --- the same readers on a real LanceDB store ---
+
+_CORE_SCHEMA = pa.schema(
+ [
+ ("collection", pa.string()),
+ ("doc_id", pa.string()),
+ ("chunk_id", pa.string()),
+ ("user_id", pa.int64()),
+ ]
+)
+
+
+def _save_collection() -> None:
+ import asyncio
+
+ from xagent.core.tools.core.RAG_tools.core.schemas import CollectionInfo
+ from xagent.core.tools.core.RAG_tools.management import collection_manager as cm
+
+ asyncio.run(
+ cm.collection_manager.save_collection(
+ CollectionInfo(
+ name="kb",
+ embedding_model_id="BAAI/old-model",
+ embedding_dimension=2,
+ embeddings=4,
+ )
+ )
+ )
+
+
+def _real_table(name: str, rows: List[Dict[str, Any]], schema: Any = SCHEMA) -> None:
+ mod.connection().create_table(name, data=pa.Table.from_pylist(rows, schema=schema))
+
+
+def _versions() -> Dict[str, int]:
+ conn = mod.connection()
+ return {name: conn.open_table(name).version for name in mod.table_names(conn)}
+
+
+def real_store() -> None:
+ _save_collection()
+
+
+def test_a_real_store_is_counted_and_left_as_it_was(real_store: None) -> None:
+ _real_table(
+ mod.source_table("BAAI/old-model"),
+ [
+ _row("c1", model="BAAI/old-model"),
+ _row("c2", model="BAAI/old-model"),
+ _row("c3", model="BAAI/old-model", user_id=None),
+ ],
+ )
+ _real_table(
+ mod.CORE_TABLE,
+ [
+ {"collection": "kb", "doc_id": "d1", "chunk_id": chunk_id, "user_id": 7}
+ for chunk_id in ("c1", "c2", "c3")
+ ],
+ schema=_CORE_SCHEMA,
+ )
+ _real_table(mod.source_table("new-model"), [_row("c1", model="new-model")])
+ before = _versions()
+
+ item = mod.scope_collection("kb", target="new-model")
+
+ assert (item.skip_reason, item.error) == (None, None)
+ assert (item.chunks, item.missing_chunks) == (3, 1)
+ assert item.skipped == {mod.SKIP_NO_OWNER: 1}
+ assert _versions() == before
+
+
+def test_a_row_another_model_wrote_in_the_target_table_is_refused_on_a_real_store(
+ real_store: None,
+) -> None:
+ # ``new-model`` and ``new_model`` are distinct bindings with one writer table.
+ _real_table(
+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]
+ )
+ _real_table(mod.source_table("new-model"), [_row("c9", model="new_model")])
+
+ eligibility = mod.precompute_eligibility("kb", target_model_id="new-model")
+
+ assert eligibility.reason == mod.SKIP_TARGET_ROWS_IN_OTHER_TABLE
+
+
+def test_a_vector_dimension_is_checked_only_when_given_on_a_real_store(
+ real_store: None,
+) -> None:
+ wide = pa.schema(
+ [
+ f if f.name != "vector" else pa.field("vector", pa.list_(pa.float32(), 3))
+ for f in SCHEMA
+ ]
+ )
+ _real_table(
+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]
+ )
+ target = _row("c1", model="new-model")
+ target["vector"] = [0.5, 0.5, 0.5]
+ _real_table(mod.source_table("new-model"), [target], schema=wide)
+
+ assert mod.target_chunk_ids("kb", "new-model", dimension=2) == set()
+ assert mod.target_chunk_ids("kb", "new-model") == {("d1", "c1")}
+ assert mod.scope_collection("kb", target="new-model").missing_chunks == 0
+
+
[email protected]("holds_rows", [True, False])
+def test_a_source_table_without_a_collection_column_is_refused_on_a_real_store(
+ real_store: None, holds_rows: bool
+) -> None:
+ narrow = pa.schema([f for f in SCHEMA if f.name != "collection"])
+ row = _row("c1", model="BAAI/old-model")
+ del row["collection"]
+ rows = [row] if holds_rows else []
+ _real_table(mod.source_table("BAAI/old-model"), rows, schema=narrow)
+
+ item = mod.scope_collection("kb")
+
+ assert (item.skip_reason, item.error) == (mod.SKIP_MISSING_COLUMN, None)
+
+
[email protected]("missing", ["doc_id", "chunk_id"])
[email protected]("table", ["core", "target"])
+def test_a_table_without_a_compared_column_is_an_error_on_a_real_store(
+ real_store: None, missing: str, table: str
+) -> None:
+ _real_table(
+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]
+ )
+ if table == "core":
+ schema = pa.schema([f for f in _CORE_SCHEMA if f.name != missing])
+ row: Dict[str, Any] = {"collection": "kb", "doc_id": "d1", "chunk_id": "c1"}
+ name = mod.CORE_TABLE
+ else:
+ schema = pa.schema([f for f in SCHEMA if f.name != missing])
+ row = _row("c1", model="new-model")
+ name = mod.source_table("new-model")
+ row.pop(missing)
+ _real_table(name, [row], schema=schema)
+
+ (item,) = mod.scope_report(collections=["kb"], target_model_id="new-model")
+
+ assert item.error and missing in item.error and item.skip_reason is None
+
+
+def test_more_rows_than_one_page_are_all_read_on_a_real_store(
+ real_store: None,
+) -> None:
+ keys = [f"c{index:02d}" for index in range(25)]
+ _real_table(
+ mod.source_table("BAAI/old-model"),
+ [_row(key, model="BAAI/old-model") for key in keys],
+ )
+ _real_table(
+ mod.CORE_TABLE,
+ [
+ {"collection": "kb", "doc_id": "d1", "chunk_id": key, "user_id": 7}
+ for key in keys
+ ],
+ schema=_CORE_SCHEMA,
+ )
+ _real_table(
+ mod.source_table("new-model"),
+ [_row(key, model="new-model") for key in keys[:20]],
+ )
+
+ item = mod.scope_collection("kb", target="new-model")
+
+ assert (item.chunks, item.missing_chunks, item.skipped) == (25, 5, {})
+
+
+# --- the table rule: the model's own tables, else every embeddings table ---
+
+STRAY = "embeddings_legacy_name"
+
+
+def test_rows_only_in_a_table_the_model_does_not_own_are_refused_on_a_real_store(
+ real_store: None,
+) -> None:
+ _real_table(STRAY, [_row("c1", model="legacy")])
+
+ assert mod.scope_collection("kb").skip_reason == mod.SKIP_ROWS_IN_OTHER_TABLE
+
+
+def test_without_an_own_table_other_collections_do_not_refuse_on_a_real_store(
+ real_store: None,
+) -> None:
+ _real_table(STRAY, [_row("c1", collection="other", model="legacy")])
+
+ item = mod.scope_collection("kb")
+
+ assert (item.skip_reason, item.error, item.chunks) == (None, None, 0)
+
+
+def test_an_own_table_hides_every_other_table_on_a_real_store(
+ real_store: None,
+) -> None:
+ _real_table(
+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]
+ )
+ _real_table(STRAY, [_row("c9", model="legacy")])
+
+ item = mod.scope_collection("kb")
+
+ assert (item.skip_reason, item.chunks) == (None, 1)
+
+
+def test_a_new_target_is_refused_when_another_table_holds_the_collection(
+ real_store: None,
+) -> None:
+ _real_table(
+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]
+ )
+ _real_table(STRAY, [_row("c9", model="legacy")])
+
+ item = mod.scope_collection("kb", target="new-model")
+
+ assert item.skip_reason == mod.SKIP_TARGET_ROWS_IN_OTHER_TABLE
+
+
+def test_a_target_table_that_is_not_the_writer_is_refused_on_a_real_store(
+ real_store: None,
+) -> None:
+ once, writer = mod.own_tables("ACME/new-model")[::-1]
+ assert once != writer
+ _real_table(
+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]
+ )
+ _real_table(once, [_row("c1", model="ACME/new-model")])
+
+ item = mod.scope_collection("kb", target="ACME/new-model")
+
+ assert item.skip_reason == mod.SKIP_TARGET_ROWS_IN_OTHER_TABLE
+
+
+def test_a_table_without_a_collection_column_holds_the_collection_on_a_real_store(
+ real_store: None,
+) -> None:
+ row = _row("c1", model="legacy")
+ del row["collection"]
+ _real_table(
+ STRAY, [row], schema=pa.schema([f for f in SCHEMA if f.name != "collection"])
+ )
+
+ assert mod.scope_collection("kb").skip_reason == mod.SKIP_ROWS_IN_OTHER_TABLE
+
+
+def test_a_target_writer_table_without_a_model_column_is_refused_on_a_real_store(
+ real_store: None,
+) -> None:
+ _real_table(
+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]
+ )
+ row = _row("c1")
+ del row["model"]
+ narrow = pa.schema([f for f in SCHEMA if f.name != "model"])
+ _real_table(mod.source_table("new-model"), [row], schema=narrow)
+
+ item = mod.scope_collection("kb", target="new-model")
+
+ assert (item.skip_reason, item.error) == (mod.SKIP_TARGET_ROWS_IN_OTHER_TABLE, None)
+
+
+def test_another_own_target_table_is_refused_beside_the_writer_on_a_real_store(
+ real_store: None,
+) -> None:
+ writer, once = mod.own_tables("ACME/new-model")
+ _real_table(
+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]
+ )
+ _real_table(writer, [_row("c1", model="ACME/new-model")])
+ _real_table(once, [_row("c9", model="ACME/new-model")])
+
+ item = mod.scope_collection("kb", target="ACME/new-model")
+
+ assert item.skip_reason == mod.SKIP_TARGET_ROWS_IN_OTHER_TABLE
+
+
+def test_a_first_run_counts_every_row_as_missing_on_a_real_store(
+ real_store: None,
+) -> None:
+ _real_table(
+ mod.source_table("BAAI/old-model"),
+ [_row("c1", model="BAAI/old-model"), _row("c2", model="BAAI/old-model")],
+ )
+ _real_table(STRAY, [_row("c9", collection="other", model="legacy")])
+
+ item = mod.scope_collection("kb", target="new-model")
+
+ assert (item.skip_reason, item.missing_chunks) == (None, 2)
+
+
+def test_an_existing_target_table_hides_every_other_table_on_a_real_store(
+ real_store: None,
+) -> None:
+ _real_table(
+ mod.source_table("BAAI/old-model"),
+ [_row("c1", model="BAAI/old-model"), _row("c2", model="BAAI/old-model")],
+ )
+ _real_table(mod.source_table("new-model"), [_row("c1", model="new-model")])
+ _real_table(STRAY, [_row("c9", model="legacy")])
+
+ item = mod.scope_collection("kb", target="new-model")
+
+ assert (item.skip_reason, item.missing_chunks) == (None, 1)
+
+
+def test_a_previous_model_can_be_the_target_on_a_real_store(
+ real_store: None,
+) -> None:
+ _real_table(
+ mod.source_table("BAAI/old-model"),
+ [_row("c1", model="BAAI/old-model"), _row("c2", model="BAAI/old-model")],
+ )
+ _real_table(mod.source_table("prev-model"), [_row("c1", model="prev-model")])
+
+ item = mod.scope_collection("kb", target="prev-model")
+
+ assert (item.skip_reason, item.missing_chunks) == (None, 1)
+
+
+# --- the command never opens the SQL database ---
+
+
+def test_the_command_runs_without_the_sql_database(
+ real_store: None, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ import sqlalchemy
+
+ _real_table(
+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]
+ )
+ _real_table(mod.source_table("new-model"), [_row("c1", model="new-model")])
+ database = tmp_path / "absent.db"
+ monkeypatch.setenv("DATABASE_URL", f"sqlite:///{database}")
+
+ def _refuse(*args: Any, **kwargs: Any) -> Any:
+ raise AssertionError("the SQL database was opened")
+
+ monkeypatch.setattr(sqlalchemy, "create_engine", _refuse)
+ monkeypatch.setattr(sqlite3, "connect", _refuse)
+
+ assert mod.main(["--model", "new-model"]) == 0
+ assert mod.main([]) == 0
+ assert not database.exists()