Enterprise agent platform for building, deploying, and operating AI teammates.
feat(kb): report an embedding migration's scope without writing (#3825)
蓝鱼bluefishe11b436fbd4e
diff --git a/src/xagent/core/tools/core/RAG_tools/kb/coordinator.py b/src/xagent/core/tools/core/RAG_tools/kb/coordinator.pyindex a107d12b8..1234b30df 100644--- a/src/xagent/core/tools/core/RAG_tools/kb/coordinator.py+++ b/src/xagent/core/tools/core/RAG_tools/kb/coordinator.py@@ -2755,6 +2755,11 @@ def reset_kb_coordinator_for_tests() -> None:_coordinator = None+def resolve_collection_backend(collection_info: object | None) -> KBStorageBackend:+ """The engine a collection is bound to, defaulting to the deployment's."""+ return get_kb_coordinator()._resolve_backend(collection_info)++def _run_in_separate_loop(awaitable: Coroutine[Any, Any, T]) -> T:"""Run a coroutine from sync code, including inside an existing event loop."""try:diff --git a/src/xagent/migrations/embedding/__init__.py b/src/xagent/migrations/embedding/__init__.pynew file mode 100644index 000000000..9c251d0ac--- /dev/null+++ b/src/xagent/migrations/embedding/__init__.py@@ -0,0 +1 @@+"""Operator commands for moving knowledge-base embeddings to another model."""diff --git a/src/xagent/migrations/embedding/scope.py b/src/xagent/migrations/embedding/scope.pynew file mode 100644index 000000000..8be64c253--- /dev/null+++ b/src/xagent/migrations/embedding/scope.py@@ -0,0 +1,650 @@+"""Read-only scope of an embedding migration, per knowledge-base collection.++ python -m xagent.migrations.embedding.scope [--model TARGET] \+ [--collection NAME ...]+"""++from __future__ import annotations++import argparse+import logging+import math+import sys+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any, Dict, Iterator, List, Optional, Sequence, Set, Tuple++import lancedb++from xagent.core.tools.core.RAG_tools.LanceDB.model_tag_utils import (+ embeddings_table_candidates,+ written_embeddings_table_name,+)+from xagent.core.tools.core.RAG_tools.LanceDB.schema_manager import _safe_close_table+from xagent.core.tools.core.RAG_tools.utils.lancedb_query_utils import list_table_names+from xagent.core.tools.core.RAG_tools.utils.string_utils import escape_lancedb_string+from xagent.providers.vector_store.lancedb import LanceDBConnectionManager++logger = logging.getLogger(__name__)++METADATA_TABLE = "collection_metadata"+CORE_TABLE = "chunks"++# Why a collection is not counted.+SKIP_OTHER_ENGINE = "other-engine"+SKIP_UNREADABLE_BINDING = "unreadable-binding"+SKIP_UNBOUND = "unbound"+SKIP_BOUND_WITHOUT_DIMENSION = "bound-without-dimension"+SKIP_ROWS_IN_OTHER_TABLE = "rows-in-another-table"+SKIP_ROWS_IN_MULTIPLE_TABLES = "rows-in-multiple-tables"+SKIP_MISSING_COLUMN = "missing-required-column"+SKIP_SHARED_TABLE = "shared-source-target-table"+SKIP_TARGET_ROWS_IN_OTHER_TABLE = "target-rows-in-another-table"+SKIP_TARGET_IS_SOURCE = "target-is-source"+SKIP_NO_METADATA = "no-metadata"+# Why a source row is not counted as missing.+SKIP_NO_DOC_ID = "no-doc-id"+SKIP_NO_CHUNK_ID = "no-chunk-id"+SKIP_NO_OWNER = "no-owner"+SKIP_NO_TEXT = "no-text"+SKIP_NO_CORE_CHUNK = "no-core-chunk"++REQUIRED_COLUMNS = ("collection", "doc_id", "chunk_id", "user_id", "text")++READ_BATCH_SIZE = 1000+EMBEDDINGS_PREFIX = "embeddings_"+++class MigrationCliError(RuntimeError):+ """A problem the command reports: exit 2 before any collection is read,+ otherwise an error on the collection it concerns."""+++class MissingCollectionError(MigrationCliError):+ """The collection has no metadata row (never existed, or was just deleted)."""+++@dataclass(frozen=True)+class SearchableChunk:+ """One row of the source model's embeddings table."""++ doc_id: str+ chunk_id: str+ text: str+ user_id: Optional[int]+++@dataclass(frozen=True)+class Eligibility:+ """Whether one collection can be counted: ``reason`` is ``None`` only then."""++ model_id: Optional[str] = None+ reason: Optional[str] = None+++@dataclass+class CollectionScope:+ """One collection's report line."""++ name: str+ source_model_id: Optional[str] = None+ skip_reason: Optional[str] = None+ usable_documents: int = 0+ chunks: int = 0+ # ``None`` unless the collection was counted against a target.+ missing_chunks: Optional[int] = None+ skipped: Dict[str, int] = field(default_factory=dict)+ error: Optional[str] = None++ @property+ def usable(self) -> bool:+ return self.skip_reason is None and self.error is None+++def connection() -> Any:+ """The application's LanceDB directory, opened only if it already exists."""+ db_dir = LanceDBConnectionManager().resolve_dir_from_env(create=False)+ if not Path(db_dir).is_dir():+ raise MigrationCliError(f"LanceDB directory {db_dir} is not a directory")+ return lancedb.connect(db_dir)+++def table_names(conn: Any) -> Set[str]:+ return set(list_table_names(conn))+++def source_table(model_id: str) -> str:+ return written_embeddings_table_name(model_id)+++def iter_rows(+ conn: Any,+ table: str,+ where: str,+ columns: Sequence[str],+) -> Iterator[Dict[str, Any]]:+ """Stream rows of ``table``; a failed read or a missing column raises."""+ opened = conn.open_table(table)+ try:+ missing = [name for name in columns if name not in opened.schema.names]+ if missing:+ raise MigrationCliError(f"table '{table}' has no {', '.join(missing)}")+ reader = (+ opened.search()+ .where(where)+ .select(list(columns))+ .limit(None)+ .to_batches(READ_BATCH_SIZE)+ )+ for batch in reader:+ yield from batch.to_pylist()+ finally:+ _safe_close_table(opened)+++def schema_names(conn: Any, table: str) -> List[str]:+ opened = conn.open_table(table)+ try:+ return list(opened.schema.names)+ finally:+ _safe_close_table(opened)+++def collection_filter(collection: str) -> str:+ return f"collection = '{escape_lancedb_string(collection)}'"+++def chunk_key(row: Dict[str, Any]) -> Tuple[str, str]:+ """The ``(doc_id, chunk_id)`` every reader compares, exactly as stored."""+ return (str(row.get("doc_id") or ""), str(row.get("chunk_id") or ""))+++def collection_info(collection: str, conn: Optional[Any] = None) -> Any:+ """The metadata row; only an absent row is :class:`MissingCollectionError`."""+ from xagent.core.tools.core.RAG_tools.core.schemas import CollectionInfo++ conn = conn if conn is not None else connection()+ if METADATA_TABLE not in table_names(conn):+ raise MissingCollectionError(f"collection '{collection}' does not exist")+ table = conn.open_table(METADATA_TABLE)+ try:+ rows = (+ table.search()+ .where(f"name = '{escape_lancedb_string(collection)}'")+ .to_arrow()+ .to_pylist()+ )+ finally:+ _safe_close_table(table)+ if not rows:+ raise MissingCollectionError(f"collection '{collection}' does not exist")+ try:+ return CollectionInfo.from_storage(rows[0])+ except Exception as exc: # noqa: BLE001 - an error on the collection+ raise MigrationCliError(+ f"collection '{collection}' metadata does not decode: {exc}"+ ) from exc+++def collection_engine(info: Any) -> str:+ """The engine a collection is bound to; ``""`` for a malformed binding."""+ from xagent.core.tools.core.RAG_tools.kb.coordinator import (+ get_kb_coordinator,+ resolve_collection_backend,+ )++ # Built outside the try: only a malformed binding may become a reason.+ get_kb_coordinator()+ try:+ return str(resolve_collection_backend(info).value)+ except ValueError:+ return ""+++def own_tables(model_id: str) -> List[str]:+ """The tables search tries for ``model_id`` before asking the model hub."""+ return embeddings_table_candidates(model_id)+++def written_by(row: Dict[str, Any], model_id: str) -> bool:+ """Whether the row's ``model`` is ``model_id``: distinct ids can share a writer+ table (``old-model`` and ``old_model``)."""+ return row.get("model") == model_id+++def _count_rows(conn: Any, table: str, where: Optional[str] = None) -> int:+ opened = conn.open_table(table)+ try:+ return int(opened.count_rows(where) if where else opened.count_rows())+ finally:+ _safe_close_table(opened)+++def holds_rows(conn: Any, table: str, collection: str) -> bool:+ if "collection" not in schema_names(conn, table):+ # Cannot attribute a row to a collection: any row counts.+ return _count_rows(conn, table) > 0+ return _count_rows(conn, table, collection_filter(collection)) > 0+++def _holds_rows_written_elsewhere(+ conn: Any, table: str, collection: str, model: str+) -> bool:+ """Whether ``table`` holds rows of this collection that ``model`` did not+ write; without a ``collection`` column any row counts."""+ if "collection" not in schema_names(conn, table):+ return _count_rows(conn, table) > 0+ if "model" not in schema_names(conn, table):+ return _count_rows(conn, table, collection_filter(collection)) > 0+ foreign = (+ f"{collection_filter(collection)} AND "+ f"(model IS NULL OR model != '{escape_lancedb_string(model)}')"+ )+ return _count_rows(conn, table, foreign) > 0+++def _embeddings_tables(names: Set[str]) -> List[str]:+ return sorted(name for name in names if name.startswith(EMBEDDINGS_PREFIX))+++def target_tables_hold_foreign_rows(+ conn: Any, collection: str, model_id: str, names: Set[str], source: List[str]+) -> bool:+ """Whether search for ``model_id`` could serve rows of this collection other+ than those ``model_id`` wrote in its writer table."""+ present = [table for table in own_tables(model_id) if table in names]+ if not present:+ # Search would ask the model hub for a table this command cannot name.+ return any(+ holds_rows(conn, table, collection)+ for table in _embeddings_tables(names)+ if table not in source+ )+ writer = source_table(model_id)+ for table in present:+ if table != writer:+ if holds_rows(conn, table, collection):+ return True+ elif _holds_rows_written_elsewhere(conn, table, collection, model_id):+ return True+ return False+++def missing_columns(conn: Any, table: str) -> List[str]:+ present = set(schema_names(conn, table))+ return [name for name in REQUIRED_COLUMNS if name not in present]+++def precompute_eligibility(+ collection: str,+ *,+ conn: Optional[Any] = None,+ target_model_id: Optional[str] = None,+) -> Eligibility:+ """:func:`_source_eligibility`, also refusing a target that is the bound model."""+ result = _source_eligibility(collection, conn=conn, target_model_id=target_model_id)+ if result.reason is None and target_model_id and target_model_id == result.model_id:+ return Eligibility(reason=SKIP_TARGET_IS_SOURCE)+ return result+++def _source_eligibility(+ collection: str,+ *,+ conn: Optional[Any] = None,+ target_model_id: Optional[str] = None,+) -> Eligibility:+ """Eligible only when bound with a dimension, with no row search could serve+ from a table but the writer's, and (with a target) no shared or foreign table."""+ from xagent.core.tools.core.RAG_tools.management.collection_manager import (+ _normalize_model_id,+ )++ conn = conn if conn is not None else connection()+ info = collection_info(collection, conn)+ engine = collection_engine(info)+ if engine != "lancedb":+ return Eligibility(+ reason=SKIP_OTHER_ENGINE if engine else SKIP_UNREADABLE_BINDING+ )+ model = _normalize_model_id(info.embedding_model_id)+ if not model:+ return Eligibility(reason=SKIP_UNBOUND)+ dimension = info.embedding_dimension+ if not isinstance(dimension, int) or dimension <= 0:+ return Eligibility(reason=SKIP_BOUND_WITHOUT_DIMENSION)+ table = source_table(model)+ own = own_tables(model)+ names = table_names(conn)+ if table in names and missing_columns(conn, table):+ return Eligibility(reason=SKIP_MISSING_COLUMN)+ if target_model_id and target_model_id != model:+ if set(own) & set(own_tables(target_model_id)):+ return Eligibility(reason=SKIP_SHARED_TABLE)+ if target_tables_hold_foreign_rows(+ conn, collection, target_model_id, names, own+ ):+ return Eligibility(reason=SKIP_TARGET_ROWS_IN_OTHER_TABLE)+ if not any(name in names for name in own):+ # Search would ask the model hub for a table this command cannot name.+ candidates = _embeddings_tables(names)+ if any(holds_rows(conn, name, collection) for name in candidates):+ return Eligibility(reason=SKIP_ROWS_IN_OTHER_TABLE)+ return Eligibility(model_id=model)+ holders = [+ name for name in own if name in names and holds_rows(conn, name, collection)+ ]+ if table not in holders:+ if holders:+ return Eligibility(reason=SKIP_ROWS_IN_OTHER_TABLE)+ return Eligibility(model_id=model)+ if len(holders) > 1:+ return Eligibility(reason=SKIP_ROWS_IN_MULTIPLE_TABLES)+ return Eligibility(model_id=model)+++@dataclass+class SourceScan:+ """One pass over a collection's source rows; ``pending`` counts the usable rows+ the target does not hold."""++ usable_documents: Set[str] = field(default_factory=set)+ pending: int = 0+ skipped: Dict[str, int] = field(default_factory=dict)+ rows: int = 0+++def scan_source(+ collection: str,+ model_id: str,+ *,+ have: Optional[Set[Tuple[str, str]]] = None,+ core: Optional[Set[Tuple[str, str]]] = None,+ conn: Optional[Any] = None,+) -> SourceScan:+ """Classify every source row once; without ``core`` (:func:`core_chunk_pairs`)+ no row is checked against the core table."""+ conn = conn if conn is not None else connection()+ scan = SourceScan()+ core_keys = core if core is not None else set()+ core_docs = {doc_id for doc_id, _ in core_keys}+ table = source_table(model_id)+ if table not in table_names(conn):+ return scan+ for row in iter_rows(+ conn,+ table,+ collection_filter(collection),+ ["doc_id", "chunk_id", "user_id", "text"],+ ):+ scan.rows += 1+ key = chunk_key(row)+ chunk = SearchableChunk(+ doc_id=key[0],+ chunk_id=key[1],+ text=str(row.get("text") or ""),+ user_id=_owner(row.get("user_id")),+ )+ reason = skip_reason(chunk)+ if reason is None and chunk.doc_id in core_docs and key not in core_keys:+ reason = SKIP_NO_CORE_CHUNK+ if reason:+ scan.skipped[reason] = scan.skipped.get(reason, 0) + 1+ continue+ scan.usable_documents.add(chunk.doc_id)+ if have is not None and key not in have:+ scan.pending += 1+ return scan+++def core_chunk_pairs(+ collection: str, *, conn: Optional[Any] = None+) -> Optional[Set[Tuple[str, str]]]:+ """Every ``(doc_id, chunk_id)`` the core chunks table holds, or ``None`` without one."""+ conn = conn if conn is not None else connection()+ if CORE_TABLE not in table_names(conn):+ return None+ return {+ chunk_key(row)+ for row in iter_rows(+ conn,+ CORE_TABLE,+ collection_filter(collection),+ ["doc_id", "chunk_id"],+ )+ }+++def skip_reason(chunk: SearchableChunk) -> Optional[str]:+ """Why this source row is excluded, or ``None`` when it is usable."""+ if not chunk.doc_id.strip():+ return SKIP_NO_DOC_ID+ if not chunk.chunk_id.strip():+ return SKIP_NO_CHUNK_ID+ if chunk.user_id is None:+ return SKIP_NO_OWNER+ if not chunk.text:+ return SKIP_NO_TEXT+ return None+++def _owner(raw: Any) -> Optional[int]:+ if raw is None:+ return None+ try:+ return int(raw)+ except (TypeError, ValueError):+ return None+++def target_chunk_ids(+ collection: str,+ model_id: str,+ *,+ conn: Optional[Any] = None,+ dimension: Optional[int] = None,+) -> Set[Tuple[str, str]]:+ """The ``(doc_id, chunk_id)`` of each row in the target's writer table that+ ``model_id`` wrote with a finite, non-zero vector (of ``dimension``, if given)."""+ conn = conn if conn is not None else connection()+ table = source_table(model_id)+ if table not in table_names(conn) or "collection" not in schema_names(conn, table):+ return set()+ held: Set[Tuple[str, str]] = set()+ columns = ["doc_id", "chunk_id", "vector", "model"]+ for row in iter_rows(conn, table, collection_filter(collection), columns):+ if not written_by(row, model_id):+ continue+ if not _usable_vector(row.get("vector"), dimension):+ continue+ held.add(chunk_key(row))+ return held+++def _usable_vector(vector: Any, dimension: Optional[int]) -> bool:+ if not isinstance(vector, list) or not vector:+ return False+ if dimension is not None and len(vector) != dimension:+ return False+ if not all(isinstance(value, (int, float)) for value in vector):+ return False+ return all(math.isfinite(value) for value in vector) and any(vector)+++def resolve_collection_names(+ *, collections: Optional[Sequence[str]] = None, conn: Optional[Any] = None+) -> List[str]:+ """The collections this run covers; a named one that does not exist raises."""+ conn = conn if conn is not None else connection()+ existing = _all_collection_names(conn)+ if not collections:+ return existing+ names: List[str] = []+ for name in collections:+ if not name.strip():+ raise MigrationCliError("--collection must not be empty")+ if name not in existing:+ raise MissingCollectionError(f"collection '{name}' does not exist")+ if name not in names:+ names.append(name)+ return names+++def _all_collection_names(conn: Any) -> List[str]:+ """Every collection name; ``list_collections`` would create missing tables."""+ if METADATA_TABLE not in table_names(conn):+ return []+ return sorted(+ {+ str(row.get("name") or "")+ for row in iter_rows(conn, METADATA_TABLE, "true", ["name"])+ if row.get("name")+ }+ )+++def resolve_target(model_id: str) -> str:+ """The target id, normalised like a binding; a blank id raises."""+ from xagent.core.tools.core.RAG_tools.management.collection_manager import (+ _normalize_model_id,+ )++ target = _normalize_model_id(model_id)+ if not target:+ raise MigrationCliError("--model must name a model")+ return target+++def scope_collection(+ name: str,+ *,+ conn: Optional[Any] = None,+ target: Optional[str] = None,+) -> CollectionScope:+ """Count one collection's source rows and, with a target, those it lacks."""+ target_model_id = target+ conn = conn if conn is not None else connection()+ eligibility = precompute_eligibility(+ name, conn=conn, target_model_id=target_model_id+ )+ item = CollectionScope(name=name)+ if eligibility.reason:+ item.skip_reason = eligibility.reason+ return item+ source_model_id = eligibility.model_id+ assert source_model_id is not None+ item.source_model_id = source_model_id+ have: Optional[Set[Tuple[str, str]]] = None+ if target_model_id:+ have = target_chunk_ids(name, target_model_id, conn=conn)+ scan = scan_source(+ name,+ source_model_id,+ have=have,+ core=core_chunk_pairs(name, conn=conn),+ conn=conn,+ )+ item.chunks = scan.rows+ item.skipped = dict(scan.skipped)+ item.usable_documents = len(scan.usable_documents)+ if have is not None:+ item.missing_chunks = scan.pending+ return item+++def scope_report(+ *,+ collections: Optional[Sequence[str]] = None,+ target_model_id: Optional[str] = None,+) -> List[CollectionScope]:+ """Count every collection in scope, keeping a failed read on its collection."""+ target = resolve_target(target_model_id) if target_model_id is not None else None+ conn = connection()+ names = resolve_collection_names(collections=collections, conn=conn)+ report: List[CollectionScope] = []+ for name in names:+ try:+ report.append(scope_collection(name, conn=conn, target=target))+ except MissingCollectionError:+ # Listed, then gone before it was read: nothing to report on.+ report.append(CollectionScope(name=name, skip_reason=SKIP_NO_METADATA))+ except Exception as exc: # noqa: BLE001 - kept on the collection, exit 1+ report.append(+ CollectionScope(name=name, error=f"{type(exc).__name__}: {exc}")+ )+ return report+++def describe(item: CollectionScope) -> str:+ parts = [f"{item.usable_documents} usable documents", f"{item.chunks} chunks"]+ if item.source_model_id:+ parts.insert(0, f"source model {item.source_model_id}")+ if item.missing_chunks is not None:+ parts.append(f"{item.missing_chunks} missing for the target model")+ if item.skipped:+ parts.append(+ "skipped " + ", ".join(f"{k}={v}" for k, v in sorted(item.skipped.items()))+ )+ return f"{item.name}: " + "; ".join(parts)+++def main(argv: Optional[List[str]] = None) -> int:+ logging.basicConfig(+ level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"+ )+ parser = argparse.ArgumentParser(+ description=(+ "Report the scope of an embedding migration.\n\n"+ "Read-only. For each collection in scope it logs the model it is\n"+ "bound to, its usable documents and its chunks; with --model, also\n"+ "how many usable chunks the target model does not hold yet.\n\n"+ "'chunks' counts every source row of the collection, including the\n"+ "rows listed after 'skipped', which are never counted as missing.\n"+ "A collection that cannot be counted is logged as skipped with a\n"+ "reason and does not change the exit code.\n\n"+ "Exit codes: 0 every collection in scope was read (not: every\n"+ "collection can be migrated); 1 a collection failed to read, or\n"+ "nothing is in scope; 2 the run could not start.\n\n"+ "Not a readiness check: the --model id is not checked against the\n"+ "configured models or their dimension, and the store is read live\n"+ "without a lock, so the counts are not a consistent snapshot."+ ),+ formatter_class=argparse.RawDescriptionHelpFormatter,+ )+ parser.add_argument(+ "--model",+ default=None,+ help="Target embedding model id; adds the missing-chunk count",+ )+ parser.add_argument(+ "--collection",+ action="append",+ metavar="NAME",+ help="Restrict to this collection; repeatable",+ )+ args = parser.parse_args(argv)++ try:+ report = scope_report(collections=args.collection, target_model_id=args.model)+ except MigrationCliError as exc:+ logger.error("%s", exc)+ return 2+ except Exception as exc: # noqa: BLE001 - report and exit non-zero+ logger.error("Scope report failed: %s", exc, exc_info=True)+ return 2++ if not report:+ logger.warning("No collections in scope")+ return 1+ for item in report:+ if item.error:+ logger.error("%s: failed: %s", item.name, item.error)+ elif item.usable:+ logger.info("%s", describe(item))+ else:+ logger.info("%s: skipped (%s)", item.name, item.skip_reason)+ return 1 if any(item.error for item in report) else 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/src/xagent/providers/vector_store/lancedb.py b/src/xagent/providers/vector_store/lancedb.pyindex 26e9a0546..71692328d 100644--- a/src/xagent/providers/vector_store/lancedb.py+++ b/src/xagent/providers/vector_store/lancedb.py@@ -130,14 +130,20 @@ class LanceDBConnectionManager:@staticmethod@functools.lru_cache(maxsize=None)def get_default_lancedb_dir() -> str:- """Get the default LanceDB directory path.+ """The default LanceDB directory, created if missing; cached."""+ path = LanceDBConnectionManager.locate_default_lancedb_dir()+ path.mkdir(parents=True, exist_ok=True)+ return str(path)++ @staticmethod+ def locate_default_lancedb_dir() -> Path:+ """Get the default LanceDB directory path, without creating it.Returns:Default LanceDB directory pathNote:Priority: legacy_location (if has data) > ~/.xagent/data/lancedb- Result is cached after first call."""# TODO: This function has confusing logic and potential issues.#@@ -162,15 +168,13 @@ class LanceDBConnectionManager:)if legacy_dir.is_dir() and list(legacy_dir.iterdir()):logger.info(f"Using legacy LanceDB location: {legacy_dir}")- return str(legacy_dir)+ return legacy_dir# Use new default location from unified config module# Note: get_lancedb_path() returns relative path, so we combine with# get_storage_root() to get absolute path.# The former is: new_dir = Path.home() / ".xagent" / "data" / "lancedb"- new_dir = get_storage_root() / get_lancedb_path()- new_dir.mkdir(parents=True, exist_ok=True)- return str(new_dir)+ return get_storage_root() / get_lancedb_path()@staticmethoddef _cleanup_expired_connections() -> None:@@ -255,11 +259,14 @@ class LanceDBConnectionManager:"""return self.get_connection(self.resolve_dir_from_env(env_var))- def resolve_dir_from_env(self, env_var: str = "LANCEDB_DIR") -> str:+ def resolve_dir_from_env(+ self, env_var: str = "LANCEDB_DIR", *, create: bool = True+ ) -> str:"""Resolve the configured directory without opening a connection.Args:env_var: Environment variable name containing database directory+ create: Create the default directory when it is the one resolvedReturns:Normalized directory path@@ -273,7 +280,11 @@ class LanceDBConnectionManager:if db_dir is None:if env_var == "LANCEDB_DIR":# Use default path only for the standard LANCEDB_DIR environment variable- db_dir = self.get_default_lancedb_dir()+ db_dir = (+ self.get_default_lancedb_dir()+ if create+ else str(self.locate_default_lancedb_dir())+ )# Reached on every connection request now that the KB stores# no longer memoise one (LanceDBVectorStore still does), so# this must be neither INFO nor an eager f-string.diff --git a/tests/migrations/test_embedding_migration_scope.py b/tests/migrations/test_embedding_migration_scope.pynew file mode 100644index 000000000..9855bdf3b--- /dev/null+++ b/tests/migrations/test_embedding_migration_scope.py@@ -0,0 +1,1373 @@+"""Scope: eligibility, strict reads and per-collection isolation."""++from __future__ import annotations++import re+import sqlite3+from pathlib import Path+from typing import Any, Dict, Iterator, List, Optional++import pyarrow as pa+import pytest++from xagent.migrations.embedding import scope as mod++SCHEMA = pa.schema(+ [+ ("collection", pa.string()),+ ("doc_id", pa.string()),+ ("chunk_id", pa.string()),+ ("parse_hash", pa.string()),+ ("user_id", pa.int64()),+ ("text", pa.string()),+ ("chunk_hash", pa.string()),+ ("metadata", pa.string()),+ ("model", pa.string()),+ ("vector", pa.list_(pa.float32(), 2)),+ ]+)+++def _row(+ chunk_id: str = "c1",+ *,+ doc_id: Optional[str] = "d1",+ user_id: Optional[int] = 7,+ text: Optional[str] = "t",+ collection: str = "kb",+ model: Optional[str] = None,+) -> Dict[str, Any]:+ return {+ "collection": collection,+ "doc_id": doc_id,+ "chunk_id": chunk_id,+ "parse_hash": "p",+ "user_id": user_id,+ "text": text,+ "chunk_hash": "h",+ "metadata": None,+ "model": model,+ "vector": [0.5, 0.5],+ }+++class _Query:+ def __init__(self, table: "_Table") -> None:+ self.table = table+ self.filter = ""+ self.columns: List[str] = []++ def where(self, filter: str) -> "_Query":+ self.filter = filter+ return self++ def select(self, columns: List[str]) -> "_Query":+ self.columns = list(columns)+ return self++ def limit(self, value: Any) -> "_Query":+ return self++ def to_batches(self, batch_size: int) -> Iterator[Any]:+ _require_fields(self.table, self.filter)+ rows = [row for row in self.table.rows if _matches(row, self.filter)]+ for start in range(0, len(rows), batch_size):+ yield pa.RecordBatch.from_pylist(+ [+ {key: row.get(key) for key in self.columns or row}+ for row in rows[start : start + batch_size]+ ]+ )++ def to_arrow(self) -> Any:+ _require_fields(self.table, self.filter)+ return pa.Table.from_pylist(+ [row for row in self.table.rows if _matches(row, self.filter)]+ )+++def _require_fields(table: "_Table", where: str) -> None:+ # LanceDB rejects a filter on a column the table lacks; so must the double.+ for name in re.findall(r"(\w+) (?:=|!=|IS NULL)", where or ""):+ if name not in table.schema.names:+ raise RuntimeError(f"No field named {name}")+++def _matches(row: Dict[str, Any], where: str) -> bool:+ if not where or where == "true":+ return True+ return all(_matches_clause(row, part) for part in where.split(" AND "))+++def _matches_clause(row: Dict[str, Any], clause: str) -> bool:+ if clause.startswith("(") and clause.endswith(")"):+ return any(_matches_clause(row, part) for part in clause[1:-1].split(" OR "))+ if clause.endswith(" IS NULL"):+ return row.get(clause[: -len(" IS NULL")].strip()) is None+ for operator in (" != ", " = "):+ if operator in clause:+ column, _, literal = clause.partition(operator)+ value = row.get(column.strip())+ if operator == " != ":+ # SQL: NULL is not "different from", it is unknown.+ return value is not None and str(value) != literal.strip("'")+ return str(value) == literal.strip("'")+ raise AssertionError(f"unsupported filter clause: {clause!r}")+++class _Table:+ def __init__(self, rows: List[Dict[str, Any]], names: Optional[List[str]] = None):+ self.rows = rows+ self.schema = pa.schema(+ [(name, pa.string()) for name in (names or list(SCHEMA.names))]+ )++ def search(self) -> _Query:+ return _Query(self)++ def count_rows(self, filter: Optional[str] = None) -> int:+ _require_fields(self, filter or "")+ return len([row for row in self.rows if _matches(row, filter or "")])+++class _Conn:+ def __init__(self, tables: Dict[str, _Table]) -> None:+ self.tables = tables++ def open_table(self, name: str) -> _Table:+ if name not in self.tables:+ raise FileNotFoundError(name)+ return self.tables[name]++ def list_tables(self) -> List[str]:+ return list(self.tables)+++def _bound(+ model_id: Optional[str] = "BAAI/old-model",+ dimension: Optional[int] = 2,+ name: str = "kb",+) -> Dict[str, Any]:+ return {+ "name": name,+ "embedding_model_id": model_id,+ "embedding_dimension": dimension,+ "extra_metadata": "{}",+ }+++def _metadata(rows: List[Dict[str, Any]]) -> _Table:+ return _Table(rows, names=["name", "embedding_model_id", "embedding_dimension"])+++def _wire(+ monkeypatch: pytest.MonkeyPatch, conn: _Conn, engine: str = "lancedb"+) -> None:+ monkeypatch.setattr(mod, "connection", lambda: conn)+ monkeypatch.setattr(mod, "collection_engine", lambda info: engine)+++def _writer(model_id: str = "BAAI/old-model") -> str:+ return mod.source_table(model_id)+++# --- eligibility ---+++def test_an_eligible_collection_is_counted(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): _Table([_row("c1"), _row("c2")]),+ }+ )+ _wire(monkeypatch, conn)++ item = mod.scope_collection("kb")++ assert item.usable+ assert item.source_model_id == "BAAI/old-model"+ assert (item.usable_documents, item.chunks) == (1, 2)+ assert mod.describe(item).startswith("kb: source model BAAI/old-model")+ assert "missing" not in mod.describe(item)+++def test_an_eligible_collection_without_rows_is_counted_empty(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()])})+ _wire(monkeypatch, conn)++ item = mod.scope_collection("kb")++ assert item.usable and (item.usable_documents, item.chunks) == (0, 0)+ assert mod.precompute_eligibility("kb").model_id == "BAAI/old-model"+++def test_an_unbound_collection_is_skipped(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound(None, None)]),+ _writer(): _Table([_row()]),+ }+ )+ _wire(monkeypatch, conn)++ assert mod.precompute_eligibility("kb").reason == mod.SKIP_UNBOUND+ assert mod.scope_collection("kb").skip_reason == mod.SKIP_UNBOUND+++def test_a_binding_without_a_dimension_is_skipped(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound("BAAI/old-model", None)]),+ _writer(): _Table([_row()]),+ }+ )+ _wire(monkeypatch, conn)++ assert mod.precompute_eligibility("kb").reason == mod.SKIP_BOUND_WITHOUT_DIMENSION+++def test_rows_in_another_table_are_skipped(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ "embeddings_BAAI_old_model": _Table([_row()]),+ }+ )+ _wire(monkeypatch, conn)++ assert _writer() == "embeddings_baai_old_model"+ assert mod.precompute_eligibility("kb").reason == mod.SKIP_ROWS_IN_OTHER_TABLE+++def test_rows_split_across_tables_are_skipped(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): _Table([_row()]),+ "embeddings_BAAI_old_model": _Table([_row("c2")]),+ }+ )+ _wire(monkeypatch, conn)++ assert mod.precompute_eligibility("kb").reason == mod.SKIP_ROWS_IN_MULTIPLE_TABLES+++def test_rows_of_another_collection_do_not_disqualify(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): _Table([_row()]),+ "embeddings_legacy_name": _Table([_row(collection="other")]),+ }+ )+ _wire(monkeypatch, conn)++ assert mod.precompute_eligibility("kb").reason is None+++ "column", ["collection", "doc_id", "chunk_id", "user_id", "text"]+)+def test_a_missing_required_column_is_skipped(+ monkeypatch: pytest.MonkeyPatch, column: str+) -> None:+ table = _Table([_row()], names=[name for name in SCHEMA.names if name != column])+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()]), _writer(): table})+ _wire(monkeypatch, conn)++ assert mod.precompute_eligibility("kb").reason == mod.SKIP_MISSING_COLUMN+++def test_another_engine_is_skipped(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()])})+ _wire(monkeypatch, conn, engine="milvus")++ assert mod.precompute_eligibility("kb").reason == mod.SKIP_OTHER_ENGINE+++def test_an_unreadable_engine_binding_is_skipped(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()])})+ _wire(monkeypatch, conn, engine="")++ assert mod.precompute_eligibility("kb").reason == mod.SKIP_UNREADABLE_BINDING+++# --- counts ---+++def test_excluded_rows_are_counted_once(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): _Table(+ [+ _row("c1"),+ _row("c2", doc_id=None),+ _row("c3", user_id=None),+ _row("c4", text=None),+ _row("", doc_id="d9"),+ _row("c5", doc_id="d9"),+ ]+ ),+ }+ )+ _wire(monkeypatch, conn)++ item = mod.scope_collection("kb")++ assert item.chunks == 6+ assert item.skipped == {+ mod.SKIP_NO_DOC_ID: 1,+ mod.SKIP_NO_CHUNK_ID: 1,+ mod.SKIP_NO_OWNER: 1,+ mod.SKIP_NO_TEXT: 1,+ }+ assert item.usable_documents == 2+ assert "2 usable documents" in mod.describe(item)+++def test_the_missing_count_uses_the_target_table(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): _Table([_row("c1"), _row("c2"), _row("c3")]),+ mod.source_table("new-model"): _Table([_row("c1", model="new-model")]),+ }+ )+ _wire(monkeypatch, conn)++ assert mod.scope_collection("kb", target="new-model").missing_chunks == 2++[email protected]("vector", [[0.0, 0.0], [float("nan"), 0.5]])+def test_a_target_row_with_a_bad_vector_is_still_missing(+ monkeypatch: pytest.MonkeyPatch, vector: List[float]+) -> None:+ bad = _row("c1", model="new-model")+ bad["vector"] = vector+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): _Table([_row("c1")]),+ mod.source_table("new-model"): _Table([bad]),+ }+ )+ _wire(monkeypatch, conn)++ assert mod.scope_collection("kb", target="new-model").missing_chunks == 1+++def test_a_chunk_id_held_under_another_document_is_still_missing(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): _Table([_row("c1", doc_id="d1"), _row("c1", doc_id="d2")]),+ mod.source_table("new-model"): _Table(+ [_row("c1", doc_id="d1", model="new-model")]+ ),+ }+ )+ _wire(monkeypatch, conn)++ assert mod.scope_collection("kb", target="new-model").missing_chunks == 1+++def test_only_rows_the_target_wrote_in_its_writer_table_are_held(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.source_table("new-model"): _Table(+ [+ _row("c1", model="new-model"),+ _row("c2", model="new_model"),+ _row("c4", model="new-model "),+ ]+ ),+ "embeddings_new_legacy": _Table([_row("c3", model="new-model")]),+ }+ )+ _wire(monkeypatch, conn)++ held = mod.target_chunk_ids("kb", "new-model", dimension=2)++ assert held == {("d1", "c1")}+++ ("target_rows", "missing"), [([], 1), ([_row("c1", model="new-model")], 0)]+)+def test_a_coreless_row_is_excluded_not_missing(+ monkeypatch: pytest.MonkeyPatch, target_rows: List[Dict[str, Any]], missing: int+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): _Table([_row("c1"), _row("c2")]),+ mod.CORE_TABLE: _Table(+ [{"collection": "kb", "doc_id": "d1", "chunk_id": "c1"}],+ names=["collection", "doc_id", "chunk_id"],+ ),+ mod.source_table("new-model"): _Table(target_rows),+ }+ )+ _wire(monkeypatch, conn)++ item = mod.scope_collection("kb", target="new-model")++ assert item.chunks == 2+ assert item.missing_chunks == missing+ assert f"{missing} missing for the target model" in mod.describe(item)+ assert item.skipped == {mod.SKIP_NO_CORE_CHUNK: 1}+++def _core(*keys: Any) -> _Table:+ return _Table(+ [{"collection": "kb", "doc_id": d, "chunk_id": c} for d, c in keys],+ names=["collection", "doc_id", "chunk_id"],+ )+++def test_rows_of_a_document_the_core_table_lacks_are_missing(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): _Table([_row("c1", doc_id="d1"), _row("c2", doc_id="d2")]),+ mod.CORE_TABLE: _core(("d1", "c1")),+ }+ )+ _wire(monkeypatch, conn)++ item = mod.scope_collection("kb", target="new-model")++ assert (item.missing_chunks, item.skipped) == (2, {})+++def test_ids_are_compared_as_stored(monkeypatch: pytest.MonkeyPatch) -> None:+ source = _Table([_row("c1", doc_id="d1 ")])+ held = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): source,+ mod.source_table("new-model"): _Table(+ [_row("c1", doc_id="d1", model="new-model")]+ ),+ }+ )+ _wire(monkeypatch, held)+ assert mod.scope_collection("kb", target="new-model").missing_chunks == 1++ backed = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): source,+ mod.CORE_TABLE: _core(("d1 ", "c2")),+ }+ )+ _wire(monkeypatch, backed)+ assert mod.scope_collection("kb").skipped == {mod.SKIP_NO_CORE_CHUNK: 1}+++# --- strict reads and isolation ---+++def test_an_absent_table_or_row_is_a_missing_collection() -> None:+ with pytest.raises(mod.MissingCollectionError):+ mod.collection_info("kb")+ _save_collection()+ with pytest.raises(mod.MissingCollectionError):+ mod.collection_info("other")+++def test_a_missing_collection_named_explicitly_stops_the_run(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _wire(monkeypatch, _Conn({}))++ assert mod.main(["--collection", "kb"]) == 2+++def test_an_unknown_name_stops_the_run_before_any_collection_is_read(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _wire(monkeypatch, _Conn({mod.METADATA_TABLE: _metadata([_bound()])}))+ scoped: List[str] = []+ monkeypatch.setattr(+ mod, "scope_collection", lambda name, *a, **k: scoped.append(name)+ )++ assert mod.main(["--collection", "kb", "--collection", "nope"]) == 2+ assert scoped == []+++def test_a_row_that_does_not_decode_is_an_error(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ import asyncio++ from xagent.core.tools.core.RAG_tools.core.schemas import CollectionInfo+ from xagent.core.tools.core.RAG_tools.management import collection_manager as cm++ asyncio.run(+ cm.collection_manager.save_collection(+ CollectionInfo(+ name="kb",+ embedding_model_id="BAAI/old-model",+ embedding_dimension=2,+ )+ )+ )+ conn = mod.connection()+ conn.open_table(mod.METADATA_TABLE).update(+ "name = 'kb'", {"extra_metadata": "{not json"}+ )+ # The manager reports this row as "not found"; the strict read must not.+ with pytest.raises(ValueError, match="not found"):+ cm.get_collection_sync("kb")+ with pytest.raises(mod.MigrationCliError) as raised:+ mod.collection_info("kb")+ assert not isinstance(raised.value, mod.MissingCollectionError)++ monkeypatch.setattr(mod, "resolve_collection_names", lambda **kwargs: ["kb"])+ (item,) = mod.scope_report()+ assert item.error and item.skip_reason is None+ assert mod.main([]) == 1+++class _FailingWhere:+ def __init__(self, table: Any) -> None:+ self._table = table++ def where(self, where: str) -> Any:+ if "'bad'" in where:+ raise OSError("metadata store unavailable")+ return self._table.search().where(where)+++def test_a_metadata_store_failure_is_not_a_missing_collection(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ import asyncio++ from xagent.core.tools.core.RAG_tools.core.schemas import CollectionInfo+ from xagent.core.tools.core.RAG_tools.management import collection_manager as cm++ for name in ("bad", "ok"):+ asyncio.run(+ cm.collection_manager.save_collection(+ CollectionInfo(+ name=name,+ embedding_model_id="BAAI/old-model",+ embedding_dimension=2,+ )+ )+ )+ real = mod.connection()++ class _Broken:+ def __init__(self, table: Any) -> None:+ self._table = table++ def search(self) -> Any:+ return _FailingWhere(self._table)++ def __getattr__(self, attr: str) -> Any:+ return getattr(self._table, attr)++ class _Conn:+ def open_table(self, name: str) -> Any:+ table = real.open_table(name)+ return _Broken(table) if name == mod.METADATA_TABLE else table++ def __getattr__(self, attr: str) -> Any:+ return getattr(real, attr)++ monkeypatch.setattr(mod, "connection", lambda: _Conn())+ monkeypatch.setattr(mod, "resolve_collection_names", lambda **kwargs: ["bad", "ok"])++ report = {item.name: item for item in mod.scope_report()}++ assert "metadata store unavailable" in (report["bad"].error or "")+ assert report["bad"].skip_reason is None+ assert report["ok"].error is None+ assert mod.main([]) == 1++[email protected]("setting", ["qdrant", "no-such-engine"])+def test_an_unusable_backend_setting_is_an_error_not_a_skip(+ monkeypatch: pytest.MonkeyPatch, setting: str+) -> None:+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()])})+ monkeypatch.setattr(mod, "connection", lambda: conn)+ monkeypatch.setenv("XAGENT_VECTOR_BACKEND", setting)++ (item,) = mod.scope_report(collections=["kb"])++ assert item.error and item.skip_reason is None+++def test_a_malformed_engine_binding_is_unreadable() -> None:+ from types import SimpleNamespace++ from xagent.core.tools.core.RAG_tools.kb.coordinator import (+ KB_STORAGE_METADATA_KEY,+ )++ info = SimpleNamespace(extra_metadata={KB_STORAGE_METADATA_KEY: "bogus"})++ assert mod.collection_engine(info) == ""+++def test_a_coordinator_that_fails_to_build_is_an_error(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from types import SimpleNamespace++ from xagent.core.tools.core.RAG_tools.kb import coordinator++ def _broken() -> Any:+ raise ValueError("cannot build the coordinator")++ monkeypatch.setattr(coordinator, "get_kb_coordinator", _broken)++ with pytest.raises(ValueError):+ mod.collection_engine(SimpleNamespace(extra_metadata={}))+++def test_a_reader_failure_does_not_stop_the_other_collections(+ monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture+) -> None:+ monkeypatch.setattr(mod, "resolve_collection_names", lambda **kwargs: ["a", "b"])++ def _scope(name: str, target: Any = None, *, conn: Any, **kwargs: Any) -> Any:+ if name == "a":+ raise RuntimeError("batch read failed")+ return mod.CollectionScope(name=name)++ monkeypatch.setattr(mod, "scope_collection", _scope)+ monkeypatch.setattr(mod, "connection", lambda: object())++ report = mod.scope_report()++ assert [item.name for item in report] == ["a", "b"]+ assert "batch read failed" in (report[0].error or "")+ assert mod.main([]) == 1+ assert caplog.text.count("batch read failed") == 1+++def test_a_discovered_collection_that_vanished_is_not_a_failure(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(+ mod, "resolve_collection_names", lambda **kwargs: ["gone", "ok"]+ )++ def _scope(name: str, target: Any = None, *, conn: Any, **kwargs: Any) -> Any:+ if name == "gone":+ raise mod.MissingCollectionError("gone")+ return mod.CollectionScope(name=name)++ monkeypatch.setattr(mod, "scope_collection", _scope)+ monkeypatch.setattr(mod, "connection", lambda: object())++ report = mod.scope_report()++ assert report[0].skip_reason == mod.SKIP_NO_METADATA and report[0].error is None+ assert mod.main([]) == 0+++def test_an_empty_scope_is_reported_as_such(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(mod, "resolve_collection_names", lambda **kwargs: [])++ assert mod.main([]) == 1+++# --- the store directory and the arguments ---++[email protected]("argv", [["--collection", "kb"], []])[email protected]("configured", [True, False])+def test_a_missing_lancedb_dir_is_left_absent(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ argv: List[str],+ configured: bool,+) -> None:+ from xagent.providers.vector_store.lancedb import LanceDBConnectionManager++ missing = tmp_path / "absent" / "lancedb"+ if configured:+ monkeypatch.setenv("LANCEDB_DIR", str(missing))+ else:+ monkeypatch.delenv("LANCEDB_DIR", raising=False)+ monkeypatch.setattr(+ LanceDBConnectionManager,+ "locate_default_lancedb_dir",+ staticmethod(lambda: missing),+ )+ models = tmp_path / "language_models"+ models.mkdir()+ monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(models))++ assert mod.main(argv) == 2++ assert not (tmp_path / "absent").exists()+ assert list(models.iterdir()) == []+++def test_the_application_still_creates_its_default_directory(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ from xagent.providers.vector_store.lancedb import LanceDBConnectionManager++ default = tmp_path / "default" / "lancedb"+ monkeypatch.delenv("LANCEDB_DIR", raising=False)+ monkeypatch.setattr(+ LanceDBConnectionManager,+ "locate_default_lancedb_dir",+ staticmethod(lambda: default),+ )+ cached = LanceDBConnectionManager.get_default_lancedb_dir+ cached.cache_clear()+ try:+ assert LanceDBConnectionManager().resolve_dir_from_env(create=False)+ assert not default.exists()+ assert LanceDBConnectionManager().resolve_dir_from_env() == str(default)+ assert default.is_dir()+ finally:+ cached.cache_clear()+++def _unreadable(*args: Any, **kwargs: Any) -> Any:+ raise OSError("unreadable")+++ ("patched", "argv"),+ [+ ("connection", []),+ ("_all_collection_names", []),+ (None, ["--collection", " "]),+ ],+)+def test_a_run_that_cannot_start_exits_2(+ monkeypatch: pytest.MonkeyPatch, patched: Optional[str], argv: List[str]+) -> None:+ _wire(monkeypatch, _Conn({mod.METADATA_TABLE: _metadata([_bound()])}))+ if patched:+ monkeypatch.setattr(mod, patched, _unreadable)++ assert mod.main(argv) == 2++[email protected]("model", ["", " ", "none"])+def test_a_blank_target_stops_the_run(+ monkeypatch: pytest.MonkeyPatch, model: str+) -> None:+ _wire(monkeypatch, _Conn({mod.METADATA_TABLE: _metadata([_bound()])}))++ assert mod.main(["--model", model]) == 2+++def test_a_target_id_is_normalised_like_a_binding() -> None:+ assert mod.resolve_target(" new-model ") == "new-model"+++def test_collection_names_are_matched_as_stored() -> None:+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound(), _bound(name=" kb")])})++ assert mod.resolve_collection_names(collections=["kb", " kb", "kb"], conn=conn) == [+ "kb",+ " kb",+ ]+ with pytest.raises(mod.MissingCollectionError):+ mod.resolve_collection_names(collections=["kb "], conn=conn)+++def test_reading_is_batched(monkeypatch: pytest.MonkeyPatch) -> None:+ rows = [_row(f"c{index}", doc_id=f"d{index}") for index in range(2500)]+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()]), _writer(): _Table(rows)})+ _wire(monkeypatch, conn)+ sizes: List[int] = []+ batches = _Query.to_batches++ def _spy(query: _Query, batch_size: int) -> Iterator[Any]:+ sizes.append(batch_size)+ return batches(query, batch_size)++ monkeypatch.setattr(_Query, "to_batches", _spy)++ assert mod.scope_collection("kb").chunks == 2500+ assert mod.READ_BATCH_SIZE in sizes+++# --- imports ---+++def test_every_module_in_the_migration_package_imports() -> None:+ import importlib+ import pkgutil++ import xagent.migrations.embedding as package++ names = {module.name for module in pkgutil.iter_modules(package.__path__)}+ assert "scope" in names+ for name in sorted(names):+ importlib.import_module(f"xagent.migrations.embedding.{name}")+++# --- source and target must not share a physical table ---+++def test_a_target_that_writes_the_source_table_is_refused(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from xagent.core.tools.core.RAG_tools.LanceDB.model_tag_utils import (+ written_embeddings_table_name,+ )++ # ``old-model`` and ``old_model`` share one writer spelling.+ assert written_embeddings_table_name("old-model") == written_embeddings_table_name(+ "old_model"+ )+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound("old-model", 2)]),+ mod.source_table("old-model"): _Table([_row("c1")]),+ }+ )+ _wire(monkeypatch, conn)++ assert (+ mod.precompute_eligibility("kb", target_model_id="old_model").reason+ == mod.SKIP_SHARED_TABLE+ )+ item = mod.scope_collection("kb", target="old_model")+ assert item.skip_reason == mod.SKIP_SHARED_TABLE+ assert item.chunks == 0+++def test_a_target_that_is_the_bound_model_is_refused(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): _Table([_row("c1")]),+ }+ )+ _wire(monkeypatch, conn)++ item = mod.scope_collection("kb", target="BAAI/old-model")++ assert item.skip_reason == mod.SKIP_TARGET_IS_SOURCE+ assert item.missing_chunks is None+++def test_the_refusal_happens_before_anything_is_counted(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound("old-model", 2)]),+ mod.source_table("old-model"): _Table([_row("c1")]),+ }+ )+ _wire(monkeypatch, conn)+ monkeypatch.setattr(mod, "resolve_target", lambda model_id: "old_model")+ monkeypatch.setattr(mod, "resolve_collection_names", lambda **kwargs: ["kb"])++ report = mod.scope_report(target_model_id="old_model")++ assert [item.error for item in report] == [None]+ assert report[0].skip_reason == mod.SKIP_SHARED_TABLE+ assert report[0].chunks == 0+++# --- a target table whose rows cannot be attributed to a collection ---++_NO_COLLECTION_COLUMN = ["doc_id", "chunk_id", "user_id", "text", "model", "vector"]+++def test_a_target_table_that_cannot_name_its_collection_is_refused(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Without the column a row cannot be proven foreign.+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): _Table([_row("c1")]),+ mod.source_table("new-model"): _Table(+ [_row("c9")], names=_NO_COLLECTION_COLUMN+ ),+ }+ )+ _wire(monkeypatch, conn)++ assert (+ mod.precompute_eligibility("kb", target_model_id="new-model").reason+ == mod.SKIP_TARGET_ROWS_IN_OTHER_TABLE+ )+++def test_a_target_row_no_model_wrote_is_refused(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): _Table([_row("c1")]),+ mod.source_table("new-model"): _Table([_row("c9", model=None)]),+ }+ )+ _wire(monkeypatch, conn)++ assert (+ mod.precompute_eligibility("kb", target_model_id="new-model").reason+ == mod.SKIP_TARGET_ROWS_IN_OTHER_TABLE+ )+++def test_an_empty_target_table_without_the_column_does_not_refuse(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): _Table([_row("c1")]),+ mod.source_table("new-model"): _Table([], names=_NO_COLLECTION_COLUMN),+ }+ )+ _wire(monkeypatch, conn)++ assert mod.precompute_eligibility("kb", target_model_id="new-model").reason is None+ assert mod.scope_collection("kb", target="new-model").missing_chunks == 1+++def test_an_empty_collection_with_an_absent_source_table_is_counted(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()])})+ _wire(monkeypatch, conn)++ eligibility = mod.precompute_eligibility("kb", target_model_id="new-model")++ assert eligibility.reason is None+ assert mod.scope_collection("kb", target="new-model").missing_chunks == 0+++# --- one pass over the source rows ---+++def test_the_scan_classifies_every_row_once(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn(+ {+ mod.METADATA_TABLE: _metadata([_bound()]),+ _writer(): _Table(+ [+ _row("c1", doc_id="d1"),+ _row("c2", doc_id="d2"),+ _row("c3", doc_id="d1"),+ _row("c4", doc_id=None),+ _row("c5", doc_id="d3", user_id=None),+ ]+ ),+ }+ )+ _wire(monkeypatch, conn)++ scan = mod.scan_source("kb", "BAAI/old-model", have={("d2", "c2")})++ assert scan.usable_documents == {"d1", "d2"}+ assert scan.pending == 2+ assert scan.skipped == {mod.SKIP_NO_DOC_ID: 1, mod.SKIP_NO_OWNER: 1}+++def test_a_missing_source_table_scans_empty(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ conn = _Conn({mod.METADATA_TABLE: _metadata([_bound()])})+ _wire(monkeypatch, conn)++ scan = mod.scan_source("kb", "BAAI/old-model", have=set())++ assert (scan.rows, scan.pending) == (0, 0)+++# --- the same readers on a real LanceDB store ---++_CORE_SCHEMA = pa.schema(+ [+ ("collection", pa.string()),+ ("doc_id", pa.string()),+ ("chunk_id", pa.string()),+ ("user_id", pa.int64()),+ ]+)+++def _save_collection() -> None:+ import asyncio++ from xagent.core.tools.core.RAG_tools.core.schemas import CollectionInfo+ from xagent.core.tools.core.RAG_tools.management import collection_manager as cm++ asyncio.run(+ cm.collection_manager.save_collection(+ CollectionInfo(+ name="kb",+ embedding_model_id="BAAI/old-model",+ embedding_dimension=2,+ embeddings=4,+ )+ )+ )+++def _real_table(name: str, rows: List[Dict[str, Any]], schema: Any = SCHEMA) -> None:+ mod.connection().create_table(name, data=pa.Table.from_pylist(rows, schema=schema))+++def _versions() -> Dict[str, int]:+ conn = mod.connection()+ return {name: conn.open_table(name).version for name in mod.table_names(conn)}+++def real_store() -> None:+ _save_collection()+++def test_a_real_store_is_counted_and_left_as_it_was(real_store: None) -> None:+ _real_table(+ mod.source_table("BAAI/old-model"),+ [+ _row("c1", model="BAAI/old-model"),+ _row("c2", model="BAAI/old-model"),+ _row("c3", model="BAAI/old-model", user_id=None),+ ],+ )+ _real_table(+ mod.CORE_TABLE,+ [+ {"collection": "kb", "doc_id": "d1", "chunk_id": chunk_id, "user_id": 7}+ for chunk_id in ("c1", "c2", "c3")+ ],+ schema=_CORE_SCHEMA,+ )+ _real_table(mod.source_table("new-model"), [_row("c1", model="new-model")])+ before = _versions()++ item = mod.scope_collection("kb", target="new-model")++ assert (item.skip_reason, item.error) == (None, None)+ assert (item.chunks, item.missing_chunks) == (3, 1)+ assert item.skipped == {mod.SKIP_NO_OWNER: 1}+ assert _versions() == before+++def test_a_row_another_model_wrote_in_the_target_table_is_refused_on_a_real_store(+ real_store: None,+) -> None:+ # ``new-model`` and ``new_model`` are distinct bindings with one writer table.+ _real_table(+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]+ )+ _real_table(mod.source_table("new-model"), [_row("c9", model="new_model")])++ eligibility = mod.precompute_eligibility("kb", target_model_id="new-model")++ assert eligibility.reason == mod.SKIP_TARGET_ROWS_IN_OTHER_TABLE+++def test_a_vector_dimension_is_checked_only_when_given_on_a_real_store(+ real_store: None,+) -> None:+ wide = pa.schema(+ [+ f if f.name != "vector" else pa.field("vector", pa.list_(pa.float32(), 3))+ for f in SCHEMA+ ]+ )+ _real_table(+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]+ )+ target = _row("c1", model="new-model")+ target["vector"] = [0.5, 0.5, 0.5]+ _real_table(mod.source_table("new-model"), [target], schema=wide)++ assert mod.target_chunk_ids("kb", "new-model", dimension=2) == set()+ assert mod.target_chunk_ids("kb", "new-model") == {("d1", "c1")}+ assert mod.scope_collection("kb", target="new-model").missing_chunks == 0++[email protected]("holds_rows", [True, False])+def test_a_source_table_without_a_collection_column_is_refused_on_a_real_store(+ real_store: None, holds_rows: bool+) -> None:+ narrow = pa.schema([f for f in SCHEMA if f.name != "collection"])+ row = _row("c1", model="BAAI/old-model")+ del row["collection"]+ rows = [row] if holds_rows else []+ _real_table(mod.source_table("BAAI/old-model"), rows, schema=narrow)++ item = mod.scope_collection("kb")++ assert (item.skip_reason, item.error) == (mod.SKIP_MISSING_COLUMN, None)++[email protected]("missing", ["doc_id", "chunk_id"])[email protected]("table", ["core", "target"])+def test_a_table_without_a_compared_column_is_an_error_on_a_real_store(+ real_store: None, missing: str, table: str+) -> None:+ _real_table(+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]+ )+ if table == "core":+ schema = pa.schema([f for f in _CORE_SCHEMA if f.name != missing])+ row: Dict[str, Any] = {"collection": "kb", "doc_id": "d1", "chunk_id": "c1"}+ name = mod.CORE_TABLE+ else:+ schema = pa.schema([f for f in SCHEMA if f.name != missing])+ row = _row("c1", model="new-model")+ name = mod.source_table("new-model")+ row.pop(missing)+ _real_table(name, [row], schema=schema)++ (item,) = mod.scope_report(collections=["kb"], target_model_id="new-model")++ assert item.error and missing in item.error and item.skip_reason is None+++def test_more_rows_than_one_page_are_all_read_on_a_real_store(+ real_store: None,+) -> None:+ keys = [f"c{index:02d}" for index in range(25)]+ _real_table(+ mod.source_table("BAAI/old-model"),+ [_row(key, model="BAAI/old-model") for key in keys],+ )+ _real_table(+ mod.CORE_TABLE,+ [+ {"collection": "kb", "doc_id": "d1", "chunk_id": key, "user_id": 7}+ for key in keys+ ],+ schema=_CORE_SCHEMA,+ )+ _real_table(+ mod.source_table("new-model"),+ [_row(key, model="new-model") for key in keys[:20]],+ )++ item = mod.scope_collection("kb", target="new-model")++ assert (item.chunks, item.missing_chunks, item.skipped) == (25, 5, {})+++# --- the table rule: the model's own tables, else every embeddings table ---++STRAY = "embeddings_legacy_name"+++def test_rows_only_in_a_table_the_model_does_not_own_are_refused_on_a_real_store(+ real_store: None,+) -> None:+ _real_table(STRAY, [_row("c1", model="legacy")])++ assert mod.scope_collection("kb").skip_reason == mod.SKIP_ROWS_IN_OTHER_TABLE+++def test_without_an_own_table_other_collections_do_not_refuse_on_a_real_store(+ real_store: None,+) -> None:+ _real_table(STRAY, [_row("c1", collection="other", model="legacy")])++ item = mod.scope_collection("kb")++ assert (item.skip_reason, item.error, item.chunks) == (None, None, 0)+++def test_an_own_table_hides_every_other_table_on_a_real_store(+ real_store: None,+) -> None:+ _real_table(+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]+ )+ _real_table(STRAY, [_row("c9", model="legacy")])++ item = mod.scope_collection("kb")++ assert (item.skip_reason, item.chunks) == (None, 1)+++def test_a_new_target_is_refused_when_another_table_holds_the_collection(+ real_store: None,+) -> None:+ _real_table(+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]+ )+ _real_table(STRAY, [_row("c9", model="legacy")])++ item = mod.scope_collection("kb", target="new-model")++ assert item.skip_reason == mod.SKIP_TARGET_ROWS_IN_OTHER_TABLE+++def test_a_target_table_that_is_not_the_writer_is_refused_on_a_real_store(+ real_store: None,+) -> None:+ once, writer = mod.own_tables("ACME/new-model")[::-1]+ assert once != writer+ _real_table(+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]+ )+ _real_table(once, [_row("c1", model="ACME/new-model")])++ item = mod.scope_collection("kb", target="ACME/new-model")++ assert item.skip_reason == mod.SKIP_TARGET_ROWS_IN_OTHER_TABLE+++def test_a_table_without_a_collection_column_holds_the_collection_on_a_real_store(+ real_store: None,+) -> None:+ row = _row("c1", model="legacy")+ del row["collection"]+ _real_table(+ STRAY, [row], schema=pa.schema([f for f in SCHEMA if f.name != "collection"])+ )++ assert mod.scope_collection("kb").skip_reason == mod.SKIP_ROWS_IN_OTHER_TABLE+++def test_a_target_writer_table_without_a_model_column_is_refused_on_a_real_store(+ real_store: None,+) -> None:+ _real_table(+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]+ )+ row = _row("c1")+ del row["model"]+ narrow = pa.schema([f for f in SCHEMA if f.name != "model"])+ _real_table(mod.source_table("new-model"), [row], schema=narrow)++ item = mod.scope_collection("kb", target="new-model")++ assert (item.skip_reason, item.error) == (mod.SKIP_TARGET_ROWS_IN_OTHER_TABLE, None)+++def test_another_own_target_table_is_refused_beside_the_writer_on_a_real_store(+ real_store: None,+) -> None:+ writer, once = mod.own_tables("ACME/new-model")+ _real_table(+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]+ )+ _real_table(writer, [_row("c1", model="ACME/new-model")])+ _real_table(once, [_row("c9", model="ACME/new-model")])++ item = mod.scope_collection("kb", target="ACME/new-model")++ assert item.skip_reason == mod.SKIP_TARGET_ROWS_IN_OTHER_TABLE+++def test_a_first_run_counts_every_row_as_missing_on_a_real_store(+ real_store: None,+) -> None:+ _real_table(+ mod.source_table("BAAI/old-model"),+ [_row("c1", model="BAAI/old-model"), _row("c2", model="BAAI/old-model")],+ )+ _real_table(STRAY, [_row("c9", collection="other", model="legacy")])++ item = mod.scope_collection("kb", target="new-model")++ assert (item.skip_reason, item.missing_chunks) == (None, 2)+++def test_an_existing_target_table_hides_every_other_table_on_a_real_store(+ real_store: None,+) -> None:+ _real_table(+ mod.source_table("BAAI/old-model"),+ [_row("c1", model="BAAI/old-model"), _row("c2", model="BAAI/old-model")],+ )+ _real_table(mod.source_table("new-model"), [_row("c1", model="new-model")])+ _real_table(STRAY, [_row("c9", model="legacy")])++ item = mod.scope_collection("kb", target="new-model")++ assert (item.skip_reason, item.missing_chunks) == (None, 1)+++def test_a_previous_model_can_be_the_target_on_a_real_store(+ real_store: None,+) -> None:+ _real_table(+ mod.source_table("BAAI/old-model"),+ [_row("c1", model="BAAI/old-model"), _row("c2", model="BAAI/old-model")],+ )+ _real_table(mod.source_table("prev-model"), [_row("c1", model="prev-model")])++ item = mod.scope_collection("kb", target="prev-model")++ assert (item.skip_reason, item.missing_chunks) == (None, 1)+++# --- the command never opens the SQL database ---+++def test_the_command_runs_without_the_sql_database(+ real_store: None, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ import sqlalchemy++ _real_table(+ mod.source_table("BAAI/old-model"), [_row("c1", model="BAAI/old-model")]+ )+ _real_table(mod.source_table("new-model"), [_row("c1", model="new-model")])+ database = tmp_path / "absent.db"+ monkeypatch.setenv("DATABASE_URL", f"sqlite:///{database}")++ def _refuse(*args: Any, **kwargs: Any) -> Any:+ raise AssertionError("the SQL database was opened")++ monkeypatch.setattr(sqlalchemy, "create_engine", _refuse)+ monkeypatch.setattr(sqlite3, "connect", _refuse)++ assert mod.main(["--model", "new-model"]) == 0+ assert mod.main([]) == 0+ assert not database.exists()