Enterprise agent platform for building, deploying, and operating AI teammates.
feat(kb): refuse a vector batch committed under a migration (#3824)
蓝鱼bluefish7c021c7d716a
diff --git a/src/xagent/core/tools/core/RAG_tools/kb/coordinator.py b/src/xagent/core/tools/core/RAG_tools/kb/coordinator.pyindex 9c5373b2c..a107d12b8 100644--- a/src/xagent/core/tools/core/RAG_tools/kb/coordinator.py+++ b/src/xagent/core/tools/core/RAG_tools/kb/coordinator.py@@ -35,6 +35,7 @@ from ..management.migration_state import (holding_leases,verify_active_leases,)+from ..storage.contracts import is_missing_collection_errorfrom ..storage.factory import StorageFactoryfrom ..utils.user_scope import resolve_user_scopefrom .api_compatibility import KBApiCompatibilityFacade@@ -1900,11 +1901,7 @@ class KBCoordinator:@staticmethoddef _is_missing_collection_error(collection: str, exc: ValueError) -> bool:- message = str(exc)- return message in {- f"Collection '{collection}' not found",- "Table 'collection_metadata' was not found",- }+ return is_missing_collection_error(collection, exc)@staticmethoddef _resolve_user_scope(request: KBContextRequest) -> KBUserScope:diff --git a/src/xagent/core/tools/core/RAG_tools/kb/maintenance_compatibility.py b/src/xagent/core/tools/core/RAG_tools/kb/maintenance_compatibility.pyindex 72b13a848..d0c9fceb0 100644--- a/src/xagent/core/tools/core/RAG_tools/kb/maintenance_compatibility.py+++ b/src/xagent/core/tools/core/RAG_tools/kb/maintenance_compatibility.py@@ -77,6 +77,16 @@ class KBMaintenanceCompatibilityFacade:with self._storage_context():return _get_collection_sync_impl(collection_name)+ def get_collection_if_present_sync(+ self, collection_name: str+ ) -> Optional["CollectionInfo"]:+ from ..management.collection_manager import (+ _get_collection_if_present_sync_impl,+ )++ with self._storage_context():+ return _get_collection_if_present_sync_impl(collection_name)+def initialize_collection_embedding_sync(self, collection_name: str, embedding_model_id: str) -> "CollectionInfo":diff --git a/src/xagent/core/tools/core/RAG_tools/management/collection_manager.py b/src/xagent/core/tools/core/RAG_tools/management/collection_manager.pyindex 99f48156a..a1bf822ce 100644--- a/src/xagent/core/tools/core/RAG_tools/management/collection_manager.py+++ b/src/xagent/core/tools/core/RAG_tools/management/collection_manager.py@@ -28,7 +28,7 @@ from ..core.exceptions import CollectionNotFrozenErrorfrom ..core.parser_registry import get_supported_parsers, validate_parser_compatibilityfrom ..core.schemas import CollectionInfofrom ..LanceDB.schema_manager import _safe_close_table-from ..storage.contracts import MetadataStore+from ..storage.contracts import MetadataStore, is_missing_collection_errorfrom ..storage.factory import get_metadata_store, get_vector_index_storefrom ..utils.lancedb_query_utils import list_table_namesfrom ..utils.model_resolution_pin import pinned_resolution@@ -913,6 +913,27 @@ def _get_collection_sync_impl(collection_name: str) -> "CollectionInfo":return _sync_wrapper(collection_manager.get_collection)(collection_name)+def get_collection_if_present_sync(collection_name: str) -> Optional["CollectionInfo"]:+ """The metadata row, ``None`` only when it or its table is missing.++ Unlike ``get_collection_sync``, a failed read is raised, not reported as absent.+ """+ return _get_maintenance_compatibility_facade().get_collection_if_present_sync(+ collection_name+ )+++def _get_collection_if_present_sync_impl(+ collection_name: str,+) -> Optional["CollectionInfo"]:+ try:+ return _sync_wrapper(get_metadata_store().get_collection)(collection_name)+ except ValueError as exc:+ if is_missing_collection_error(collection_name, exc):+ return None+ raise++def initialize_collection_embedding_sync(collection_name: str, embedding_model_id: str) -> "CollectionInfo":diff --git a/src/xagent/core/tools/core/RAG_tools/storage/contracts.py b/src/xagent/core/tools/core/RAG_tools/storage/contracts.pyindex a53f3e938..c9f0463a7 100644--- a/src/xagent/core/tools/core/RAG_tools/storage/contracts.py+++ b/src/xagent/core/tools/core/RAG_tools/storage/contracts.py@@ -292,6 +292,14 @@ FilterExpression = Union[]+def is_missing_collection_error(collection_name: str, exc: BaseException) -> bool:+ """Whether a ``get_collection`` error means the row or its table is missing."""+ return str(exc) in {+ f"Collection '{collection_name}' not found",+ "Table 'collection_metadata' was not found",+ }++class MetadataStore(ABC):"""Control-plane metadata storage contract."""diff --git a/src/xagent/core/tools/core/RAG_tools/vector_storage/vector_manager.py b/src/xagent/core/tools/core/RAG_tools/vector_storage/vector_manager.pyindex e92e5842b..157bf5ce1 100644--- a/src/xagent/core/tools/core/RAG_tools/vector_storage/vector_manager.py+++ b/src/xagent/core/tools/core/RAG_tools/vector_storage/vector_manager.py@@ -84,6 +84,8 @@ def write_vectors_to_db(embeddings: List[ChunkEmbeddingData],create_index: bool = True,user_id: Optional[int] = None,+ *,+ verify_bound: bool = True,) -> EmbeddingWriteResponse:"""Write embedding vectors to database with idempotency through the facade.@@ -92,16 +94,70 @@ def write_vectors_to_db((including the create_index build) can raise CommitConflict. The dominantcost, embedding computation, already finished before this call, so the lockonly serializes the cheap write and barely dents throughput.++ With ``verify_bound``, returns only if, after the commit, the migration row is not+ frozen and the collection has no metadata row, no binding, or the batch model bound."""+ from ..kb.coordinator import KBCoordinatorfrom ..management.collection_manager import _get_collection_thread_lock+ # The handle writes under this name; the guard must read the same rows.+ collection = KBCoordinator._normalize_collection(collection)with _get_collection_thread_lock(collection):- return _get_vector_storage_compatibility_facade().write_vectors_to_db(+ response = _get_vector_storage_compatibility_facade().write_vectors_to_db(collection=collection,embeddings=embeddings,create_index=create_index,user_id=user_id,)+ if verify_bound and embeddings:+ _refuse_unless_still_bound(collection, embeddings)+ return response+++def _refuse_unless_still_bound(+ collection: str, embeddings: List[ChunkEmbeddingData]+) -> None:+ from ..core.exceptions import CollectionMigratingError+ from ..management.collection_manager import (+ _normalize_model_id,+ get_collection_if_present_sync,+ )+ from ..management.migration_state import FROZEN, read_migration_state++ written = {_normalize_model_id(item.model) for item in embeddings}+ error: Optional[Exception] = None+ read = "migration row"+ try:+ # Row before binding: a switch flips the binding before it unfreezes.+ if read_migration_state(collection).state == FROZEN:+ reason = "is frozen for an embedding migration"+ else:+ read = "binding"+ info = get_collection_if_present_sync(collection)+ if info is None:+ return+ bound = _normalize_model_id(info.embedding_model_id)+ # No binding cannot be a finished switch: a switch always leaves one.+ if bound is None or {bound} == written:+ return+ reason = f"is now bound to '{info.embedding_model_id}'"+ except Exception as exc: # noqa: BLE001 - unreadable proves nothing either way+ error = exc+ reason = f"could not be checked ({read} read: {type(exc).__name__})"+ logger.warning(+ "Write guard on collection %s: %s read failed",+ collection,+ read,+ exc_info=True,+ )+ raise CollectionMigratingError(+ f"Collection '{collection}' {reason}; this write is refused, retry it.",+ details={+ "collection": collection,+ "written_models": sorted({item.model for item in embeddings}),+ },+ ) from errordef commit_vectors_to_db(diff --git a/tests/core/tools/core/RAG_tools/kb/test_maintenance_compatibility.py b/tests/core/tools/core/RAG_tools/kb/test_maintenance_compatibility.pyindex 847216517..d7d5048a5 100644--- a/tests/core/tools/core/RAG_tools/kb/test_maintenance_compatibility.py+++ b/tests/core/tools/core/RAG_tools/kb/test_maintenance_compatibility.py@@ -29,6 +29,10 @@ def test_kb_maintenance_compatibility_methods_match_public_helper_signatures() -facade = KBMaintenanceCompatibilityFacade()pairs = [(facade.get_collection_sync, collection_manager.get_collection_sync),+ (+ facade.get_collection_if_present_sync,+ collection_manager.get_collection_if_present_sync,+ ),(facade.initialize_collection_embedding_sync,collection_manager.initialize_collection_embedding_sync,diff --git a/tests/core/tools/core/RAG_tools/management/test_collection_manager.py b/tests/core/tools/core/RAG_tools/management/test_collection_manager.pyindex 7d1d2ab62..9e01f1fcd 100644--- a/tests/core/tools/core/RAG_tools/management/test_collection_manager.py+++ b/tests/core/tools/core/RAG_tools/management/test_collection_manager.py@@ -736,3 +736,48 @@ class TestRebuildCollectionMetadata:assert result.name == "rebuild_test_collection"assert result.documents == 5assert result.processed_documents == 3+++def test_get_collection_if_present_reads_a_present_row() -> None:+ import asyncio++ from xagent.core.tools.core.RAG_tools.management.collection_manager import (+ collection_manager,+ get_collection_if_present_sync,+ )++ asyncio.run(+ collection_manager.save_collection(+ CollectionInfo(name="present", embedding_model_id="m")+ )+ )++ info = get_collection_if_present_sync("present")++ assert info is not None and info.embedding_model_id == "m"+++def test_get_collection_if_present_reports_an_absent_row_as_none() -> None:+ from xagent.core.tools.core.RAG_tools.management.collection_manager import (+ get_collection_if_present_sync,+ )++ assert get_collection_if_present_sync("never-created") is None++[email protected]("error", [OSError("store gone"), ValueError("boom")])+def test_get_collection_if_present_reraises_a_failed_read(+ monkeypatch: pytest.MonkeyPatch, error: Exception+) -> None:+ from xagent.core.tools.core.RAG_tools.management.collection_manager import (+ get_collection_if_present_sync,+ )+ from xagent.core.tools.core.RAG_tools.storage.factory import get_metadata_store++ async def _unreadable(name: str) -> CollectionInfo:+ raise error++ monkeypatch.setattr(get_metadata_store(), "get_collection", _unreadable)++ with pytest.raises(type(error), match=str(error)):+ get_collection_if_present_sync("anything")diff --git a/tests/core/tools/core/RAG_tools/pipelines/test_document_ingestion.py b/tests/core/tools/core/RAG_tools/pipelines/test_document_ingestion.pyindex d7137c912..3bc258e6f 100644--- a/tests/core/tools/core/RAG_tools/pipelines/test_document_ingestion.py+++ b/tests/core/tools/core/RAG_tools/pipelines/test_document_ingestion.py@@ -877,6 +877,37 @@ def test_process_document_fails_without_success_when_the_commit_fails(]+def test_a_write_refused_by_the_migration_check_fails_the_ingestion(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from xagent.core.tools.core.RAG_tools.core.exceptions import (+ CollectionMigratingError,+ )++ _patch_pipeline_dependencies(monkeypatch)+ refusal = (+ "Collection 'demo' is frozen for an embedding migration; "+ "this write is refused, retry it."+ )++ def _refused(**_kwargs: object) -> None:+ raise CollectionMigratingError(refusal)++ monkeypatch.setattr(document_ingestion, "write_vectors_to_db", _refused)++ result = document_ingestion.process_document(+ collection="demo",+ source_path="/tmp/doc.pdf",+ config=IngestionConfig(),+ )++ assert result.status == "partial"+ assert result.failed_step == "write_vectors_to_db"+ assert result.message == refusal+ assert result.vector_count == 0+ assert STATUS_EVENTS[-1]["status"] == DocumentProcessingStatus.FAILED.value++def test_process_document_commit_failure_keeps_completed_steps_in_order(monkeypatch: pytest.MonkeyPatch,) -> None:diff --git a/tests/core/tools/core/RAG_tools/vector_storage/test_vector_manager.py b/tests/core/tools/core/RAG_tools/vector_storage/test_vector_manager.pyindex bf367452d..0a942bd26 100644--- a/tests/core/tools/core/RAG_tools/vector_storage/test_vector_manager.py+++ b/tests/core/tools/core/RAG_tools/vector_storage/test_vector_manager.py@@ -233,3 +233,366 @@ class TestVectorValidation:# Backward-compatible signature: model_tag/conn accepted, ignored.validate_query_vector([1.0, 2.0, 3.0])validate_query_vector([1.0, 2.0, 3.0], model_tag="test_model")+++def _bound_collection_with_a_chunk(model: str | None = "m-old") -> None:+ import asyncio+ from datetime import datetime, timezone++ from xagent.core.tools.core.RAG_tools.core.schemas import CollectionInfo+ from xagent.core.tools.core.RAG_tools.management.collection_manager import (+ collection_manager,+ )+ from xagent.core.tools.core.RAG_tools.storage.factory import (+ get_vector_index_store,+ )++ asyncio.run(+ collection_manager.save_collection(+ CollectionInfo(name="kb", embedding_model_id=model, embedding_dimension=3)+ )+ )+ get_vector_index_store().upsert_chunks(+ [+ {+ "collection": "kb",+ "doc_id": "d1",+ "parse_hash": "h1",+ "chunk_id": "c0",+ "index": 0,+ "text": "hello",+ "page_number": None,+ "section": None,+ "anchor": None,+ "json_path": None,+ "chunk_hash": "ch-c0",+ "config_hash": "cfg1",+ "created_at": datetime.now(timezone.utc),+ "metadata": None,+ "user_id": None,+ }+ ]+ )+++def _write_batch(+ create_index: bool = False, *, verify_bound: bool = True, collection: str = "kb"+) -> EmbeddingWriteResponse:+ return write_vectors_to_db(+ collection=collection,+ embeddings=[+ ChunkEmbeddingData(+ doc_id="d1",+ chunk_id="c0",+ parse_hash="h1",+ model="m-old",+ vector=[0.1, 0.2, 0.3],+ text="hello",+ chunk_hash="ch-c0",+ metadata=None,+ )+ ],+ create_index=create_index,+ verify_bound=verify_bound,+ )+++def _vectorless_chunks(model: str = "m-old") -> int:+ return read_chunks_for_embedding(+ collection="kb", doc_id="d1", parse_hash="h1", model=model, is_admin=True+ ).pending_count+++def _freeze_from_another_connection() -> None:+ import os+ from datetime import datetime++ import lancedb++ from xagent.core.tools.core.RAG_tools.management import migration_state++ migration_state._merge_row(+ lancedb.connect(os.environ["LANCEDB_DIR"]),+ {+ "name": "kb",+ "state": migration_state.FROZEN,+ "source_model_id": "m-old",+ "target_model_id": None,+ "updated_at": datetime.now(),+ },+ )+++_SWITCH_IN_ANOTHER_PROCESS = """+import sys++from xagent.core.tools.core.RAG_tools.management import collection_manager+from xagent.core.tools.core.RAG_tools.management import migration_state as ms++ms.write_migration_state(+ "kb", ms.CollectionMigrationState(state=ms.FROZEN, source_model_id="m-old")+)+if sys.argv[1] == "switch":+ assert collection_manager.write_collection_binding_sync(+ "kb", "m-new", 3, expected_model_id="m-old"+ )+ ms.clear_migration_state("kb")+"""+++class TestWriteAfterMigration:+ """Real LanceDB: a refused write raises and leaves its rows in place."""++ def test_untouched_collection_keeps_the_batch(self) -> None:+ _bound_collection_with_a_chunk()++ assert _write_batch().upsert_count == 1+ assert _vectorless_chunks() == 0++ def test_a_collection_without_a_binding_keeps_the_batch(self) -> None:+ _bound_collection_with_a_chunk(model=None)++ assert _write_batch().upsert_count == 1+ assert _vectorless_chunks() == 0++ def test_a_frozen_collection_without_a_binding_refuses_the_batch(self) -> None:+ from xagent.core.tools.core.RAG_tools.core.exceptions import (+ CollectionMigratingError,+ )++ _bound_collection_with_a_chunk(model=None)+ _freeze_from_another_connection()++ with pytest.raises(CollectionMigratingError, match="frozen"):+ _write_batch()++ def test_another_attempts_rows_survive_a_refused_batch_of_the_document(+ self,+ ) -> None:+ from xagent.core.tools.core.RAG_tools.core.exceptions import (+ CollectionMigratingError,+ )++ _bound_collection_with_a_chunk()+ assert _write_batch().upsert_count == 1+ _freeze_from_another_connection()++ # A stalled attempt of the same document wakes up and commits the row.+ with pytest.raises(CollectionMigratingError):+ _write_batch()+ assert _vectorless_chunks() == 0++ # The last batch of an ingestion builds the index before the guard reads.+ @pytest.mark.parametrize("create_index", [False, True])+ def test_frozen_collection_refuses_the_batch(self, create_index: bool) -> None:+ from xagent.core.tools.core.RAG_tools.core.exceptions import (+ CollectionMigratingError,+ )++ _bound_collection_with_a_chunk()+ _freeze_from_another_connection()++ with pytest.raises(CollectionMigratingError, match="frozen"):+ _write_batch(create_index)+ assert _vectorless_chunks() == 0++ @pytest.mark.parametrize("create_index", [False, True])+ def test_moved_binding_refuses_the_batch(self, create_index: bool) -> None:+ from xagent.core.tools.core.RAG_tools.core.exceptions import (+ CollectionMigratingError,+ )+ from xagent.core.tools.core.RAG_tools.management.collection_manager import (+ write_collection_binding_sync,+ )+ from xagent.core.tools.core.RAG_tools.management.migration_state import (+ clear_migration_state,+ )++ _bound_collection_with_a_chunk()+ _freeze_from_another_connection()+ assert write_collection_binding_sync(+ "kb", "m-new", 3, expected_model_id="m-old"+ )+ # Unfrozen by deleting the row: it then reads like a never-migrated one.+ clear_migration_state("kb")++ with pytest.raises(CollectionMigratingError, match="m-new"):+ _write_batch(create_index)+ assert _vectorless_chunks() == 0+ # The retry embeds under the new binding; the old row does not count.+ assert _vectorless_chunks("m-new") == 1++ def test_unreadable_migration_row_refuses_the_batch(self) -> None:+ import pyarrow as pa++ from xagent.core.tools.core.RAG_tools.core.exceptions import (+ CollectionMigratingError,+ )+ from xagent.core.tools.core.RAG_tools.management.migration_state import (+ MIGRATION_TABLE,+ )+ from xagent.core.tools.core.RAG_tools.storage.factory import (+ get_metadata_store,+ )++ _bound_collection_with_a_chunk()+ get_metadata_store().get_raw_connection().create_table(+ MIGRATION_TABLE, schema=pa.schema([("other", pa.string())])+ )++ with pytest.raises(CollectionMigratingError, match="migration row read: "):+ _write_batch()+ assert _vectorless_chunks() == 0++ def test_unreadable_binding_refuses_the_batch(+ self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture+ ) -> None:+ from xagent.core.tools.core.RAG_tools.core.exceptions import (+ CollectionMigratingError,+ )+ from xagent.core.tools.core.RAG_tools.management import collection_manager++ _bound_collection_with_a_chunk()++ def _unreadable(name: str) -> None:+ raise OSError("metadata store gone")++ monkeypatch.setattr(+ collection_manager, "get_collection_if_present_sync", _unreadable+ )++ with pytest.raises(CollectionMigratingError) as raised:+ _write_batch()+ assert "could not be checked (binding read: OSError)" in str(raised.value)+ assert "metadata store gone" not in str(raised.value)+ assert any(+ "metadata store gone" in str(record.exc_info[1])+ for record in caplog.records+ if record.exc_info+ )+ assert _vectorless_chunks() == 0++ @pytest.mark.parametrize("change", ["freeze", "switch"])+ def test_a_padded_name_is_checked_as_the_canonical_collection(+ self, change: str+ ) -> None:+ from xagent.core.tools.core.RAG_tools.core.exceptions import (+ CollectionMigratingError,+ )+ from xagent.core.tools.core.RAG_tools.management.collection_manager import (+ write_collection_binding_sync,+ )+ from xagent.core.tools.core.RAG_tools.management.migration_state import (+ clear_migration_state,+ )++ _bound_collection_with_a_chunk()+ _freeze_from_another_connection()+ if change == "switch":+ assert write_collection_binding_sync(+ "kb", "m-new", 3, expected_model_id="m-old"+ )+ clear_migration_state("kb")++ with pytest.raises(+ CollectionMigratingError, match="frozen" if change == "freeze" else "m-new"+ ) as raised:+ _write_batch(collection=" kb ")+ assert raised.value.details["collection"] == "kb"+ # The padded write landed in the canonical collection's table.+ assert _vectorless_chunks() == 0++ def test_an_unverified_write_ignores_a_frozen_collection(self) -> None:+ _bound_collection_with_a_chunk()+ _freeze_from_another_connection()++ assert _write_batch(verify_bound=False).upsert_count == 1+ assert _vectorless_chunks() == 0++ @pytest.mark.parametrize("change", ["freeze", "switch"])+ def test_a_change_made_by_another_process_is_seen(self, change: str) -> None:+ import os+ import subprocess+ import sys+ from pathlib import Path++ from xagent.core.tools.core.RAG_tools.core.exceptions import (+ CollectionMigratingError,+ )++ env = dict(os.environ)+ src = str(Path(__file__).resolve().parents[6] / "src")+ env["PYTHONPATH"] = os.pathsep.join(filter(None, [src, env.get("PYTHONPATH")]))+ _bound_collection_with_a_chunk()+ # Warms this process's cached connection before the other process writes.+ assert _write_batch().upsert_count == 1+ subprocess.run(+ [sys.executable, "-c", _SWITCH_IN_ANOTHER_PROCESS, change],+ env=env,+ check=True,+ capture_output=True,+ timeout=120,+ )++ with pytest.raises(+ CollectionMigratingError, match="frozen" if change == "freeze" else "m-new"+ ):+ _write_batch()++ def test_a_switch_finishing_between_the_two_reads_is_caught(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ from xagent.core.tools.core.RAG_tools.core.exceptions import (+ CollectionMigratingError,+ )+ from xagent.core.tools.core.RAG_tools.kb.vector_storage_compatibility import (+ KBVectorStorageCompatibilityFacade,+ )+ from xagent.core.tools.core.RAG_tools.management import (+ collection_manager,+ migration_state,+ )++ _bound_collection_with_a_chunk()+ _freeze_from_another_connection()+ written: list[bool] = []+ switched: list[bool] = []+ real_write = KBVectorStorageCompatibilityFacade.write_vectors_to_db++ def _write(self: object, **kwargs: object) -> object:+ response = real_write(self, **kwargs) # type: ignore[arg-type]+ written.append(True)+ return response++ def _then_switch(read): # type: ignore[no-untyped-def]+ def _read(*args: object, **kwargs: object) -> object:+ result = read(*args, **kwargs)+ if written and not switched:+ switched.append(True)+ collection_manager.write_collection_binding_sync(+ "kb", "m-new", 3, expected_model_id="m-old"+ )+ migration_state.clear_migration_state("kb")+ return result++ return _read++ monkeypatch.setattr(+ KBVectorStorageCompatibilityFacade, "write_vectors_to_db", _write+ )+ # Whichever the guard reads first, the switch completes right after it.+ monkeypatch.setattr(+ migration_state,+ "read_migration_state",+ _then_switch(migration_state.read_migration_state),+ )+ monkeypatch.setattr(+ collection_manager,+ "get_collection_if_present_sync",+ _then_switch(collection_manager.get_collection_if_present_sync),+ )++ with pytest.raises(CollectionMigratingError):+ _write_batch()+ assert switched == [True]+ assert _vectorless_chunks() == 0