Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix(vector_store): drop qdrant migrate wiring, keep iter_all only
Zohaib Hassnainec9e63e16fab
diff --git a/semantica/cli.py b/semantica/cli.pyindex 957878f4..d05a00f1 100644--- a/semantica/cli.py+++ b/semantica/cli.py@@ -5,7 +5,6 @@ This module provides the command-line interface for the Semantica framework,enabling users to interact with the framework via terminal commands."""-import itertoolsimport jsonimport osimport sys@@ -3715,7 +3714,7 @@ def store_stats(cli_ctx: CLIContext, backend: str, fmt: str, local_json: bool) -_run_with_error_handling(_action)-_MIGRATE_SUPPORTED_BACKENDS = {"faiss", "sqlite", "pgvector", "qdrant"}+_MIGRATE_SUPPORTED_BACKENDS = {"faiss", "sqlite", "pgvector"}_MIGRATE_BATCH_SIZE = 500@@ -3760,10 +3759,12 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str,namespace: Optional[str], local_dry: bool, local_json: bool) -> None:"""Migrate data between backends.- Wired up between faiss, sqlite, pgvector and qdrant. Migrating to or from- pinecone, milvus or weaviate still needs the export/reindex workaround- below, since each of those needs its own enumeration design (Pinecone- list, Milvus query_iterator, Weaviate cursor) that is not built yet.+ Direct migration is only wired up between faiss, sqlite, and pgvector -+ these are the backends whose storage contract supports paging through+ every stored vector. Migrating to or from qdrant, pinecone, milvus, or+ weaviate still needs the export/reindex workaround below, since each of+ those needs its own enumeration design (Qdrant scroll, Pinecone list,+ etc.) that hasn't been built yet.\bExample:@@ -3803,18 +3804,7 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str,if source_index_path:source._backend_store.load_index(source_index_path)- # Pull the first record before building the destination: qdrant, milvus- # and weaviate expose no `dimension` attribute, so without this the- # destination falls back to the default 768 and rejects every insert.- # The data itself is the reliable source of truth.- source_iter = source.iter_vectors(batch_size=_MIGRATE_BATCH_SIZE)- first_item = next(source_iter, None)-source_dimension = getattr(source._backend_store, "dimension", None)- if not source_dimension and first_item is not None:- first_vector = first_item.get("vector")- if first_vector is not None:- source_dimension = len(first_vector)if source_dimension and "dimension" not in dest_cfg:dest_cfg["dimension"] = source_dimension@@ -3837,12 +3827,7 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str,metadata_batch.clear()ids_batch.clear()- items = (- source_iter- if first_item is None- else itertools.chain([first_item], source_iter)- )- for item in items:+ for item in source.iter_vectors(batch_size=_MIGRATE_BATCH_SIZE):meta = dict(item.get("metadata") or {})if namespace and "namespace" not in meta:meta["namespace"] = namespacediff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.pyindex a90673c4..4a054be1 100644--- a/tests/test_cli_commands.py+++ b/tests/test_cli_commands.py@@ -1360,11 +1360,10 @@ class TestStore:assert result.exit_code != 0def test_migrate_refuses_unsupported_backend_pair(self, runner):- # qdrant is wired up now, so weaviate stands in as the unsupported side.result = runner.invoke(cli_module.main, ["store", "migrate",- "--from", "faiss", "--to", "weaviate"])+ "--from", "faiss", "--to", "qdrant"])assert result.exit_code != 0- assert "faiss, pgvector, qdrant, sqlite" in result.output+ assert "faiss, pgvector, sqlite" in result.outputdef _fake_migrate_store_module(self, source_items, stored, dest_configs=None):class _FakeBackendStore:@@ -1446,97 +1445,6 @@ class TestStore:assert result.exit_code != 0assert "index_path" in result.output- def _fake_dimensionless_store_module(self, source_items, dest_configs, stored):- """Fake VectorStore whose backend store has NO `dimension` attribute.-- Matches qdrant/milvus/weaviate, none of which expose one.- """-- class _DimensionlessBackendStore:- pass-- class _FakeStore:- def __init__(self, backend, config=None, **kw):- self.backend = backend- self._config = config or {}- self._backend_store = _DimensionlessBackendStore()- dest_configs[backend] = dict(self._config)-- def iter_vectors(self, batch_size=500):- if self.backend == "qdrant":- yield from source_items- return- return- yield # pragma: no cover - makes this a generator-- def store_vectors(self, vectors, metadata, ids=None):- for vec_id, vec in zip(ids, vectors):- stored[vec_id] = vec-- return _fake_module(VectorStore=_FakeStore)-- def test_migrate_infers_dimension_from_first_vector(self, runner, monkeypatch):- source_items = [- {"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}},- {"id": "b", "vector": [0.4, 0.5, 0.6], "metadata": {}},- ]- dest_configs, stored = {}, {}- fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored)- monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)-- result = runner.invoke(cli_module.main, ["store", "migrate",- "--from", "qdrant", "--to", "sqlite", "--json"])- _ok(result)- assert dest_configs["sqlite"].get("dimension") == 3-- def test_migrate_does_not_drop_the_peeked_first_record(self, runner, monkeypatch):- """The first record is consumed to infer dimension, so it must be- chained back into the migration loop rather than lost."""- source_items = [- {"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}},- {"id": "b", "vector": [0.4, 0.5, 0.6], "metadata": {}},- ]- dest_configs, stored = {}, {}- fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored)- monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)-- result = runner.invoke(cli_module.main, ["store", "migrate",- "--from", "qdrant", "--to", "sqlite", "--json"])- _ok(result)- assert _json_output(result)["migrated"] == 2- assert sorted(stored) == ["a", "b"]-- def test_migrate_keeps_explicit_dest_dimension_over_inference(self, runner, monkeypatch):- source_items = [{"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}}]- dest_configs, stored = {}, {}- fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored)- monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)- monkeypatch.setattr(- cli_module.Config, "to_dict",- lambda self: {"vector_store": {"qdrant": {}, "sqlite": {"dimension": 128}}},- )-- result = runner.invoke(cli_module.main, ["store", "migrate",- "--from", "qdrant", "--to", "sqlite", "--json"])- _ok(result)- assert dest_configs["sqlite"]["dimension"] == 128-- def test_migrate_qdrant_is_now_supported(self, runner, monkeypatch):- dest_configs, stored = {}, {}- fake_vs = self._fake_dimensionless_store_module([], dest_configs, stored)- monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)-- result = runner.invoke(cli_module.main, ["store", "migrate",- "--from", "qdrant", "--to", "pgvector", "--json"])- _ok(result)- assert _json_output(result)["migrated"] == 0-- def test_migrate_still_refuses_milvus(self, runner):- result = runner.invoke(cli_module.main, ["store", "migrate",- "--from", "qdrant", "--to", "milvus"])- assert result.exit_code != 0- assert "milvus" in result.output-def test_migrate_faiss_dest_requires_index_path(self, runner, monkeypatch):fake_vs = _fake_module(VectorStore=lambda **kw: MagicMock())monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)