Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

fix(vector_store): drop qdrant migrate wiring, keep iter_all only

Zohaib Hassnainec9e63e16fab
diff --git a/semantica/cli.py b/semantica/cli.py
index 957878f4..d05a00f1 100644
--- a/semantica/cli.py
+++ b/semantica/cli.py
@@ -5,7 +5,6 @@ This module provides the command-line interface for the Semantica framework,
enabling users to interact with the framework via terminal commands.
"""
-import itertools
import json
import os
import sys
@@ -3715,7 +3714,7 @@ def store_stats(cli_ctx: CLIContext, backend: str, fmt: str, local_json: bool) -
_run_with_error_handling(_action)
-_MIGRATE_SUPPORTED_BACKENDS = {"faiss", "sqlite", "pgvector", "qdrant"}
+_MIGRATE_SUPPORTED_BACKENDS = {"faiss", "sqlite", "pgvector"}
_MIGRATE_BATCH_SIZE = 500
@@ -3760,10 +3759,12 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str,
namespace: Optional[str], local_dry: bool, local_json: bool) -> None:
"""Migrate data between backends.
- Wired up between faiss, sqlite, pgvector and qdrant. Migrating to or from
- pinecone, milvus or weaviate still needs the export/reindex workaround
- below, since each of those needs its own enumeration design (Pinecone
- list, Milvus query_iterator, Weaviate cursor) that is not built yet.
+ Direct migration is only wired up between faiss, sqlite, and pgvector -
+ these are the backends whose storage contract supports paging through
+ every stored vector. Migrating to or from qdrant, pinecone, milvus, or
+ weaviate still needs the export/reindex workaround below, since each of
+ those needs its own enumeration design (Qdrant scroll, Pinecone list,
+ etc.) that hasn't been built yet.
\b
Example:
@@ -3803,18 +3804,7 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str,
if source_index_path:
source._backend_store.load_index(source_index_path)
- # Pull the first record before building the destination: qdrant, milvus
- # and weaviate expose no `dimension` attribute, so without this the
- # destination falls back to the default 768 and rejects every insert.
- # The data itself is the reliable source of truth.
- source_iter = source.iter_vectors(batch_size=_MIGRATE_BATCH_SIZE)
- first_item = next(source_iter, None)
-
source_dimension = getattr(source._backend_store, "dimension", None)
- if not source_dimension and first_item is not None:
- first_vector = first_item.get("vector")
- if first_vector is not None:
- source_dimension = len(first_vector)
if source_dimension and "dimension" not in dest_cfg:
dest_cfg["dimension"] = source_dimension
@@ -3837,12 +3827,7 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str,
metadata_batch.clear()
ids_batch.clear()
- items = (
- source_iter
- if first_item is None
- else itertools.chain([first_item], source_iter)
- )
- for item in items:
+ for item in source.iter_vectors(batch_size=_MIGRATE_BATCH_SIZE):
meta = dict(item.get("metadata") or {})
if namespace and "namespace" not in meta:
meta["namespace"] = namespace
diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py
index a90673c4..4a054be1 100644
--- a/tests/test_cli_commands.py
+++ b/tests/test_cli_commands.py
@@ -1360,11 +1360,10 @@ class TestStore:
assert result.exit_code != 0
def test_migrate_refuses_unsupported_backend_pair(self, runner):
- # qdrant is wired up now, so weaviate stands in as the unsupported side.
result = runner.invoke(cli_module.main, ["store", "migrate",
- "--from", "faiss", "--to", "weaviate"])
+ "--from", "faiss", "--to", "qdrant"])
assert result.exit_code != 0
- assert "faiss, pgvector, qdrant, sqlite" in result.output
+ assert "faiss, pgvector, sqlite" in result.output
def _fake_migrate_store_module(self, source_items, stored, dest_configs=None):
class _FakeBackendStore:
@@ -1446,97 +1445,6 @@ class TestStore:
assert result.exit_code != 0
assert "index_path" in result.output
- def _fake_dimensionless_store_module(self, source_items, dest_configs, stored):
- """Fake VectorStore whose backend store has NO `dimension` attribute.
-
- Matches qdrant/milvus/weaviate, none of which expose one.
- """
-
- class _DimensionlessBackendStore:
- pass
-
- class _FakeStore:
- def __init__(self, backend, config=None, **kw):
- self.backend = backend
- self._config = config or {}
- self._backend_store = _DimensionlessBackendStore()
- dest_configs[backend] = dict(self._config)
-
- def iter_vectors(self, batch_size=500):
- if self.backend == "qdrant":
- yield from source_items
- return
- return
- yield # pragma: no cover - makes this a generator
-
- def store_vectors(self, vectors, metadata, ids=None):
- for vec_id, vec in zip(ids, vectors):
- stored[vec_id] = vec
-
- return _fake_module(VectorStore=_FakeStore)
-
- def test_migrate_infers_dimension_from_first_vector(self, runner, monkeypatch):
- source_items = [
- {"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}},
- {"id": "b", "vector": [0.4, 0.5, 0.6], "metadata": {}},
- ]
- dest_configs, stored = {}, {}
- fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored)
- monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)
-
- result = runner.invoke(cli_module.main, ["store", "migrate",
- "--from", "qdrant", "--to", "sqlite", "--json"])
- _ok(result)
- assert dest_configs["sqlite"].get("dimension") == 3
-
- def test_migrate_does_not_drop_the_peeked_first_record(self, runner, monkeypatch):
- """The first record is consumed to infer dimension, so it must be
- chained back into the migration loop rather than lost."""
- source_items = [
- {"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}},
- {"id": "b", "vector": [0.4, 0.5, 0.6], "metadata": {}},
- ]
- dest_configs, stored = {}, {}
- fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored)
- monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)
-
- result = runner.invoke(cli_module.main, ["store", "migrate",
- "--from", "qdrant", "--to", "sqlite", "--json"])
- _ok(result)
- assert _json_output(result)["migrated"] == 2
- assert sorted(stored) == ["a", "b"]
-
- def test_migrate_keeps_explicit_dest_dimension_over_inference(self, runner, monkeypatch):
- source_items = [{"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}}]
- dest_configs, stored = {}, {}
- fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored)
- monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)
- monkeypatch.setattr(
- cli_module.Config, "to_dict",
- lambda self: {"vector_store": {"qdrant": {}, "sqlite": {"dimension": 128}}},
- )
-
- result = runner.invoke(cli_module.main, ["store", "migrate",
- "--from", "qdrant", "--to", "sqlite", "--json"])
- _ok(result)
- assert dest_configs["sqlite"]["dimension"] == 128
-
- def test_migrate_qdrant_is_now_supported(self, runner, monkeypatch):
- dest_configs, stored = {}, {}
- fake_vs = self._fake_dimensionless_store_module([], dest_configs, stored)
- monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)
-
- result = runner.invoke(cli_module.main, ["store", "migrate",
- "--from", "qdrant", "--to", "pgvector", "--json"])
- _ok(result)
- assert _json_output(result)["migrated"] == 0
-
- def test_migrate_still_refuses_milvus(self, runner):
- result = runner.invoke(cli_module.main, ["store", "migrate",
- "--from", "qdrant", "--to", "milvus"])
- assert result.exit_code != 0
- assert "milvus" in result.output
-
def test_migrate_faiss_dest_requires_index_path(self, runner, monkeypatch):
fake_vs = _fake_module(VectorStore=lambda **kw: MagicMock())
monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)