From ec9e63e16fabbec2aaa56f48773a44d5a848b951 Mon Sep 17 00:00:00 2001 From: Zohaib Hassnain <109234410+ZohaibHassan16@users.noreply.github.com> Date: Mon, 31 Aug 2026 02:54:29 +0500 Subject: [PATCH] fix(vector_store): drop qdrant migrate wiring, keep iter_all only VectorStore cannot actually migrate to or from qdrant yet. _init_backend_store constructs QdrantStore without connecting or selecting a collection, so reads raise a Collection not initialized error, and the facade store_vectors dispatches only to add/add_vectors while QdrantStore exposes insert_vectors, so writes raise NotImplementedError. Both are pre-existing facade gaps that nothing had exposed, since migrate previously only allowed faiss/sqlite/pgvector. Adding qdrant to the allowlist claimed support that does not work end to end, so it is removed along with the dimension inference that only fires for backends missing a .dimension attribute. Tracked separately; this PR keeps just the iter_all primitive. --- semantica/cli.py | 31 ++++-------- tests/test_cli_commands.py | 96 +------------------------------------- 2 files changed, 10 insertions(+), 117 deletions(-) diff --git a/semantica/cli.py b/semantica/cli.py index 957878f4..d05a00f1 100644 --- a/semantica/cli.py +++ b/semantica/cli.py @@ -5,7 +5,6 @@ This module provides the command-line interface for the Semantica framework, enabling users to interact with the framework via terminal commands. """ -import itertools import json import os import sys @@ -3715,7 +3714,7 @@ def store_stats(cli_ctx: CLIContext, backend: str, fmt: str, local_json: bool) - _run_with_error_handling(_action) -_MIGRATE_SUPPORTED_BACKENDS = {"faiss", "sqlite", "pgvector", "qdrant"} +_MIGRATE_SUPPORTED_BACKENDS = {"faiss", "sqlite", "pgvector"} _MIGRATE_BATCH_SIZE = 500 @@ -3760,10 +3759,12 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str, namespace: Optional[str], local_dry: bool, local_json: bool) -> None: """Migrate data between backends. - Wired up between faiss, sqlite, pgvector and qdrant. Migrating to or from - pinecone, milvus or weaviate still needs the export/reindex workaround - below, since each of those needs its own enumeration design (Pinecone - list, Milvus query_iterator, Weaviate cursor) that is not built yet. + Direct migration is only wired up between faiss, sqlite, and pgvector - + these are the backends whose storage contract supports paging through + every stored vector. Migrating to or from qdrant, pinecone, milvus, or + weaviate still needs the export/reindex workaround below, since each of + those needs its own enumeration design (Qdrant scroll, Pinecone list, + etc.) that hasn't been built yet. \b Example: @@ -3803,18 +3804,7 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str, if source_index_path: source._backend_store.load_index(source_index_path) - # Pull the first record before building the destination: qdrant, milvus - # and weaviate expose no `dimension` attribute, so without this the - # destination falls back to the default 768 and rejects every insert. - # The data itself is the reliable source of truth. - source_iter = source.iter_vectors(batch_size=_MIGRATE_BATCH_SIZE) - first_item = next(source_iter, None) - source_dimension = getattr(source._backend_store, "dimension", None) - if not source_dimension and first_item is not None: - first_vector = first_item.get("vector") - if first_vector is not None: - source_dimension = len(first_vector) if source_dimension and "dimension" not in dest_cfg: dest_cfg["dimension"] = source_dimension @@ -3837,12 +3827,7 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str, metadata_batch.clear() ids_batch.clear() - items = ( - source_iter - if first_item is None - else itertools.chain([first_item], source_iter) - ) - for item in items: + for item in source.iter_vectors(batch_size=_MIGRATE_BATCH_SIZE): meta = dict(item.get("metadata") or {}) if namespace and "namespace" not in meta: meta["namespace"] = namespace diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index a90673c4..4a054be1 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -1360,11 +1360,10 @@ class TestStore: assert result.exit_code != 0 def test_migrate_refuses_unsupported_backend_pair(self, runner): - # qdrant is wired up now, so weaviate stands in as the unsupported side. result = runner.invoke(cli_module.main, ["store", "migrate", - "--from", "faiss", "--to", "weaviate"]) + "--from", "faiss", "--to", "qdrant"]) assert result.exit_code != 0 - assert "faiss, pgvector, qdrant, sqlite" in result.output + assert "faiss, pgvector, sqlite" in result.output def _fake_migrate_store_module(self, source_items, stored, dest_configs=None): class _FakeBackendStore: @@ -1446,97 +1445,6 @@ class TestStore: assert result.exit_code != 0 assert "index_path" in result.output - def _fake_dimensionless_store_module(self, source_items, dest_configs, stored): - """Fake VectorStore whose backend store has NO `dimension` attribute. - - Matches qdrant/milvus/weaviate, none of which expose one. - """ - - class _DimensionlessBackendStore: - pass - - class _FakeStore: - def __init__(self, backend, config=None, **kw): - self.backend = backend - self._config = config or {} - self._backend_store = _DimensionlessBackendStore() - dest_configs[backend] = dict(self._config) - - def iter_vectors(self, batch_size=500): - if self.backend == "qdrant": - yield from source_items - return - return - yield # pragma: no cover - makes this a generator - - def store_vectors(self, vectors, metadata, ids=None): - for vec_id, vec in zip(ids, vectors): - stored[vec_id] = vec - - return _fake_module(VectorStore=_FakeStore) - - def test_migrate_infers_dimension_from_first_vector(self, runner, monkeypatch): - source_items = [ - {"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}}, - {"id": "b", "vector": [0.4, 0.5, 0.6], "metadata": {}}, - ] - dest_configs, stored = {}, {} - fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored) - monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs) - - result = runner.invoke(cli_module.main, ["store", "migrate", - "--from", "qdrant", "--to", "sqlite", "--json"]) - _ok(result) - assert dest_configs["sqlite"].get("dimension") == 3 - - def test_migrate_does_not_drop_the_peeked_first_record(self, runner, monkeypatch): - """The first record is consumed to infer dimension, so it must be - chained back into the migration loop rather than lost.""" - source_items = [ - {"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}}, - {"id": "b", "vector": [0.4, 0.5, 0.6], "metadata": {}}, - ] - dest_configs, stored = {}, {} - fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored) - monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs) - - result = runner.invoke(cli_module.main, ["store", "migrate", - "--from", "qdrant", "--to", "sqlite", "--json"]) - _ok(result) - assert _json_output(result)["migrated"] == 2 - assert sorted(stored) == ["a", "b"] - - def test_migrate_keeps_explicit_dest_dimension_over_inference(self, runner, monkeypatch): - source_items = [{"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}}] - dest_configs, stored = {}, {} - fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored) - monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs) - monkeypatch.setattr( - cli_module.Config, "to_dict", - lambda self: {"vector_store": {"qdrant": {}, "sqlite": {"dimension": 128}}}, - ) - - result = runner.invoke(cli_module.main, ["store", "migrate", - "--from", "qdrant", "--to", "sqlite", "--json"]) - _ok(result) - assert dest_configs["sqlite"]["dimension"] == 128 - - def test_migrate_qdrant_is_now_supported(self, runner, monkeypatch): - dest_configs, stored = {}, {} - fake_vs = self._fake_dimensionless_store_module([], dest_configs, stored) - monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs) - - result = runner.invoke(cli_module.main, ["store", "migrate", - "--from", "qdrant", "--to", "pgvector", "--json"]) - _ok(result) - assert _json_output(result)["migrated"] == 0 - - def test_migrate_still_refuses_milvus(self, runner): - result = runner.invoke(cli_module.main, ["store", "migrate", - "--from", "qdrant", "--to", "milvus"]) - assert result.exit_code != 0 - assert "milvus" in result.output - def test_migrate_faiss_dest_requires_index_path(self, runner, monkeypatch): fake_vs = _fake_module(VectorStore=lambda **kw: MagicMock()) monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)