mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-12 04:01:35 +00:00
fix(vector_store): drop qdrant migrate wiring, keep iter_all only
VectorStore cannot actually migrate to or from qdrant yet. _init_backend_store constructs QdrantStore without connecting or selecting a collection, so reads raise a Collection not initialized error, and the facade store_vectors dispatches only to add/add_vectors while QdrantStore exposes insert_vectors, so writes raise NotImplementedError. Both are pre-existing facade gaps that nothing had exposed, since migrate previously only allowed faiss/sqlite/pgvector. Adding qdrant to the allowlist claimed support that does not work end to end, so it is removed along with the dimension inference that only fires for backends missing a .dimension attribute. Tracked separately; this PR keeps just the iter_all primitive.
This commit is contained in:
Zohaib Hassnain
+8
-23
@@ -5,7 +5,6 @@ This module provides the command-line interface for the Semantica framework,
|
||||
enabling users to interact with the framework via terminal commands.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
@@ -3715,7 +3714,7 @@ def store_stats(cli_ctx: CLIContext, backend: str, fmt: str, local_json: bool) -
|
||||
_run_with_error_handling(_action)
|
||||
|
||||
|
||||
_MIGRATE_SUPPORTED_BACKENDS = {"faiss", "sqlite", "pgvector", "qdrant"}
|
||||
_MIGRATE_SUPPORTED_BACKENDS = {"faiss", "sqlite", "pgvector"}
|
||||
_MIGRATE_BATCH_SIZE = 500
|
||||
|
||||
|
||||
@@ -3760,10 +3759,12 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str,
|
||||
namespace: Optional[str], local_dry: bool, local_json: bool) -> None:
|
||||
"""Migrate data between backends.
|
||||
|
||||
Wired up between faiss, sqlite, pgvector and qdrant. Migrating to or from
|
||||
pinecone, milvus or weaviate still needs the export/reindex workaround
|
||||
below, since each of those needs its own enumeration design (Pinecone
|
||||
list, Milvus query_iterator, Weaviate cursor) that is not built yet.
|
||||
Direct migration is only wired up between faiss, sqlite, and pgvector -
|
||||
these are the backends whose storage contract supports paging through
|
||||
every stored vector. Migrating to or from qdrant, pinecone, milvus, or
|
||||
weaviate still needs the export/reindex workaround below, since each of
|
||||
those needs its own enumeration design (Qdrant scroll, Pinecone list,
|
||||
etc.) that hasn't been built yet.
|
||||
|
||||
\b
|
||||
Example:
|
||||
@@ -3803,18 +3804,7 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str,
|
||||
if source_index_path:
|
||||
source._backend_store.load_index(source_index_path)
|
||||
|
||||
# Pull the first record before building the destination: qdrant, milvus
|
||||
# and weaviate expose no `dimension` attribute, so without this the
|
||||
# destination falls back to the default 768 and rejects every insert.
|
||||
# The data itself is the reliable source of truth.
|
||||
source_iter = source.iter_vectors(batch_size=_MIGRATE_BATCH_SIZE)
|
||||
first_item = next(source_iter, None)
|
||||
|
||||
source_dimension = getattr(source._backend_store, "dimension", None)
|
||||
if not source_dimension and first_item is not None:
|
||||
first_vector = first_item.get("vector")
|
||||
if first_vector is not None:
|
||||
source_dimension = len(first_vector)
|
||||
if source_dimension and "dimension" not in dest_cfg:
|
||||
dest_cfg["dimension"] = source_dimension
|
||||
|
||||
@@ -3837,12 +3827,7 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str,
|
||||
metadata_batch.clear()
|
||||
ids_batch.clear()
|
||||
|
||||
items = (
|
||||
source_iter
|
||||
if first_item is None
|
||||
else itertools.chain([first_item], source_iter)
|
||||
)
|
||||
for item in items:
|
||||
for item in source.iter_vectors(batch_size=_MIGRATE_BATCH_SIZE):
|
||||
meta = dict(item.get("metadata") or {})
|
||||
if namespace and "namespace" not in meta:
|
||||
meta["namespace"] = namespace
|
||||
|
||||
@@ -1360,11 +1360,10 @@ class TestStore:
|
||||
assert result.exit_code != 0
|
||||
|
||||
def test_migrate_refuses_unsupported_backend_pair(self, runner):
|
||||
# qdrant is wired up now, so weaviate stands in as the unsupported side.
|
||||
result = runner.invoke(cli_module.main, ["store", "migrate",
|
||||
"--from", "faiss", "--to", "weaviate"])
|
||||
"--from", "faiss", "--to", "qdrant"])
|
||||
assert result.exit_code != 0
|
||||
assert "faiss, pgvector, qdrant, sqlite" in result.output
|
||||
assert "faiss, pgvector, sqlite" in result.output
|
||||
|
||||
def _fake_migrate_store_module(self, source_items, stored, dest_configs=None):
|
||||
class _FakeBackendStore:
|
||||
@@ -1446,97 +1445,6 @@ class TestStore:
|
||||
assert result.exit_code != 0
|
||||
assert "index_path" in result.output
|
||||
|
||||
def _fake_dimensionless_store_module(self, source_items, dest_configs, stored):
|
||||
"""Fake VectorStore whose backend store has NO `dimension` attribute.
|
||||
|
||||
Matches qdrant/milvus/weaviate, none of which expose one.
|
||||
"""
|
||||
|
||||
class _DimensionlessBackendStore:
|
||||
pass
|
||||
|
||||
class _FakeStore:
|
||||
def __init__(self, backend, config=None, **kw):
|
||||
self.backend = backend
|
||||
self._config = config or {}
|
||||
self._backend_store = _DimensionlessBackendStore()
|
||||
dest_configs[backend] = dict(self._config)
|
||||
|
||||
def iter_vectors(self, batch_size=500):
|
||||
if self.backend == "qdrant":
|
||||
yield from source_items
|
||||
return
|
||||
return
|
||||
yield # pragma: no cover - makes this a generator
|
||||
|
||||
def store_vectors(self, vectors, metadata, ids=None):
|
||||
for vec_id, vec in zip(ids, vectors):
|
||||
stored[vec_id] = vec
|
||||
|
||||
return _fake_module(VectorStore=_FakeStore)
|
||||
|
||||
def test_migrate_infers_dimension_from_first_vector(self, runner, monkeypatch):
|
||||
source_items = [
|
||||
{"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}},
|
||||
{"id": "b", "vector": [0.4, 0.5, 0.6], "metadata": {}},
|
||||
]
|
||||
dest_configs, stored = {}, {}
|
||||
fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored)
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)
|
||||
|
||||
result = runner.invoke(cli_module.main, ["store", "migrate",
|
||||
"--from", "qdrant", "--to", "sqlite", "--json"])
|
||||
_ok(result)
|
||||
assert dest_configs["sqlite"].get("dimension") == 3
|
||||
|
||||
def test_migrate_does_not_drop_the_peeked_first_record(self, runner, monkeypatch):
|
||||
"""The first record is consumed to infer dimension, so it must be
|
||||
chained back into the migration loop rather than lost."""
|
||||
source_items = [
|
||||
{"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}},
|
||||
{"id": "b", "vector": [0.4, 0.5, 0.6], "metadata": {}},
|
||||
]
|
||||
dest_configs, stored = {}, {}
|
||||
fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored)
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)
|
||||
|
||||
result = runner.invoke(cli_module.main, ["store", "migrate",
|
||||
"--from", "qdrant", "--to", "sqlite", "--json"])
|
||||
_ok(result)
|
||||
assert _json_output(result)["migrated"] == 2
|
||||
assert sorted(stored) == ["a", "b"]
|
||||
|
||||
def test_migrate_keeps_explicit_dest_dimension_over_inference(self, runner, monkeypatch):
|
||||
source_items = [{"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}}]
|
||||
dest_configs, stored = {}, {}
|
||||
fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored)
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)
|
||||
monkeypatch.setattr(
|
||||
cli_module.Config, "to_dict",
|
||||
lambda self: {"vector_store": {"qdrant": {}, "sqlite": {"dimension": 128}}},
|
||||
)
|
||||
|
||||
result = runner.invoke(cli_module.main, ["store", "migrate",
|
||||
"--from", "qdrant", "--to", "sqlite", "--json"])
|
||||
_ok(result)
|
||||
assert dest_configs["sqlite"]["dimension"] == 128
|
||||
|
||||
def test_migrate_qdrant_is_now_supported(self, runner, monkeypatch):
|
||||
dest_configs, stored = {}, {}
|
||||
fake_vs = self._fake_dimensionless_store_module([], dest_configs, stored)
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)
|
||||
|
||||
result = runner.invoke(cli_module.main, ["store", "migrate",
|
||||
"--from", "qdrant", "--to", "pgvector", "--json"])
|
||||
_ok(result)
|
||||
assert _json_output(result)["migrated"] == 0
|
||||
|
||||
def test_migrate_still_refuses_milvus(self, runner):
|
||||
result = runner.invoke(cli_module.main, ["store", "migrate",
|
||||
"--from", "qdrant", "--to", "milvus"])
|
||||
assert result.exit_code != 0
|
||||
assert "milvus" in result.output
|
||||
|
||||
def test_migrate_faiss_dest_requires_index_path(self, runner, monkeypatch):
|
||||
fake_vs = _fake_module(VectorStore=lambda **kw: MagicMock())
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs)
|
||||
|
||||
Reference in New Issue
Block a user