diff --git a/semantica/cli.py b/semantica/cli.py index d05a00f1..957878f4 100644 --- a/semantica/cli.py +++ b/semantica/cli.py @@ -5,6 +5,7 @@ This module provides the command-line interface for the Semantica framework, enabling users to interact with the framework via terminal commands. """ +import itertools import json import os import sys @@ -3714,7 +3715,7 @@ def store_stats(cli_ctx: CLIContext, backend: str, fmt: str, local_json: bool) - _run_with_error_handling(_action) -_MIGRATE_SUPPORTED_BACKENDS = {"faiss", "sqlite", "pgvector"} +_MIGRATE_SUPPORTED_BACKENDS = {"faiss", "sqlite", "pgvector", "qdrant"} _MIGRATE_BATCH_SIZE = 500 @@ -3759,12 +3760,10 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str, namespace: Optional[str], local_dry: bool, local_json: bool) -> None: """Migrate data between backends. - Direct migration is only wired up between faiss, sqlite, and pgvector - - these are the backends whose storage contract supports paging through - every stored vector. Migrating to or from qdrant, pinecone, milvus, or - weaviate still needs the export/reindex workaround below, since each of - those needs its own enumeration design (Qdrant scroll, Pinecone list, - etc.) that hasn't been built yet. + Wired up between faiss, sqlite, pgvector and qdrant. Migrating to or from + pinecone, milvus or weaviate still needs the export/reindex workaround + below, since each of those needs its own enumeration design (Pinecone + list, Milvus query_iterator, Weaviate cursor) that is not built yet. \b Example: @@ -3804,7 +3803,18 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str, if source_index_path: source._backend_store.load_index(source_index_path) + # Pull the first record before building the destination: qdrant, milvus + # and weaviate expose no `dimension` attribute, so without this the + # destination falls back to the default 768 and rejects every insert. + # The data itself is the reliable source of truth. + source_iter = source.iter_vectors(batch_size=_MIGRATE_BATCH_SIZE) + first_item = next(source_iter, None) + source_dimension = getattr(source._backend_store, "dimension", None) + if not source_dimension and first_item is not None: + first_vector = first_item.get("vector") + if first_vector is not None: + source_dimension = len(first_vector) if source_dimension and "dimension" not in dest_cfg: dest_cfg["dimension"] = source_dimension @@ -3827,7 +3837,12 @@ def store_migrate(cli_ctx: CLIContext, from_backend: str, to_backend: str, metadata_batch.clear() ids_batch.clear() - for item in source.iter_vectors(batch_size=_MIGRATE_BATCH_SIZE): + items = ( + source_iter + if first_item is None + else itertools.chain([first_item], source_iter) + ) + for item in items: meta = dict(item.get("metadata") or {}) if namespace and "namespace" not in meta: meta["namespace"] = namespace diff --git a/semantica/vector_store/qdrant_store.py b/semantica/vector_store/qdrant_store.py index 1779a0bc..85ce7302 100644 --- a/semantica/vector_store/qdrant_store.py +++ b/semantica/vector_store/qdrant_store.py @@ -604,6 +604,62 @@ class QdrantStore: self.logger.warning(f"Failed to scroll Qdrant points by metadata filter: {e}") return [] + def iter_all(self, batch_size: int = 500): + """ + Iterate over every stored point using Qdrant's native scroll cursor. + + Qdrant paginates by point-ID cursor, not by row offset, so this is + exposed instead of scan_vectors(offset, limit). An integer passed to + scroll()'s offset is a point ID rather than a rank, so there is no way + to seek to "the Nth record" without walking from the start. + VectorStore.iter_vectors() prefers this method when it is present. + + Assumes a single unnamed vector per point, matching how insert_vectors() + writes them and how get_vector() reads them back. Collections configured + with named or multi-vectors are not handled here. + + Args: + batch_size: Points to request per scroll call + + Yields: + Result dicts with 'id', 'metadata', and 'vector', in scroll order + + Raises: + ProcessingError: If the collection or client is not initialized. + Errors are raised rather than swallowed because a scan that + silently yields nothing is indistinguishable from an empty + source, which would let a caller such as `store migrate` + report success having copied nothing (issue #1083). + """ + if self.collection is None or self.client is None or not QDRANT_AVAILABLE: + raise ProcessingError( + "Collection not initialized. Call create_collection() or get_collection() first." + ) + + next_offset = None + while True: + records, next_offset = self.client.scroll( + collection_name=self.collection.collection_name, + limit=batch_size, + offset=next_offset, + with_payload=True, + with_vectors=True, + ) + + for rec in records: + yield { + "id": str(rec.id), + "metadata": rec.payload or {}, + "vector": np.array(rec.vector) if rec.vector is not None else None, + } + + # The final page can carry records while already reporting no next + # cursor, so those records are yielded above before stopping here. + # Calling scroll() again with offset=None would restart from the + # beginning rather than continue past the end. + if next_offset is None or not records: + return + def delete_vectors( self, point_ids: List[Union[str, int]], **options ) -> Dict[str, Any]: diff --git a/semantica/vector_store/vector_store.py b/semantica/vector_store/vector_store.py index d0257240..cd703eda 100644 --- a/semantica/vector_store/vector_store.py +++ b/semantica/vector_store/vector_store.py @@ -867,12 +867,25 @@ class VectorStore: """ Iterate over every stored vector, one page at a time. + Backends whose native pagination is cursor based (Qdrant, Pinecone, + Milvus, Weaviate) cannot honestly implement the positional + scan_vectors(offset, limit) contract, so they expose iter_all() + instead and it is preferred here when present. Backends with real + positional access (inmemory, FAISS, SQLite-vec, PgVector) fall + through to the offset loop below. + Args: - batch_size: Number of vectors to fetch per underlying scan_vectors() call + batch_size: Number of vectors to fetch per underlying call Yields: Result dicts with 'id', 'metadata', and 'vector', in scan order """ + if self.backend != "inmemory" and self._backend_store is not None: + iter_all = getattr(self._backend_store, "iter_all", None) + if callable(iter_all): + yield from iter_all(batch_size=batch_size) + return + offset = 0 while True: page = self.scan_vectors(offset=offset, limit=batch_size) diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 4a054be1..a90673c4 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -1360,10 +1360,11 @@ class TestStore: assert result.exit_code != 0 def test_migrate_refuses_unsupported_backend_pair(self, runner): + # qdrant is wired up now, so weaviate stands in as the unsupported side. result = runner.invoke(cli_module.main, ["store", "migrate", - "--from", "faiss", "--to", "qdrant"]) + "--from", "faiss", "--to", "weaviate"]) assert result.exit_code != 0 - assert "faiss, pgvector, sqlite" in result.output + assert "faiss, pgvector, qdrant, sqlite" in result.output def _fake_migrate_store_module(self, source_items, stored, dest_configs=None): class _FakeBackendStore: @@ -1445,6 +1446,97 @@ class TestStore: assert result.exit_code != 0 assert "index_path" in result.output + def _fake_dimensionless_store_module(self, source_items, dest_configs, stored): + """Fake VectorStore whose backend store has NO `dimension` attribute. + + Matches qdrant/milvus/weaviate, none of which expose one. + """ + + class _DimensionlessBackendStore: + pass + + class _FakeStore: + def __init__(self, backend, config=None, **kw): + self.backend = backend + self._config = config or {} + self._backend_store = _DimensionlessBackendStore() + dest_configs[backend] = dict(self._config) + + def iter_vectors(self, batch_size=500): + if self.backend == "qdrant": + yield from source_items + return + return + yield # pragma: no cover - makes this a generator + + def store_vectors(self, vectors, metadata, ids=None): + for vec_id, vec in zip(ids, vectors): + stored[vec_id] = vec + + return _fake_module(VectorStore=_FakeStore) + + def test_migrate_infers_dimension_from_first_vector(self, runner, monkeypatch): + source_items = [ + {"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}}, + {"id": "b", "vector": [0.4, 0.5, 0.6], "metadata": {}}, + ] + dest_configs, stored = {}, {} + fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored) + monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs) + + result = runner.invoke(cli_module.main, ["store", "migrate", + "--from", "qdrant", "--to", "sqlite", "--json"]) + _ok(result) + assert dest_configs["sqlite"].get("dimension") == 3 + + def test_migrate_does_not_drop_the_peeked_first_record(self, runner, monkeypatch): + """The first record is consumed to infer dimension, so it must be + chained back into the migration loop rather than lost.""" + source_items = [ + {"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}}, + {"id": "b", "vector": [0.4, 0.5, 0.6], "metadata": {}}, + ] + dest_configs, stored = {}, {} + fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored) + monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs) + + result = runner.invoke(cli_module.main, ["store", "migrate", + "--from", "qdrant", "--to", "sqlite", "--json"]) + _ok(result) + assert _json_output(result)["migrated"] == 2 + assert sorted(stored) == ["a", "b"] + + def test_migrate_keeps_explicit_dest_dimension_over_inference(self, runner, monkeypatch): + source_items = [{"id": "a", "vector": [0.1, 0.2, 0.3], "metadata": {}}] + dest_configs, stored = {}, {} + fake_vs = self._fake_dimensionless_store_module(source_items, dest_configs, stored) + monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs) + monkeypatch.setattr( + cli_module.Config, "to_dict", + lambda self: {"vector_store": {"qdrant": {}, "sqlite": {"dimension": 128}}}, + ) + + result = runner.invoke(cli_module.main, ["store", "migrate", + "--from", "qdrant", "--to", "sqlite", "--json"]) + _ok(result) + assert dest_configs["sqlite"]["dimension"] == 128 + + def test_migrate_qdrant_is_now_supported(self, runner, monkeypatch): + dest_configs, stored = {}, {} + fake_vs = self._fake_dimensionless_store_module([], dest_configs, stored) + monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs) + + result = runner.invoke(cli_module.main, ["store", "migrate", + "--from", "qdrant", "--to", "pgvector", "--json"]) + _ok(result) + assert _json_output(result)["migrated"] == 0 + + def test_migrate_still_refuses_milvus(self, runner): + result = runner.invoke(cli_module.main, ["store", "migrate", + "--from", "qdrant", "--to", "milvus"]) + assert result.exit_code != 0 + assert "milvus" in result.output + def test_migrate_faiss_dest_requires_index_path(self, runner, monkeypatch): fake_vs = _fake_module(VectorStore=lambda **kw: MagicMock()) monkeypatch.setitem(__import__("sys").modules, "semantica.vector_store", fake_vs) diff --git a/tests/vector_store/test_qdrant_store.py b/tests/vector_store/test_qdrant_store.py new file mode 100644 index 00000000..89a9fee3 --- /dev/null +++ b/tests/vector_store/test_qdrant_store.py @@ -0,0 +1,152 @@ +"""Tests for QdrantStore.iter_all() cursor enumeration. + +Qdrant is not installed in this environment, so these drive the real +QdrantStore against a MagicMock standing in for the qdrant_client, following +the pattern already used for qdrant in test_backend_metadata_filtering.py. +""" + +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest + +from semantica.utils.exceptions import ProcessingError +from semantica.vector_store.qdrant_store import QdrantStore + + +def _record(point_id, payload=None, vector=None): + """Build a stand-in for a qdrant_client Record.""" + rec = MagicMock() + rec.id = point_id + rec.payload = payload + rec.vector = vector + return rec + + +def _store_with_scroll(*pages): + """QdrantStore whose client.scroll() returns the given (records, cursor) pages.""" + store = QdrantStore() + store.client = MagicMock() + store.client.scroll.side_effect = list(pages) + store.collection = MagicMock() + store.collection.collection_name = "test_collection" + return store + + +@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True) +def test_iter_all_threads_cursor_across_pages(): + """The next call must continue from the previous page's next_page_offset.""" + store = _store_with_scroll( + ([_record(1), _record(2)], "cursor-1"), + ([_record(3)], None), + ) + + result = list(store.iter_all(batch_size=2)) + + assert [item["id"] for item in result] == ["1", "2", "3"] + calls = store.client.scroll.call_args_list + assert len(calls) == 2 + assert calls[0][1]["offset"] is None + assert calls[0][1]["limit"] == 2 + assert calls[1][1]["offset"] == "cursor-1" + + +@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True) +def test_iter_all_yields_final_page_that_reports_no_next_cursor(): + """Qdrant can return records and a null cursor on the same page. + + Those records must still be yielded. Treating a null cursor as "stop + before this page" would silently drop the tail of every scan. + """ + store = _store_with_scroll(([_record(1), _record(2)], None)) + + result = list(store.iter_all(batch_size=10)) + + assert [item["id"] for item in result] == ["1", "2"] + assert store.client.scroll.call_count == 1 + + +@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True) +def test_iter_all_converts_records_to_the_shared_result_shape(): + store = _store_with_scroll( + ([_record(7, payload={"tag": "x"}, vector=[0.1, 0.2, 0.3])], None), + ) + + item = list(store.iter_all())[0] + + assert item["id"] == "7" + assert item["metadata"] == {"tag": "x"} + np.testing.assert_allclose(item["vector"], np.array([0.1, 0.2, 0.3])) + + +@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True) +def test_iter_all_handles_missing_payload_and_vector(): + store = _store_with_scroll(([_record(1, payload=None, vector=None)], None)) + + item = list(store.iter_all())[0] + + assert item["metadata"] == {} + assert item["vector"] is None + + +@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True) +def test_iter_all_empty_collection_yields_nothing(): + store = _store_with_scroll(([], None)) + + assert list(store.iter_all()) == [] + + +@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True) +def test_iter_all_stops_on_empty_page_even_with_a_cursor(): + """Defensive: an empty page ends the scan rather than looping forever.""" + store = _store_with_scroll(([], "cursor-that-never-clears")) + + assert list(store.iter_all()) == [] + assert store.client.scroll.call_count == 1 + + +@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True) +def test_iter_all_raises_when_collection_not_initialized(): + """Must fail loudly, not yield nothing. + + An empty scan is indistinguishable from an empty source, which would let + `store migrate` report success having copied nothing (issue #1083). + """ + store = QdrantStore() + + with pytest.raises(ProcessingError, match="Collection not initialized"): + list(store.iter_all()) + + +@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", False) +def test_iter_all_raises_when_qdrant_unavailable(): + store = QdrantStore() + store.client = MagicMock() + store.collection = MagicMock() + + with pytest.raises(ProcessingError): + list(store.iter_all()) + + +@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True) +def test_iter_all_propagates_scroll_errors(): + store = QdrantStore() + store.client = MagicMock() + store.client.scroll.side_effect = RuntimeError("connection reset") + store.collection = MagicMock() + store.collection.collection_name = "test_collection" + + with pytest.raises(RuntimeError, match="connection reset"): + list(store.iter_all()) + + +@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True) +def test_iter_all_requests_payload_and_vectors(): + store = _store_with_scroll(([], None)) + + list(store.iter_all()) + + kwargs = store.client.scroll.call_args[1] + assert kwargs["with_payload"] is True + assert kwargs["with_vectors"] is True + assert kwargs["collection_name"] == "test_collection" diff --git a/tests/vector_store/test_vector_manager_persistent.py b/tests/vector_store/test_vector_manager_persistent.py index 6a432939..f0586ad2 100644 --- a/tests/vector_store/test_vector_manager_persistent.py +++ b/tests/vector_store/test_vector_manager_persistent.py @@ -26,6 +26,7 @@ from unittest.mock import MagicMock, patch import numpy as np +from semantica.utils.exceptions import ProcessingError from semantica.vector_store.vector_store import VectorStore, VectorManager @@ -138,6 +139,38 @@ class _NonScanningBackendStore: """Fake persistent backend store without any scan capability.""" +class _IterAllBackendStore: + """Fake cursor-based backend store exposing iter_all() but not scan_vectors(). + + Mirrors qdrant/pinecone/milvus/weaviate, which cannot honour a positional + offset and therefore expose native iteration instead. + """ + + def __init__(self, items): + self._items = items + self.batch_sizes = [] + + def iter_all(self, batch_size=500): + self.batch_sizes.append(batch_size) + for item in self._items: + yield item + + def scan_vectors(self, offset=0, limit=100): + raise AssertionError("scan_vectors() must not be called when iter_all() exists") + + +class _MisShapedIterAllBackendStore: + """Backend store whose ``iter_all`` attribute is not callable.""" + + iter_all = 42 # plain attribute, not a method + + def __init__(self, items): + self._items = items + + def scan_vectors(self, offset=0, limit=100): + return self._items[offset:offset + limit] + + class VectorStoreScanVectorsTests(unittest.TestCase): """VectorStore.scan_vectors() / iter_vectors() backend-agnostic accessors.""" @@ -192,6 +225,81 @@ class VectorStoreScanVectorsTests(unittest.TestCase): self.assertEqual(list(store.iter_vectors(batch_size=2)), []) +# --------------------------------------------------------------------------- +# VectorStore.iter_vectors() preference for a native iter_all() +# --------------------------------------------------------------------------- + +class VectorStoreIterAllDispatchTests(unittest.TestCase): + """iter_vectors() prefers a backend's native iter_all() when present. + + Cursor-based backends cannot implement scan_vectors(offset, limit) + honestly, so they expose iter_all() instead and iter_vectors() routes to + it rather than walking offsets. + """ + + def _persistent_store(self, backend_store, backend_name="qdrant"): + store = VectorStore(backend="inmemory", dimension=2) + store.backend = backend_name + store._backend_store = backend_store + return store + + def test_iter_vectors_uses_iter_all_when_available(self): + items = [ + {"id": "a", "vector": None, "metadata": {"n": 1}}, + {"id": "b", "vector": None, "metadata": {"n": 2}}, + ] + backend = _IterAllBackendStore(items) + store = self._persistent_store(backend) + + self.assertEqual(list(store.iter_vectors(batch_size=7)), items) + + def test_iter_vectors_forwards_batch_size_to_iter_all(self): + backend = _IterAllBackendStore([]) + store = self._persistent_store(backend) + + list(store.iter_vectors(batch_size=32)) + + self.assertEqual(backend.batch_sizes, [32]) + + def test_iter_vectors_falls_back_to_scan_vectors_without_iter_all(self): + items = [{"id": "a", "vector": None, "metadata": {}}] + store = self._persistent_store(_ScanningBackendStore(items)) + + self.assertEqual(list(store.iter_vectors(batch_size=2)), items) + + def test_iter_vectors_falls_back_when_iter_all_not_callable(self): + # A mis-shaped adapter exposing a non-callable ``iter_all`` must not be + # invoked; the offset path still has to work. Mirrors the count() + # precedent in _MisShapedBackendStore. + items = [{"id": "a", "vector": None, "metadata": {}}] + store = self._persistent_store(_MisShapedIterAllBackendStore(items)) + + self.assertEqual(list(store.iter_vectors(batch_size=2)), items) + + def test_iter_vectors_inmemory_ignores_iter_all(self): + store = VectorStore(backend="inmemory", dimension=2) + store.store_vectors([np.array([1.0, 0.0])], [{"type": "a"}]) + store._backend_store = _IterAllBackendStore([{"id": "wrong"}]) + + collected = list(store.iter_vectors(batch_size=2)) + + self.assertEqual([item["metadata"] for item in collected], [{"type": "a"}]) + + def test_iter_vectors_propagates_iter_all_errors(self): + # A scan that silently yields nothing is indistinguishable from an + # empty source, which would let `store migrate` report success having + # copied nothing (issue #1083). + class _FailingIterAll: + def iter_all(self, batch_size=500): + raise ProcessingError("backend unreachable") + yield # pragma: no cover - makes this a generator + + store = self._persistent_store(_FailingIterAll()) + + with self.assertRaises(ProcessingError): + list(store.iter_vectors(batch_size=2)) + + # --------------------------------------------------------------------------- # VectorManager tests — inmemory backend # ---------------------------------------------------------------------------