fix(vector_store): dedupe qdrant record conversion, don't abort iter_all on a live cursor with an empty page

VectorStore cannot actually migrate to or from qdrant yet. _init_backend_store constructs QdrantStore without connecting or selecting a collection, so reads raise a Collection not initialized error, and the facade store_vectors dispatches only to add/add_vectors while QdrantStore exposes insert_vectors, so writes raise NotImplementedError.

Both are pre-existing facade gaps that nothing had exposed, since migrate previously only allowed faiss/sqlite/pgvector. Adding qdrant to the allowlist claimed support that does not work end to end, so it is removed along with the dimension inference that only fires for backends missing a .dimension attribute. Tracked separately; this PR keeps just the iter_all primitive.

.github/requirements/README.md

@@ -30,8 +30,7 @@ each file's own autogenerated header comment for its exact command).
 | `pep517-build.txt` | ci.yml, benchmark.yml, Dockerfile | exact `[build-system] requires` from `pyproject.toml` (setuptools, wheel) - installed with `--no-build-isolation` before any `pip install -e .` / `pip install .`, since `--no-deps` alone doesn't stop pip's PEP 517 build isolation from fetching those two *unhashed* |
 | `explorer-extra-py311.txt` | ci.yml | semantica's base deps + the `explorer` extra, resolved for python 3.11 |
 | `explorer-extra-py313.txt` | Dockerfile | the same, resolved for python 3.13 (the image's actual interpreter) |
-| `pgvector-extra.txt` | integration.yml | semantica's base deps + the `vectorstore-pgvector` extra, resolved for python 3.11 |
-| `pytest-tool.txt` | ci.yml, integration.yml | pytest, for the pre-all-extras deterministic test |
+| `pytest-tool.txt` | ci.yml | pytest, for the pre-all-extras deterministic test |
 | `uv-tool.txt` | ci.yml | uv, to verify requirements-ci.txt is current |
 | `build-tools.txt` | ci.yml, release.yml | build, wheel |
 | `twine.txt` | release.yml | twine |

.github/workflows/integration.yml

@@ -1,91 +0,0 @@
-name: Integration Tests
-
-# Separate from ci.yml, which is a required check: a slow image pull or a
-# container flake must not block unrelated merges.
-
-permissions:
-  contents: read
-
-on:
-  pull_request:
-    branches: [main]
-    paths-ignore:
-      - 'docs/**'
-      - 'docs_check.py'
-      - '**/*.md'
-  schedule:
-    - cron: '0 5 * * 1'
-  workflow_dispatch:
-
-jobs:
-  pgvector:
-    name: pgvector (live PostgreSQL)
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-
-    services:
-      postgres:
-        # pgvector/pgvector:pg16 as published 2026-08-13. Pinned by digest like
-        # the action pins, though verify-action-pins.sh does not check images.
-        image: pgvector/pgvector@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_DB: test
-          # Throwaway container reachable only from this job, so trust auth
-          # avoids putting a credential in the workflow at all.
-          POSTGRES_HOST_AUTH_METHOD: trust
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd "pg_isready -U postgres -d test"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 10
-
-    env:
-      TEST_PGVECTOR_URL: postgresql://postgres@localhost:5432/test
-
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-
-      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
-        with:
-          python-version: '3.11'
-          cache: 'pip'
-
-      - name: Install semantica with the pgvector extra
-        # Hash-verified installs throughout, matching ci.yml/security.yml/etc
-        # (OpenSSF Scorecard's Pinned-Dependencies check). --no-deps here
-        # skips runtime dependency resolution for the editable install itself
-        # (nothing to hash); pep517-build.txt + --no-build-isolation stops
-        # its PEP 517 build from separately fetching an unhashed
-        # setuptools/wheel via build isolation.
-        run: |
-          pip install -r .github/requirements/bootstrap.txt --require-hashes
-          pip install -r .github/requirements/pep517-build.txt --require-hashes
-          pip install --no-deps --no-build-isolation -e .
-          pip install -r .github/requirements/pgvector-extra.txt --require-hashes
-          pip install -r .github/requirements/pytest-tool.txt --require-hashes
-
-      - name: Create the vector extension
-        # PgVectorStore._verify_pgvector_extension() requires it and refuses to
-        # create it. Doubles as the connectivity gate.
-        run: |
-          python - <<'PY'
-          import os
-
-          import psycopg
-
-          with psycopg.connect(os.environ["TEST_PGVECTOR_URL"]) as conn:
-              conn.execute("CREATE EXTENSION IF NOT EXISTS vector")
-              conn.commit()
-          print("vector extension ready")
-          PY
-
-      - name: Run the live pgvector suite
-        # pg_available raises rather than skipping when TEST_PGVECTOR_URL was
-        # set explicitly (which this job always does), so a service that's
-        # actually unreachable fails this step instead of the suite quietly
-        # reporting green having run nothing.
-        run: |
-          pytest tests/vector_store/test_pgvector_store.py -v -rs

semantica/vector_store/qdrant_store.py

@@ -590,20 +590,77 @@ class QdrantStore:
                 with_payload=True,
                 with_vectors=True,
             )
-            results = []
-            for rec in records:
-                results.append(
-                    {
-                        "id": str(rec.id),
-                        "metadata": rec.payload or {},
-                        "vector": np.array(rec.vector) if rec.vector is not None else None,
-                    }
-                )
-            return results
+            return [self._record_to_result(rec) for rec in records]
         except Exception as e:
             self.logger.warning(f"Failed to scroll Qdrant points by metadata filter: {e}")
             return []
 
+    @staticmethod
+    def _record_to_result(rec: Any) -> Dict[str, Any]:
+        return {
+            "id": str(rec.id),
+            "metadata": rec.payload or {},
+            "vector": np.array(rec.vector) if rec.vector is not None else None,
+        }
+
+    def iter_all(self, batch_size: int = 500):
+        """
+        Iterate over every stored point using Qdrant's scroll cursor.
+
+        Paginates by point-ID cursor rather than row offset, which is why this
+        exists instead of scan_vectors(offset, limit). An integer offset is a
+        point ID, not a rank.
+
+        Assumes a single unnamed vector per point, as insert_vectors() and
+        get_vector() already do. Named and multi-vector collections are not
+        handled.
+
+        Args:
+            batch_size: Points to request per scroll call
+
+        Yields:
+            Result dicts with 'id', 'metadata', and 'vector', in scroll order
+
+        Raises:
+            ProcessingError: If the collection or client is not initialized, or
+                if the cursor stops advancing before the scan completes.
+        """
+        if self.collection is None or self.client is None or not QDRANT_AVAILABLE:
+            raise ProcessingError(
+                "Collection not initialized. Call create_collection() or get_collection() first."
+            )
+
+        next_offset = None
+        last_offset = object()
+        while True:
+            records, next_offset = self.client.scroll(
+                collection_name=self.collection.collection_name,
+                limit=batch_size,
+                offset=next_offset,
+                with_payload=True,
+                with_vectors=True,
+            )
+
+            for rec in records:
+                yield self._record_to_result(rec)
+
+            # A final page can carry records alongside a null cursor, so they
+            # are yielded above before stopping. Passing offset=None back to
+            # scroll() would restart from the beginning, not continue.
+            if next_offset is None:
+                return
+
+            # An empty page with a live cursor isn't necessarily truncation —
+            # a batch window that lands entirely on deleted points comes back
+            # this way too, and there's more to scan past it. Only treat it as
+            # stuck if the cursor itself stops moving.
+            if not records and next_offset == last_offset:
+                raise ProcessingError(
+                    "Qdrant scroll cursor stopped advancing without reaching "
+                    "the end of the collection, so the scan cannot complete."
+                )
+            last_offset = next_offset
+
     def delete_vectors(
         self, point_ids: List[Union[str, int]], **options
     ) -> Dict[str, Any]:

semantica/vector_store/vector_store.py

@@ -867,12 +867,22 @@ class VectorStore:
         """
         Iterate over every stored vector, one page at a time.
 
+        Cursor-based backends expose iter_all() because they cannot support a
+        positional offset; it takes precedence when present. Everything else
+        falls through to the scan_vectors() offset loop.
+
         Args:
-            batch_size: Number of vectors to fetch per underlying scan_vectors() call
+            batch_size: Number of vectors to fetch per underlying call
 
         Yields:
             Result dicts with 'id', 'metadata', and 'vector', in scan order
         """
+        if self.backend != "inmemory" and self._backend_store is not None:
+            iter_all = getattr(self._backend_store, "iter_all", None)
+            if callable(iter_all):
+                yield from iter_all(batch_size=batch_size)
+                return
+
         offset = 0
         while True:
             page = self.scan_vectors(offset=offset, limit=batch_size)

tests/vector_store/test_pgvector_store.py

@@ -10,7 +10,7 @@ To run these tests locally with Docker:
         -e POSTGRES_PASSWORD=postgres \
         -e POSTGRES_DB=test \
         -p 5432:5432 \
-        pgvector/pgvector:pg16
+        ankane/pgvector:latest
 
     pytest tests/vector_store/test_pgvector_store.py -v
 
@@ -63,37 +63,29 @@ TEST_CONNECTION_STRING = os.getenv(
 
 @pytest.fixture(scope="module")
 def pg_available() -> bool:
-    """Check if PostgreSQL with pgvector is available.
-
-    A connection failure only means "skip" when TEST_PGVECTOR_URL wasn't set
-    explicitly, i.e. this is a local run falling back to the documented
-    default. CI sets it on purpose, so a failure there means the service is
-    genuinely broken and the suite should fail loudly instead of skipping.
-    """
+    """Check if PostgreSQL with pgvector is available."""
     if not psycopg_available:
         return False
 
-    explicit_url = "TEST_PGVECTOR_URL" in os.environ
-
     try:
-        try:
-            import psycopg
+        if psycopg_available:
+            try:
+                import psycopg
 
-            conn = psycopg.connect(TEST_CONNECTION_STRING, connect_timeout=5)
-        except ImportError:
-            import psycopg2
+                conn = psycopg.connect(TEST_CONNECTION_STRING, connect_timeout=5)
+            except ImportError:
+                import psycopg2
 
-            conn = psycopg2.connect(TEST_CONNECTION_STRING, connect_timeout=5)
+                conn = psycopg2.connect(TEST_CONNECTION_STRING, connect_timeout=5)
 
-        cur = conn.cursor()
-        cur.execute("SELECT 1")
-        cur.close()
-        conn.close()
-        return True
+            cur = conn.cursor()
+            cur.execute("SELECT 1")
+            cur.close()
+            conn.close()
+            return True
     except Exception:
-        if explicit_url:
-            raise
         return False
+    return False
 
 
 @pytest.fixture
@@ -199,13 +191,7 @@ class TestPgVectorStoreAdd:
         ids = store.add(vectors, metadata)
 
         assert len(ids) == 5
-        assert len(set(ids)) == 5
-        # add() assigns uuid4 identifiers, not a "vec_" prefix
-        for vector_id in ids:
-            try:
-                uuid.UUID(vector_id)
-            except ValueError:
-                pytest.fail(f"{vector_id!r} is not a valid uuid4 id")
+        assert all(id.startswith("vec_") for id in ids)
 
     def test_add_auto_generate_ids(self, store):
         """Test that IDs are auto-generated if not provided."""
@@ -304,37 +290,38 @@ class TestPgVectorStoreSearch:
         if not pg_available:
             pytest.skip("PostgreSQL not available")
 
-        from semantica.vector_store.pgvector_store import PgVectorStore, psycopg_sql
+        from semantica.vector_store.pgvector_store import PgVectorStore
 
-        # setup_vectors is autouse and seeds unique_table_name, and fixtures are
-        # cached per test, so this needs a table of its own to be empty at all.
-        empty_table = f"{unique_table_name}_empty"
         empty_store = PgVectorStore(
             connection_string=TEST_CONNECTION_STRING,
-            table_name=empty_table,
+            table_name=unique_table_name,
             dimension=128,
             distance_metric="cosine",
         )
 
-        try:
-            query = np.random.rand(128).astype(np.float32)
-            results = empty_store.search(query, top_k=5)
+        query = np.random.rand(128).astype(np.float32)
+        results = empty_store.search(query, top_k=5)
 
-            assert len(results) == 0
-        finally:
-            try:
-                with empty_store._get_connection() as conn:
-                    cur = conn.cursor()
-                    cur.execute(
-                        psycopg_sql.SQL("DROP TABLE IF EXISTS {}").format(
-                            psycopg_sql.Identifier(empty_table)
-                        )
-                    )
-                    conn.commit()
-                    cur.close()
-                empty_store.close()
-            except Exception:
-                pass
+        assert len(results) == 0
+
+        # Cleanup: Drop test table after test completes
+        # Uses best-effort cleanup - failures are silently ignored since
+        # this is teardown of optional test resources
+        try:
+            with empty_store._get_connection() as conn:
+                cur = conn.cursor()
+                from semantica.vector_store.pgvector_store import psycopg_sql
+                drop_sql = psycopg_sql.SQL("DROP TABLE IF EXISTS {}").format(
+                    psycopg_sql.Identifier(unique_table_name)
+                )
+                cur.execute(drop_sql)
+                conn.commit()
+                cur.close()
+            empty_store.close()
+        except Exception:
+            # Best-effort cleanup: PostgreSQL may be unavailable during teardown
+            # This is expected when tests are skipped or connection is lost
+            pass
 
 
 class TestPgVectorStoreGet:

tests/vector_store/test_qdrant_store.py

@@ -0,0 +1,160 @@
+"""Tests for QdrantStore.iter_all() cursor enumeration.
+
+Qdrant is not installed in this environment, so these drive the real
+QdrantStore against a MagicMock standing in for the qdrant_client, following
+the pattern already used for qdrant in test_backend_metadata_filtering.py.
+"""
+
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+import pytest
+
+from semantica.utils.exceptions import ProcessingError
+from semantica.vector_store.qdrant_store import QdrantStore
+
+
+def _record(point_id, payload=None, vector=None):
+    """Build a stand-in for a qdrant_client Record."""
+    rec = MagicMock()
+    rec.id = point_id
+    rec.payload = payload
+    rec.vector = vector
+    return rec
+
+
+def _store_with_scroll(*pages):
+    """QdrantStore whose client.scroll() returns the given (records, cursor) pages."""
+    store = QdrantStore()
+    store.client = MagicMock()
+    store.client.scroll.side_effect = list(pages)
+    store.collection = MagicMock()
+    store.collection.collection_name = "test_collection"
+    return store
+
+
+@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
+def test_iter_all_threads_cursor_across_pages():
+    """The next call continues from the previous page's cursor."""
+    store = _store_with_scroll(
+        ([_record(1), _record(2)], "cursor-1"),
+        ([_record(3)], None),
+    )
+
+    result = list(store.iter_all(batch_size=2))
+
+    assert [item["id"] for item in result] == ["1", "2", "3"]
+    calls = store.client.scroll.call_args_list
+    assert len(calls) == 2
+    assert calls[0][1]["offset"] is None
+    assert calls[0][1]["limit"] == 2
+    assert calls[1][1]["offset"] == "cursor-1"
+
+
+@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
+def test_iter_all_yields_final_page_that_reports_no_next_cursor():
+    """Records and a null cursor can arrive together; those records must still
+    be yielded or every scan loses its tail."""
+    store = _store_with_scroll(([_record(1), _record(2)], None))
+
+    result = list(store.iter_all(batch_size=10))
+
+    assert [item["id"] for item in result] == ["1", "2"]
+    assert store.client.scroll.call_count == 1
+
+
+@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
+def test_iter_all_converts_records_to_the_shared_result_shape():
+    store = _store_with_scroll(
+        ([_record(7, payload={"tag": "x"}, vector=[0.1, 0.2, 0.3])], None),
+    )
+
+    item = list(store.iter_all())[0]
+
+    assert item["id"] == "7"
+    assert item["metadata"] == {"tag": "x"}
+    np.testing.assert_allclose(item["vector"], np.array([0.1, 0.2, 0.3]))
+
+
+@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
+def test_iter_all_handles_missing_payload_and_vector():
+    store = _store_with_scroll(([_record(1, payload=None, vector=None)], None))
+
+    item = list(store.iter_all())[0]
+
+    assert item["metadata"] == {}
+    assert item["vector"] is None
+
+
+@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
+def test_iter_all_empty_collection_yields_nothing():
+    store = _store_with_scroll(([], None))
+
+    assert list(store.iter_all()) == []
+
+
+@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
+def test_iter_all_continues_past_empty_page_with_advancing_cursor():
+    store = _store_with_scroll(
+        ([], "cursor-1"),
+        ([_record(1)], None),
+    )
+
+    result = list(store.iter_all())
+
+    assert [item["id"] for item in result] == ["1"]
+    assert store.client.scroll.call_count == 2
+
+
+@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
+def test_iter_all_raises_when_cursor_stops_advancing():
+    store = _store_with_scroll(
+        ([], "stuck-cursor"),
+        ([], "stuck-cursor"),
+    )
+
+    with pytest.raises(ProcessingError, match="stopped advancing"):
+        list(store.iter_all())
+
+
+@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
+def test_iter_all_raises_when_collection_not_initialized():
+    """Must fail loudly: an empty scan reads the same as an empty source."""
+    store = QdrantStore()
+
+    with pytest.raises(ProcessingError, match="Collection not initialized"):
+        list(store.iter_all())
+
+
+@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", False)
+def test_iter_all_raises_when_qdrant_unavailable():
+    store = QdrantStore()
+    store.client = MagicMock()
+    store.collection = MagicMock()
+
+    with pytest.raises(ProcessingError):
+        list(store.iter_all())
+
+
+@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
+def test_iter_all_propagates_scroll_errors():
+    store = QdrantStore()
+    store.client = MagicMock()
+    store.client.scroll.side_effect = RuntimeError("connection reset")
+    store.collection = MagicMock()
+    store.collection.collection_name = "test_collection"
+
+    with pytest.raises(RuntimeError, match="connection reset"):
+        list(store.iter_all())
+
+
+@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
+def test_iter_all_requests_payload_and_vectors():
+    store = _store_with_scroll(([], None))
+
+    list(store.iter_all())
+
+    kwargs = store.client.scroll.call_args[1]
+    assert kwargs["with_payload"] is True
+    assert kwargs["with_vectors"] is True
+    assert kwargs["collection_name"] == "test_collection"

tests/vector_store/test_vector_manager_persistent.py

@@ -26,6 +26,7 @@ from unittest.mock import MagicMock, patch
 
 import numpy as np
 
+from semantica.utils.exceptions import ProcessingError
 from semantica.vector_store.vector_store import VectorStore, VectorManager
 
 
@@ -138,6 +139,34 @@ class _NonScanningBackendStore:
     """Fake persistent backend store without any scan capability."""
 
 
+class _IterAllBackendStore:
+    """Fake cursor-based store: iter_all() only, no usable scan_vectors()."""
+
+    def __init__(self, items):
+        self._items = items
+        self.batch_sizes = []
+
+    def iter_all(self, batch_size=500):
+        self.batch_sizes.append(batch_size)
+        for item in self._items:
+            yield item
+
+    def scan_vectors(self, offset=0, limit=100):
+        raise AssertionError("scan_vectors() must not be called when iter_all() exists")
+
+
+class _MisShapedIterAllBackendStore:
+    """Backend store whose ``iter_all`` attribute is not callable."""
+
+    iter_all = 42  # plain attribute, not a method
+
+    def __init__(self, items):
+        self._items = items
+
+    def scan_vectors(self, offset=0, limit=100):
+        return self._items[offset:offset + limit]
+
+
 class VectorStoreScanVectorsTests(unittest.TestCase):
     """VectorStore.scan_vectors() / iter_vectors() backend-agnostic accessors."""
 
@@ -192,6 +221,72 @@ class VectorStoreScanVectorsTests(unittest.TestCase):
         self.assertEqual(list(store.iter_vectors(batch_size=2)), [])
 
 
+# ---------------------------------------------------------------------------
+# VectorStore.iter_vectors() preference for a native iter_all()
+# ---------------------------------------------------------------------------
+
+class VectorStoreIterAllDispatchTests(unittest.TestCase):
+    """iter_vectors() prefers a backend's native iter_all() when present."""
+
+    def _persistent_store(self, backend_store, backend_name="qdrant"):
+        store = VectorStore(backend="inmemory", dimension=2)
+        store.backend = backend_name
+        store._backend_store = backend_store
+        return store
+
+    def test_iter_vectors_uses_iter_all_when_available(self):
+        items = [
+            {"id": "a", "vector": None, "metadata": {"n": 1}},
+            {"id": "b", "vector": None, "metadata": {"n": 2}},
+        ]
+        backend = _IterAllBackendStore(items)
+        store = self._persistent_store(backend)
+
+        self.assertEqual(list(store.iter_vectors(batch_size=7)), items)
+
+    def test_iter_vectors_forwards_batch_size_to_iter_all(self):
+        backend = _IterAllBackendStore([])
+        store = self._persistent_store(backend)
+
+        list(store.iter_vectors(batch_size=32))
+
+        self.assertEqual(backend.batch_sizes, [32])
+
+    def test_iter_vectors_falls_back_to_scan_vectors_without_iter_all(self):
+        items = [{"id": "a", "vector": None, "metadata": {}}]
+        store = self._persistent_store(_ScanningBackendStore(items))
+
+        self.assertEqual(list(store.iter_vectors(batch_size=2)), items)
+
+    def test_iter_vectors_falls_back_when_iter_all_not_callable(self):
+        # Mirrors the count() precedent in _MisShapedBackendStore.
+        items = [{"id": "a", "vector": None, "metadata": {}}]
+        store = self._persistent_store(_MisShapedIterAllBackendStore(items))
+
+        self.assertEqual(list(store.iter_vectors(batch_size=2)), items)
+
+    def test_iter_vectors_inmemory_ignores_iter_all(self):
+        store = VectorStore(backend="inmemory", dimension=2)
+        store.store_vectors([np.array([1.0, 0.0])], [{"type": "a"}])
+        store._backend_store = _IterAllBackendStore([{"id": "wrong"}])
+
+        collected = list(store.iter_vectors(batch_size=2))
+
+        self.assertEqual([item["metadata"] for item in collected], [{"type": "a"}])
+
+    def test_iter_vectors_propagates_iter_all_errors(self):
+        # Silently yielding nothing would read as an empty source (#1083).
+        class _FailingIterAll:
+            def iter_all(self, batch_size=500):
+                raise ProcessingError("backend unreachable")
+                yield  # pragma: no cover - makes this a generator
+
+        store = self._persistent_store(_FailingIterAll())
+
+        with self.assertRaises(ProcessingError):
+            list(store.iter_vectors(batch_size=2))
+
+
 # ---------------------------------------------------------------------------
 # VectorManager tests — inmemory backend
 # ---------------------------------------------------------------------------