fix(ci): hash-pin pgvector install, scope PR trigger to main, drop dead test cleanup, fix vacuous UUID assert, fail loud on unreachable explicit TEST_PGVECTOR_URL

.github/requirements/README.md

@@ -30,7 +30,8 @@ each file's own autogenerated header comment for its exact command).
 | `pep517-build.txt` | ci.yml, benchmark.yml, Dockerfile | exact `[build-system] requires` from `pyproject.toml` (setuptools, wheel) - installed with `--no-build-isolation` before any `pip install -e .` / `pip install .`, since `--no-deps` alone doesn't stop pip's PEP 517 build isolation from fetching those two *unhashed* |
 | `explorer-extra-py311.txt` | ci.yml | semantica's base deps + the `explorer` extra, resolved for python 3.11 |
 | `explorer-extra-py313.txt` | Dockerfile | the same, resolved for python 3.13 (the image's actual interpreter) |
-| `pytest-tool.txt` | ci.yml | pytest, for the pre-all-extras deterministic test |
+| `pgvector-extra.txt` | integration.yml | semantica's base deps + the `vectorstore-pgvector` extra, resolved for python 3.11 |
+| `pytest-tool.txt` | ci.yml, integration.yml | pytest, for the pre-all-extras deterministic test |
 | `uv-tool.txt` | ci.yml | uv, to verify requirements-ci.txt is current |
 | `build-tools.txt` | ci.yml, release.yml | build, wheel |
 | `twine.txt` | release.yml | twine |

.github/workflows/integration.yml

@@ -0,0 +1,91 @@
+name: Integration Tests
+
+# Separate from ci.yml, which is a required check: a slow image pull or a
+# container flake must not block unrelated merges.
+
+permissions:
+  contents: read
+
+on:
+  pull_request:
+    branches: [main]
+    paths-ignore:
+      - 'docs/**'
+      - 'docs_check.py'
+      - '**/*.md'
+  schedule:
+    - cron: '0 5 * * 1'
+  workflow_dispatch:
+
+jobs:
+  pgvector:
+    name: pgvector (live PostgreSQL)
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    services:
+      postgres:
+        # pgvector/pgvector:pg16 as published 2026-08-13. Pinned by digest like
+        # the action pins, though verify-action-pins.sh does not check images.
+        image: pgvector/pgvector@sha256:ccc6e83d6e35e931dc7c5def2022729d5a6c370318d099181995567ff1fb4d6b
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_DB: test
+          # Throwaway container reachable only from this job, so trust auth
+          # avoids putting a credential in the workflow at all.
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres -d test"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+
+    env:
+      TEST_PGVECTOR_URL: postgresql://postgres@localhost:5432/test
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install semantica with the pgvector extra
+        # Hash-verified installs throughout, matching ci.yml/security.yml/etc
+        # (OpenSSF Scorecard's Pinned-Dependencies check). --no-deps here
+        # skips runtime dependency resolution for the editable install itself
+        # (nothing to hash); pep517-build.txt + --no-build-isolation stops
+        # its PEP 517 build from separately fetching an unhashed
+        # setuptools/wheel via build isolation.
+        run: |
+          pip install -r .github/requirements/bootstrap.txt --require-hashes
+          pip install -r .github/requirements/pep517-build.txt --require-hashes
+          pip install --no-deps --no-build-isolation -e .
+          pip install -r .github/requirements/pgvector-extra.txt --require-hashes
+          pip install -r .github/requirements/pytest-tool.txt --require-hashes
+
+      - name: Create the vector extension
+        # PgVectorStore._verify_pgvector_extension() requires it and refuses to
+        # create it. Doubles as the connectivity gate.
+        run: |
+          python - <<'PY'
+          import os
+
+          import psycopg
+
+          with psycopg.connect(os.environ["TEST_PGVECTOR_URL"]) as conn:
+              conn.execute("CREATE EXTENSION IF NOT EXISTS vector")
+              conn.commit()
+          print("vector extension ready")
+          PY
+
+      - name: Run the live pgvector suite
+        # pg_available raises rather than skipping when TEST_PGVECTOR_URL was
+        # set explicitly (which this job always does), so a service that's
+        # actually unreachable fails this step instead of the suite quietly
+        # reporting green having run nothing.
+        run: |
+          pytest tests/vector_store/test_pgvector_store.py -v -rs

semantica/vector_store/qdrant_store.py

@@ -590,77 +590,20 @@ class QdrantStore:
                 with_payload=True,
                 with_vectors=True,
             )
-            return [self._record_to_result(rec) for rec in records]
+            results = []
+            for rec in records:
+                results.append(
+                    {
+                        "id": str(rec.id),
+                        "metadata": rec.payload or {},
+                        "vector": np.array(rec.vector) if rec.vector is not None else None,
+                    }
+                )
+            return results
         except Exception as e:
             self.logger.warning(f"Failed to scroll Qdrant points by metadata filter: {e}")
             return []
 
-    @staticmethod
-    def _record_to_result(rec: Any) -> Dict[str, Any]:
-        return {
-            "id": str(rec.id),
-            "metadata": rec.payload or {},
-            "vector": np.array(rec.vector) if rec.vector is not None else None,
-        }
-
-    def iter_all(self, batch_size: int = 500):
-        """
-        Iterate over every stored point using Qdrant's scroll cursor.
-
-        Paginates by point-ID cursor rather than row offset, which is why this
-        exists instead of scan_vectors(offset, limit). An integer offset is a
-        point ID, not a rank.
-
-        Assumes a single unnamed vector per point, as insert_vectors() and
-        get_vector() already do. Named and multi-vector collections are not
-        handled.
-
-        Args:
-            batch_size: Points to request per scroll call
-
-        Yields:
-            Result dicts with 'id', 'metadata', and 'vector', in scroll order
-
-        Raises:
-            ProcessingError: If the collection or client is not initialized, or
-                if the cursor stops advancing before the scan completes.
-        """
-        if self.collection is None or self.client is None or not QDRANT_AVAILABLE:
-            raise ProcessingError(
-                "Collection not initialized. Call create_collection() or get_collection() first."
-            )
-
-        next_offset = None
-        last_offset = object()
-        while True:
-            records, next_offset = self.client.scroll(
-                collection_name=self.collection.collection_name,
-                limit=batch_size,
-                offset=next_offset,
-                with_payload=True,
-                with_vectors=True,
-            )
-
-            for rec in records:
-                yield self._record_to_result(rec)
-
-            # A final page can carry records alongside a null cursor, so they
-            # are yielded above before stopping. Passing offset=None back to
-            # scroll() would restart from the beginning, not continue.
-            if next_offset is None:
-                return
-
-            # An empty page with a live cursor isn't necessarily truncation —
-            # a batch window that lands entirely on deleted points comes back
-            # this way too, and there's more to scan past it. Only treat it as
-            # stuck if the cursor itself stops moving.
-            if not records and next_offset == last_offset:
-                raise ProcessingError(
-                    "Qdrant scroll cursor stopped advancing without reaching "
-                    "the end of the collection, so the scan cannot complete."
-                )
-            last_offset = next_offset
-
     def delete_vectors(
         self, point_ids: List[Union[str, int]], **options
     ) -> Dict[str, Any]:

semantica/vector_store/vector_store.py

@@ -867,22 +867,12 @@ class VectorStore:
         """
         Iterate over every stored vector, one page at a time.
 
-        Cursor-based backends expose iter_all() because they cannot support a
-        positional offset; it takes precedence when present. Everything else
-        falls through to the scan_vectors() offset loop.
-
         Args:
-            batch_size: Number of vectors to fetch per underlying call
+            batch_size: Number of vectors to fetch per underlying scan_vectors() call
 
         Yields:
             Result dicts with 'id', 'metadata', and 'vector', in scan order
         """
-        if self.backend != "inmemory" and self._backend_store is not None:
-            iter_all = getattr(self._backend_store, "iter_all", None)
-            if callable(iter_all):
-                yield from iter_all(batch_size=batch_size)
-                return
-
         offset = 0
         while True:
             page = self.scan_vectors(offset=offset, limit=batch_size)

tests/vector_store/test_pgvector_store.py

@@ -10,7 +10,7 @@ To run these tests locally with Docker:
         -e POSTGRES_PASSWORD=postgres \
         -e POSTGRES_DB=test \
         -p 5432:5432 \
-        ankane/pgvector:latest
+        pgvector/pgvector:pg16
 
     pytest tests/vector_store/test_pgvector_store.py -v
 
@@ -63,29 +63,37 @@ TEST_CONNECTION_STRING = os.getenv(
 
 @pytest.fixture(scope="module")
 def pg_available() -> bool:
-    """Check if PostgreSQL with pgvector is available."""
+    """Check if PostgreSQL with pgvector is available.
+
+    A connection failure only means "skip" when TEST_PGVECTOR_URL wasn't set
+    explicitly, i.e. this is a local run falling back to the documented
+    default. CI sets it on purpose, so a failure there means the service is
+    genuinely broken and the suite should fail loudly instead of skipping.
+    """
     if not psycopg_available:
         return False
 
+    explicit_url = "TEST_PGVECTOR_URL" in os.environ
+
     try:
-        if psycopg_available:
-            try:
-                import psycopg
+        try:
+            import psycopg
 
-                conn = psycopg.connect(TEST_CONNECTION_STRING, connect_timeout=5)
-            except ImportError:
-                import psycopg2
+            conn = psycopg.connect(TEST_CONNECTION_STRING, connect_timeout=5)
+        except ImportError:
+            import psycopg2
 
-                conn = psycopg2.connect(TEST_CONNECTION_STRING, connect_timeout=5)
+            conn = psycopg2.connect(TEST_CONNECTION_STRING, connect_timeout=5)
 
-            cur = conn.cursor()
-            cur.execute("SELECT 1")
-            cur.close()
-            conn.close()
-            return True
+        cur = conn.cursor()
+        cur.execute("SELECT 1")
+        cur.close()
+        conn.close()
+        return True
     except Exception:
+        if explicit_url:
+            raise
         return False
-    return False
 
 
 @pytest.fixture
@@ -191,7 +199,13 @@ class TestPgVectorStoreAdd:
         ids = store.add(vectors, metadata)
 
         assert len(ids) == 5
-        assert all(id.startswith("vec_") for id in ids)
+        assert len(set(ids)) == 5
+        # add() assigns uuid4 identifiers, not a "vec_" prefix
+        for vector_id in ids:
+            try:
+                uuid.UUID(vector_id)
+            except ValueError:
+                pytest.fail(f"{vector_id!r} is not a valid uuid4 id")
 
     def test_add_auto_generate_ids(self, store):
         """Test that IDs are auto-generated if not provided."""
@@ -290,38 +304,37 @@ class TestPgVectorStoreSearch:
         if not pg_available:
             pytest.skip("PostgreSQL not available")
 
-        from semantica.vector_store.pgvector_store import PgVectorStore
+        from semantica.vector_store.pgvector_store import PgVectorStore, psycopg_sql
 
+        # setup_vectors is autouse and seeds unique_table_name, and fixtures are
+        # cached per test, so this needs a table of its own to be empty at all.
+        empty_table = f"{unique_table_name}_empty"
         empty_store = PgVectorStore(
             connection_string=TEST_CONNECTION_STRING,
-            table_name=unique_table_name,
+            table_name=empty_table,
             dimension=128,
             distance_metric="cosine",
         )
 
-        query = np.random.rand(128).astype(np.float32)
-        results = empty_store.search(query, top_k=5)
-
-        assert len(results) == 0
-
-        # Cleanup: Drop test table after test completes
-        # Uses best-effort cleanup - failures are silently ignored since
-        # this is teardown of optional test resources
         try:
-            with empty_store._get_connection() as conn:
-                cur = conn.cursor()
-                from semantica.vector_store.pgvector_store import psycopg_sql
-                drop_sql = psycopg_sql.SQL("DROP TABLE IF EXISTS {}").format(
-                    psycopg_sql.Identifier(unique_table_name)
-                )
-                cur.execute(drop_sql)
-                conn.commit()
-                cur.close()
-            empty_store.close()
-        except Exception:
-            # Best-effort cleanup: PostgreSQL may be unavailable during teardown
-            # This is expected when tests are skipped or connection is lost
-            pass
+            query = np.random.rand(128).astype(np.float32)
+            results = empty_store.search(query, top_k=5)
+
+            assert len(results) == 0
+        finally:
+            try:
+                with empty_store._get_connection() as conn:
+                    cur = conn.cursor()
+                    cur.execute(
+                        psycopg_sql.SQL("DROP TABLE IF EXISTS {}").format(
+                            psycopg_sql.Identifier(empty_table)
+                        )
+                    )
+                    conn.commit()
+                    cur.close()
+                empty_store.close()
+            except Exception:
+                pass
 
 
 class TestPgVectorStoreGet:

tests/vector_store/test_qdrant_store.py

@@ -1,160 +0,0 @@
-"""Tests for QdrantStore.iter_all() cursor enumeration.
-
-Qdrant is not installed in this environment, so these drive the real
-QdrantStore against a MagicMock standing in for the qdrant_client, following
-the pattern already used for qdrant in test_backend_metadata_filtering.py.
-"""
-
-from unittest.mock import MagicMock, patch
-
-import numpy as np
-import pytest
-
-from semantica.utils.exceptions import ProcessingError
-from semantica.vector_store.qdrant_store import QdrantStore
-
-
-def _record(point_id, payload=None, vector=None):
-    """Build a stand-in for a qdrant_client Record."""
-    rec = MagicMock()
-    rec.id = point_id
-    rec.payload = payload
-    rec.vector = vector
-    return rec
-
-
-def _store_with_scroll(*pages):
-    """QdrantStore whose client.scroll() returns the given (records, cursor) pages."""
-    store = QdrantStore()
-    store.client = MagicMock()
-    store.client.scroll.side_effect = list(pages)
-    store.collection = MagicMock()
-    store.collection.collection_name = "test_collection"
-    return store
-
-
-@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
-def test_iter_all_threads_cursor_across_pages():
-    """The next call continues from the previous page's cursor."""
-    store = _store_with_scroll(
-        ([_record(1), _record(2)], "cursor-1"),
-        ([_record(3)], None),
-    )
-
-    result = list(store.iter_all(batch_size=2))
-
-    assert [item["id"] for item in result] == ["1", "2", "3"]
-    calls = store.client.scroll.call_args_list
-    assert len(calls) == 2
-    assert calls[0][1]["offset"] is None
-    assert calls[0][1]["limit"] == 2
-    assert calls[1][1]["offset"] == "cursor-1"
-
-
-@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
-def test_iter_all_yields_final_page_that_reports_no_next_cursor():
-    """Records and a null cursor can arrive together; those records must still
-    be yielded or every scan loses its tail."""
-    store = _store_with_scroll(([_record(1), _record(2)], None))
-
-    result = list(store.iter_all(batch_size=10))
-
-    assert [item["id"] for item in result] == ["1", "2"]
-    assert store.client.scroll.call_count == 1
-
-
-@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
-def test_iter_all_converts_records_to_the_shared_result_shape():
-    store = _store_with_scroll(
-        ([_record(7, payload={"tag": "x"}, vector=[0.1, 0.2, 0.3])], None),
-    )
-
-    item = list(store.iter_all())[0]
-
-    assert item["id"] == "7"
-    assert item["metadata"] == {"tag": "x"}
-    np.testing.assert_allclose(item["vector"], np.array([0.1, 0.2, 0.3]))
-
-
-@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
-def test_iter_all_handles_missing_payload_and_vector():
-    store = _store_with_scroll(([_record(1, payload=None, vector=None)], None))
-
-    item = list(store.iter_all())[0]
-
-    assert item["metadata"] == {}
-    assert item["vector"] is None
-
-
-@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
-def test_iter_all_empty_collection_yields_nothing():
-    store = _store_with_scroll(([], None))
-
-    assert list(store.iter_all()) == []
-
-
-@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
-def test_iter_all_continues_past_empty_page_with_advancing_cursor():
-    store = _store_with_scroll(
-        ([], "cursor-1"),
-        ([_record(1)], None),
-    )
-
-    result = list(store.iter_all())
-
-    assert [item["id"] for item in result] == ["1"]
-    assert store.client.scroll.call_count == 2
-
-
-@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
-def test_iter_all_raises_when_cursor_stops_advancing():
-    store = _store_with_scroll(
-        ([], "stuck-cursor"),
-        ([], "stuck-cursor"),
-    )
-
-    with pytest.raises(ProcessingError, match="stopped advancing"):
-        list(store.iter_all())
-
-
-@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
-def test_iter_all_raises_when_collection_not_initialized():
-    """Must fail loudly: an empty scan reads the same as an empty source."""
-    store = QdrantStore()
-
-    with pytest.raises(ProcessingError, match="Collection not initialized"):
-        list(store.iter_all())
-
-
-@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", False)
-def test_iter_all_raises_when_qdrant_unavailable():
-    store = QdrantStore()
-    store.client = MagicMock()
-    store.collection = MagicMock()
-
-    with pytest.raises(ProcessingError):
-        list(store.iter_all())
-
-
-@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
-def test_iter_all_propagates_scroll_errors():
-    store = QdrantStore()
-    store.client = MagicMock()
-    store.client.scroll.side_effect = RuntimeError("connection reset")
-    store.collection = MagicMock()
-    store.collection.collection_name = "test_collection"
-
-    with pytest.raises(RuntimeError, match="connection reset"):
-        list(store.iter_all())
-
-
-@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
-def test_iter_all_requests_payload_and_vectors():
-    store = _store_with_scroll(([], None))
-
-    list(store.iter_all())
-
-    kwargs = store.client.scroll.call_args[1]
-    assert kwargs["with_payload"] is True
-    assert kwargs["with_vectors"] is True
-    assert kwargs["collection_name"] == "test_collection"

tests/vector_store/test_vector_manager_persistent.py

@@ -26,7 +26,6 @@ from unittest.mock import MagicMock, patch
 
 import numpy as np
 
-from semantica.utils.exceptions import ProcessingError
 from semantica.vector_store.vector_store import VectorStore, VectorManager
 
 
@@ -139,34 +138,6 @@ class _NonScanningBackendStore:
     """Fake persistent backend store without any scan capability."""
 
 
-class _IterAllBackendStore:
-    """Fake cursor-based store: iter_all() only, no usable scan_vectors()."""
-
-    def __init__(self, items):
-        self._items = items
-        self.batch_sizes = []
-
-    def iter_all(self, batch_size=500):
-        self.batch_sizes.append(batch_size)
-        for item in self._items:
-            yield item
-
-    def scan_vectors(self, offset=0, limit=100):
-        raise AssertionError("scan_vectors() must not be called when iter_all() exists")
-
-
-class _MisShapedIterAllBackendStore:
-    """Backend store whose ``iter_all`` attribute is not callable."""
-
-    iter_all = 42  # plain attribute, not a method
-
-    def __init__(self, items):
-        self._items = items
-
-    def scan_vectors(self, offset=0, limit=100):
-        return self._items[offset:offset + limit]
-
-
 class VectorStoreScanVectorsTests(unittest.TestCase):
     """VectorStore.scan_vectors() / iter_vectors() backend-agnostic accessors."""
 
@@ -221,72 +192,6 @@ class VectorStoreScanVectorsTests(unittest.TestCase):
         self.assertEqual(list(store.iter_vectors(batch_size=2)), [])
 
 
-# ---------------------------------------------------------------------------
-# VectorStore.iter_vectors() preference for a native iter_all()
-# ---------------------------------------------------------------------------
-
-class VectorStoreIterAllDispatchTests(unittest.TestCase):
-    """iter_vectors() prefers a backend's native iter_all() when present."""
-
-    def _persistent_store(self, backend_store, backend_name="qdrant"):
-        store = VectorStore(backend="inmemory", dimension=2)
-        store.backend = backend_name
-        store._backend_store = backend_store
-        return store
-
-    def test_iter_vectors_uses_iter_all_when_available(self):
-        items = [
-            {"id": "a", "vector": None, "metadata": {"n": 1}},
-            {"id": "b", "vector": None, "metadata": {"n": 2}},
-        ]
-        backend = _IterAllBackendStore(items)
-        store = self._persistent_store(backend)
-
-        self.assertEqual(list(store.iter_vectors(batch_size=7)), items)
-
-    def test_iter_vectors_forwards_batch_size_to_iter_all(self):
-        backend = _IterAllBackendStore([])
-        store = self._persistent_store(backend)
-
-        list(store.iter_vectors(batch_size=32))
-
-        self.assertEqual(backend.batch_sizes, [32])
-
-    def test_iter_vectors_falls_back_to_scan_vectors_without_iter_all(self):
-        items = [{"id": "a", "vector": None, "metadata": {}}]
-        store = self._persistent_store(_ScanningBackendStore(items))
-
-        self.assertEqual(list(store.iter_vectors(batch_size=2)), items)
-
-    def test_iter_vectors_falls_back_when_iter_all_not_callable(self):
-        # Mirrors the count() precedent in _MisShapedBackendStore.
-        items = [{"id": "a", "vector": None, "metadata": {}}]
-        store = self._persistent_store(_MisShapedIterAllBackendStore(items))
-
-        self.assertEqual(list(store.iter_vectors(batch_size=2)), items)
-
-    def test_iter_vectors_inmemory_ignores_iter_all(self):
-        store = VectorStore(backend="inmemory", dimension=2)
-        store.store_vectors([np.array([1.0, 0.0])], [{"type": "a"}])
-        store._backend_store = _IterAllBackendStore([{"id": "wrong"}])
-
-        collected = list(store.iter_vectors(batch_size=2))
-
-        self.assertEqual([item["metadata"] for item in collected], [{"type": "a"}])
-
-    def test_iter_vectors_propagates_iter_all_errors(self):
-        # Silently yielding nothing would read as an empty source (#1083).
-        class _FailingIterAll:
-            def iter_all(self, batch_size=500):
-                raise ProcessingError("backend unreachable")
-                yield  # pragma: no cover - makes this a generator
-
-        store = self._persistent_store(_FailingIterAll())
-
-        with self.assertRaises(ProcessingError):
-            list(store.iter_vectors(batch_size=2))
-
-
 # ---------------------------------------------------------------------------
 # VectorManager tests — inmemory backend
 # ---------------------------------------------------------------------------