From 4dd88375e107d821d86f115d5ffaf8ef71fc4c3a Mon Sep 17 00:00:00 2001
From: KaifAhmad1 <kaifahmad087@gmail.com>
Date: Wed, 2 Sep 2026 19:54:24 +0530
Subject: [PATCH] fix(vector_store): don't short-circuit pinecone iter_all() on
 an empty page

if not vector_ids: return fired before the continuation token was ever
checked. Pinecone's actual pagination contract is that a scan is only
exhausted when the response carries no pagination token -- a page can
legitimately list zero ids while pagination.next is still set (sparse
or filtered namespaces, eventual-consistency windows on serverless
indexes). This was flagged in review but the fix commit that followed
only addressed the separate repeated-token stall case, not this one.

Reproduced concretely against the unfixed code: a page with data,
followed by an empty page with a live token, followed by a page with
more data -- the last page was silently dropped with no error raised,
exactly the #1083 failure mode (store migrate reporting success after
copying only part of a collection).

Now the empty-page case skips the pointless fetch() call but still
falls through to the same next_token check every other path already
goes through, so a live token continues the scan and only a genuinely
absent token (or one that's stopped advancing) ends it.

Added test_continues_past_an_empty_page_with_a_live_token, the
"empty page + non-None next token" case the original review asked for
and that wasn't otherwise covered.
---
 semantica/vector_store/pinecone_store.py  | 37 ++++++++++++++---------
 tests/vector_store/test_pinecone_store.py | 25 +++++++++++++++
 2 files changed, 47 insertions(+), 15 deletions(-)

diff --git a/semantica/vector_store/pinecone_store.py b/semantica/vector_store/pinecone_store.py
index 131e3dd0..6df7ad85 100644
--- a/semantica/vector_store/pinecone_store.py
+++ b/semantica/vector_store/pinecone_store.py
@@ -820,23 +820,30 @@ class PineconeStore:
 
             response = list_paginated(**kwargs)
             vector_ids = _pinecone_listed_ids(response)
-            if not vector_ids:
-                return
 
-            fetched = self.index.fetch_vectors(vector_ids, namespace=namespace)
-            vectors = fetched.get("vectors") or {}
+            # A page listing zero ids is not necessarily exhaustion: Pinecone's
+            # contract is that a scan ends only when there's no pagination
+            # token, and a page can legitimately come back empty while
+            # pagination.next is still set (sparse/filtered namespaces,
+            # eventual-consistency windows on serverless indexes). Skip the
+            # fetch (nothing to hydrate) but still fall through to the token
+            # check below instead of returning early, or a gap like that
+            # silently truncates the scan with no error.
+            if vector_ids:
+                fetched = self.index.fetch_vectors(vector_ids, namespace=namespace)
+                vectors = fetched.get("vectors") or {}
 
-            for vector_id in vector_ids:
-                entry = vectors.get(vector_id)
-                if entry is None:
-                    # fetch() omits ids it cannot find: deleted since listing.
-                    continue
-                values = entry.get("values")
-                yield {
-                    "id": vector_id,
-                    "metadata": entry.get("metadata") or {},
-                    "vector": np.array(values) if values is not None else None,
-                }
+                for vector_id in vector_ids:
+                    entry = vectors.get(vector_id)
+                    if entry is None:
+                        # fetch() omits ids it cannot find: deleted since listing.
+                        continue
+                    values = entry.get("values")
+                    yield {
+                        "id": vector_id,
+                        "metadata": entry.get("metadata") or {},
+                        "vector": np.array(values) if values is not None else None,
+                    }
 
             next_token = _pinecone_next_token(response)
             if not next_token:
diff --git a/tests/vector_store/test_pinecone_store.py b/tests/vector_store/test_pinecone_store.py
index 04e990a7..86758a93 100644
--- a/tests/vector_store/test_pinecone_store.py
+++ b/tests/vector_store/test_pinecone_store.py
@@ -350,6 +350,31 @@ class TestPineconeIterAll(unittest.TestCase):
         self.assertEqual(list(store.iter_all()), [])
         wrapper.fetch_vectors.assert_not_called()
 
+    @patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
+    def test_continues_past_an_empty_page_with_a_live_token(self):
+        """An empty page is not necessarily the end: Pinecone can legitimately
+        list zero ids for a page while pagination.next is still set (sparse
+        or filtered namespaces, eventual-consistency windows on serverless
+        indexes). Only the absence of a next token means exhaustion."""
+        store, wrapper, raw_index = self._store(
+            [
+                self._page(["a"], "token-1"),
+                self._page([], "token-2"),  # empty page, but the token still advances
+                self._page(["b"], None),
+            ],
+            [
+                {"vectors": {"a": {"values": [0.1], "metadata": {}}}},
+                {"vectors": {"b": {"values": [0.2], "metadata": {}}}},
+            ],
+        )
+
+        result = list(store.iter_all(batch_size=1))
+
+        self.assertEqual([item["id"] for item in result], ["a", "b"])
+        self.assertEqual(raw_index.list_paginated.call_count, 3)
+        # Nothing to hydrate on the empty page, so only two fetches happen.
+        self.assertEqual(wrapper.fetch_vectors.call_count, 2)
+
     @patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
     def test_accepts_plain_string_ids_from_listing(self):
         """SDK generations differ on what listing yields."""