From 8e507eb8629e50d5aa6a8d8a64cebc079cbfa7a6 Mon Sep 17 00:00:00 2001
From: Zohaib Hassnain <109234410+ZohaibHassan16@users.noreply.github.com>
Date: Mon, 31 Aug 2026 13:17:13 +0500
Subject: [PATCH] fix(vector_store): raise on stalled pinecone pagination
 instead of truncating

---
 semantica/vector_store/pinecone_store.py  | 14 +++++++++++---
 tests/vector_store/test_pinecone_store.py | 14 ++++++++++----
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/semantica/vector_store/pinecone_store.py b/semantica/vector_store/pinecone_store.py
index 8e1f2041..c9ff2475 100644
--- a/semantica/vector_store/pinecone_store.py
+++ b/semantica/vector_store/pinecone_store.py
@@ -848,10 +848,18 @@ class PineconeStore:
                 }
 
             next_token = _pinecone_next_token(response)
-            # A token that repeats means the listing is not advancing; stop
-            # rather than re-reading the same page forever.
-            if not next_token or next_token == token:
+            if not next_token:
                 return
+            if next_token == token:
+                # Distinct from exhaustion above: the listing is not advancing.
+                # Returning here would yield a partial scan that a caller cannot
+                # tell apart from a complete one, and `store migrate` would flush
+                # it and report success having copied only part of the index.
+                raise ProcessingError(
+                    "Pinecone returned the same pagination token twice, so the "
+                    "listing is not advancing. Refusing to return a truncated "
+                    "scan."
+                )
             token = next_token
 
     def fetch_vectors(
diff --git a/tests/vector_store/test_pinecone_store.py b/tests/vector_store/test_pinecone_store.py
index 45dbf470..b5bcfcd0 100644
--- a/tests/vector_store/test_pinecone_store.py
+++ b/tests/vector_store/test_pinecone_store.py
@@ -327,8 +327,14 @@ class TestPineconeIterAll(unittest.TestCase):
         self.assertEqual([item["id"] for item in result], ["a"])
 
     @patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
-    def test_stops_when_pagination_token_repeats(self):
-        """A token that stops advancing must not loop forever."""
+    def test_raises_when_pagination_token_repeats(self):
+        """A token that stops advancing must not loop forever, and must not
+        quietly return a partial scan either.
+
+        Truncating silently is indistinguishable from a complete enumeration,
+        which would let store migrate copy part of an index and report success
+        (issue #1083).
+        """
         store, _, raw_index = self._store(
             [self._page(["a"], "same"), self._page(["b"], "same")],
             [
@@ -337,10 +343,10 @@ class TestPineconeIterAll(unittest.TestCase):
             ],
         )
 
-        result = list(store.iter_all())
+        with self.assertRaises(ProcessingError):
+            list(store.iter_all())
 
         self.assertEqual(raw_index.list_paginated.call_count, 2)
-        self.assertEqual(len(result), 2)
 
     @patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
     def test_empty_listing_yields_nothing_without_fetching(self):