diff --git a/semantica/vector_store/pinecone_store.py b/semantica/vector_store/pinecone_store.py index 8e1f2041..c9ff2475 100644 --- a/semantica/vector_store/pinecone_store.py +++ b/semantica/vector_store/pinecone_store.py @@ -848,10 +848,18 @@ class PineconeStore: } next_token = _pinecone_next_token(response) - # A token that repeats means the listing is not advancing; stop - # rather than re-reading the same page forever. - if not next_token or next_token == token: + if not next_token: return + if next_token == token: + # Distinct from exhaustion above: the listing is not advancing. + # Returning here would yield a partial scan that a caller cannot + # tell apart from a complete one, and `store migrate` would flush + # it and report success having copied only part of the index. + raise ProcessingError( + "Pinecone returned the same pagination token twice, so the " + "listing is not advancing. Refusing to return a truncated " + "scan." + ) token = next_token def fetch_vectors( diff --git a/tests/vector_store/test_pinecone_store.py b/tests/vector_store/test_pinecone_store.py index 45dbf470..b5bcfcd0 100644 --- a/tests/vector_store/test_pinecone_store.py +++ b/tests/vector_store/test_pinecone_store.py @@ -327,8 +327,14 @@ class TestPineconeIterAll(unittest.TestCase): self.assertEqual([item["id"] for item in result], ["a"]) @patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True) - def test_stops_when_pagination_token_repeats(self): - """A token that stops advancing must not loop forever.""" + def test_raises_when_pagination_token_repeats(self): + """A token that stops advancing must not loop forever, and must not + quietly return a partial scan either. + + Truncating silently is indistinguishable from a complete enumeration, + which would let store migrate copy part of an index and report success + (issue #1083). + """ store, _, raw_index = self._store( [self._page(["a"], "same"), self._page(["b"], "same")], [ @@ -337,10 +343,10 @@ class TestPineconeIterAll(unittest.TestCase): ], ) - result = list(store.iter_all()) + with self.assertRaises(ProcessingError): + list(store.iter_all()) self.assertEqual(raw_index.list_paginated.call_count, 2) - self.assertEqual(len(result), 2) @patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True) def test_empty_listing_yields_nothing_without_fetching(self):