Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

fix(vector_store): don't short-circuit pinecone iter_all() on an empty page

KaifAhmad14dd88375e107
diff --git a/semantica/vector_store/pinecone_store.py b/semantica/vector_store/pinecone_store.py
index 131e3dd0..6df7ad85 100644
--- a/semantica/vector_store/pinecone_store.py
+++ b/semantica/vector_store/pinecone_store.py
@@ -820,23 +820,30 @@ class PineconeStore:
response = list_paginated(**kwargs)
vector_ids = _pinecone_listed_ids(response)
- if not vector_ids:
- return
- fetched = self.index.fetch_vectors(vector_ids, namespace=namespace)
- vectors = fetched.get("vectors") or {}
-
- for vector_id in vector_ids:
- entry = vectors.get(vector_id)
- if entry is None:
- # fetch() omits ids it cannot find: deleted since listing.
- continue
- values = entry.get("values")
- yield {
- "id": vector_id,
- "metadata": entry.get("metadata") or {},
- "vector": np.array(values) if values is not None else None,
- }
+ # A page listing zero ids is not necessarily exhaustion: Pinecone's
+ # contract is that a scan ends only when there's no pagination
+ # token, and a page can legitimately come back empty while
+ # pagination.next is still set (sparse/filtered namespaces,
+ # eventual-consistency windows on serverless indexes). Skip the
+ # fetch (nothing to hydrate) but still fall through to the token
+ # check below instead of returning early, or a gap like that
+ # silently truncates the scan with no error.
+ if vector_ids:
+ fetched = self.index.fetch_vectors(vector_ids, namespace=namespace)
+ vectors = fetched.get("vectors") or {}
+
+ for vector_id in vector_ids:
+ entry = vectors.get(vector_id)
+ if entry is None:
+ # fetch() omits ids it cannot find: deleted since listing.
+ continue
+ values = entry.get("values")
+ yield {
+ "id": vector_id,
+ "metadata": entry.get("metadata") or {},
+ "vector": np.array(values) if values is not None else None,
+ }
next_token = _pinecone_next_token(response)
if not next_token:
diff --git a/tests/vector_store/test_pinecone_store.py b/tests/vector_store/test_pinecone_store.py
index 04e990a7..86758a93 100644
--- a/tests/vector_store/test_pinecone_store.py
+++ b/tests/vector_store/test_pinecone_store.py
@@ -350,6 +350,31 @@ class TestPineconeIterAll(unittest.TestCase):
self.assertEqual(list(store.iter_all()), [])
wrapper.fetch_vectors.assert_not_called()
+ @patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
+ def test_continues_past_an_empty_page_with_a_live_token(self):
+ """An empty page is not necessarily the end: Pinecone can legitimately
+ list zero ids for a page while pagination.next is still set (sparse
+ or filtered namespaces, eventual-consistency windows on serverless
+ indexes). Only the absence of a next token means exhaustion."""
+ store, wrapper, raw_index = self._store(
+ [
+ self._page(["a"], "token-1"),
+ self._page([], "token-2"), # empty page, but the token still advances
+ self._page(["b"], None),
+ ],
+ [
+ {"vectors": {"a": {"values": [0.1], "metadata": {}}}},
+ {"vectors": {"b": {"values": [0.2], "metadata": {}}}},
+ ],
+ )
+
+ result = list(store.iter_all(batch_size=1))
+
+ self.assertEqual([item["id"] for item in result], ["a", "b"])
+ self.assertEqual(raw_index.list_paginated.call_count, 3)
+ # Nothing to hydrate on the empty page, so only two fetches happen.
+ self.assertEqual(wrapper.fetch_vectors.call_count, 2)
+
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
def test_accepts_plain_string_ids_from_listing(self):
"""SDK generations differ on what listing yields."""