Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix(vector_store): don't short-circuit pinecone iter_all() on an empty page
KaifAhmad14dd88375e107
diff --git a/semantica/vector_store/pinecone_store.py b/semantica/vector_store/pinecone_store.pyindex 131e3dd0..6df7ad85 100644--- a/semantica/vector_store/pinecone_store.py+++ b/semantica/vector_store/pinecone_store.py@@ -820,23 +820,30 @@ class PineconeStore:response = list_paginated(**kwargs)vector_ids = _pinecone_listed_ids(response)- if not vector_ids:- return- fetched = self.index.fetch_vectors(vector_ids, namespace=namespace)- vectors = fetched.get("vectors") or {}-- for vector_id in vector_ids:- entry = vectors.get(vector_id)- if entry is None:- # fetch() omits ids it cannot find: deleted since listing.- continue- values = entry.get("values")- yield {- "id": vector_id,- "metadata": entry.get("metadata") or {},- "vector": np.array(values) if values is not None else None,- }+ # A page listing zero ids is not necessarily exhaustion: Pinecone's+ # contract is that a scan ends only when there's no pagination+ # token, and a page can legitimately come back empty while+ # pagination.next is still set (sparse/filtered namespaces,+ # eventual-consistency windows on serverless indexes). Skip the+ # fetch (nothing to hydrate) but still fall through to the token+ # check below instead of returning early, or a gap like that+ # silently truncates the scan with no error.+ if vector_ids:+ fetched = self.index.fetch_vectors(vector_ids, namespace=namespace)+ vectors = fetched.get("vectors") or {}++ for vector_id in vector_ids:+ entry = vectors.get(vector_id)+ if entry is None:+ # fetch() omits ids it cannot find: deleted since listing.+ continue+ values = entry.get("values")+ yield {+ "id": vector_id,+ "metadata": entry.get("metadata") or {},+ "vector": np.array(values) if values is not None else None,+ }next_token = _pinecone_next_token(response)if not next_token:diff --git a/tests/vector_store/test_pinecone_store.py b/tests/vector_store/test_pinecone_store.pyindex 04e990a7..86758a93 100644--- a/tests/vector_store/test_pinecone_store.py+++ b/tests/vector_store/test_pinecone_store.py@@ -350,6 +350,31 @@ class TestPineconeIterAll(unittest.TestCase):self.assertEqual(list(store.iter_all()), [])wrapper.fetch_vectors.assert_not_called()+ @patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)+ def test_continues_past_an_empty_page_with_a_live_token(self):+ """An empty page is not necessarily the end: Pinecone can legitimately+ list zero ids for a page while pagination.next is still set (sparse+ or filtered namespaces, eventual-consistency windows on serverless+ indexes). Only the absence of a next token means exhaustion."""+ store, wrapper, raw_index = self._store(+ [+ self._page(["a"], "token-1"),+ self._page([], "token-2"), # empty page, but the token still advances+ self._page(["b"], None),+ ],+ [+ {"vectors": {"a": {"values": [0.1], "metadata": {}}}},+ {"vectors": {"b": {"values": [0.2], "metadata": {}}}},+ ],+ )++ result = list(store.iter_all(batch_size=1))++ self.assertEqual([item["id"] for item in result], ["a", "b"])+ self.assertEqual(raw_index.list_paginated.call_count, 3)+ # Nothing to hydrate on the empty page, so only two fetches happen.+ self.assertEqual(wrapper.fetch_vectors.call_count, 2)+@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)def test_accepts_plain_string_ids_from_listing(self):"""SDK generations differ on what listing yields."""