💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Fix Cluster.shard TypeError on tuple documents and empty-string ids (#1146)
Andrew Chen77965f44ddc8
diff --git a/src/python/txtai/api/cluster.py b/src/python/txtai/api/cluster.pyindex d98c7a40..80718186 100644--- a/src/python/txtai/api/cluster.py+++ b/src/python/txtai/api/cluster.py@@ -196,11 +196,11 @@ class Cluster:shards = [[] for _ in range(len(self.shards))]for document in documents:uid = document.get("id") if isinstance(document, dict) else document- if uid and isinstance(uid, str):- # Quick int hash of string to help derive shard id+ if isinstance(uid, str):+ # Quick int hash of string to help derive shard id (handles empty strings too)uid = zlib.adler32(uid.encode("utf-8"))- elif uid is None:- # Get random shard id when uid isn't set+ elif not isinstance(uid, int):+ # Get random shard id when uid isn't a usable int (None, a tuple document, etc.)uid = random.randint(0, len(shards) - 1)shards[uid % len(self.shards)].append(document)diff --git a/test/python/testapi/testcluster.py b/test/python/testapi/testcluster.pyindex 15108213..646d035e 100644--- a/test/python/testapi/testcluster.py+++ b/test/python/testapi/testcluster.py@@ -147,6 +147,21 @@ class TestCluster(unittest.TestCase):cls.httpd1.shutdown()cls.httpd2.shutdown()+ def testShard(self):+ """+ Test sharding handles all id shapes without crashing+ """++ from txtai.api.cluster import Cluster++ cluster = Cluster({"shards": [{"base": "h1"}, {"base": "h2"}, {"base": "h3"}]})++ # str id, tuple document, empty-string id, int id and a missing id must all be placed.+ documents = [{"id": "doc1"}, (1, "text", None), {"id": ""}, {"id": 0}, {"other": "x"}]+ shards = cluster.shard(documents)++ self.assertEqual(sum(len(shard) for shard in shards), len(documents))+def testCount(self):"""Test cluster count