Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Fix Cluster.shard TypeError on tuple documents and empty-string ids (#1146)

Andrew Chen77965f44ddc8
diff --git a/src/python/txtai/api/cluster.py b/src/python/txtai/api/cluster.py
index d98c7a40..80718186 100644
--- a/src/python/txtai/api/cluster.py
+++ b/src/python/txtai/api/cluster.py
@@ -196,11 +196,11 @@ class Cluster:
shards = [[] for _ in range(len(self.shards))]
for document in documents:
uid = document.get("id") if isinstance(document, dict) else document
- if uid and isinstance(uid, str):
- # Quick int hash of string to help derive shard id
+ if isinstance(uid, str):
+ # Quick int hash of string to help derive shard id (handles empty strings too)
uid = zlib.adler32(uid.encode("utf-8"))
- elif uid is None:
- # Get random shard id when uid isn't set
+ elif not isinstance(uid, int):
+ # Get random shard id when uid isn't a usable int (None, a tuple document, etc.)
uid = random.randint(0, len(shards) - 1)
shards[uid % len(self.shards)].append(document)
diff --git a/test/python/testapi/testcluster.py b/test/python/testapi/testcluster.py
index 15108213..646d035e 100644
--- a/test/python/testapi/testcluster.py
+++ b/test/python/testapi/testcluster.py
@@ -147,6 +147,21 @@ class TestCluster(unittest.TestCase):
cls.httpd1.shutdown()
cls.httpd2.shutdown()
+ def testShard(self):
+ """
+ Test sharding handles all id shapes without crashing
+ """
+
+ from txtai.api.cluster import Cluster
+
+ cluster = Cluster({"shards": [{"base": "h1"}, {"base": "h2"}, {"base": "h3"}]})
+
+ # str id, tuple document, empty-string id, int id and a missing id must all be placed.
+ documents = [{"id": "doc1"}, (1, "text", None), {"id": ""}, {"id": 0}, {"other": "x"}]
+ shards = cluster.shard(documents)
+
+ self.assertEqual(sum(len(shard) for shard in shards), len(documents))
+
def testCount(self):
"""
Test cluster count