Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Fix Transform.stream() yielding a None text value into the encoder (#1231)

Amir Fathi2bae6e4d43a1
diff --git a/src/python/txtai/embeddings/index/transform.py b/src/python/txtai/embeddings/index/transform.py
index 2b29013d..87c0e4f7 100644
--- a/src/python/txtai/embeddings/index/transform.py
+++ b/src/python/txtai/embeddings/index/transform.py
@@ -147,10 +147,10 @@ class Transform:
if not self.indexing and not document[1].get(self.text):
document[1][self.text] = str(document[0])
- if self.text in document[1]:
+ if document[1].get(self.text) is not None:
yield (document[0], document[1][self.text], document[2])
offset += 1
- elif self.object in document[1]:
+ elif document[1].get(self.object) is not None:
yield (document[0], document[1][self.object], document[2])
offset += 1
else:
diff --git a/test/python/testembeddings.py b/test/python/testembeddings.py
index 10211599..682deb18 100644
--- a/test/python/testembeddings.py
+++ b/test/python/testembeddings.py
@@ -226,6 +226,19 @@ class TestEmbeddings(unittest.TestCase):
self.embeddings.index([(0, {"text": ""}, None)])
self.assertTrue(self.embeddings.search("test"))
+ def testNoneText(self):
+ """
+ Test dict documents where the text field is present but set to None
+ """
+
+ # None text mixed with real text
+ self.embeddings.index([(0, {"text": "alpha content"}, None), (1, {"text": None}, None), (2, {"text": "gamma content"}, None)])
+
+ # Only the two documents with real text should be indexed
+ self.assertEqual(self.embeddings.count(), 2)
+ self.assertEqual(self.embeddings.search("alpha content", 1)[0][0], 0)
+ self.assertEqual(self.embeddings.search("gamma content", 1)[0][0], 2)
+
def testExternal(self):
"""
Test embeddings backed by external vectors