💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Fix Transform.stream() yielding a None text value into the encoder (#1231)
Amir Fathi2bae6e4d43a1
diff --git a/src/python/txtai/embeddings/index/transform.py b/src/python/txtai/embeddings/index/transform.pyindex 2b29013d..87c0e4f7 100644--- a/src/python/txtai/embeddings/index/transform.py+++ b/src/python/txtai/embeddings/index/transform.py@@ -147,10 +147,10 @@ class Transform:if not self.indexing and not document[1].get(self.text):document[1][self.text] = str(document[0])- if self.text in document[1]:+ if document[1].get(self.text) is not None:yield (document[0], document[1][self.text], document[2])offset += 1- elif self.object in document[1]:+ elif document[1].get(self.object) is not None:yield (document[0], document[1][self.object], document[2])offset += 1else:diff --git a/test/python/testembeddings.py b/test/python/testembeddings.pyindex 10211599..682deb18 100644--- a/test/python/testembeddings.py+++ b/test/python/testembeddings.py@@ -226,6 +226,19 @@ class TestEmbeddings(unittest.TestCase):self.embeddings.index([(0, {"text": ""}, None)])self.assertTrue(self.embeddings.search("test"))+ def testNoneText(self):+ """+ Test dict documents where the text field is present but set to None+ """++ # None text mixed with real text+ self.embeddings.index([(0, {"text": "alpha content"}, None), (1, {"text": None}, None), (2, {"text": "gamma content"}, None)])++ # Only the two documents with real text should be indexed+ self.assertEqual(self.embeddings.count(), 2)+ self.assertEqual(self.embeddings.search("alpha content", 1)[0][0], 0)+ self.assertEqual(self.embeddings.search("gamma content", 1)[0][0], 2)+def testExternal(self):"""Test embeddings backed by external vectors