💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Fix issue with empty string embeddings indexing, closes #496
davidmezzetti56f0bceee398
diff --git a/src/python/txtai/database/filedb.py b/src/python/txtai/database/filedb.pyindex f48f55f2..2aacd000 100644--- a/src/python/txtai/database/filedb.py+++ b/src/python/txtai/database/filedb.py@@ -377,15 +377,12 @@ class FileDB(Database):if document:self.cursor.execute(FileDB.INSERT_DOCUMENT, [uid, json.dumps(document, allow_nan=False), tags, entry])- # Get value of text field- text = document.get("text")-- # If both text and object are set, insert object as it won't otherwise be used- if text and obj:+ # If text and object are both available, insert object as it won't otherwise be used+ if "text" in document and obj:self.insertobject(uid, obj, tags, entry)# Return value to use for section - use text if available otherwise use object- return text if text else obj+ return document["text"] if "text" in document else objdef insertobject(self, uid, obj, tags, entry):"""diff --git a/test/python/testdatabase/testsqlite.py b/test/python/testdatabase/testsqlite.pyindex 587cc558..3a1ab5d9 100644--- a/test/python/testdatabase/testsqlite.py+++ b/test/python/testdatabase/testsqlite.py@@ -146,6 +146,19 @@ class TestSQLite(unittest.TestCase):embeddings.upsert([])self.assertIsNotNone(embeddings.ann)+ def testEmptyString(self):+ """+ Test empty string indexing+ """++ # Test empty string+ self.embeddings.index([(0, "", None)])+ self.assertTrue(self.embeddings.search("test"))++ # Test empty string with dict+ self.embeddings.index([(0, {"text": ""}, None)])+ self.assertTrue(self.embeddings.search("test"))+def testExplain(self):"""Test query explaindiff --git a/test/python/testembeddings.py b/test/python/testembeddings.pyindex 8161c0be..cda30ac2 100644--- a/test/python/testembeddings.py+++ b/test/python/testembeddings.py@@ -92,6 +92,19 @@ class TestEmbeddings(unittest.TestCase):embeddings.upsert([])self.assertIsNotNone(embeddings.ann)+ def testEmptyString(self):+ """+ Test empty string indexing+ """++ # Test empty string+ self.embeddings.index([(0, "", None)])+ self.assertTrue(self.embeddings.search("test"))++ # Test empty string with dict+ self.embeddings.index([(0, {"text": ""}, None)])+ self.assertTrue(self.embeddings.search("test"))+def testExternal(self):"""Test embeddings backed by external vectors