Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Fix issue with empty string embeddings indexing, closes #496

davidmezzetti56f0bceee398
diff --git a/src/python/txtai/database/filedb.py b/src/python/txtai/database/filedb.py
index f48f55f2..2aacd000 100644
--- a/src/python/txtai/database/filedb.py
+++ b/src/python/txtai/database/filedb.py
@@ -377,15 +377,12 @@ class FileDB(Database):
if document:
self.cursor.execute(FileDB.INSERT_DOCUMENT, [uid, json.dumps(document, allow_nan=False), tags, entry])
- # Get value of text field
- text = document.get("text")
-
- # If both text and object are set, insert object as it won't otherwise be used
- if text and obj:
+ # If text and object are both available, insert object as it won't otherwise be used
+ if "text" in document and obj:
self.insertobject(uid, obj, tags, entry)
# Return value to use for section - use text if available otherwise use object
- return text if text else obj
+ return document["text"] if "text" in document else obj
def insertobject(self, uid, obj, tags, entry):
"""
diff --git a/test/python/testdatabase/testsqlite.py b/test/python/testdatabase/testsqlite.py
index 587cc558..3a1ab5d9 100644
--- a/test/python/testdatabase/testsqlite.py
+++ b/test/python/testdatabase/testsqlite.py
@@ -146,6 +146,19 @@ class TestSQLite(unittest.TestCase):
embeddings.upsert([])
self.assertIsNotNone(embeddings.ann)
+ def testEmptyString(self):
+ """
+ Test empty string indexing
+ """
+
+ # Test empty string
+ self.embeddings.index([(0, "", None)])
+ self.assertTrue(self.embeddings.search("test"))
+
+ # Test empty string with dict
+ self.embeddings.index([(0, {"text": ""}, None)])
+ self.assertTrue(self.embeddings.search("test"))
+
def testExplain(self):
"""
Test query explain
diff --git a/test/python/testembeddings.py b/test/python/testembeddings.py
index 8161c0be..cda30ac2 100644
--- a/test/python/testembeddings.py
+++ b/test/python/testembeddings.py
@@ -92,6 +92,19 @@ class TestEmbeddings(unittest.TestCase):
embeddings.upsert([])
self.assertIsNotNone(embeddings.ann)
+ def testEmptyString(self):
+ """
+ Test empty string indexing
+ """
+
+ # Test empty string
+ self.embeddings.index([(0, "", None)])
+ self.assertTrue(self.embeddings.search("test"))
+
+ # Test empty string with dict
+ self.embeddings.index([(0, {"text": ""}, None)])
+ self.assertTrue(self.embeddings.search("test"))
+
def testExternal(self):
"""
Test embeddings backed by external vectors