💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Keep documents with a None text field and an object in all stores (#1435)
digit505409ddbba13a
diff --git a/src/python/txtai/database/rdbms.py b/src/python/txtai/database/rdbms.pyindex 5752993e..3729ad03 100644--- a/src/python/txtai/database/rdbms.py+++ b/src/python/txtai/database/rdbms.py@@ -338,11 +338,11 @@ class RDBMS(Database):self.insertdocument(uid, json.dumps(data, allow_nan=False), tags, entry)# If text and object are both available, load object as it won't otherwise be used- if self.text in document and obj:+ if document.get(self.text) is not None and obj:self.loadobject(uid, obj, tags, entry)# Return value to use for section - use text if available otherwise use object- return document[self.text] if self.text in document else obj+ return document[self.text] if document.get(self.text) is not None else objdef insertdocument(self, uid, data, tags, entry):"""diff --git a/src/python/txtai/embeddings/index/indexes.py b/src/python/txtai/embeddings/index/indexes.pyindex fd5983c7..27669d97 100644--- a/src/python/txtai/embeddings/index/indexes.py+++ b/src/python/txtai/embeddings/index/indexes.py@@ -118,9 +118,10 @@ class Indexes:batch = []for _, document, _ in documents:# Add to documents collection if text or object field is set- parent = document- if isinstance(parent, dict):- parent = parent.get(self.text, document.get(self.object))+ if isinstance(document, dict):+ parent = document[self.text] if document.get(self.text) is not None else document.get(self.object)+ else:+ parent = document# Add if field is available or top-level indexing is disabledif parent is not None or not indexing:diff --git a/src/python/txtai/graph/base.py b/src/python/txtai/graph/base.pyindex a25f0645..17fd5e28 100644--- a/src/python/txtai/graph/base.py+++ b/src/python/txtai/graph/base.py@@ -422,7 +422,7 @@ class Graph:}# Require text or object field- document = document.get(self.text, document.get(self.object))+ document = document[self.text] if document.get(self.text) is not None else document.get(self.object)if document is not None:if isinstance(document, list):diff --git a/src/python/txtai/scoring/pgtext.py b/src/python/txtai/scoring/pgtext.pyindex 68b56d1e..840f526e 100644--- a/src/python/txtai/scoring/pgtext.py+++ b/src/python/txtai/scoring/pgtext.py@@ -48,7 +48,7 @@ class PGText(Scoring):for uid, document, _ in documents:# Extract text, if necessaryif isinstance(document, dict):- document = document.get(self.text, document.get(self.object))+ document = document[self.text] if document.get(self.text) is not None else document.get(self.object)if document is not None:# If index is passed, use indexid, otherwise use iddiff --git a/src/python/txtai/scoring/sparse.py b/src/python/txtai/scoring/sparse.pyindex 5101618d..f49173e3 100644--- a/src/python/txtai/scoring/sparse.py+++ b/src/python/txtai/scoring/sparse.py@@ -66,7 +66,7 @@ class Sparse(Scoring):for uid, document, tags in documents:# Extract text, if necessaryif isinstance(document, dict):- document = document.get(self.text, document.get(self.object))+ document = document[self.text] if document.get(self.text) is not None else document.get(self.object)if document is not None:# Add datadiff --git a/src/python/txtai/scoring/tfidf.py b/src/python/txtai/scoring/tfidf.pyindex f4eac335..4b7bd4e8 100644--- a/src/python/txtai/scoring/tfidf.py+++ b/src/python/txtai/scoring/tfidf.py@@ -65,7 +65,7 @@ class TFIDF(Scoring):for uid, document, tags in documents:# Extract text, if necessaryif isinstance(document, dict):- document = document.get(self.text, document.get(self.object))+ document = document[self.text] if document.get(self.text) is not None else document.get(self.object)if document is not None:# If index is passed, use indexid, otherwise use iddiff --git a/test/python/testembeddings.py b/test/python/testembeddings.pyindex de00783b..376086ab 100644--- a/test/python/testembeddings.py+++ b/test/python/testembeddings.py@@ -238,6 +238,52 @@ class TestEmbeddings(unittest.TestCase):self.assertEqual(self.embeddings.search("alpha content", 1)[0][0], 0)self.assertEqual(self.embeddings.search("gamma content", 1)[0][0], 2)+ def testNoneTextObject(self):+ """+ Test dict documents where the text field is set to None and an object is set+ """++ def transform(data):+ # Dummy embedding per input using sum and mean of byte values+ embeddings = []+ for x in data:+ values = list(x if isinstance(x, bytes) else x.encode())+ embeddings.append([sum(values), sum(values) / len(values)])++ return np.array(embeddings, dtype=np.float32)++ # Graph and scoring are tested separately, graph edge inference doesn't support object queries with scoring+ for config in [{"graph": {"approximate": False}}, {"scoring": {"method": "bm25", "terms": True}}]:+ embeddings = Embeddings(method="external", transform=transform, content=True, backend="numpy", **config)+ embeddings.index(+ [+ ("d0", {"text": "alpha content"}, None),+ ("d1", {"text": None, "object": b"objbytes"}, None),+ ("d2", {"text": "gamma content"}, None),+ ]+ )++ # The object document is vectorized and kept with its own id everywhere+ self.assertEqual(embeddings.ann.count(), 3)+ self.assertEqual(embeddings.count(), 3)+ self.assertEqual(+ embeddings.search("select indexid, id, text from txtai", 10),+ [+ {"indexid": 0, "id": "d0", "text": "alpha content"},+ {"indexid": 1, "id": "d1", "text": None},+ {"indexid": 2, "id": "d2", "text": "gamma content"},+ ],+ )++ # Vectors, database rows and search results should all line up+ self.assertEqual(embeddings.search("select id from txtai where similar('alpha content')", 1), [{"id": "d0"}])+ self.assertEqual(embeddings.search("select id from txtai where similar('gamma content')", 1), [{"id": "d2"}])++ if embeddings.graph:+ self.assertEqual([embeddings.graph.attribute(node, "id") for node in range(3)], ["d0", "d1", "d2"])+ else:+ self.assertEqual(embeddings.scoring.search("gamma", 1)[0][0], 2)+def testExternal(self):"""Test embeddings backed by external vectors