Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Keep documents with a None text field and an object in all stores (#1435)

digit505409ddbba13a
diff --git a/src/python/txtai/database/rdbms.py b/src/python/txtai/database/rdbms.py
index 5752993e..3729ad03 100644
--- a/src/python/txtai/database/rdbms.py
+++ b/src/python/txtai/database/rdbms.py
@@ -338,11 +338,11 @@ class RDBMS(Database):
self.insertdocument(uid, json.dumps(data, allow_nan=False), tags, entry)
# If text and object are both available, load object as it won't otherwise be used
- if self.text in document and obj:
+ if document.get(self.text) is not None and obj:
self.loadobject(uid, obj, tags, entry)
# Return value to use for section - use text if available otherwise use object
- return document[self.text] if self.text in document else obj
+ return document[self.text] if document.get(self.text) is not None else obj
def insertdocument(self, uid, data, tags, entry):
"""
diff --git a/src/python/txtai/embeddings/index/indexes.py b/src/python/txtai/embeddings/index/indexes.py
index fd5983c7..27669d97 100644
--- a/src/python/txtai/embeddings/index/indexes.py
+++ b/src/python/txtai/embeddings/index/indexes.py
@@ -118,9 +118,10 @@ class Indexes:
batch = []
for _, document, _ in documents:
# Add to documents collection if text or object field is set
- parent = document
- if isinstance(parent, dict):
- parent = parent.get(self.text, document.get(self.object))
+ if isinstance(document, dict):
+ parent = document[self.text] if document.get(self.text) is not None else document.get(self.object)
+ else:
+ parent = document
# Add if field is available or top-level indexing is disabled
if parent is not None or not indexing:
diff --git a/src/python/txtai/graph/base.py b/src/python/txtai/graph/base.py
index a25f0645..17fd5e28 100644
--- a/src/python/txtai/graph/base.py
+++ b/src/python/txtai/graph/base.py
@@ -422,7 +422,7 @@ class Graph:
}
# Require text or object field
- document = document.get(self.text, document.get(self.object))
+ document = document[self.text] if document.get(self.text) is not None else document.get(self.object)
if document is not None:
if isinstance(document, list):
diff --git a/src/python/txtai/scoring/pgtext.py b/src/python/txtai/scoring/pgtext.py
index 68b56d1e..840f526e 100644
--- a/src/python/txtai/scoring/pgtext.py
+++ b/src/python/txtai/scoring/pgtext.py
@@ -48,7 +48,7 @@ class PGText(Scoring):
for uid, document, _ in documents:
# Extract text, if necessary
if isinstance(document, dict):
- document = document.get(self.text, document.get(self.object))
+ document = document[self.text] if document.get(self.text) is not None else document.get(self.object)
if document is not None:
# If index is passed, use indexid, otherwise use id
diff --git a/src/python/txtai/scoring/sparse.py b/src/python/txtai/scoring/sparse.py
index 5101618d..f49173e3 100644
--- a/src/python/txtai/scoring/sparse.py
+++ b/src/python/txtai/scoring/sparse.py
@@ -66,7 +66,7 @@ class Sparse(Scoring):
for uid, document, tags in documents:
# Extract text, if necessary
if isinstance(document, dict):
- document = document.get(self.text, document.get(self.object))
+ document = document[self.text] if document.get(self.text) is not None else document.get(self.object)
if document is not None:
# Add data
diff --git a/src/python/txtai/scoring/tfidf.py b/src/python/txtai/scoring/tfidf.py
index f4eac335..4b7bd4e8 100644
--- a/src/python/txtai/scoring/tfidf.py
+++ b/src/python/txtai/scoring/tfidf.py
@@ -65,7 +65,7 @@ class TFIDF(Scoring):
for uid, document, tags in documents:
# Extract text, if necessary
if isinstance(document, dict):
- document = document.get(self.text, document.get(self.object))
+ document = document[self.text] if document.get(self.text) is not None else document.get(self.object)
if document is not None:
# If index is passed, use indexid, otherwise use id
diff --git a/test/python/testembeddings.py b/test/python/testembeddings.py
index de00783b..376086ab 100644
--- a/test/python/testembeddings.py
+++ b/test/python/testembeddings.py
@@ -238,6 +238,52 @@ class TestEmbeddings(unittest.TestCase):
self.assertEqual(self.embeddings.search("alpha content", 1)[0][0], 0)
self.assertEqual(self.embeddings.search("gamma content", 1)[0][0], 2)
+ def testNoneTextObject(self):
+ """
+ Test dict documents where the text field is set to None and an object is set
+ """
+
+ def transform(data):
+ # Dummy embedding per input using sum and mean of byte values
+ embeddings = []
+ for x in data:
+ values = list(x if isinstance(x, bytes) else x.encode())
+ embeddings.append([sum(values), sum(values) / len(values)])
+
+ return np.array(embeddings, dtype=np.float32)
+
+ # Graph and scoring are tested separately, graph edge inference doesn't support object queries with scoring
+ for config in [{"graph": {"approximate": False}}, {"scoring": {"method": "bm25", "terms": True}}]:
+ embeddings = Embeddings(method="external", transform=transform, content=True, backend="numpy", **config)
+ embeddings.index(
+ [
+ ("d0", {"text": "alpha content"}, None),
+ ("d1", {"text": None, "object": b"objbytes"}, None),
+ ("d2", {"text": "gamma content"}, None),
+ ]
+ )
+
+ # The object document is vectorized and kept with its own id everywhere
+ self.assertEqual(embeddings.ann.count(), 3)
+ self.assertEqual(embeddings.count(), 3)
+ self.assertEqual(
+ embeddings.search("select indexid, id, text from txtai", 10),
+ [
+ {"indexid": 0, "id": "d0", "text": "alpha content"},
+ {"indexid": 1, "id": "d1", "text": None},
+ {"indexid": 2, "id": "d2", "text": "gamma content"},
+ ],
+ )
+
+ # Vectors, database rows and search results should all line up
+ self.assertEqual(embeddings.search("select id from txtai where similar('alpha content')", 1), [{"id": "d0"}])
+ self.assertEqual(embeddings.search("select id from txtai where similar('gamma content')", 1), [{"id": "d2"}])
+
+ if embeddings.graph:
+ self.assertEqual([embeddings.graph.attribute(node, "id") for node in range(3)], ["d0", "d1", "d2"])
+ else:
+ self.assertEqual(embeddings.scoring.search("gamma", 1)[0][0], 2)
+
def testExternal(self):
"""
Test embeddings backed by external vectors