Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Fix bugs with embeddings graphs. Closes #421, closes #435, closes #453

davidmezzetti33d83d75cbd5
diff --git a/docs/embeddings/configuration/index.md b/docs/embeddings/configuration/index.md
index 68bd42b5..9e373f19 100644
--- a/docs/embeddings/configuration/index.md
+++ b/docs/embeddings/configuration/index.md
@@ -218,6 +218,7 @@ topics:
labels: scoring index method used to build topic labels (string)
options are bm25 (default), tfidf, sif
terms: number of frequent terms to use for topic labels (int), defaults to 4
+ stopwords: optional list of stop words to exclude from topic labels
categories: optional list of categories used to group topics, allows
granular topics with broad categories grouping topics
```
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.py
index c5706fdf..8126fab4 100644
--- a/src/python/txtai/embeddings/base.py
+++ b/src/python/txtai/embeddings/base.py
@@ -62,6 +62,9 @@ class Embeddings:
# Document database
self.database = None
+ # Resolvable functions
+ self.functions = None
+
# Graph network
self.graph = None
@@ -151,8 +154,8 @@ class Embeddings:
documents: list of (id, data, tags)
"""
- # Run standard insert if index doesn't exist
- if not self.ann:
+ # Run standard insert if index doesn't exist or it has no records
+ if not self.count():
self.index(documents)
return
@@ -179,7 +182,7 @@ class Embeddings:
# Graph upsert, if necessary
if self.graph:
- self.graph.upsert(Search(self, True))
+ self.graph.upsert(Search(self, True), self.batchsimilarity)
def delete(self, ids):
"""
@@ -252,6 +255,10 @@ class Embeddings:
# Reset configuration
self.configure(config)
+ # Reset function references
+ if self.functions:
+ self.functions.reset()
+
# Reindex
if function:
self.index(function(self.database.reindex(columns)), True)
@@ -579,7 +586,7 @@ class Embeddings:
# Close database connection if open
if self.database:
self.database.close()
- self.database = None
+ self.database, self.functions = None, None
def info(self):
"""
@@ -761,9 +768,10 @@ class Embeddings:
config = self.config.copy()
- # Resolve callable functions
- if "functions" in config:
- config["functions"] = Functions(self)(config)
+ # Create references to callable functions
+ self.functions = Functions(self) if "functions" in config else None
+ if self.functions:
+ config["functions"] = self.functions(config)
# Create database from config and return
return DatabaseFactory.create(config)
diff --git a/src/python/txtai/embeddings/functions.py b/src/python/txtai/embeddings/functions.py
index c9d4c31d..28d1a913 100644
--- a/src/python/txtai/embeddings/functions.py
+++ b/src/python/txtai/embeddings/functions.py
@@ -20,6 +20,9 @@ class Functions:
self.embeddings = embeddings
+ # Handle to all reference objects
+ self.references = None
+
def __call__(self, config):
"""
Resolves a list of functions to function references.
@@ -31,6 +34,9 @@ class Functions:
list of function references
"""
+ # Initialize stored references array
+ self.references = []
+
# Resolve callable functions
functions = []
for fn in config["functions"]:
@@ -43,6 +49,15 @@ class Functions:
return functions
+ def reset(self):
+ """
+ Clears all resolved references.
+ """
+
+ if self.references:
+ for reference in self.references:
+ reference.reset()
+
def function(self, function):
"""
Resolves function configuration. If function is a string, it's split on '.' and each part
@@ -61,12 +76,14 @@ class Functions:
if hasattr(self.embeddings, parts[0]):
m = Reference(self.embeddings, parts[0])
+ self.references.append(m)
else:
module = ".".join(parts[:-1])
m = __import__(module)
for comp in parts[1:]:
m = Reference(m, comp)
+ self.references.append(m)
return m
@@ -92,6 +109,9 @@ class Reference:
self.obj = obj
self.attribute = attribute
+ # Keep a handle to the original inputs
+ self.inputs = (obj, attribute)
+
# True if the object and attribute have been resolved
self.resolved = False
@@ -125,3 +145,11 @@ class Reference:
# If attribute is a function, execute and return, otherwise return attribute
return attribute(*args) if self.function else attribute
+
+ def reset(self):
+ """
+ Clears resolved references.
+ """
+
+ self.obj, self.attribute = self.inputs
+ self.resolved = False
diff --git a/src/python/txtai/graph/base.py b/src/python/txtai/graph/base.py
index 4149efde..28a4ef93 100644
--- a/src/python/txtai/graph/base.py
+++ b/src/python/txtai/graph/base.py
@@ -377,8 +377,13 @@ class Graph:
# Delete from topics
topic = self.attribute(uid, "topic")
if topic and self.topics:
+ # Delete id from topic
self.topics[topic].remove(uid)
+ # Also delete topic, if it's empty
+ if not self.topics[topic]:
+ self.topics.pop(topic)
+
# Delete node
self.removenode(uid)
@@ -398,12 +403,13 @@ class Graph:
if "topics" in self.config:
self.addtopics(similarity)
- def upsert(self, search):
+ def upsert(self, search, similarity=None):
"""
Adds relationships for new graph nodes using a score-based search function.
Args:
search: batch search function - takes a list of queries and returns lists of (id, scores) to use as edge weights
+ similarity: batch similarity function - takes a list of text and labels and returns best matches
"""
# Detect if topics processing is enabled
@@ -414,7 +420,11 @@ class Graph:
# Infer topics with topics of connected nodes
if hastopics:
- self.infertopics()
+ # Infer topics if there is at least one topic, otherwise rebuild
+ if self.topics:
+ self.infertopics()
+ else:
+ self.addtopics(similarity)
def addedges(self, nodes, search, attributes=None):
"""
diff --git a/test/python/testgraph.py b/test/python/testgraph.py
index 6ed7a3b1..a12f5472 100644
--- a/test/python/testgraph.py
+++ b/test/python/testgraph.py
@@ -131,7 +131,26 @@ class TestGraph(unittest.TestCase):
# Create an index for the list of text
self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
- # Test similar
+ # Test function
+ result = self.embeddings.search("select category, topic, topicrank from txtai where id = 0", 1)[0]
+
+ # Check columns have a value
+ self.assertIsNotNone(result["category"])
+ self.assertIsNotNone(result["topic"])
+ self.assertIsNotNone(result["topicrank"])
+
+ def testFunctionReindex(self):
+ """
+ Test running graph functions with SQL after reindex
+ """
+
+ # Create an index for the list of text
+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
+
+ # Test functions reset with a reindex
+ self.embeddings.reindex(self.embeddings.config)
+
+ # Test function
result = self.embeddings.search("select category, topic, topicrank from txtai where id = 0", 1)[0]
# Check columns have a value
@@ -182,6 +201,16 @@ class TestGraph(unittest.TestCase):
self.assertRaises(NotImplementedError, graph.loadgraph, None)
self.assertRaises(NotImplementedError, graph.savegraph, None)
+ def testResetTopics(self):
+ """
+ Test resetting of topics
+ """
+
+ # Create an index for the list of text
+ self.embeddings.index([(1, "text", None)])
+ self.embeddings.upsert([(1, "graph", None)])
+ self.assertEqual(list(self.embeddings.graph.topics.keys()), ["graph"])
+
def testSave(self):
"""
Test save