💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Fix bugs with embeddings graphs. Closes #421, closes #435, closes #453
davidmezzetti33d83d75cbd5
diff --git a/docs/embeddings/configuration/index.md b/docs/embeddings/configuration/index.mdindex 68bd42b5..9e373f19 100644--- a/docs/embeddings/configuration/index.md+++ b/docs/embeddings/configuration/index.md@@ -218,6 +218,7 @@ topics:labels: scoring index method used to build topic labels (string)options are bm25 (default), tfidf, sifterms: number of frequent terms to use for topic labels (int), defaults to 4+ stopwords: optional list of stop words to exclude from topic labelscategories: optional list of categories used to group topics, allowsgranular topics with broad categories grouping topics```diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.pyindex c5706fdf..8126fab4 100644--- a/src/python/txtai/embeddings/base.py+++ b/src/python/txtai/embeddings/base.py@@ -62,6 +62,9 @@ class Embeddings:# Document databaseself.database = None+ # Resolvable functions+ self.functions = None+# Graph networkself.graph = None@@ -151,8 +154,8 @@ class Embeddings:documents: list of (id, data, tags)"""- # Run standard insert if index doesn't exist- if not self.ann:+ # Run standard insert if index doesn't exist or it has no records+ if not self.count():self.index(documents)return@@ -179,7 +182,7 @@ class Embeddings:# Graph upsert, if necessaryif self.graph:- self.graph.upsert(Search(self, True))+ self.graph.upsert(Search(self, True), self.batchsimilarity)def delete(self, ids):"""@@ -252,6 +255,10 @@ class Embeddings:# Reset configurationself.configure(config)+ # Reset function references+ if self.functions:+ self.functions.reset()+# Reindexif function:self.index(function(self.database.reindex(columns)), True)@@ -579,7 +586,7 @@ class Embeddings:# Close database connection if openif self.database:self.database.close()- self.database = None+ self.database, self.functions = None, Nonedef info(self):"""@@ -761,9 +768,10 @@ class Embeddings:config = self.config.copy()- # Resolve callable functions- if "functions" in config:- config["functions"] = Functions(self)(config)+ # Create references to callable functions+ self.functions = Functions(self) if "functions" in config else None+ if self.functions:+ config["functions"] = self.functions(config)# Create database from config and returnreturn DatabaseFactory.create(config)diff --git a/src/python/txtai/embeddings/functions.py b/src/python/txtai/embeddings/functions.pyindex c9d4c31d..28d1a913 100644--- a/src/python/txtai/embeddings/functions.py+++ b/src/python/txtai/embeddings/functions.py@@ -20,6 +20,9 @@ class Functions:self.embeddings = embeddings+ # Handle to all reference objects+ self.references = None+def __call__(self, config):"""Resolves a list of functions to function references.@@ -31,6 +34,9 @@ class Functions:list of function references"""+ # Initialize stored references array+ self.references = []+# Resolve callable functionsfunctions = []for fn in config["functions"]:@@ -43,6 +49,15 @@ class Functions:return functions+ def reset(self):+ """+ Clears all resolved references.+ """++ if self.references:+ for reference in self.references:+ reference.reset()+def function(self, function):"""Resolves function configuration. If function is a string, it's split on '.' and each part@@ -61,12 +76,14 @@ class Functions:if hasattr(self.embeddings, parts[0]):m = Reference(self.embeddings, parts[0])+ self.references.append(m)else:module = ".".join(parts[:-1])m = __import__(module)for comp in parts[1:]:m = Reference(m, comp)+ self.references.append(m)return m@@ -92,6 +109,9 @@ class Reference:self.obj = objself.attribute = attribute+ # Keep a handle to the original inputs+ self.inputs = (obj, attribute)+# True if the object and attribute have been resolvedself.resolved = False@@ -125,3 +145,11 @@ class Reference:# If attribute is a function, execute and return, otherwise return attributereturn attribute(*args) if self.function else attribute++ def reset(self):+ """+ Clears resolved references.+ """++ self.obj, self.attribute = self.inputs+ self.resolved = Falsediff --git a/src/python/txtai/graph/base.py b/src/python/txtai/graph/base.pyindex 4149efde..28a4ef93 100644--- a/src/python/txtai/graph/base.py+++ b/src/python/txtai/graph/base.py@@ -377,8 +377,13 @@ class Graph:# Delete from topicstopic = self.attribute(uid, "topic")if topic and self.topics:+ # Delete id from topicself.topics[topic].remove(uid)+ # Also delete topic, if it's empty+ if not self.topics[topic]:+ self.topics.pop(topic)+# Delete nodeself.removenode(uid)@@ -398,12 +403,13 @@ class Graph:if "topics" in self.config:self.addtopics(similarity)- def upsert(self, search):+ def upsert(self, search, similarity=None):"""Adds relationships for new graph nodes using a score-based search function.Args:search: batch search function - takes a list of queries and returns lists of (id, scores) to use as edge weights+ similarity: batch similarity function - takes a list of text and labels and returns best matches"""# Detect if topics processing is enabled@@ -414,7 +420,11 @@ class Graph:# Infer topics with topics of connected nodesif hastopics:- self.infertopics()+ # Infer topics if there is at least one topic, otherwise rebuild+ if self.topics:+ self.infertopics()+ else:+ self.addtopics(similarity)def addedges(self, nodes, search, attributes=None):"""diff --git a/test/python/testgraph.py b/test/python/testgraph.pyindex 6ed7a3b1..a12f5472 100644--- a/test/python/testgraph.py+++ b/test/python/testgraph.py@@ -131,7 +131,26 @@ class TestGraph(unittest.TestCase):# Create an index for the list of textself.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])- # Test similar+ # Test function+ result = self.embeddings.search("select category, topic, topicrank from txtai where id = 0", 1)[0]++ # Check columns have a value+ self.assertIsNotNone(result["category"])+ self.assertIsNotNone(result["topic"])+ self.assertIsNotNone(result["topicrank"])++ def testFunctionReindex(self):+ """+ Test running graph functions with SQL after reindex+ """++ # Create an index for the list of text+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])++ # Test functions reset with a reindex+ self.embeddings.reindex(self.embeddings.config)++ # Test functionresult = self.embeddings.search("select category, topic, topicrank from txtai where id = 0", 1)[0]# Check columns have a value@@ -182,6 +201,16 @@ class TestGraph(unittest.TestCase):self.assertRaises(NotImplementedError, graph.loadgraph, None)self.assertRaises(NotImplementedError, graph.savegraph, None)+ def testResetTopics(self):+ """+ Test resetting of topics+ """++ # Create an index for the list of text+ self.embeddings.index([(1, "text", None)])+ self.embeddings.upsert([(1, "graph", None)])+ self.assertEqual(list(self.embeddings.graph.topics.keys()), ["graph"])+def testSave(self):"""Test save