💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add a semantic graph component, closes #339
davidmezzetti8bd6bc7b388e
diff --git a/setup.py b/setup.pyindex 2b80eb22..fc93923f 100644--- a/setup.py+++ b/setup.py@@ -25,6 +25,8 @@ extras["console"] = ["rich>=12.0.1"]extras["database"] = ["pillow>=7.1.2"]+extras["graph"] = ["networkx>=2.6.3", "python-louvain>=0.16"]+extras["model"] = ["onnxruntime>=1.8.1"]extras["pipeline"] = [diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.pyindex f692547f..59466cbb 100644--- a/src/python/txtai/embeddings/base.py+++ b/src/python/txtai/embeddings/base.py@@ -14,11 +14,13 @@ from .. import __pickle__from ..ann import ANNFactoryfrom ..database import DatabaseFactory+from ..graph import GraphFactoryfrom ..scoring import ScoringFactoryfrom ..vectors import VectorsFactoryfrom .archive import Archivefrom .explain import Explain+from .functions import Functionsfrom .reducer import Reducerfrom .query import Queryfrom .search import Search@@ -46,7 +48,7 @@ class Embeddings:# Index configurationself.config = None- # Dimensionality reduction and scoring models - word vectors only+ # Dimensionality reduction and scoring index - word vectors onlyself.reducer, self.scoring = None, None# Embeddings vector model - transforms data into similarity vectors@@ -58,6 +60,9 @@ class Embeddings:# Document databaseself.database = None+ # Graph network+ self.graph = None+# Query modelself.query = None@@ -95,6 +100,9 @@ class Embeddings:# Reset archive since this is a new indexself.archive = None+ # Create graph, if necessary+ self.graph = self.creategraph()+# Create transform actiontransform = Transform(self, Action.REINDEX if reindex else Action.INDEX)@@ -123,6 +131,10 @@ class Embeddings:if not reindex and not self.database:self.config["ids"] = ids+ # Index graph, if necessary+ if self.graph:+ self.graph.index(Search(self, True), self.batchsimilarity)+def upsert(self, documents):"""Runs an embeddings upsert operation. If the index exists, new data is@@ -155,6 +167,10 @@ class Embeddings:if not self.database:self.config["ids"] = self.config["ids"] + ids+ # Graph upsert, if necessary+ if self.graph:+ self.graph.upsert(Search(self, True))+def delete(self, ids):"""Deletes from an embeddings index. Returns list of ids deleted.@@ -200,6 +216,10 @@ class Embeddings:# Delete ids from indexself.ann.delete(indices)+ # Delete ids from graph+ if self.graph:+ self.graph.delete(indices)+return deletesdef reindex(self, config, columns=None, function=None):@@ -427,7 +447,7 @@ class Embeddings:self.reducer = Reducer()self.reducer.load(f"{path}/lsa")- # Embedding scoring model - word vectors only+ # Embedding scoring index - word vectors onlyif self.config.get("scoring"):self.scoring = ScoringFactory.create(self.config["scoring"])self.scoring.load(f"{path}/scoring")@@ -443,6 +463,11 @@ class Embeddings:if self.database:self.database.load(f"{path}/documents")+ # Graph network - stores relationships+ self.graph = self.creategraph()+ if self.graph:+ self.graph.load(f"{path}/graph")+def save(self, path, cloud=None):"""Saves an index in a directory at path unless path ends with tar.gz, tar.bz2, tar.xz or zip.@@ -477,7 +502,7 @@ class Embeddings:if self.reducer:self.reducer.save(f"{path}/lsa")- # Save embedding scoring model (word vectors only)+ # Save embedding scoring index (word vectors only)if self.scoring:self.scoring.save(f"{path}/scoring")@@ -485,6 +510,10 @@ class Embeddings:if self.database:self.database.save(f"{path}/documents")+ # Save graph+ if self.graph:+ self.graph.save(f"{path}/graph")+# If this is an archive, save itif apath:self.archive.save(apath, cloud)@@ -494,7 +523,8 @@ class Embeddings:Closes this embeddings index and frees all resources."""- self.config, self.reducer, self.scoring, self.model, self.ann, self.query, self.archive = None, None, None, None, None, None, None+ self.config, self.reducer, self.scoring, self.model = None, None, None, None+ self.ann, self.graph, self.query, self.archive = None, None, None, None# Close database connection if openif self.database:@@ -597,8 +627,24 @@ class Embeddings:if self.database:self.database.close()+ config = self.config.copy()++ # Resolve callable functions+ if "functions" in config:+ config["functions"] = Functions(self)(config)+# Create database from config and return- return DatabaseFactory.create(self.config)+ return DatabaseFactory.create(config)++ def creategraph(self):+ """+ Creates a graph from config.++ Returns:+ new graph, if enabled in config+ """++ return GraphFactory.create(self.config["graph"]) if "graph" in self.config else Nonedef normalize(self, embeddings):"""diff --git a/src/python/txtai/embeddings/search.py b/src/python/txtai/embeddings/search.pyindex 74f77340..2f8884f2 100644--- a/src/python/txtai/embeddings/search.py+++ b/src/python/txtai/embeddings/search.py@@ -15,15 +15,17 @@ class Search:Executes a batch search action. A search can be both approximate nearest neighbor and/or database driven."""- def __init__(self, embeddings):+ def __init__(self, embeddings, indexids=False):"""Creates a new search action.Args:embeddings: embeddings instance+ indexids: searches return indexids when True, otherwise run standard search"""self.embeddings = embeddings+ self.indexids = indexids# Alias embeddings attributesself.config = embeddings.config@@ -49,7 +51,7 @@ class Search:if not self.ann:return [[]] * len(queries)- if self.database:+ if not self.indexids and self.database:return self.dbsearch(queries, limit)# Default: execute an approximate nearest neighbor search@@ -77,7 +79,7 @@ class Search:results = [[(i, score) for i, score in r if score > 0] for r in results]# Map indexids to ids if "ids" available- if "ids" in self.config:+ if not self.indexids and "ids" in self.config:lookup = self.config["ids"]return [[(lookup[i], score) for i, score in r] for r in results]diff --git a/src/python/txtai/embeddings/transform.py b/src/python/txtai/embeddings/transform.pyindex 7237da23..d9ddcfbe 100644--- a/src/python/txtai/embeddings/transform.py+++ b/src/python/txtai/embeddings/transform.py@@ -31,6 +31,7 @@ class Transform:self.delete = self.embeddings.deleteself.model = self.embeddings.modelself.database = self.embeddings.database+ self.graph = self.embeddings.graph# Get config parametersself.offset = self.embeddings.config.get("offset", 0) if self.action == Action.UPSERT else 0@@ -139,7 +140,13 @@ class Transform:# Load batch into database except if this is a reindexif self.database and self.action != Action.REINDEX:self.database.insert(batch, self.offset)- self.offset += offset++ # Load batch into graph+ if self.graph:+ self.graph.insert(batch, self.offset)++ # Increment offset+ self.offset += offsetclass Action(Enum):diff --git a/src/python/txtai/graph/__init__.py b/src/python/txtai/graph/__init__.pynew file mode 100644index 00000000..80019ad6--- /dev/null+++ b/src/python/txtai/graph/__init__.py@@ -0,0 +1,8 @@+"""+Graph imports+"""++from .base import Graph+from .factory import GraphFactory+from .networkx import NetworkX+from .topics import Topicsdiff --git a/src/python/txtai/graph/base.py b/src/python/txtai/graph/base.pynew file mode 100644index 00000000..af15d93f--- /dev/null+++ b/src/python/txtai/graph/base.py@@ -0,0 +1,552 @@+"""+Graph module+"""++import os+import pickle+import tarfile++from collections import Counter+from tempfile import TemporaryDirectory++from .. import __pickle__++from .topics import Topics++# pylint: disable=R0904+class Graph:+ """+ Base class for Graph instances.+ """++ def __init__(self, config):+ """+ Creates a new Graph.++ Args:+ config: graph configuration+ """++ # Graph configuration+ self.config = config++ # Graph backend+ self.backend = None++ # Topic modeling+ self.categories = None+ self.topics = None++ def create(self):+ """+ Creates the graph network.+ """++ raise NotImplementedError++ def count(self):+ """+ Returns the total number of nodes in graph.++ Returns:+ total nodes in graph+ """++ raise NotImplementedError++ def scan(self, attribute=None):+ """+ Iterates over nodes that match a criteria. If no criteria specified, all nodes+ are returned.++ Args:+ attribute: if specified, nodes having this attribute are returned++ Returns:+ node iterator+ """++ raise NotImplementedError++ def node(self, uid):+ """+ Get node with id uid. Returns None if not found.++ Args:+ uid: node id++ Returns:+ graph node+ """++ raise NotImplementedError++ def addnode(self, uid, **attrs):+ """+ Adds a node to the graph.++ Args:+ uid: node id+ attrs: node attributes+ """++ raise NotImplementedError++ def removenode(self, uid):+ """+ Removes a node and all it's edges from graph.++ Args:+ uid: node id+ """++ raise NotImplementedError++ def hasnode(self, uid):+ """+ Returns True if node found, False otherwise.++ Args:+ uid: node id++ Returns:+ True if node found, False otherwise+ """++ raise NotImplementedError++ def attribute(self, uid, field):+ """+ Gets a node attribute.++ Args:+ uid: node id+ field: attribute name++ Returns:+ attribute value+ """++ raise NotImplementedError++ def addattribute(self, uid, field, value):+ """+ Adds an attribute to node.++ Args:+ uid: node id+ field: attribute name+ value: attribute value+ """++ raise NotImplementedError++ def removeattribute(self, uid, field):+ """+ Removes an attribute from node.++ Args:+ uid: node id+ field: attribute name++ Returns:+ attribute value or None if not present+ """++ raise NotImplementedError++ def edgecount(self):+ """+ Returns the total number of edges.++ Returns:+ total number of edges in graph+ """++ raise NotImplementedError++ def edges(self, uid):+ """+ Gets edges of node by id.++ Args:+ uid: node id++ Returns:+ list of edge node ids+ """++ raise NotImplementedError++ def addedge(self, source, target, **attrs):+ """+ Adds an edge to graph.++ Args:+ source: node 1 id+ target: node 2 id+ """++ raise NotImplementedError++ def hasedge(self, source, target=None):+ """+ Returns True if edge found, False otherwise. If target is None, this method+ returns True if any edge is found.++ Args:+ source: node 1 id+ target: node 2 id++ Returns:+ True if edge found, False otherwise+ """++ raise NotImplementedError++ def centrality(self):+ """+ Runs a centrality algorithm on the graph.++ Returns:+ dict of {node id: centrality score}+ """++ raise NotImplementedError++ def pagerank(self):+ """+ Runs the pagerank algorithm on the graph.++ Returns:+ dict of {node id, page rank score}+ """++ raise NotImplementedError++ def showpath(self, source, target):+ """+ Gets the shortest path between source and target.++ Args:+ source: start node id+ target: end node id++ Returns:+ list of node ids representing the shortest path+ """++ raise NotImplementedError++ def communities(self, config):+ """+ Run community detection on the graph.++ Args:+ config: configuration++ Returns:+ dictionary of {topic name:[ids]}+ """++ raise NotImplementedError++ def loadgraph(self, path):+ """+ Loads a graph backend at path.++ Args:+ path: path to graph backend+ """++ raise NotImplementedError++ def savegraph(self, path):+ """+ Saves graph backend to path.++ Args:+ path: path to save graph backend+ """++ raise NotImplementedError++ def initialize(self):+ """+ Initialize graph instance.+ """++ if not self.backend:+ self.backend = self.create()++ def load(self, path):+ """+ Loads a graph at path.++ Args:+ path: path to graph+ """++ # Extract files to temporary directory and load content+ with TemporaryDirectory() as directory:+ # Extract tar contents+ with tarfile.open(path, "r") as tar:+ tar.extractall(directory)++ # Load graph backend+ self.loadgraph(f"{directory}/graph")++ # Load categories, if necessary+ path = f"{directory}/categories"+ if os.path.exists(path):+ with open(path, "rb") as handle:+ self.categories = pickle.load(handle)++ # Load topics, if necessary+ path = f"{directory}/topics"+ if os.path.exists(path):+ with open(path, "rb") as handle:+ self.topics = pickle.load(handle)++ def save(self, path):+ """+ Saves a graph at path.++ Args:+ path: path to save graph+ """++ # Save files to temporary directory and combine into TAR+ with TemporaryDirectory() as directory:+ # Save graph+ self.savegraph(f"{directory}/graph")++ # Save categories, if necessary+ if self.categories:+ with open(f"{directory}/categories", "wb") as handle:+ pickle.dump(self.categories, handle, protocol=__pickle__)++ # Save topics, if necessary+ if self.topics:+ with open(f"{directory}/topics", "wb") as handle:+ pickle.dump(self.topics, handle, protocol=__pickle__)++ # Combine output as TAR+ with tarfile.open(path, "w") as tar:+ tar.add(directory, arcname=".")++ def insert(self, documents, index=0):+ """+ Insert graph nodes for each document.++ Args:+ documents: list of (id, data, tags)+ index: indexid offset, used for node ids+ """++ # Initialize graph backend+ self.initialize()++ for _, document, _ in documents:+ if isinstance(document, dict):+ # Require text or object field+ document = document.get("text", document.get("object"))++ if document is not None:+ if isinstance(document, list):+ # Join tokens as text+ document = " ".join(document)++ # Create node+ self.addnode(index, data=document)+ index += 1++ def delete(self, ids):+ """+ Deletes ids from graph.++ Args:+ ids: node ids to delete+ """++ for uid in ids:+ # Remove existing node, if it exists+ if self.hasnode(uid):+ # Delete from topics+ topic = self.attribute(uid, "topic")+ if topic and self.topics:+ self.topics[topic].remove(uid)++ # Delete node+ self.removenode(uid)++ def index(self, search, similarity):+ """+ Build relationships between graph nodes using a score-based search function.++ Args:+ search: batch search function - takes a list of queries and returns lists of (id, scores) to use as edge weights+ similarity: batch similarity function - takes a list of text and labels and returns best matches+ """++ # Add node edges+ self.addedges(self.scan(), search)++ # Label categories/topics+ if "topics" in self.config:+ self.addtopics(similarity)++ def upsert(self, search):+ """+ Adds relationships for new graph nodes using a score-based search function.++ Args:+ search: batch search function - takes a list of queries and returns lists of (id, scores) to use as edge weights+ """++ # Detect if topics processing is enabled+ hastopics = "topics" in self.config++ # Add node edges using new/updated nodes, set updated flag for topic processing, if necessary+ self.addedges(self.scan(attribute="data"), search, {"updated": True} if hastopics else None)++ # Infer topics with topics of connected nodes+ if hastopics:+ self.infertopics()++ def addedges(self, nodes, search, attributes=None):+ """+ Adds edges for a list of nodes using a score-based search function.++ Args:+ nodes: list of nodes+ search: search function to use to identify edges+ attribute: dictionary of attributes to add to each node+ """++ # Read graph parameters+ batchsize, limit, minscore = self.config.get("batchsize", 256), self.config.get("limit", 15), self.config.get("minscore", 0.1)+ approximate = self.config.get("approximate", True)++ batch = []+ for uid in nodes:+ # Get data attribute+ data = self.removeattribute(uid, "data")++ # Set text field when data is a string+ if isinstance(data, str):+ self.addattribute(uid, "text", data)++ # Add additional attributes, if specified+ if attributes:+ for field, value in attributes.items():+ self.addattribute(uid, field, value)++ # Skip nodes with existing edges when building an approximate network+ if not self.hasedge(uid) or not approximate:+ batch.append((uid, data))++ # Process batch+ if len(batch) == batchsize:+ self.addbatch(search, batch, limit, minscore)+ batch = []++ if batch:+ self.addbatch(search, batch, limit, minscore)++ def addbatch(self, search, batch, limit, minscore):+ """+ Adds batch of documents to graph. This method runs the search function for each item in batch+ and adds node edges between the input and each search result.++ Args:+ search: search function to use to identify edges+ batch: batch to add+ limit: max edges to add per node+ minscore: min score to add node edge+ """++ for x, result in enumerate(search([data for _, data in batch], limit)):+ # Get input node id+ x, _ = batch[x]++ # Add edges for each input uid and result uid pair that meets specified criteria+ for y, score in result:+ if x != y and score > minscore and not self.hasedge(x, y):+ self.addedge(x, y, weight=score)++ def addtopics(self, similarity=None):+ """+ Identifies and adds topics using community detection.++ Args:+ similarity: similarity function for labeling categories+ """++ # Clear previous topics, if any+ self.cleartopics()++ # Use community detection o get topics+ config = self.config["topics"]+ topics = Topics(config)+ self.topics = topics(self)++ # Label each topic with a higher level category+ if "categories" in config and similarity:+ self.categories = []+ results = similarity(self.topics.keys(), config["categories"])+ for result in results:+ self.categories.append(config["categories"][result[0][0]])++ # Add topic-related node attributes+ for x, topic in enumerate(self.topics):+ for r, uid in enumerate(self.topics[topic]):+ self.addattribute(uid, "topic", topic)+ self.addattribute(uid, "topicrank", r)++ if self.categories:+ self.addattribute(uid, "category", self.categories[x])++ def cleartopics(self):+ """+ Clears topic fields from all nodes.+ """++ # Clear previous topics, if any+ if self.topics:+ for uid in self.scan():+ self.removeattribute(uid, "topic")+ self.removeattribute(uid, "topicrank")++ if self.categories:+ self.removeattribute(uid, "category")++ self.topics, self.categories = None, None++ def infertopics(self):+ """+ Infers topics for all nodes with an "updated" attribute. This method analyzes the direct node+ neighbors and set the most commonly occuring topic and category for each node.+ """++ # Iterate over nodes missing topic attribute (only occurs for new nodes)+ for uid in self.scan(attribute="updated"):+ # Remove updated attribute+ self.removeattribute(uid, "updated")++ # Get list of neighboring nodes+ ids = self.edges(uid)++ # Infer topic+ topic = Counter(self.attribute(x, "topic") for x in ids).most_common(1)[0][0]+ if topic:+ # Add id to topic list and set topic attribute+ self.topics[topic].append(uid)+ self.addattribute(uid, "topic", topic)++ # Set topic rank+ self.addattribute(uid, "topicrank", len(self.topics[topic]))++ # Infer category+ category = Counter(self.attribute(x, "category") for x in ids).most_common(1)[0][0]+ self.addattribute(uid, "category", category)diff --git a/src/python/txtai/graph/factory.py b/src/python/txtai/graph/factory.pynew file mode 100644index 00000000..3c4ced69--- /dev/null+++ b/src/python/txtai/graph/factory.py@@ -0,0 +1,35 @@+"""+Factory module+"""++from .networkx import NetworkX+++class GraphFactory:+ """+ Methods to create graphs.+ """++ @staticmethod+ def create(config):+ """+ Create a graph.++ Args:+ config: graph configuration++ Returns:+ Graph+ """++ # Graph instance+ graph = None+ backend = config.get("backend", "networkx")++ # Create graph instance+ graph = NetworkX(config)++ # Store config back+ config["backend"] = backend++ return graphdiff --git a/src/python/txtai/graph/networkx.py b/src/python/txtai/graph/networkx.pynew file mode 100644index 00000000..6b10943e--- /dev/null+++ b/src/python/txtai/graph/networkx.py@@ -0,0 +1,170 @@+"""+NetworkX module+"""++import pickle++# Conditional import+try:+ import networkx as nx++ from community import community_louvain+ from networkx.algorithms.community import asyn_lpa_communities, greedy_modularity_communities++ NETWORKX = True+except ImportError:+ NETWORKX = False++from .. import __pickle__++from .base import Graph+++# pylint: disable=R0904+class NetworkX(Graph):+ """+ Graph instance backed by NetworkX.+ """++ def __init__(self, config):+ super().__init__(config)++ if not NETWORKX:+ raise ImportError('NetworkX is not available - install "graph" extra to enable')++ def create(self):+ return nx.Graph()++ def count(self):+ return self.backend.number_of_nodes()++ def scan(self, attribute=None):+ # Nodes containing an attribute+ if attribute:+ return nx.subgraph_view(self.backend, filter_node=lambda x: attribute in self.node(x))++ # Return all nodes+ return self.backend++ def node(self, uid):+ return self.backend.nodes.get(uid)++ def hasnode(self, uid):+ return self.backend.has_node(uid)++ def addnode(self, uid, **attrs):+ self.backend.add_node(uid, **attrs)++ def removenode(self, uid):+ if self.hasnode(uid):+ self.backend.remove_node(uid)++ def attribute(self, uid, field):+ return self.node(uid).get(field) if self.hasnode(uid) else None++ def addattribute(self, uid, field, value):+ if self.hasnode(uid):+ self.node(uid)[field] = value++ def removeattribute(self, uid, field):+ return self.node(uid).pop(field, None) if self.hasnode(uid) else None++ def edgecount(self):+ return self.backend.number_of_edges()++ def edges(self, uid):+ edges = self.backend.adj.get(uid)+ if edges:+ return dict(sorted(edges.items(), key=lambda x: x[1]["weight"]), reverse="True").keys()++ return None++ def hasedge(self, source, target=None):+ if not target:+ edges = self.backend.adj.get(source)+ return len(edges) > 0 if edges else False++ return self.backend.has_edge(source, target)++ def addedge(self, source, target, **attrs):+ self.backend.add_edge(source, target, **attrs)++ def centrality(self):+ rank = nx.degree_centrality(self.backend)+ return dict(sorted(rank.items(), key=lambda x: x[1], reverse=True))++ def pagerank(self):+ rank = nx.pagerank(self.backend, weight="weight")+ return dict(sorted(rank.items(), key=lambda x: x[1], reverse=True))++ def showpath(self, source, target):+ # pylint: disable=E1121+ return nx.shortest_path(self.backend, source, target, self.distance)++ def communities(self, config):+ # Get community detection algorithm+ algorithm = config.get("algorithm")++ if algorithm == "greedy":+ communities = greedy_modularity_communities(self.backend, weight="weight", resolution=config.get("resolution", 100))+ elif algorithm == "lpa":+ communities = asyn_lpa_communities(self.backend, weight="weight", seed=0)+ else:+ communities = self.louvain(config)++ return communities++ def loadgraph(self, path):+ # Load graph network+ with open(path, "rb") as handle:+ self.backend = pickle.load(handle)++ def savegraph(self, path):+ # Save graph+ with open(path, "wb") as handle:+ pickle.dump(self.backend, handle, protocol=__pickle__)++ def louvain(self, config):+ """+ Runs the Louvain community detection algorithm.++ Args:+ config: topic configuration++ Returns:+ dictionary of {topic name:[ids]}+ """++ # Partition level to use+ level = config.get("level", "best")++ # Run community detection+ results = community_louvain.generate_dendrogram(self.backend, weight="weight", resolution=config.get("resolution", 100), random_state=0)++ # Get partition level (first or best)+ results = results[0] if level == "first" else community_louvain.partition_at_level(results, len(results) - 1)++ # Build mapping of community to list of ids+ communities = {}+ for k, v in results.items():+ communities[v] = [k] if v not in communities else communities[v] + [k]++ return communities.values()++ # pylint: disable=W0613+ def distance(self, source, target, attrs):+ """+ Computes distance between source and target nodes using weight.++ Args:+ source: source node+ target: target node+ attrs: edge attributes++ Returns:+ distance between source and target+ """++ # Distance is 1 - score. Skip minimal distances as they are near duplicates.+ distance = max(1.0 - attrs["weight"], 0.0)+ return distance if distance >= 0.15 else 1.00diff --git a/src/python/txtai/graph/topics.py b/src/python/txtai/graph/topics.pynew file mode 100644index 00000000..a84215e8--- /dev/null+++ b/src/python/txtai/graph/topics.py@@ -0,0 +1,164 @@+"""+Topics module+"""++import re++from ..pipeline import Tokenizer+from ..scoring import ScoringFactory+++class Topics:+ """+ Topic modeling using community detection.+ """++ def __init__(self, config):+ """+ Creates a new Topics instance.++ Args:+ config: topic configuration+ """++ self.config = config+ self.pattern = re.compile(r"(?u)\b\w\w+\b")+ self.tokenizer = Tokenizer()++ def __call__(self, graph):+ """+ Runs topic modeling for input graph.++ Args:+ graph: Graph instance++ Returns:+ dictionary of {topic name: [ids]}+ """++ # Detect communities+ communities = graph.communities(self.config)++ # Sort by community size, largest to smallest+ communities = sorted(communities, key=len, reverse=True)++ # Calculate centrality of graph+ centrality = graph.centrality()++ # Score communities and generate topn terms+ topics = [self.score(graph, x, community, centrality) for x, community in enumerate(communities)]++ # Merge duplicate topics and return+ return self.merge(topics)++ def score(self, graph, index, community, centrality):+ """+ Scores a community of nodes and generates the topn terms in the community.++ Args:+ graph: Graph instance+ index: community index+ community: community of nodes+ centrality: node centrality scores++ Returns:+ (topn topic terms, topic ids sorted by score descending)+ """++ # Tokenize input and build scoring index+ scoring = ScoringFactory.create({"method": self.config.get("labels", "bm25"), "terms": True})+ scoring.index(((node, self.tokenize(graph, node), None) for node in community))++ # Check if scoring index has data+ if scoring.idf:+ # Sort by most commonly occurring terms (i.e. lowest score)+ idf = sorted(scoring.idf, key=scoring.idf.get)++ # Term count for generating topic labels+ topn = self.config.get("terms", 4)++ # Get topn terms+ terms = self.topn(idf, topn)++ # Sort community by score descending+ community = [uid for uid, _ in scoring.search(terms, len(community))]+ else:+ # No text found for topic, generate topic name+ terms = ["topic", str(index)]++ # Sort community by centrality scores+ community = sorted(community, key=lambda x: centrality[x], reverse=True)++ return (terms, community)++ def tokenize(self, graph, node):+ """+ Tokenizes node text.++ Args:+ graph: Graph instance+ node: node uid++ Returns:+ list of node tokens+ """++ text = graph.attribute(node, "text")+ return self.pattern.findall(text.lower()) if text else []++ def topn(self, terms, n):+ """+ Gets topn terms.++ Args:+ terms: list of terms+ n: topn++ Returns:+ topn terms+ """++ topn = []++ for term in terms:+ # Add terms that pass tokenization rules+ if self.tokenizer(term):+ topn.append(term)++ # Break once topn terms collected+ if len(topn) == n:+ break++ return topn++ def merge(self, topics):+ """+ Merges duplicate topics++ Args:+ topics: list of (topn terms, topic ids)++ Returns:+ dictionary of {topic name:[ids]}+ """++ merge, termslist = {}, {}++ for terms, uids in topics:+ # Use topic terms as key+ key = frozenset(terms)++ # Add key to merged topics, if necessary+ if key not in merge:+ merge[key], termslist[key] = [], terms++ # Merge communities+ merge[key].extend(uids)++ # Sort communities largest to smallest since the order could have changed with merges+ results = {}+ for k, v in sorted(merge.items(), key=lambda x: len(x[1]), reverse=True):+ # Create composite string key using topic terms and store ids+ results["_".join(termslist[k])] = v++ return resultsdiff --git a/test/python/testgraph.py b/test/python/testgraph.pynew file mode 100644index 00000000..6d95f185--- /dev/null+++ b/test/python/testgraph.py@@ -0,0 +1,229 @@+"""+Graph module tests+"""++import os+import itertools+import tempfile+import unittest++from txtai.embeddings import Embeddings+from txtai.graph import Graph, GraphFactory+++class TestGraph(unittest.TestCase):+ """+ Graph tests.+ """++ @classmethod+ def setUpClass(cls):+ """+ Initialize test data.+ """++ cls.data = [+ "US tops 5 million confirmed virus cases",+ "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",+ "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",+ "The National Park Service warns against sacrificing slower friends in a bear attack",+ "Maine man wins $1M from $25 lottery ticket",+ "Make huge profits without work, earn up to $100,000 a day",+ ]++ cls.config = {+ "path": "sentence-transformers/nli-mpnet-base-v2",+ "content": True,+ "functions": [{"name": "graph", "function": "graph.attribute"}],+ "expressions": [+ {"name": "category", "expression": "graph(indexid, 'category')"},+ {"name": "topic", "expression": "graph(indexid, 'topic')"},+ {"name": "topicrank", "expression": "graph(indexid, 'topicrank')"},+ ],+ "graph": {"limit": 5, "minscore": 0.2, "batchsize": 4, "approximate": False, "topics": {"categories": ["News"]}},+ }++ # Create embeddings instance+ cls.embeddings = Embeddings(cls.config)++ def testAnalysis(self):+ """+ Test analysis methods+ """++ # Create an index for the list of text+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])++ # Graph centrality+ graph = self.embeddings.graph+ centrality = graph.centrality()+ self.assertEqual(list(centrality.keys())[0], 5)++ # Page Rank+ pagerank = graph.pagerank()+ self.assertEqual(list(pagerank.keys())[0], 5)++ # Path between nodes+ path = graph.showpath(4, 5)+ self.assertEqual(len(path), 2)++ def testCommunity(self):+ """+ Test community detection+ """++ # Create an index for the list of text+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])++ # Get graph reference+ graph = self.embeddings.graph++ # Rebuild topics with updated graph settings+ graph.config = {"topics": {"algorithm": "greedy"}}+ graph.addtopics()+ self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 6)++ graph.config = {"topics": {"algorithm": "lpa"}}+ graph.addtopics()+ self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 4)++ def testDelete(self):+ """+ Test delete+ """++ # Create an index for the list of text+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])++ # Delete row+ self.embeddings.delete([4])++ # Validate counts+ graph = self.embeddings.graph+ self.assertEqual(graph.count(), 5)+ self.assertEqual(graph.edgecount(), 1)+ self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 5)+ self.assertEqual(len(graph.categories), 6)++ def testFunction(self):+ """+ Test running graph functions with SQL+ """++ # Create an index for the list of text+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])++ # Test similar+ result = self.embeddings.search("select category, topic, topicrank from txtai where id = 0", 1)[0]++ # Check columns have a value+ self.assertIsNotNone(result["category"])+ self.assertIsNotNone(result["topic"])+ self.assertIsNotNone(result["topicrank"])++ def testIndex(self):+ """+ Test index+ """++ # Create an index for the list of text+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])++ # Validate counts+ graph = self.embeddings.graph+ self.assertEqual(graph.count(), 6)+ self.assertEqual(graph.edgecount(), 2)+ self.assertEqual(len(graph.topics), 6)+ self.assertEqual(len(graph.categories), 6)++ def testNotImplemented(self):+ """+ Tests exceptions for non-implemented methods+ """++ graph = Graph({})++ self.assertRaises(NotImplementedError, graph.create)+ self.assertRaises(NotImplementedError, graph.count)+ self.assertRaises(NotImplementedError, graph.scan, None)+ self.assertRaises(NotImplementedError, graph.node, None)+ self.assertRaises(NotImplementedError, graph.addnode, None)+ self.assertRaises(NotImplementedError, graph.removenode, None)+ self.assertRaises(NotImplementedError, graph.hasnode, None)+ self.assertRaises(NotImplementedError, graph.attribute, None, None)+ self.assertRaises(NotImplementedError, graph.addattribute, None, None, None)+ self.assertRaises(NotImplementedError, graph.removeattribute, None, None)+ self.assertRaises(NotImplementedError, graph.edgecount)+ self.assertRaises(NotImplementedError, graph.edges, None)+ self.assertRaises(NotImplementedError, graph.addedge, None, None)+ self.assertRaises(NotImplementedError, graph.hasedge, None, None)+ self.assertRaises(NotImplementedError, graph.centrality)+ self.assertRaises(NotImplementedError, graph.pagerank)+ self.assertRaises(NotImplementedError, graph.showpath, None, None)+ self.assertRaises(NotImplementedError, graph.communities, None)+ self.assertRaises(NotImplementedError, graph.loadgraph, None)+ self.assertRaises(NotImplementedError, graph.savegraph, None)++ def testSave(self):+ """+ Test save+ """++ # Create an index for the list of text+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])++ # Generate temp file path+ index = os.path.join(tempfile.gettempdir(), "graph")++ # Save and reload index+ self.embeddings.save(index)+ self.embeddings.load(index)++ # Validate counts+ graph = self.embeddings.graph+ self.assertEqual(graph.count(), 6)+ self.assertEqual(graph.edgecount(), 2)+ self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 6)+ self.assertEqual(len(graph.categories), 6)++ def testSimple(self):+ """+ Tests creating a simple graph+ """++ graph = GraphFactory.create({"topics": {}})++ # Initialize the graph+ graph.initialize()++ for x in range(5):+ graph.addnode(x)++ for x, y in itertools.combinations(range(5), 2):+ graph.addedge(x, y)++ # Validate counts+ self.assertEqual(graph.count(), 5)+ self.assertEqual(graph.edgecount(), 10)++ # Test missing edge+ self.assertIsNone(graph.edges(100))++ # Test topics with no text+ graph.addtopics()+ self.assertEqual(len(graph.topics), 5)++ def testUpsert(self):+ """+ Test upsert+ """++ # Update data+ self.embeddings.upsert([(0, {"text": "Canadian ice shelf collapses".split()}, None)])++ # Validate counts+ graph = self.embeddings.graph+ self.assertEqual(graph.count(), 6)+ self.assertEqual(graph.edgecount(), 2)+ self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 6)+ self.assertEqual(len(graph.categories), 6)diff --git a/test/python/testoptional.py b/test/python/testoptional.pyindex cc868dfa..22e3d578 100644--- a/test/python/testoptional.py+++ b/test/python/testoptional.py@@ -29,6 +29,7 @@ class TestOptional(unittest.TestCase):"imagehash","nltk","libcloud.storage.providers",+ "networkx","onnxmltools","onnxruntime","pandas",@@ -108,6 +109,16 @@ class TestOptional(unittest.TestCase):with self.assertRaises(ImportError):ImageEncoder()+ def testGraph(self):+ """+ Test missing graph dependencies+ """++ from txtai.graph import GraphFactory++ with self.assertRaises(ImportError):+ GraphFactory.create({"backend": "networkx"})+def testModel(self):"""Test missing model dependencies