Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add a semantic graph component, closes #339

davidmezzetti8bd6bc7b388e
diff --git a/setup.py b/setup.py
index 2b80eb22..fc93923f 100644
--- a/setup.py
+++ b/setup.py
@@ -25,6 +25,8 @@ extras["console"] = ["rich>=12.0.1"]
extras["database"] = ["pillow>=7.1.2"]
+extras["graph"] = ["networkx>=2.6.3", "python-louvain>=0.16"]
+
extras["model"] = ["onnxruntime>=1.8.1"]
extras["pipeline"] = [
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.py
index f692547f..59466cbb 100644
--- a/src/python/txtai/embeddings/base.py
+++ b/src/python/txtai/embeddings/base.py
@@ -14,11 +14,13 @@ from .. import __pickle__
from ..ann import ANNFactory
from ..database import DatabaseFactory
+from ..graph import GraphFactory
from ..scoring import ScoringFactory
from ..vectors import VectorsFactory
from .archive import Archive
from .explain import Explain
+from .functions import Functions
from .reducer import Reducer
from .query import Query
from .search import Search
@@ -46,7 +48,7 @@ class Embeddings:
# Index configuration
self.config = None
- # Dimensionality reduction and scoring models - word vectors only
+ # Dimensionality reduction and scoring index - word vectors only
self.reducer, self.scoring = None, None
# Embeddings vector model - transforms data into similarity vectors
@@ -58,6 +60,9 @@ class Embeddings:
# Document database
self.database = None
+ # Graph network
+ self.graph = None
+
# Query model
self.query = None
@@ -95,6 +100,9 @@ class Embeddings:
# Reset archive since this is a new index
self.archive = None
+ # Create graph, if necessary
+ self.graph = self.creategraph()
+
# Create transform action
transform = Transform(self, Action.REINDEX if reindex else Action.INDEX)
@@ -123,6 +131,10 @@ class Embeddings:
if not reindex and not self.database:
self.config["ids"] = ids
+ # Index graph, if necessary
+ if self.graph:
+ self.graph.index(Search(self, True), self.batchsimilarity)
+
def upsert(self, documents):
"""
Runs an embeddings upsert operation. If the index exists, new data is
@@ -155,6 +167,10 @@ class Embeddings:
if not self.database:
self.config["ids"] = self.config["ids"] + ids
+ # Graph upsert, if necessary
+ if self.graph:
+ self.graph.upsert(Search(self, True))
+
def delete(self, ids):
"""
Deletes from an embeddings index. Returns list of ids deleted.
@@ -200,6 +216,10 @@ class Embeddings:
# Delete ids from index
self.ann.delete(indices)
+ # Delete ids from graph
+ if self.graph:
+ self.graph.delete(indices)
+
return deletes
def reindex(self, config, columns=None, function=None):
@@ -427,7 +447,7 @@ class Embeddings:
self.reducer = Reducer()
self.reducer.load(f"{path}/lsa")
- # Embedding scoring model - word vectors only
+ # Embedding scoring index - word vectors only
if self.config.get("scoring"):
self.scoring = ScoringFactory.create(self.config["scoring"])
self.scoring.load(f"{path}/scoring")
@@ -443,6 +463,11 @@ class Embeddings:
if self.database:
self.database.load(f"{path}/documents")
+ # Graph network - stores relationships
+ self.graph = self.creategraph()
+ if self.graph:
+ self.graph.load(f"{path}/graph")
+
def save(self, path, cloud=None):
"""
Saves an index in a directory at path unless path ends with tar.gz, tar.bz2, tar.xz or zip.
@@ -477,7 +502,7 @@ class Embeddings:
if self.reducer:
self.reducer.save(f"{path}/lsa")
- # Save embedding scoring model (word vectors only)
+ # Save embedding scoring index (word vectors only)
if self.scoring:
self.scoring.save(f"{path}/scoring")
@@ -485,6 +510,10 @@ class Embeddings:
if self.database:
self.database.save(f"{path}/documents")
+ # Save graph
+ if self.graph:
+ self.graph.save(f"{path}/graph")
+
# If this is an archive, save it
if apath:
self.archive.save(apath, cloud)
@@ -494,7 +523,8 @@ class Embeddings:
Closes this embeddings index and frees all resources.
"""
- self.config, self.reducer, self.scoring, self.model, self.ann, self.query, self.archive = None, None, None, None, None, None, None
+ self.config, self.reducer, self.scoring, self.model = None, None, None, None
+ self.ann, self.graph, self.query, self.archive = None, None, None, None
# Close database connection if open
if self.database:
@@ -597,8 +627,24 @@ class Embeddings:
if self.database:
self.database.close()
+ config = self.config.copy()
+
+ # Resolve callable functions
+ if "functions" in config:
+ config["functions"] = Functions(self)(config)
+
# Create database from config and return
- return DatabaseFactory.create(self.config)
+ return DatabaseFactory.create(config)
+
+ def creategraph(self):
+ """
+ Creates a graph from config.
+
+ Returns:
+ new graph, if enabled in config
+ """
+
+ return GraphFactory.create(self.config["graph"]) if "graph" in self.config else None
def normalize(self, embeddings):
"""
diff --git a/src/python/txtai/embeddings/search.py b/src/python/txtai/embeddings/search.py
index 74f77340..2f8884f2 100644
--- a/src/python/txtai/embeddings/search.py
+++ b/src/python/txtai/embeddings/search.py
@@ -15,15 +15,17 @@ class Search:
Executes a batch search action. A search can be both approximate nearest neighbor and/or database driven.
"""
- def __init__(self, embeddings):
+ def __init__(self, embeddings, indexids=False):
"""
Creates a new search action.
Args:
embeddings: embeddings instance
+ indexids: searches return indexids when True, otherwise run standard search
"""
self.embeddings = embeddings
+ self.indexids = indexids
# Alias embeddings attributes
self.config = embeddings.config
@@ -49,7 +51,7 @@ class Search:
if not self.ann:
return [[]] * len(queries)
- if self.database:
+ if not self.indexids and self.database:
return self.dbsearch(queries, limit)
# Default: execute an approximate nearest neighbor search
@@ -77,7 +79,7 @@ class Search:
results = [[(i, score) for i, score in r if score > 0] for r in results]
# Map indexids to ids if "ids" available
- if "ids" in self.config:
+ if not self.indexids and "ids" in self.config:
lookup = self.config["ids"]
return [[(lookup[i], score) for i, score in r] for r in results]
diff --git a/src/python/txtai/embeddings/transform.py b/src/python/txtai/embeddings/transform.py
index 7237da23..d9ddcfbe 100644
--- a/src/python/txtai/embeddings/transform.py
+++ b/src/python/txtai/embeddings/transform.py
@@ -31,6 +31,7 @@ class Transform:
self.delete = self.embeddings.delete
self.model = self.embeddings.model
self.database = self.embeddings.database
+ self.graph = self.embeddings.graph
# Get config parameters
self.offset = self.embeddings.config.get("offset", 0) if self.action == Action.UPSERT else 0
@@ -139,7 +140,13 @@ class Transform:
# Load batch into database except if this is a reindex
if self.database and self.action != Action.REINDEX:
self.database.insert(batch, self.offset)
- self.offset += offset
+
+ # Load batch into graph
+ if self.graph:
+ self.graph.insert(batch, self.offset)
+
+ # Increment offset
+ self.offset += offset
class Action(Enum):
diff --git a/src/python/txtai/graph/__init__.py b/src/python/txtai/graph/__init__.py
new file mode 100644
index 00000000..80019ad6
--- /dev/null
+++ b/src/python/txtai/graph/__init__.py
@@ -0,0 +1,8 @@
+"""
+Graph imports
+"""
+
+from .base import Graph
+from .factory import GraphFactory
+from .networkx import NetworkX
+from .topics import Topics
diff --git a/src/python/txtai/graph/base.py b/src/python/txtai/graph/base.py
new file mode 100644
index 00000000..af15d93f
--- /dev/null
+++ b/src/python/txtai/graph/base.py
@@ -0,0 +1,552 @@
+"""
+Graph module
+"""
+
+import os
+import pickle
+import tarfile
+
+from collections import Counter
+from tempfile import TemporaryDirectory
+
+from .. import __pickle__
+
+from .topics import Topics
+
+# pylint: disable=R0904
+class Graph:
+ """
+ Base class for Graph instances.
+ """
+
+ def __init__(self, config):
+ """
+ Creates a new Graph.
+
+ Args:
+ config: graph configuration
+ """
+
+ # Graph configuration
+ self.config = config
+
+ # Graph backend
+ self.backend = None
+
+ # Topic modeling
+ self.categories = None
+ self.topics = None
+
+ def create(self):
+ """
+ Creates the graph network.
+ """
+
+ raise NotImplementedError
+
+ def count(self):
+ """
+ Returns the total number of nodes in graph.
+
+ Returns:
+ total nodes in graph
+ """
+
+ raise NotImplementedError
+
+ def scan(self, attribute=None):
+ """
+ Iterates over nodes that match a criteria. If no criteria specified, all nodes
+ are returned.
+
+ Args:
+ attribute: if specified, nodes having this attribute are returned
+
+ Returns:
+ node iterator
+ """
+
+ raise NotImplementedError
+
+ def node(self, uid):
+ """
+ Get node with id uid. Returns None if not found.
+
+ Args:
+ uid: node id
+
+ Returns:
+ graph node
+ """
+
+ raise NotImplementedError
+
+ def addnode(self, uid, **attrs):
+ """
+ Adds a node to the graph.
+
+ Args:
+ uid: node id
+ attrs: node attributes
+ """
+
+ raise NotImplementedError
+
+ def removenode(self, uid):
+ """
+ Removes a node and all it's edges from graph.
+
+ Args:
+ uid: node id
+ """
+
+ raise NotImplementedError
+
+ def hasnode(self, uid):
+ """
+ Returns True if node found, False otherwise.
+
+ Args:
+ uid: node id
+
+ Returns:
+ True if node found, False otherwise
+ """
+
+ raise NotImplementedError
+
+ def attribute(self, uid, field):
+ """
+ Gets a node attribute.
+
+ Args:
+ uid: node id
+ field: attribute name
+
+ Returns:
+ attribute value
+ """
+
+ raise NotImplementedError
+
+ def addattribute(self, uid, field, value):
+ """
+ Adds an attribute to node.
+
+ Args:
+ uid: node id
+ field: attribute name
+ value: attribute value
+ """
+
+ raise NotImplementedError
+
+ def removeattribute(self, uid, field):
+ """
+ Removes an attribute from node.
+
+ Args:
+ uid: node id
+ field: attribute name
+
+ Returns:
+ attribute value or None if not present
+ """
+
+ raise NotImplementedError
+
+ def edgecount(self):
+ """
+ Returns the total number of edges.
+
+ Returns:
+ total number of edges in graph
+ """
+
+ raise NotImplementedError
+
+ def edges(self, uid):
+ """
+ Gets edges of node by id.
+
+ Args:
+ uid: node id
+
+ Returns:
+ list of edge node ids
+ """
+
+ raise NotImplementedError
+
+ def addedge(self, source, target, **attrs):
+ """
+ Adds an edge to graph.
+
+ Args:
+ source: node 1 id
+ target: node 2 id
+ """
+
+ raise NotImplementedError
+
+ def hasedge(self, source, target=None):
+ """
+ Returns True if edge found, False otherwise. If target is None, this method
+ returns True if any edge is found.
+
+ Args:
+ source: node 1 id
+ target: node 2 id
+
+ Returns:
+ True if edge found, False otherwise
+ """
+
+ raise NotImplementedError
+
+ def centrality(self):
+ """
+ Runs a centrality algorithm on the graph.
+
+ Returns:
+ dict of {node id: centrality score}
+ """
+
+ raise NotImplementedError
+
+ def pagerank(self):
+ """
+ Runs the pagerank algorithm on the graph.
+
+ Returns:
+ dict of {node id, page rank score}
+ """
+
+ raise NotImplementedError
+
+ def showpath(self, source, target):
+ """
+ Gets the shortest path between source and target.
+
+ Args:
+ source: start node id
+ target: end node id
+
+ Returns:
+ list of node ids representing the shortest path
+ """
+
+ raise NotImplementedError
+
+ def communities(self, config):
+ """
+ Run community detection on the graph.
+
+ Args:
+ config: configuration
+
+ Returns:
+ dictionary of {topic name:[ids]}
+ """
+
+ raise NotImplementedError
+
+ def loadgraph(self, path):
+ """
+ Loads a graph backend at path.
+
+ Args:
+ path: path to graph backend
+ """
+
+ raise NotImplementedError
+
+ def savegraph(self, path):
+ """
+ Saves graph backend to path.
+
+ Args:
+ path: path to save graph backend
+ """
+
+ raise NotImplementedError
+
+ def initialize(self):
+ """
+ Initialize graph instance.
+ """
+
+ if not self.backend:
+ self.backend = self.create()
+
+ def load(self, path):
+ """
+ Loads a graph at path.
+
+ Args:
+ path: path to graph
+ """
+
+ # Extract files to temporary directory and load content
+ with TemporaryDirectory() as directory:
+ # Extract tar contents
+ with tarfile.open(path, "r") as tar:
+ tar.extractall(directory)
+
+ # Load graph backend
+ self.loadgraph(f"{directory}/graph")
+
+ # Load categories, if necessary
+ path = f"{directory}/categories"
+ if os.path.exists(path):
+ with open(path, "rb") as handle:
+ self.categories = pickle.load(handle)
+
+ # Load topics, if necessary
+ path = f"{directory}/topics"
+ if os.path.exists(path):
+ with open(path, "rb") as handle:
+ self.topics = pickle.load(handle)
+
+ def save(self, path):
+ """
+ Saves a graph at path.
+
+ Args:
+ path: path to save graph
+ """
+
+ # Save files to temporary directory and combine into TAR
+ with TemporaryDirectory() as directory:
+ # Save graph
+ self.savegraph(f"{directory}/graph")
+
+ # Save categories, if necessary
+ if self.categories:
+ with open(f"{directory}/categories", "wb") as handle:
+ pickle.dump(self.categories, handle, protocol=__pickle__)
+
+ # Save topics, if necessary
+ if self.topics:
+ with open(f"{directory}/topics", "wb") as handle:
+ pickle.dump(self.topics, handle, protocol=__pickle__)
+
+ # Combine output as TAR
+ with tarfile.open(path, "w") as tar:
+ tar.add(directory, arcname=".")
+
+ def insert(self, documents, index=0):
+ """
+ Insert graph nodes for each document.
+
+ Args:
+ documents: list of (id, data, tags)
+ index: indexid offset, used for node ids
+ """
+
+ # Initialize graph backend
+ self.initialize()
+
+ for _, document, _ in documents:
+ if isinstance(document, dict):
+ # Require text or object field
+ document = document.get("text", document.get("object"))
+
+ if document is not None:
+ if isinstance(document, list):
+ # Join tokens as text
+ document = " ".join(document)
+
+ # Create node
+ self.addnode(index, data=document)
+ index += 1
+
+ def delete(self, ids):
+ """
+ Deletes ids from graph.
+
+ Args:
+ ids: node ids to delete
+ """
+
+ for uid in ids:
+ # Remove existing node, if it exists
+ if self.hasnode(uid):
+ # Delete from topics
+ topic = self.attribute(uid, "topic")
+ if topic and self.topics:
+ self.topics[topic].remove(uid)
+
+ # Delete node
+ self.removenode(uid)
+
+ def index(self, search, similarity):
+ """
+ Build relationships between graph nodes using a score-based search function.
+
+ Args:
+ search: batch search function - takes a list of queries and returns lists of (id, scores) to use as edge weights
+ similarity: batch similarity function - takes a list of text and labels and returns best matches
+ """
+
+ # Add node edges
+ self.addedges(self.scan(), search)
+
+ # Label categories/topics
+ if "topics" in self.config:
+ self.addtopics(similarity)
+
+ def upsert(self, search):
+ """
+ Adds relationships for new graph nodes using a score-based search function.
+
+ Args:
+ search: batch search function - takes a list of queries and returns lists of (id, scores) to use as edge weights
+ """
+
+ # Detect if topics processing is enabled
+ hastopics = "topics" in self.config
+
+ # Add node edges using new/updated nodes, set updated flag for topic processing, if necessary
+ self.addedges(self.scan(attribute="data"), search, {"updated": True} if hastopics else None)
+
+ # Infer topics with topics of connected nodes
+ if hastopics:
+ self.infertopics()
+
+ def addedges(self, nodes, search, attributes=None):
+ """
+ Adds edges for a list of nodes using a score-based search function.
+
+ Args:
+ nodes: list of nodes
+ search: search function to use to identify edges
+ attribute: dictionary of attributes to add to each node
+ """
+
+ # Read graph parameters
+ batchsize, limit, minscore = self.config.get("batchsize", 256), self.config.get("limit", 15), self.config.get("minscore", 0.1)
+ approximate = self.config.get("approximate", True)
+
+ batch = []
+ for uid in nodes:
+ # Get data attribute
+ data = self.removeattribute(uid, "data")
+
+ # Set text field when data is a string
+ if isinstance(data, str):
+ self.addattribute(uid, "text", data)
+
+ # Add additional attributes, if specified
+ if attributes:
+ for field, value in attributes.items():
+ self.addattribute(uid, field, value)
+
+ # Skip nodes with existing edges when building an approximate network
+ if not self.hasedge(uid) or not approximate:
+ batch.append((uid, data))
+
+ # Process batch
+ if len(batch) == batchsize:
+ self.addbatch(search, batch, limit, minscore)
+ batch = []
+
+ if batch:
+ self.addbatch(search, batch, limit, minscore)
+
+ def addbatch(self, search, batch, limit, minscore):
+ """
+ Adds batch of documents to graph. This method runs the search function for each item in batch
+ and adds node edges between the input and each search result.
+
+ Args:
+ search: search function to use to identify edges
+ batch: batch to add
+ limit: max edges to add per node
+ minscore: min score to add node edge
+ """
+
+ for x, result in enumerate(search([data for _, data in batch], limit)):
+ # Get input node id
+ x, _ = batch[x]
+
+ # Add edges for each input uid and result uid pair that meets specified criteria
+ for y, score in result:
+ if x != y and score > minscore and not self.hasedge(x, y):
+ self.addedge(x, y, weight=score)
+
+ def addtopics(self, similarity=None):
+ """
+ Identifies and adds topics using community detection.
+
+ Args:
+ similarity: similarity function for labeling categories
+ """
+
+ # Clear previous topics, if any
+ self.cleartopics()
+
+ # Use community detection o get topics
+ config = self.config["topics"]
+ topics = Topics(config)
+ self.topics = topics(self)
+
+ # Label each topic with a higher level category
+ if "categories" in config and similarity:
+ self.categories = []
+ results = similarity(self.topics.keys(), config["categories"])
+ for result in results:
+ self.categories.append(config["categories"][result[0][0]])
+
+ # Add topic-related node attributes
+ for x, topic in enumerate(self.topics):
+ for r, uid in enumerate(self.topics[topic]):
+ self.addattribute(uid, "topic", topic)
+ self.addattribute(uid, "topicrank", r)
+
+ if self.categories:
+ self.addattribute(uid, "category", self.categories[x])
+
+ def cleartopics(self):
+ """
+ Clears topic fields from all nodes.
+ """
+
+ # Clear previous topics, if any
+ if self.topics:
+ for uid in self.scan():
+ self.removeattribute(uid, "topic")
+ self.removeattribute(uid, "topicrank")
+
+ if self.categories:
+ self.removeattribute(uid, "category")
+
+ self.topics, self.categories = None, None
+
+ def infertopics(self):
+ """
+ Infers topics for all nodes with an "updated" attribute. This method analyzes the direct node
+ neighbors and set the most commonly occuring topic and category for each node.
+ """
+
+ # Iterate over nodes missing topic attribute (only occurs for new nodes)
+ for uid in self.scan(attribute="updated"):
+ # Remove updated attribute
+ self.removeattribute(uid, "updated")
+
+ # Get list of neighboring nodes
+ ids = self.edges(uid)
+
+ # Infer topic
+ topic = Counter(self.attribute(x, "topic") for x in ids).most_common(1)[0][0]
+ if topic:
+ # Add id to topic list and set topic attribute
+ self.topics[topic].append(uid)
+ self.addattribute(uid, "topic", topic)
+
+ # Set topic rank
+ self.addattribute(uid, "topicrank", len(self.topics[topic]))
+
+ # Infer category
+ category = Counter(self.attribute(x, "category") for x in ids).most_common(1)[0][0]
+ self.addattribute(uid, "category", category)
diff --git a/src/python/txtai/graph/factory.py b/src/python/txtai/graph/factory.py
new file mode 100644
index 00000000..3c4ced69
--- /dev/null
+++ b/src/python/txtai/graph/factory.py
@@ -0,0 +1,35 @@
+"""
+Factory module
+"""
+
+from .networkx import NetworkX
+
+
+class GraphFactory:
+ """
+ Methods to create graphs.
+ """
+
+ @staticmethod
+ def create(config):
+ """
+ Create a graph.
+
+ Args:
+ config: graph configuration
+
+ Returns:
+ Graph
+ """
+
+ # Graph instance
+ graph = None
+ backend = config.get("backend", "networkx")
+
+ # Create graph instance
+ graph = NetworkX(config)
+
+ # Store config back
+ config["backend"] = backend
+
+ return graph
diff --git a/src/python/txtai/graph/networkx.py b/src/python/txtai/graph/networkx.py
new file mode 100644
index 00000000..6b10943e
--- /dev/null
+++ b/src/python/txtai/graph/networkx.py
@@ -0,0 +1,170 @@
+"""
+NetworkX module
+"""
+
+import pickle
+
+# Conditional import
+try:
+ import networkx as nx
+
+ from community import community_louvain
+ from networkx.algorithms.community import asyn_lpa_communities, greedy_modularity_communities
+
+ NETWORKX = True
+except ImportError:
+ NETWORKX = False
+
+from .. import __pickle__
+
+from .base import Graph
+
+
+# pylint: disable=R0904
+class NetworkX(Graph):
+ """
+ Graph instance backed by NetworkX.
+ """
+
+ def __init__(self, config):
+ super().__init__(config)
+
+ if not NETWORKX:
+ raise ImportError('NetworkX is not available - install "graph" extra to enable')
+
+ def create(self):
+ return nx.Graph()
+
+ def count(self):
+ return self.backend.number_of_nodes()
+
+ def scan(self, attribute=None):
+ # Nodes containing an attribute
+ if attribute:
+ return nx.subgraph_view(self.backend, filter_node=lambda x: attribute in self.node(x))
+
+ # Return all nodes
+ return self.backend
+
+ def node(self, uid):
+ return self.backend.nodes.get(uid)
+
+ def hasnode(self, uid):
+ return self.backend.has_node(uid)
+
+ def addnode(self, uid, **attrs):
+ self.backend.add_node(uid, **attrs)
+
+ def removenode(self, uid):
+ if self.hasnode(uid):
+ self.backend.remove_node(uid)
+
+ def attribute(self, uid, field):
+ return self.node(uid).get(field) if self.hasnode(uid) else None
+
+ def addattribute(self, uid, field, value):
+ if self.hasnode(uid):
+ self.node(uid)[field] = value
+
+ def removeattribute(self, uid, field):
+ return self.node(uid).pop(field, None) if self.hasnode(uid) else None
+
+ def edgecount(self):
+ return self.backend.number_of_edges()
+
+ def edges(self, uid):
+ edges = self.backend.adj.get(uid)
+ if edges:
+ return dict(sorted(edges.items(), key=lambda x: x[1]["weight"]), reverse="True").keys()
+
+ return None
+
+ def hasedge(self, source, target=None):
+ if not target:
+ edges = self.backend.adj.get(source)
+ return len(edges) > 0 if edges else False
+
+ return self.backend.has_edge(source, target)
+
+ def addedge(self, source, target, **attrs):
+ self.backend.add_edge(source, target, **attrs)
+
+ def centrality(self):
+ rank = nx.degree_centrality(self.backend)
+ return dict(sorted(rank.items(), key=lambda x: x[1], reverse=True))
+
+ def pagerank(self):
+ rank = nx.pagerank(self.backend, weight="weight")
+ return dict(sorted(rank.items(), key=lambda x: x[1], reverse=True))
+
+ def showpath(self, source, target):
+ # pylint: disable=E1121
+ return nx.shortest_path(self.backend, source, target, self.distance)
+
+ def communities(self, config):
+ # Get community detection algorithm
+ algorithm = config.get("algorithm")
+
+ if algorithm == "greedy":
+ communities = greedy_modularity_communities(self.backend, weight="weight", resolution=config.get("resolution", 100))
+ elif algorithm == "lpa":
+ communities = asyn_lpa_communities(self.backend, weight="weight", seed=0)
+ else:
+ communities = self.louvain(config)
+
+ return communities
+
+ def loadgraph(self, path):
+ # Load graph network
+ with open(path, "rb") as handle:
+ self.backend = pickle.load(handle)
+
+ def savegraph(self, path):
+ # Save graph
+ with open(path, "wb") as handle:
+ pickle.dump(self.backend, handle, protocol=__pickle__)
+
+ def louvain(self, config):
+ """
+ Runs the Louvain community detection algorithm.
+
+ Args:
+ config: topic configuration
+
+ Returns:
+ dictionary of {topic name:[ids]}
+ """
+
+ # Partition level to use
+ level = config.get("level", "best")
+
+ # Run community detection
+ results = community_louvain.generate_dendrogram(self.backend, weight="weight", resolution=config.get("resolution", 100), random_state=0)
+
+ # Get partition level (first or best)
+ results = results[0] if level == "first" else community_louvain.partition_at_level(results, len(results) - 1)
+
+ # Build mapping of community to list of ids
+ communities = {}
+ for k, v in results.items():
+ communities[v] = [k] if v not in communities else communities[v] + [k]
+
+ return communities.values()
+
+ # pylint: disable=W0613
+ def distance(self, source, target, attrs):
+ """
+ Computes distance between source and target nodes using weight.
+
+ Args:
+ source: source node
+ target: target node
+ attrs: edge attributes
+
+ Returns:
+ distance between source and target
+ """
+
+ # Distance is 1 - score. Skip minimal distances as they are near duplicates.
+ distance = max(1.0 - attrs["weight"], 0.0)
+ return distance if distance >= 0.15 else 1.00
diff --git a/src/python/txtai/graph/topics.py b/src/python/txtai/graph/topics.py
new file mode 100644
index 00000000..a84215e8
--- /dev/null
+++ b/src/python/txtai/graph/topics.py
@@ -0,0 +1,164 @@
+"""
+Topics module
+"""
+
+import re
+
+from ..pipeline import Tokenizer
+from ..scoring import ScoringFactory
+
+
+class Topics:
+ """
+ Topic modeling using community detection.
+ """
+
+ def __init__(self, config):
+ """
+ Creates a new Topics instance.
+
+ Args:
+ config: topic configuration
+ """
+
+ self.config = config
+ self.pattern = re.compile(r"(?u)\b\w\w+\b")
+ self.tokenizer = Tokenizer()
+
+ def __call__(self, graph):
+ """
+ Runs topic modeling for input graph.
+
+ Args:
+ graph: Graph instance
+
+ Returns:
+ dictionary of {topic name: [ids]}
+ """
+
+ # Detect communities
+ communities = graph.communities(self.config)
+
+ # Sort by community size, largest to smallest
+ communities = sorted(communities, key=len, reverse=True)
+
+ # Calculate centrality of graph
+ centrality = graph.centrality()
+
+ # Score communities and generate topn terms
+ topics = [self.score(graph, x, community, centrality) for x, community in enumerate(communities)]
+
+ # Merge duplicate topics and return
+ return self.merge(topics)
+
+ def score(self, graph, index, community, centrality):
+ """
+ Scores a community of nodes and generates the topn terms in the community.
+
+ Args:
+ graph: Graph instance
+ index: community index
+ community: community of nodes
+ centrality: node centrality scores
+
+ Returns:
+ (topn topic terms, topic ids sorted by score descending)
+ """
+
+ # Tokenize input and build scoring index
+ scoring = ScoringFactory.create({"method": self.config.get("labels", "bm25"), "terms": True})
+ scoring.index(((node, self.tokenize(graph, node), None) for node in community))
+
+ # Check if scoring index has data
+ if scoring.idf:
+ # Sort by most commonly occurring terms (i.e. lowest score)
+ idf = sorted(scoring.idf, key=scoring.idf.get)
+
+ # Term count for generating topic labels
+ topn = self.config.get("terms", 4)
+
+ # Get topn terms
+ terms = self.topn(idf, topn)
+
+ # Sort community by score descending
+ community = [uid for uid, _ in scoring.search(terms, len(community))]
+ else:
+ # No text found for topic, generate topic name
+ terms = ["topic", str(index)]
+
+ # Sort community by centrality scores
+ community = sorted(community, key=lambda x: centrality[x], reverse=True)
+
+ return (terms, community)
+
+ def tokenize(self, graph, node):
+ """
+ Tokenizes node text.
+
+ Args:
+ graph: Graph instance
+ node: node uid
+
+ Returns:
+ list of node tokens
+ """
+
+ text = graph.attribute(node, "text")
+ return self.pattern.findall(text.lower()) if text else []
+
+ def topn(self, terms, n):
+ """
+ Gets topn terms.
+
+ Args:
+ terms: list of terms
+ n: topn
+
+ Returns:
+ topn terms
+ """
+
+ topn = []
+
+ for term in terms:
+ # Add terms that pass tokenization rules
+ if self.tokenizer(term):
+ topn.append(term)
+
+ # Break once topn terms collected
+ if len(topn) == n:
+ break
+
+ return topn
+
+ def merge(self, topics):
+ """
+ Merges duplicate topics
+
+ Args:
+ topics: list of (topn terms, topic ids)
+
+ Returns:
+ dictionary of {topic name:[ids]}
+ """
+
+ merge, termslist = {}, {}
+
+ for terms, uids in topics:
+ # Use topic terms as key
+ key = frozenset(terms)
+
+ # Add key to merged topics, if necessary
+ if key not in merge:
+ merge[key], termslist[key] = [], terms
+
+ # Merge communities
+ merge[key].extend(uids)
+
+ # Sort communities largest to smallest since the order could have changed with merges
+ results = {}
+ for k, v in sorted(merge.items(), key=lambda x: len(x[1]), reverse=True):
+ # Create composite string key using topic terms and store ids
+ results["_".join(termslist[k])] = v
+
+ return results
diff --git a/test/python/testgraph.py b/test/python/testgraph.py
new file mode 100644
index 00000000..6d95f185
--- /dev/null
+++ b/test/python/testgraph.py
@@ -0,0 +1,229 @@
+"""
+Graph module tests
+"""
+
+import os
+import itertools
+import tempfile
+import unittest
+
+from txtai.embeddings import Embeddings
+from txtai.graph import Graph, GraphFactory
+
+
+class TestGraph(unittest.TestCase):
+ """
+ Graph tests.
+ """
+
+ @classmethod
+ def setUpClass(cls):
+ """
+ Initialize test data.
+ """
+
+ cls.data = [
+ "US tops 5 million confirmed virus cases",
+ "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",
+ "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",
+ "The National Park Service warns against sacrificing slower friends in a bear attack",
+ "Maine man wins $1M from $25 lottery ticket",
+ "Make huge profits without work, earn up to $100,000 a day",
+ ]
+
+ cls.config = {
+ "path": "sentence-transformers/nli-mpnet-base-v2",
+ "content": True,
+ "functions": [{"name": "graph", "function": "graph.attribute"}],
+ "expressions": [
+ {"name": "category", "expression": "graph(indexid, 'category')"},
+ {"name": "topic", "expression": "graph(indexid, 'topic')"},
+ {"name": "topicrank", "expression": "graph(indexid, 'topicrank')"},
+ ],
+ "graph": {"limit": 5, "minscore": 0.2, "batchsize": 4, "approximate": False, "topics": {"categories": ["News"]}},
+ }
+
+ # Create embeddings instance
+ cls.embeddings = Embeddings(cls.config)
+
+ def testAnalysis(self):
+ """
+ Test analysis methods
+ """
+
+ # Create an index for the list of text
+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
+
+ # Graph centrality
+ graph = self.embeddings.graph
+ centrality = graph.centrality()
+ self.assertEqual(list(centrality.keys())[0], 5)
+
+ # Page Rank
+ pagerank = graph.pagerank()
+ self.assertEqual(list(pagerank.keys())[0], 5)
+
+ # Path between nodes
+ path = graph.showpath(4, 5)
+ self.assertEqual(len(path), 2)
+
+ def testCommunity(self):
+ """
+ Test community detection
+ """
+
+ # Create an index for the list of text
+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
+
+ # Get graph reference
+ graph = self.embeddings.graph
+
+ # Rebuild topics with updated graph settings
+ graph.config = {"topics": {"algorithm": "greedy"}}
+ graph.addtopics()
+ self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 6)
+
+ graph.config = {"topics": {"algorithm": "lpa"}}
+ graph.addtopics()
+ self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 4)
+
+ def testDelete(self):
+ """
+ Test delete
+ """
+
+ # Create an index for the list of text
+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
+
+ # Delete row
+ self.embeddings.delete([4])
+
+ # Validate counts
+ graph = self.embeddings.graph
+ self.assertEqual(graph.count(), 5)
+ self.assertEqual(graph.edgecount(), 1)
+ self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 5)
+ self.assertEqual(len(graph.categories), 6)
+
+ def testFunction(self):
+ """
+ Test running graph functions with SQL
+ """
+
+ # Create an index for the list of text
+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
+
+ # Test similar
+ result = self.embeddings.search("select category, topic, topicrank from txtai where id = 0", 1)[0]
+
+ # Check columns have a value
+ self.assertIsNotNone(result["category"])
+ self.assertIsNotNone(result["topic"])
+ self.assertIsNotNone(result["topicrank"])
+
+ def testIndex(self):
+ """
+ Test index
+ """
+
+ # Create an index for the list of text
+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
+
+ # Validate counts
+ graph = self.embeddings.graph
+ self.assertEqual(graph.count(), 6)
+ self.assertEqual(graph.edgecount(), 2)
+ self.assertEqual(len(graph.topics), 6)
+ self.assertEqual(len(graph.categories), 6)
+
+ def testNotImplemented(self):
+ """
+ Tests exceptions for non-implemented methods
+ """
+
+ graph = Graph({})
+
+ self.assertRaises(NotImplementedError, graph.create)
+ self.assertRaises(NotImplementedError, graph.count)
+ self.assertRaises(NotImplementedError, graph.scan, None)
+ self.assertRaises(NotImplementedError, graph.node, None)
+ self.assertRaises(NotImplementedError, graph.addnode, None)
+ self.assertRaises(NotImplementedError, graph.removenode, None)
+ self.assertRaises(NotImplementedError, graph.hasnode, None)
+ self.assertRaises(NotImplementedError, graph.attribute, None, None)
+ self.assertRaises(NotImplementedError, graph.addattribute, None, None, None)
+ self.assertRaises(NotImplementedError, graph.removeattribute, None, None)
+ self.assertRaises(NotImplementedError, graph.edgecount)
+ self.assertRaises(NotImplementedError, graph.edges, None)
+ self.assertRaises(NotImplementedError, graph.addedge, None, None)
+ self.assertRaises(NotImplementedError, graph.hasedge, None, None)
+ self.assertRaises(NotImplementedError, graph.centrality)
+ self.assertRaises(NotImplementedError, graph.pagerank)
+ self.assertRaises(NotImplementedError, graph.showpath, None, None)
+ self.assertRaises(NotImplementedError, graph.communities, None)
+ self.assertRaises(NotImplementedError, graph.loadgraph, None)
+ self.assertRaises(NotImplementedError, graph.savegraph, None)
+
+ def testSave(self):
+ """
+ Test save
+ """
+
+ # Create an index for the list of text
+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
+
+ # Generate temp file path
+ index = os.path.join(tempfile.gettempdir(), "graph")
+
+ # Save and reload index
+ self.embeddings.save(index)
+ self.embeddings.load(index)
+
+ # Validate counts
+ graph = self.embeddings.graph
+ self.assertEqual(graph.count(), 6)
+ self.assertEqual(graph.edgecount(), 2)
+ self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 6)
+ self.assertEqual(len(graph.categories), 6)
+
+ def testSimple(self):
+ """
+ Tests creating a simple graph
+ """
+
+ graph = GraphFactory.create({"topics": {}})
+
+ # Initialize the graph
+ graph.initialize()
+
+ for x in range(5):
+ graph.addnode(x)
+
+ for x, y in itertools.combinations(range(5), 2):
+ graph.addedge(x, y)
+
+ # Validate counts
+ self.assertEqual(graph.count(), 5)
+ self.assertEqual(graph.edgecount(), 10)
+
+ # Test missing edge
+ self.assertIsNone(graph.edges(100))
+
+ # Test topics with no text
+ graph.addtopics()
+ self.assertEqual(len(graph.topics), 5)
+
+ def testUpsert(self):
+ """
+ Test upsert
+ """
+
+ # Update data
+ self.embeddings.upsert([(0, {"text": "Canadian ice shelf collapses".split()}, None)])
+
+ # Validate counts
+ graph = self.embeddings.graph
+ self.assertEqual(graph.count(), 6)
+ self.assertEqual(graph.edgecount(), 2)
+ self.assertEqual(sum((len(graph.topics[x]) for x in graph.topics)), 6)
+ self.assertEqual(len(graph.categories), 6)
diff --git a/test/python/testoptional.py b/test/python/testoptional.py
index cc868dfa..22e3d578 100644
--- a/test/python/testoptional.py
+++ b/test/python/testoptional.py
@@ -29,6 +29,7 @@ class TestOptional(unittest.TestCase):
"imagehash",
"nltk",
"libcloud.storage.providers",
+ "networkx",
"onnxmltools",
"onnxruntime",
"pandas",
@@ -108,6 +109,16 @@ class TestOptional(unittest.TestCase):
with self.assertRaises(ImportError):
ImageEncoder()
+ def testGraph(self):
+ """
+ Test missing graph dependencies
+ """
+
+ from txtai.graph import GraphFactory
+
+ with self.assertRaises(ImportError):
+ GraphFactory.create({"backend": "networkx"})
+
def testModel(self):
"""
Test missing model dependencies