💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add RDBMS Graph, closes #699
davidmezzetti9aa2edd4fbe2
diff --git a/src/python/txtai/database/client.py b/src/python/txtai/database/client.pyindex cfd16985..1cb9ebf8 100644--- a/src/python/txtai/database/client.py+++ b/src/python/txtai/database/client.py@@ -105,7 +105,7 @@ class Client(RDBMS):def connect(self, path=None):# Connection URL- content = self.config["content"]+ content = self.config.get("content")# Read ENV variable, if necessarycontent = os.environ.get("CLIENT_URL") if content == "client" else contentdiff --git a/src/python/txtai/graph/__init__.py b/src/python/txtai/graph/__init__.pyindex 80019ad6..773bb66a 100644--- a/src/python/txtai/graph/__init__.py+++ b/src/python/txtai/graph/__init__.py@@ -5,4 +5,5 @@ Graph importsfrom .base import Graphfrom .factory import GraphFactoryfrom .networkx import NetworkX+from .rdbms import RDBMSfrom .topics import Topicsdiff --git a/src/python/txtai/graph/base.py b/src/python/txtai/graph/base.pyindex b9d98f93..3b3602b9 100644--- a/src/python/txtai/graph/base.py+++ b/src/python/txtai/graph/base.py@@ -637,7 +637,8 @@ class Graph:edges.append((node, target, relation))# Add relationships- self.addedges(edges)+ if edges:+ self.addedges(edges)# Clear temporary relationship storageself.relations = {}diff --git a/src/python/txtai/graph/factory.py b/src/python/txtai/graph/factory.pyindex 30ad46cc..4dd3fde8 100644--- a/src/python/txtai/graph/factory.py+++ b/src/python/txtai/graph/factory.py@@ -5,6 +5,7 @@ Factory modulefrom ..util import Resolverfrom .networkx import NetworkX+from .rdbms import RDBMSclass GraphFactory:@@ -31,6 +32,8 @@ class GraphFactory:# Create graph instanceif backend == "networkx":graph = NetworkX(config)+ elif backend == "rdbms":+ graph = RDBMS(config)else:graph = GraphFactory.resolve(backend, config)diff --git a/src/python/txtai/graph/networkx.py b/src/python/txtai/graph/networkx.pyindex 72edec38..72afb0a2 100644--- a/src/python/txtai/graph/networkx.py+++ b/src/python/txtai/graph/networkx.py@@ -80,7 +80,7 @@ class NetworkX(Graph):def edges(self, node):edges = self.backend.adj.get(node)if edges:- return dict(sorted(edges.items(), key=lambda x: x[1]["weight"], reverse=True))+ return dict(sorted(edges.items(), key=lambda x: x[1].get("weight", 0), reverse=True))return Nonediff --git a/src/python/txtai/graph/rdbms.py b/src/python/txtai/graph/rdbms.pynew file mode 100644index 00000000..3aa64535--- /dev/null+++ b/src/python/txtai/graph/rdbms.py@@ -0,0 +1,89 @@+"""+RDBMS module+"""++import os++# Conditional import+try:+ from grand import Graph+ from grand.backends import SQLBackend, InMemoryCachedBackend++ from sqlalchemy import text, StaticPool++ ORM = True+except ImportError:+ ORM = False++from .networkx import NetworkX+++class RDBMS(NetworkX):+ """+ Graph instance backed by a relational database.+ """++ def __init__(self, config):+ # Check before super() in case those required libraries are also not available+ if not ORM:+ raise ImportError('RDBMS is not available - install "graph" extra to enable')++ super().__init__(config)++ # Graph and database instances+ self.graph = None+ self.database = None++ def __del__(self):+ if self.database:+ self.database.close()++ def create(self):+ # Create graph instance+ self.graph, self.database = self.connect()++ # Clear previous graph, if available+ for table in [self.config.get("nodes", "nodes"), self.config.get("edges", "edges")]:+ self.database.execute(text(f"DELETE FROM {table}"))++ # Return NetworkX compatible backend+ return self.graph.nx++ def scan(self, attribute=None):+ if attribute:+ for node in self.backend:+ if attribute in self.node(node):+ yield node+ else:+ yield from super().scan(attribute)++ def load(self, path):+ # Create graph instance+ self.graph, self.database = self.connect()++ # Store NetworkX compatible backend+ self.backend = self.graph.nx++ def save(self, path):+ self.database.commit()++ def filter(self, nodes, graph=None):+ return super().filter(nodes, graph if graph else NetworkX(self.config))++ def connect(self):+ """+ Connects to a graph backed by a relational database.++ Args:+ Graph database instance+ """++ backend = SQLBackend(+ db_url=self.config.get("url", os.environ.get("GRAPH_URL")),+ node_table_name=self.config.get("nodes", "nodes"),+ edge_table_name=self.config.get("edges", "edges"),+ sqlalchemy_kwargs={"poolclass": StaticPool, "echo": False},+ )++ # pylint: disable=W0212+ return Graph(backend=InMemoryCachedBackend(backend, maxsize=None)), backend._connectiondiff --git a/test/python/testgraph.py b/test/python/testgraph.pyindex 322eb9bd..98ba4555 100644--- a/test/python/testgraph.py+++ b/test/python/testgraph.py@@ -106,6 +106,34 @@ class TestGraph(unittest.TestCase):graph = GraphFactory.create({"backend": "notfound.graph"})graph.initialize()+ def testDatabase(self):+ """+ Test creating a Graph backed by a relational database+ """++ # Generate graph database+ path = os.path.join(tempfile.gettempdir(), "graph.sqlite")+ graph = GraphFactory.create({"backend": "rdbms", "url": f"sqlite:///{path}"})++ # Initialize the graph+ graph.initialize()++ for x in range(5):+ graph.addnode(x, field=x)++ for x, y in itertools.combinations(range(5), 2):+ graph.addedge(x, y)++ # Test methods+ self.assertEqual(list(graph.scan()), [str(x) for x in range(5)])+ self.assertEqual(list(graph.scan(attribute="field")), [str(x) for x in range(5)])+ self.assertEqual(list(graph.filter([0]).scan()), [0])++ # Test save/load+ graph.save(None)+ graph.load(None)+ self.assertEqual(list(graph.scan()), [str(x) for x in range(5)])+def testDelete(self):"""Test deletediff --git a/test/python/testoptional.py b/test/python/testoptional.pyindex d7981008..ee44685b 100644--- a/test/python/testoptional.py+++ b/test/python/testoptional.py@@ -26,6 +26,8 @@ class TestOptional(unittest.TestCase):"duckdb","fastapi","fasttext",+ "grand-cypher",+ "grand-graph","hnswlib","imagehash","libcloud.storage.providers",@@ -38,6 +40,7 @@ class TestOptional(unittest.TestCase):"onnxruntime.quantization","pandas","peft",+ "pgvector","PIL","rich","sklearn.decomposition",@@ -78,6 +81,22 @@ class TestOptional(unittest.TestCase):else:del sys.modules[key]+ def testANN(self):+ """+ Test missing ANN dependencies+ """++ from txtai.ann import ANNFactory++ with self.assertRaises(ImportError):+ ANNFactory.create({"backend": "annoy"})++ with self.assertRaises(ImportError):+ ANNFactory.create({"backend": "hnsw"})++ with self.assertRaises(ImportError):+ ANNFactory.create({"backend": "pgvector"})+def testApi(self):"""Test missing api dependencies@@ -132,6 +151,9 @@ class TestOptional(unittest.TestCase):with self.assertRaises(ImportError):GraphFactory.create({"backend": "networkx"})+ with self.assertRaises(ImportError):+ GraphFactory.create({"backend": "rdbms"})+def testModel(self):"""Test missing model dependencies@@ -210,20 +232,13 @@ class TestOptional(unittest.TestCase):with self.assertRaises(ImportError):Translation().detect(["test"])- def testSimilarity(self):+ def testVectors(self):"""- Test missing similarity dependencies+ Test missing vector dependencies"""- from txtai.ann import ANNFactoryfrom txtai.vectors import VectorsFactory- with self.assertRaises(ImportError):- ANNFactory.create({"backend": "annoy"})-- with self.assertRaises(ImportError):- ANNFactory.create({"backend": "hnsw"})-with self.assertRaises(ImportError):VectorsFactory.create({"method": "words"}, None)