Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add RDBMS Graph, closes #699

davidmezzetti9aa2edd4fbe2
diff --git a/src/python/txtai/database/client.py b/src/python/txtai/database/client.py
index cfd16985..1cb9ebf8 100644
--- a/src/python/txtai/database/client.py
+++ b/src/python/txtai/database/client.py
@@ -105,7 +105,7 @@ class Client(RDBMS):
def connect(self, path=None):
# Connection URL
- content = self.config["content"]
+ content = self.config.get("content")
# Read ENV variable, if necessary
content = os.environ.get("CLIENT_URL") if content == "client" else content
diff --git a/src/python/txtai/graph/__init__.py b/src/python/txtai/graph/__init__.py
index 80019ad6..773bb66a 100644
--- a/src/python/txtai/graph/__init__.py
+++ b/src/python/txtai/graph/__init__.py
@@ -5,4 +5,5 @@ Graph imports
from .base import Graph
from .factory import GraphFactory
from .networkx import NetworkX
+from .rdbms import RDBMS
from .topics import Topics
diff --git a/src/python/txtai/graph/base.py b/src/python/txtai/graph/base.py
index b9d98f93..3b3602b9 100644
--- a/src/python/txtai/graph/base.py
+++ b/src/python/txtai/graph/base.py
@@ -637,7 +637,8 @@ class Graph:
edges.append((node, target, relation))
# Add relationships
- self.addedges(edges)
+ if edges:
+ self.addedges(edges)
# Clear temporary relationship storage
self.relations = {}
diff --git a/src/python/txtai/graph/factory.py b/src/python/txtai/graph/factory.py
index 30ad46cc..4dd3fde8 100644
--- a/src/python/txtai/graph/factory.py
+++ b/src/python/txtai/graph/factory.py
@@ -5,6 +5,7 @@ Factory module
from ..util import Resolver
from .networkx import NetworkX
+from .rdbms import RDBMS
class GraphFactory:
@@ -31,6 +32,8 @@ class GraphFactory:
# Create graph instance
if backend == "networkx":
graph = NetworkX(config)
+ elif backend == "rdbms":
+ graph = RDBMS(config)
else:
graph = GraphFactory.resolve(backend, config)
diff --git a/src/python/txtai/graph/networkx.py b/src/python/txtai/graph/networkx.py
index 72edec38..72afb0a2 100644
--- a/src/python/txtai/graph/networkx.py
+++ b/src/python/txtai/graph/networkx.py
@@ -80,7 +80,7 @@ class NetworkX(Graph):
def edges(self, node):
edges = self.backend.adj.get(node)
if edges:
- return dict(sorted(edges.items(), key=lambda x: x[1]["weight"], reverse=True))
+ return dict(sorted(edges.items(), key=lambda x: x[1].get("weight", 0), reverse=True))
return None
diff --git a/src/python/txtai/graph/rdbms.py b/src/python/txtai/graph/rdbms.py
new file mode 100644
index 00000000..3aa64535
--- /dev/null
+++ b/src/python/txtai/graph/rdbms.py
@@ -0,0 +1,89 @@
+"""
+RDBMS module
+"""
+
+import os
+
+# Conditional import
+try:
+ from grand import Graph
+ from grand.backends import SQLBackend, InMemoryCachedBackend
+
+ from sqlalchemy import text, StaticPool
+
+ ORM = True
+except ImportError:
+ ORM = False
+
+from .networkx import NetworkX
+
+
+class RDBMS(NetworkX):
+ """
+ Graph instance backed by a relational database.
+ """
+
+ def __init__(self, config):
+ # Check before super() in case those required libraries are also not available
+ if not ORM:
+ raise ImportError('RDBMS is not available - install "graph" extra to enable')
+
+ super().__init__(config)
+
+ # Graph and database instances
+ self.graph = None
+ self.database = None
+
+ def __del__(self):
+ if self.database:
+ self.database.close()
+
+ def create(self):
+ # Create graph instance
+ self.graph, self.database = self.connect()
+
+ # Clear previous graph, if available
+ for table in [self.config.get("nodes", "nodes"), self.config.get("edges", "edges")]:
+ self.database.execute(text(f"DELETE FROM {table}"))
+
+ # Return NetworkX compatible backend
+ return self.graph.nx
+
+ def scan(self, attribute=None):
+ if attribute:
+ for node in self.backend:
+ if attribute in self.node(node):
+ yield node
+ else:
+ yield from super().scan(attribute)
+
+ def load(self, path):
+ # Create graph instance
+ self.graph, self.database = self.connect()
+
+ # Store NetworkX compatible backend
+ self.backend = self.graph.nx
+
+ def save(self, path):
+ self.database.commit()
+
+ def filter(self, nodes, graph=None):
+ return super().filter(nodes, graph if graph else NetworkX(self.config))
+
+ def connect(self):
+ """
+ Connects to a graph backed by a relational database.
+
+ Args:
+ Graph database instance
+ """
+
+ backend = SQLBackend(
+ db_url=self.config.get("url", os.environ.get("GRAPH_URL")),
+ node_table_name=self.config.get("nodes", "nodes"),
+ edge_table_name=self.config.get("edges", "edges"),
+ sqlalchemy_kwargs={"poolclass": StaticPool, "echo": False},
+ )
+
+ # pylint: disable=W0212
+ return Graph(backend=InMemoryCachedBackend(backend, maxsize=None)), backend._connection
diff --git a/test/python/testgraph.py b/test/python/testgraph.py
index 322eb9bd..98ba4555 100644
--- a/test/python/testgraph.py
+++ b/test/python/testgraph.py
@@ -106,6 +106,34 @@ class TestGraph(unittest.TestCase):
graph = GraphFactory.create({"backend": "notfound.graph"})
graph.initialize()
+ def testDatabase(self):
+ """
+ Test creating a Graph backed by a relational database
+ """
+
+ # Generate graph database
+ path = os.path.join(tempfile.gettempdir(), "graph.sqlite")
+ graph = GraphFactory.create({"backend": "rdbms", "url": f"sqlite:///{path}"})
+
+ # Initialize the graph
+ graph.initialize()
+
+ for x in range(5):
+ graph.addnode(x, field=x)
+
+ for x, y in itertools.combinations(range(5), 2):
+ graph.addedge(x, y)
+
+ # Test methods
+ self.assertEqual(list(graph.scan()), [str(x) for x in range(5)])
+ self.assertEqual(list(graph.scan(attribute="field")), [str(x) for x in range(5)])
+ self.assertEqual(list(graph.filter([0]).scan()), [0])
+
+ # Test save/load
+ graph.save(None)
+ graph.load(None)
+ self.assertEqual(list(graph.scan()), [str(x) for x in range(5)])
+
def testDelete(self):
"""
Test delete
diff --git a/test/python/testoptional.py b/test/python/testoptional.py
index d7981008..ee44685b 100644
--- a/test/python/testoptional.py
+++ b/test/python/testoptional.py
@@ -26,6 +26,8 @@ class TestOptional(unittest.TestCase):
"duckdb",
"fastapi",
"fasttext",
+ "grand-cypher",
+ "grand-graph",
"hnswlib",
"imagehash",
"libcloud.storage.providers",
@@ -38,6 +40,7 @@ class TestOptional(unittest.TestCase):
"onnxruntime.quantization",
"pandas",
"peft",
+ "pgvector",
"PIL",
"rich",
"sklearn.decomposition",
@@ -78,6 +81,22 @@ class TestOptional(unittest.TestCase):
else:
del sys.modules[key]
+ def testANN(self):
+ """
+ Test missing ANN dependencies
+ """
+
+ from txtai.ann import ANNFactory
+
+ with self.assertRaises(ImportError):
+ ANNFactory.create({"backend": "annoy"})
+
+ with self.assertRaises(ImportError):
+ ANNFactory.create({"backend": "hnsw"})
+
+ with self.assertRaises(ImportError):
+ ANNFactory.create({"backend": "pgvector"})
+
def testApi(self):
"""
Test missing api dependencies
@@ -132,6 +151,9 @@ class TestOptional(unittest.TestCase):
with self.assertRaises(ImportError):
GraphFactory.create({"backend": "networkx"})
+ with self.assertRaises(ImportError):
+ GraphFactory.create({"backend": "rdbms"})
+
def testModel(self):
"""
Test missing model dependencies
@@ -210,20 +232,13 @@ class TestOptional(unittest.TestCase):
with self.assertRaises(ImportError):
Translation().detect(["test"])
- def testSimilarity(self):
+ def testVectors(self):
"""
- Test missing similarity dependencies
+ Test missing vector dependencies
"""
- from txtai.ann import ANNFactory
from txtai.vectors import VectorsFactory
- with self.assertRaises(ImportError):
- ANNFactory.create({"backend": "annoy"})
-
- with self.assertRaises(ImportError):
- ANNFactory.create({"backend": "hnsw"})
-
with self.assertRaises(ImportError):
VectorsFactory.create({"method": "words"}, None)