💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Persist NetworkX graph component with MessagePack, closes #775
davidmezzetti88dc320a43ad
diff --git a/src/python/txtai/graph/base.py b/src/python/txtai/graph/base.pyindex 73939923..dd0888a9 100644--- a/src/python/txtai/graph/base.py+++ b/src/python/txtai/graph/base.py@@ -2,14 +2,7 @@Graph module"""-import os-import pickle-from collections import Counter-from tempfile import TemporaryDirectory--from ..archive import ArchiveFactory-from ..version import __pickle__from .topics import Topics@@ -30,7 +23,7 @@ class Graph:"""# Graph configuration- self.config = config if config else {}+ self.config = config if config is not None else {}# Graph backendself.backend = None@@ -313,22 +306,22 @@ class Graph:raise NotImplementedError- def loadgraph(self, path):+ def load(self, path):"""- Loads a graph backend at path.+ Loads a graph at path.Args:- path: path to graph backend+ path: path to graph"""raise NotImplementedError- def savegraph(self, path):+ def save(self, path):"""- Saves graph backend to path.+ Saves a graph at path.Args:- path: path to save graph backend+ path: path to save graph"""raise NotImplementedError@@ -361,62 +354,6 @@ class Graph:if not self.backend:self.backend = self.create()- def load(self, path):- """- Loads a graph at path.-- Args:- path: path to graph- """-- # Extract files to temporary directory and load content- with TemporaryDirectory() as directory:- # Unpack files- archive = ArchiveFactory.create(directory)- archive.load(path, "tar")-- # Load graph backend- self.loadgraph(f"{directory}/graph")-- # Load categories, if necessary- path = f"{directory}/categories"- if os.path.exists(path):- with open(path, "rb") as handle:- self.categories = pickle.load(handle)-- # Load topics, if necessary- path = f"{directory}/topics"- if os.path.exists(path):- with open(path, "rb") as handle:- self.topics = pickle.load(handle)-- def save(self, path):- """- Saves a graph at path.-- Args:- path: path to save graph- """-- # Save files to temporary directory and combine into TAR- with TemporaryDirectory() as directory:- # Save graph- self.savegraph(f"{directory}/graph")-- # Save categories, if necessary- if self.categories:- with open(f"{directory}/categories", "wb") as handle:- pickle.dump(self.categories, handle, protocol=__pickle__)-- # Save topics, if necessary- if self.topics:- with open(f"{directory}/topics", "wb") as handle:- pickle.dump(self.topics, handle, protocol=__pickle__)-- # Pack files- archive = ArchiveFactory.create(directory)- archive.save(path, "tar")-def close(self):"""Closes this graph.diff --git a/src/python/txtai/graph/factory.py b/src/python/txtai/graph/factory.pyindex 4dd3fde8..d03e7bd9 100644--- a/src/python/txtai/graph/factory.py+++ b/src/python/txtai/graph/factory.py@@ -16,7 +16,7 @@ class GraphFactory:@staticmethoddef create(config):"""- Create a graph.+ Create a Graph.Args:config: graph configurationdiff --git a/src/python/txtai/graph/networkx.py b/src/python/txtai/graph/networkx.pyindex 72afb0a2..ae95e5ae 100644--- a/src/python/txtai/graph/networkx.py+++ b/src/python/txtai/graph/networkx.py@@ -2,7 +2,9 @@NetworkX module"""-import pickle+import os++from tempfile import TemporaryDirectory# Conditional importtry:@@ -17,7 +19,8 @@ try:except ImportError:NETWORKX = False-from ..version import __pickle__+from ..archive import ArchiveFactory+from ..serialize import SerializeError, SerializeFactoryfrom .base import Graph@@ -149,22 +152,44 @@ class NetworkX(Graph):return communities- def loadgraph(self, path):- # Load graph network- with open(path, "rb") as handle:- self.backend = pickle.load(handle)-- def savegraph(self, path):- # Save graph- with open(path, "wb") as handle:- pickle.dump(self.backend, handle, protocol=__pickle__)+ def load(self, path):+ try:+ # Load graph data+ data = SerializeFactory.create().load(path)++ # Add data to graph+ self.backend = self.create()+ self.backend.add_nodes_from(data["nodes"])+ self.backend.add_edges_from(data["edges"])++ # Load categories+ self.categories = data.get("categories")++ # Load topics+ self.topics = data.get("topics")++ except SerializeError:+ # Backwards compatible support for legacy TAR format+ self.loadtar(path)++ def save(self, path):+ # Save graph data+ SerializeFactory.create().save(+ {+ "nodes": [(uid, self.node(uid)) for uid in self.scan()],+ "edges": list(self.backend.edges(data=True)),+ "categories": self.categories,+ "topics": self.topics,+ },+ path,+ )def loaddict(self, data):- self.backend = json_graph.node_link_graph(data)+ self.backend = json_graph.node_link_graph(data, name="indexid")self.categories, self.topics = data.get("categories"), data.get("topics")def savedict(self):- data = json_graph.node_link_data(self.backend)+ data = json_graph.node_link_data(self.backend, name="indexid")data["categories"] = self.categoriesdata["topics"] = self.topics@@ -214,3 +239,33 @@ class NetworkX(Graph):# Distance is 1 - score. Skip minimal distances as they are near duplicates.distance = max(1.0 - attrs["weight"], 0.0)return distance if distance >= 0.15 else 1.00++ def loadtar(self, path):+ """+ Loads a graph from the legacy TAR file.++ Args:+ path: path to graph+ """++ # Pickle serialization - backwards compatible+ serializer = SerializeFactory.create("pickle")++ # Extract files to temporary directory and load content+ with TemporaryDirectory() as directory:+ # Unpack files+ archive = ArchiveFactory.create(directory)+ archive.load(path, "tar")++ # Load graph backend+ self.backend = serializer.load(f"{directory}/graph")++ # Load categories, if necessary+ path = f"{directory}/categories"+ if os.path.exists(path):+ self.categories = serializer.load(path)++ # Load topics, if necessary+ path = f"{directory}/topics"+ if os.path.exists(path):+ self.topics = serializer.load(path)diff --git a/test/python/testgraph.py b/test/python/testgraph.pyindex 1df86344..2cb6d4b6 100644--- a/test/python/testgraph.py+++ b/test/python/testgraph.py@@ -7,8 +7,12 @@ import itertoolsimport tempfileimport unittest+from unittest.mock import patch++from txtai.archive import ArchiveFactoryfrom txtai.embeddings import Embeddingsfrom txtai.graph import Graph, GraphFactory+from txtai.serialize import SerializeFactory# pylint: disable=R0904@@ -232,6 +236,48 @@ class TestGraph(unittest.TestCase):self.assertEqual(len(graph.topics), 6)self.assertEqual(len(graph.categories), 6)+ @patch.dict(os.environ, {"ALLOW_PICKLE": "True"})+ def testLegacy(self):+ """+ Test loading a legacy graph in TAR format+ """++ # Create graph+ graph = GraphFactory.create({})+ graph.initialize()+ graph.addedge(0, 1)++ categories = ["C1"]+ topics = {"T1": [0, 1]}++ serializer = SerializeFactory.create("pickle", allowpickle=True)++ # Save files to temporary directory and combine into TAR+ path = os.path.join(tempfile.gettempdir(), "graph.tar")+ with tempfile.TemporaryDirectory() as directory:+ # Save graph+ serializer.save(graph.backend, f"{directory}/graph")++ # Save categories, if necessary+ serializer.save(categories, f"{directory}/categories")++ # Save topics, if necessary+ serializer.save(topics, f"{directory}/topics")++ # Pack files+ archive = ArchiveFactory.create(directory)+ archive.save(path, "tar")++ # Load loading legacy format+ graph = GraphFactory.create({})+ graph.load(path)++ # Validate graph data is correct+ self.assertEqual(graph.count(), 2)+ self.assertEqual(graph.edgecount(), 1)+ self.assertEqual(graph.topics, topics)+ self.assertEqual(graph.categories, categories)+def testNotImplemented(self):"""Test exceptions for non-implemented methods@@ -260,8 +306,8 @@ class TestGraph(unittest.TestCase):self.assertRaises(NotImplementedError, graph.showpath, None, None)self.assertRaises(NotImplementedError, graph.search, None)self.assertRaises(NotImplementedError, graph.communities, None)- self.assertRaises(NotImplementedError, graph.loadgraph, None)- self.assertRaises(NotImplementedError, graph.savegraph, None)+ self.assertRaises(NotImplementedError, graph.load, None)+ self.assertRaises(NotImplementedError, graph.save, None)self.assertRaises(NotImplementedError, graph.loaddict, None)self.assertRaises(NotImplementedError, graph.savedict)