Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Persist NetworkX graph component with MessagePack, closes #775

davidmezzetti88dc320a43ad
diff --git a/src/python/txtai/graph/base.py b/src/python/txtai/graph/base.py
index 73939923..dd0888a9 100644
--- a/src/python/txtai/graph/base.py
+++ b/src/python/txtai/graph/base.py
@@ -2,14 +2,7 @@
Graph module
"""
-import os
-import pickle
-
from collections import Counter
-from tempfile import TemporaryDirectory
-
-from ..archive import ArchiveFactory
-from ..version import __pickle__
from .topics import Topics
@@ -30,7 +23,7 @@ class Graph:
"""
# Graph configuration
- self.config = config if config else {}
+ self.config = config if config is not None else {}
# Graph backend
self.backend = None
@@ -313,22 +306,22 @@ class Graph:
raise NotImplementedError
- def loadgraph(self, path):
+ def load(self, path):
"""
- Loads a graph backend at path.
+ Loads a graph at path.
Args:
- path: path to graph backend
+ path: path to graph
"""
raise NotImplementedError
- def savegraph(self, path):
+ def save(self, path):
"""
- Saves graph backend to path.
+ Saves a graph at path.
Args:
- path: path to save graph backend
+ path: path to save graph
"""
raise NotImplementedError
@@ -361,62 +354,6 @@ class Graph:
if not self.backend:
self.backend = self.create()
- def load(self, path):
- """
- Loads a graph at path.
-
- Args:
- path: path to graph
- """
-
- # Extract files to temporary directory and load content
- with TemporaryDirectory() as directory:
- # Unpack files
- archive = ArchiveFactory.create(directory)
- archive.load(path, "tar")
-
- # Load graph backend
- self.loadgraph(f"{directory}/graph")
-
- # Load categories, if necessary
- path = f"{directory}/categories"
- if os.path.exists(path):
- with open(path, "rb") as handle:
- self.categories = pickle.load(handle)
-
- # Load topics, if necessary
- path = f"{directory}/topics"
- if os.path.exists(path):
- with open(path, "rb") as handle:
- self.topics = pickle.load(handle)
-
- def save(self, path):
- """
- Saves a graph at path.
-
- Args:
- path: path to save graph
- """
-
- # Save files to temporary directory and combine into TAR
- with TemporaryDirectory() as directory:
- # Save graph
- self.savegraph(f"{directory}/graph")
-
- # Save categories, if necessary
- if self.categories:
- with open(f"{directory}/categories", "wb") as handle:
- pickle.dump(self.categories, handle, protocol=__pickle__)
-
- # Save topics, if necessary
- if self.topics:
- with open(f"{directory}/topics", "wb") as handle:
- pickle.dump(self.topics, handle, protocol=__pickle__)
-
- # Pack files
- archive = ArchiveFactory.create(directory)
- archive.save(path, "tar")
-
def close(self):
"""
Closes this graph.
diff --git a/src/python/txtai/graph/factory.py b/src/python/txtai/graph/factory.py
index 4dd3fde8..d03e7bd9 100644
--- a/src/python/txtai/graph/factory.py
+++ b/src/python/txtai/graph/factory.py
@@ -16,7 +16,7 @@ class GraphFactory:
@staticmethod
def create(config):
"""
- Create a graph.
+ Create a Graph.
Args:
config: graph configuration
diff --git a/src/python/txtai/graph/networkx.py b/src/python/txtai/graph/networkx.py
index 72afb0a2..ae95e5ae 100644
--- a/src/python/txtai/graph/networkx.py
+++ b/src/python/txtai/graph/networkx.py
@@ -2,7 +2,9 @@
NetworkX module
"""
-import pickle
+import os
+
+from tempfile import TemporaryDirectory
# Conditional import
try:
@@ -17,7 +19,8 @@ try:
except ImportError:
NETWORKX = False
-from ..version import __pickle__
+from ..archive import ArchiveFactory
+from ..serialize import SerializeError, SerializeFactory
from .base import Graph
@@ -149,22 +152,44 @@ class NetworkX(Graph):
return communities
- def loadgraph(self, path):
- # Load graph network
- with open(path, "rb") as handle:
- self.backend = pickle.load(handle)
-
- def savegraph(self, path):
- # Save graph
- with open(path, "wb") as handle:
- pickle.dump(self.backend, handle, protocol=__pickle__)
+ def load(self, path):
+ try:
+ # Load graph data
+ data = SerializeFactory.create().load(path)
+
+ # Add data to graph
+ self.backend = self.create()
+ self.backend.add_nodes_from(data["nodes"])
+ self.backend.add_edges_from(data["edges"])
+
+ # Load categories
+ self.categories = data.get("categories")
+
+ # Load topics
+ self.topics = data.get("topics")
+
+ except SerializeError:
+ # Backwards compatible support for legacy TAR format
+ self.loadtar(path)
+
+ def save(self, path):
+ # Save graph data
+ SerializeFactory.create().save(
+ {
+ "nodes": [(uid, self.node(uid)) for uid in self.scan()],
+ "edges": list(self.backend.edges(data=True)),
+ "categories": self.categories,
+ "topics": self.topics,
+ },
+ path,
+ )
def loaddict(self, data):
- self.backend = json_graph.node_link_graph(data)
+ self.backend = json_graph.node_link_graph(data, name="indexid")
self.categories, self.topics = data.get("categories"), data.get("topics")
def savedict(self):
- data = json_graph.node_link_data(self.backend)
+ data = json_graph.node_link_data(self.backend, name="indexid")
data["categories"] = self.categories
data["topics"] = self.topics
@@ -214,3 +239,33 @@ class NetworkX(Graph):
# Distance is 1 - score. Skip minimal distances as they are near duplicates.
distance = max(1.0 - attrs["weight"], 0.0)
return distance if distance >= 0.15 else 1.00
+
+ def loadtar(self, path):
+ """
+ Loads a graph from the legacy TAR file.
+
+ Args:
+ path: path to graph
+ """
+
+ # Pickle serialization - backwards compatible
+ serializer = SerializeFactory.create("pickle")
+
+ # Extract files to temporary directory and load content
+ with TemporaryDirectory() as directory:
+ # Unpack files
+ archive = ArchiveFactory.create(directory)
+ archive.load(path, "tar")
+
+ # Load graph backend
+ self.backend = serializer.load(f"{directory}/graph")
+
+ # Load categories, if necessary
+ path = f"{directory}/categories"
+ if os.path.exists(path):
+ self.categories = serializer.load(path)
+
+ # Load topics, if necessary
+ path = f"{directory}/topics"
+ if os.path.exists(path):
+ self.topics = serializer.load(path)
diff --git a/test/python/testgraph.py b/test/python/testgraph.py
index 1df86344..2cb6d4b6 100644
--- a/test/python/testgraph.py
+++ b/test/python/testgraph.py
@@ -7,8 +7,12 @@ import itertools
import tempfile
import unittest
+from unittest.mock import patch
+
+from txtai.archive import ArchiveFactory
from txtai.embeddings import Embeddings
from txtai.graph import Graph, GraphFactory
+from txtai.serialize import SerializeFactory
# pylint: disable=R0904
@@ -232,6 +236,48 @@ class TestGraph(unittest.TestCase):
self.assertEqual(len(graph.topics), 6)
self.assertEqual(len(graph.categories), 6)
+ @patch.dict(os.environ, {"ALLOW_PICKLE": "True"})
+ def testLegacy(self):
+ """
+ Test loading a legacy graph in TAR format
+ """
+
+ # Create graph
+ graph = GraphFactory.create({})
+ graph.initialize()
+ graph.addedge(0, 1)
+
+ categories = ["C1"]
+ topics = {"T1": [0, 1]}
+
+ serializer = SerializeFactory.create("pickle", allowpickle=True)
+
+ # Save files to temporary directory and combine into TAR
+ path = os.path.join(tempfile.gettempdir(), "graph.tar")
+ with tempfile.TemporaryDirectory() as directory:
+ # Save graph
+ serializer.save(graph.backend, f"{directory}/graph")
+
+ # Save categories, if necessary
+ serializer.save(categories, f"{directory}/categories")
+
+ # Save topics, if necessary
+ serializer.save(topics, f"{directory}/topics")
+
+ # Pack files
+ archive = ArchiveFactory.create(directory)
+ archive.save(path, "tar")
+
+ # Load loading legacy format
+ graph = GraphFactory.create({})
+ graph.load(path)
+
+ # Validate graph data is correct
+ self.assertEqual(graph.count(), 2)
+ self.assertEqual(graph.edgecount(), 1)
+ self.assertEqual(graph.topics, topics)
+ self.assertEqual(graph.categories, categories)
+
def testNotImplemented(self):
"""
Test exceptions for non-implemented methods
@@ -260,8 +306,8 @@ class TestGraph(unittest.TestCase):
self.assertRaises(NotImplementedError, graph.showpath, None, None)
self.assertRaises(NotImplementedError, graph.search, None)
self.assertRaises(NotImplementedError, graph.communities, None)
- self.assertRaises(NotImplementedError, graph.loadgraph, None)
- self.assertRaises(NotImplementedError, graph.savegraph, None)
+ self.assertRaises(NotImplementedError, graph.load, None)
+ self.assertRaises(NotImplementedError, graph.save, None)
self.assertRaises(NotImplementedError, graph.loaddict, None)
self.assertRaises(NotImplementedError, graph.savedict)