Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Consolidate compression logic into single package, closes #376

davidmezzettide8b2ee5991b
diff --git a/src/python/txtai/compress/__init__.py b/src/python/txtai/compress/__init__.py
new file mode 100644
index 00000000..0d1b78bd
--- /dev/null
+++ b/src/python/txtai/compress/__init__.py
@@ -0,0 +1,8 @@
+"""
+Compress imports
+"""
+
+from .base import Compress
+from .factory import CompressFactory
+from .tar import Tar
+from .zip import Zip
diff --git a/src/python/txtai/compress/base.py b/src/python/txtai/compress/base.py
new file mode 100644
index 00000000..739b201b
--- /dev/null
+++ b/src/python/txtai/compress/base.py
@@ -0,0 +1,51 @@
+"""
+Compress module
+"""
+
+import os
+
+
+class Compress:
+ """
+ Base class for Compress instances.
+ """
+
+ def pack(self, path, output):
+ """
+ Compresses files in directory path to file output.
+
+ Args:
+ path: input directory path
+ output: output file
+ """
+
+ raise NotImplementedError
+
+ def unpack(self, path, output):
+ """
+ Extracts all files in path to output.
+
+ Args:
+ path: input file path
+ output: output directory
+ """
+
+ raise NotImplementedError
+
+ def validate(self, directory, path):
+ """
+ Validates path is under directory.
+
+ Args:
+ directory: base directory
+ path: path to validate
+
+ Returns:
+ True if path is under directory, False otherwise
+ """
+
+ directory = os.path.abspath(directory)
+ path = os.path.abspath(path)
+ prefix = os.path.commonprefix([directory, path])
+
+ return prefix == directory
diff --git a/src/python/txtai/compress/factory.py b/src/python/txtai/compress/factory.py
new file mode 100644
index 00000000..2e29a69a
--- /dev/null
+++ b/src/python/txtai/compress/factory.py
@@ -0,0 +1,27 @@
+"""
+Factory module
+"""
+
+from .tar import Tar
+from .zip import Zip
+
+
+class CompressFactory:
+ """
+ Methods to create Compress instances.
+ """
+
+ @staticmethod
+ def create(path):
+ """
+ Factory method to construct a Compress instance.
+
+ Args:
+ path: file path
+
+ Returns:
+ Compression
+ """
+
+ compression = path.lower().split(".")[-1]
+ return Zip() if compression == "zip" else Tar()
diff --git a/src/python/txtai/compress/tar.py b/src/python/txtai/compress/tar.py
new file mode 100644
index 00000000..4652b3d6
--- /dev/null
+++ b/src/python/txtai/compress/tar.py
@@ -0,0 +1,49 @@
+"""
+Tar module
+"""
+
+import os
+import tarfile
+
+from .base import Compress
+
+
+class Tar(Compress):
+ """
+ Tar compression
+ """
+
+ def pack(self, path, output):
+ # Infer compression type
+ compression = self.compression(output)
+
+ with tarfile.open(output, f"w:{compression}" if compression else "w") as tar:
+ tar.add(path, arcname=".")
+
+ def unpack(self, path, output):
+ # Infer compression type
+ compression = self.compression(path)
+
+ with tarfile.open(path, f"r:{compression}" if compression else "r") as tar:
+ # Validate paths
+ for member in tar.getmembers():
+ fullpath = os.path.join(path, member.name)
+ if not self.validate(path, fullpath):
+ raise IOError(f"Invalid tar entry: {member.name}")
+
+ tar.extractall(output)
+
+ def compression(self, path):
+ """
+ Gets compression type for path.
+
+ Args:
+ path: path to file
+
+ Returns:
+ compression type
+ """
+
+ # Infer compression type from last path component. Limit to supported types.
+ compression = path.lower().split(".")[-1]
+ return compression if compression in ("bz2", "gz", "xz") else None
diff --git a/src/python/txtai/compress/zip.py b/src/python/txtai/compress/zip.py
new file mode 100644
index 00000000..e88f9376
--- /dev/null
+++ b/src/python/txtai/compress/zip.py
@@ -0,0 +1,30 @@
+"""
+Zip module
+"""
+
+import os
+
+from zipfile import ZipFile, ZIP_DEFLATED
+
+from .base import Compress
+
+
+class Zip(Compress):
+ """
+ Zip compression
+ """
+
+ def pack(self, path, output):
+ with ZipFile(output, "w", ZIP_DEFLATED) as zfile:
+ for root, _, files in sorted(os.walk(path)):
+ for f in files:
+ zfile.write(os.path.join(root, f), arcname=f)
+
+ def unpack(self, path, output):
+ with ZipFile(path, "r") as zfile:
+ # Validate path if directory specified
+ for fullpath in zfile.namelist():
+ if os.path.dirname(fullpath) and not self.validate(path, fullpath):
+ raise IOError(f"Invalid zip entry: {fullpath}")
+
+ zfile.extractall(output)
diff --git a/src/python/txtai/embeddings/archive.py b/src/python/txtai/embeddings/archive.py
index 0689b216..b52ba488 100644
--- a/src/python/txtai/embeddings/archive.py
+++ b/src/python/txtai/embeddings/archive.py
@@ -3,10 +3,10 @@ Archive module
"""
import os
-import tarfile
from tempfile import TemporaryDirectory
-from zipfile import ZipFile, ZIP_DEFLATED
+
+from ..compress import CompressFactory
from .cloud import Cloud
@@ -89,18 +89,9 @@ class Archive:
if cloud:
cloud.load(path)
- # Get compression type
- compression = self.compression(path)
-
- # Zip files
- if compression == "zip":
- with ZipFile(path, "r") as zfile:
- zfile.extractall(self.path())
-
- # Tar files
- else:
- with tarfile.open(path, f"r:{compression}") as tar:
- tar.extractall(self.path())
+ # Unpack compressed file
+ compress = CompressFactory().create(path)
+ compress.unpack(path, self.path())
def save(self, path, config):
"""
@@ -111,44 +102,20 @@ class Archive:
config: additional configuration
"""
- # Get compression type
- compression = self.compression(path)
-
# Create output directory, if necessary
output = os.path.dirname(path)
if output:
os.makedirs(output, exist_ok=True)
- # Zip files
- if compression == "zip":
- with ZipFile(path, "w", ZIP_DEFLATED) as zfile:
- for root, _, files in sorted(os.walk(self.path())):
- for f in files:
- zfile.write(os.path.join(root, f), arcname=f)
-
- # Tar files
- else:
- with tarfile.open(path, f"w:{compression}") as tar:
- tar.add(self.path(), arcname=".")
+ # Pack into compressed file
+ compress = CompressFactory().create(path)
+ compress.pack(self.path(), path)
# Save file to cloud storage, if necessary
cloud = self.cloud(config)
if cloud:
cloud.save(path)
- def compression(self, path):
- """
- Gets compression type for path.
-
- Args:
- path: path to archive file
-
- Returns:
- compression type
- """
-
- return path.lower().split(".")[-1]
-
def cloud(self, config):
"""
Gets a cloud storage instance, if specified in config.
diff --git a/src/python/txtai/graph/base.py b/src/python/txtai/graph/base.py
index 37bcc141..ae152330 100644
--- a/src/python/txtai/graph/base.py
+++ b/src/python/txtai/graph/base.py
@@ -4,12 +4,12 @@ Graph module
import os
import pickle
-import tarfile
from collections import Counter
from tempfile import TemporaryDirectory
from .. import __pickle__
+from ..compress import CompressFactory
from .topics import Topics
@@ -289,9 +289,9 @@ class Graph:
# Extract files to temporary directory and load content
with TemporaryDirectory() as directory:
- # Extract tar contents
- with tarfile.open(path, "r") as tar:
- tar.extractall(directory)
+ # Unpack files
+ compress = CompressFactory().create("tar")
+ compress.unpack(path, directory)
# Load graph backend
self.loadgraph(f"{directory}/graph")
@@ -331,9 +331,9 @@ class Graph:
with open(f"{directory}/topics", "wb") as handle:
pickle.dump(self.topics, handle, protocol=__pickle__)
- # Combine output as TAR
- with tarfile.open(path, "w") as tar:
- tar.add(directory, arcname=".")
+ # Pack files
+ compress = CompressFactory().create("tar")
+ compress.pack(directory, path)
def insert(self, documents, index=0):
"""
diff --git a/test/python/testcompress.py b/test/python/testcompress.py
new file mode 100644
index 00000000..58e9ab44
--- /dev/null
+++ b/test/python/testcompress.py
@@ -0,0 +1,61 @@
+"""
+Compress module tests
+"""
+
+import os
+import tarfile
+import tempfile
+import unittest
+
+from zipfile import ZipFile, ZIP_DEFLATED
+
+from txtai.compress import Compress, CompressFactory
+
+# pylint: disable = C0411
+from utils import Utils
+
+
+class TestCompress(unittest.TestCase):
+ """
+ Compress tests.
+ """
+
+ def testInvalidTar(self):
+ """
+ Test invalid tar file
+ """
+
+ path = os.path.join(tempfile.gettempdir(), "badtar")
+ with tarfile.open(path, "w") as tar:
+ tar.add(Utils.PATH, arcname="..")
+
+ compress = CompressFactory.create("tar")
+
+ # Validate error is thrown for file
+ with self.assertRaises(IOError):
+ compress.unpack(path, path)
+
+ def testInvalidZip(self):
+ """
+ Test invalid zip file
+ """
+
+ path = os.path.join(tempfile.gettempdir(), "badzip")
+ with ZipFile(path, "w", ZIP_DEFLATED) as zfile:
+ zfile.write(Utils.PATH + "/article.pdf", arcname="../article.pdf")
+
+ compress = CompressFactory.create("zip")
+
+ # Validate error is thrown for file
+ with self.assertRaises(IOError):
+ compress.unpack(path, path)
+
+ def testNotImplemented(self):
+ """
+ Test exceptions for non-implemented methods
+ """
+
+ compress = Compress()
+
+ self.assertRaises(NotImplementedError, compress.pack, None, None)
+ self.assertRaises(NotImplementedError, compress.unpack, None, None)