💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Consolidate compression logic into single package, closes #376
davidmezzettide8b2ee5991b
diff --git a/src/python/txtai/compress/__init__.py b/src/python/txtai/compress/__init__.pynew file mode 100644index 00000000..0d1b78bd--- /dev/null+++ b/src/python/txtai/compress/__init__.py@@ -0,0 +1,8 @@+"""+Compress imports+"""++from .base import Compress+from .factory import CompressFactory+from .tar import Tar+from .zip import Zipdiff --git a/src/python/txtai/compress/base.py b/src/python/txtai/compress/base.pynew file mode 100644index 00000000..739b201b--- /dev/null+++ b/src/python/txtai/compress/base.py@@ -0,0 +1,51 @@+"""+Compress module+"""++import os+++class Compress:+ """+ Base class for Compress instances.+ """++ def pack(self, path, output):+ """+ Compresses files in directory path to file output.++ Args:+ path: input directory path+ output: output file+ """++ raise NotImplementedError++ def unpack(self, path, output):+ """+ Extracts all files in path to output.++ Args:+ path: input file path+ output: output directory+ """++ raise NotImplementedError++ def validate(self, directory, path):+ """+ Validates path is under directory.++ Args:+ directory: base directory+ path: path to validate++ Returns:+ True if path is under directory, False otherwise+ """++ directory = os.path.abspath(directory)+ path = os.path.abspath(path)+ prefix = os.path.commonprefix([directory, path])++ return prefix == directorydiff --git a/src/python/txtai/compress/factory.py b/src/python/txtai/compress/factory.pynew file mode 100644index 00000000..2e29a69a--- /dev/null+++ b/src/python/txtai/compress/factory.py@@ -0,0 +1,27 @@+"""+Factory module+"""++from .tar import Tar+from .zip import Zip+++class CompressFactory:+ """+ Methods to create Compress instances.+ """++ @staticmethod+ def create(path):+ """+ Factory method to construct a Compress instance.++ Args:+ path: file path++ Returns:+ Compression+ """++ compression = path.lower().split(".")[-1]+ return Zip() if compression == "zip" else Tar()diff --git a/src/python/txtai/compress/tar.py b/src/python/txtai/compress/tar.pynew file mode 100644index 00000000..4652b3d6--- /dev/null+++ b/src/python/txtai/compress/tar.py@@ -0,0 +1,49 @@+"""+Tar module+"""++import os+import tarfile++from .base import Compress+++class Tar(Compress):+ """+ Tar compression+ """++ def pack(self, path, output):+ # Infer compression type+ compression = self.compression(output)++ with tarfile.open(output, f"w:{compression}" if compression else "w") as tar:+ tar.add(path, arcname=".")++ def unpack(self, path, output):+ # Infer compression type+ compression = self.compression(path)++ with tarfile.open(path, f"r:{compression}" if compression else "r") as tar:+ # Validate paths+ for member in tar.getmembers():+ fullpath = os.path.join(path, member.name)+ if not self.validate(path, fullpath):+ raise IOError(f"Invalid tar entry: {member.name}")++ tar.extractall(output)++ def compression(self, path):+ """+ Gets compression type for path.++ Args:+ path: path to file++ Returns:+ compression type+ """++ # Infer compression type from last path component. Limit to supported types.+ compression = path.lower().split(".")[-1]+ return compression if compression in ("bz2", "gz", "xz") else Nonediff --git a/src/python/txtai/compress/zip.py b/src/python/txtai/compress/zip.pynew file mode 100644index 00000000..e88f9376--- /dev/null+++ b/src/python/txtai/compress/zip.py@@ -0,0 +1,30 @@+"""+Zip module+"""++import os++from zipfile import ZipFile, ZIP_DEFLATED++from .base import Compress+++class Zip(Compress):+ """+ Zip compression+ """++ def pack(self, path, output):+ with ZipFile(output, "w", ZIP_DEFLATED) as zfile:+ for root, _, files in sorted(os.walk(path)):+ for f in files:+ zfile.write(os.path.join(root, f), arcname=f)++ def unpack(self, path, output):+ with ZipFile(path, "r") as zfile:+ # Validate path if directory specified+ for fullpath in zfile.namelist():+ if os.path.dirname(fullpath) and not self.validate(path, fullpath):+ raise IOError(f"Invalid zip entry: {fullpath}")++ zfile.extractall(output)diff --git a/src/python/txtai/embeddings/archive.py b/src/python/txtai/embeddings/archive.pyindex 0689b216..b52ba488 100644--- a/src/python/txtai/embeddings/archive.py+++ b/src/python/txtai/embeddings/archive.py@@ -3,10 +3,10 @@ Archive module"""import os-import tarfilefrom tempfile import TemporaryDirectory-from zipfile import ZipFile, ZIP_DEFLATED++from ..compress import CompressFactoryfrom .cloud import Cloud@@ -89,18 +89,9 @@ class Archive:if cloud:cloud.load(path)- # Get compression type- compression = self.compression(path)-- # Zip files- if compression == "zip":- with ZipFile(path, "r") as zfile:- zfile.extractall(self.path())-- # Tar files- else:- with tarfile.open(path, f"r:{compression}") as tar:- tar.extractall(self.path())+ # Unpack compressed file+ compress = CompressFactory().create(path)+ compress.unpack(path, self.path())def save(self, path, config):"""@@ -111,44 +102,20 @@ class Archive:config: additional configuration"""- # Get compression type- compression = self.compression(path)-# Create output directory, if necessaryoutput = os.path.dirname(path)if output:os.makedirs(output, exist_ok=True)- # Zip files- if compression == "zip":- with ZipFile(path, "w", ZIP_DEFLATED) as zfile:- for root, _, files in sorted(os.walk(self.path())):- for f in files:- zfile.write(os.path.join(root, f), arcname=f)-- # Tar files- else:- with tarfile.open(path, f"w:{compression}") as tar:- tar.add(self.path(), arcname=".")+ # Pack into compressed file+ compress = CompressFactory().create(path)+ compress.pack(self.path(), path)# Save file to cloud storage, if necessarycloud = self.cloud(config)if cloud:cloud.save(path)- def compression(self, path):- """- Gets compression type for path.-- Args:- path: path to archive file-- Returns:- compression type- """-- return path.lower().split(".")[-1]-def cloud(self, config):"""Gets a cloud storage instance, if specified in config.diff --git a/src/python/txtai/graph/base.py b/src/python/txtai/graph/base.pyindex 37bcc141..ae152330 100644--- a/src/python/txtai/graph/base.py+++ b/src/python/txtai/graph/base.py@@ -4,12 +4,12 @@ Graph moduleimport osimport pickle-import tarfilefrom collections import Counterfrom tempfile import TemporaryDirectoryfrom .. import __pickle__+from ..compress import CompressFactoryfrom .topics import Topics@@ -289,9 +289,9 @@ class Graph:# Extract files to temporary directory and load contentwith TemporaryDirectory() as directory:- # Extract tar contents- with tarfile.open(path, "r") as tar:- tar.extractall(directory)+ # Unpack files+ compress = CompressFactory().create("tar")+ compress.unpack(path, directory)# Load graph backendself.loadgraph(f"{directory}/graph")@@ -331,9 +331,9 @@ class Graph:with open(f"{directory}/topics", "wb") as handle:pickle.dump(self.topics, handle, protocol=__pickle__)- # Combine output as TAR- with tarfile.open(path, "w") as tar:- tar.add(directory, arcname=".")+ # Pack files+ compress = CompressFactory().create("tar")+ compress.pack(directory, path)def insert(self, documents, index=0):"""diff --git a/test/python/testcompress.py b/test/python/testcompress.pynew file mode 100644index 00000000..58e9ab44--- /dev/null+++ b/test/python/testcompress.py@@ -0,0 +1,61 @@+"""+Compress module tests+"""++import os+import tarfile+import tempfile+import unittest++from zipfile import ZipFile, ZIP_DEFLATED++from txtai.compress import Compress, CompressFactory++# pylint: disable = C0411+from utils import Utils+++class TestCompress(unittest.TestCase):+ """+ Compress tests.+ """++ def testInvalidTar(self):+ """+ Test invalid tar file+ """++ path = os.path.join(tempfile.gettempdir(), "badtar")+ with tarfile.open(path, "w") as tar:+ tar.add(Utils.PATH, arcname="..")++ compress = CompressFactory.create("tar")++ # Validate error is thrown for file+ with self.assertRaises(IOError):+ compress.unpack(path, path)++ def testInvalidZip(self):+ """+ Test invalid zip file+ """++ path = os.path.join(tempfile.gettempdir(), "badzip")+ with ZipFile(path, "w", ZIP_DEFLATED) as zfile:+ zfile.write(Utils.PATH + "/article.pdf", arcname="../article.pdf")++ compress = CompressFactory.create("zip")++ # Validate error is thrown for file+ with self.assertRaises(IOError):+ compress.unpack(path, path)++ def testNotImplemented(self):+ """+ Test exceptions for non-implemented methods+ """++ compress = Compress()++ self.assertRaises(NotImplementedError, compress.pack, None, None)+ self.assertRaises(NotImplementedError, compress.unpack, None, None)