💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Refactor archive module into separate package, closes #429
davidmezzetti399dc56bef77
diff --git a/src/python/txtai/archive/__init__.py b/src/python/txtai/archive/__init__.pynew file mode 100644index 00000000..398bb580--- /dev/null+++ b/src/python/txtai/archive/__init__.py@@ -0,0 +1,9 @@+"""+Archive imports+"""++from .base import Archive+from .compress import Compress+from .factory import ArchiveFactory+from .tar import Tar+from .zip import Zipdiff --git a/src/python/txtai/embeddings/archive.py b/src/python/txtai/archive/base.pysimilarity index 52%rename from src/python/txtai/embeddings/archive.pyrename to src/python/txtai/archive/base.pyindex b52ba488..8afd68da 100644--- a/src/python/txtai/embeddings/archive.py+++ b/src/python/txtai/archive/base.py@@ -6,14 +6,13 @@ import osfrom tempfile import TemporaryDirectory-from ..compress import CompressFactory--from .cloud import Cloud+from .tar import Tar+from .zip import Zipclass Archive:"""- Methods to load and save archive files.+ Base class for archive instances."""def __init__(self, directory=None):@@ -37,7 +36,7 @@ class Archive:True if the path ends with an archive extension, False otherwise"""- return any(path.lower().endswith(extension) for extension in [".tar.bz2", ".tar.gz", ".tar.xz", ".zip"])+ return path and any(path.lower().endswith(extension) for extension in [".tar.bz2", ".tar.gz", ".tar.xz", ".zip"])def path(self):"""@@ -53,53 +52,28 @@ class Archive:# pylint: disable=R1732self.directory = TemporaryDirectory()- return self.directory.name-- def exists(self, path, config):- """- Checks if file at path exists.-- Args:- path: path to archive file- config: additional configuration-- Returns:- True if path exists, False otherwise- """-- # Check if archive exists in cloud storage- cloud = self.cloud(config)- if cloud:- return cloud.exists(path)-- # Check if archive exists locally- return os.path.exists(path)+ return self.directory.name if isinstance(self.directory, TemporaryDirectory) else self.directory- def load(self, path, config):+ def load(self, path, compression=None):"""Extracts file at path to archive working directory.Args:path: path to archive file- config: additional configuration+ compression: compression format, infers from path if not provided"""- # Retrieve archive from cloud storage, if necessary- cloud = self.cloud(config)- if cloud:- cloud.load(path)-# Unpack compressed file- compress = CompressFactory().create(path)+ compress = self.create(path, compression)compress.unpack(path, self.path())- def save(self, path, config):+ def save(self, path, compression=None):"""Archives files in archive working directory to file at path.Args:path: path to archive file- config: additional configuration+ compression: compression format, infers from path if not provided"""# Create output directory, if necessary@@ -108,24 +82,23 @@ class Archive:os.makedirs(output, exist_ok=True)# Pack into compressed file- compress = CompressFactory().create(path)+ compress = self.create(path, compression)compress.pack(self.path(), path)- # Save file to cloud storage, if necessary- cloud = self.cloud(config)- if cloud:- cloud.save(path)-- def cloud(self, config):+ def create(self, path, compression):"""- Gets a cloud storage instance, if specified in config.+ Method to construct a Compress instance.Args:- config: cloud storage configuration+ path: file path+ compression: compression format, infers using file extension if not providedReturns:- Cloud storage instance+ Compress"""- # Create cloud storage connection, if necessary- return Cloud(config) if config and "provider" in config else None+ # Infer compression format from path if not provided+ compression = compression if compression else path.lower().split(".")[-1]++ # Create compression instance+ return Zip() if compression == "zip" else Tar()diff --git a/src/python/txtai/compress/base.py b/src/python/txtai/archive/compress.pysimilarity index 100%rename from src/python/txtai/compress/base.pyrename to src/python/txtai/archive/compress.pydiff --git a/src/python/txtai/archive/factory.py b/src/python/txtai/archive/factory.pynew file mode 100644index 00000000..57b63a94--- /dev/null+++ b/src/python/txtai/archive/factory.py@@ -0,0 +1,25 @@+"""+Factory module+"""++from .base import Archive+++class ArchiveFactory:+ """+ Methods to create Archive instances.+ """++ @staticmethod+ def create(directory=None):+ """+ Create a new Archive instance.++ Args:+ directory: optional default working directory, otherwise uses a temporary directory++ Returns:+ Archive+ """++ return Archive(directory)diff --git a/src/python/txtai/compress/tar.py b/src/python/txtai/archive/tar.pysimilarity index 97%rename from src/python/txtai/compress/tar.pyrename to src/python/txtai/archive/tar.pyindex 4652b3d6..676c0cdc 100644--- a/src/python/txtai/compress/tar.py+++ b/src/python/txtai/archive/tar.py@@ -5,7 +5,7 @@ Tar moduleimport osimport tarfile-from .base import Compress+from .compress import Compressclass Tar(Compress):diff --git a/src/python/txtai/compress/zip.py b/src/python/txtai/archive/zip.pysimilarity index 91%rename from src/python/txtai/compress/zip.pyrename to src/python/txtai/archive/zip.pyindex e88f9376..3c627f20 100644--- a/src/python/txtai/compress/zip.py+++ b/src/python/txtai/archive/zip.py@@ -6,12 +6,12 @@ import osfrom zipfile import ZipFile, ZIP_DEFLATED-from .base import Compress+from .compress import Compressclass Zip(Compress):"""- Zip compression+ Zip archival with compression"""def pack(self, path, output):diff --git a/src/python/txtai/compress/__init__.py b/src/python/txtai/compress/__init__.pydeleted file mode 100644index 0d1b78bd..00000000--- a/src/python/txtai/compress/__init__.py+++ /dev/null@@ -1,8 +0,0 @@-"""-Compress imports-"""--from .base import Compress-from .factory import CompressFactory-from .tar import Tar-from .zip import Zipdiff --git a/src/python/txtai/compress/factory.py b/src/python/txtai/compress/factory.pydeleted file mode 100644index 2e29a69a..00000000--- a/src/python/txtai/compress/factory.py+++ /dev/null@@ -1,27 +0,0 @@-"""-Factory module-"""--from .tar import Tar-from .zip import Zip---class CompressFactory:- """- Methods to create Compress instances.- """-- @staticmethod- def create(path):- """- Factory method to construct a Compress instance.-- Args:- path: file path-- Returns:- Compression- """-- compression = path.lower().split(".")[-1]- return Zip() if compression == "zip" else Tar()