Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Refactor archive module into separate package, closes #429

davidmezzetti399dc56bef77
diff --git a/src/python/txtai/archive/__init__.py b/src/python/txtai/archive/__init__.py
new file mode 100644
index 00000000..398bb580
--- /dev/null
+++ b/src/python/txtai/archive/__init__.py
@@ -0,0 +1,9 @@
+"""
+Archive imports
+"""
+
+from .base import Archive
+from .compress import Compress
+from .factory import ArchiveFactory
+from .tar import Tar
+from .zip import Zip
diff --git a/src/python/txtai/embeddings/archive.py b/src/python/txtai/archive/base.py
similarity index 52%
rename from src/python/txtai/embeddings/archive.py
rename to src/python/txtai/archive/base.py
index b52ba488..8afd68da 100644
--- a/src/python/txtai/embeddings/archive.py
+++ b/src/python/txtai/archive/base.py
@@ -6,14 +6,13 @@ import os
from tempfile import TemporaryDirectory
-from ..compress import CompressFactory
-
-from .cloud import Cloud
+from .tar import Tar
+from .zip import Zip
class Archive:
"""
- Methods to load and save archive files.
+ Base class for archive instances.
"""
def __init__(self, directory=None):
@@ -37,7 +36,7 @@ class Archive:
True if the path ends with an archive extension, False otherwise
"""
- return any(path.lower().endswith(extension) for extension in [".tar.bz2", ".tar.gz", ".tar.xz", ".zip"])
+ return path and any(path.lower().endswith(extension) for extension in [".tar.bz2", ".tar.gz", ".tar.xz", ".zip"])
def path(self):
"""
@@ -53,53 +52,28 @@ class Archive:
# pylint: disable=R1732
self.directory = TemporaryDirectory()
- return self.directory.name
-
- def exists(self, path, config):
- """
- Checks if file at path exists.
-
- Args:
- path: path to archive file
- config: additional configuration
-
- Returns:
- True if path exists, False otherwise
- """
-
- # Check if archive exists in cloud storage
- cloud = self.cloud(config)
- if cloud:
- return cloud.exists(path)
-
- # Check if archive exists locally
- return os.path.exists(path)
+ return self.directory.name if isinstance(self.directory, TemporaryDirectory) else self.directory
- def load(self, path, config):
+ def load(self, path, compression=None):
"""
Extracts file at path to archive working directory.
Args:
path: path to archive file
- config: additional configuration
+ compression: compression format, infers from path if not provided
"""
- # Retrieve archive from cloud storage, if necessary
- cloud = self.cloud(config)
- if cloud:
- cloud.load(path)
-
# Unpack compressed file
- compress = CompressFactory().create(path)
+ compress = self.create(path, compression)
compress.unpack(path, self.path())
- def save(self, path, config):
+ def save(self, path, compression=None):
"""
Archives files in archive working directory to file at path.
Args:
path: path to archive file
- config: additional configuration
+ compression: compression format, infers from path if not provided
"""
# Create output directory, if necessary
@@ -108,24 +82,23 @@ class Archive:
os.makedirs(output, exist_ok=True)
# Pack into compressed file
- compress = CompressFactory().create(path)
+ compress = self.create(path, compression)
compress.pack(self.path(), path)
- # Save file to cloud storage, if necessary
- cloud = self.cloud(config)
- if cloud:
- cloud.save(path)
-
- def cloud(self, config):
+ def create(self, path, compression):
"""
- Gets a cloud storage instance, if specified in config.
+ Method to construct a Compress instance.
Args:
- config: cloud storage configuration
+ path: file path
+ compression: compression format, infers using file extension if not provided
Returns:
- Cloud storage instance
+ Compress
"""
- # Create cloud storage connection, if necessary
- return Cloud(config) if config and "provider" in config else None
+ # Infer compression format from path if not provided
+ compression = compression if compression else path.lower().split(".")[-1]
+
+ # Create compression instance
+ return Zip() if compression == "zip" else Tar()
diff --git a/src/python/txtai/compress/base.py b/src/python/txtai/archive/compress.py
similarity index 100%
rename from src/python/txtai/compress/base.py
rename to src/python/txtai/archive/compress.py
diff --git a/src/python/txtai/archive/factory.py b/src/python/txtai/archive/factory.py
new file mode 100644
index 00000000..57b63a94
--- /dev/null
+++ b/src/python/txtai/archive/factory.py
@@ -0,0 +1,25 @@
+"""
+Factory module
+"""
+
+from .base import Archive
+
+
+class ArchiveFactory:
+ """
+ Methods to create Archive instances.
+ """
+
+ @staticmethod
+ def create(directory=None):
+ """
+ Create a new Archive instance.
+
+ Args:
+ directory: optional default working directory, otherwise uses a temporary directory
+
+ Returns:
+ Archive
+ """
+
+ return Archive(directory)
diff --git a/src/python/txtai/compress/tar.py b/src/python/txtai/archive/tar.py
similarity index 97%
rename from src/python/txtai/compress/tar.py
rename to src/python/txtai/archive/tar.py
index 4652b3d6..676c0cdc 100644
--- a/src/python/txtai/compress/tar.py
+++ b/src/python/txtai/archive/tar.py
@@ -5,7 +5,7 @@ Tar module
import os
import tarfile
-from .base import Compress
+from .compress import Compress
class Tar(Compress):
diff --git a/src/python/txtai/compress/zip.py b/src/python/txtai/archive/zip.py
similarity index 91%
rename from src/python/txtai/compress/zip.py
rename to src/python/txtai/archive/zip.py
index e88f9376..3c627f20 100644
--- a/src/python/txtai/compress/zip.py
+++ b/src/python/txtai/archive/zip.py
@@ -6,12 +6,12 @@ import os
from zipfile import ZipFile, ZIP_DEFLATED
-from .base import Compress
+from .compress import Compress
class Zip(Compress):
"""
- Zip compression
+ Zip archival with compression
"""
def pack(self, path, output):
diff --git a/src/python/txtai/compress/__init__.py b/src/python/txtai/compress/__init__.py
deleted file mode 100644
index 0d1b78bd..00000000
--- a/src/python/txtai/compress/__init__.py
+++ /dev/null
@@ -1,8 +0,0 @@
-"""
-Compress imports
-"""
-
-from .base import Compress
-from .factory import CompressFactory
-from .tar import Tar
-from .zip import Zip
diff --git a/src/python/txtai/compress/factory.py b/src/python/txtai/compress/factory.py
deleted file mode 100644
index 2e29a69a..00000000
--- a/src/python/txtai/compress/factory.py
+++ /dev/null
@@ -1,27 +0,0 @@
-"""
-Factory module
-"""
-
-from .tar import Tar
-from .zip import Zip
-
-
-class CompressFactory:
- """
- Methods to create Compress instances.
- """
-
- @staticmethod
- def create(path):
- """
- Factory method to construct a Compress instance.
-
- Args:
- path: file path
-
- Returns:
- Compression
- """
-
- compression = path.lower().split(".")[-1]
- return Zip() if compression == "zip" else Tar()