Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add support for loading/storing embedding indexes on cloud storage, closes #229

davidmezzetti41e93f097cf5
diff --git a/setup.py b/setup.py
index 088f62e7..d2f185b8 100644
--- a/setup.py
+++ b/setup.py
@@ -20,6 +20,8 @@ extras["api"] = [
"uvicorn>=0.12.1",
]
+extras["cloud"] = ["apache-libcloud>=3.3.1"]
+
extras["database"] = ["pillow>=7.2.0"]
extras["model"] = ["onnxruntime>=1.8.1"]
@@ -58,7 +60,9 @@ extras["workflow"] = [
"xmltodict>=0.12.0",
]
-extras["all"] = extras["api"] + extras["database"] + extras["model"] + extras["pipeline"] + extras["similarity"] + extras["workflow"]
+extras["all"] = (
+ extras["api"] + extras["cloud"] + extras["database"] + extras["model"] + extras["pipeline"] + extras["similarity"] + extras["workflow"]
+)
setup(
name="txtai",
diff --git a/src/python/txtai/api/base.py b/src/python/txtai/api/base.py
index e8df6d58..6fc5e3f3 100644
--- a/src/python/txtai/api/base.py
+++ b/src/python/txtai/api/base.py
@@ -47,12 +47,13 @@ class API:
# Return unmodified
return data
- def __init__(self, config):
+ def __init__(self, config, loaddata=True):
"""
Creates an API instance, which encapsulates embeddings, pipelines and workflows.
Args:
config: index configuration
+ loaddata: If True (default), load existing index data, if available. Otherwise, only load models.
"""
# Initialize member variables
@@ -65,10 +66,10 @@ class API:
self.pool = None
# Local embeddings index
- if self.config.get("path") and Embeddings().exists(self.config["path"]):
+ if loaddata and self.config.get("path") and Embeddings().exists(self.config["path"], self.config.get("archive")):
# Load existing index if available
self.embeddings = Embeddings()
- self.embeddings.load(self.config["path"])
+ self.embeddings.load(self.config["path"], self.config.get("archive"))
elif self.config.get("embeddings"):
# Initialize empty embeddings
self.embeddings = Embeddings(self.config["embeddings"])
@@ -348,7 +349,7 @@ class API:
# Save index if path available, otherwise this is an memory-only index
if self.config.get("path"):
- self.embeddings.save(self.config["path"])
+ self.embeddings.save(self.config["path"], self.config.get("archive"))
# Reset document stream
self.documents.close()
diff --git a/src/python/txtai/embeddings/__init__.py b/src/python/txtai/embeddings/__init__.py
index cf310c3d..929cdc48 100644
--- a/src/python/txtai/embeddings/__init__.py
+++ b/src/python/txtai/embeddings/__init__.py
@@ -4,5 +4,6 @@ Embeddings imports
from .archive import Archive
from .base import Embeddings
+from .cloud import Cloud
from .documents import Documents
from .reducer import Reducer
diff --git a/src/python/txtai/embeddings/archive.py b/src/python/txtai/embeddings/archive.py
index c96753e9..0689b216 100644
--- a/src/python/txtai/embeddings/archive.py
+++ b/src/python/txtai/embeddings/archive.py
@@ -8,6 +8,8 @@ import tarfile
from tempfile import TemporaryDirectory
from zipfile import ZipFile, ZIP_DEFLATED
+from .cloud import Cloud
+
class Archive:
"""
@@ -53,14 +55,40 @@ class Archive:
return self.directory.name
- def load(self, path):
+ def exists(self, path, config):
+ """
+ Checks if file at path exists.
+
+ Args:
+ path: path to archive file
+ config: additional configuration
+
+ Returns:
+ True if path exists, False otherwise
+ """
+
+ # Check if archive exists in cloud storage
+ cloud = self.cloud(config)
+ if cloud:
+ return cloud.exists(path)
+
+ # Check if archive exists locally
+ return os.path.exists(path)
+
+ def load(self, path, config):
"""
Extracts file at path to archive working directory.
Args:
path: path to archive file
+ config: additional configuration
"""
+ # Retrieve archive from cloud storage, if necessary
+ cloud = self.cloud(config)
+ if cloud:
+ cloud.load(path)
+
# Get compression type
compression = self.compression(path)
@@ -74,12 +102,13 @@ class Archive:
with tarfile.open(path, f"r:{compression}") as tar:
tar.extractall(self.path())
- def save(self, path):
+ def save(self, path, config):
"""
Archives files in archive working directory to file at path.
Args:
path: path to archive file
+ config: additional configuration
"""
# Get compression type
@@ -102,6 +131,11 @@ class Archive:
with tarfile.open(path, f"w:{compression}") as tar:
tar.add(self.path(), arcname=".")
+ # Save file to cloud storage, if necessary
+ cloud = self.cloud(config)
+ if cloud:
+ cloud.save(path)
+
def compression(self, path):
"""
Gets compression type for path.
@@ -114,3 +148,17 @@ class Archive:
"""
return path.lower().split(".")[-1]
+
+ def cloud(self, config):
+ """
+ Gets a cloud storage instance, if specified in config.
+
+ Args:
+ config: cloud storage configuration
+
+ Returns:
+ Cloud storage instance
+ """
+
+ # Create cloud storage connection, if necessary
+ return Cloud(config) if config and "provider" in config else None
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.py
index df12d111..7fd2ace1 100644
--- a/src/python/txtai/embeddings/base.py
+++ b/src/python/txtai/embeddings/base.py
@@ -340,18 +340,38 @@ class Embeddings:
# Add index and sort desc based on score
return [sorted(enumerate(score), key=lambda x: x[1], reverse=True) for score in scores]
- def load(self, path):
+ def exists(self, path, archive=None):
+ """
+ Checks if an index exists at path.
+
+ Args:
+ path: input path
+ archive: archive configuration
+
+ Returns:
+ True if index exists, False otherwise
+ """
+
+ # Check if this is an archive file and exists
+ path, apath = self.checkarchive(path)
+ if apath:
+ return self.archive.exists(apath, archive)
+
+ return os.path.exists(f"{path}/config") and os.path.exists(f"{path}/embeddings")
+
+ def load(self, path, archive=None):
"""
Loads an existing index from path.
Args:
path: input path
+ archive: archive configuration
"""
# Check if this is an archive file and extract
path, apath = self.checkarchive(path)
if apath:
- self.archive.load(apath)
+ self.archive.load(apath, archive)
# Index configuration
with open(f"{path}/config", "rb") as handle:
@@ -383,25 +403,13 @@ class Embeddings:
if self.database:
self.database.load(f"{path}/documents")
- def exists(self, path):
- """
- Checks if an index exists at path.
-
- Args:
- path: input path
-
- Returns:
- True if index exists, False otherwise
- """
-
- return os.path.exists(f"{path}/config") and os.path.exists(f"{path}/embeddings")
-
- def save(self, path):
+ def save(self, path, archive=None):
"""
Saves an index.
Args:
path: output path
+ archive: archive configuration
"""
if self.config:
@@ -438,7 +446,7 @@ class Embeddings:
# If this is an archive, save it
if apath:
- self.archive.save(apath)
+ self.archive.save(apath, archive)
def close(self):
"""
diff --git a/src/python/txtai/embeddings/cloud.py b/src/python/txtai/embeddings/cloud.py
new file mode 100644
index 00000000..a7f8aade
--- /dev/null
+++ b/src/python/txtai/embeddings/cloud.py
@@ -0,0 +1,88 @@
+"""
+Cloud module
+"""
+
+import os
+
+# Conditional import
+try:
+ from libcloud.storage.providers import get_driver
+ from libcloud.storage.types import ContainerDoesNotExistError, ObjectDoesNotExistError
+
+ LIBCLOUD = True
+except ImportError:
+ LIBCLOUD = False
+
+
+class Cloud:
+ """
+ Methods to sync files with cloud storage.
+ """
+
+ def __init__(self, config):
+ """
+ Creates a new cloud storage client connection.
+
+ Args:
+ config: cloud storage configuration
+ """
+
+ if not LIBCLOUD:
+ raise ImportError('Cloud storage is not available - install "cloud" extra to enable')
+
+ self.config = config
+
+ # Get driver for provider
+ driver = get_driver(config["provider"])
+
+ # Get client connection
+ self.client = driver(
+ config.get("key", os.environ.get("access_key")), config.get("secret", os.environ.get("access_secret")), region=config.get("region")
+ )
+
+ def exists(self, path):
+ """
+ Checks if path exists.
+
+ Args:
+ path: path to check
+
+ Returns:
+ True if path exists, False otherwise
+ """
+
+ try:
+ self.client.get_object(self.config["container"], os.path.basename(path))
+ return True
+ except (ContainerDoesNotExistError, ObjectDoesNotExistError):
+ return False
+
+ def load(self, path):
+ """
+ Retrieves file from cloud storage and stores locally.
+
+ Args:
+ path: path to retrieve
+ """
+
+ # Download object
+ obj = self.client.get_object(self.config["container"], os.path.basename(path))
+ obj.download(path, overwrite_existing=True)
+
+ def save(self, path):
+ """
+ Sends local file to cloud storage.
+
+ Args:
+ path: path to send
+ """
+
+ # Get or create container
+ try:
+ container = self.client.get_container(self.config["container"])
+ except ContainerDoesNotExistError:
+ container = self.client.create_container(self.config["container"])
+
+ # Upload object
+ with open(path, "rb") as iterator:
+ self.client.upload_object_via_stream(iterator=iterator, container=container, object_name=os.path.basename(path))
diff --git a/src/python/txtai/workflow/task/storage.py b/src/python/txtai/workflow/task/storage.py
index 9327084f..dd1217dc 100644
--- a/src/python/txtai/workflow/task/storage.py
+++ b/src/python/txtai/workflow/task/storage.py
@@ -75,8 +75,8 @@ class StorageTask(Task):
key, container = os.path.dirname(path), os.path.basename(path)
- client = get_driver(provider)
- driver = client(key)
+ driver = get_driver(provider)
+ client = driver(key)
- container = driver.get_container(container_name=container)
- return [driver.get_object_cdn_url(obj) for obj in driver.list_container_objects(container=container)]
+ container = client.get_container(container_name=container)
+ return [client.get_object_cdn_url(obj) for obj in client.list_container_objects(container=container)]
diff --git a/test/python/testdatabase/testembeddings.py b/test/python/testdatabase/testembeddings.py
index 30802fcf..60c18adf 100644
--- a/test/python/testdatabase/testembeddings.py
+++ b/test/python/testdatabase/testembeddings.py
@@ -6,8 +6,11 @@ import contextlib
import io
import os
import tempfile
+import time
import unittest
+from libcloud.storage.types import ContainerDoesNotExistError, ObjectDoesNotExistError
+
from txtai.embeddings import Embeddings
from txtai.database import Database, SQLException
@@ -68,6 +71,43 @@ class TestEmbeddings(unittest.TestCase):
self.embeddings.upsert([(0, "Looking out into the dreadful abyss", None)])
self.assertEqual(self.embeddings.count(), len(self.data))
+ def testArchiveCloud(self):
+ """
+ Tests embeddings index archiving with cloud storage
+ """
+
+ # Create an index for the list of text
+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
+
+ # Generate temp file path
+ index = os.path.join(tempfile.gettempdir(), "cloud.tar.gz")
+
+ # Test save with local cloud provider
+ cloud = {"provider": "local", "container": f"cloud.{time.time()}", "key": tempfile.gettempdir()}
+
+ # Test exists handles missing cloud storage object
+ self.assertFalse(self.embeddings.exists(index, cloud))
+
+ # Test exception raised when trying to load index and doesn't exist in cloud storage
+ with self.assertRaises((ContainerDoesNotExistError, ObjectDoesNotExistError)):
+ self.embeddings.load(index, cloud)
+
+ # Save index
+ self.embeddings.save(index, cloud)
+
+ # Test object exists in cloud storage
+ self.assertTrue(self.embeddings.exists(index, cloud))
+
+ # Test object exists locally
+ self.assertTrue(self.embeddings.exists(index))
+
+ # Test index can be reloaded
+ self.embeddings.load(index, cloud)
+
+ # Search for best match
+ result = self.embeddings.search("feel good story", 1)[0]
+ self.assertEqual(result["text"], self.data[4])
+
def testClose(self):
"""
Tests embeddings close
diff --git a/test/python/testoptional.py b/test/python/testoptional.py
index 86fccc42..98607497 100644
--- a/test/python/testoptional.py
+++ b/test/python/testoptional.py
@@ -77,6 +77,16 @@ class TestOptional(unittest.TestCase):
with self.assertRaises(ImportError):
import txtai.api
+ def testCloud(self):
+ """
+ Test missing cloud dependencies
+ """
+
+ from txtai.embeddings import Cloud
+
+ with self.assertRaises(ImportError):
+ Cloud(None)
+
def testDatabase(self):
"""
Test missing database dependencies