💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add support for loading/storing embedding indexes on cloud storage, closes #229
davidmezzetti41e93f097cf5
diff --git a/setup.py b/setup.pyindex 088f62e7..d2f185b8 100644--- a/setup.py+++ b/setup.py@@ -20,6 +20,8 @@ extras["api"] = ["uvicorn>=0.12.1",]+extras["cloud"] = ["apache-libcloud>=3.3.1"]+extras["database"] = ["pillow>=7.2.0"]extras["model"] = ["onnxruntime>=1.8.1"]@@ -58,7 +60,9 @@ extras["workflow"] = ["xmltodict>=0.12.0",]-extras["all"] = extras["api"] + extras["database"] + extras["model"] + extras["pipeline"] + extras["similarity"] + extras["workflow"]+extras["all"] = (+ extras["api"] + extras["cloud"] + extras["database"] + extras["model"] + extras["pipeline"] + extras["similarity"] + extras["workflow"]+)setup(name="txtai",diff --git a/src/python/txtai/api/base.py b/src/python/txtai/api/base.pyindex e8df6d58..6fc5e3f3 100644--- a/src/python/txtai/api/base.py+++ b/src/python/txtai/api/base.py@@ -47,12 +47,13 @@ class API:# Return unmodifiedreturn data- def __init__(self, config):+ def __init__(self, config, loaddata=True):"""Creates an API instance, which encapsulates embeddings, pipelines and workflows.Args:config: index configuration+ loaddata: If True (default), load existing index data, if available. Otherwise, only load models."""# Initialize member variables@@ -65,10 +66,10 @@ class API:self.pool = None# Local embeddings index- if self.config.get("path") and Embeddings().exists(self.config["path"]):+ if loaddata and self.config.get("path") and Embeddings().exists(self.config["path"], self.config.get("archive")):# Load existing index if availableself.embeddings = Embeddings()- self.embeddings.load(self.config["path"])+ self.embeddings.load(self.config["path"], self.config.get("archive"))elif self.config.get("embeddings"):# Initialize empty embeddingsself.embeddings = Embeddings(self.config["embeddings"])@@ -348,7 +349,7 @@ class API:# Save index if path available, otherwise this is an memory-only indexif self.config.get("path"):- self.embeddings.save(self.config["path"])+ self.embeddings.save(self.config["path"], self.config.get("archive"))# Reset document streamself.documents.close()diff --git a/src/python/txtai/embeddings/__init__.py b/src/python/txtai/embeddings/__init__.pyindex cf310c3d..929cdc48 100644--- a/src/python/txtai/embeddings/__init__.py+++ b/src/python/txtai/embeddings/__init__.py@@ -4,5 +4,6 @@ Embeddings importsfrom .archive import Archivefrom .base import Embeddings+from .cloud import Cloudfrom .documents import Documentsfrom .reducer import Reducerdiff --git a/src/python/txtai/embeddings/archive.py b/src/python/txtai/embeddings/archive.pyindex c96753e9..0689b216 100644--- a/src/python/txtai/embeddings/archive.py+++ b/src/python/txtai/embeddings/archive.py@@ -8,6 +8,8 @@ import tarfilefrom tempfile import TemporaryDirectoryfrom zipfile import ZipFile, ZIP_DEFLATED+from .cloud import Cloud+class Archive:"""@@ -53,14 +55,40 @@ class Archive:return self.directory.name- def load(self, path):+ def exists(self, path, config):+ """+ Checks if file at path exists.++ Args:+ path: path to archive file+ config: additional configuration++ Returns:+ True if path exists, False otherwise+ """++ # Check if archive exists in cloud storage+ cloud = self.cloud(config)+ if cloud:+ return cloud.exists(path)++ # Check if archive exists locally+ return os.path.exists(path)++ def load(self, path, config):"""Extracts file at path to archive working directory.Args:path: path to archive file+ config: additional configuration"""+ # Retrieve archive from cloud storage, if necessary+ cloud = self.cloud(config)+ if cloud:+ cloud.load(path)+# Get compression typecompression = self.compression(path)@@ -74,12 +102,13 @@ class Archive:with tarfile.open(path, f"r:{compression}") as tar:tar.extractall(self.path())- def save(self, path):+ def save(self, path, config):"""Archives files in archive working directory to file at path.Args:path: path to archive file+ config: additional configuration"""# Get compression type@@ -102,6 +131,11 @@ class Archive:with tarfile.open(path, f"w:{compression}") as tar:tar.add(self.path(), arcname=".")+ # Save file to cloud storage, if necessary+ cloud = self.cloud(config)+ if cloud:+ cloud.save(path)+def compression(self, path):"""Gets compression type for path.@@ -114,3 +148,17 @@ class Archive:"""return path.lower().split(".")[-1]++ def cloud(self, config):+ """+ Gets a cloud storage instance, if specified in config.++ Args:+ config: cloud storage configuration++ Returns:+ Cloud storage instance+ """++ # Create cloud storage connection, if necessary+ return Cloud(config) if config and "provider" in config else Nonediff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.pyindex df12d111..7fd2ace1 100644--- a/src/python/txtai/embeddings/base.py+++ b/src/python/txtai/embeddings/base.py@@ -340,18 +340,38 @@ class Embeddings:# Add index and sort desc based on scorereturn [sorted(enumerate(score), key=lambda x: x[1], reverse=True) for score in scores]- def load(self, path):+ def exists(self, path, archive=None):+ """+ Checks if an index exists at path.++ Args:+ path: input path+ archive: archive configuration++ Returns:+ True if index exists, False otherwise+ """++ # Check if this is an archive file and exists+ path, apath = self.checkarchive(path)+ if apath:+ return self.archive.exists(apath, archive)++ return os.path.exists(f"{path}/config") and os.path.exists(f"{path}/embeddings")++ def load(self, path, archive=None):"""Loads an existing index from path.Args:path: input path+ archive: archive configuration"""# Check if this is an archive file and extractpath, apath = self.checkarchive(path)if apath:- self.archive.load(apath)+ self.archive.load(apath, archive)# Index configurationwith open(f"{path}/config", "rb") as handle:@@ -383,25 +403,13 @@ class Embeddings:if self.database:self.database.load(f"{path}/documents")- def exists(self, path):- """- Checks if an index exists at path.-- Args:- path: input path-- Returns:- True if index exists, False otherwise- """-- return os.path.exists(f"{path}/config") and os.path.exists(f"{path}/embeddings")-- def save(self, path):+ def save(self, path, archive=None):"""Saves an index.Args:path: output path+ archive: archive configuration"""if self.config:@@ -438,7 +446,7 @@ class Embeddings:# If this is an archive, save itif apath:- self.archive.save(apath)+ self.archive.save(apath, archive)def close(self):"""diff --git a/src/python/txtai/embeddings/cloud.py b/src/python/txtai/embeddings/cloud.pynew file mode 100644index 00000000..a7f8aade--- /dev/null+++ b/src/python/txtai/embeddings/cloud.py@@ -0,0 +1,88 @@+"""+Cloud module+"""++import os++# Conditional import+try:+ from libcloud.storage.providers import get_driver+ from libcloud.storage.types import ContainerDoesNotExistError, ObjectDoesNotExistError++ LIBCLOUD = True+except ImportError:+ LIBCLOUD = False+++class Cloud:+ """+ Methods to sync files with cloud storage.+ """++ def __init__(self, config):+ """+ Creates a new cloud storage client connection.++ Args:+ config: cloud storage configuration+ """++ if not LIBCLOUD:+ raise ImportError('Cloud storage is not available - install "cloud" extra to enable')++ self.config = config++ # Get driver for provider+ driver = get_driver(config["provider"])++ # Get client connection+ self.client = driver(+ config.get("key", os.environ.get("access_key")), config.get("secret", os.environ.get("access_secret")), region=config.get("region")+ )++ def exists(self, path):+ """+ Checks if path exists.++ Args:+ path: path to check++ Returns:+ True if path exists, False otherwise+ """++ try:+ self.client.get_object(self.config["container"], os.path.basename(path))+ return True+ except (ContainerDoesNotExistError, ObjectDoesNotExistError):+ return False++ def load(self, path):+ """+ Retrieves file from cloud storage and stores locally.++ Args:+ path: path to retrieve+ """++ # Download object+ obj = self.client.get_object(self.config["container"], os.path.basename(path))+ obj.download(path, overwrite_existing=True)++ def save(self, path):+ """+ Sends local file to cloud storage.++ Args:+ path: path to send+ """++ # Get or create container+ try:+ container = self.client.get_container(self.config["container"])+ except ContainerDoesNotExistError:+ container = self.client.create_container(self.config["container"])++ # Upload object+ with open(path, "rb") as iterator:+ self.client.upload_object_via_stream(iterator=iterator, container=container, object_name=os.path.basename(path))diff --git a/src/python/txtai/workflow/task/storage.py b/src/python/txtai/workflow/task/storage.pyindex 9327084f..dd1217dc 100644--- a/src/python/txtai/workflow/task/storage.py+++ b/src/python/txtai/workflow/task/storage.py@@ -75,8 +75,8 @@ class StorageTask(Task):key, container = os.path.dirname(path), os.path.basename(path)- client = get_driver(provider)- driver = client(key)+ driver = get_driver(provider)+ client = driver(key)- container = driver.get_container(container_name=container)- return [driver.get_object_cdn_url(obj) for obj in driver.list_container_objects(container=container)]+ container = client.get_container(container_name=container)+ return [client.get_object_cdn_url(obj) for obj in client.list_container_objects(container=container)]diff --git a/test/python/testdatabase/testembeddings.py b/test/python/testdatabase/testembeddings.pyindex 30802fcf..60c18adf 100644--- a/test/python/testdatabase/testembeddings.py+++ b/test/python/testdatabase/testembeddings.py@@ -6,8 +6,11 @@ import contextlibimport ioimport osimport tempfile+import timeimport unittest+from libcloud.storage.types import ContainerDoesNotExistError, ObjectDoesNotExistError+from txtai.embeddings import Embeddingsfrom txtai.database import Database, SQLException@@ -68,6 +71,43 @@ class TestEmbeddings(unittest.TestCase):self.embeddings.upsert([(0, "Looking out into the dreadful abyss", None)])self.assertEqual(self.embeddings.count(), len(self.data))+ def testArchiveCloud(self):+ """+ Tests embeddings index archiving with cloud storage+ """++ # Create an index for the list of text+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])++ # Generate temp file path+ index = os.path.join(tempfile.gettempdir(), "cloud.tar.gz")++ # Test save with local cloud provider+ cloud = {"provider": "local", "container": f"cloud.{time.time()}", "key": tempfile.gettempdir()}++ # Test exists handles missing cloud storage object+ self.assertFalse(self.embeddings.exists(index, cloud))++ # Test exception raised when trying to load index and doesn't exist in cloud storage+ with self.assertRaises((ContainerDoesNotExistError, ObjectDoesNotExistError)):+ self.embeddings.load(index, cloud)++ # Save index+ self.embeddings.save(index, cloud)++ # Test object exists in cloud storage+ self.assertTrue(self.embeddings.exists(index, cloud))++ # Test object exists locally+ self.assertTrue(self.embeddings.exists(index))++ # Test index can be reloaded+ self.embeddings.load(index, cloud)++ # Search for best match+ result = self.embeddings.search("feel good story", 1)[0]+ self.assertEqual(result["text"], self.data[4])+def testClose(self):"""Tests embeddings closediff --git a/test/python/testoptional.py b/test/python/testoptional.pyindex 86fccc42..98607497 100644--- a/test/python/testoptional.py+++ b/test/python/testoptional.py@@ -77,6 +77,16 @@ class TestOptional(unittest.TestCase):with self.assertRaises(ImportError):import txtai.api+ def testCloud(self):+ """+ Test missing cloud dependencies+ """++ from txtai.embeddings import Cloud++ with self.assertRaises(ImportError):+ Cloud(None)+def testDatabase(self):"""Test missing database dependencies