💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add huggingface-hub integration, closes #430. Add cloud object storage support for standard embeddings indexes, closes #431. Add support for custom cloud providers, closes #432. Add support for storing embeddings config as JSON, closes #433.
davidmezzetti95697322c9b3
diff --git a/src/python/txtai/app/base.py b/src/python/txtai/app/base.pyindex 4b31a80e..3fd52a60 100644--- a/src/python/txtai/app/base.py+++ b/src/python/txtai/app/base.py@@ -185,11 +185,11 @@ class Application:# Resolve transform functionconfig["transform"] = self.function(config["transform"])- # Local embeddings index- if loaddata and self.config.get("path") and Embeddings().exists(self.config["path"], self.config.get("cloud")):+ # Load embeddings index if loaddata and index exists+ if loaddata and Embeddings().exists(self.config.get("path"), self.config.get("cloud")):# Load existing index if availableself.embeddings = Embeddings()- self.embeddings.load(self.config["path"], self.config.get("cloud"))+ self.embeddings.load(self.config.get("path"), self.config.get("cloud"))elif config:# Initialize empty embeddingsself.embeddings = Embeddings(config)diff --git a/src/python/txtai/cloud/__init__.py b/src/python/txtai/cloud/__init__.pynew file mode 100644index 00000000..e2fd6d71--- /dev/null+++ b/src/python/txtai/cloud/__init__.py@@ -0,0 +1,8 @@+"""+Cloud imports+"""++from .base import Cloud+from .factory import CloudFactory+from .hub import HuggingFaceHub+from .storage import ObjectStoragediff --git a/src/python/txtai/cloud/base.py b/src/python/txtai/cloud/base.pynew file mode 100644index 00000000..0c5e53dc--- /dev/null+++ b/src/python/txtai/cloud/base.py@@ -0,0 +1,106 @@+"""+Cloud module+"""++import os++from ..archive import ArchiveFactory+++class Cloud:+ """+ Base class for cloud providers. Cloud providers sync content between local and remote storage.+ """++ def __init__(self, config):+ """+ Creates a new cloud connection.++ Args:+ config: cloud configuration+ """++ self.config = config++ def exists(self, path=None):+ """+ Checks if path exists in cloud. If path is None, this method checks if the container exists.++ Args:+ path: path to check++ Returns:+ True if path or container exists, False otherwise+ """++ return self.metadata(path) is not None++ def metadata(self, path=None):+ """+ Returns metadata for path from cloud. If path is None, this method returns metadata+ for container.++ Args:+ path: retrieve metadata for this path++ Returns:+ path or container metadata if available, otherwise returns None+ """++ raise NotImplementedError++ def load(self, path=None):+ """+ Retrieves content from cloud and stores locally. If path is empty, this method retrieves+ all content in the container.++ Args:+ path: path to retrieve++ Returns:+ local path which can be different than input path+ """++ raise NotImplementedError++ def save(self, path):+ """+ Sends local content stored in path to cloud.++ Args:+ path: local path to sync+ """++ raise NotImplementedError++ def isarchive(self, path):+ """+ Check if path is an archive file.++ Args:+ path: path to check++ Returns:+ True if path ends with an archive extension, false otherwise+ """++ return ArchiveFactory.create().isarchive(path)++ def listfiles(self, path):+ """+ Lists files in path. If path is a file, this method returns a single element list+ containing path.++ Args:+ path: path to list++ Returns:+ List of files+ """++ # List all files if path is a directory+ if os.path.isdir(path):+ return [os.path.join(path, f) for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]++ # Path is a file+ return [path]diff --git a/src/python/txtai/cloud/factory.py b/src/python/txtai/cloud/factory.pynew file mode 100644index 00000000..ff967b9a--- /dev/null+++ b/src/python/txtai/cloud/factory.py@@ -0,0 +1,70 @@+"""+Factory module+"""++from ..util import Resolver++from .hub import HuggingFaceHub+from .storage import ObjectStorage, LIBCLOUD+++class CloudFactory:+ """+ Methods to create Cloud instances.+ """++ @staticmethod+ def create(config):+ """+ Creates a Cloud instance.++ Args:+ config: cloud configuration++ Returns:+ Cloud+ """++ # Cloud instance+ cloud = None++ provider = config.get("provider", "")++ # Hugging Face Hub+ if provider.lower() == "huggingface-hub":+ cloud = HuggingFaceHub(config)++ # Cloud object storage+ elif ObjectStorage.isprovider(provider):+ cloud = ObjectStorage(config)++ # External provider+ elif provider:+ cloud = CloudFactory.resolve(provider, config)++ return cloud++ @staticmethod+ def resolve(backend, config):+ """+ Attempt to resolve a custom cloud backend.++ Args:+ backend: backend class+ config: configuration parameters++ Returns:+ Cloud+ """++ try:+ return Resolver()(backend)(config)++ except Exception as e:+ # Failure message+ message = f'Unable to resolve cloud backend: "{backend}".'++ # Append message if LIBCLOUD is not installed+ message += ' Cloud storage is not available - install "cloud" extra to enable' if not LIBCLOUD else ""++ raise ImportError(message) from ediff --git a/src/python/txtai/cloud/hub.py b/src/python/txtai/cloud/hub.pynew file mode 100644index 00000000..e69fb95c--- /dev/null+++ b/src/python/txtai/cloud/hub.py@@ -0,0 +1,65 @@+"""+Hugging Face Hub module+"""++import os++import huggingface_hub++from huggingface_hub.utils import RepositoryNotFoundError++from .base import Cloud+++class HuggingFaceHub(Cloud):+ """+ Hugging Face Hub cloud provider.+ """++ def metadata(self, path=None):+ try:+ # If this is an archive file path, get file metadata+ if self.isarchive(path):+ url = huggingface_hub.hf_hub_url(+ repo_id=self.config["container"], filename=os.path.basename(path), revision=self.config.get("revision")+ )++ return huggingface_hub.get_hf_file_metadata(url=url, token=self.config.get("token"))++ # Otherwise return repository metadata+ return huggingface_hub.model_info(repo_id=self.config["container"], revision=self.config.get("revision"), token=self.config.get("token"))++ except RepositoryNotFoundError:+ return None++ def load(self, path=None):+ # Download archvie file and return local path+ if self.isarchive(path):+ return huggingface_hub.hf_hub_download(+ repo_id=self.config["container"],+ filename=os.path.basename(path),+ revision=self.config.get("revision"),+ cache_dir=self.config.get("cache"),+ token=self.config.get("token"),+ )++ # Download repository and return cached path+ return huggingface_hub.snapshot_download(+ repo_id=self.config["container"], revision=self.config.get("revision"), cache_dir=self.config.get("cache"), token=self.config.get("token")+ )++ def save(self, path):+ # Get or create repository+ huggingface_hub.create_repo(+ repo_id=self.config["container"], token=self.config.get("token"), private=self.config.get("private", True), exist_ok=True+ )++ # Upload files+ for f in self.listfiles(path):+ huggingface_hub.upload_file(+ repo_id=self.config["container"],+ revision=self.config.get("revision"),+ token=self.config.get("token"),+ path_or_fileobj=f,+ path_in_repo=os.path.basename(f),+ )diff --git a/src/python/txtai/cloud/storage.py b/src/python/txtai/cloud/storage.pynew file mode 100644index 00000000..ac13b1ec--- /dev/null+++ b/src/python/txtai/cloud/storage.py@@ -0,0 +1,96 @@+"""+Object storage module+"""++import os++# Conditional import+try:+ from libcloud.storage.providers import get_driver, DRIVERS+ from libcloud.storage.types import ContainerDoesNotExistError, ObjectDoesNotExistError++ LIBCLOUD = True+except ImportError:+ LIBCLOUD, DRIVERS = False, None+++from .base import Cloud+++class ObjectStorage(Cloud):+ """+ Object storage cloud provider backed by Apache libcloud.+ """++ @staticmethod+ def isprovider(provider):+ """+ Checks if this provider is an object storage provider.++ Args:+ provider: provider name++ Returns:+ True if this is an object storage provider+ """++ return LIBCLOUD and provider and provider in DRIVERS++ def __init__(self, config):+ super().__init__(config)++ if not LIBCLOUD:+ raise ImportError('Cloud object storage is not available - install "cloud" extra to enable')++ # Get driver for provider+ driver = get_driver(config["provider"])++ # Get client connection+ self.client = driver(+ config.get("key", os.environ.get("ACCESS_KEY")),+ config.get("secret", os.environ.get("ACCESS_SECRET")),+ host=config.get("host"),+ port=config.get("port"),+ token=config.get("token"),+ region=config.get("region"),+ )++ def metadata(self, path=None):+ try:+ # If this is an archive path, check if file exists+ if self.isarchive(path):+ return self.client.get_object(self.config["container"], os.path.basename(path))++ # Otherwise check if container exists+ return self.client.get_container(self.config["container"])+ except (ContainerDoesNotExistError, ObjectDoesNotExistError):+ return None++ def load(self, path=None):+ # Download archive file+ if self.isarchive(path):+ obj = self.client.get_object(self.config["container"], os.path.basename(path))+ obj.download(path, overwrite_existing=True)++ # Download all files in container+ else:+ # Create local directory, if necessary+ os.makedirs(path, exist_ok=True)++ container = self.client.get_container(self.config["container"])+ for obj in container.list_objects():+ obj.download(os.path.join(path, obj.name), overwrite_existing=True)++ return path++ def save(self, path):+ # Get or create container+ try:+ container = self.client.get_container(self.config["container"])+ except ContainerDoesNotExistError:+ container = self.client.create_container(self.config["container"])++ # Upload files+ for f in self.listfiles(path):+ with open(f, "rb") as iterator:+ self.client.upload_object_via_stream(iterator=iterator, container=container, object_name=os.path.basename(f))diff --git a/src/python/txtai/embeddings/__init__.py b/src/python/txtai/embeddings/__init__.pyindex 6152117e..55d3c39c 100644--- a/src/python/txtai/embeddings/__init__.py+++ b/src/python/txtai/embeddings/__init__.py@@ -2,11 +2,10 @@Embeddings imports"""-from .archive import Archivefrom .base import Embeddings-from .cloud import Cloudfrom .documents import Documentsfrom .explain import Explain+from .functions import *from .query import Queryfrom .reducer import Reducerfrom .search import Searchdiff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.pyindex e7721fc6..9629ebce 100644--- a/src/python/txtai/embeddings/base.py+++ b/src/python/txtai/embeddings/base.py@@ -13,12 +13,13 @@ import numpy as npfrom .. import __pickle__from ..ann import ANNFactory+from ..archive import ArchiveFactory+from ..cloud import CloudFactoryfrom ..database import DatabaseFactoryfrom ..graph import GraphFactoryfrom ..scoring import ScoringFactoryfrom ..vectors import VectorsFactory-from .archive import Archivefrom .explain import Explainfrom .functions import Functionsfrom .reducer import Reducer@@ -269,19 +270,20 @@ class Embeddings:return self.batchtransform([document])[0]- def batchtransform(self, documents):+ def batchtransform(self, documents, category=None):"""Transforms documents into embeddings vectors.Args:documents: list of (id, data, tags)+ category: category for instruction-based embeddingsReturns:embeddings vectors"""# Convert documents into sentence embeddings- embeddings = self.model.batchtransform(documents)+ embeddings = self.model.batchtransform(documents, category)# Reduce the dimensionality of the embeddings. Scale the embeddings using this# model to reduce the noise of common but less relevant terms.@@ -365,8 +367,8 @@ class Embeddings:"""# Convert queries to embedding vectors- queries = self.batchtransform((None, query, None) for query in queries)- data = self.batchtransform((None, row, None) for row in data)+ queries = self.batchtransform(((None, query, None) for query in queries), "query")+ data = self.batchtransform(((None, row, None) for row in data), "data")# Dot product on normalized vectors is equal to cosine similarityscores = np.dot(queries, data.T).tolist()@@ -405,46 +407,54 @@ class Embeddings:return Explain(self)(queries, texts, limit)- def exists(self, path, cloud=None):+ def exists(self, path=None, cloud=None, **kwargs):"""Checks if an index exists at path.Args:path: input pathcloud: cloud storage configuration+ kwargs: additional configuration as keyword argsReturns:True if index exists, False otherwise"""+ # Check if this exists in a cloud instance+ cloud = self.createcloud(cloud=cloud, **kwargs)+ if cloud:+ return cloud.exists(path)+# Check if this is an archive file and existspath, apath = self.checkarchive(path)if apath:- return self.archive.exists(apath, cloud)+ return os.path.exists(apath)- return os.path.exists(f"{path}/config") and os.path.exists(f"{path}/embeddings")+ # Return true if path has a config or config.json file and an embeddings file+ return path and (os.path.exists(f"{path}/config") or os.path.exists(f"{path}/config.json")) and os.path.exists(f"{path}/embeddings")- def load(self, path, cloud=None):+ def load(self, path=None, cloud=None, **kwargs):"""Loads an existing index from path.Args:path: input pathcloud: cloud storage configuration+ kwargs: additional configuration as keyword args"""+ # Load from cloud, if configured+ cloud = self.createcloud(cloud=cloud, **kwargs)+ if cloud:+ path = cloud.load(path)+# Check if this is an archive file and extractpath, apath = self.checkarchive(path)if apath:- self.archive.load(apath, cloud)+ self.archive.load(apath)- # Index configuration- with open(f"{path}/config", "rb") as handle:- self.config = pickle.load(handle)-- # Build full path to embedding vectors file- if self.config.get("storevectors"):- self.config["path"] = os.path.join(path, self.config["path"])+ # Load index configuration+ self.config = self.loadconfig(path)# Approximate nearest neighbor index - stores embeddings vectorsself.ann = ANNFactory.create(self.config)@@ -476,7 +486,7 @@ class Embeddings:if self.graph:self.graph.load(f"{path}/graph")- def save(self, path, cloud=None):+ def save(self, path, cloud=None, **kwargs):"""Saves an index in a directory at path unless path ends with tar.gz, tar.bz2, tar.xz or zip.In those cases, the index is stored as a compressed file.@@ -484,6 +494,7 @@ class Embeddings:Args:path: output pathcloud: cloud storage configuration+ kwargs: additional configuration as keyword args"""if self.config:@@ -499,9 +510,8 @@ class Embeddings:self.config["path"] = os.path.basename(self.config["path"])- # Write index configuration- with open(f"{path}/config", "wb") as handle:- pickle.dump(self.config, handle, protocol=__pickle__)+ # Save index configuration+ self.saveconfig(path)# Save approximate nearest neighbor indexself.ann.save(f"{path}/embeddings")@@ -524,7 +534,12 @@ class Embeddings:# If this is an archive, save itif apath:- self.archive.save(apath, cloud)+ self.archive.save(apath)++ # Save to cloud, if configured+ cloud = self.createcloud(cloud=cloud, **kwargs)+ if cloud:+ cloud.save(apath if apath else path)def close(self):"""@@ -589,6 +604,62 @@ class Embeddings:return {"path": "sentence-transformers/all-MiniLM-L6-v2"}+ def loadconfig(self, path):+ """+ Loads index configuration. This method supports both config pickle files and config.json files.++ Args:+ path: path to directory++ Returns:+ dict+ """++ # Configuration+ config = None++ # Determine if config is json or pickle+ jsonconfig = os.path.exists(f"{path}/config.json")++ # Set config file name+ name = "config.json" if jsonconfig else "config"++ # Load configuration+ with open(f"{path}/{name}", "r" if jsonconfig else "rb") as handle:+ config = json.load(handle) if jsonconfig else pickle.load(handle)++ # Build full path to embedding vectors file+ if config.get("storevectors"):+ config["path"] = os.path.join(path, config["path"])++ return config++ def saveconfig(self, path):+ """+ Saves index configuration. This method saves to JSON if possible, otherwise it falls back to pickle.++ Args:+ path: path to directory++ Returns:+ dict+ """++ # Default to pickle config+ jsonconfig = self.config.get("format", "pickle") == "json"++ # Set config file name+ name = "config.json" if jsonconfig else "config"++ # Write configuration+ with open(f"{path}/{name}", "w" if jsonconfig else "wb", encoding="utf-8" if jsonconfig else None) as handle:+ if jsonconfig:+ # Write config as JSON+ json.dump(self.config, handle, default=str, indent=2)+ else:+ # Write config as pickle format+ pickle.dump(self.config, handle, protocol=__pickle__)+def loadvectors(self):"""Loads a vector model set in config.@@ -624,7 +695,7 @@ class Embeddings:"""# Create archive instance, if necessary- self.archive = self.archive if self.archive else Archive()+ self.archive = ArchiveFactory.create()# Check if path is an archive fileif self.archive.isarchive(path):@@ -633,6 +704,22 @@ class Embeddings:return path, None+ def createcloud(self, **cloud):+ """+ Creates a cloud instance from config.++ Args:+ cloud: cloud configuration+ """++ # Merge keyword args and keys under the cloud parameter+ config = cloud+ if "cloud" in config and config["cloud"]:+ config.update(config.pop("cloud"))++ # Create cloud instance from config and return+ return CloudFactory.create(config) if config else None+def createdatabase(self):"""Creates a database from config. This method will also close any existing database connection.diff --git a/src/python/txtai/embeddings/cloud.py b/src/python/txtai/embeddings/cloud.pydeleted file mode 100644index bd6efadc..00000000--- a/src/python/txtai/embeddings/cloud.py+++ /dev/null@@ -1,93 +0,0 @@-"""-Cloud module-"""--import os--# Conditional import-try:- from libcloud.storage.providers import get_driver- from libcloud.storage.types import ContainerDoesNotExistError, ObjectDoesNotExistError-- LIBCLOUD = True-except ImportError:- LIBCLOUD = False---class Cloud:- """- Methods to sync files with cloud storage.- """-- def __init__(self, config):- """- Creates a new cloud storage client connection.-- Args:- config: cloud storage configuration- """-- if not LIBCLOUD:- raise ImportError('Cloud storage is not available - install "cloud" extra to enable')-- self.config = config-- # Get driver for provider- driver = get_driver(config["provider"])-- # Get client connection- self.client = driver(- config.get("key", os.environ.get("ACCESS_KEY")),- config.get("secret", os.environ.get("ACCESS_SECRET")),- host=config.get("host"),- port=config.get("port"),- token=config.get("token"),- region=config.get("region"),- )-- def exists(self, path):- """- Checks if path exists.-- Args:- path: path to check-- Returns:- True if path exists, False otherwise- """-- try:- self.client.get_object(self.config["container"], os.path.basename(path))- return True- except (ContainerDoesNotExistError, ObjectDoesNotExistError):- return False-- def load(self, path):- """- Retrieves file from cloud storage and stores locally.-- Args:- path: path to retrieve- """-- # Download object- obj = self.client.get_object(self.config["container"], os.path.basename(path))- obj.download(path, overwrite_existing=True)-- def save(self, path):- """- Sends local file to cloud storage.-- Args:- path: path to send- """-- # Get or create container- try:- container = self.client.get_container(self.config["container"])- except ContainerDoesNotExistError:- container = self.client.create_container(self.config["container"])-- # Upload object- with open(path, "rb") as iterator:- self.client.upload_object_via_stream(iterator=iterator, container=container, object_name=os.path.basename(path))