Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add huggingface-hub integration, closes #430. Add cloud object storage support for standard embeddings indexes, closes #431. Add support for custom cloud providers, closes #432. Add support for storing embeddings config as JSON, closes #433.

davidmezzetti95697322c9b3
diff --git a/src/python/txtai/app/base.py b/src/python/txtai/app/base.py
index 4b31a80e..3fd52a60 100644
--- a/src/python/txtai/app/base.py
+++ b/src/python/txtai/app/base.py
@@ -185,11 +185,11 @@ class Application:
# Resolve transform function
config["transform"] = self.function(config["transform"])
- # Local embeddings index
- if loaddata and self.config.get("path") and Embeddings().exists(self.config["path"], self.config.get("cloud")):
+ # Load embeddings index if loaddata and index exists
+ if loaddata and Embeddings().exists(self.config.get("path"), self.config.get("cloud")):
# Load existing index if available
self.embeddings = Embeddings()
- self.embeddings.load(self.config["path"], self.config.get("cloud"))
+ self.embeddings.load(self.config.get("path"), self.config.get("cloud"))
elif config:
# Initialize empty embeddings
self.embeddings = Embeddings(config)
diff --git a/src/python/txtai/cloud/__init__.py b/src/python/txtai/cloud/__init__.py
new file mode 100644
index 00000000..e2fd6d71
--- /dev/null
+++ b/src/python/txtai/cloud/__init__.py
@@ -0,0 +1,8 @@
+"""
+Cloud imports
+"""
+
+from .base import Cloud
+from .factory import CloudFactory
+from .hub import HuggingFaceHub
+from .storage import ObjectStorage
diff --git a/src/python/txtai/cloud/base.py b/src/python/txtai/cloud/base.py
new file mode 100644
index 00000000..0c5e53dc
--- /dev/null
+++ b/src/python/txtai/cloud/base.py
@@ -0,0 +1,106 @@
+"""
+Cloud module
+"""
+
+import os
+
+from ..archive import ArchiveFactory
+
+
+class Cloud:
+ """
+ Base class for cloud providers. Cloud providers sync content between local and remote storage.
+ """
+
+ def __init__(self, config):
+ """
+ Creates a new cloud connection.
+
+ Args:
+ config: cloud configuration
+ """
+
+ self.config = config
+
+ def exists(self, path=None):
+ """
+ Checks if path exists in cloud. If path is None, this method checks if the container exists.
+
+ Args:
+ path: path to check
+
+ Returns:
+ True if path or container exists, False otherwise
+ """
+
+ return self.metadata(path) is not None
+
+ def metadata(self, path=None):
+ """
+ Returns metadata for path from cloud. If path is None, this method returns metadata
+ for container.
+
+ Args:
+ path: retrieve metadata for this path
+
+ Returns:
+ path or container metadata if available, otherwise returns None
+ """
+
+ raise NotImplementedError
+
+ def load(self, path=None):
+ """
+ Retrieves content from cloud and stores locally. If path is empty, this method retrieves
+ all content in the container.
+
+ Args:
+ path: path to retrieve
+
+ Returns:
+ local path which can be different than input path
+ """
+
+ raise NotImplementedError
+
+ def save(self, path):
+ """
+ Sends local content stored in path to cloud.
+
+ Args:
+ path: local path to sync
+ """
+
+ raise NotImplementedError
+
+ def isarchive(self, path):
+ """
+ Check if path is an archive file.
+
+ Args:
+ path: path to check
+
+ Returns:
+ True if path ends with an archive extension, false otherwise
+ """
+
+ return ArchiveFactory.create().isarchive(path)
+
+ def listfiles(self, path):
+ """
+ Lists files in path. If path is a file, this method returns a single element list
+ containing path.
+
+ Args:
+ path: path to list
+
+ Returns:
+ List of files
+ """
+
+ # List all files if path is a directory
+ if os.path.isdir(path):
+ return [os.path.join(path, f) for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
+
+ # Path is a file
+ return [path]
diff --git a/src/python/txtai/cloud/factory.py b/src/python/txtai/cloud/factory.py
new file mode 100644
index 00000000..ff967b9a
--- /dev/null
+++ b/src/python/txtai/cloud/factory.py
@@ -0,0 +1,70 @@
+"""
+Factory module
+"""
+
+from ..util import Resolver
+
+from .hub import HuggingFaceHub
+from .storage import ObjectStorage, LIBCLOUD
+
+
+class CloudFactory:
+ """
+ Methods to create Cloud instances.
+ """
+
+ @staticmethod
+ def create(config):
+ """
+ Creates a Cloud instance.
+
+ Args:
+ config: cloud configuration
+
+ Returns:
+ Cloud
+ """
+
+ # Cloud instance
+ cloud = None
+
+ provider = config.get("provider", "")
+
+ # Hugging Face Hub
+ if provider.lower() == "huggingface-hub":
+ cloud = HuggingFaceHub(config)
+
+ # Cloud object storage
+ elif ObjectStorage.isprovider(provider):
+ cloud = ObjectStorage(config)
+
+ # External provider
+ elif provider:
+ cloud = CloudFactory.resolve(provider, config)
+
+ return cloud
+
+ @staticmethod
+ def resolve(backend, config):
+ """
+ Attempt to resolve a custom cloud backend.
+
+ Args:
+ backend: backend class
+ config: configuration parameters
+
+ Returns:
+ Cloud
+ """
+
+ try:
+ return Resolver()(backend)(config)
+
+ except Exception as e:
+ # Failure message
+ message = f'Unable to resolve cloud backend: "{backend}".'
+
+ # Append message if LIBCLOUD is not installed
+ message += ' Cloud storage is not available - install "cloud" extra to enable' if not LIBCLOUD else ""
+
+ raise ImportError(message) from e
diff --git a/src/python/txtai/cloud/hub.py b/src/python/txtai/cloud/hub.py
new file mode 100644
index 00000000..e69fb95c
--- /dev/null
+++ b/src/python/txtai/cloud/hub.py
@@ -0,0 +1,65 @@
+"""
+Hugging Face Hub module
+"""
+
+import os
+
+import huggingface_hub
+
+from huggingface_hub.utils import RepositoryNotFoundError
+
+from .base import Cloud
+
+
+class HuggingFaceHub(Cloud):
+ """
+ Hugging Face Hub cloud provider.
+ """
+
+ def metadata(self, path=None):
+ try:
+ # If this is an archive file path, get file metadata
+ if self.isarchive(path):
+ url = huggingface_hub.hf_hub_url(
+ repo_id=self.config["container"], filename=os.path.basename(path), revision=self.config.get("revision")
+ )
+
+ return huggingface_hub.get_hf_file_metadata(url=url, token=self.config.get("token"))
+
+ # Otherwise return repository metadata
+ return huggingface_hub.model_info(repo_id=self.config["container"], revision=self.config.get("revision"), token=self.config.get("token"))
+
+ except RepositoryNotFoundError:
+ return None
+
+ def load(self, path=None):
+ # Download archvie file and return local path
+ if self.isarchive(path):
+ return huggingface_hub.hf_hub_download(
+ repo_id=self.config["container"],
+ filename=os.path.basename(path),
+ revision=self.config.get("revision"),
+ cache_dir=self.config.get("cache"),
+ token=self.config.get("token"),
+ )
+
+ # Download repository and return cached path
+ return huggingface_hub.snapshot_download(
+ repo_id=self.config["container"], revision=self.config.get("revision"), cache_dir=self.config.get("cache"), token=self.config.get("token")
+ )
+
+ def save(self, path):
+ # Get or create repository
+ huggingface_hub.create_repo(
+ repo_id=self.config["container"], token=self.config.get("token"), private=self.config.get("private", True), exist_ok=True
+ )
+
+ # Upload files
+ for f in self.listfiles(path):
+ huggingface_hub.upload_file(
+ repo_id=self.config["container"],
+ revision=self.config.get("revision"),
+ token=self.config.get("token"),
+ path_or_fileobj=f,
+ path_in_repo=os.path.basename(f),
+ )
diff --git a/src/python/txtai/cloud/storage.py b/src/python/txtai/cloud/storage.py
new file mode 100644
index 00000000..ac13b1ec
--- /dev/null
+++ b/src/python/txtai/cloud/storage.py
@@ -0,0 +1,96 @@
+"""
+Object storage module
+"""
+
+import os
+
+# Conditional import
+try:
+ from libcloud.storage.providers import get_driver, DRIVERS
+ from libcloud.storage.types import ContainerDoesNotExistError, ObjectDoesNotExistError
+
+ LIBCLOUD = True
+except ImportError:
+ LIBCLOUD, DRIVERS = False, None
+
+
+from .base import Cloud
+
+
+class ObjectStorage(Cloud):
+ """
+ Object storage cloud provider backed by Apache libcloud.
+ """
+
+ @staticmethod
+ def isprovider(provider):
+ """
+ Checks if this provider is an object storage provider.
+
+ Args:
+ provider: provider name
+
+ Returns:
+ True if this is an object storage provider
+ """
+
+ return LIBCLOUD and provider and provider in DRIVERS
+
+ def __init__(self, config):
+ super().__init__(config)
+
+ if not LIBCLOUD:
+ raise ImportError('Cloud object storage is not available - install "cloud" extra to enable')
+
+ # Get driver for provider
+ driver = get_driver(config["provider"])
+
+ # Get client connection
+ self.client = driver(
+ config.get("key", os.environ.get("ACCESS_KEY")),
+ config.get("secret", os.environ.get("ACCESS_SECRET")),
+ host=config.get("host"),
+ port=config.get("port"),
+ token=config.get("token"),
+ region=config.get("region"),
+ )
+
+ def metadata(self, path=None):
+ try:
+ # If this is an archive path, check if file exists
+ if self.isarchive(path):
+ return self.client.get_object(self.config["container"], os.path.basename(path))
+
+ # Otherwise check if container exists
+ return self.client.get_container(self.config["container"])
+ except (ContainerDoesNotExistError, ObjectDoesNotExistError):
+ return None
+
+ def load(self, path=None):
+ # Download archive file
+ if self.isarchive(path):
+ obj = self.client.get_object(self.config["container"], os.path.basename(path))
+ obj.download(path, overwrite_existing=True)
+
+ # Download all files in container
+ else:
+ # Create local directory, if necessary
+ os.makedirs(path, exist_ok=True)
+
+ container = self.client.get_container(self.config["container"])
+ for obj in container.list_objects():
+ obj.download(os.path.join(path, obj.name), overwrite_existing=True)
+
+ return path
+
+ def save(self, path):
+ # Get or create container
+ try:
+ container = self.client.get_container(self.config["container"])
+ except ContainerDoesNotExistError:
+ container = self.client.create_container(self.config["container"])
+
+ # Upload files
+ for f in self.listfiles(path):
+ with open(f, "rb") as iterator:
+ self.client.upload_object_via_stream(iterator=iterator, container=container, object_name=os.path.basename(f))
diff --git a/src/python/txtai/embeddings/__init__.py b/src/python/txtai/embeddings/__init__.py
index 6152117e..55d3c39c 100644
--- a/src/python/txtai/embeddings/__init__.py
+++ b/src/python/txtai/embeddings/__init__.py
@@ -2,11 +2,10 @@
Embeddings imports
"""
-from .archive import Archive
from .base import Embeddings
-from .cloud import Cloud
from .documents import Documents
from .explain import Explain
+from .functions import *
from .query import Query
from .reducer import Reducer
from .search import Search
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.py
index e7721fc6..9629ebce 100644
--- a/src/python/txtai/embeddings/base.py
+++ b/src/python/txtai/embeddings/base.py
@@ -13,12 +13,13 @@ import numpy as np
from .. import __pickle__
from ..ann import ANNFactory
+from ..archive import ArchiveFactory
+from ..cloud import CloudFactory
from ..database import DatabaseFactory
from ..graph import GraphFactory
from ..scoring import ScoringFactory
from ..vectors import VectorsFactory
-from .archive import Archive
from .explain import Explain
from .functions import Functions
from .reducer import Reducer
@@ -269,19 +270,20 @@ class Embeddings:
return self.batchtransform([document])[0]
- def batchtransform(self, documents):
+ def batchtransform(self, documents, category=None):
"""
Transforms documents into embeddings vectors.
Args:
documents: list of (id, data, tags)
+ category: category for instruction-based embeddings
Returns:
embeddings vectors
"""
# Convert documents into sentence embeddings
- embeddings = self.model.batchtransform(documents)
+ embeddings = self.model.batchtransform(documents, category)
# Reduce the dimensionality of the embeddings. Scale the embeddings using this
# model to reduce the noise of common but less relevant terms.
@@ -365,8 +367,8 @@ class Embeddings:
"""
# Convert queries to embedding vectors
- queries = self.batchtransform((None, query, None) for query in queries)
- data = self.batchtransform((None, row, None) for row in data)
+ queries = self.batchtransform(((None, query, None) for query in queries), "query")
+ data = self.batchtransform(((None, row, None) for row in data), "data")
# Dot product on normalized vectors is equal to cosine similarity
scores = np.dot(queries, data.T).tolist()
@@ -405,46 +407,54 @@ class Embeddings:
return Explain(self)(queries, texts, limit)
- def exists(self, path, cloud=None):
+ def exists(self, path=None, cloud=None, **kwargs):
"""
Checks if an index exists at path.
Args:
path: input path
cloud: cloud storage configuration
+ kwargs: additional configuration as keyword args
Returns:
True if index exists, False otherwise
"""
+ # Check if this exists in a cloud instance
+ cloud = self.createcloud(cloud=cloud, **kwargs)
+ if cloud:
+ return cloud.exists(path)
+
# Check if this is an archive file and exists
path, apath = self.checkarchive(path)
if apath:
- return self.archive.exists(apath, cloud)
+ return os.path.exists(apath)
- return os.path.exists(f"{path}/config") and os.path.exists(f"{path}/embeddings")
+ # Return true if path has a config or config.json file and an embeddings file
+ return path and (os.path.exists(f"{path}/config") or os.path.exists(f"{path}/config.json")) and os.path.exists(f"{path}/embeddings")
- def load(self, path, cloud=None):
+ def load(self, path=None, cloud=None, **kwargs):
"""
Loads an existing index from path.
Args:
path: input path
cloud: cloud storage configuration
+ kwargs: additional configuration as keyword args
"""
+ # Load from cloud, if configured
+ cloud = self.createcloud(cloud=cloud, **kwargs)
+ if cloud:
+ path = cloud.load(path)
+
# Check if this is an archive file and extract
path, apath = self.checkarchive(path)
if apath:
- self.archive.load(apath, cloud)
+ self.archive.load(apath)
- # Index configuration
- with open(f"{path}/config", "rb") as handle:
- self.config = pickle.load(handle)
-
- # Build full path to embedding vectors file
- if self.config.get("storevectors"):
- self.config["path"] = os.path.join(path, self.config["path"])
+ # Load index configuration
+ self.config = self.loadconfig(path)
# Approximate nearest neighbor index - stores embeddings vectors
self.ann = ANNFactory.create(self.config)
@@ -476,7 +486,7 @@ class Embeddings:
if self.graph:
self.graph.load(f"{path}/graph")
- def save(self, path, cloud=None):
+ def save(self, path, cloud=None, **kwargs):
"""
Saves an index in a directory at path unless path ends with tar.gz, tar.bz2, tar.xz or zip.
In those cases, the index is stored as a compressed file.
@@ -484,6 +494,7 @@ class Embeddings:
Args:
path: output path
cloud: cloud storage configuration
+ kwargs: additional configuration as keyword args
"""
if self.config:
@@ -499,9 +510,8 @@ class Embeddings:
self.config["path"] = os.path.basename(self.config["path"])
- # Write index configuration
- with open(f"{path}/config", "wb") as handle:
- pickle.dump(self.config, handle, protocol=__pickle__)
+ # Save index configuration
+ self.saveconfig(path)
# Save approximate nearest neighbor index
self.ann.save(f"{path}/embeddings")
@@ -524,7 +534,12 @@ class Embeddings:
# If this is an archive, save it
if apath:
- self.archive.save(apath, cloud)
+ self.archive.save(apath)
+
+ # Save to cloud, if configured
+ cloud = self.createcloud(cloud=cloud, **kwargs)
+ if cloud:
+ cloud.save(apath if apath else path)
def close(self):
"""
@@ -589,6 +604,62 @@ class Embeddings:
return {"path": "sentence-transformers/all-MiniLM-L6-v2"}
+ def loadconfig(self, path):
+ """
+ Loads index configuration. This method supports both config pickle files and config.json files.
+
+ Args:
+ path: path to directory
+
+ Returns:
+ dict
+ """
+
+ # Configuration
+ config = None
+
+ # Determine if config is json or pickle
+ jsonconfig = os.path.exists(f"{path}/config.json")
+
+ # Set config file name
+ name = "config.json" if jsonconfig else "config"
+
+ # Load configuration
+ with open(f"{path}/{name}", "r" if jsonconfig else "rb") as handle:
+ config = json.load(handle) if jsonconfig else pickle.load(handle)
+
+ # Build full path to embedding vectors file
+ if config.get("storevectors"):
+ config["path"] = os.path.join(path, config["path"])
+
+ return config
+
+ def saveconfig(self, path):
+ """
+ Saves index configuration. This method saves to JSON if possible, otherwise it falls back to pickle.
+
+ Args:
+ path: path to directory
+
+ Returns:
+ dict
+ """
+
+ # Default to pickle config
+ jsonconfig = self.config.get("format", "pickle") == "json"
+
+ # Set config file name
+ name = "config.json" if jsonconfig else "config"
+
+ # Write configuration
+ with open(f"{path}/{name}", "w" if jsonconfig else "wb", encoding="utf-8" if jsonconfig else None) as handle:
+ if jsonconfig:
+ # Write config as JSON
+ json.dump(self.config, handle, default=str, indent=2)
+ else:
+ # Write config as pickle format
+ pickle.dump(self.config, handle, protocol=__pickle__)
+
def loadvectors(self):
"""
Loads a vector model set in config.
@@ -624,7 +695,7 @@ class Embeddings:
"""
# Create archive instance, if necessary
- self.archive = self.archive if self.archive else Archive()
+ self.archive = ArchiveFactory.create()
# Check if path is an archive file
if self.archive.isarchive(path):
@@ -633,6 +704,22 @@ class Embeddings:
return path, None
+ def createcloud(self, **cloud):
+ """
+ Creates a cloud instance from config.
+
+ Args:
+ cloud: cloud configuration
+ """
+
+ # Merge keyword args and keys under the cloud parameter
+ config = cloud
+ if "cloud" in config and config["cloud"]:
+ config.update(config.pop("cloud"))
+
+ # Create cloud instance from config and return
+ return CloudFactory.create(config) if config else None
+
def createdatabase(self):
"""
Creates a database from config. This method will also close any existing database connection.
diff --git a/src/python/txtai/embeddings/cloud.py b/src/python/txtai/embeddings/cloud.py
deleted file mode 100644
index bd6efadc..00000000
--- a/src/python/txtai/embeddings/cloud.py
+++ /dev/null
@@ -1,93 +0,0 @@
-"""
-Cloud module
-"""
-
-import os
-
-# Conditional import
-try:
- from libcloud.storage.providers import get_driver
- from libcloud.storage.types import ContainerDoesNotExistError, ObjectDoesNotExistError
-
- LIBCLOUD = True
-except ImportError:
- LIBCLOUD = False
-
-
-class Cloud:
- """
- Methods to sync files with cloud storage.
- """
-
- def __init__(self, config):
- """
- Creates a new cloud storage client connection.
-
- Args:
- config: cloud storage configuration
- """
-
- if not LIBCLOUD:
- raise ImportError('Cloud storage is not available - install "cloud" extra to enable')
-
- self.config = config
-
- # Get driver for provider
- driver = get_driver(config["provider"])
-
- # Get client connection
- self.client = driver(
- config.get("key", os.environ.get("ACCESS_KEY")),
- config.get("secret", os.environ.get("ACCESS_SECRET")),
- host=config.get("host"),
- port=config.get("port"),
- token=config.get("token"),
- region=config.get("region"),
- )
-
- def exists(self, path):
- """
- Checks if path exists.
-
- Args:
- path: path to check
-
- Returns:
- True if path exists, False otherwise
- """
-
- try:
- self.client.get_object(self.config["container"], os.path.basename(path))
- return True
- except (ContainerDoesNotExistError, ObjectDoesNotExistError):
- return False
-
- def load(self, path):
- """
- Retrieves file from cloud storage and stores locally.
-
- Args:
- path: path to retrieve
- """
-
- # Download object
- obj = self.client.get_object(self.config["container"], os.path.basename(path))
- obj.download(path, overwrite_existing=True)
-
- def save(self, path):
- """
- Sends local file to cloud storage.
-
- Args:
- path: path to send
- """
-
- # Get or create container
- try:
- container = self.client.get_container(self.config["container"])
- except ContainerDoesNotExistError:
- container = self.client.create_container(self.config["container"])
-
- # Upload object
- with open(path, "rb") as iterator:
- self.client.upload_object_via_stream(iterator=iterator, container=container, object_name=os.path.basename(path))