Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Reduce dependencies to just numpy for minimal install, closes #1093

davidmezzetti65b72bfdfaab
diff --git a/setup.py b/setup.py
index a2249fdd..3c69b26e 100644
--- a/setup.py
+++ b/setup.py
@@ -8,10 +8,19 @@ with open("README.md", "r", encoding="utf-8") as f:
DESCRIPTION = "".join([line for line in f if "gh-dark-mode-only" not in line])
# Required dependencies
-install = ["huggingface-hub>=0.34.0", "numpy>=1.18.4", "pyyaml>=5.3", "regex>=2022.8.17", "safetensors>=0.4.5"]
+install = ["numpy>=1.18.4"]
# Default dependencies
-default = ["faiss-cpu>=1.7.1.post2", "msgpack>=1.0.7", "torch>=2.4", "transformers>=4.56.2"]
+default = [
+ "faiss-cpu>=1.7.1.post2",
+ "huggingface-hub>=0.34.0",
+ "msgpack>=1.0.7",
+ "regex>=2022.8.17",
+ "pyyaml>=5.3",
+ "safetensors>=0.4.5",
+ "torch>=2.4",
+ "transformers>=4.56.2",
+]
# Optional dependencies
extras = {}
diff --git a/src/python/txtai/agent/tool/skill.py b/src/python/txtai/agent/tool/skill.py
index 13b616bb..9edbad56 100644
--- a/src/python/txtai/agent/tool/skill.py
+++ b/src/python/txtai/agent/tool/skill.py
@@ -2,10 +2,13 @@
Skill imports
"""
-import yaml
-
from smolagents import Tool
+# Conditional imports
+from ...util import TransformersLib
+
+yaml = TransformersLib().yaml()
+
class SkillTool(Tool):
"""
diff --git a/src/python/txtai/ann/dense/faiss.py b/src/python/txtai/ann/dense/faiss.py
index a60a5825..68bcd803 100644
--- a/src/python/txtai/ann/dense/faiss.py
+++ b/src/python/txtai/ann/dense/faiss.py
@@ -37,7 +37,7 @@ class Faiss(ANN):
super().__init__(config)
if not FAISS:
- raise ImportError('Faiss is not available - install "ann" extra to enable')
+ raise ImportError("Faiss is not available - install faiss-cpu to enable")
# Scalar quantization
quantize = self.config.get("quantize")
diff --git a/src/python/txtai/ann/dense/numpy.py b/src/python/txtai/ann/dense/numpy.py
index 166e652c..2773beca 100644
--- a/src/python/txtai/ann/dense/numpy.py
+++ b/src/python/txtai/ann/dense/numpy.py
@@ -4,13 +4,15 @@ NumPy module
import numpy as np
-from safetensors import safe_open
-from safetensors.numpy import save_file
-
from ...serialize import SerializeFactory
from ..base import ANN
+# Conditional imports
+from ...util import TransformersLib
+
+safetensors = TransformersLib().safetensors()
+
class NumPy(ANN):
"""
@@ -157,7 +159,7 @@ class NumPy(ANN):
"""
# Merge metadata and tensors into single dictionary
- with safe_open(path, framework="np") as f:
+ with safetensors.safe_open(path, framework="np") as f:
return {**(f.metadata() if f.metadata() else {}), **{k: f.get_tensor(k) for k in f.keys()}}
def savesafetensors(self, data, path, metadata=None):
@@ -170,7 +172,7 @@ class NumPy(ANN):
metadata: additional metadata to save
"""
- save_file(data, path, metadata)
+ safetensors.numpy.save_file(data, path, metadata)
def hammingscore(self, queries):
"""
diff --git a/src/python/txtai/app/base.py b/src/python/txtai/app/base.py
index e8068627..e54837b6 100644
--- a/src/python/txtai/app/base.py
+++ b/src/python/txtai/app/base.py
@@ -7,13 +7,16 @@ import os
from multiprocessing.pool import ThreadPool
from threading import RLock
-import yaml
-
from ..agent import Agent
from ..embeddings import Documents, Embeddings
from ..pipeline import PipelineFactory
from ..workflow import WorkflowFactory
+# Conditional imports
+from ..util import TransformersLib
+
+yaml = TransformersLib().yaml()
+
# pylint: disable=R0904
class Application:
diff --git a/src/python/txtai/cloud/hub.py b/src/python/txtai/cloud/hub.py
index ba0dea06..4cae96c3 100644
--- a/src/python/txtai/cloud/hub.py
+++ b/src/python/txtai/cloud/hub.py
@@ -5,11 +5,12 @@ Hugging Face Hub module
import os
import tempfile
-import huggingface_hub
+from .base import Cloud
-from huggingface_hub.utils import RepositoryNotFoundError
+# Conditional imports
+from ..util import TransformersLib
-from .base import Cloud
+huggingface_hub = TransformersLib().huggingface_hub()
class HuggingFaceHub(Cloud):
@@ -30,11 +31,11 @@ class HuggingFaceHub(Cloud):
# Otherwise return repository metadata
return huggingface_hub.model_info(repo_id=self.config["container"], revision=self.config.get("revision"), token=self.config.get("token"))
- except RepositoryNotFoundError:
+ except huggingface_hub.utils.RepositoryNotFoundError:
return None
def load(self, path=None):
- # Download archvie file and return local path
+ # Download archive file and return local path
if self.isarchive(path):
return huggingface_hub.hf_hub_download(
repo_id=self.config["container"],
diff --git a/src/python/txtai/models/pooling/base.py b/src/python/txtai/models/pooling/base.py
index ed39c612..783cad0b 100644
--- a/src/python/txtai/models/pooling/base.py
+++ b/src/python/txtai/models/pooling/base.py
@@ -6,12 +6,10 @@ import json
import numpy as np
-from huggingface_hub.errors import HFValidationError
-
from ..models import Models
# Conditional imports
-from ...util import Download, TransformersLib
+from ...util import Download, DownloadError, TransformersLib
transformerslib = TransformersLib()
torch = transformerslib.torch()
@@ -178,7 +176,7 @@ class Pooling(Module):
config = json.load(f)
# Ignore this error - invalid repo or directory
- except (HFValidationError, OSError):
+ except DownloadError:
pass
return config
diff --git a/src/python/txtai/models/pooling/factory.py b/src/python/txtai/models/pooling/factory.py
index 92fcde8c..0e635e7f 100644
--- a/src/python/txtai/models/pooling/factory.py
+++ b/src/python/txtai/models/pooling/factory.py
@@ -5,15 +5,13 @@ Factory module
import json
import os
-from huggingface_hub.errors import HFValidationError
-
from .base import Pooling
from .cls import ClsPooling
from .last import LastPooling
from .late import LatePooling
from .mean import MeanPooling
-from ...util import Download
+from ...util import Download, DownloadError
class PoolingFactory:
@@ -148,7 +146,7 @@ class PoolingFactory:
config = json.load(f)
# Ignore this error - invalid repo or directory
- except (HFValidationError, OSError):
+ except DownloadError:
pass
return config
diff --git a/src/python/txtai/models/pooling/late.py b/src/python/txtai/models/pooling/late.py
index fccae3d0..c57f770f 100644
--- a/src/python/txtai/models/pooling/late.py
+++ b/src/python/txtai/models/pooling/late.py
@@ -4,8 +4,6 @@ Late module
import numpy as np
-from safetensors import safe_open
-
from .base import Pooling
from .muvera import Muvera
@@ -13,6 +11,7 @@ from .muvera import Muvera
from ...util import Download, TransformersLib
transformerslib = TransformersLib()
+safetensors = transformerslib.safetensors()
torch = transformerslib.torch()
@@ -44,7 +43,7 @@ class LatePooling(Pooling):
# Load linear layer
path = Download()(path, name)
- with safe_open(filename=path, framework="pt") as f:
+ with safetensors.safe_open(filename=path, framework="pt") as f:
weights = f.get_tensor("linear.weight")
# Load weights into linear layer
diff --git a/src/python/txtai/pipeline/audio/texttospeech.py b/src/python/txtai/pipeline/audio/texttospeech.py
index 1a1cb936..0718bf3b 100644
--- a/src/python/txtai/pipeline/audio/texttospeech.py
+++ b/src/python/txtai/pipeline/audio/texttospeech.py
@@ -20,20 +20,17 @@ import logging
from io import BytesIO
-import yaml
-
import numpy as np
-from huggingface_hub.errors import HFValidationError
-
from ..base import Pipeline
# Conditional imports
-from ...util import Download, TransformersLib
+from ...util import Download, DownloadError, TransformersLib
transformerslib = TransformersLib()
torch = transformerslib.torch()
transformers = transformerslib.transformers()
+yaml = transformerslib.yaml()
# Logging configuration
logger = logging.getLogger(__name__)
@@ -136,7 +133,7 @@ class TextToSpeech(Pipeline):
try:
# Check if file exists
exists = Download()(path, name) is not None
- except (HFValidationError, OSError):
+ except DownloadError:
return False
return exists
diff --git a/src/python/txtai/pipeline/data/tokenizer.py b/src/python/txtai/pipeline/data/tokenizer.py
index c5c979e8..f292afbd 100644
--- a/src/python/txtai/pipeline/data/tokenizer.py
+++ b/src/python/txtai/pipeline/data/tokenizer.py
@@ -5,10 +5,13 @@ Tokenizer module
import re
import string
-import regex
-
from ..base import Pipeline
+# Conditional imports
+from ...util import TransformersLib
+
+regex = TransformersLib().regex()
+
class Tokenizer(Pipeline):
"""
diff --git a/src/python/txtai/pipeline/text/entity.py b/src/python/txtai/pipeline/text/entity.py
index b52dd70e..57564a18 100644
--- a/src/python/txtai/pipeline/text/entity.py
+++ b/src/python/txtai/pipeline/text/entity.py
@@ -10,10 +10,8 @@ try:
except ImportError:
GLINER = False
-from huggingface_hub.errors import HFValidationError
-
from ...models import Models
-from ...util import Download
+from ...util import Download, DownloadError
from ..hfpipeline import HFPipeline
@@ -90,7 +88,7 @@ class Entity(HFPipeline):
return Download()(path, "gliner_config.json") is not None
# Ignore this error - invalid repo or directory
- except (HFValidationError, OSError):
+ except DownloadError:
pass
return False
diff --git a/src/python/txtai/pipeline/text/translation.py b/src/python/txtai/pipeline/text/translation.py
index 6b1a7c10..b069e24c 100644
--- a/src/python/txtai/pipeline/text/translation.py
+++ b/src/python/txtai/pipeline/text/translation.py
@@ -10,11 +10,14 @@ try:
except ImportError:
STATICVECTORS = False
-from huggingface_hub.hf_api import HfApi
-
from ...models import Models
from ..hfmodel import HFModel
+# Conditional imports
+from ...util import TransformersLib
+
+huggingface_hub = TransformersLib().huggingface_hub()
+
class Translation(HFModel):
"""
@@ -110,7 +113,7 @@ class Translation(HFModel):
list of source-target language model ids
"""
- ids = [x.id for x in HfApi().list_models(author="Helsinki-NLP")] if self.findmodels else []
+ ids = [x.id for x in huggingface_hub.hf_api.HfApi().list_models(author="Helsinki-NLP")] if self.findmodels else []
return set(ids)
def detect(self, texts):
diff --git a/src/python/txtai/serialize/messagepack.py b/src/python/txtai/serialize/messagepack.py
index 0d4a5897..dd952443 100644
--- a/src/python/txtai/serialize/messagepack.py
+++ b/src/python/txtai/serialize/messagepack.py
@@ -26,7 +26,7 @@ class MessagePack(Serialize):
super().__init__()
if not MSGPACK:
- raise ImportError("MessagePack is not available - install to msgpack enable")
+ raise ImportError("MessagePack is not available - install msgpack to enable")
# Streaming unpacker
self.streaming = streaming
diff --git a/src/python/txtai/util/__init__.py b/src/python/txtai/util/__init__.py
index b6bcbce3..baab7b33 100644
--- a/src/python/txtai/util/__init__.py
+++ b/src/python/txtai/util/__init__.py
@@ -2,7 +2,7 @@
Utility imports
"""
-from .download import Download
+from .download import *
from .resolver import Resolver
from .sparsearray import SparseArray
from .template import TemplateFormatter
diff --git a/src/python/txtai/util/download.py b/src/python/txtai/util/download.py
index 01330fd4..add12a0f 100644
--- a/src/python/txtai/util/download.py
+++ b/src/python/txtai/util/download.py
@@ -4,7 +4,12 @@ Download module
import os
-from huggingface_hub import hf_hub_download
+# Conditional imports
+from .transformerslib import TransformersLib
+
+transformerslib = TransformersLib()
+huggingface_hub = transformerslib.huggingface_hub()
+HFValidationError = transformerslib.hferror()
class Download:
@@ -18,7 +23,7 @@ class Download:
Args:
path: model path or repo
- name: file name to load
+ name: file name
Returns:
local cached model path, if available
@@ -40,5 +45,15 @@ class Download:
path, name = "/".join(parts[:repo]), "/".join(parts[repo:])
# Download (if necessary) and return local file path
- local = os.path.join(path, name)
- return local if os.path.exists(local) else hf_hub_download(repo_id=path, filename=name)
+ try:
+ local = os.path.join(path, name)
+ return local if os.path.exists(local) else huggingface_hub.hf_hub_download(repo_id=path, filename=name)
+
+ except (HFValidationError, OSError) as e:
+ raise DownloadError(f"Error locating file with parameters: path={path}, name={name}") from e
+
+
+class DownloadError(Exception):
+ """
+ Exception raised when a local or remote file path is not valid.
+ """
diff --git a/src/python/txtai/util/transformerslib.py b/src/python/txtai/util/transformerslib.py
index 71418edf..b761b2fb 100644
--- a/src/python/txtai/util/transformerslib.py
+++ b/src/python/txtai/util/transformerslib.py
@@ -11,7 +11,7 @@ class TransformersLib:
def arguments(self):
"""
- Imports transformers.TrainingArguments
+ Imports transformers.TrainingArguments.
"""
try:
@@ -66,6 +66,48 @@ class TransformersLib:
return Dataset
+ def hferror(self):
+ """
+ Import huggingface_hub.errors.HFValidationError.
+ """
+
+ try:
+ from huggingface_hub.errors import HFValidationError
+
+ except ImportError:
+
+ class HFValidationError(Exception):
+ """
+ Stub for HFValidationError
+ """
+
+ return HFValidationError
+
+ def huggingface_hub(self):
+ """
+ Imports huggingface_hub.
+
+ Returns:
+ torch
+ """
+
+ try:
+ import huggingface_hub
+
+ except ImportError:
+
+ class HuggingFaceHub:
+ """
+ Stub for HuggingFaceHub
+ """
+
+ def __getattr__(self, name):
+ raise ImportError("Hugging Face Hub is not installed, install huggingface-hub to use this module")
+
+ huggingface_hub = HuggingFaceHub()
+
+ return huggingface_hub
+
def model(self):
"""
Imports transformers.modeling_utils.PreTrainedModel.
@@ -88,7 +130,7 @@ class TransformersLib:
def module(self):
"""
- Imports torch.nn.Module
+ Imports torch.nn.Module.
Returns:
Module
@@ -109,6 +151,56 @@ class TransformersLib:
return Module
+ def regex(self):
+ """
+ Imports regex.
+
+ Returns:
+ regex
+ """
+
+ try:
+ import regex
+
+ except ImportError:
+
+ class Regex:
+ """
+ Stub for Regex
+ """
+
+ def __getattr__(self, name):
+ raise ImportError("Regex is not installed, install regex to use this module")
+
+ regex = Regex()
+
+ return regex
+
+ def safetensors(self):
+ """
+ Imports safetensors.
+
+ Returns:
+ safetensors
+ """
+
+ try:
+ import safetensors
+
+ except ImportError:
+
+ class Safetensors:
+ """
+ Stub for Safetensors
+ """
+
+ def __getattr__(self, name):
+ raise ImportError("Safetensors is not installed, install safetensors to use this module")
+
+ safetensors = Safetensors()
+
+ return safetensors
+
def torch(self):
"""
Imports torch.
@@ -158,3 +250,28 @@ class TransformersLib:
transformers = Transformers()
return transformers
+
+ def yaml(self):
+ """
+ Imports yaml.
+
+ Returns:
+ yaml
+ """
+
+ try:
+ import yaml
+
+ except ImportError:
+
+ class YAML:
+ """
+ Stub for yaml
+ """
+
+ def __getattr__(self, name):
+ raise ImportError("PyYAML is not installed, install pyyaml to use this module")
+
+ yaml = YAML()
+
+ return yaml
diff --git a/src/python/txtai/vectors/dense/m2v.py b/src/python/txtai/vectors/dense/m2v.py
index 17f5cddf..20bfaf2d 100644
--- a/src/python/txtai/vectors/dense/m2v.py
+++ b/src/python/txtai/vectors/dense/m2v.py
@@ -4,8 +4,6 @@ Model2Vec module
import json
-from huggingface_hub.errors import HFValidationError
-
# Conditional import
try:
from model2vec import StaticModel
@@ -14,7 +12,7 @@ try:
except ImportError:
MODEL2VEC = False
-from ...util import Download
+from ...util import Download, DownloadError
from ..base import Vectors
@@ -44,7 +42,7 @@ class Model2Vec(Vectors):
return config.get("model_type") == "model2vec"
# Ignore this error - invalid repo or directory
- except (HFValidationError, OSError):
+ except DownloadError:
pass
return False
diff --git a/src/python/txtai/vectors/dense/words.py b/src/python/txtai/vectors/dense/words.py
index f94555d9..f43140df 100644
--- a/src/python/txtai/vectors/dense/words.py
+++ b/src/python/txtai/vectors/dense/words.py
@@ -11,8 +11,6 @@ from multiprocessing import Pool
import numpy as np
-from huggingface_hub.errors import HFValidationError
-
# Conditional import
try:
from staticvectors import Database, StaticVectors
@@ -22,7 +20,7 @@ except ImportError:
STATICVECTORS = False
from ...pipeline import Tokenizer
-from ...util import Download
+from ...util import Download, DownloadError
from ..base import Vectors
@@ -99,7 +97,7 @@ class WordVectors(Vectors):
return config.get("model_type") == "staticvectors"
# Ignore this error - invalid repo or directory
- except (HFValidationError, OSError):
+ except DownloadError:
pass
return False
diff --git a/test/python/testcloud.py b/test/python/testcloud.py
index e748c2c2..81a700c0 100644
--- a/test/python/testcloud.py
+++ b/test/python/testcloud.py
@@ -9,6 +9,7 @@ import unittest
from unittest.mock import patch
+from huggingface_hub import hf_hub_download
from txtai.cloud import Cloud
from txtai.embeddings import Embeddings
@@ -107,8 +108,16 @@ class TestCloud(unittest.TestCase):
if "Invalid" in kwargs["repo_id"]:
raise FileNotFoundError
- # Return either .gitattributes file or index
- return attributes if kwargs["filename"] == ".gitattributes" else index
+ # Check for .gitattributes file
+ if kwargs["filename"] == ".gitattributes":
+ return attributes
+
+ # Check for cloud index path
+ if any(kwargs["filename"] == x for x in paths):
+ return index
+
+ # Use original method for all other requests
+ return hf_hub_download(**kwargs)
# Patch write methods since token will not be available
create.return_value = None
@@ -130,7 +139,8 @@ class TestCloud(unittest.TestCase):
attributes = tmp.name
# Run tests with uncompressed and compressed index
- for path in [f"cloud.{provider}", f"cloud.{provider}.tar.gz"]:
+ paths = [f"cloud.{provider}", f"cloud.{provider}.tar.gz"]
+ for path in paths:
self.runTests(path, {"provider": provider, "container": "neuml/txtai-intro"})
def runTests(self, path, cloud):
diff --git a/test/python/testtransformers.py b/test/python/testtransformers.py
index 33dfb5cc..e7240d95 100644
--- a/test/python/testtransformers.py
+++ b/test/python/testtransformers.py
@@ -21,6 +21,10 @@ class TestTransformers(unittest.TestCase):
"""
modules = [
+ "huggingface_hub",
+ "huggingface_hub.errors",
+ "regex",
+ "safetensors",
"transformers",
"transformers.configuration_utils",
"transformers.modeling_utils",
@@ -29,6 +33,7 @@ class TestTransformers(unittest.TestCase):
"torch.nn",
"torch.onnx",
"torch.utils.data",
+ "yaml",
]
# Get handle to all currently loaded txtai modules
@@ -65,18 +70,29 @@ class TestTransformers(unittest.TestCase):
Test transformers not installed
"""
+ # pylint: disable=W0106
from txtai.util import TransformersLib
lib = TransformersLib()
# Test transformers stubs
- for x in [lib.arguments(), lib.config(), lib.dataset(), lib.module(), lib.model()]:
+ for x in [lib.arguments(), lib.config(), lib.dataset(), lib.hferror(), lib.module(), lib.model()]:
self.assertTrue(x.__module__.endswith("transformerslib"))
- # pylint: disable=W0106
with self.assertRaises(ImportError):
- lib.transformers().AutoModel
+ lib.huggingface_hub().hf_hub_download
+
+ with self.assertRaises(ImportError):
+ lib.regex().compile
+
+ with self.assertRaises(ImportError):
+ lib.safetensors().safe_open
- # pylint: disable=W0106
with self.assertRaises(ImportError):
lib.torch().device
+
+ with self.assertRaises(ImportError):
+ lib.transformers().AutoModel
+
+ with self.assertRaises(ImportError):
+ lib.yaml().safe_open