💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Reduce dependencies to just numpy for minimal install, closes #1093
davidmezzetti65b72bfdfaab
diff --git a/setup.py b/setup.pyindex a2249fdd..3c69b26e 100644--- a/setup.py+++ b/setup.py@@ -8,10 +8,19 @@ with open("README.md", "r", encoding="utf-8") as f:DESCRIPTION = "".join([line for line in f if "gh-dark-mode-only" not in line])# Required dependencies-install = ["huggingface-hub>=0.34.0", "numpy>=1.18.4", "pyyaml>=5.3", "regex>=2022.8.17", "safetensors>=0.4.5"]+install = ["numpy>=1.18.4"]# Default dependencies-default = ["faiss-cpu>=1.7.1.post2", "msgpack>=1.0.7", "torch>=2.4", "transformers>=4.56.2"]+default = [+ "faiss-cpu>=1.7.1.post2",+ "huggingface-hub>=0.34.0",+ "msgpack>=1.0.7",+ "regex>=2022.8.17",+ "pyyaml>=5.3",+ "safetensors>=0.4.5",+ "torch>=2.4",+ "transformers>=4.56.2",+]# Optional dependenciesextras = {}diff --git a/src/python/txtai/agent/tool/skill.py b/src/python/txtai/agent/tool/skill.pyindex 13b616bb..9edbad56 100644--- a/src/python/txtai/agent/tool/skill.py+++ b/src/python/txtai/agent/tool/skill.py@@ -2,10 +2,13 @@Skill imports"""-import yaml-from smolagents import Tool+# Conditional imports+from ...util import TransformersLib++yaml = TransformersLib().yaml()+class SkillTool(Tool):"""diff --git a/src/python/txtai/ann/dense/faiss.py b/src/python/txtai/ann/dense/faiss.pyindex a60a5825..68bcd803 100644--- a/src/python/txtai/ann/dense/faiss.py+++ b/src/python/txtai/ann/dense/faiss.py@@ -37,7 +37,7 @@ class Faiss(ANN):super().__init__(config)if not FAISS:- raise ImportError('Faiss is not available - install "ann" extra to enable')+ raise ImportError("Faiss is not available - install faiss-cpu to enable")# Scalar quantizationquantize = self.config.get("quantize")diff --git a/src/python/txtai/ann/dense/numpy.py b/src/python/txtai/ann/dense/numpy.pyindex 166e652c..2773beca 100644--- a/src/python/txtai/ann/dense/numpy.py+++ b/src/python/txtai/ann/dense/numpy.py@@ -4,13 +4,15 @@ NumPy moduleimport numpy as np-from safetensors import safe_open-from safetensors.numpy import save_file-from ...serialize import SerializeFactoryfrom ..base import ANN+# Conditional imports+from ...util import TransformersLib++safetensors = TransformersLib().safetensors()+class NumPy(ANN):"""@@ -157,7 +159,7 @@ class NumPy(ANN):"""# Merge metadata and tensors into single dictionary- with safe_open(path, framework="np") as f:+ with safetensors.safe_open(path, framework="np") as f:return {**(f.metadata() if f.metadata() else {}), **{k: f.get_tensor(k) for k in f.keys()}}def savesafetensors(self, data, path, metadata=None):@@ -170,7 +172,7 @@ class NumPy(ANN):metadata: additional metadata to save"""- save_file(data, path, metadata)+ safetensors.numpy.save_file(data, path, metadata)def hammingscore(self, queries):"""diff --git a/src/python/txtai/app/base.py b/src/python/txtai/app/base.pyindex e8068627..e54837b6 100644--- a/src/python/txtai/app/base.py+++ b/src/python/txtai/app/base.py@@ -7,13 +7,16 @@ import osfrom multiprocessing.pool import ThreadPoolfrom threading import RLock-import yaml-from ..agent import Agentfrom ..embeddings import Documents, Embeddingsfrom ..pipeline import PipelineFactoryfrom ..workflow import WorkflowFactory+# Conditional imports+from ..util import TransformersLib++yaml = TransformersLib().yaml()+# pylint: disable=R0904class Application:diff --git a/src/python/txtai/cloud/hub.py b/src/python/txtai/cloud/hub.pyindex ba0dea06..4cae96c3 100644--- a/src/python/txtai/cloud/hub.py+++ b/src/python/txtai/cloud/hub.py@@ -5,11 +5,12 @@ Hugging Face Hub moduleimport osimport tempfile-import huggingface_hub+from .base import Cloud-from huggingface_hub.utils import RepositoryNotFoundError+# Conditional imports+from ..util import TransformersLib-from .base import Cloud+huggingface_hub = TransformersLib().huggingface_hub()class HuggingFaceHub(Cloud):@@ -30,11 +31,11 @@ class HuggingFaceHub(Cloud):# Otherwise return repository metadatareturn huggingface_hub.model_info(repo_id=self.config["container"], revision=self.config.get("revision"), token=self.config.get("token"))- except RepositoryNotFoundError:+ except huggingface_hub.utils.RepositoryNotFoundError:return Nonedef load(self, path=None):- # Download archvie file and return local path+ # Download archive file and return local pathif self.isarchive(path):return huggingface_hub.hf_hub_download(repo_id=self.config["container"],diff --git a/src/python/txtai/models/pooling/base.py b/src/python/txtai/models/pooling/base.pyindex ed39c612..783cad0b 100644--- a/src/python/txtai/models/pooling/base.py+++ b/src/python/txtai/models/pooling/base.py@@ -6,12 +6,10 @@ import jsonimport numpy as np-from huggingface_hub.errors import HFValidationError-from ..models import Models# Conditional imports-from ...util import Download, TransformersLib+from ...util import Download, DownloadError, TransformersLibtransformerslib = TransformersLib()torch = transformerslib.torch()@@ -178,7 +176,7 @@ class Pooling(Module):config = json.load(f)# Ignore this error - invalid repo or directory- except (HFValidationError, OSError):+ except DownloadError:passreturn configdiff --git a/src/python/txtai/models/pooling/factory.py b/src/python/txtai/models/pooling/factory.pyindex 92fcde8c..0e635e7f 100644--- a/src/python/txtai/models/pooling/factory.py+++ b/src/python/txtai/models/pooling/factory.py@@ -5,15 +5,13 @@ Factory moduleimport jsonimport os-from huggingface_hub.errors import HFValidationError-from .base import Poolingfrom .cls import ClsPoolingfrom .last import LastPoolingfrom .late import LatePoolingfrom .mean import MeanPooling-from ...util import Download+from ...util import Download, DownloadErrorclass PoolingFactory:@@ -148,7 +146,7 @@ class PoolingFactory:config = json.load(f)# Ignore this error - invalid repo or directory- except (HFValidationError, OSError):+ except DownloadError:passreturn configdiff --git a/src/python/txtai/models/pooling/late.py b/src/python/txtai/models/pooling/late.pyindex fccae3d0..c57f770f 100644--- a/src/python/txtai/models/pooling/late.py+++ b/src/python/txtai/models/pooling/late.py@@ -4,8 +4,6 @@ Late moduleimport numpy as np-from safetensors import safe_open-from .base import Poolingfrom .muvera import Muvera@@ -13,6 +11,7 @@ from .muvera import Muverafrom ...util import Download, TransformersLibtransformerslib = TransformersLib()+safetensors = transformerslib.safetensors()torch = transformerslib.torch()@@ -44,7 +43,7 @@ class LatePooling(Pooling):# Load linear layerpath = Download()(path, name)- with safe_open(filename=path, framework="pt") as f:+ with safetensors.safe_open(filename=path, framework="pt") as f:weights = f.get_tensor("linear.weight")# Load weights into linear layerdiff --git a/src/python/txtai/pipeline/audio/texttospeech.py b/src/python/txtai/pipeline/audio/texttospeech.pyindex 1a1cb936..0718bf3b 100644--- a/src/python/txtai/pipeline/audio/texttospeech.py+++ b/src/python/txtai/pipeline/audio/texttospeech.py@@ -20,20 +20,17 @@ import loggingfrom io import BytesIO-import yaml-import numpy as np-from huggingface_hub.errors import HFValidationError-from ..base import Pipeline# Conditional imports-from ...util import Download, TransformersLib+from ...util import Download, DownloadError, TransformersLibtransformerslib = TransformersLib()torch = transformerslib.torch()transformers = transformerslib.transformers()+yaml = transformerslib.yaml()# Logging configurationlogger = logging.getLogger(__name__)@@ -136,7 +133,7 @@ class TextToSpeech(Pipeline):try:# Check if file existsexists = Download()(path, name) is not None- except (HFValidationError, OSError):+ except DownloadError:return Falsereturn existsdiff --git a/src/python/txtai/pipeline/data/tokenizer.py b/src/python/txtai/pipeline/data/tokenizer.pyindex c5c979e8..f292afbd 100644--- a/src/python/txtai/pipeline/data/tokenizer.py+++ b/src/python/txtai/pipeline/data/tokenizer.py@@ -5,10 +5,13 @@ Tokenizer moduleimport reimport string-import regex-from ..base import Pipeline+# Conditional imports+from ...util import TransformersLib++regex = TransformersLib().regex()+class Tokenizer(Pipeline):"""diff --git a/src/python/txtai/pipeline/text/entity.py b/src/python/txtai/pipeline/text/entity.pyindex b52dd70e..57564a18 100644--- a/src/python/txtai/pipeline/text/entity.py+++ b/src/python/txtai/pipeline/text/entity.py@@ -10,10 +10,8 @@ try:except ImportError:GLINER = False-from huggingface_hub.errors import HFValidationError-from ...models import Models-from ...util import Download+from ...util import Download, DownloadErrorfrom ..hfpipeline import HFPipeline@@ -90,7 +88,7 @@ class Entity(HFPipeline):return Download()(path, "gliner_config.json") is not None# Ignore this error - invalid repo or directory- except (HFValidationError, OSError):+ except DownloadError:passreturn Falsediff --git a/src/python/txtai/pipeline/text/translation.py b/src/python/txtai/pipeline/text/translation.pyindex 6b1a7c10..b069e24c 100644--- a/src/python/txtai/pipeline/text/translation.py+++ b/src/python/txtai/pipeline/text/translation.py@@ -10,11 +10,14 @@ try:except ImportError:STATICVECTORS = False-from huggingface_hub.hf_api import HfApi-from ...models import Modelsfrom ..hfmodel import HFModel+# Conditional imports+from ...util import TransformersLib++huggingface_hub = TransformersLib().huggingface_hub()+class Translation(HFModel):"""@@ -110,7 +113,7 @@ class Translation(HFModel):list of source-target language model ids"""- ids = [x.id for x in HfApi().list_models(author="Helsinki-NLP")] if self.findmodels else []+ ids = [x.id for x in huggingface_hub.hf_api.HfApi().list_models(author="Helsinki-NLP")] if self.findmodels else []return set(ids)def detect(self, texts):diff --git a/src/python/txtai/serialize/messagepack.py b/src/python/txtai/serialize/messagepack.pyindex 0d4a5897..dd952443 100644--- a/src/python/txtai/serialize/messagepack.py+++ b/src/python/txtai/serialize/messagepack.py@@ -26,7 +26,7 @@ class MessagePack(Serialize):super().__init__()if not MSGPACK:- raise ImportError("MessagePack is not available - install to msgpack enable")+ raise ImportError("MessagePack is not available - install msgpack to enable")# Streaming unpackerself.streaming = streamingdiff --git a/src/python/txtai/util/__init__.py b/src/python/txtai/util/__init__.pyindex b6bcbce3..baab7b33 100644--- a/src/python/txtai/util/__init__.py+++ b/src/python/txtai/util/__init__.py@@ -2,7 +2,7 @@Utility imports"""-from .download import Download+from .download import *from .resolver import Resolverfrom .sparsearray import SparseArrayfrom .template import TemplateFormatterdiff --git a/src/python/txtai/util/download.py b/src/python/txtai/util/download.pyindex 01330fd4..add12a0f 100644--- a/src/python/txtai/util/download.py+++ b/src/python/txtai/util/download.py@@ -4,7 +4,12 @@ Download moduleimport os-from huggingface_hub import hf_hub_download+# Conditional imports+from .transformerslib import TransformersLib++transformerslib = TransformersLib()+huggingface_hub = transformerslib.huggingface_hub()+HFValidationError = transformerslib.hferror()class Download:@@ -18,7 +23,7 @@ class Download:Args:path: model path or repo- name: file name to load+ name: file nameReturns:local cached model path, if available@@ -40,5 +45,15 @@ class Download:path, name = "/".join(parts[:repo]), "/".join(parts[repo:])# Download (if necessary) and return local file path- local = os.path.join(path, name)- return local if os.path.exists(local) else hf_hub_download(repo_id=path, filename=name)+ try:+ local = os.path.join(path, name)+ return local if os.path.exists(local) else huggingface_hub.hf_hub_download(repo_id=path, filename=name)++ except (HFValidationError, OSError) as e:+ raise DownloadError(f"Error locating file with parameters: path={path}, name={name}") from e+++class DownloadError(Exception):+ """+ Exception raised when a local or remote file path is not valid.+ """diff --git a/src/python/txtai/util/transformerslib.py b/src/python/txtai/util/transformerslib.pyindex 71418edf..b761b2fb 100644--- a/src/python/txtai/util/transformerslib.py+++ b/src/python/txtai/util/transformerslib.py@@ -11,7 +11,7 @@ class TransformersLib:def arguments(self):"""- Imports transformers.TrainingArguments+ Imports transformers.TrainingArguments."""try:@@ -66,6 +66,48 @@ class TransformersLib:return Dataset+ def hferror(self):+ """+ Import huggingface_hub.errors.HFValidationError.+ """++ try:+ from huggingface_hub.errors import HFValidationError++ except ImportError:++ class HFValidationError(Exception):+ """+ Stub for HFValidationError+ """++ return HFValidationError++ def huggingface_hub(self):+ """+ Imports huggingface_hub.++ Returns:+ torch+ """++ try:+ import huggingface_hub++ except ImportError:++ class HuggingFaceHub:+ """+ Stub for HuggingFaceHub+ """++ def __getattr__(self, name):+ raise ImportError("Hugging Face Hub is not installed, install huggingface-hub to use this module")++ huggingface_hub = HuggingFaceHub()++ return huggingface_hub+def model(self):"""Imports transformers.modeling_utils.PreTrainedModel.@@ -88,7 +130,7 @@ class TransformersLib:def module(self):"""- Imports torch.nn.Module+ Imports torch.nn.Module.Returns:Module@@ -109,6 +151,56 @@ class TransformersLib:return Module+ def regex(self):+ """+ Imports regex.++ Returns:+ regex+ """++ try:+ import regex++ except ImportError:++ class Regex:+ """+ Stub for Regex+ """++ def __getattr__(self, name):+ raise ImportError("Regex is not installed, install regex to use this module")++ regex = Regex()++ return regex++ def safetensors(self):+ """+ Imports safetensors.++ Returns:+ safetensors+ """++ try:+ import safetensors++ except ImportError:++ class Safetensors:+ """+ Stub for Safetensors+ """++ def __getattr__(self, name):+ raise ImportError("Safetensors is not installed, install safetensors to use this module")++ safetensors = Safetensors()++ return safetensors+def torch(self):"""Imports torch.@@ -158,3 +250,28 @@ class TransformersLib:transformers = Transformers()return transformers++ def yaml(self):+ """+ Imports yaml.++ Returns:+ yaml+ """++ try:+ import yaml++ except ImportError:++ class YAML:+ """+ Stub for yaml+ """++ def __getattr__(self, name):+ raise ImportError("PyYAML is not installed, install pyyaml to use this module")++ yaml = YAML()++ return yamldiff --git a/src/python/txtai/vectors/dense/m2v.py b/src/python/txtai/vectors/dense/m2v.pyindex 17f5cddf..20bfaf2d 100644--- a/src/python/txtai/vectors/dense/m2v.py+++ b/src/python/txtai/vectors/dense/m2v.py@@ -4,8 +4,6 @@ Model2Vec moduleimport json-from huggingface_hub.errors import HFValidationError-# Conditional importtry:from model2vec import StaticModel@@ -14,7 +12,7 @@ try:except ImportError:MODEL2VEC = False-from ...util import Download+from ...util import Download, DownloadErrorfrom ..base import Vectors@@ -44,7 +42,7 @@ class Model2Vec(Vectors):return config.get("model_type") == "model2vec"# Ignore this error - invalid repo or directory- except (HFValidationError, OSError):+ except DownloadError:passreturn Falsediff --git a/src/python/txtai/vectors/dense/words.py b/src/python/txtai/vectors/dense/words.pyindex f94555d9..f43140df 100644--- a/src/python/txtai/vectors/dense/words.py+++ b/src/python/txtai/vectors/dense/words.py@@ -11,8 +11,6 @@ from multiprocessing import Poolimport numpy as np-from huggingface_hub.errors import HFValidationError-# Conditional importtry:from staticvectors import Database, StaticVectors@@ -22,7 +20,7 @@ except ImportError:STATICVECTORS = Falsefrom ...pipeline import Tokenizer-from ...util import Download+from ...util import Download, DownloadErrorfrom ..base import Vectors@@ -99,7 +97,7 @@ class WordVectors(Vectors):return config.get("model_type") == "staticvectors"# Ignore this error - invalid repo or directory- except (HFValidationError, OSError):+ except DownloadError:passreturn Falsediff --git a/test/python/testcloud.py b/test/python/testcloud.pyindex e748c2c2..81a700c0 100644--- a/test/python/testcloud.py+++ b/test/python/testcloud.py@@ -9,6 +9,7 @@ import unittestfrom unittest.mock import patch+from huggingface_hub import hf_hub_downloadfrom txtai.cloud import Cloudfrom txtai.embeddings import Embeddings@@ -107,8 +108,16 @@ class TestCloud(unittest.TestCase):if "Invalid" in kwargs["repo_id"]:raise FileNotFoundError- # Return either .gitattributes file or index- return attributes if kwargs["filename"] == ".gitattributes" else index+ # Check for .gitattributes file+ if kwargs["filename"] == ".gitattributes":+ return attributes++ # Check for cloud index path+ if any(kwargs["filename"] == x for x in paths):+ return index++ # Use original method for all other requests+ return hf_hub_download(**kwargs)# Patch write methods since token will not be availablecreate.return_value = None@@ -130,7 +139,8 @@ class TestCloud(unittest.TestCase):attributes = tmp.name# Run tests with uncompressed and compressed index- for path in [f"cloud.{provider}", f"cloud.{provider}.tar.gz"]:+ paths = [f"cloud.{provider}", f"cloud.{provider}.tar.gz"]+ for path in paths:self.runTests(path, {"provider": provider, "container": "neuml/txtai-intro"})def runTests(self, path, cloud):diff --git a/test/python/testtransformers.py b/test/python/testtransformers.pyindex 33dfb5cc..e7240d95 100644--- a/test/python/testtransformers.py+++ b/test/python/testtransformers.py@@ -21,6 +21,10 @@ class TestTransformers(unittest.TestCase):"""modules = [+ "huggingface_hub",+ "huggingface_hub.errors",+ "regex",+ "safetensors","transformers","transformers.configuration_utils","transformers.modeling_utils",@@ -29,6 +33,7 @@ class TestTransformers(unittest.TestCase):"torch.nn","torch.onnx","torch.utils.data",+ "yaml",]# Get handle to all currently loaded txtai modules@@ -65,18 +70,29 @@ class TestTransformers(unittest.TestCase):Test transformers not installed"""+ # pylint: disable=W0106from txtai.util import TransformersLiblib = TransformersLib()# Test transformers stubs- for x in [lib.arguments(), lib.config(), lib.dataset(), lib.module(), lib.model()]:+ for x in [lib.arguments(), lib.config(), lib.dataset(), lib.hferror(), lib.module(), lib.model()]:self.assertTrue(x.__module__.endswith("transformerslib"))- # pylint: disable=W0106with self.assertRaises(ImportError):- lib.transformers().AutoModel+ lib.huggingface_hub().hf_hub_download++ with self.assertRaises(ImportError):+ lib.regex().compile++ with self.assertRaises(ImportError):+ lib.safetensors().safe_open- # pylint: disable=W0106with self.assertRaises(ImportError):lib.torch().device++ with self.assertRaises(ImportError):+ lib.transformers().AutoModel++ with self.assertRaises(ImportError):+ lib.yaml().safe_open