DSPy: The framework for programming—not prompting—language models
Python
feat: FastEmbedVectorizer
Anush0085d3a1f9b9477
diff --git a/dsp/modules/sentence_vectorizer.py b/dsp/modules/sentence_vectorizer.pyindex b878420f7..c5b70cccb 100644--- a/dsp/modules/sentence_vectorizer.py+++ b/dsp/modules/sentence_vectorizer.py@@ -203,3 +203,48 @@ class OpenAIVectorizer(BaseSentenceVectorizer):embeddings = np.array(embeddings_list, dtype=np.float32)return embeddings++class FastEmbedVectorizer(BaseSentenceVectorizer):+ """Sentence vectorizer implementaion using FastEmbed - https://qdrant.github.io/fastembed."""++ def __init__(+ self,+ model_name: str = "BAAI/bge-small-en-v1.5",+ batch_size: int = 256,+ cache_dir: Optional[str] = None,+ threads: Optional[int] = None,+ parallel: Optional[int] = None,+ **kwargs,+ ):+ """Initialize fastembed.TextEmbedding.++ Args:+ model_name (str): The name of the model to use. Defaults to `"BAAI/bge-small-en-v1.5"`.+ batch_size (int): Batch size for encoding. Higher values will use more memory, but be faster.\+ Defaults to 256.+ cache_dir (str, optional): The path to the model cache directory.\+ Can also be set using the `FASTEMBED_CACHE_PATH` env variable.+ threads (int, optional): The number of threads single onnxruntime session can use.+ parallel (int, optional): If `>1`, data-parallel encoding will be used, recommended for large datasets.\+ If `0`, use all available cores.\+ If `None`, don't use data-parallel processing, use default onnxruntime threading.\+ Defaults to None.+ **kwargs: Additional options to pass to fastembed.TextEmbedding+ Raises:+ ValueError: If the model_name is not in the format <org>/<model> e.g. BAAI/bge-small-en-v1.5.+ """+ try:+ from fastembed import TextEmbedding+ except ImportError as e:+ raise ValueError(+ "The 'fastembed' package is not installed. Please install it with `pip install fastembed`",+ ) from e+ self._batch_size = batch_size+ self._parallel = parallel+ self._model = TextEmbedding(model_name=model_name, cache_dir=cache_dir, threads=threads, **kwargs)++ def __call__(self, inp_examples: List["Example"]) -> np.ndarray:+ texts_to_vectorize = self._extract_text_from_examples(inp_examples)+ embeddings = self._model.embed(texts_to_vectorize, batch_size=self._batch_size, parallel=self._parallel)++ return np.array([embedding.tolist() for embedding in embeddings], dtype=np.float32)\ No newline at end of filediff --git a/pyproject.toml b/pyproject.tomlindex f99df941a..8aaf8b44d 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -62,6 +62,7 @@ docs = ["sphinx-automodapi==0.16.0",]dev = ["pytest>=6.2.5"]+fastembed = ["fastembed>=0.1.0"][project.urls]homepage = "https://github.com/stanfordnlp/dspy"@@ -151,6 +152,7 @@ docs = ["sphinx-reredirects","sphinx-automodapi",]+fastembed = ["fastembed"][tool.poetry.group.doc.dependencies]mkdocs = ">=1.5.3"diff --git a/setup.py b/setup.pyindex 7b2438131..4dcce888c 100644--- a/setup.py+++ b/setup.py@@ -31,6 +31,7 @@ setup("faiss-cpu": ["sentence_transformers", "faiss-cpu"],"milvus": ["pymilvus~=2.3.7"],"google-vertex-ai": ["google-cloud-aiplatform==1.43.0"],+ "fastembed": ["fastembed"],},classifiers=["Development Status :: 3 - Alpha",diff --git a/tests/modules/vectorizer/integration_test_fastembed.py b/tests/modules/vectorizer/integration_test_fastembed.pynew file mode 100644index 000000000..d6064cc66--- /dev/null+++ b/tests/modules/vectorizer/integration_test_fastembed.py@@ -0,0 +1,43 @@+from dsp.modules.sentence_vectorizer import FastEmbedVectorizer+import pytest++from dspy.primitives.example import Example++# Skip the test if the 'fastembed' package is not installed+pytest.importorskip("fastembed", reason="'fastembed' is not installed. Use `pip install fastembed` to install it.")+++ "n_dims,model_name", [(384, "BAAI/bge-small-en-v1.5"), (512, "jinaai/jina-embeddings-v2-small-en")]+)+def test_fastembed_with_examples(n_dims, model_name):+ vectorizer = FastEmbedVectorizer(model_name)++ examples = [+ Example(query="What's the price today?", response="The price is $10.00").with_inputs("query", "response"),+ Example(query="What's the weather today?", response="The weather is sunny").with_inputs("query", "response"),+ Example(query="Who was leading the team?", response="It was Jim. Rather enthusiastic guy.").with_inputs(+ "query", "response"+ ),+ ]++ embeddings = vectorizer(examples)++ assert embeddings.shape == (len(examples), n_dims)+++ "n_dims,model_name", [(384, "BAAI/bge-small-en-v1.5"), (512, "jinaai/jina-embeddings-v2-small-en")]+)+def test_fastembed_with_strings(n_dims, model_name):+ vectorizer = FastEmbedVectorizer(model_name)++ inputs = [+ "Jonathan Kent is a fictional character appearing in American comic books published by DC Comics.",+ "Clark Kent is a fictional character appearing in American comic books published by DC Comics.",+ "Martha Kent is a fictional character appearing in American comic books published by DC Comics.",+ ]++ embeddings = vectorizer(inputs)++ assert embeddings.shape == (len(inputs), n_dims)