Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

sentence-transformers integration added for KNN-based DEMONSTRATIONs sampling

Igor Kotenkov0cb534a13136
diff --git a/dsp/primitives/demonstrate.py b/dsp/primitives/demonstrate.py
index d8ea517d2..582c0fc23 100644
--- a/dsp/primitives/demonstrate.py
+++ b/dsp/primitives/demonstrate.py
@@ -4,8 +4,8 @@ from typing import Callable, List
import numpy as np
import dsp
-from dsp.utils import (EM, DPR_normalize, create_faiss_index, dotdict,
- has_answer, normalize_text)
+from dsp.utils import (EM, DPR_normalize, create_faiss_index,
+ determine_devices, dotdict, has_answer, normalize_text)
class Example(dotdict):
@@ -131,32 +131,56 @@ def vectorize_naive_get_field(inp_examples: List[Example]) -> np.ndarray:
return embeddings
+def get_sentence_transformers_vectorize_func(
+ model_name_or_path: str = 'all-MiniLM-L6-v2',
+ vectorize_bs: int = 256,
+ max_gpu_devices: int = 1,
+ normalize_embeddings: bool = False
+) -> Callable[[List[Example]], np.ndarray]:
+ from sentence_transformers import SentenceTransformer
+ num_devices, is_gpu = determine_devices(max_gpu_devices)
+ proxy_device = 'cuda' if is_gpu else 'cpu'
+ model = SentenceTransformer(model_name_or_path, device=proxy_device)
+
+ def vectorize_with_sentence_transformers(inp_examples: List[Example]) -> np.ndarray:
+ text_to_vectorize = [example.text_to_vectorize for example in inp_examples]
+ if is_gpu and num_devices > 1:
+ pool = model.start_multi_process_pool(target_devices=list(range(num_devices)))
+ # Compute the embeddings using the multi-process pool
+ emb = model.encode_multi_process(text_to_vectorize, pool, batch_size=vectorize_bs)
+ model.stop_multi_process_pool(pool)
+ # for some reason, multi-GPU setup doesn't accept normalize_embeddings parameter
+ if normalize_embeddings:
+ emb = emb / np.linalg.norm(emb)
+
+ return emb
+ else:
+ emb = model.encode(
+ sentences=text_to_vectorize,
+ batch_size=vectorize_bs,
+ normalize_embeddings=normalize_embeddings
+ )
+ return emb
+
+ return vectorize_with_sentence_transformers
+
+
def knn(
train: List[Example],
cast: Callable[[Example], Example] = cast_naive_get_only_question_text,
vectorize: Callable[[List[Example]], np.ndarray] = vectorize_naive_get_field,
- vectorize_train_bs: int = 128,
**knn_args
) -> Callable[[Example, int], List[Example]]:
- need_add_one = len(train) % vectorize_train_bs != 0
- n_batches = len(train) // vectorize_train_bs + int(need_add_one)
- emb_dim = vectorize(train[:1]).shape[1]
- all_vectors = np.empty((len(train), emb_dim), dtype=np.float32)
- # batched vectorization for train Examples
- for batch_idx in range(n_batches):
- batch_start_idx = batch_idx * vectorize_train_bs
- batch_end_idx = (batch_idx + 1) * vectorize_train_bs
- cur_batch = train[batch_start_idx: batch_end_idx]
- cur_batch = [cast(cur_elem) for cur_elem in cur_batch]
- cur_batch_vectors = vectorize(cur_batch)
- all_vectors[batch_start_idx:batch_end_idx, :] = cur_batch_vectors
-
- index = create_faiss_index(emb_dim=emb_dim, n_objects=len(train), **knn_args)
+ train_casted_to_vectorize = [cast(cur_elem) for cur_elem in train]
+
+ all_vectors = vectorize(train_casted_to_vectorize).astype(np.float32)
+
+ index = create_faiss_index(emb_dim=all_vectors.shape[1], n_objects=len(train), **knn_args)
index.train(all_vectors)
index.add(all_vectors)
def inner_knn_search(inp_example: Example, k: int) -> List[Example]:
- inp_example_vector = vectorize([inp_example])
+ inp_example_vector = vectorize([cast(inp_example)])
_, nearest_samples_idxs = index.search(inp_example_vector, k)
train_sampled = [train[cur_idx] for cur_idx in nearest_samples_idxs[0]]
return train_sampled
diff --git a/dsp/utils/ann_utils.py b/dsp/utils/ann_utils.py
index 89924b019..1f38cd1c6 100644
--- a/dsp/utils/ann_utils.py
+++ b/dsp/utils/ann_utils.py
@@ -1,11 +1,10 @@
-from typing import Optional, Tuple
+from typing import Tuple
import faiss
-import numpy as np
from faiss import Index
-def _determine_devices(max_gpu_devices: int = 0) -> Tuple[int, bool]:
+def determine_devices(max_gpu_devices: int = 0) -> Tuple[int, bool]:
"""
Determine which device we should use
Args:
@@ -118,7 +117,7 @@ def create_faiss_index(
index.nprobe = n_probe
- num_devices, is_gpu = _determine_devices(max_gpu_devices)
+ num_devices, is_gpu = determine_devices(max_gpu_devices)
if is_gpu:
cloner_options = faiss.GpuMultipleClonerOptions()
cloner_options.shard = True # split (not replicate) one index between GPUs
diff --git a/knn_demonstrations_test.ipynb b/knn_demonstrations_test.ipynb
index 6e01a2707..28b85ac18 100644
--- a/knn_demonstrations_test.ipynb
+++ b/knn_demonstrations_test.ipynb
@@ -112,6 +112,107 @@
"[train.index(i) for i in knn_res_train] # as expected"
]
},
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### SentenceTransformer-based knn"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# we need to add some relevant and irrelevant questions to double-check the logic\n",
+ "\n",
+ "train = [\n",
+ " # base questions\n",
+ " ('Who produced the album that included a re-recording of \"Lithium\"?', ['Butch Vig']),\n",
+ " ('Who was the director of the 2009 movie featuring Peter Outerbridge as William Easton?', ['Kevin Greutert']),\n",
+ " ('The heir to the Du Pont family fortune sponsored what wrestling team?', ['Foxcatcher', 'Team Foxcatcher', 'Foxcatcher Team']),\n",
+ " ('In what year was the star of To Hell and Back born?', ['1925']),\n",
+ " ('Which award did the first book of Gary Zukav receive?', ['U.S. National Book Award', 'National Book Award']),\n",
+ " ('What city was the victim of Joseph Druces working in?', ['Boston, Massachusetts', 'Boston']),\n",
+ " # extra\n",
+ " ('Who founded Smart Studios in Madison, while still performing drums in Spooner at night and driving a taxi cab during the day?', ['Butch Vig']),\n",
+ " ('There were two founders of recording studio called \"Smart Studios\". The first one was Butch Vig. Who was the second founder?', ['Steve Marker']),\n",
+ "\n",
+ " ('What is the first book of Gary Zukav?', ['\"The Dancing Wu Li Masters\"', \"'The Dancing Wu Li Masters'\", \"The Dancing Wu Li Masters\"]),\n",
+ " ('When did Zukav returned to the United States?', ['1970'])\n",
+ "]\n",
+ "\n",
+ "train = [dsp.Example(question=question, answer=answer) for question, answer in train]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "transformer_vectorize = dsp.get_sentence_transformers_vectorize_func()\n",
+ "knn_func = dsp.knn(train, vectorize=transformer_vectorize)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Who produced the album that included a re-recording of \"Lithium\"?\n",
+ "--------------------------------------------------------------------------------\n",
+ "Who produced the album that included a re-recording of \"Lithium\"?\n",
+ "Who founded Smart Studios in Madison, while still performing drums in Spooner at night and driving a taxi cab during the day?\n",
+ "There were two founders of recording studio called \"Smart Studios\". The first one was Butch Vig. Who was the second founder?\n"
+ ]
+ }
+ ],
+ "source": [
+ "QUERY_IDX = 0\n",
+ "\n",
+ "knn_res_train_vec = knn_func(train[QUERY_IDX], 3)\n",
+ "print(train[QUERY_IDX].question) # the query itself\n",
+ "# similar questions\n",
+ "print('-'*80)\n",
+ "for i in [train[train.index(i)].question for i in knn_res_train_vec]:\n",
+ " print(i)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Which award did the first book of Gary Zukav receive?\n",
+ "--------------------------------------------------------------------------------\n",
+ "Which award did the first book of Gary Zukav receive?\n",
+ "What is the first book of Gary Zukav?\n",
+ "When did Zukav returned to the United States?\n"
+ ]
+ }
+ ],
+ "source": [
+ "QUERY_IDX = 4\n",
+ "\n",
+ "knn_res_train_vec = knn_func(train[QUERY_IDX], 3)\n",
+ "print(train[QUERY_IDX].question) # the query itself\n",
+ "# similar questions\n",
+ "print('-'*80)\n",
+ "for i in [train[train.index(i)].question for i in knn_res_train_vec]:\n",
+ " print(i)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/setup.py b/setup.py
index a82ee4bb6..e191ebe37 100644
--- a/setup.py
+++ b/setup.py
@@ -12,11 +12,12 @@ setup(
python_requires='>=3.8',
install_requires=[
"backoff",
- "faiss",
+ "faiss-cpu",
"joblib",
"jupyter",
"openai",
"pandas",
+ "sentence-transformers",
"spacy",
"regex",
],