DSPy: The framework for programming—not prompting—language models
Python
sentence-transformers integration added for KNN-based DEMONSTRATIONs sampling
Igor Kotenkov0cb534a13136
diff --git a/dsp/primitives/demonstrate.py b/dsp/primitives/demonstrate.pyindex d8ea517d2..582c0fc23 100644--- a/dsp/primitives/demonstrate.py+++ b/dsp/primitives/demonstrate.py@@ -4,8 +4,8 @@ from typing import Callable, Listimport numpy as npimport dsp-from dsp.utils import (EM, DPR_normalize, create_faiss_index, dotdict,- has_answer, normalize_text)+from dsp.utils import (EM, DPR_normalize, create_faiss_index,+ determine_devices, dotdict, has_answer, normalize_text)class Example(dotdict):@@ -131,32 +131,56 @@ def vectorize_naive_get_field(inp_examples: List[Example]) -> np.ndarray:return embeddings+def get_sentence_transformers_vectorize_func(+ model_name_or_path: str = 'all-MiniLM-L6-v2',+ vectorize_bs: int = 256,+ max_gpu_devices: int = 1,+ normalize_embeddings: bool = False+) -> Callable[[List[Example]], np.ndarray]:+ from sentence_transformers import SentenceTransformer+ num_devices, is_gpu = determine_devices(max_gpu_devices)+ proxy_device = 'cuda' if is_gpu else 'cpu'+ model = SentenceTransformer(model_name_or_path, device=proxy_device)++ def vectorize_with_sentence_transformers(inp_examples: List[Example]) -> np.ndarray:+ text_to_vectorize = [example.text_to_vectorize for example in inp_examples]+ if is_gpu and num_devices > 1:+ pool = model.start_multi_process_pool(target_devices=list(range(num_devices)))+ # Compute the embeddings using the multi-process pool+ emb = model.encode_multi_process(text_to_vectorize, pool, batch_size=vectorize_bs)+ model.stop_multi_process_pool(pool)+ # for some reason, multi-GPU setup doesn't accept normalize_embeddings parameter+ if normalize_embeddings:+ emb = emb / np.linalg.norm(emb)++ return emb+ else:+ emb = model.encode(+ sentences=text_to_vectorize,+ batch_size=vectorize_bs,+ normalize_embeddings=normalize_embeddings+ )+ return emb++ return vectorize_with_sentence_transformers++def knn(train: List[Example],cast: Callable[[Example], Example] = cast_naive_get_only_question_text,vectorize: Callable[[List[Example]], np.ndarray] = vectorize_naive_get_field,- vectorize_train_bs: int = 128,**knn_args) -> Callable[[Example, int], List[Example]]:- need_add_one = len(train) % vectorize_train_bs != 0- n_batches = len(train) // vectorize_train_bs + int(need_add_one)- emb_dim = vectorize(train[:1]).shape[1]- all_vectors = np.empty((len(train), emb_dim), dtype=np.float32)- # batched vectorization for train Examples- for batch_idx in range(n_batches):- batch_start_idx = batch_idx * vectorize_train_bs- batch_end_idx = (batch_idx + 1) * vectorize_train_bs- cur_batch = train[batch_start_idx: batch_end_idx]- cur_batch = [cast(cur_elem) for cur_elem in cur_batch]- cur_batch_vectors = vectorize(cur_batch)- all_vectors[batch_start_idx:batch_end_idx, :] = cur_batch_vectors-- index = create_faiss_index(emb_dim=emb_dim, n_objects=len(train), **knn_args)+ train_casted_to_vectorize = [cast(cur_elem) for cur_elem in train]++ all_vectors = vectorize(train_casted_to_vectorize).astype(np.float32)++ index = create_faiss_index(emb_dim=all_vectors.shape[1], n_objects=len(train), **knn_args)index.train(all_vectors)index.add(all_vectors)def inner_knn_search(inp_example: Example, k: int) -> List[Example]:- inp_example_vector = vectorize([inp_example])+ inp_example_vector = vectorize([cast(inp_example)])_, nearest_samples_idxs = index.search(inp_example_vector, k)train_sampled = [train[cur_idx] for cur_idx in nearest_samples_idxs[0]]return train_sampleddiff --git a/dsp/utils/ann_utils.py b/dsp/utils/ann_utils.pyindex 89924b019..1f38cd1c6 100644--- a/dsp/utils/ann_utils.py+++ b/dsp/utils/ann_utils.py@@ -1,11 +1,10 @@-from typing import Optional, Tuple+from typing import Tupleimport faiss-import numpy as npfrom faiss import Index-def _determine_devices(max_gpu_devices: int = 0) -> Tuple[int, bool]:+def determine_devices(max_gpu_devices: int = 0) -> Tuple[int, bool]:"""Determine which device we should useArgs:@@ -118,7 +117,7 @@ def create_faiss_index(index.nprobe = n_probe- num_devices, is_gpu = _determine_devices(max_gpu_devices)+ num_devices, is_gpu = determine_devices(max_gpu_devices)if is_gpu:cloner_options = faiss.GpuMultipleClonerOptions()cloner_options.shard = True # split (not replicate) one index between GPUsdiff --git a/knn_demonstrations_test.ipynb b/knn_demonstrations_test.ipynbindex 6e01a2707..28b85ac18 100644--- a/knn_demonstrations_test.ipynb+++ b/knn_demonstrations_test.ipynb@@ -112,6 +112,107 @@"[train.index(i) for i in knn_res_train] # as expected"]},+ {+ "attachments": {},+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "### SentenceTransformer-based knn"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 8,+ "metadata": {},+ "outputs": [],+ "source": [+ "# we need to add some relevant and irrelevant questions to double-check the logic\n",+ "\n",+ "train = [\n",+ " # base questions\n",+ " ('Who produced the album that included a re-recording of \"Lithium\"?', ['Butch Vig']),\n",+ " ('Who was the director of the 2009 movie featuring Peter Outerbridge as William Easton?', ['Kevin Greutert']),\n",+ " ('The heir to the Du Pont family fortune sponsored what wrestling team?', ['Foxcatcher', 'Team Foxcatcher', 'Foxcatcher Team']),\n",+ " ('In what year was the star of To Hell and Back born?', ['1925']),\n",+ " ('Which award did the first book of Gary Zukav receive?', ['U.S. National Book Award', 'National Book Award']),\n",+ " ('What city was the victim of Joseph Druces working in?', ['Boston, Massachusetts', 'Boston']),\n",+ " # extra\n",+ " ('Who founded Smart Studios in Madison, while still performing drums in Spooner at night and driving a taxi cab during the day?', ['Butch Vig']),\n",+ " ('There were two founders of recording studio called \"Smart Studios\". The first one was Butch Vig. Who was the second founder?', ['Steve Marker']),\n",+ "\n",+ " ('What is the first book of Gary Zukav?', ['\"The Dancing Wu Li Masters\"', \"'The Dancing Wu Li Masters'\", \"The Dancing Wu Li Masters\"]),\n",+ " ('When did Zukav returned to the United States?', ['1970'])\n",+ "]\n",+ "\n",+ "train = [dsp.Example(question=question, answer=answer) for question, answer in train]"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 9,+ "metadata": {},+ "outputs": [],+ "source": [+ "transformer_vectorize = dsp.get_sentence_transformers_vectorize_func()\n",+ "knn_func = dsp.knn(train, vectorize=transformer_vectorize)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 10,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Who produced the album that included a re-recording of \"Lithium\"?\n",+ "--------------------------------------------------------------------------------\n",+ "Who produced the album that included a re-recording of \"Lithium\"?\n",+ "Who founded Smart Studios in Madison, while still performing drums in Spooner at night and driving a taxi cab during the day?\n",+ "There were two founders of recording studio called \"Smart Studios\". The first one was Butch Vig. Who was the second founder?\n"+ ]+ }+ ],+ "source": [+ "QUERY_IDX = 0\n",+ "\n",+ "knn_res_train_vec = knn_func(train[QUERY_IDX], 3)\n",+ "print(train[QUERY_IDX].question) # the query itself\n",+ "# similar questions\n",+ "print('-'*80)\n",+ "for i in [train[train.index(i)].question for i in knn_res_train_vec]:\n",+ " print(i)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 11,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Which award did the first book of Gary Zukav receive?\n",+ "--------------------------------------------------------------------------------\n",+ "Which award did the first book of Gary Zukav receive?\n",+ "What is the first book of Gary Zukav?\n",+ "When did Zukav returned to the United States?\n"+ ]+ }+ ],+ "source": [+ "QUERY_IDX = 4\n",+ "\n",+ "knn_res_train_vec = knn_func(train[QUERY_IDX], 3)\n",+ "print(train[QUERY_IDX].question) # the query itself\n",+ "# similar questions\n",+ "print('-'*80)\n",+ "for i in [train[train.index(i)].question for i in knn_res_train_vec]:\n",+ " print(i)"+ ]+ },{"cell_type": "code","execution_count": null,diff --git a/setup.py b/setup.pyindex a82ee4bb6..e191ebe37 100644--- a/setup.py+++ b/setup.py@@ -12,11 +12,12 @@ setup(python_requires='>=3.8',install_requires=["backoff",- "faiss",+ "faiss-cpu","joblib","jupyter","openai","pandas",+ "sentence-transformers","spacy","regex",],