DSPy: The framework for programming—not prompting—language models
Python
added KNN for DEMONSTRATE (base version, need proper vectorization)
Igor Kotenkov355425799405
diff --git a/dsp/primitives/demonstrate.py b/dsp/primitives/demonstrate.pyindex 8b73653e7..2daebd61c 100644--- a/dsp/primitives/demonstrate.py+++ b/dsp/primitives/demonstrate.py@@ -1,7 +1,8 @@import dspimport random--from dsp.utils import dotdict, has_answer, DPR_normalize, normalize_text, EM+from typing import List, Callable+import numpy as np+from dsp.utils import dotdict, has_answer, DPR_normalize, normalize_text, EM, create_faiss_indexclass Example(dotdict):@@ -108,3 +109,44 @@ def answer_match(prediction, answers):def passage_has_answers(passage, answers):return has_answer([DPR_normalize(normalize_text(ans)) for ans in answers], normalize_text(passage))+++def cast_naive_get_only_question_text(inp_example: Example) -> str:+ return inp_example.question++def vectorize_naive_get_field(inp_examples: List[Example]) -> np.ndarray:+ embeddings = [cur_example.vectorized.reshape(1, -1) for cur_example in inp_examples]+ embeddings = np.concatenate(embeddings, axis=0).astype(np.float32)+ return embeddings++def knn(+ train: List[Example],+ cast: Callable[[Example], str] = cast_naive_get_only_question_text,+ vectorize: Callable[[List[Example]], np.ndarray] = vectorize_naive_get_field,+ vectorize_train_bs: int = 128,+ **knn_args+) -> Callable[[Example, int], List[Example]]:+ need_add_one = len(train) % vectorize_train_bs != 0+ n_batches = len(train) // vectorize_train_bs + int(need_add_one)+ emb_dim = vectorize(train[:1]).shape[1]+ all_vectors = np.empty((len(train), emb_dim), dtype=np.float32)+ # batched vectorization for train Examples+ for batch_idx in range(n_batches):+ batch_start_idx = batch_idx * vectorize_train_bs+ batch_end_idx = (batch_idx + 1) * vectorize_train_bs+ cur_batch = train[batch_start_idx : batch_end_idx]+ # cur_batch = [cast(cur_elem) for cur_elem in cur_batch]+ cur_batch_vectors = vectorize(cur_batch)+ all_vectors[batch_start_idx:batch_end_idx, :] = cur_batch_vectors++ index = create_faiss_index(emb_dim=emb_dim, n_objects=len(train), **knn_args)+ index.train(all_vectors)+ index.add(all_vectors)++ def inner_knn_search(inp_example: Example, k: int) -> List[Example]:+ inp_example_vector = vectorize([inp_example])+ _, nearest_samples_idxs = index.search(inp_example_vector, k)+ train_sampled = [train[cur_idx] for cur_idx in nearest_samples_idxs[0]]+ return train_sampled++ return inner_knn_searchdiff --git a/dsp/utils/__init__.py b/dsp/utils/__init__.pyindex dc24ec025..86acee3d9 100644--- a/dsp/utils/__init__.py+++ b/dsp/utils/__init__.py@@ -1,4 +1,5 @@from dsp.utils.settings import *from dsp.utils.utils import *from dsp.utils.metrics import *-from dsp.utils.dpr import *\ No newline at end of file+from dsp.utils.dpr import *+from dsp.utils.ann_utils import *diff --git a/dsp/utils/ann_utils.py b/dsp/utils/ann_utils.pynew file mode 100644index 000000000..2586b551e--- /dev/null+++ b/dsp/utils/ann_utils.py@@ -0,0 +1,126 @@+from typing import Tuple, Optional++import faiss+import numpy as np+from faiss import Index+++def _determine_devices(max_gpu_devices: int = 0) -> Tuple[int, bool]:+ """+ Determine which device we should use+ Args:+ max_gpu_devices: an integer value, define how many GPUs we'll use.+ -1 means all devices. 0 means there are no GPUs. Default is 0.++ Returns: number of devices and is it allowed to use CUDA device (True if yes)+ """+ n_devices_total = faiss.get_num_gpus()+ is_gpu = n_devices_total > 0++ if max_gpu_devices > 0 and is_gpu:+ num_devices = min(n_devices_total, max_gpu_devices)+ elif max_gpu_devices == -1 and is_gpu:+ num_devices = n_devices_total+ else:+ num_devices = 1+ is_gpu = False+ return num_devices, is_gpu++def _get_brute_index(emb_dim: int, dist_type: str) -> Index:+ if dist_type.lower() == 'ip':+ index = faiss.IndexFlatIP(emb_dim)+ elif dist_type.lower() == 'l2':+ index = faiss.IndexFlatL2(emb_dim)+ else:+ raise ValueError(f'Wrong distance type for FAISS Flat Index: {dist_type}')++ return index+++def _get_ivf_index(+ emb_dim: int,+ n_objects: int,+ in_list_dist_type: str,+ centroid_dist_type: str,+ encode_residuals: bool+) -> Index:+ # according to the FAISS doc, this should be OK+ n_list = 4 * (n_objects ** 0.5)++ if in_list_dist_type.lower() == 'ip':+ quannizer = faiss.IndexFlatIP(emb_dim)+ elif in_list_dist_type.lower() == 'l2':+ quannizer = faiss.IndexFlatL2(emb_dim)+ else:+ raise ValueError(f'Wrong distance type for FAISS quantizer: {in_list_dist_type}')++ if centroid_dist_type.lower() == 'ip':+ centroid_metric = faiss.METRIC_INNER_PRODUCT+ elif centroid_dist_type.lower() == 'l2':+ centroid_metric = faiss.METRIC_L2+ else:+ raise ValueError(f'Wrong distance type for FAISS index: {centroid_dist_type}')++ index = faiss.IndexIVFScalarQuantizer(+ quannizer,+ emb_dim,+ n_list,+ faiss.ScalarQuantizer.QT_fp16, # TODO: should be optional?+ centroid_metric,+ encode_residuals=encode_residuals+ )+ return index+++def create_faiss_index(+ emb_dim: int,+ n_objects: int,+ n_probe: int = 10,+ max_gpu_devices: int = 0,+ encode_residuals: bool = True,+ in_list_dist_type: str = 'L2',+ centroid_dist_type: str = 'L2'+) -> Index:+ """+ Create IVF index (with IP or L2 dist), without adding data and training+ Args:+ emb_dim: size of each embedding+ n_objects: size of a trainset for index. Used to determine optimal type+ of index and its settings (will use bruteforce if n_objects is less than 20_000).+ n_probe: number of closest IVF-clusters to check for neighbours.+ Doesn't affect bruteforce-based search.+ max_gpu_devices: maximum amount of GPUs to use for ANN-index. 0 if run on CPU.+ encode_residuals: whether or not compute residuals. The residual vector is+ the difference between a vector and the reconstruction that can be+ decoded from its representation in the index.+ in_list_dist_type: type of distance to calculate simmilarities within one IVF.+ Can be IP (for inner product) or L2 distance. Case insensetive.+ If the index type is bruteforce (n_objects < 20_000), this variable will define+ the distane type for that bruteforce index.+ centroid_dist_type: type of distance to calculate simmilarities between a query+ and cluster centroids. Can be IP (for inner product) or L2 distance.+ Case insensetive.+ Returns: untrained FAISS-index+ """+ if n_objects < 20_000:+ # if less than 20_000 / (4 * sqrt(20_000)) ~= 35 points per cluster - make bruteforce+ # https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index#if-below-1m-vectors-ivfk+ index = _get_brute_index(emb_dim=emb_dim, dist_type=in_list_dist_type)+ else:+ index = _get_ivf_index(+ emb_dim=emb_dim,+ n_objects=n_objects,+ in_list_dist_type=in_list_dist_type,+ centroid_dist_type=centroid_dist_type,+ encode_residuals=encode_residuals+ )++ index.nprobe = n_probe++ num_devices, is_gpu = _determine_devices(max_gpu_devices)+ if is_gpu:+ cloner_options = faiss.GpuMultipleClonerOptions()+ cloner_options.shard = True # split (not replicate) one index between GPUs+ index = faiss.index_cpu_to_gpus_list(index, cloner_options, range(num_devices))++ return indexdiff --git a/knn_demonstrations_test.ipynb b/knn_demonstrations_test.ipynbnew file mode 100644index 000000000..6e01a2707--- /dev/null+++ b/knn_demonstrations_test.ipynb@@ -0,0 +1,150 @@+{+ "cells": [+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {},+ "outputs": [],+ "source": [+ "import dsp\n",+ "import os\n",+ "\n",+ "os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE'"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "metadata": {},+ "outputs": [],+ "source": [+ "train = [('Who produced the album that included a re-recording of \"Lithium\"?', ['Butch Vig']),\n",+ " ('Who was the director of the 2009 movie featuring Peter Outerbridge as William Easton?', ['Kevin Greutert']),\n",+ " ('The heir to the Du Pont family fortune sponsored what wrestling team?', ['Foxcatcher', 'Team Foxcatcher', 'Foxcatcher Team']),\n",+ " ('In what year was the star of To Hell and Back born?', ['1925']),\n",+ " ('Which award did the first book of Gary Zukav receive?', ['U.S. National Book Award', 'National Book Award']),\n",+ " ('What city was the victim of Joseph Druces working in?', ['Boston, Massachusetts', 'Boston']),]\n",+ "\n",+ "train = [dsp.Example(question=question, answer=answer) for question, answer in train]"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {},+ "outputs": [],+ "source": [+ "import numpy as np\n",+ "\n",+ "EMB_DIM = 768\n",+ "np.random.seed(42)\n",+ "\n",+ "\n",+ "three_closest_embeddings = np.random.randn(3, EMB_DIM)\n",+ "# shift the last three so they won't be the closest ones\n",+ "three_furthest_embeddings = three_closest_embeddings.copy() + 10\n",+ "train_embeddings = np.concatenate([three_closest_embeddings, three_furthest_embeddings], axis=0)\n",+ "\n",+ "for idx, example in enumerate(train):\n",+ " example.vectorized = train_embeddings[idx]"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 4,+ "metadata": {},+ "outputs": [],+ "source": [+ "dev = [('Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?', ['E. L. Doctorow', 'E.L. Doctorow', 'Doctorow'])]\n",+ "\n",+ "dev = [dsp.Example(question=question, answer=answer) for question, answer in dev]\n",+ "dev[0].vectorized = train_embeddings[1] # the same as in train, 2nd will be closest 100%"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "metadata": {},+ "outputs": [],+ "source": [+ "knn_func = dsp.knn(train)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 6,+ "metadata": {},+ "outputs": [+ {+ "data": {+ "text/plain": [+ "[1, 0, 2]"+ ]+ },+ "execution_count": 6,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "knn_res_dev = knn_func(dev[0], 3)\n",+ "[train.index(i) for i in knn_res_dev] # first 3 train examples are the closest, the first one w index 1 is the first in the list"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 7,+ "metadata": {},+ "outputs": [+ {+ "data": {+ "text/plain": [+ "[4, 3, 5]"+ ]+ },+ "execution_count": 7,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "knn_res_train = knn_func(train[-2], 3)\n",+ "[train.index(i) for i in knn_res_train] # as expected"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": []+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": "base",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.8.5"+ },+ "orig_nbformat": 4,+ "vscode": {+ "interpreter": {+ "hash": "3b53cdb0bf9b815b12de7512a3c2ebfaf76b70a6c1108d7bdac63e2ca9602dd2"+ }+ }+ },+ "nbformat": 4,+ "nbformat_minor": 2+}diff --git a/setup.py b/setup.pyindex 97ad971a7..a82ee4bb6 100644--- a/setup.py+++ b/setup.py@@ -12,6 +12,7 @@ setup(python_requires='>=3.8',install_requires=["backoff",+ "faiss","joblib","jupyter","openai",