Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

added KNN for DEMONSTRATE (base version, need proper vectorization)

Igor Kotenkov355425799405
diff --git a/dsp/primitives/demonstrate.py b/dsp/primitives/demonstrate.py
index 8b73653e7..2daebd61c 100644
--- a/dsp/primitives/demonstrate.py
+++ b/dsp/primitives/demonstrate.py
@@ -1,7 +1,8 @@
import dsp
import random
-
-from dsp.utils import dotdict, has_answer, DPR_normalize, normalize_text, EM
+from typing import List, Callable
+import numpy as np
+from dsp.utils import dotdict, has_answer, DPR_normalize, normalize_text, EM, create_faiss_index
class Example(dotdict):
@@ -108,3 +109,44 @@ def answer_match(prediction, answers):
def passage_has_answers(passage, answers):
return has_answer([DPR_normalize(normalize_text(ans)) for ans in answers], normalize_text(passage))
+
+
+def cast_naive_get_only_question_text(inp_example: Example) -> str:
+ return inp_example.question
+
+def vectorize_naive_get_field(inp_examples: List[Example]) -> np.ndarray:
+ embeddings = [cur_example.vectorized.reshape(1, -1) for cur_example in inp_examples]
+ embeddings = np.concatenate(embeddings, axis=0).astype(np.float32)
+ return embeddings
+
+def knn(
+ train: List[Example],
+ cast: Callable[[Example], str] = cast_naive_get_only_question_text,
+ vectorize: Callable[[List[Example]], np.ndarray] = vectorize_naive_get_field,
+ vectorize_train_bs: int = 128,
+ **knn_args
+) -> Callable[[Example, int], List[Example]]:
+ need_add_one = len(train) % vectorize_train_bs != 0
+ n_batches = len(train) // vectorize_train_bs + int(need_add_one)
+ emb_dim = vectorize(train[:1]).shape[1]
+ all_vectors = np.empty((len(train), emb_dim), dtype=np.float32)
+ # batched vectorization for train Examples
+ for batch_idx in range(n_batches):
+ batch_start_idx = batch_idx * vectorize_train_bs
+ batch_end_idx = (batch_idx + 1) * vectorize_train_bs
+ cur_batch = train[batch_start_idx : batch_end_idx]
+ # cur_batch = [cast(cur_elem) for cur_elem in cur_batch]
+ cur_batch_vectors = vectorize(cur_batch)
+ all_vectors[batch_start_idx:batch_end_idx, :] = cur_batch_vectors
+
+ index = create_faiss_index(emb_dim=emb_dim, n_objects=len(train), **knn_args)
+ index.train(all_vectors)
+ index.add(all_vectors)
+
+ def inner_knn_search(inp_example: Example, k: int) -> List[Example]:
+ inp_example_vector = vectorize([inp_example])
+ _, nearest_samples_idxs = index.search(inp_example_vector, k)
+ train_sampled = [train[cur_idx] for cur_idx in nearest_samples_idxs[0]]
+ return train_sampled
+
+ return inner_knn_search
diff --git a/dsp/utils/__init__.py b/dsp/utils/__init__.py
index dc24ec025..86acee3d9 100644
--- a/dsp/utils/__init__.py
+++ b/dsp/utils/__init__.py
@@ -1,4 +1,5 @@
from dsp.utils.settings import *
from dsp.utils.utils import *
from dsp.utils.metrics import *
-from dsp.utils.dpr import *
\ No newline at end of file
+from dsp.utils.dpr import *
+from dsp.utils.ann_utils import *
diff --git a/dsp/utils/ann_utils.py b/dsp/utils/ann_utils.py
new file mode 100644
index 000000000..2586b551e
--- /dev/null
+++ b/dsp/utils/ann_utils.py
@@ -0,0 +1,126 @@
+from typing import Tuple, Optional
+
+import faiss
+import numpy as np
+from faiss import Index
+
+
+def _determine_devices(max_gpu_devices: int = 0) -> Tuple[int, bool]:
+ """
+ Determine which device we should use
+ Args:
+ max_gpu_devices: an integer value, define how many GPUs we'll use.
+ -1 means all devices. 0 means there are no GPUs. Default is 0.
+
+ Returns: number of devices and is it allowed to use CUDA device (True if yes)
+ """
+ n_devices_total = faiss.get_num_gpus()
+ is_gpu = n_devices_total > 0
+
+ if max_gpu_devices > 0 and is_gpu:
+ num_devices = min(n_devices_total, max_gpu_devices)
+ elif max_gpu_devices == -1 and is_gpu:
+ num_devices = n_devices_total
+ else:
+ num_devices = 1
+ is_gpu = False
+ return num_devices, is_gpu
+
+def _get_brute_index(emb_dim: int, dist_type: str) -> Index:
+ if dist_type.lower() == 'ip':
+ index = faiss.IndexFlatIP(emb_dim)
+ elif dist_type.lower() == 'l2':
+ index = faiss.IndexFlatL2(emb_dim)
+ else:
+ raise ValueError(f'Wrong distance type for FAISS Flat Index: {dist_type}')
+
+ return index
+
+
+def _get_ivf_index(
+ emb_dim: int,
+ n_objects: int,
+ in_list_dist_type: str,
+ centroid_dist_type: str,
+ encode_residuals: bool
+) -> Index:
+ # according to the FAISS doc, this should be OK
+ n_list = 4 * (n_objects ** 0.5)
+
+ if in_list_dist_type.lower() == 'ip':
+ quannizer = faiss.IndexFlatIP(emb_dim)
+ elif in_list_dist_type.lower() == 'l2':
+ quannizer = faiss.IndexFlatL2(emb_dim)
+ else:
+ raise ValueError(f'Wrong distance type for FAISS quantizer: {in_list_dist_type}')
+
+ if centroid_dist_type.lower() == 'ip':
+ centroid_metric = faiss.METRIC_INNER_PRODUCT
+ elif centroid_dist_type.lower() == 'l2':
+ centroid_metric = faiss.METRIC_L2
+ else:
+ raise ValueError(f'Wrong distance type for FAISS index: {centroid_dist_type}')
+
+ index = faiss.IndexIVFScalarQuantizer(
+ quannizer,
+ emb_dim,
+ n_list,
+ faiss.ScalarQuantizer.QT_fp16, # TODO: should be optional?
+ centroid_metric,
+ encode_residuals=encode_residuals
+ )
+ return index
+
+
+def create_faiss_index(
+ emb_dim: int,
+ n_objects: int,
+ n_probe: int = 10,
+ max_gpu_devices: int = 0,
+ encode_residuals: bool = True,
+ in_list_dist_type: str = 'L2',
+ centroid_dist_type: str = 'L2'
+) -> Index:
+ """
+ Create IVF index (with IP or L2 dist), without adding data and training
+ Args:
+ emb_dim: size of each embedding
+ n_objects: size of a trainset for index. Used to determine optimal type
+ of index and its settings (will use bruteforce if n_objects is less than 20_000).
+ n_probe: number of closest IVF-clusters to check for neighbours.
+ Doesn't affect bruteforce-based search.
+ max_gpu_devices: maximum amount of GPUs to use for ANN-index. 0 if run on CPU.
+ encode_residuals: whether or not compute residuals. The residual vector is
+ the difference between a vector and the reconstruction that can be
+ decoded from its representation in the index.
+ in_list_dist_type: type of distance to calculate simmilarities within one IVF.
+ Can be IP (for inner product) or L2 distance. Case insensetive.
+ If the index type is bruteforce (n_objects < 20_000), this variable will define
+ the distane type for that bruteforce index.
+ centroid_dist_type: type of distance to calculate simmilarities between a query
+ and cluster centroids. Can be IP (for inner product) or L2 distance.
+ Case insensetive.
+ Returns: untrained FAISS-index
+ """
+ if n_objects < 20_000:
+ # if less than 20_000 / (4 * sqrt(20_000)) ~= 35 points per cluster - make bruteforce
+ # https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index#if-below-1m-vectors-ivfk
+ index = _get_brute_index(emb_dim=emb_dim, dist_type=in_list_dist_type)
+ else:
+ index = _get_ivf_index(
+ emb_dim=emb_dim,
+ n_objects=n_objects,
+ in_list_dist_type=in_list_dist_type,
+ centroid_dist_type=centroid_dist_type,
+ encode_residuals=encode_residuals
+ )
+
+ index.nprobe = n_probe
+
+ num_devices, is_gpu = _determine_devices(max_gpu_devices)
+ if is_gpu:
+ cloner_options = faiss.GpuMultipleClonerOptions()
+ cloner_options.shard = True # split (not replicate) one index between GPUs
+ index = faiss.index_cpu_to_gpus_list(index, cloner_options, range(num_devices))
+
+ return index
diff --git a/knn_demonstrations_test.ipynb b/knn_demonstrations_test.ipynb
new file mode 100644
index 000000000..6e01a2707
--- /dev/null
+++ b/knn_demonstrations_test.ipynb
@@ -0,0 +1,150 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import dsp\n",
+ "import os\n",
+ "\n",
+ "os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train = [('Who produced the album that included a re-recording of \"Lithium\"?', ['Butch Vig']),\n",
+ " ('Who was the director of the 2009 movie featuring Peter Outerbridge as William Easton?', ['Kevin Greutert']),\n",
+ " ('The heir to the Du Pont family fortune sponsored what wrestling team?', ['Foxcatcher', 'Team Foxcatcher', 'Foxcatcher Team']),\n",
+ " ('In what year was the star of To Hell and Back born?', ['1925']),\n",
+ " ('Which award did the first book of Gary Zukav receive?', ['U.S. National Book Award', 'National Book Award']),\n",
+ " ('What city was the victim of Joseph Druces working in?', ['Boston, Massachusetts', 'Boston']),]\n",
+ "\n",
+ "train = [dsp.Example(question=question, answer=answer) for question, answer in train]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "\n",
+ "EMB_DIM = 768\n",
+ "np.random.seed(42)\n",
+ "\n",
+ "\n",
+ "three_closest_embeddings = np.random.randn(3, EMB_DIM)\n",
+ "# shift the last three so they won't be the closest ones\n",
+ "three_furthest_embeddings = three_closest_embeddings.copy() + 10\n",
+ "train_embeddings = np.concatenate([three_closest_embeddings, three_furthest_embeddings], axis=0)\n",
+ "\n",
+ "for idx, example in enumerate(train):\n",
+ " example.vectorized = train_embeddings[idx]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dev = [('Who has a broader scope of profession: E. L. Doctorow or Julia Peterkin?', ['E. L. Doctorow', 'E.L. Doctorow', 'Doctorow'])]\n",
+ "\n",
+ "dev = [dsp.Example(question=question, answer=answer) for question, answer in dev]\n",
+ "dev[0].vectorized = train_embeddings[1] # the same as in train, 2nd will be closest 100%"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "knn_func = dsp.knn(train)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[1, 0, 2]"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "knn_res_dev = knn_func(dev[0], 3)\n",
+ "[train.index(i) for i in knn_res_dev] # first 3 train examples are the closest, the first one w index 1 is the first in the list"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[4, 3, 5]"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "knn_res_train = knn_func(train[-2], 3)\n",
+ "[train.index(i) for i in knn_res_train] # as expected"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "base",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.5"
+ },
+ "orig_nbformat": 4,
+ "vscode": {
+ "interpreter": {
+ "hash": "3b53cdb0bf9b815b12de7512a3c2ebfaf76b70a6c1108d7bdac63e2ca9602dd2"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/setup.py b/setup.py
index 97ad971a7..a82ee4bb6 100644
--- a/setup.py
+++ b/setup.py
@@ -12,6 +12,7 @@ setup(
python_requires='>=3.8',
install_requires=[
"backoff",
+ "faiss",
"joblib",
"jupyter",
"openai",