Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Merge branch 'main' into feature/knn-demonstrations-sampler

Igor Kotenkov7a58f4147d36
diff --cc dsp/primitives/demonstrate.py
index 10a725a47,cc15ccd2a..ea20a246e
--- a/dsp/primitives/demonstrate.py
+++ b/dsp/primitives/demonstrate.py
@@@ -1,11 -1,7 +1,11 @@@
-import dsp
import random
+from typing import Callable, List
+
+import numpy as np
-from dsp.utils import dotdict, has_answer, DPR_normalize, normalize_text, EM, F1
+import dsp
- from dsp.utils import (EM, DPR_normalize, create_faiss_index, dotdict,
++from dsp.utils import (EM, F1, DPR_normalize, create_faiss_index, dotdict,
+ has_answer, normalize_text)
class Example(dotdict):
@@@ -113,68 -101,14 +113,71 @@@ def passage_match(passages, answers)
# answers = example.answers
return any(passage_has_answers(psg, answers) for psg in passages)
-
- def answer_match(prediction, answers):
+ def answer_match(prediction, answers, frac=1.0):
# pred = example.prediction
# answers = example.answers
- return EM(prediction, answers)
+
+ if frac >= 1.0:
+ return EM(prediction, answers)
+
+ return F1(prediction, answers) >= frac
+
def passage_has_answers(passage, answers):
- return has_answer([DPR_normalize(normalize_text(ans)) for ans in answers], normalize_text(passage))
+ return has_answer(
+ tokenized_answers=[DPR_normalize(normalize_text(ans)) for ans in answers],
+ text=normalize_text(passage)
+ )
+
+
+def cast_naive_get_only_question_text(inp_example: Example) -> Example:
+ '''
+ Extracts question as a field to vectorize with Vectorizer object. `question` field is used.
+ '''
+ return inp_example.copy(text_to_vectorize=inp_example.question)
+
+
+def cast_naive_get_question_and_answer(inp_example: Example) -> Example:
+ '''
+ Extracts question and answer as fields to vectorize with Vectorizer object.
+ `question` and `answer` fields are used. They will be concatenated with the word "Answer"
+ between.
+ '''
+ text_to_vectorize = inp_example.question.strip() + " Answer: " + inp_example.answer.strip()
+ return inp_example.copy(text_to_vectorize=text_to_vectorize)
+
+
+def knn(
+ train: List[Example],
+ cast: Callable[[Example], Example] = cast_naive_get_only_question_text,
+ **knn_args
+) -> Callable[[Example, int], List[Example]]:
+ '''
+ A function that vectorizes train data using `dsm.settings.vectorizer`, then build an ANN/KNN
+ index to search similar questions among `train` samples.
+
+ Args:
+ train: a bunch of questions to put in index & search later
+ cast: function that contructs text before vectorization. By default,
+ it uses only question. Check `cast_naive_get_question_and_answer` for more details.
+ n_probe: number of closest IVF-clusters to check for neighbours.
+ Doesn't affect bruteforce-based search.
+ knn_args: check `create_faiss_index` function for details on ANN/KNN arguments.
+ Returns: function to search similar Examples from `train` in FAISS-index.
+ '''
+ train_casted_to_vectorize = [cast(cur_elem) for cur_elem in train]
+
+ vectorizer: "BaseSentenceVectorizer" = dsp.settings.vectorizer
+ all_vectors = vectorizer(train_casted_to_vectorize).astype(np.float32)
+
+ index = create_faiss_index(emb_dim=all_vectors.shape[1], n_objects=len(train), **knn_args)
+ index.train(all_vectors)
+ index.add(all_vectors)
+
+ def inner_knn_search(inp_example: Example, k: int) -> List[Example]:
+ inp_example_vector = vectorizer([cast(inp_example)])
+ _, nearest_samples_idxs = index.search(inp_example_vector, k)
+ train_sampled = [train[cur_idx] for cur_idx in nearest_samples_idxs[0]]
+ return train_sampled
+
+ return inner_knn_search
diff --cc setup.py
index e191ebe37,6db14be91..a544e19d7
--- a/setup.py
+++ b/setup.py
@@@ -17,9 -16,11 +17,12 @@@ setup
"jupyter",
"openai",
"pandas",
+ "sentence-transformers",
"spacy",
"regex",
+ "ujson",
+ "tqdm",
+ "datasets"
],
classifiers=[
"Development Status :: 3 - Alpha",