DSPy: The framework for programming—not prompting—language models
Python
Merge branch 'main' into feature/knn-demonstrations-sampler
Igor Kotenkov7a58f4147d36
diff --cc dsp/primitives/demonstrate.pyindex 10a725a47,cc15ccd2a..ea20a246e--- a/dsp/primitives/demonstrate.py+++ b/dsp/primitives/demonstrate.py@@@ -1,11 -1,7 +1,11 @@@-import dspimport random+from typing import Callable, List++import numpy as np-from dsp.utils import dotdict, has_answer, DPR_normalize, normalize_text, EM, F1+import dsp- from dsp.utils import (EM, DPR_normalize, create_faiss_index, dotdict,++from dsp.utils import (EM, F1, DPR_normalize, create_faiss_index, dotdict,+ has_answer, normalize_text)class Example(dotdict):@@@ -113,68 -101,14 +113,71 @@@ def passage_match(passages, answers)# answers = example.answersreturn any(passage_has_answers(psg, answers) for psg in passages)-- def answer_match(prediction, answers):+ def answer_match(prediction, answers, frac=1.0):# pred = example.prediction# answers = example.answers- return EM(prediction, answers)++ if frac >= 1.0:+ return EM(prediction, answers)++ return F1(prediction, answers) >= frac+def passage_has_answers(passage, answers):- return has_answer([DPR_normalize(normalize_text(ans)) for ans in answers], normalize_text(passage))+ return has_answer(+ tokenized_answers=[DPR_normalize(normalize_text(ans)) for ans in answers],+ text=normalize_text(passage)+ )+++def cast_naive_get_only_question_text(inp_example: Example) -> Example:+ '''+ Extracts question as a field to vectorize with Vectorizer object. `question` field is used.+ '''+ return inp_example.copy(text_to_vectorize=inp_example.question)+++def cast_naive_get_question_and_answer(inp_example: Example) -> Example:+ '''+ Extracts question and answer as fields to vectorize with Vectorizer object.+ `question` and `answer` fields are used. They will be concatenated with the word "Answer"+ between.+ '''+ text_to_vectorize = inp_example.question.strip() + " Answer: " + inp_example.answer.strip()+ return inp_example.copy(text_to_vectorize=text_to_vectorize)+++def knn(+ train: List[Example],+ cast: Callable[[Example], Example] = cast_naive_get_only_question_text,+ **knn_args+) -> Callable[[Example, int], List[Example]]:+ '''+ A function that vectorizes train data using `dsm.settings.vectorizer`, then build an ANN/KNN+ index to search similar questions among `train` samples.++ Args:+ train: a bunch of questions to put in index & search later+ cast: function that contructs text before vectorization. By default,+ it uses only question. Check `cast_naive_get_question_and_answer` for more details.+ n_probe: number of closest IVF-clusters to check for neighbours.+ Doesn't affect bruteforce-based search.+ knn_args: check `create_faiss_index` function for details on ANN/KNN arguments.+ Returns: function to search similar Examples from `train` in FAISS-index.+ '''+ train_casted_to_vectorize = [cast(cur_elem) for cur_elem in train]++ vectorizer: "BaseSentenceVectorizer" = dsp.settings.vectorizer+ all_vectors = vectorizer(train_casted_to_vectorize).astype(np.float32)++ index = create_faiss_index(emb_dim=all_vectors.shape[1], n_objects=len(train), **knn_args)+ index.train(all_vectors)+ index.add(all_vectors)++ def inner_knn_search(inp_example: Example, k: int) -> List[Example]:+ inp_example_vector = vectorizer([cast(inp_example)])+ _, nearest_samples_idxs = index.search(inp_example_vector, k)+ train_sampled = [train[cur_idx] for cur_idx in nearest_samples_idxs[0]]+ return train_sampled++ return inner_knn_searchdiff --cc setup.pyindex e191ebe37,6db14be91..a544e19d7--- a/setup.py+++ b/setup.py@@@ -17,9 -16,11 +17,12 @@@ setup"jupyter","openai","pandas",+ "sentence-transformers","spacy","regex",+ "ujson",+ "tqdm",+ "datasets"],classifiers=["Development Status :: 3 - Alpha",