DSPy: The framework for programming—not prompting—language models
Python
Docstrings & small fixes
Igor Kotenkovabaf8d7e5b7f
diff --git a/dsp/modules/sentence_vectorizer.py b/dsp/modules/sentence_vectorizer.pyindex 691340843..f45d3b696 100644--- a/dsp/modules/sentence_vectorizer.py+++ b/dsp/modules/sentence_vectorizer.py@@ -8,6 +8,12 @@ from dsp.utils import determine_devicesclass BaseSentenceVectorizer(abc.ABC):+ '''+ Base Class for Vectorizers. The main purpose is to vectorize text (doc/query)+ for ANN/KNN indexes. `__call__` method takes `List[Example]` as a single input, then extracts+ `field_to_vectorize` from every Example and convert them into embeddings.+ You can customize extraction logic in the `_extract_text_from_examples` method.+ '''# embeddings will be computed based on the string in this attribute of Example objectfield_to_vectorize = 'text_to_vectorize'@@ -27,6 +33,12 @@ class BaseSentenceVectorizer(abc.ABC):class SentenceTransformersVectorizer(BaseSentenceVectorizer):+ '''+ Vectorizer bsaed on `SentenceTransformers` models. You can pick any model from this link:+ https://huggingface.co/sentence-transformers+ More details about models:+ https://www.sbert.net/docs/pretrained_models.html+ '''def __init__(self,model_name_or_path: str = 'all-MiniLM-L6-v2',@@ -75,6 +87,10 @@ class SentenceTransformersVectorizer(BaseSentenceVectorizer):class NaiveGetFieldVectorizer(BaseSentenceVectorizer):+ '''+ If embeddings were precomputed, then we could just extract them from the proper field+ (set by `field_with_embedding`) from each `Example`.+ '''def __init__(self, field_with_embedding: str = 'vectorized'):self.field_with_embedding = field_with_embedding@@ -88,8 +104,14 @@ class NaiveGetFieldVectorizer(BaseSentenceVectorizer):class OpenAIVectorizer(BaseSentenceVectorizer):+ '''+ This vectorizer uses OpenAI API to convert texts to embeddings. Changing `model` is not+ recommended. More about the model: https://openai.com/blog/new-and-improved-embedding-model/+ `api_key` should be passed as an argument or as env variable (`OPENAI_API_KEY`).+ '''def __init__(- self, model: str = 'text-embedding-ada-002',+ self,+ model: str = 'text-embedding-ada-002',embed_batch_size: int = 1024,api_key: Optional[str] = None):@@ -109,7 +131,7 @@ class OpenAIVectorizer(BaseSentenceVectorizer):start_idx = cur_batch_idx * self.embed_batch_sizeend_idx = (cur_batch_idx + 1) * self.embed_batch_sizecur_batch = text_to_vectorize[start_idx: end_idx]- # cached API call:+ # OpenAI API call:response = openai.Embedding.create(model=self.model,input=cur_batchdiff --git a/dsp/primitives/demonstrate.py b/dsp/primitives/demonstrate.pyindex 7d42003eb..10a725a47 100644--- a/dsp/primitives/demonstrate.py+++ b/dsp/primitives/demonstrate.py@@ -75,8 +75,11 @@ def annotate(*transformations):def sample(train, k: int):- branch_idx = hasattr(dsp.settings, 'branch_idx') and isinstance(- dsp.settings.branch_idx, int) and dsp.settings.branch_idx+ branch_idx = (+ hasattr(dsp.settings, 'branch_idx')+ and isinstance(dsp.settings.branch_idx, int)+ and dsp.settings.branch_idx+ )branch_idx = branch_idx or 0# print(f"branch_idx = {branch_idx}")@@ -93,8 +96,13 @@ def sample(train, k: int):def all_but(train, x):# output = [y for y in train if y.question != x.question]- output = [y for y in train- if not set.intersection(set(x.get('history', []) + [x.question]), set(y.get('history', []) + [y.question]))]+ output = [+ y for y in train+ if not set.intersection(+ set(x.get('history', []) + [x.question]),+ set(y.get('history', []) + [y.question])+ )+ ]# print(len(output))return output@@ -113,14 +121,25 @@ def answer_match(prediction, answers):def passage_has_answers(passage, answers):- return has_answer([DPR_normalize(normalize_text(ans)) for ans in answers], normalize_text(passage))+ return has_answer(+ tokenized_answers=[DPR_normalize(normalize_text(ans)) for ans in answers],+ text=normalize_text(passage)+ )def cast_naive_get_only_question_text(inp_example: Example) -> Example:+ '''+ Extracts question as a field to vectorize with Vectorizer object. `question` field is used.+ '''return inp_example.copy(text_to_vectorize=inp_example.question)def cast_naive_get_question_and_answer(inp_example: Example) -> Example:+ '''+ Extracts question and answer as fields to vectorize with Vectorizer object.+ `question` and `answer` fields are used. They will be concatenated with the word "Answer"+ between.+ '''text_to_vectorize = inp_example.question.strip() + " Answer: " + inp_example.answer.strip()return inp_example.copy(text_to_vectorize=text_to_vectorize)@@ -130,6 +149,19 @@ def knn(cast: Callable[[Example], Example] = cast_naive_get_only_question_text,**knn_args) -> Callable[[Example, int], List[Example]]:+ '''+ A function that vectorizes train data using `dsm.settings.vectorizer`, then build an ANN/KNN+ index to search similar questions among `train` samples.++ Args:+ train: a bunch of questions to put in index & search later+ cast: function that contructs text before vectorization. By default,+ it uses only question. Check `cast_naive_get_question_and_answer` for more details.+ n_probe: number of closest IVF-clusters to check for neighbours.+ Doesn't affect bruteforce-based search.+ knn_args: check `create_faiss_index` function for details on ANN/KNN arguments.+ Returns: function to search similar Examples from `train` in FAISS-index.+ '''train_casted_to_vectorize = [cast(cur_elem) for cur_elem in train]vectorizer: "BaseSentenceVectorizer" = dsp.settings.vectorizer