DSPy: The framework for programming—not prompting—language models
Python
deleted unused file
Hanna Moazam2c0305a61a63
diff --git a/build_utils/tests/intro.py b/build_utils/tests/intro.pydeleted file mode 100644index a49f16307..000000000--- a/build_utils/tests/intro.py+++ /dev/null@@ -1,223 +0,0 @@-###-# Copy of intro notebook for the sake of testing new versions of the package.-###--import sys-import os--# Set up the cache directory, using the environment variable set in the workflow-cache_dir = os.getenv("DSP_NOTEBOOK_CACHEDIR")-if not os.path.exists(cache_dir):- os.makedirs(cache_dir)--# Now import dspy or any other required modules-import dspy--turbo = dspy.OpenAI(model='gpt-3.5-turbo')-colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')--dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts)---from dspy.datasets import HotPotQA--# Load the dataset.-dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)--# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.-trainset = [x.with_inputs('question') for x in dataset.train]-devset = [x.with_inputs('question') for x in dataset.dev]---train_example = trainset[0]-print(f"Question: {train_example.question}")-print(f"Answer: {train_example.answer}")--dev_example = devset[18]-print(f"Question: {dev_example.question}")-print(f"Answer: {dev_example.answer}")-print(f"Relevant Wikipedia Titles: {dev_example.gold_titles}")--print(f"For this dataset, training examples have input keys {train_example.inputs().keys()} and label keys {train_example.labels().keys()}")-print(f"For this dataset, dev examples have input keys {dev_example.inputs().keys()} and label keys {dev_example.labels().keys()}")---class BasicQA(dspy.Signature):- """Answer questions with short factoid answers."""-- question = dspy.InputField()- answer = dspy.OutputField(desc="often between 1 and 5 words")--# Define the predictor.-generate_answer = dspy.Predict(BasicQA)--# Call the predictor on a particular input.-pred = generate_answer(question=dev_example.question)--# Print the input and the prediction.-print(f"Question: {dev_example.question}")-print(f"Predicted Answer: {pred.answer}")--# Define the predictor. Notice we're just changing the class. The signature BasicQA is unchanged.-generate_answer_with_chain_of_thought = dspy.ChainOfThought(BasicQA)--# Call the predictor on the same input.-pred = generate_answer_with_chain_of_thought(question=dev_example.question)--# Print the input, the chain of thought, and the prediction.-print(f"Question: {dev_example.question}")-print(f"Thought: {pred.rationale.split('.', 1)[1].strip()}")-print(f"Predicted Answer: {pred.answer}")--retrieve = dspy.Retrieve(k=3)-topK_passages = retrieve(dev_example.question).passages--print(f"Top {retrieve.k} passages for question: {dev_example.question} \n", '-' * 30, '\n')--for idx, passage in enumerate(topK_passages):- print(f'{idx+1}]', passage, '\n')---retrieve("When was the first FIFA World Cup held?").passages[0]--class GenerateAnswer(dspy.Signature):- """Answer questions with short factoid answers."""-- context = dspy.InputField(desc="may contain relevant facts")- question = dspy.InputField()- answer = dspy.OutputField(desc="often between 1 and 5 words")---class RAG(dspy.Module):- def __init__(self, num_passages=3):- super().__init__()-- self.retrieve = dspy.Retrieve(k=num_passages)- self.generate_answer = dspy.ChainOfThought(GenerateAnswer)-- def forward(self, question):- context = self.retrieve(question).passages- prediction = self.generate_answer(context=context, question=question)- return dspy.Prediction(context=context, answer=prediction.answer)--from dspy.teleprompt import BootstrapFewShot--# Validation logic: check that the predicted answer is correct.-# Also check that the retrieved context does actually contain that answer.-def validate_context_and_answer(example, pred, trace=None):- answer_EM = dspy.evaluate.answer_exact_match(example, pred)- answer_PM = dspy.evaluate.answer_passage_match(example, pred)- return answer_EM and answer_PM--# Set up a basic teleprompter, which will compile our RAG program.-teleprompter = BootstrapFewShot(metric=validate_context_and_answer)--# Compile!-compiled_rag = teleprompter.compile(RAG(), trainset=trainset)--# Ask any question you like to this simple RAG program.-my_question = "What castle did David Gregory inherit?"--# Get the prediction. This contains `pred.context` and `pred.answer`.-pred = compiled_rag(my_question)--# Print the contexts and the answer.-print(f"Question: {my_question}")-print(f"Predicted Answer: {pred.answer}")-print(f"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}")---for name, parameter in compiled_rag.named_predictors():- print(name)- print(parameter.demos[0])- print()--from dspy.evaluate.evaluate import Evaluate--# Set up the `evaluate_on_hotpotqa` function. We'll use this many times below.-evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=1, display_progress=True, display_table=5)--# Evaluate the `compiled_rag` program with the `answer_exact_match` metric.-metric = dspy.evaluate.answer_exact_match-evaluate_on_hotpotqa(compiled_rag, metric=metric)---def gold_passages_retrieved(example, pred, trace=None):- gold_titles = set(map(dspy.evaluate.normalize_text, example['gold_titles']))- found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.context]))-- return gold_titles.issubset(found_titles)--compiled_rag_retrieval_score = evaluate_on_hotpotqa(compiled_rag, metric=gold_passages_retrieved)---class GenerateSearchQuery(dspy.Signature):- """Write a simple search query that will help answer a complex question."""-- context = dspy.InputField(desc="may contain relevant facts")- question = dspy.InputField()- query = dspy.OutputField()---from dsp.utils import deduplicate--class SimplifiedBaleen(dspy.Module):- def __init__(self, passages_per_hop=3, max_hops=2):- super().__init__()-- self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]- self.retrieve = dspy.Retrieve(k=passages_per_hop)- self.generate_answer = dspy.ChainOfThought(GenerateAnswer)- self.max_hops = max_hops-- def forward(self, question):- context = []-- for hop in range(self.max_hops):- query = self.generate_query[hop](context=context, question=question).query- passages = self.retrieve(query).passages- context = deduplicate(context + passages)-- pred = self.generate_answer(context=context, question=question)- return dspy.Prediction(context=context, answer=pred.answer)---# Ask any question you like to this simple RAG program.-my_question = "How many storeys are in the castle that David Gregory inherited?"--# Get the prediction. This contains `pred.context` and `pred.answer`.-uncompiled_baleen = SimplifiedBaleen() # uncompiled (i.e., zero-shot) program-pred = uncompiled_baleen(my_question)--# Print the contexts and the answer.-print(f"Question: {my_question}")-print(f"Predicted Answer: {pred.answer}")-print(f"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}")---def validate_context_and_answer_and_hops(example, pred, trace=None):- if not dspy.evaluate.answer_exact_match(example, pred): return False- if not dspy.evaluate.answer_passage_match(example, pred): return False-- hops = [example.question] + [outputs.query for *_, outputs in trace if 'query' in outputs]-- if max([len(h) for h in hops]) > 100: return False- if any(dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))): return False-- return True--teleprompter = BootstrapFewShot(metric=validate_context_and_answer_and_hops)-compiled_baleen = teleprompter.compile(SimplifiedBaleen(), teacher=SimplifiedBaleen(passages_per_hop=2), trainset=trainset)--uncompiled_baleen_retrieval_score = evaluate_on_hotpotqa(uncompiled_baleen, metric=gold_passages_retrieved)--compiled_baleen_retrieval_score = evaluate_on_hotpotqa(compiled_baleen, metric=gold_passages_retrieved)--print(f"## Retrieval Score for RAG: {compiled_rag_retrieval_score}") # note that for RAG, compilation has no effect on the retrieval step-print(f"## Retrieval Score for uncompiled Baleen: {uncompiled_baleen_retrieval_score}")-print(f"## Retrieval Score for compiled Baleen: {compiled_baleen_retrieval_score}")--print(compiled_baleen("How many storeys are in the castle that David Gregory inherited?"))---