Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

deleted unused file

Hanna Moazam2c0305a61a63
diff --git a/build_utils/tests/intro.py b/build_utils/tests/intro.py
deleted file mode 100644
index a49f16307..000000000
--- a/build_utils/tests/intro.py
+++ /dev/null
@@ -1,223 +0,0 @@
-###
-# Copy of intro notebook for the sake of testing new versions of the package.
-###
-
-import sys
-import os
-
-# Set up the cache directory, using the environment variable set in the workflow
-cache_dir = os.getenv("DSP_NOTEBOOK_CACHEDIR")
-if not os.path.exists(cache_dir):
- os.makedirs(cache_dir)
-
-# Now import dspy or any other required modules
-import dspy
-
-turbo = dspy.OpenAI(model='gpt-3.5-turbo')
-colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')
-
-dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts)
-
-
-from dspy.datasets import HotPotQA
-
-# Load the dataset.
-dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)
-
-# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.
-trainset = [x.with_inputs('question') for x in dataset.train]
-devset = [x.with_inputs('question') for x in dataset.dev]
-
-
-train_example = trainset[0]
-print(f"Question: {train_example.question}")
-print(f"Answer: {train_example.answer}")
-
-dev_example = devset[18]
-print(f"Question: {dev_example.question}")
-print(f"Answer: {dev_example.answer}")
-print(f"Relevant Wikipedia Titles: {dev_example.gold_titles}")
-
-print(f"For this dataset, training examples have input keys {train_example.inputs().keys()} and label keys {train_example.labels().keys()}")
-print(f"For this dataset, dev examples have input keys {dev_example.inputs().keys()} and label keys {dev_example.labels().keys()}")
-
-
-class BasicQA(dspy.Signature):
- """Answer questions with short factoid answers."""
-
- question = dspy.InputField()
- answer = dspy.OutputField(desc="often between 1 and 5 words")
-
-# Define the predictor.
-generate_answer = dspy.Predict(BasicQA)
-
-# Call the predictor on a particular input.
-pred = generate_answer(question=dev_example.question)
-
-# Print the input and the prediction.
-print(f"Question: {dev_example.question}")
-print(f"Predicted Answer: {pred.answer}")
-
-# Define the predictor. Notice we're just changing the class. The signature BasicQA is unchanged.
-generate_answer_with_chain_of_thought = dspy.ChainOfThought(BasicQA)
-
-# Call the predictor on the same input.
-pred = generate_answer_with_chain_of_thought(question=dev_example.question)
-
-# Print the input, the chain of thought, and the prediction.
-print(f"Question: {dev_example.question}")
-print(f"Thought: {pred.rationale.split('.', 1)[1].strip()}")
-print(f"Predicted Answer: {pred.answer}")
-
-retrieve = dspy.Retrieve(k=3)
-topK_passages = retrieve(dev_example.question).passages
-
-print(f"Top {retrieve.k} passages for question: {dev_example.question} \n", '-' * 30, '\n')
-
-for idx, passage in enumerate(topK_passages):
- print(f'{idx+1}]', passage, '\n')
-
-
-retrieve("When was the first FIFA World Cup held?").passages[0]
-
-class GenerateAnswer(dspy.Signature):
- """Answer questions with short factoid answers."""
-
- context = dspy.InputField(desc="may contain relevant facts")
- question = dspy.InputField()
- answer = dspy.OutputField(desc="often between 1 and 5 words")
-
-
-class RAG(dspy.Module):
- def __init__(self, num_passages=3):
- super().__init__()
-
- self.retrieve = dspy.Retrieve(k=num_passages)
- self.generate_answer = dspy.ChainOfThought(GenerateAnswer)
-
- def forward(self, question):
- context = self.retrieve(question).passages
- prediction = self.generate_answer(context=context, question=question)
- return dspy.Prediction(context=context, answer=prediction.answer)
-
-from dspy.teleprompt import BootstrapFewShot
-
-# Validation logic: check that the predicted answer is correct.
-# Also check that the retrieved context does actually contain that answer.
-def validate_context_and_answer(example, pred, trace=None):
- answer_EM = dspy.evaluate.answer_exact_match(example, pred)
- answer_PM = dspy.evaluate.answer_passage_match(example, pred)
- return answer_EM and answer_PM
-
-# Set up a basic teleprompter, which will compile our RAG program.
-teleprompter = BootstrapFewShot(metric=validate_context_and_answer)
-
-# Compile!
-compiled_rag = teleprompter.compile(RAG(), trainset=trainset)
-
-# Ask any question you like to this simple RAG program.
-my_question = "What castle did David Gregory inherit?"
-
-# Get the prediction. This contains `pred.context` and `pred.answer`.
-pred = compiled_rag(my_question)
-
-# Print the contexts and the answer.
-print(f"Question: {my_question}")
-print(f"Predicted Answer: {pred.answer}")
-print(f"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}")
-
-
-for name, parameter in compiled_rag.named_predictors():
- print(name)
- print(parameter.demos[0])
- print()
-
-from dspy.evaluate.evaluate import Evaluate
-
-# Set up the `evaluate_on_hotpotqa` function. We'll use this many times below.
-evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=1, display_progress=True, display_table=5)
-
-# Evaluate the `compiled_rag` program with the `answer_exact_match` metric.
-metric = dspy.evaluate.answer_exact_match
-evaluate_on_hotpotqa(compiled_rag, metric=metric)
-
-
-def gold_passages_retrieved(example, pred, trace=None):
- gold_titles = set(map(dspy.evaluate.normalize_text, example['gold_titles']))
- found_titles = set(map(dspy.evaluate.normalize_text, [c.split(' | ')[0] for c in pred.context]))
-
- return gold_titles.issubset(found_titles)
-
-compiled_rag_retrieval_score = evaluate_on_hotpotqa(compiled_rag, metric=gold_passages_retrieved)
-
-
-class GenerateSearchQuery(dspy.Signature):
- """Write a simple search query that will help answer a complex question."""
-
- context = dspy.InputField(desc="may contain relevant facts")
- question = dspy.InputField()
- query = dspy.OutputField()
-
-
-from dsp.utils import deduplicate
-
-class SimplifiedBaleen(dspy.Module):
- def __init__(self, passages_per_hop=3, max_hops=2):
- super().__init__()
-
- self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]
- self.retrieve = dspy.Retrieve(k=passages_per_hop)
- self.generate_answer = dspy.ChainOfThought(GenerateAnswer)
- self.max_hops = max_hops
-
- def forward(self, question):
- context = []
-
- for hop in range(self.max_hops):
- query = self.generate_query[hop](context=context, question=question).query
- passages = self.retrieve(query).passages
- context = deduplicate(context + passages)
-
- pred = self.generate_answer(context=context, question=question)
- return dspy.Prediction(context=context, answer=pred.answer)
-
-
-# Ask any question you like to this simple RAG program.
-my_question = "How many storeys are in the castle that David Gregory inherited?"
-
-# Get the prediction. This contains `pred.context` and `pred.answer`.
-uncompiled_baleen = SimplifiedBaleen() # uncompiled (i.e., zero-shot) program
-pred = uncompiled_baleen(my_question)
-
-# Print the contexts and the answer.
-print(f"Question: {my_question}")
-print(f"Predicted Answer: {pred.answer}")
-print(f"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}")
-
-
-def validate_context_and_answer_and_hops(example, pred, trace=None):
- if not dspy.evaluate.answer_exact_match(example, pred): return False
- if not dspy.evaluate.answer_passage_match(example, pred): return False
-
- hops = [example.question] + [outputs.query for *_, outputs in trace if 'query' in outputs]
-
- if max([len(h) for h in hops]) > 100: return False
- if any(dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))): return False
-
- return True
-
-teleprompter = BootstrapFewShot(metric=validate_context_and_answer_and_hops)
-compiled_baleen = teleprompter.compile(SimplifiedBaleen(), teacher=SimplifiedBaleen(passages_per_hop=2), trainset=trainset)
-
-uncompiled_baleen_retrieval_score = evaluate_on_hotpotqa(uncompiled_baleen, metric=gold_passages_retrieved)
-
-compiled_baleen_retrieval_score = evaluate_on_hotpotqa(compiled_baleen, metric=gold_passages_retrieved)
-
-print(f"## Retrieval Score for RAG: {compiled_rag_retrieval_score}") # note that for RAG, compilation has no effect on the retrieval step
-print(f"## Retrieval Score for uncompiled Baleen: {uncompiled_baleen_retrieval_score}")
-print(f"## Retrieval Score for compiled Baleen: {compiled_baleen_retrieval_score}")
-
-print(compiled_baleen("How many storeys are in the castle that David Gregory inherited?"))
-
-
-