DSPy: The framework for programming—not prompting—language models
Python
fixed parsing issues
manishshettym479eca0e6c6c
diff --git a/assert_gsm8k.py b/assert_gsm8k.pyindex 8dc7ba188..98528b55b 100644--- a/assert_gsm8k.py+++ b/assert_gsm8k.py@@ -6,21 +6,21 @@ from dspy.teleprompt import LabeledFewShot, BootstrapFewShotWithRandomSearchimport datasets# pipeline configs-turbo = dspy.OpenAI(model="gpt-3.5-turbo")+turbo = dspy.OpenAI(model="gpt-3.5-turbo-16k")dspy.settings.configure(lm=turbo)# load up the gsm8k dataset from hub into a dspy-compatible formatdataset = datasets.load_dataset("gsm8k", 'main')-training_data = dataset["train"].shuffle(seed=42).select(range(200))-dev_data = dataset["train"].shuffle(seed=42).select(range(200, 500))+training_data = dataset["train"].shuffle(seed=42).select(range(2))+dev_data = dataset["train"].shuffle(seed=42).select(range(2, 5))trainset = [Example(x).with_inputs("question") for x in training_data]devset = [Example(x).with_inputs("question") for x in dev_data]testset = [Example(x).with_inputs("question") for x in dataset["test"]]# validation function-def gsm8k_accuracy(example, pred):+def gsm8k_accuracy(example, pred, trace=None):ANS_RE = re.compile(r"#### (\-?[0-9\.\,]+)")INVALID_ANS = "[invalid]"@@ -37,31 +37,36 @@ def gsm8k_accuracy(example, pred):assert gt_answer != INVALID_ANSpred_answer = extract_answer(pred.answer)- print("GT:", gt_answer, "PRED:", pred.answer, "PRED-ANS:", pred_answer)return pred_answer == gt_answer++class GenerateAnswer(dspy.Signature):+ """Answer questions with a single numeric value."""+ question = dspy.InputField()+ answer = dspy.OutputField(desc="#### ${answer}")+class ThoughtReflection(dspy.Module):def __init__(self, num_attempts):- self.predict = dspy.ChainOfThought("question -> answer", n=num_attempts)- self.compare = dspy.MultiChainComparison("question -> answer", M=num_attempts)+ super().__init__()++ self.predict = dspy.ChainOfThought(GenerateAnswer, n=num_attempts)+ self.compare = dspy.MultiChainComparison(GenerateAnswer, M=num_attempts)def forward(self, question):- completions = self.predict(question=question).completions+ completions = self.predict(question=question).completionsanswer = self.compare(question=question, completions=completions)return answer-# teleprompter = LabeledFewShot(k=8)+# teleprompter = LabeledFewShot(k=8) # gives error?teleprompter = BootstrapFewShotWithRandomSearch(metric=gsm8k_accuracy)prog = ThoughtReflection(num_attempts=5)-compiled_prog = teleprompter.compile(prog, trainset=trainset, valset=devset)+compiled_prog = teleprompter.compile(prog, trainset=trainset) #, valset=devset)my_question = testset[0].questionpred = compiled_prog(my_question)print("Question:", my_question)turbo.inspect_history(n=1)--# TODO: Debug the parsing of the output!\ No newline at end of file