Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

fixed parsing issues

manishshettym479eca0e6c6c
diff --git a/assert_gsm8k.py b/assert_gsm8k.py
index 8dc7ba188..98528b55b 100644
--- a/assert_gsm8k.py
+++ b/assert_gsm8k.py
@@ -6,21 +6,21 @@ from dspy.teleprompt import LabeledFewShot, BootstrapFewShotWithRandomSearch
import datasets
# pipeline configs
-turbo = dspy.OpenAI(model="gpt-3.5-turbo")
+turbo = dspy.OpenAI(model="gpt-3.5-turbo-16k")
dspy.settings.configure(lm=turbo)
# load up the gsm8k dataset from hub into a dspy-compatible format
dataset = datasets.load_dataset("gsm8k", 'main')
-training_data = dataset["train"].shuffle(seed=42).select(range(200))
-dev_data = dataset["train"].shuffle(seed=42).select(range(200, 500))
+training_data = dataset["train"].shuffle(seed=42).select(range(2))
+dev_data = dataset["train"].shuffle(seed=42).select(range(2, 5))
trainset = [Example(x).with_inputs("question") for x in training_data]
devset = [Example(x).with_inputs("question") for x in dev_data]
testset = [Example(x).with_inputs("question") for x in dataset["test"]]
# validation function
-def gsm8k_accuracy(example, pred):
+def gsm8k_accuracy(example, pred, trace=None):
ANS_RE = re.compile(r"#### (\-?[0-9\.\,]+)")
INVALID_ANS = "[invalid]"
@@ -37,31 +37,36 @@ def gsm8k_accuracy(example, pred):
assert gt_answer != INVALID_ANS
pred_answer = extract_answer(pred.answer)
- print("GT:", gt_answer, "PRED:", pred.answer, "PRED-ANS:", pred_answer)
return pred_answer == gt_answer
+
+class GenerateAnswer(dspy.Signature):
+ """Answer questions with a single numeric value."""
+ question = dspy.InputField()
+ answer = dspy.OutputField(desc="#### ${answer}")
+
class ThoughtReflection(dspy.Module):
def __init__(self, num_attempts):
- self.predict = dspy.ChainOfThought("question -> answer", n=num_attempts)
- self.compare = dspy.MultiChainComparison("question -> answer", M=num_attempts)
+ super().__init__()
+
+ self.predict = dspy.ChainOfThought(GenerateAnswer, n=num_attempts)
+ self.compare = dspy.MultiChainComparison(GenerateAnswer, M=num_attempts)
def forward(self, question):
- completions = self.predict(question=question).completions
+ completions = self.predict(question=question).completions
answer = self.compare(question=question, completions=completions)
return answer
-# teleprompter = LabeledFewShot(k=8)
+# teleprompter = LabeledFewShot(k=8) # gives error?
teleprompter = BootstrapFewShotWithRandomSearch(metric=gsm8k_accuracy)
prog = ThoughtReflection(num_attempts=5)
-compiled_prog = teleprompter.compile(prog, trainset=trainset, valset=devset)
+compiled_prog = teleprompter.compile(prog, trainset=trainset) #, valset=devset)
my_question = testset[0].question
pred = compiled_prog(my_question)
print("Question:", my_question)
turbo.inspect_history(n=1)
-
-# TODO: Debug the parsing of the output!
\ No newline at end of file