DSPy: The framework for programming—not prompting—language models
Python
updating tests for new optimizers
klopsahlonga5f75af4339e
diff --git a/tests/teleprompt/test_signature_opt.py b/tests/teleprompt/test_signature_opt.pyindex d7f347551..c0fe712bc 100644--- a/tests/teleprompt/test_signature_opt.py+++ b/tests/teleprompt/test_signature_opt.py@@ -1,6 +1,6 @@import textwrapimport dspy-from dspy.teleprompt.signature_opt import SignatureOptimizer+from dspy.teleprompt.signature_opt import COPROfrom dspy.utils.dummies import DummyLMfrom dspy import Example@@ -16,7 +16,7 @@ trainset = []def test_signature_optimizer_initialization():- optimizer = SignatureOptimizer(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)+ optimizer = COPRO(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)assert optimizer.metric == simple_metric, "Metric not correctly initialized"assert optimizer.breadth == 2, "Breadth not correctly initialized"assert optimizer.depth == 1, "Depth not correctly initialized"@@ -25,20 +25,20 @@ def test_signature_optimizer_initialization():class SimpleModule(dspy.Module):def __init__(self, signature):super().__init__()- # SignatureOptimizer doesn't work with dspy.Predict+ # COPRO doesn't work with dspy.Predictself.predictor = dspy.ChainOfThought(signature)def forward(self, **kwargs):return self.predictor(**kwargs)def test_signature_optimizer_optimization_process():- optimizer = SignatureOptimizer(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)+ optimizer = COPRO(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)dspy.settings.configure(lm=DummyLM(["Optimized instruction 1", "Optimized instruction 2"]))student = SimpleModule("input -> output")- # Assuming the compile method of SignatureOptimizer requires a student module, a development set, and evaluation kwargs- optimized_student = optimizer.compile(student, devset=trainset, eval_kwargs={"num_threads": 1, "display_progress": False})+ # Assuming the compile method of COPRO requires a student module, a development set, and evaluation kwargs+ optimized_student = optimizer.compile(student, trainset=trainset, eval_kwargs={"num_threads": 1, "display_progress": False})# Check that the optimized student has been modified from the original# This check can be more specific based on how the optimization modifies the student@@ -48,12 +48,12 @@ def test_signature_optimizer_optimization_process():# such as checking the instructions of the optimized student's predictors.def test_signature_optimizer_statistics_tracking():- optimizer = SignatureOptimizer(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)+ optimizer = COPRO(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)optimizer.track_stats = True # Enable statistics trackingdspy.settings.configure(lm=DummyLM(["Optimized instruction"]))student = SimpleModule("input -> output")- optimized_student = optimizer.compile(student, devset=trainset, eval_kwargs={"num_threads": 1, "display_progress": False})+ optimized_student = optimizer.compile(student, trainset=trainset, eval_kwargs={"num_threads": 1, "display_progress": False})# Verify that statistics have been tracked and attached to the optimized studentassert hasattr(optimized_student, 'total_calls'), "Total calls statistic not tracked"@@ -67,12 +67,12 @@ def test_optimization_and_output_verification():"Optimized Prefix",])dspy.settings.configure(lm=lm)- optimizer = SignatureOptimizer(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)+ optimizer = COPRO(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)student = SimpleModule("input -> output")# Compile the student with the optimizer- optimized_student = optimizer.compile(student, devset=trainset, eval_kwargs={"num_threads": 1, "display_progress": False})+ optimized_student = optimizer.compile(student, trainset=trainset, eval_kwargs={"num_threads": 1, "display_progress": False})# Simulate calling the optimized student with a new inputtest_input = "What is the capital of France?"@@ -102,11 +102,11 @@ def test_optimization_and_output_verification():def test_statistics_tracking_during_optimization():dspy.settings.configure(lm=DummyLM(["Optimized instruction for stats tracking"]))- optimizer = SignatureOptimizer(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)+ optimizer = COPRO(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)optimizer.track_stats = True # Enable statistics trackingstudent = SimpleModule("input -> output")- optimized_student = optimizer.compile(student, devset=trainset, eval_kwargs={"num_threads": 1, "display_progress": False})+ optimized_student = optimizer.compile(student, trainset=trainset, eval_kwargs={"num_threads": 1, "display_progress": False})# Verify that statistics have been trackedassert hasattr(optimized_student, 'total_calls'), "Optimizer did not track total metric calls"diff --git a/tests/teleprompt/test_signature_opt_bayesian.py b/tests/teleprompt/test_signature_opt_bayesian.pyindex 63baf7a5b..c1cb61de0 100644--- a/tests/teleprompt/test_signature_opt_bayesian.py+++ b/tests/teleprompt/test_signature_opt_bayesian.py@@ -53,10 +53,18 @@ class ConditionalLM(LM):answer = " summarizing..."else:pairs = re.findall(r"Input: (.*)\nOutput: (.*)", prompt)+ # pairs = re.findall(r"(?<=\n|^)---\n\nInput: (.*?)\nOutput: (.*?)(?=\n\n---|\n*$)", prompt, re.DOTALL)+ # pairs = re.findall(r"Input: (.*?)\n(?:Reasoning: .*?\n)?Output: (.*)", prompt, re.DOTALL)+ pairs = re.findall(r"Input: (.*?)\n(?:Reasoning:.*?\n)?Output: (.*?)\n", prompt, re.DOTALL)++ # breakpoint()print("PROMPT:", prompt)print("PAIRS:", pairs)+ # if "What is the capital of Spain?" in prompt:+ # breakpoint()+last = re.search(r"Input: (.*)\nReasoning: (.*)$", prompt)current_question = last.group(1)@@ -227,34 +235,34 @@ def test_optimization_and_output_verification():assert prediction.output == "Madrid"- assert lm.get_convo(-1) == textwrap.dedent(+ expected_lm_output = textwrap.dedent("""\Input:----+Follow the following format.-+Input: ${input}Reasoning: Let's think step by step in order to ${produce the output}. We ...Output: ${output}---- Input: What is the capital of Norway?+ Input: What is the capital of France?Reasoning: Let's think step by step in order to think deeply.- Output: Oslo+ Output: Paris---- Input: What is the capital of Sweden?+ Input: What is the capital of Norway?Reasoning: Let's think step by step in order to think deeply.- Output: Stockholm+ Output: Oslo---- Input: What is the capital of France?- Output: Paris+ Input: What does the fox say?+ Output: Ring-ding-ding-ding-dingeringeding!---@@ -262,3 +270,5 @@ def test_optimization_and_output_verification():Reasoning: Let's think step by step in order to think deeply.Output: Madrid""")++ assert lm.get_convo(-1) == expected_lm_output\ No newline at end of file