DSPy: The framework for programming—not prompting—language models
Python
Better dummy lm for bayesian opt
Thomas D Ahle51902e174401
diff --git a/dspy/functional/functional.py b/dspy/functional/functional.pyindex c4388ee86..bc10def22 100644--- a/dspy/functional/functional.py+++ b/dspy/functional/functional.py@@ -71,7 +71,7 @@ class TypedPredictor(dspy.Module):f". Respond with a single JSON object using the schema "+ json.dumps(type_.model_json_schema())),- format=lambda x: x if isinstance(x, str) else x.json(),+ format=lambda x: x if isinstance(x, str) else x.model_dump_json(),parser=lambda x: unwrap(type_.model_validate_json(_unwrap_json(x))),@@ -81,7 +81,7 @@ class TypedPredictor(dspy.Module):if type_ in (list[str], tuple[str]):format = passages2textelif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):- format = lambda x: x if isinstance(x, str) else x.json()+ format = lambda x: x if isinstance(x, str) else x.model_dump_json()signature = signature.with_updated_fields(name, format=format)if self.chain_of_thought:diff --git a/dspy/teleprompt/signature_opt_bayesian.py b/dspy/teleprompt/signature_opt_bayesian.pyindex 045461fce..d5245bc93 100644--- a/dspy/teleprompt/signature_opt_bayesian.py+++ b/dspy/teleprompt/signature_opt_bayesian.py@@ -325,8 +325,11 @@ class BayesianSignatureOptimizer(Teleprompter):p_demo_candidates = demo_candidates[id(p_old)]# Suggest the index of the instruction candidate to use in our trial- instruction_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_instruction",range(len(p_instruction_candidates)))- demos_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_demos",range(len(p_demo_candidates)))+ #instruction_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_instruction",range(len(p_instruction_candidates)))+ #demos_idx = trial.suggest_categorical(f"{id(p_old)}_predictor_demos",range(len(p_demo_candidates)))+ instruction_idx = trial.suggest_int(f"{id(p_old)}_predictor_instruction",low=0, high=len(p_instruction_candidates)-1)+ demos_idx = trial.suggest_int(f"{id(p_old)}_predictor_demos",low=0, high=len(p_demo_candidates)-1)+trial_logs[trial_num][f"{id(p_old)}_predictor_instruction"] = instruction_idxtrial_logs[trial_num][f"{id(p_old)}_predictor_demos"] = demos_idxdiff --git a/tests/teleprompt/test_signature_opt_bayesian.py b/tests/teleprompt/test_signature_opt_bayesian.pyindex abbcf11d1..0cf655784 100644--- a/tests/teleprompt/test_signature_opt_bayesian.py+++ b/tests/teleprompt/test_signature_opt_bayesian.py@@ -1,6 +1,8 @@import textwrapimport pytest+import reimport dspy+from dsp.modules import LMfrom dspy.teleprompt.signature_opt_bayesian import BayesianSignatureOptimizerfrom dspy.utils.dummies import DummyLMfrom dspy import Example@@ -11,6 +13,20 @@ def simple_metric(example, prediction, trace=None):# Simplified metric for testing: true if prediction matches expected outputreturn example.output == prediction.output+# Some example data+capitals = {+ "Germany": "Berlin",+ "France": "Paris",+ "Denmark": "Copenhagen",+ "Sweden": "Stockholm",+ "Norway": "Oslo",+}+# Not used for training data+extra_capitals = {+ "Spain": "Madrid",+ "Portugal": "Lisbon",+ "Italy": "Rome",+}# Example training and validation setstrainset = [@@ -18,13 +34,82 @@ trainset = [Example(input="What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!").with_inputs("input"),- Example(input="What is the capital of France?", output="Paris").with_inputs(- "input"- ),- Example(input="What is the capital of Germany?", output="Berlin").with_inputs(- "input"- ),-]+] + [Example(input=f"What is the capital of {country}?", output=capital).with_inputs("input") for country, capital in capitals.items()]+++class ConditionalLM(LM):+ def __init__(self):+ super().__init__("conditional-lm")++ def basic_request(self, prompt, n=1, **kwargs):+ # If we are in the "optimization" stage, we don't say much.+ if prompt.endswith("Observations:"):+ answer = " (*silence*)"+ elif prompt.endswith("Proposed Instruction:"):+ answer = " Input: "+ elif prompt.endswith("Proposed Prefix For Output Field:"):+ answer = " Output: "+ elif prompt.endswith("Summary:"):+ answer = " summarizing..."+ else:+ pairs = re.findall(r"Input: (.*)\nOutput: (.*)", prompt)++ print("PROMPT:", prompt)+ print("PAIRS:", pairs)++ last = re.search(r"Input: (.*)\nReasoning: (.*)$", prompt)+ current_question = last.group(1)++ if match := re.match(r"What is the capital of (.*?)\?", current_question):+ country = match.group(1)+ # If we had a previous example of a question about a capital, the model+ # has learned the format, and will answer with question correctly.+ if any("capital" in question for question, _ in pairs):+ answer = (capitals | extra_capitals)[country]+ # Otherwise, it is confused and will answer with the country's name.+ else:+ answer = country+ # For other questions, the model will answer with the last word of the question.+ else:+ answer = current_question.split()[-1]++ answer = "think deeply.\nOutput: " + answer++ RED, GREEN, RESET = '\033[91m', '\033[92m', '\033[0m'+ print("=== DummyLM ===")+ print(prompt, end="")+ print(f"{RED}{answer}{RESET}")+ print("===")++ dummy_response = {"choices": []}+ for _ in range(n):+ dummy_response["choices"].append(+ {+ "text": answer,+ "finish_reason": "done",+ }+ )++ # Simulate processing and storing the request and response.+ history_entry = {+ "prompt": prompt,+ "response": dummy_response,+ "kwargs": kwargs,+ "raw_kwargs": kwargs,+ }+ self.history.append(history_entry)++ return dummy_response++ def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):+ response = self.basic_request(prompt, **kwargs)+ return [choice["text"] for choice in response["choices"]]++ def get_convo(self, index):+ """get the prompt + anwer from the ith message"""+ return self.history[index]['prompt'] \+ + " " \+ + self.history[index]['response']['choices'][0]['text']def test_bayesian_signature_optimizer_initialization():@@ -51,8 +136,8 @@ class SimpleModule(dspy.Module):def test_signature_optimizer_optimization_process():- # Make LM that is always right- dspy.settings.configure(lm=DummyLM({ex.input: ex.output for ex in trainset}))+ lm = ConditionalLM()+ dspy.settings.configure(lm=lm)student = SimpleModule(signature="input -> output")@@ -74,7 +159,7 @@ def test_signature_optimizer_optimization_process():eval_kwargs={"num_threads": 1, "display_progress": False},)- assert len(optimized_student.predictor.demos) == 4+ assert len(optimized_student.predictor.demos) == 5def test_signature_optimizer_bad_lm():@@ -107,7 +192,7 @@ def test_signature_optimizer_bad_lm():def test_optimization_and_output_verification():# Make a language model that is always right, except on the last# example in the train set.- lm = DummyLM({ex.input: ex.output for ex in trainset[:-1]}, follow_examples=True)+ lm = ConditionalLM()dspy.settings.configure(lm=lm)optimizer = BayesianSignatureOptimizer(@@ -124,35 +209,24 @@ def test_optimization_and_output_verification():optimized_student = optimizer.compile(student=student,devset=trainset,- optuna_trials_num=10,- max_bootstrapped_demos=3,- max_labeled_demos=5,+ optuna_trials_num=4,+ max_bootstrapped_demos=2,+ max_labeled_demos=3,eval_kwargs={"num_threads": 1, "display_progress": False},)# Simulate calling the optimized student with a new input- test_input = "What is the capital of France?"+ test_input = "What is the capital of Spain?"prediction = optimized_student(input=test_input)print("CORRECT ANSWER")print(lm.get_convo(-1))- assert prediction.output == "blue"+ assert prediction.output == "Madrid"assert lm.get_convo(-1) == textwrap.dedent("""\- Given the fields `input`, produce the fields `output`.-- ----- Input: What does the fox say?- Output: Ring-ding-ding-ding-dingeringeding!-- Input: What is the capital of Germany?- Output: Berlin-- Input: What is the capital of France?- Output: Paris+ Input:---@@ -164,13 +238,24 @@ def test_optimization_and_output_verification():---- Input: What is the color of the sky?- Reasoning: Let's think step by step in order to blue- Output: blue+ Input: What is the capital of Norway?+ Reasoning: Let's think step by step in order to think deeply.+ Output: Oslo++ ---++ Input: What is the capital of Sweden?+ Reasoning: Let's think step by step in order to think deeply.+ Output: Stockholm---Input: What is the capital of France?- Reasoning: Let's think step by step in order to blue- Output: blue"""+ Output: Paris++ ---++ Input: What is the capital of Spain?+ Reasoning: Let's think step by step in order to think deeply.+ Output: Madrid""")