DSPy: The framework for programming—not prompting—language models
Python
adding in hackercup tutorial + fixing verbosity of mipro
klopsahlong0899f1c6f50b
diff --git a/dsp/adapters/experimental_adapter.py b/dsp/adapters/experimental_adapter.pyindex 6e2eb3a33..ecc46aff6 100644--- a/dsp/adapters/experimental_adapter.py+++ b/dsp/adapters/experimental_adapter.py@@ -75,18 +75,6 @@ class ExperimentalAdapter(BaseTemplate):"""example = dsp.Example(example)- raw_pred = raw_pred.strip()- parts = raw_pred.split('\n')- adjusted_parts = []- for part in parts:- trimmed_part = part.strip()- if trimmed_part:- if adjusted_parts:- adjusted_parts.append('\n' + trimmed_part)- else:- adjusted_parts.append(trimmed_part)- raw_pred = '\n'.join(adjusted_parts)-idx = 0while idx < len(self.fields):if self.fields[idx].input_variable not in example or example[self.fields[idx].input_variable] is None:diff --git a/dspy/propose/dataset_summary_generator.py b/dspy/propose/dataset_summary_generator.pyindex 83a39b9b2..6c6d5fcaf 100644--- a/dspy/propose/dataset_summary_generator.py+++ b/dspy/propose/dataset_summary_generator.py@@ -45,7 +45,7 @@ def order_input_keys_in_string(unordered_repr):return ordered_repr-def create_dataset_summary(trainset, view_data_batch_size, prompt_model, log_file=None):+def create_dataset_summary(trainset, view_data_batch_size, prompt_model, log_file=None, verbose=False):upper_lim = min(len(trainset), view_data_batch_size)with dspy.settings.context(lm=prompt_model):observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=order_input_keys_in_string(trainset[0:upper_lim].__repr__()))@@ -62,7 +62,7 @@ def create_dataset_summary(trainset, view_data_batch_size, prompt_model, log_filcalls+=1if calls >= max_calls:break- print(f"b: {b}")+ if verbose: print(f"b: {b}")upper_lim = min(len(trainset), b+view_data_batch_size)with dspy.settings.context(lm=prompt_model):output = dspy.Predict(DatasetDescriptorWithPriorObservations, n=1, temperature=1.0)(prior_observations=observations, examples=order_input_keys_in_string(trainset[b:upper_lim].__repr__()))@@ -76,11 +76,11 @@ def create_dataset_summary(trainset, view_data_batch_size, prompt_model, log_filif log_file:log_file.write(f"observations {observations}\n")except Exception as e:- print(f"e {e}. using observations from past round for a summary.")+ if verbose: print(f"e {e}. using observations from past round for a summary.")with dspy.settings.context(lm=prompt_model):summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)- print(f"summary: {summary}")+ if verbose: print(f"summary: {summary}")if log_file:log_file.write(f"summary: {summary}\n")diff --git a/dspy/propose/grounded_proposer.py b/dspy/propose/grounded_proposer.pyindex 130c20ad8..1745d9254 100644--- a/dspy/propose/grounded_proposer.py+++ b/dspy/propose/grounded_proposer.py@@ -136,6 +136,7 @@ class GenerateModuleInstruction(dspy.Module):use_task_demos=True,use_instruct_history=True,use_tip=True,+ verbose=False,):super().__init__()self.use_dataset_summary = use_dataset_summary@@ -143,6 +144,7 @@ class GenerateModuleInstruction(dspy.Module):self.use_task_demos = use_task_demosself.use_instruct_history = use_instruct_historyself.use_tip = use_tip+ self.verbose = verboseself.program_code_string = program_code_stringself.describe_program = dspy.Predict(DescribeProgram)@@ -189,7 +191,7 @@ class GenerateModuleInstruction(dspy.Module):program_code=self.program_code_string, program_example=task_demos,).program_description,)- print(f"PROGRAM DESCRIPTION: {program_description}")+ if self.verbose: print(f"PROGRAM DESCRIPTION: {program_description}")# Identify all modulesinit_pattern = r"def __init__\(.*?\):([\s\S]*?)(?=^\s*def|\Z)"@@ -209,7 +211,7 @@ class GenerateModuleInstruction(dspy.Module):).module_description# Generate an instruction for our chosen module- print(f"task_demos {task_demos}")+ if self.verbose: print(f"task_demos {task_demos}")instruct = self.generate_module_instruction(dataset_description=data_summary,program_code=self.program_code_string,@@ -223,9 +225,8 @@ class GenerateModuleInstruction(dspy.Module):)if hasattr(instruct, "module_description"):module_description = strip_prefix(instruct.module_description)- print(f"MODULE DESCRIPTION: {module_description}")+ if self.verbose: print(f"MODULE DESCRIPTION: {module_description}")proposed_instruction = strip_prefix(instruct.proposed_instruction)- # print(f"PROPOSED INSTRUCTION: {proposed_instruction}")return dspy.Prediction(proposed_instruction=proposed_instruction)@@ -245,6 +246,7 @@ class GroundedProposer(Proposer):use_tip=True,set_tip_randomly=True,set_history_randomly=True,+ verbose=False,):super().__init__()self.program_aware = program_aware@@ -254,13 +256,14 @@ class GroundedProposer(Proposer):self.use_tip = use_tipself.set_tip_randomly=set_tip_randomlyself.set_history_randomly=set_history_randomly+ self.verbose = verboseself.prompt_model = prompt_modelself.program_code_string = program_code_stringself.data_summary = create_dataset_summary(trainset=trainset, view_data_batch_size=view_data_batch_size, prompt_model=prompt_model,)- print(f"DATA SUMMARY: {self.data_summary}")+ if self.verbose: print(f"DATA SUMMARY: {self.data_summary}")def propose_instructions_for_program(self,@@ -278,21 +281,21 @@ class GroundedProposer(Proposer):proposed_instructions = {}if self.set_tip_randomly:- print("Using a randomly generated configuration for our grounded proposer.")+ if self.verbose: print("Using a randomly generated configuration for our grounded proposer.")# Randomly select the tipselected_tip_key = random.choice(list(TIPS.keys()))selected_tip = TIPS[selected_tip_key]self.use_tip = bool(selected_tip,)- print(f"Selected tip: {selected_tip_key}")+ if self.verbose: print(f"Selected tip: {selected_tip_key}")if self.set_history_randomly:# Randomly select whether or not we're using instruction historyuse_history = random.random() < 0.5self.use_instruct_history = use_history- print(f"Use history T/F: {self.use_instruct_history}")-+ if self.verbose: print(f"Use history T/F: {self.use_instruct_history}")+# Create an instruction for each predictorfor pred_i, predictor in enumerate(program.predictors()):for demo_set_i in range(len(demo_candidates[0])):@@ -340,6 +343,7 @@ class GroundedProposer(Proposer):use_task_demos=self.use_task_demos,use_instruct_history=self.use_instruct_history and instruction_history,use_tip=self.use_tip,+ verbose=self.verbose)# Generate a new instruction for our predictor, using the temperature specified for this round@@ -359,6 +363,6 @@ class GroundedProposer(Proposer):# Log the trace used to generate the new instruction, along with the new instruction itselfprompt_model.inspect_history(n=1)- print(f"PROPOSED INSTRUCTION: {proposed_instruction}")+ if self.verbose: print(f"PROPOSED INSTRUCTION: {proposed_instruction}")return strip_prefix(proposed_instruction)diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.pyindex 994cb74c6..18239d98f 100644--- a/dspy/teleprompt/mipro_optimizer_v2.py+++ b/dspy/teleprompt/mipro_optimizer_v2.py@@ -210,7 +210,7 @@ class MIPROv2(Teleprompter):try:self.program_code_string = get_dspy_source_code(student)if self.verbose:- print("SOURCE CODE:",self.program_code_string)+ if self.verbose: print("SOURCE CODE:",self.program_code_string)except Exception as e:print(f"Error getting source code: {e}.\n\nRunning without program aware proposer.")self.program_code_string = None@@ -228,6 +228,7 @@ class MIPROv2(Teleprompter):)# Setup logging+ logging.getLogger('openai').setLevel(logging.WARNING)logging.basicConfig(level=logging.WARNING)setup_logging(self.log_dir)@@ -382,9 +383,9 @@ class MIPROv2(Teleprompter):p_new.demos = p_demo_candidates[demos_idx]# Log assembled program- print("CANDIDATE PROGRAM:")- print_full_program(candidate_program)- print("...")+ if self.verbose: print("CANDIDATE PROGRAM:")+ if self.verbose: print_full_program(candidate_program)+ if self.verbose: print("...")# Save the candidate programtrial_logs[trial.number]["program_path"] = save_candidate_program(@@ -396,25 +397,27 @@ class MIPROv2(Teleprompter):# Evaluate the candidate program with relevant batch sizebatch_size = self._get_batch_size(minibatch, trainset)score = eval_candidate_program(- batch_size, trainset, candidate_program, evaluate,+ batch_size, trainset, candidate_program,+ evaluate,)+ if self.verbose: print(f"SCORE {score}, BATCH SIZE {batch_size}")# Print out a full trace of the program in use- print("FULL TRACE")- full_trace = get_task_model_history_for_full_example(- candidate_program, self.task_model, trainset, evaluate,- )- print("...")+ if self.verbose:+ print("FULL TRACE")+ get_task_model_history_for_full_example(+ candidate_program, self.task_model, trainset, evaluate,+ )+ print("...")# Log relevant information- print(f"Score {score}")+ if self.verbose: print(f"Score {score}")categorical_key = ",".join(map(str, chosen_params))param_score_dict[categorical_key].append((score, candidate_program),)trial_logs[trial.number]["num_eval_calls"] = batch_sizetrial_logs[trial.number]["full_eval"] = batch_size >= len(trainset)- trial_logs[trial.number]["eval_example_call"] = full_tracetrial_logs[trial.number]["score"] = scoretrial_logs[trial.number]["pruned"] = Falsetotal_eval_calls += trial_logs[trial.number]["num_eval_calls"]@@ -432,7 +435,7 @@ class MIPROv2(Teleprompter):# Update the best program if the current score is better, and if we're not using minibatchingbest_score_updated = Falseif score > best_score and trial_logs[trial.number]["full_eval"] and not minibatch:- print("Updating best score")+ if self.verbose: print("Updating best score")best_score = scorebest_program = candidate_program.deepcopy()best_score_updated = True@@ -462,7 +465,7 @@ class MIPROv2(Teleprompter):trial_logs[trial.number]["score"] = full_train_scoreif full_train_score > best_score:- print(f"UPDATING BEST SCORE WITH {full_train_score}")+ if self.verbose: print(f"UPDATING BEST SCORE WITH {full_train_score}")best_score = full_train_scorebest_program = highest_mean_program.deepcopy()best_score_updated = True@@ -490,10 +493,9 @@ class MIPROv2(Teleprompter):objective_function = create_objective(program, instruction_candidates, demo_candidates, evaluate, trainset,)-sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True)study = optuna.create_study(direction="maximize", sampler=sampler)- score = study.optimize(objective_function, n_trials=num_batches)+ _ = study.optimize(objective_function, n_trials=num_batches)if best_program is not None and self.track_stats:best_program.trial_logs = trial_logs@@ -501,7 +503,6 @@ class MIPROv2(Teleprompter):best_program.prompt_model_total_calls = self.prompt_model_total_callsbest_program.total_calls = self.total_calls- # program_file_path = os.path.join(self.log_dir, 'best_program.pickle')if self.log_dir:program_file_path = os.path.join(self.log_dir, "best_program")best_program.save(program_file_path)diff --git a/dspy/teleprompt/utils.py b/dspy/teleprompt/utils.pyindex 2d9ed0a78..0d7000293 100644--- a/dspy/teleprompt/utils.py+++ b/dspy/teleprompt/utils.py@@ -227,18 +227,18 @@ def setup_logging(log_dir):return# Create a loggerlogger = logging.getLogger()- logger.setLevel(logging.DEBUG)+ logger.setLevel(logging.WARNING)# Create a file handler that logs debug and higher level messagesfile_handler = logging.FileHandler(f"{log_dir}/logs.txt")- file_handler.setLevel(logging.DEBUG)+ file_handler.setLevel(logging.WARNING)file_formatter = logging.Formatter("%(asctime)s - %(message)s")file_handler.setFormatter(file_formatter)logger.addHandler(file_handler)# Create a console handler with a higher log levelconsole_handler = logging.StreamHandler()- console_handler.setLevel(logging.INFO)+ console_handler.setLevel(logging.WARNING)console_formatter = logging.Formatter("%(message)s")console_handler.setFormatter(console_formatter)logger.addHandler(console_handler)diff --git a/examples/coding/hackercup.py b/examples/coding/hackercup.pynew file mode 100644index 000000000..d19edffb1--- /dev/null+++ b/examples/coding/hackercup.py@@ -0,0 +1,433 @@+import datasets+import time+import asyncio+from typing import List, Optional+import os+import openai+import dspy+from dspy import InputField, OutputField, Signature+from dspy.functional import TypedPredictor+import pydantic+from dspy import Example+from dspy.evaluate.evaluate import Evaluate+from dspy.teleprompt.random_search import BootstrapFewShotWithRandomSearch+import re+import traceback+from datetime import datetime+import random+from dspy.teleprompt import MIPROv2+from hackercup_utils import extract_code, run, check_solution+from datetime import datetime++openai.api_key = os.environ.get("OPENAI_API_KEY")+openai.api_base = os.environ.get("OPENAI_API_BASE")++### DEFINING HELPER FUNCTIONS ###++def format_mistakes(solution_results):+ offending_cases = ""+ for offending_case in solution_results["offending_cases"]:+ offending_cases += f"Incorrect Output: {offending_case[1]} -> Expected Output: {offending_case[0]}\n"+ return offending_cases+++def get_expected_behavior_str(sample_input, sample_output):+ return f"""+input = {str(sample_input)} # input is of type {type(sample_input)}+output = solve(input)+print(output) # Output is of type string+# Prints: {sample_output}+"""+++### DEFINE SIMPLE PIPELINE ###++class GenerateCodeSignature(Signature):+ """You are an expert problem solver. Your task is creating the code to solve the problem at hand in python.++ The program should have a single `solve` method that has the following signature:+ input: [str]: The same Input provided above+ output [str]: The same Output provided above++ Here's an example of the format we'd expect for a simple python program that adds 1 to a number:+ ```python def solve(x: int):\n return x + 1```++ Note:+ * Do NOT print the Output, instead return it.+ * Make sure that your proposed solution is both time and memory efficient.+ """++ problem_description: str = InputField(format=str)+ expected_behavior: str = InputField(format=str)+ solution: str = OutputField(+ format=str, desc="A plan for how we should go about solving this problem."+ )+ python_program: str = OutputField(format=str)+++class SimpleGenerateCode(dspy.Module):+ def __init__(self):+ super().__init__()+ self.generate_code = dspy.Predict(GenerateCodeSignature)++ def forward(self, problem_description, sample_input, sample_output):+ expected_behavior = get_expected_behavior_str(sample_input, sample_output)+ python_code = extract_code(+ self.generate_code(+ problem_description=problem_description,+ expected_behavior=expected_behavior,+ ).python_program+ )++ return dspy.Prediction(solution=python_code)+++### DEFINE ADVANCED PIPELINE ###++class GenerateCodeString1(Signature):+ """You are an expert problem solver. Your task is creating the code to solve the problem at hand in python.++ The program should have a single `solve` method that has the following signature:+ input: [str]: The same Input provided above+ output [str]: The same Output provided above++ Here's an example of the format we'd expect for a simple python program that adds 1 to a number:+ ```python def solve(x: int):\n return x + 1```++ Note:+ * Do NOT print( the Output, instead return it.+ * Make sure that your proposed solution is both time and memory efficient.+ """++ # """Given a description of a coding challenge to solve, please respond with the """+ problem_description: str = InputField(format=str)+ expected_behavior: str = InputField(format=str)+ solution: str = OutputField(+ format=str, desc="A plan for how we should go about solving this problem."+ )+ python_program: str = OutputField(format=str)+++class GenerateCodeString2(Signature):+ """You are an expert leetcoder who is competing in the HackerCup.++ Your task is to write the code to solve the problem described below.++ The program should have a single `solve` method that has the following signature:+ input: [str]: The same Input provided above+ output [str]: The same Output provided above++ Here's an example of the format we'd expect for a simple python program that adds 1 to a number:+ ```python def solve(x: int):\n return x + 1```++ Note:+ * Do NOT print( the Output, instead return it.+ * Make sure that your proposed solution is both time and memory efficient.+ """++ # """Given a description of a coding challenge to solve, please respond with the """+ problem_description: str = InputField(format=str)+ expected_behavior: str = InputField(format=str)+ solution: str = OutputField(+ format=str, desc="A plan for how we should go about solving this problem."+ )+ python_program: str = OutputField(format=str)+++class GenerateCodeString3(Signature):+ """Given the leetcode problem below, write a solution in python.++ The program should have a single `solve` method that has the following signature:+ input: [str]: The same Input provided above+ output [str]: The same Output provided above++ Here's an example of the format we'd expect for a simple python program that adds 1 to a number:+ ```python def solve(x: int):\n return x + 1```++ Note:+ * Do NOT print( the Output, instead return it.+ * Make sure that your proposed solution is both time and memory efficient.+ """++ # """Given a description of a coding challenge to solve, please respond with the """+ problem_description: str = InputField(format=str)+ expected_behavior: str = InputField(format=str)+ solution: str = OutputField(+ format=str, desc="A plan for how we should go about solving this problem."+ )+ python_program: str = OutputField(format=str)++class FixCodeWithStackTrace(Signature):+ """We are trying to get this python code to run properly. Please examine the current code and the error, and propose new code."""++ problem_description = InputField()+ current_code = InputField(+ prefix="Current Incorrect Code",+ desc="Current code that is producing the incorrect behavior",+ )+ expected_behavior = InputField()+ stack_trace = InputField()+ explanation = OutputField(desc="Explanation for why the current error is occuring.")+ fixed_code = OutputField()+++class FixCodeWithCases(Signature):+ """We are trying to get this python code to run properly. Please examine the current code and the cases that are currently being missed. Propose new code that fixes the error."""++ problem_description = InputField(format=str)+ current_code = InputField(+ prefix="Current Incorrect Code:",+ desc="Current code that is producing the incorrect behavior",+ )+ expected_behavior = InputField(format=str)+ current_incorrect_results = InputField(format=str)+ problems = OutputField(+ prefix="Problem(s):",+ desc="Problem(s) with current code that have led to the incorrect results",+ format=str,+ )+ solutions = OutputField(+ prefix="Solution(s):",+ desc="Solution(s) to problems with existing code",+ format=str,+ )+ fixes = OutputField(+ prefix="Specific Changes to Make to Code:",+ desc="Specific changes we will be making to the code - this can include changes to particular code snippets.",+ )+ fixed_code = OutputField(+ format=str,+ prefix="New Code w/ Fixes Integrated:",+ desc="Code with fixes applied to correct the incorrect behavior. This code MUST be different from the code above, as it should have the fixes integrated. Include a comment next to the changes made to point them out.",+ )+++class GenerateCode(dspy.Module):+ def __init__(self, max_tries=3, num_ensembles=5):+ super().__init__()+ # Initialize variables+ self.max_tries = max_tries+ self.num_ensembles = num_ensembles++ # Initialize layers+ # self.generate_code = dspy.Predict(GenerateCodeSignature, n=self.num_ensembles)+ self.generate_code1 = dspy.Predict(GenerateCodeString1)+ self.generate_code2 = dspy.Predict(GenerateCodeString2)+ self.generate_code3 = dspy.Predict(GenerateCodeString3)+ self.fix_code_with_error = dspy.Predict(FixCodeWithStackTrace)+ self.fix_code_with_cases = dspy.Predict(FixCodeWithCases)++ def forward(self, problem_description, sample_input, sample_output):++ expected_behavior = get_expected_behavior_str(sample_input, sample_output)+ python_code1 = extract_code(self.generate_code1(problem_description=problem_description, expected_behavior = expected_behavior).python_program)+ python_code2 = extract_code(self.generate_code2(problem_description=problem_description, expected_behavior = expected_behavior).python_program)+ python_code3 = extract_code(self.generate_code3(problem_description=problem_description, expected_behavior = expected_behavior).python_program)++ python_solutions = [python_code1, python_code2, python_code3]++ for i, python_code in enumerate(python_solutions):+ for try_iter in range(self.max_tries):+ # Test our generated code, get a result+ result_dict = run(code=python_code, input=sample_input, timeout=5)+ error, result, stack_trace = (+ result_dict["error"],+ result_dict["result"],+ result_dict["stack_trace"],+ )+ if error: # Running code led to an exception, fix code+ python_code = extract_code(+ self.fix_code_with_error(+ problem_description=problem_description,+ current_code=python_code,+ expected_behavior=expected_behavior,+ stack_trace=stack_trace,+ ).fixed_code+ )+ elif result is None: # Nothing was returned by program+ python_code = extract_code(+ self.fix_code_with_cases(+ problem_description=problem_description,+ current_code=python_code,+ expected_behavior=expected_behavior,+ current_incorrect_results="Nothing was returned!",+ ).fixed_code+ )+ elif not isinstance(result, str): # Wrong type returned by program+ python_code = extract_code(+ self.fix_code_with_cases(+ problem_description=problem_description,+ current_code=python_code,+ expected_behavior=expected_behavior,+ current_incorrect_results=f"Returned type {type(result)}, but the result should be a string.",+ ).fixed_code+ )+ elif check_solution(sample_output, result)[ # Found a solutiond!+ "matches"+ ]:+ print(+ f"CORRECT SOLN W/ CODE OPTION {i}, DEBUGGING TRY {try_iter+1}."+ )+ return dspy.Prediction(solution=python_code)+ else: # Otherwise, we should be able to check the solution+ solution_results = check_solution(sample_output, result)+ # with dspy.context(lm=gpt4):+ python_code = extract_code(+ self.fix_code_with_cases(+ problem_description=problem_description,+ current_code=python_code,+ expected_behavior=expected_behavior,+ current_incorrect_results=format_mistakes(solution_results),+ ).fixed_code+ )+ return dspy.Prediction(solution=python_code)++### OPTIMIZATION ###++def optimize_with_mipro(program, prompt_model, task_model, metric, trainset):+ teleprompter = MIPROv2(+ prompt_model=prompt_model,+ task_model=task_model,+ metric=metric,+ num_candidates=5,+ init_temperature=0.5,+ verbose=False,+ log_dir="/lfs/0/kristaoo/dspy/examples/functional/logs",+ )++ optimized_program = teleprompter.compile(+ program.deepcopy(),+ trainset=trainset,+ eval_kwargs=dict(num_threads=16),+ max_bootstrapped_demos=0, # 0-shot optimization+ max_labeled_demos=0,+ num_batches=20,+ minibatch=False, # turning this off bc we have a small trainset already+ seed=9+ )++ now = datetime.now()+ date_time = now.strftime("%Y%m%d_%H%M%S")++ optimized_program.save(f"mipro_optimized_{date_time}")++ return optimized_program++def optimize_with_bootstrap_fewshot(program, task_model, teacher_model, metric, trainset):+ rs_optimizer = BootstrapFewShotWithRandomSearch(+ metric=test_code(timeout=5),+ num_threads=8,+ num_candidate_programs=5,+ max_labeled_demos=0,+ max_bootstrapped_demos=2,+ max_errors =10000,+ teacher_settings=dict(lm=teacher_model)+ )++ optimized_program = rs_optimizer.compile(+ program,+ trainset=trainset,+ )++ now = datetime.now()+ date_time = now.strftime("%Y%m%d_%H%M%S")++ optimized_program.save(f"fewshot_optimized_{date_time}")+++ return optimized_program++### DEFINING FUNCTION FOR TESTING CODE TO USE AS METRIC ###+### TODO: why this syntax?+def test_code(timeout=5):+ def metric(example, pred, trace=None):+ if pred.solution is None:+ return 0+ solution_code = pred.solution+ result_dict = run(+ code=solution_code, input=example.sample_input, timeout=timeout+ )+ if not result_dict["result"]:+ return 0+ return int(+ check_solution(example.sample_output, result_dict["result"])["matches"]+ )++ return metric+++if __name__ == "__main__":++ ### LOAD AND PREPARE DATA ###+ ds = datasets.load_dataset("hackercupai/hackercup")++ # Shuffle data+ ds_full_list = list(ds["full"])+ rng = random.Random(0)+ rng.shuffle(ds_full_list)++ # Format dataset to use in DSPy+ # TODO: what does this syntax mean+ sample_ds = [+ Example(+ problem_description=example["statement"],+ sample_input=example["sample_input"].strip().split("\n"),+ sample_output=example["sample_output"],+ ).with_inputs("problem_description", "sample_input", "sample_output")+ for example in ds["sample"]+ if example["sample_input"]+ ]++ full_ds = [+ Example(+ problem_description=example["statement"],+ sample_input=example["sample_input"].strip().split("\n"),+ sample_output=example["sample_output"],+ ).with_inputs("problem_description", "sample_input", "sample_output")+ for example in ds_full_list+ if example["sample_input"]+ ]++ trainset = sample_ds + full_ds[0:40] # use sample in train because it's easier+ testset = full_ds[40:90]++ # Configure our dspy settings (particularly LM we're using)+ lm = dspy.OpenAI(+ model="gpt-4o-mini-2024-07-18", # Note: didn't find much a difference btwn mini & full gpt-4o+ max_tokens=4000,+ temperature=0.1,+ )++ dspy.settings.configure(lm=lm)+ dspy.configure(experimental=True)++ # Setup evaluation function+ evaluate = Evaluate(+ devset=testset,+ num_threads=1, # Note: Set this to 1 for debugging purposes+ display_progress=True,+ display_table=5,+ metric=test_code(timeout=5)+ )++ # Try out a simple program (7.5% on 40 ex)+ # simple_program = SimpleGenerateCode()+ # print(f"Evaluating Simple Program on test...")+ # evaluate(program=simple_program, devset=testset)++ # Try out more advanced pipeline (22.5% on 40 ex)+ multi_stage_program = GenerateCode()+ print(f"Evaluating Multi-Stage Program on test...")+ evaluate(program=multi_stage_program, devset=testset)++ # OPTIONAL: Optimize program w/ MIPROv2 (0-shot)+ # multi_stage_program = GenerateCode()+ # mipro_optimized_multi_stage_program = optimize_with_mipro(multi_stage_program, lm, lm, test_code(timeout=5), trainset)+ # print(f"Evaluating MIPRO optimized Multi-Stage Program on test...")+ # evaluate(program=mipro_optimized_multi_stage_program, devset=testset)++ # OPTIONAL: Optimize program w/ MIPROv2 (0-shot)+ # multi_stage_program = GenerateCode()+ # bootstrap_optimized_multi_stage_program = optimize_with_bootstrap_fewshot(multi_stage_program, lm, lm, test_code(timeout=5), trainset)+ # print(f"Evaluating Bootstrap Few-Shot optimized Multi-Stage Program on test...")+ # evaluate(program=bootstrap_optimized_multi_stage_program, devset=testset)