Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

adding in hackercup tutorial + fixing verbosity of mipro

klopsahlong0899f1c6f50b
diff --git a/dsp/adapters/experimental_adapter.py b/dsp/adapters/experimental_adapter.py
index 6e2eb3a33..ecc46aff6 100644
--- a/dsp/adapters/experimental_adapter.py
+++ b/dsp/adapters/experimental_adapter.py
@@ -75,18 +75,6 @@ class ExperimentalAdapter(BaseTemplate):
"""
example = dsp.Example(example)
- raw_pred = raw_pred.strip()
- parts = raw_pred.split('\n')
- adjusted_parts = []
- for part in parts:
- trimmed_part = part.strip()
- if trimmed_part:
- if adjusted_parts:
- adjusted_parts.append('\n' + trimmed_part)
- else:
- adjusted_parts.append(trimmed_part)
- raw_pred = '\n'.join(adjusted_parts)
-
idx = 0
while idx < len(self.fields):
if self.fields[idx].input_variable not in example or example[self.fields[idx].input_variable] is None:
diff --git a/dspy/propose/dataset_summary_generator.py b/dspy/propose/dataset_summary_generator.py
index 83a39b9b2..6c6d5fcaf 100644
--- a/dspy/propose/dataset_summary_generator.py
+++ b/dspy/propose/dataset_summary_generator.py
@@ -45,7 +45,7 @@ def order_input_keys_in_string(unordered_repr):
return ordered_repr
-def create_dataset_summary(trainset, view_data_batch_size, prompt_model, log_file=None):
+def create_dataset_summary(trainset, view_data_batch_size, prompt_model, log_file=None, verbose=False):
upper_lim = min(len(trainset), view_data_batch_size)
with dspy.settings.context(lm=prompt_model):
observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=order_input_keys_in_string(trainset[0:upper_lim].__repr__()))
@@ -62,7 +62,7 @@ def create_dataset_summary(trainset, view_data_batch_size, prompt_model, log_fil
calls+=1
if calls >= max_calls:
break
- print(f"b: {b}")
+ if verbose: print(f"b: {b}")
upper_lim = min(len(trainset), b+view_data_batch_size)
with dspy.settings.context(lm=prompt_model):
output = dspy.Predict(DatasetDescriptorWithPriorObservations, n=1, temperature=1.0)(prior_observations=observations, examples=order_input_keys_in_string(trainset[b:upper_lim].__repr__()))
@@ -76,11 +76,11 @@ def create_dataset_summary(trainset, view_data_batch_size, prompt_model, log_fil
if log_file:
log_file.write(f"observations {observations}\n")
except Exception as e:
- print(f"e {e}. using observations from past round for a summary.")
+ if verbose: print(f"e {e}. using observations from past round for a summary.")
with dspy.settings.context(lm=prompt_model):
summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)
- print(f"summary: {summary}")
+ if verbose: print(f"summary: {summary}")
if log_file:
log_file.write(f"summary: {summary}\n")
diff --git a/dspy/propose/grounded_proposer.py b/dspy/propose/grounded_proposer.py
index 130c20ad8..1745d9254 100644
--- a/dspy/propose/grounded_proposer.py
+++ b/dspy/propose/grounded_proposer.py
@@ -136,6 +136,7 @@ class GenerateModuleInstruction(dspy.Module):
use_task_demos=True,
use_instruct_history=True,
use_tip=True,
+ verbose=False,
):
super().__init__()
self.use_dataset_summary = use_dataset_summary
@@ -143,6 +144,7 @@ class GenerateModuleInstruction(dspy.Module):
self.use_task_demos = use_task_demos
self.use_instruct_history = use_instruct_history
self.use_tip = use_tip
+ self.verbose = verbose
self.program_code_string = program_code_string
self.describe_program = dspy.Predict(DescribeProgram)
@@ -189,7 +191,7 @@ class GenerateModuleInstruction(dspy.Module):
program_code=self.program_code_string, program_example=task_demos,
).program_description,
)
- print(f"PROGRAM DESCRIPTION: {program_description}")
+ if self.verbose: print(f"PROGRAM DESCRIPTION: {program_description}")
# Identify all modules
init_pattern = r"def __init__\(.*?\):([\s\S]*?)(?=^\s*def|\Z)"
@@ -209,7 +211,7 @@ class GenerateModuleInstruction(dspy.Module):
).module_description
# Generate an instruction for our chosen module
- print(f"task_demos {task_demos}")
+ if self.verbose: print(f"task_demos {task_demos}")
instruct = self.generate_module_instruction(
dataset_description=data_summary,
program_code=self.program_code_string,
@@ -223,9 +225,8 @@ class GenerateModuleInstruction(dspy.Module):
)
if hasattr(instruct, "module_description"):
module_description = strip_prefix(instruct.module_description)
- print(f"MODULE DESCRIPTION: {module_description}")
+ if self.verbose: print(f"MODULE DESCRIPTION: {module_description}")
proposed_instruction = strip_prefix(instruct.proposed_instruction)
- # print(f"PROPOSED INSTRUCTION: {proposed_instruction}")
return dspy.Prediction(proposed_instruction=proposed_instruction)
@@ -245,6 +246,7 @@ class GroundedProposer(Proposer):
use_tip=True,
set_tip_randomly=True,
set_history_randomly=True,
+ verbose=False,
):
super().__init__()
self.program_aware = program_aware
@@ -254,13 +256,14 @@ class GroundedProposer(Proposer):
self.use_tip = use_tip
self.set_tip_randomly=set_tip_randomly
self.set_history_randomly=set_history_randomly
+ self.verbose = verbose
self.prompt_model = prompt_model
self.program_code_string = program_code_string
self.data_summary = create_dataset_summary(
trainset=trainset, view_data_batch_size=view_data_batch_size, prompt_model=prompt_model,
)
- print(f"DATA SUMMARY: {self.data_summary}")
+ if self.verbose: print(f"DATA SUMMARY: {self.data_summary}")
def propose_instructions_for_program(
self,
@@ -278,21 +281,21 @@ class GroundedProposer(Proposer):
proposed_instructions = {}
if self.set_tip_randomly:
- print("Using a randomly generated configuration for our grounded proposer.")
+ if self.verbose: print("Using a randomly generated configuration for our grounded proposer.")
# Randomly select the tip
selected_tip_key = random.choice(list(TIPS.keys()))
selected_tip = TIPS[selected_tip_key]
self.use_tip = bool(
selected_tip,
)
- print(f"Selected tip: {selected_tip_key}")
+ if self.verbose: print(f"Selected tip: {selected_tip_key}")
if self.set_history_randomly:
# Randomly select whether or not we're using instruction history
use_history = random.random() < 0.5
self.use_instruct_history = use_history
- print(f"Use history T/F: {self.use_instruct_history}")
-
+ if self.verbose: print(f"Use history T/F: {self.use_instruct_history}")
+
# Create an instruction for each predictor
for pred_i, predictor in enumerate(program.predictors()):
for demo_set_i in range(len(demo_candidates[0])):
@@ -340,6 +343,7 @@ class GroundedProposer(Proposer):
use_task_demos=self.use_task_demos,
use_instruct_history=self.use_instruct_history and instruction_history,
use_tip=self.use_tip,
+ verbose=self.verbose
)
# Generate a new instruction for our predictor, using the temperature specified for this round
@@ -359,6 +363,6 @@ class GroundedProposer(Proposer):
# Log the trace used to generate the new instruction, along with the new instruction itself
prompt_model.inspect_history(n=1)
- print(f"PROPOSED INSTRUCTION: {proposed_instruction}")
+ if self.verbose: print(f"PROPOSED INSTRUCTION: {proposed_instruction}")
return strip_prefix(proposed_instruction)
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.py
index 994cb74c6..18239d98f 100644
--- a/dspy/teleprompt/mipro_optimizer_v2.py
+++ b/dspy/teleprompt/mipro_optimizer_v2.py
@@ -210,7 +210,7 @@ class MIPROv2(Teleprompter):
try:
self.program_code_string = get_dspy_source_code(student)
if self.verbose:
- print("SOURCE CODE:",self.program_code_string)
+ if self.verbose: print("SOURCE CODE:",self.program_code_string)
except Exception as e:
print(f"Error getting source code: {e}.\n\nRunning without program aware proposer.")
self.program_code_string = None
@@ -228,6 +228,7 @@ class MIPROv2(Teleprompter):
)
# Setup logging
+ logging.getLogger('openai').setLevel(logging.WARNING)
logging.basicConfig(level=logging.WARNING)
setup_logging(self.log_dir)
@@ -382,9 +383,9 @@ class MIPROv2(Teleprompter):
p_new.demos = p_demo_candidates[demos_idx]
# Log assembled program
- print("CANDIDATE PROGRAM:")
- print_full_program(candidate_program)
- print("...")
+ if self.verbose: print("CANDIDATE PROGRAM:")
+ if self.verbose: print_full_program(candidate_program)
+ if self.verbose: print("...")
# Save the candidate program
trial_logs[trial.number]["program_path"] = save_candidate_program(
@@ -396,25 +397,27 @@ class MIPROv2(Teleprompter):
# Evaluate the candidate program with relevant batch size
batch_size = self._get_batch_size(minibatch, trainset)
score = eval_candidate_program(
- batch_size, trainset, candidate_program, evaluate,
+ batch_size, trainset, candidate_program,
+ evaluate,
)
+ if self.verbose: print(f"SCORE {score}, BATCH SIZE {batch_size}")
# Print out a full trace of the program in use
- print("FULL TRACE")
- full_trace = get_task_model_history_for_full_example(
- candidate_program, self.task_model, trainset, evaluate,
- )
- print("...")
+ if self.verbose:
+ print("FULL TRACE")
+ get_task_model_history_for_full_example(
+ candidate_program, self.task_model, trainset, evaluate,
+ )
+ print("...")
# Log relevant information
- print(f"Score {score}")
+ if self.verbose: print(f"Score {score}")
categorical_key = ",".join(map(str, chosen_params))
param_score_dict[categorical_key].append(
(score, candidate_program),
)
trial_logs[trial.number]["num_eval_calls"] = batch_size
trial_logs[trial.number]["full_eval"] = batch_size >= len(trainset)
- trial_logs[trial.number]["eval_example_call"] = full_trace
trial_logs[trial.number]["score"] = score
trial_logs[trial.number]["pruned"] = False
total_eval_calls += trial_logs[trial.number]["num_eval_calls"]
@@ -432,7 +435,7 @@ class MIPROv2(Teleprompter):
# Update the best program if the current score is better, and if we're not using minibatching
best_score_updated = False
if score > best_score and trial_logs[trial.number]["full_eval"] and not minibatch:
- print("Updating best score")
+ if self.verbose: print("Updating best score")
best_score = score
best_program = candidate_program.deepcopy()
best_score_updated = True
@@ -462,7 +465,7 @@ class MIPROv2(Teleprompter):
trial_logs[trial.number]["score"] = full_train_score
if full_train_score > best_score:
- print(f"UPDATING BEST SCORE WITH {full_train_score}")
+ if self.verbose: print(f"UPDATING BEST SCORE WITH {full_train_score}")
best_score = full_train_score
best_program = highest_mean_program.deepcopy()
best_score_updated = True
@@ -490,10 +493,9 @@ class MIPROv2(Teleprompter):
objective_function = create_objective(
program, instruction_candidates, demo_candidates, evaluate, trainset,
)
-
sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True)
study = optuna.create_study(direction="maximize", sampler=sampler)
- score = study.optimize(objective_function, n_trials=num_batches)
+ _ = study.optimize(objective_function, n_trials=num_batches)
if best_program is not None and self.track_stats:
best_program.trial_logs = trial_logs
@@ -501,7 +503,6 @@ class MIPROv2(Teleprompter):
best_program.prompt_model_total_calls = self.prompt_model_total_calls
best_program.total_calls = self.total_calls
- # program_file_path = os.path.join(self.log_dir, 'best_program.pickle')
if self.log_dir:
program_file_path = os.path.join(self.log_dir, "best_program")
best_program.save(program_file_path)
diff --git a/dspy/teleprompt/utils.py b/dspy/teleprompt/utils.py
index 2d9ed0a78..0d7000293 100644
--- a/dspy/teleprompt/utils.py
+++ b/dspy/teleprompt/utils.py
@@ -227,18 +227,18 @@ def setup_logging(log_dir):
return
# Create a logger
logger = logging.getLogger()
- logger.setLevel(logging.DEBUG)
+ logger.setLevel(logging.WARNING)
# Create a file handler that logs debug and higher level messages
file_handler = logging.FileHandler(f"{log_dir}/logs.txt")
- file_handler.setLevel(logging.DEBUG)
+ file_handler.setLevel(logging.WARNING)
file_formatter = logging.Formatter("%(asctime)s - %(message)s")
file_handler.setFormatter(file_formatter)
logger.addHandler(file_handler)
# Create a console handler with a higher log level
console_handler = logging.StreamHandler()
- console_handler.setLevel(logging.INFO)
+ console_handler.setLevel(logging.WARNING)
console_formatter = logging.Formatter("%(message)s")
console_handler.setFormatter(console_formatter)
logger.addHandler(console_handler)
diff --git a/examples/coding/hackercup.py b/examples/coding/hackercup.py
new file mode 100644
index 000000000..d19edffb1
--- /dev/null
+++ b/examples/coding/hackercup.py
@@ -0,0 +1,433 @@
+import datasets
+import time
+import asyncio
+from typing import List, Optional
+import os
+import openai
+import dspy
+from dspy import InputField, OutputField, Signature
+from dspy.functional import TypedPredictor
+import pydantic
+from dspy import Example
+from dspy.evaluate.evaluate import Evaluate
+from dspy.teleprompt.random_search import BootstrapFewShotWithRandomSearch
+import re
+import traceback
+from datetime import datetime
+import random
+from dspy.teleprompt import MIPROv2
+from hackercup_utils import extract_code, run, check_solution
+from datetime import datetime
+
+openai.api_key = os.environ.get("OPENAI_API_KEY")
+openai.api_base = os.environ.get("OPENAI_API_BASE")
+
+### DEFINING HELPER FUNCTIONS ###
+
+def format_mistakes(solution_results):
+ offending_cases = ""
+ for offending_case in solution_results["offending_cases"]:
+ offending_cases += f"Incorrect Output: {offending_case[1]} -> Expected Output: {offending_case[0]}\n"
+ return offending_cases
+
+
+def get_expected_behavior_str(sample_input, sample_output):
+ return f"""
+input = {str(sample_input)} # input is of type {type(sample_input)}
+output = solve(input)
+print(output) # Output is of type string
+# Prints: {sample_output}
+"""
+
+
+### DEFINE SIMPLE PIPELINE ###
+
+class GenerateCodeSignature(Signature):
+ """You are an expert problem solver. Your task is creating the code to solve the problem at hand in python.
+
+ The program should have a single `solve` method that has the following signature:
+ input: [str]: The same Input provided above
+ output [str]: The same Output provided above
+
+ Here's an example of the format we'd expect for a simple python program that adds 1 to a number:
+ ```python def solve(x: int):\n return x + 1```
+
+ Note:
+ * Do NOT print the Output, instead return it.
+ * Make sure that your proposed solution is both time and memory efficient.
+ """
+
+ problem_description: str = InputField(format=str)
+ expected_behavior: str = InputField(format=str)
+ solution: str = OutputField(
+ format=str, desc="A plan for how we should go about solving this problem."
+ )
+ python_program: str = OutputField(format=str)
+
+
+class SimpleGenerateCode(dspy.Module):
+ def __init__(self):
+ super().__init__()
+ self.generate_code = dspy.Predict(GenerateCodeSignature)
+
+ def forward(self, problem_description, sample_input, sample_output):
+ expected_behavior = get_expected_behavior_str(sample_input, sample_output)
+ python_code = extract_code(
+ self.generate_code(
+ problem_description=problem_description,
+ expected_behavior=expected_behavior,
+ ).python_program
+ )
+
+ return dspy.Prediction(solution=python_code)
+
+
+### DEFINE ADVANCED PIPELINE ###
+
+class GenerateCodeString1(Signature):
+ """You are an expert problem solver. Your task is creating the code to solve the problem at hand in python.
+
+ The program should have a single `solve` method that has the following signature:
+ input: [str]: The same Input provided above
+ output [str]: The same Output provided above
+
+ Here's an example of the format we'd expect for a simple python program that adds 1 to a number:
+ ```python def solve(x: int):\n return x + 1```
+
+ Note:
+ * Do NOT print( the Output, instead return it.
+ * Make sure that your proposed solution is both time and memory efficient.
+ """
+
+ # """Given a description of a coding challenge to solve, please respond with the """
+ problem_description: str = InputField(format=str)
+ expected_behavior: str = InputField(format=str)
+ solution: str = OutputField(
+ format=str, desc="A plan for how we should go about solving this problem."
+ )
+ python_program: str = OutputField(format=str)
+
+
+class GenerateCodeString2(Signature):
+ """You are an expert leetcoder who is competing in the HackerCup.
+
+ Your task is to write the code to solve the problem described below.
+
+ The program should have a single `solve` method that has the following signature:
+ input: [str]: The same Input provided above
+ output [str]: The same Output provided above
+
+ Here's an example of the format we'd expect for a simple python program that adds 1 to a number:
+ ```python def solve(x: int):\n return x + 1```
+
+ Note:
+ * Do NOT print( the Output, instead return it.
+ * Make sure that your proposed solution is both time and memory efficient.
+ """
+
+ # """Given a description of a coding challenge to solve, please respond with the """
+ problem_description: str = InputField(format=str)
+ expected_behavior: str = InputField(format=str)
+ solution: str = OutputField(
+ format=str, desc="A plan for how we should go about solving this problem."
+ )
+ python_program: str = OutputField(format=str)
+
+
+class GenerateCodeString3(Signature):
+ """Given the leetcode problem below, write a solution in python.
+
+ The program should have a single `solve` method that has the following signature:
+ input: [str]: The same Input provided above
+ output [str]: The same Output provided above
+
+ Here's an example of the format we'd expect for a simple python program that adds 1 to a number:
+ ```python def solve(x: int):\n return x + 1```
+
+ Note:
+ * Do NOT print( the Output, instead return it.
+ * Make sure that your proposed solution is both time and memory efficient.
+ """
+
+ # """Given a description of a coding challenge to solve, please respond with the """
+ problem_description: str = InputField(format=str)
+ expected_behavior: str = InputField(format=str)
+ solution: str = OutputField(
+ format=str, desc="A plan for how we should go about solving this problem."
+ )
+ python_program: str = OutputField(format=str)
+
+class FixCodeWithStackTrace(Signature):
+ """We are trying to get this python code to run properly. Please examine the current code and the error, and propose new code."""
+
+ problem_description = InputField()
+ current_code = InputField(
+ prefix="Current Incorrect Code",
+ desc="Current code that is producing the incorrect behavior",
+ )
+ expected_behavior = InputField()
+ stack_trace = InputField()
+ explanation = OutputField(desc="Explanation for why the current error is occuring.")
+ fixed_code = OutputField()
+
+
+class FixCodeWithCases(Signature):
+ """We are trying to get this python code to run properly. Please examine the current code and the cases that are currently being missed. Propose new code that fixes the error."""
+
+ problem_description = InputField(format=str)
+ current_code = InputField(
+ prefix="Current Incorrect Code:",
+ desc="Current code that is producing the incorrect behavior",
+ )
+ expected_behavior = InputField(format=str)
+ current_incorrect_results = InputField(format=str)
+ problems = OutputField(
+ prefix="Problem(s):",
+ desc="Problem(s) with current code that have led to the incorrect results",
+ format=str,
+ )
+ solutions = OutputField(
+ prefix="Solution(s):",
+ desc="Solution(s) to problems with existing code",
+ format=str,
+ )
+ fixes = OutputField(
+ prefix="Specific Changes to Make to Code:",
+ desc="Specific changes we will be making to the code - this can include changes to particular code snippets.",
+ )
+ fixed_code = OutputField(
+ format=str,
+ prefix="New Code w/ Fixes Integrated:",
+ desc="Code with fixes applied to correct the incorrect behavior. This code MUST be different from the code above, as it should have the fixes integrated. Include a comment next to the changes made to point them out.",
+ )
+
+
+class GenerateCode(dspy.Module):
+ def __init__(self, max_tries=3, num_ensembles=5):
+ super().__init__()
+ # Initialize variables
+ self.max_tries = max_tries
+ self.num_ensembles = num_ensembles
+
+ # Initialize layers
+ # self.generate_code = dspy.Predict(GenerateCodeSignature, n=self.num_ensembles)
+ self.generate_code1 = dspy.Predict(GenerateCodeString1)
+ self.generate_code2 = dspy.Predict(GenerateCodeString2)
+ self.generate_code3 = dspy.Predict(GenerateCodeString3)
+ self.fix_code_with_error = dspy.Predict(FixCodeWithStackTrace)
+ self.fix_code_with_cases = dspy.Predict(FixCodeWithCases)
+
+ def forward(self, problem_description, sample_input, sample_output):
+
+ expected_behavior = get_expected_behavior_str(sample_input, sample_output)
+ python_code1 = extract_code(self.generate_code1(problem_description=problem_description, expected_behavior = expected_behavior).python_program)
+ python_code2 = extract_code(self.generate_code2(problem_description=problem_description, expected_behavior = expected_behavior).python_program)
+ python_code3 = extract_code(self.generate_code3(problem_description=problem_description, expected_behavior = expected_behavior).python_program)
+
+ python_solutions = [python_code1, python_code2, python_code3]
+
+ for i, python_code in enumerate(python_solutions):
+ for try_iter in range(self.max_tries):
+ # Test our generated code, get a result
+ result_dict = run(code=python_code, input=sample_input, timeout=5)
+ error, result, stack_trace = (
+ result_dict["error"],
+ result_dict["result"],
+ result_dict["stack_trace"],
+ )
+ if error: # Running code led to an exception, fix code
+ python_code = extract_code(
+ self.fix_code_with_error(
+ problem_description=problem_description,
+ current_code=python_code,
+ expected_behavior=expected_behavior,
+ stack_trace=stack_trace,
+ ).fixed_code
+ )
+ elif result is None: # Nothing was returned by program
+ python_code = extract_code(
+ self.fix_code_with_cases(
+ problem_description=problem_description,
+ current_code=python_code,
+ expected_behavior=expected_behavior,
+ current_incorrect_results="Nothing was returned!",
+ ).fixed_code
+ )
+ elif not isinstance(result, str): # Wrong type returned by program
+ python_code = extract_code(
+ self.fix_code_with_cases(
+ problem_description=problem_description,
+ current_code=python_code,
+ expected_behavior=expected_behavior,
+ current_incorrect_results=f"Returned type {type(result)}, but the result should be a string.",
+ ).fixed_code
+ )
+ elif check_solution(sample_output, result)[ # Found a solutiond!
+ "matches"
+ ]:
+ print(
+ f"CORRECT SOLN W/ CODE OPTION {i}, DEBUGGING TRY {try_iter+1}."
+ )
+ return dspy.Prediction(solution=python_code)
+ else: # Otherwise, we should be able to check the solution
+ solution_results = check_solution(sample_output, result)
+ # with dspy.context(lm=gpt4):
+ python_code = extract_code(
+ self.fix_code_with_cases(
+ problem_description=problem_description,
+ current_code=python_code,
+ expected_behavior=expected_behavior,
+ current_incorrect_results=format_mistakes(solution_results),
+ ).fixed_code
+ )
+ return dspy.Prediction(solution=python_code)
+
+### OPTIMIZATION ###
+
+def optimize_with_mipro(program, prompt_model, task_model, metric, trainset):
+ teleprompter = MIPROv2(
+ prompt_model=prompt_model,
+ task_model=task_model,
+ metric=metric,
+ num_candidates=5,
+ init_temperature=0.5,
+ verbose=False,
+ log_dir="/lfs/0/kristaoo/dspy/examples/functional/logs",
+ )
+
+ optimized_program = teleprompter.compile(
+ program.deepcopy(),
+ trainset=trainset,
+ eval_kwargs=dict(num_threads=16),
+ max_bootstrapped_demos=0, # 0-shot optimization
+ max_labeled_demos=0,
+ num_batches=20,
+ minibatch=False, # turning this off bc we have a small trainset already
+ seed=9
+ )
+
+ now = datetime.now()
+ date_time = now.strftime("%Y%m%d_%H%M%S")
+
+ optimized_program.save(f"mipro_optimized_{date_time}")
+
+ return optimized_program
+
+def optimize_with_bootstrap_fewshot(program, task_model, teacher_model, metric, trainset):
+ rs_optimizer = BootstrapFewShotWithRandomSearch(
+ metric=test_code(timeout=5),
+ num_threads=8,
+ num_candidate_programs=5,
+ max_labeled_demos=0,
+ max_bootstrapped_demos=2,
+ max_errors =10000,
+ teacher_settings=dict(lm=teacher_model)
+ )
+
+ optimized_program = rs_optimizer.compile(
+ program,
+ trainset=trainset,
+ )
+
+ now = datetime.now()
+ date_time = now.strftime("%Y%m%d_%H%M%S")
+
+ optimized_program.save(f"fewshot_optimized_{date_time}")
+
+
+ return optimized_program
+
+### DEFINING FUNCTION FOR TESTING CODE TO USE AS METRIC ###
+### TODO: why this syntax?
+def test_code(timeout=5):
+ def metric(example, pred, trace=None):
+ if pred.solution is None:
+ return 0
+ solution_code = pred.solution
+ result_dict = run(
+ code=solution_code, input=example.sample_input, timeout=timeout
+ )
+ if not result_dict["result"]:
+ return 0
+ return int(
+ check_solution(example.sample_output, result_dict["result"])["matches"]
+ )
+
+ return metric
+
+
+if __name__ == "__main__":
+
+ ### LOAD AND PREPARE DATA ###
+ ds = datasets.load_dataset("hackercupai/hackercup")
+
+ # Shuffle data
+ ds_full_list = list(ds["full"])
+ rng = random.Random(0)
+ rng.shuffle(ds_full_list)
+
+ # Format dataset to use in DSPy
+ # TODO: what does this syntax mean
+ sample_ds = [
+ Example(
+ problem_description=example["statement"],
+ sample_input=example["sample_input"].strip().split("\n"),
+ sample_output=example["sample_output"],
+ ).with_inputs("problem_description", "sample_input", "sample_output")
+ for example in ds["sample"]
+ if example["sample_input"]
+ ]
+
+ full_ds = [
+ Example(
+ problem_description=example["statement"],
+ sample_input=example["sample_input"].strip().split("\n"),
+ sample_output=example["sample_output"],
+ ).with_inputs("problem_description", "sample_input", "sample_output")
+ for example in ds_full_list
+ if example["sample_input"]
+ ]
+
+ trainset = sample_ds + full_ds[0:40] # use sample in train because it's easier
+ testset = full_ds[40:90]
+
+ # Configure our dspy settings (particularly LM we're using)
+ lm = dspy.OpenAI(
+ model="gpt-4o-mini-2024-07-18", # Note: didn't find much a difference btwn mini & full gpt-4o
+ max_tokens=4000,
+ temperature=0.1,
+ )
+
+ dspy.settings.configure(lm=lm)
+ dspy.configure(experimental=True)
+
+ # Setup evaluation function
+ evaluate = Evaluate(
+ devset=testset,
+ num_threads=1, # Note: Set this to 1 for debugging purposes
+ display_progress=True,
+ display_table=5,
+ metric=test_code(timeout=5)
+ )
+
+ # Try out a simple program (7.5% on 40 ex)
+ # simple_program = SimpleGenerateCode()
+ # print(f"Evaluating Simple Program on test...")
+ # evaluate(program=simple_program, devset=testset)
+
+ # Try out more advanced pipeline (22.5% on 40 ex)
+ multi_stage_program = GenerateCode()
+ print(f"Evaluating Multi-Stage Program on test...")
+ evaluate(program=multi_stage_program, devset=testset)
+
+ # OPTIONAL: Optimize program w/ MIPROv2 (0-shot)
+ # multi_stage_program = GenerateCode()
+ # mipro_optimized_multi_stage_program = optimize_with_mipro(multi_stage_program, lm, lm, test_code(timeout=5), trainset)
+ # print(f"Evaluating MIPRO optimized Multi-Stage Program on test...")
+ # evaluate(program=mipro_optimized_multi_stage_program, devset=testset)
+
+ # OPTIONAL: Optimize program w/ MIPROv2 (0-shot)
+ # multi_stage_program = GenerateCode()
+ # bootstrap_optimized_multi_stage_program = optimize_with_bootstrap_fewshot(multi_stage_program, lm, lm, test_code(timeout=5), trainset)
+ # print(f"Evaluating Bootstrap Few-Shot optimized Multi-Stage Program on test...")
+ # evaluate(program=bootstrap_optimized_multi_stage_program, devset=testset)