DSPy: The framework for programming—not prompting—language models
Python
SIMBA Improvements: Bug fixes + Adding Trial Logs (#8019)
klopsahlongfeaf41287b53
diff --git a/dspy/teleprompt/simba.py b/dspy/teleprompt/simba.pyindex 9d41c563a..8554d0b1e 100644--- a/dspy/teleprompt/simba.py+++ b/dspy/teleprompt/simba.py@@ -47,7 +47,10 @@ class SIMBA(Teleprompter):self.temperature_for_sampling = temperature_for_samplingself.temperature_for_candidates = temperature_for_candidates- self.strategies = [append_a_demo(demo_input_field_maxlen), append_a_rule]+ if self.max_demos > 0:+ self.strategies = [append_a_demo(demo_input_field_maxlen), append_a_rule]+ else:+ self.strategies = [append_a_rule]def compile(self, student: dspy.Module, *, trainset: list[dspy.Example], seed: int = 0):# Basic checks@@ -117,7 +120,10 @@ class SIMBA(Teleprompter):# Parallel runnerrun_parallel = dspy.Parallel(access_examples=False, num_threads=self.num_threads)+ trial_logs = {}for batch_idx in range(self.max_steps):+ trial_logs[batch_idx] = {}+logger.info(f"Starting batch {batch_idx+1} of {self.max_steps}.")# STEP 1: Get next batch@@ -205,12 +211,14 @@ class SIMBA(Teleprompter):name2predictor = {}num_demos_list = []+ max_demos_tmp = self.max_demos if self.max_demos > 0 else 3+for name, predictor in system_candidate.named_predictors():name2predictor[name] = predictornum_demos_list.append(len(predictor.demos))num_demos = max(num_demos_list) if num_demos_list else 0- num_demos_to_drop = max(rng_np.poisson(num_demos / self.max_demos), int(num_demos >= self.max_demos))+ num_demos_to_drop = max(rng_np.poisson(num_demos / max_demos_tmp), int(num_demos >= max_demos_tmp))num_demos_to_drop = min(num_demos_to_drop, num_demos)demos_to_drop = [rng.randrange(num_demos) for _ in range(num_demos_to_drop)]@@ -268,7 +276,7 @@ class SIMBA(Teleprompter):if candidate_scores:best_idx_among_candidates = candidate_scores.index(max(candidate_scores))best_program = system_candidates[best_idx_among_candidates]- winning_programs.append(best_program)+ winning_programs.append(best_program.deepcopy())# STEP 8: Register all new candidate systems in our global poolfor idx_cand, cand_sys in enumerate(system_candidates):@@ -276,7 +284,7 @@ class SIMBA(Teleprompter):end = (idx_cand + 1) * self.bsizesys_scores = [outputs[i]["score"] for i in range(start, end)]register_new_program(cand_sys, sys_scores)-+M = len(winning_programs) - 1N = self.num_candidates + 1if M < 1:@@ -286,7 +294,7 @@ class SIMBA(Teleprompter):program_idxs = [round(i * M / (N - 1)) for i in range(N)]program_idxs = list(dict.fromkeys(program_idxs))- candidate_programs = [winning_programs[i] for i in program_idxs]+ candidate_programs = [winning_programs[i].deepcopy() for i in program_idxs]logger.info(f"VALIDATION: Evaluating {len(candidate_programs)} programs on the full trainset.")exec_pairs = [(wrap_program(sys, self.metric), ex) for sys in candidate_programs for ex in trainset]outputs = run_parallel(exec_pairs)@@ -298,7 +306,9 @@ class SIMBA(Teleprompter):sys_scores = [outputs[i]["score"] for i in range(start, end)]avg_score = sum(sys_scores) / len(sys_scores) if sys_scores else 0.0scores.append(avg_score)-+ if idx_prog != 0:+ trial_logs[idx_prog-1]["train_score"] = avg_score+best_idx = scores.index(max(scores)) if scores else 0best_program = candidate_programs[best_idx]logger.info(@@ -309,5 +319,6 @@ class SIMBA(Teleprompter):# FIXME: Attach all program candidates in decreasing average score to the best program.best_program.candidate_programs = candidate_programsbest_program.winning_programs = winning_programs+ best_program.trial_logs = trial_logsreturn best_programdiff --git a/dspy/teleprompt/utils.py b/dspy/teleprompt/utils.pyindex bbf9ac503..954ce5604 100644--- a/dspy/teleprompt/utils.py+++ b/dspy/teleprompt/utils.py@@ -5,7 +5,6 @@ import osimport randomimport shutilimport sys-import numpy as nptry:@@ -250,6 +249,46 @@ def setup_logging(log_dir):console_handler.setFormatter(console_formatter)logger.addHandler(console_handler)+def get_token_usage(model) -> tuple[int, int]:+ """+ Extract total input tokens and output tokens from a model's interaction history.+ Returns (total_input_tokens, total_output_tokens).+ """+ if not hasattr(model, "history"):+ return 0, 0++ input_tokens = []+ output_tokens = []+ for interaction in model.history:+ usage = interaction.get("usage", {})+ _input_tokens = usage.get("prompt_tokens", 0)+ _output_tokens = usage.get("completion_tokens", 0)+ input_tokens.append(_input_tokens)+ output_tokens.append(_output_tokens)++ total_input_tokens = int(np.sum(input_tokens))+ total_output_tokens = int(np.sum(output_tokens))++ return total_input_tokens, total_output_tokens+++def log_token_usage(trial_logs, trial_num, model_dict):+ """+ Extract total input and output tokens used by each model and log to trial_logs[trial_num]["token_usage"].+ """++ token_usage_dict = {}++ for model_name, model in model_dict.items():+ in_tokens, out_tokens = get_token_usage(model)+ token_usage_dict[model_name] = {+ "total_input_tokens": in_tokens,+ "total_output_tokens": out_tokens+ }++ # Store token usage info in trial logs+ trial_logs[trial_num]["token_usage"] = token_usage_dict+### OTHER UTILS ###