Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Fix style errors of dspy/teleprompt (#8188)

TomuHirata9914282986b7
diff --git a/dspy/teleprompt/__init__.py b/dspy/teleprompt/__init__.py
index 3168cd1c4..a5bc4da7f 100644
--- a/dspy/teleprompt/__init__.py
+++ b/dspy/teleprompt/__init__.py
@@ -4,13 +4,11 @@ from dspy.teleprompt.bootstrap import BootstrapFewShot
from dspy.teleprompt.bootstrap_finetune import BootstrapFinetune
from dspy.teleprompt.copro_optimizer import COPRO
from dspy.teleprompt.ensemble import Ensemble
+from dspy.teleprompt.infer_rules import InferRules
from dspy.teleprompt.knn_fewshot import KNNFewShot
-from dspy.teleprompt.simba import SIMBA
-
from dspy.teleprompt.mipro_optimizer_v2 import MIPROv2
from dspy.teleprompt.random_search import BootstrapFewShotWithRandomSearch
-from dspy.teleprompt.infer_rules import InferRules
-
+from dspy.teleprompt.simba import SIMBA
from dspy.teleprompt.teleprompt import Teleprompter
from dspy.teleprompt.teleprompt_optuna import BootstrapFewShotWithOptuna
from dspy.teleprompt.vanilla import LabeledFewShot
diff --git a/dspy/teleprompt/avatar_optimizer.py b/dspy/teleprompt/avatar_optimizer.py
index 43fa1d57b..5d0490941 100644
--- a/dspy/teleprompt/avatar_optimizer.py
+++ b/dspy/teleprompt/avatar_optimizer.py
@@ -1,15 +1,14 @@
-import dspy
-
-from tqdm import tqdm
+from concurrent.futures import ThreadPoolExecutor
from copy import deepcopy
from random import sample
+from typing import Callable, List, Optional, Tuple
+
from pydantic import BaseModel
-from concurrent.futures import ThreadPoolExecutor
-from typing import Callable, List, Tuple, Optional
+from tqdm import tqdm
-from .teleprompt import Teleprompter
+import dspy
from dspy.predict.avatar import ActionOutput
-
+from dspy.teleprompt.teleprompt import Teleprompter
DEFAULT_MAX_EXAMPLES = 10
@@ -80,8 +79,8 @@ class AvatarOptimizer(Teleprompter):
max_iters: int = 10,
lower_bound: int = 0,
upper_bound: int = 1,
- max_positive_inputs: int = None,
- max_negative_inputs: int = None,
+ max_positive_inputs: Optional[int] = None,
+ max_negative_inputs: Optional[int] = None,
optimize_for: str = "max",
):
assert metric is not None, "`metric` argument cannot be None. Please provide a metric function."
@@ -113,7 +112,7 @@ class AvatarOptimizer(Teleprompter):
except Exception as e:
print(e)
-
+
if return_outputs:
return example, None, 0
else:
@@ -128,7 +127,7 @@ class AvatarOptimizer(Teleprompter):
with ThreadPoolExecutor(max_workers=num_threads) as executor:
futures = [executor.submit(self.process_example, actor, example, return_outputs) for example in devset]
-
+
for future in tqdm(futures, total=total_examples, desc="Processing examples"):
result = future.result()
if return_outputs:
@@ -137,9 +136,9 @@ class AvatarOptimizer(Teleprompter):
results.append((example, prediction, score))
else:
total_score += result
-
+
avg_metric = total_score / total_examples
-
+
if return_outputs:
return avg_metric, results
else:
@@ -147,13 +146,13 @@ class AvatarOptimizer(Teleprompter):
def _get_pos_neg_results(
- self,
- actor: dspy.Module,
+ self,
+ actor: dspy.Module,
trainset: List[dspy.Example]
) -> Tuple[float, List[EvalResult], List[EvalResult]]:
pos_inputs = []
neg_inputs = []
-
+
avg_score, results = self.thread_safe_evaluator(trainset, actor, return_outputs=True)
print(f"Average Score: {avg_score}")
@@ -179,14 +178,14 @@ class AvatarOptimizer(Teleprompter):
raise ValueError("No positive examples found, try lowering the upper_bound or providing more training data")
if len(neg_inputs) == 0:
raise ValueError("No negative examples found, try raising the lower_bound or providing more training data")
-
+
return (avg_score, pos_inputs, neg_inputs)
-
+
def compile(self, student, *, trainset):
best_actor = deepcopy(student)
best_score = -999 if self.optimize_for == "max" else 999
-
+
for i in range(self.max_iters):
print(20*'=')
print(f"Iteration {i+1}/{self.max_iters}")
@@ -220,7 +219,7 @@ class AvatarOptimizer(Teleprompter):
best_actor.actor.signature = best_actor.actor.signature.with_instructions(new_instruction)
best_actor.actor_clone = deepcopy(best_actor.actor)
best_score = score
-
+
print(f"Best Actor: {best_actor}")
return best_actor
diff --git a/dspy/teleprompt/bettertogether.py b/dspy/teleprompt/bettertogether.py
index 64c613030..6737c9ca6 100644
--- a/dspy/teleprompt/bettertogether.py
+++ b/dspy/teleprompt/bettertogether.py
@@ -5,7 +5,13 @@ from typing import Callable, List, Optional
import dspy
from dspy.primitives.example import Example
from dspy.primitives.program import Program
-from dspy.teleprompt.bootstrap_finetune import BootstrapFinetune, prepare_student, all_predictors_have_lms, launch_lms, kill_lms
+from dspy.teleprompt.bootstrap_finetune import (
+ BootstrapFinetune,
+ all_predictors_have_lms,
+ kill_lms,
+ launch_lms,
+ prepare_student,
+)
from dspy.teleprompt.random_search import BootstrapFewShotWithRandomSearch
from dspy.teleprompt.teleprompt import Teleprompter
@@ -27,7 +33,7 @@ class BetterTogether(Teleprompter):
# TODO: Note that the BetterTogether optimizer is meaningful when
# BootstrapFinetune uses a metric to filter the training data before
- # fine-tuning. However, one can also choose to run this optimizer with
+ # fine-tuning. However, one can also choose to run this optimizer with
# a BoostrapFinetune without a metric, say, if there aren't labels
# available for the training data. Should this be noted somewhere?
# TODO: We should re-consider if the metric should be required.
@@ -56,7 +62,7 @@ class BetterTogether(Teleprompter):
logger.info("Validating the strategy")
parsed_strategy = strategy.lower().split(self.STRAT_SEP)
- if not all([s in ["p", "w"] for s in parsed_strategy]):
+ if not all(s in ["p", "w"] for s in parsed_strategy):
raise ValueError(
f"The strategy should be a sequence of 'p' and 'w' separated by '{self.STRAT_SEP}', but "
f"found: {strategy}"
@@ -73,10 +79,10 @@ class BetterTogether(Teleprompter):
trainset = trainset[:]
logger.info("Compiling the student program...")
student = self._run_strategies(parsed_strategy, student, trainset, valset_ratio)
-
+
logger.info("BetterTogether has finished compiling the student program")
return student
-
+
def _run_strategies(self, parsed_strategy, student, trainset, valset_ratio) -> Program:
# Keep track of all the partial strategies/programs in parsed_strategy
# "" corresponds to the initial student program
@@ -115,7 +121,7 @@ class BetterTogether(Teleprompter):
student.candidate_programs = candidate_programs
return student
-
+
def _compile_prompt_optimizer(self, student, trainset, valset_ratio) -> Program:
logger.info("Preparing for prompt optimization...")
@@ -141,7 +147,7 @@ class BetterTogether(Teleprompter):
pred.lm = lm
return student
-
+
def _compile_weight_optimizer(self, student, trainset) -> Program:
logger.info("Preparing for weight optimization...")
@@ -152,7 +158,7 @@ class BetterTogether(Teleprompter):
# prompt optimizers are accepting a valset or encode a way to check if
# a valset should be passed to an optimizer's compile.
logger.info("Compiling the weight optimizer...")
- student = self.weight_optimizer.compile(student, trainset=trainset)
+ student = self.weight_optimizer.compile(student, trainset=trainset)
# Updating the train kwargs for the new LMs. This is needed because the
# train_kwargs of the optimizer is configured for the original LMs.
diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.py
index 81ab80a86..5fe2c8fd0 100644
--- a/dspy/teleprompt/bootstrap.py
+++ b/dspy/teleprompt/bootstrap.py
@@ -6,8 +6,8 @@ from typing import Dict, Optional
import tqdm
import dspy
+from dspy.teleprompt.teleprompt import Teleprompter
-from .teleprompt import Teleprompter
from .vanilla import LabeledFewShot
# TODO: metrics should return an object with __bool__ basically, but fine if they're more complex.
@@ -184,7 +184,7 @@ class BootstrapFewShot(Teleprompter):
with dspy.settings.context(trace=[], **self.teacher_settings):
lm = dspy.settings.lm
lm = lm.copy(temperature=0.7 + 0.001 * round_idx) if round_idx > 0 else lm
- new_settings = dict(lm=lm) if round_idx > 0 else {}
+ new_settings = {"lm": lm} if round_idx > 0 else {}
with dspy.settings.context(**new_settings):
for name, predictor in teacher.named_predictors():
diff --git a/dspy/teleprompt/bootstrap_finetune.py b/dspy/teleprompt/bootstrap_finetune.py
index 7ce5787cc..83b3808a6 100644
--- a/dspy/teleprompt/bootstrap_finetune.py
+++ b/dspy/teleprompt/bootstrap_finetune.py
@@ -87,12 +87,12 @@ class BootstrapFinetune(FinetuneTeleprompter):
trace_data=trace_data, lm=pred.lm, pred_ind=data_pred_ind
)
logger.info(f"Using {len(train_data)} data points for fine-tuning the model: {pred.lm.model}")
- finetune_kwargs = dict(
- lm=pred.lm,
- train_data=train_data,
- train_data_format=data_format,
- train_kwargs=self.train_kwargs[pred.lm],
- )
+ finetune_kwargs = {
+ "lm": pred.lm,
+ "train_data": train_data,
+ "train_data_format": data_format,
+ "train_kwargs": self.train_kwargs[pred.lm],
+ }
key_to_data[training_key] = finetune_kwargs
logger.info("Starting LM fine-tuning...")
@@ -308,7 +308,7 @@ def bootstrap_trace_data(
raise ve
else:
continue
- data_dict = dict(example=example, prediction=prediction, trace=trace, example_ind=example_ind)
+ data_dict = {"example": example, "prediction": prediction, "trace": trace, "example_ind": example_ind}
if metric:
data_dict["score"] = score
data.append(data_dict)
diff --git a/dspy/teleprompt/grpo.py b/dspy/teleprompt/grpo.py
index d693779cc..7ee397845 100644
--- a/dspy/teleprompt/grpo.py
+++ b/dspy/teleprompt/grpo.py
@@ -1,18 +1,23 @@
-from collections import Counter
import logging
import random
+from collections import Counter
from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
from dspy.adapters.base import Adapter
from dspy.adapters.chat_adapter import ChatAdapter
from dspy.clients.lm import LM
-from dspy.clients.utils_finetune import TrainDataFormat, GRPOGroup
+from dspy.clients.utils_finetune import GRPOGroup, TrainDataFormat
from dspy.dsp.utils.settings import settings
from dspy.evaluate.evaluate import Evaluate
from dspy.primitives.example import Example
from dspy.primitives.program import Program
-from dspy.teleprompt.bootstrap_finetune import FinetuneTeleprompter, all_predictors_have_lms, assert_structural_equivalency, bootstrap_trace_data, FailedPrediction
-
+from dspy.teleprompt.bootstrap_finetune import (
+ FailedPrediction,
+ FinetuneTeleprompter,
+ all_predictors_have_lms,
+ assert_structural_equivalency,
+ bootstrap_trace_data,
+)
logger = logging.getLogger(__name__)
@@ -94,13 +99,13 @@ class GRPO(FinetuneTeleprompter):
assert "trace" in trace_data[0][0][0], "Trace data does not contain the 'trace' key"
assert len(trace_data[0][0][0]["trace"]) > 0, "Trace data is empty"
assert len(trace_data[0][0][0]["trace"][0]) == 3, f"Trace tuple length {len(trace_data[0][0][0]['trace'][0])} does not match the expected length 3"
-
+
for example_data in trace_data:
for teacher_data in example_data:
for sample in teacher_data:
for t in sample["trace"]:
assert hash(t[0].signature) in pred_signature_hash_to_ind
-
+
def report_validation_metrics(self, student, trainset, valset, logger, step_idx=-1):
if step_idx == -1 or step_idx == self.num_train_steps - 1 or (step_idx + 1) % self.num_steps_for_val == 0:
pass
@@ -192,7 +197,7 @@ class GRPO(FinetuneTeleprompter):
assert not self.use_train_as_val, "If report_train_scores is False, use_train_as_val must be False."
if step_idx == -1:
logger.info("Not using any validation set and not reporting train scores.")
-
+
def update_shuffled_trainset(self, original_trainset):
self.shuffled_trainset_ids = list(range(len(original_trainset)))
self.rng.shuffle(self.shuffled_trainset_ids)
@@ -299,7 +304,7 @@ class GRPO(FinetuneTeleprompter):
for t in teachers:
assert_structural_equivalency(student, t)
all_predictors_have_lms(t)
-
+
# Ensure that the teachers list contain the student program
assert student in teachers, f"Student program {student} is not in the list of teachers {teachers}. Please provide the student program as one of the teachers. Alternatively, you can leave the teacher argument as None, and the student program will be used as the teacher program."
assert self.num_rollouts_per_grpo_step % len(teachers) == 0, (
@@ -308,7 +313,7 @@ class GRPO(FinetuneTeleprompter):
"Please provide a number of examples that is divisible by the number of teachers."
)
num_samples_per_input = self.num_rollouts_per_grpo_step // len(teachers)
-
+
# We will disable the LM cache for all programs (student and teachers)
# These will be reverted to their original state at the end of the
# training
@@ -336,7 +341,7 @@ class GRPO(FinetuneTeleprompter):
train_kwargs = self.train_kwargs[pred.lm]
job = pred.lm.reinforce(train_kwargs=train_kwargs)
grpo_training_jobs[job_key] = job
-
+
self.report_validation_metrics(
student=student,
trainset=trainset,
@@ -405,7 +410,7 @@ class GRPO(FinetuneTeleprompter):
assert sample["example_ind"] == example_ind, f"Example index {sample['example_ind']} does not match the expected index {example_ind}"
trace_instances_for_current_pred = [(*t, sample["score"]) for t in sample["trace"] if hash(t[0].signature) == hash(student.predictors()[pred_id].signature)]
-
+
predictor_example_invocations.append(trace_instances_for_current_pred)
if len(predictor_example_invocations) == 0:
@@ -424,9 +429,9 @@ class GRPO(FinetuneTeleprompter):
predictor_example_invocations = [invocation[:min_len] for invocation in predictor_example_invocations]
elif self.variably_invoked_predictor_grouping_mode == 'fill':
if self.variably_invoked_predictor_fill_strategy == 'randint':
- selector = lambda l: self.rng.choice(l)
+ selector = lambda l: self.rng.choice(l) # noqa: E731, E741
else:
- selector = lambda l: l[-1]
+ selector = lambda l: l[-1] # noqa: E731, E741
predictor_example_invocations = [
invocation + [selector(invocation) for _ in range(max_len - len(invocation))]
for invocation in predictor_example_invocations
@@ -454,8 +459,8 @@ class GRPO(FinetuneTeleprompter):
# TODO(GRPO Team): Use build_call_data_from_trace (from bootstrap_finetune) instead of
# dealing with the message formatting ourselves.
inp_messages = adapter.format(
- signature=trace_instance[0].signature,
- inputs=trace_instance[1],
+ signature=trace_instance[0].signature,
+ inputs=trace_instance[1],
demos=[] # TODO: Add support for demos
)
@@ -488,10 +493,10 @@ class GRPO(FinetuneTeleprompter):
},
"reward": float(score),
})
-
+
train_batch_per_predictor[pred_id].extend(example_training_data)
-
- if len(sum(train_batch_per_predictor, [])) == 0:
+
+ if not any(train_batch_per_predictor):
logger.warning("No training data found for this training step. This means that the model did not generate valid formatted responses for any of the examples in the training set. This is a critical error. Please check the model and the training set.")
continue
@@ -519,8 +524,8 @@ class GRPO(FinetuneTeleprompter):
# addition to updating any teacher programs that share the same
# LM.
logger.info("Invoking GRPO training step...")
- for (lm, data_key), job in grpo_training_jobs.items():
- train_data: List[GRPOGroup] = sum(train_batch_per_predictor, []) if data_key is None else train_batch_per_predictor[data_key]
+ for (_, data_key), job in grpo_training_jobs.items():
+ train_data: List[GRPOGroup] = sum(train_batch_per_predictor, []) if data_key is None else train_batch_per_predictor[data_key] #noqa: RUF017
for group in train_data:
if len(group) != self.num_rollouts_per_grpo_step:
# TODO(GRPO Team): This is very undesirable. This occurs only because in some of the generations, the model does not follow the correct dspy format.
@@ -533,13 +538,13 @@ class GRPO(FinetuneTeleprompter):
job.step(train_data=train_data, train_data_format=TrainDataFormat.GRPO_CHAT)
- for (lm, data_key), job in grpo_training_jobs.items():
+ for (lm, _), job in grpo_training_jobs.items():
if (train_step_idx + 1) % self.train_kwargs[lm]["update_interval"] == 0 and train_step_idx != 0:
logger.info(f"Current train step is {train_step_idx + 1}. Updating the model...")
job.update_model()
logger.info(f"GRPO training step {train_step_idx + 1}/{self.num_train_steps} completed.")
-
+
self.report_validation_metrics(
student=student,
trainset=trainset,
@@ -549,7 +554,7 @@ class GRPO(FinetuneTeleprompter):
)
logger.info("Done with the iterations! Retrieving the final model(s)...")
- for (lm, data_key), job in grpo_training_jobs.items():
+ for _, job in grpo_training_jobs.items():
job.terminate()
# Revert cache states to their initial values
diff --git a/dspy/teleprompt/knn_fewshot.py b/dspy/teleprompt/knn_fewshot.py
index ab08fd2fc..65ff27ccb 100644
--- a/dspy/teleprompt/knn_fewshot.py
+++ b/dspy/teleprompt/knn_fewshot.py
@@ -4,8 +4,7 @@ from dspy.clients import Embedder
from dspy.predict.knn import KNN
from dspy.primitives import Example
from dspy.teleprompt import BootstrapFewShot
-
-from .teleprompt import Teleprompter
+from dspy.teleprompt.teleprompt import Teleprompter
class KNNFewShot(Teleprompter):
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.py
index 711b227f5..7bcff1a32 100644
--- a/dspy/teleprompt/mipro_optimizer_v2.py
+++ b/dspy/teleprompt/mipro_optimizer_v2.py
@@ -1,15 +1,16 @@
import logging
import random
-import textwrap
-from collections import defaultdict
-from typing import Any, Callable, Dict, List, Literal, Optional, Tuple
import select
import sys
+import textwrap
import time
+from collections import defaultdict
+from typing import Any, Callable, Dict, List, Literal, Optional, Tuple
import numpy as np
import optuna
from optuna.distributions import CategoricalDistribution
+
import dspy
from dspy.evaluate.evaluate import Evaluate
from dspy.propose import GroundedProposer
@@ -52,7 +53,7 @@ class MIPROv2(Teleprompter):
metric: Callable,
prompt_model: Optional[Any] = None,
task_model: Optional[Any] = None,
- teacher_settings: Dict = {},
+ teacher_settings: Optional[dict] = None,
max_bootstrapped_demos: int = 4,
max_labeled_demos: int = 4,
auto: Optional[Literal["light", "medium", "heavy"]] = "light",
@@ -83,7 +84,7 @@ class MIPROv2(Teleprompter):
self.verbose = verbose
self.track_stats = track_stats
self.log_dir = log_dir
- self.teacher_settings = teacher_settings
+ self.teacher_settings = teacher_settings or {}
self.prompt_model_total_calls = 0
self.total_calls = 0
self.num_threads = num_threads
@@ -114,21 +115,21 @@ class MIPROv2(Teleprompter):
requires_permission_to_run: bool = True,
provide_traceback: Optional[bool] = None,
) -> Any:
-
+
zeroshot_opt = (self.max_bootstrapped_demos == 0) and (self.max_labeled_demos == 0)
-
+
# If auto is None, and num_trials is not provided (but num_candidates is), raise an error that suggests a good num_trials value
if self.auto is None and (self.num_candidates is not None and num_trials is None):
raise ValueError(f"If auto is None, num_trials must also be provided. Given num_candidates={self.num_candidates}, we'd recommend setting num_trials to ~{self._set_num_trials_from_num_candidates(student, zeroshot_opt, self.num_candidates)}.")
-
+
# If auto is None, and num_candidates or num_trials is None, raise an error
if self.auto is None and (self.num_candidates is None or num_trials is None):
raise ValueError("If auto is None, num_candidates must also be provided.")
-
+
# If auto is provided, and either num_candidates or num_trials is not None, raise an error
if self.auto is not None and (self.num_candidates is not None or num_trials is not None):
raise ValueError("If auto is not None, num_candidates and num_trials cannot be set, since they would be overrided by the auto settings. Please either set auto to None, or do not specify num_candidates and num_trials.")
-
+
# Set random seeds
seed = seed or self.seed
self._set_random_seeds(seed)
@@ -226,7 +227,7 @@ class MIPROv2(Teleprompter):
num_trials = int(max(2 * num_vars * np.log2(num_candidates), 1.5 * num_candidates))
return num_trials
-
+
def _set_hyperparams_from_run_mode(
self,
program: Any,
@@ -239,15 +240,15 @@ class MIPROv2(Teleprompter):
return num_trials, valset, minibatch
auto_settings = AUTO_RUN_SETTINGS[self.auto]
-
+
valset = create_minibatch(valset, batch_size=auto_settings["val_size"], rng=self.rng)
minibatch = len(valset) > MIN_MINIBATCH_SIZE
-
+
# Set num instruct candidates to 1/2 of N if optimizing with few-shot examples, otherwise set to N
# This is because we've found that it's generally better to spend optimization budget on few-shot examples
# When they are allowed.
self.num_instruct_candidates = auto_settings["n"] if zeroshot_opt else int(auto_settings["n"] * 0.5)
- self.num_fewshot_candidates = auto_settings["n"]
+ self.num_fewshot_candidates = auto_settings["n"]
num_trials = self._set_num_trials_from_num_candidates(program, zeroshot_opt, auto_settings["n"])
@@ -381,7 +382,7 @@ class MIPROv2(Teleprompter):
)
print(f"{user_message}\n{user_confirmation_message}\nDo you wish to continue? (y/n): ", end='', flush=True)
-
+
# Wait for input with timeout
start_time = time.time()
while time.time() - start_time < 20:
@@ -389,7 +390,7 @@ class MIPROv2(Teleprompter):
user_input = sys.stdin.readline().strip().lower()
return user_input == "y"
time.sleep(0.1)
-
+
print("\nNo input received within 20 seconds. Proceeding with execution...")
return True
diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.py
index caba1474b..89fcd5eae 100644
--- a/dspy/teleprompt/random_search.py
+++ b/dspy/teleprompt/random_search.py
@@ -27,7 +27,7 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
def __init__(
self,
metric,
- teacher_settings={},
+ teacher_settings=None,
max_bootstrapped_demos=4,
max_labeled_demos=16,
max_rounds=1,
@@ -38,7 +38,7 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
metric_threshold=None,
):
self.metric = metric
- self.teacher_settings = teacher_settings
+ self.teacher_settings = teacher_settings or {}
self.max_rounds = max_rounds
self.num_threads = num_threads
diff --git a/dspy/teleprompt/simba.py b/dspy/teleprompt/simba.py
index 05aabe248..eb080521a 100644
--- a/dspy/teleprompt/simba.py
+++ b/dspy/teleprompt/simba.py
@@ -1,11 +1,12 @@
-import dspy
-import random
import logging
+import random
+from typing import Callable
import numpy as np
-from typing import Callable
+
+import dspy
+from dspy.teleprompt.simba_utils import append_a_demo, append_a_rule, prepare_models_for_resampling, wrap_program
from dspy.teleprompt.teleprompt import Teleprompter
-from dspy.teleprompt.simba_utils import prepare_models_for_resampling, wrap_program, append_a_demo, append_a_rule
logger = logging.getLogger(__name__)
@@ -169,7 +170,7 @@ class SIMBA(Teleprompter):
batch_90th_percentile_score = np.percentile([float(o["score"]) for o in outputs], 90)
# We'll chunk `outputs` by example index, each chunk has length = num_candidates
- for idx, example in enumerate(batch):
+ for idx, _ in enumerate(batch):
# gather all results for this example
bucket = [outputs[i] for i in range(idx, len(outputs), self.bsize)]
bucket.sort(key=lambda x: x["score"], reverse=True)
@@ -222,7 +223,7 @@ class SIMBA(Teleprompter):
num_demos_to_drop = min(num_demos_to_drop, num_demos)
demos_to_drop = [rng.randrange(num_demos) for _ in range(num_demos_to_drop)]
- for name, predictor in name2predictor.items():
+ for _, predictor in name2predictor.items():
predictor.demos = [demo for idxd, demo in enumerate(predictor.demos) if idxd not in demos_to_drop]
# Pick a strategy
@@ -260,7 +261,7 @@ class SIMBA(Teleprompter):
# STEP 6: Compute average mini-batch scores for each new candidate
candidate_scores = []
- for idx_cand, cand_sys in enumerate(system_candidates):
+ for idx_cand, _ in enumerate(system_candidates):
start = idx_cand * self.bsize
end = (idx_cand + 1) * self.bsize
sys_scores = [outputs[i]["score"] for i in range(start, end)]
@@ -284,9 +285,9 @@ class SIMBA(Teleprompter):
end = (idx_cand + 1) * self.bsize
sys_scores = [outputs[i]["score"] for i in range(start, end)]
register_new_program(cand_sys, sys_scores)
-
- M = len(winning_programs) - 1
- N = self.num_candidates + 1
+
+ M = len(winning_programs) - 1 # noqa: N806
+ N = self.num_candidates + 1 # noqa: N806
if M < 1:
program_idxs = [0] * N
else:
@@ -300,7 +301,7 @@ class SIMBA(Teleprompter):
outputs = run_parallel(exec_pairs)
scores = []
- for idx_prog, prog in enumerate(candidate_programs):
+ for idx_prog, _ in enumerate(candidate_programs):
start = idx_prog * len(trainset)
end = (idx_prog + 1) * len(trainset)
sys_scores = [outputs[i]["score"] for i in range(start, end)]
diff --git a/dspy/teleprompt/simba_utils.py b/dspy/teleprompt/simba_utils.py
index 3765a33f1..b32f30609 100644
--- a/dspy/teleprompt/simba_utils.py
+++ b/dspy/teleprompt/simba_utils.py
@@ -1,12 +1,13 @@
-import dspy
-import ujson
import inspect
import logging
import textwrap
+from typing import Callable
+
+import ujson
+import dspy
from dspy.adapters.utils import get_field_description_string
from dspy.signatures import InputField, OutputField
-from typing import Callable
logger = logging.getLogger(__name__)
@@ -69,7 +70,7 @@ def append_a_demo(demo_input_field_maxlen):
logger.info(f"Added {len(name2demo)} demos (one each) across all predictors.")
return True
-
+
return append_a_demo_
@@ -97,27 +98,27 @@ def append_a_rule(bucket, system, **kwargs):
good["prediction"] = {"N/A": "Prediction not available"}
better_trajectory = [
- dict(module_name=predictor2name[id(p)], inputs=i, outputs=dict(o))
+ {"module_name": predictor2name[id(p)], "inputs": i, "outputs": dict(o)}
for p, i, o in good["trace"]
]
worse_trajectory = [
- dict(module_name=predictor2name[id(p)], inputs=i, outputs=dict(o))
+ {"module_name": predictor2name[id(p)], "inputs": i, "outputs": dict(o)}
for p, i, o in bad["trace"]
]
- kwargs = dict(
- program_code=inspect.getsource(system.__class__),
- modules_defn=inspect_modules(system),
- program_inputs={**example.inputs()},
- oracle_metadata={**example.labels()},
- better_program_trajectory=better_trajectory,
- better_program_outputs=dict(good["prediction"]),
- worse_program_trajectory=worse_trajectory,
- worse_program_outputs=dict(bad["prediction"] or {}),
- worse_reward_value=bad["score"],
- better_reward_value=good["score"],
- module_names=module_names,
- )
+ kwargs = {
+ "program_code": inspect.getsource(system.__class__),
+ "modules_defn": inspect_modules(system),
+ "program_inputs": {**example.inputs()},
+ "oracle_metadata": {**example.labels()},
+ "better_program_trajectory": better_trajectory,
+ "better_program_outputs": dict(good["prediction"]),
+ "worse_program_trajectory": worse_trajectory,
+ "worse_program_outputs": dict(bad["prediction"] or {}),
+ "worse_reward_value": bad["score"],
+ "better_reward_value": good["score"],
+ "module_names": module_names,
+ }
kwargs = {k: v if isinstance(v, str) else ujson.dumps(recursive_mask(v), indent=2)
for k, v in kwargs.items()}
@@ -174,7 +175,7 @@ def inspect_modules(program):
separator = "-" * 80
output = [separator]
- for idx, (name, predictor) in enumerate(program.named_predictors()):
+ for name, predictor in program.named_predictors():
signature = predictor.signature
instructions = textwrap.dedent(signature.instructions)
instructions = ("\n" + "\t" * 2).join([""] + instructions.splitlines())
diff --git a/dspy/teleprompt/teleprompt.py b/dspy/teleprompt/teleprompt.py
index 3c27226fb..74ed78eaf 100644
--- a/dspy/teleprompt/teleprompt.py
+++ b/dspy/teleprompt/teleprompt.py
@@ -1,5 +1,7 @@
-from typing import Optional, Any
-from dspy.primitives import Module, Example
+from typing import Any, Optional
+
+from dspy.primitives import Example, Module
+
class Teleprompter:
def __init__(self):
@@ -14,12 +16,12 @@ class Teleprompter:
trainset: The training set to use for optimization.
teacher: The teacher program to use for optimization.
valset: The validation set to use for optimization.
-
+
Returns:
The optimized student program.
"""
raise NotImplementedError
-
+
def get_params(self) -> dict[str, Any]:
"""
Get the parameters of the teleprompter.
diff --git a/dspy/teleprompt/teleprompt_optuna.py b/dspy/teleprompt/teleprompt_optuna.py
index b82d122dc..c733df0e5 100644
--- a/dspy/teleprompt/teleprompt_optuna.py
+++ b/dspy/teleprompt/teleprompt_optuna.py
@@ -10,7 +10,7 @@ class BootstrapFewShotWithOptuna(Teleprompter):
def __init__(
self,
metric,
- teacher_settings={},
+ teacher_settings=None,
max_bootstrapped_demos=4,
max_labeled_demos=16,
max_rounds=1,
@@ -18,7 +18,7 @@ class BootstrapFewShotWithOptuna(Teleprompter):
num_threads=None,
):
self.metric = metric
- self.teacher_settings = teacher_settings
+ self.teacher_settings = teacher_settings or {}
self.max_rounds = max_rounds
self.num_threads = num_threads
self.min_num_samples = 1
diff --git a/dspy/teleprompt/utils.py b/dspy/teleprompt/utils.py
index 954ce5604..b6578303f 100644
--- a/dspy/teleprompt/utils.py
+++ b/dspy/teleprompt/utils.py
@@ -5,6 +5,7 @@ import os
import random
import shutil
import sys
+
import numpy as np
try:
@@ -404,7 +405,7 @@ def old_getfile(object):
if inspect.ismodule(object):
if getattr(object, '__file__', None):
return object.__file__
- raise TypeError('{!r} is a built-in module'.format(object))
+ raise TypeError(f'{object!r} is a built-in module')
if inspect.isclass(object):
if hasattr(object, '__module__'):
module = sys.modules.get(object.__module__)
@@ -412,7 +413,7 @@ def old_getfile(object):
return module.__file__
if object.__module__ == '__main__':
raise OSError('source code not available')
- raise TypeError('{!r} is a built-in class'.format(object))
+ raise TypeError(f'{object!r} is a built-in class')
if inspect.ismethod(object):
object = object.__func__
if inspect.isfunction(object):
@@ -424,23 +425,22 @@ def old_getfile(object):
if inspect.iscode(object):
return object.co_filename
raise TypeError('module, class, method, function, traceback, frame, or '
- 'code object was expected, got {}'.format(
- type(object).__name__))
+ f'code object was expected, got {type(object).__name__}')
def new_getfile(object):
if not inspect.isclass(object):
return old_getfile(object)
-
+
# Lookup by parent module (as in current inspect)
if hasattr(object, '__module__'):
object_ = sys.modules.get(object.__module__)
if hasattr(object_, '__file__'):
return object_.__file__
-
+
# If parent module is __main__, lookup by methods (NEW)
- for name, member in inspect.getmembers(object):
+ for _, member in inspect.getmembers(object):
if inspect.isfunction(member) and object.__qualname__ + '.' + member.__name__ == member.__qualname__:
return inspect.getfile(member)
raise TypeError(f'Source for {object!r} not found')
-inspect.getfile = new_getfile
\ No newline at end of file
+inspect.getfile = new_getfile
diff --git a/dspy/teleprompt/vanilla.py b/dspy/teleprompt/vanilla.py
index 4f437f9c1..3d5118695 100644
--- a/dspy/teleprompt/vanilla.py
+++ b/dspy/teleprompt/vanilla.py
@@ -1,6 +1,6 @@
import random
-from .teleprompt import Teleprompter
+from dspy.teleprompt.teleprompt import Teleprompter
class LabeledFewShot(Teleprompter):