Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

updates to mipro

klopsahlong07550964c0ed
diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.py
index f7928ed17..82e5c78b4 100644
--- a/dspy/adapters/chat_adapter.py
+++ b/dspy/adapters/chat_adapter.py
@@ -1,7 +1,7 @@
import re
from .base import Adapter
-field_header_pattern = re.compile(r'\[\[\[\[ #### (\w+) #### \]\]\]\]')
+field_header_pattern = re.compile(r'\[\[\[ ### (\w+) ### \]\]\]')
class ChatAdapter(Adapter):
@@ -42,7 +42,7 @@ class ChatAdapter(Adapter):
def format_fields(fields):
- return '\n\n'.join([f"[[[[ #### {k} #### ]]]]\n{v}" for k, v in fields.items()]).strip()
+ return '\n\n'.join([f"[[[ ### {k} ### ]]]\n{v}" for k, v in fields.items()]).strip()
def format_chat_turn(field_names, values):
if not set(values).issuperset(set(field_names)):
diff --git a/dspy/propose/dataset_summary_generator.py b/dspy/propose/dataset_summary_generator.py
index 6c6d5fcaf..7a4c03a96 100644
--- a/dspy/propose/dataset_summary_generator.py
+++ b/dspy/propose/dataset_summary_generator.py
@@ -3,7 +3,6 @@ import re
import dspy
from dspy.propose.utils import strip_prefix
-
class ObservationSummarizer(dspy.Signature):
("""Given a series of observations I have made about my dataset, please summarize them into a brief 2-3 sentence summary which highlights only the most important details.""")
observations = dspy.InputField(desc="Observations I have made about my dataset")
@@ -46,7 +45,9 @@ def order_input_keys_in_string(unordered_repr):
return ordered_repr
def create_dataset_summary(trainset, view_data_batch_size, prompt_model, log_file=None, verbose=False):
+ if verbose: print("\nBootstrapping dataset summary (this will be used to generate instructions)...")
upper_lim = min(len(trainset), view_data_batch_size)
+ prompt_model = prompt_model if prompt_model else dspy.settings.lm
with dspy.settings.context(lm=prompt_model):
observation = dspy.Predict(DatasetDescriptor, n=1, temperature=1.0)(examples=order_input_keys_in_string(trainset[0:upper_lim].__repr__()))
observations = observation["observations"]
@@ -78,10 +79,15 @@ def create_dataset_summary(trainset, view_data_batch_size, prompt_model, log_fil
except Exception as e:
if verbose: print(f"e {e}. using observations from past round for a summary.")
- with dspy.settings.context(lm=prompt_model):
+ if prompt_model:
+ with dspy.settings.context(lm=prompt_model):
+ summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)
+ else:
summary = dspy.Predict(ObservationSummarizer, n=1, temperature=1.0)(observations=observations)
if verbose: print(f"summary: {summary}")
if log_file:
log_file.write(f"summary: {summary}\n")
+
+ if verbose: print(f"\nGenerated summary: {strip_prefix(summary.summary)}\n")
return strip_prefix(summary.summary)
diff --git a/dspy/propose/grounded_proposer.py b/dspy/propose/grounded_proposer.py
index 1378d6ca9..0fa17a48d 100644
--- a/dspy/propose/grounded_proposer.py
+++ b/dspy/propose/grounded_proposer.py
@@ -3,8 +3,8 @@ import re
import dspy
from dspy.propose.dataset_summary_generator import create_dataset_summary
-from dspy.propose.utils import create_example_string, create_predictor_level_history_string, strip_prefix
-from dspy.teleprompt.utils import get_signature
+from dspy.propose.utils import create_example_string, create_predictor_level_history_string, strip_prefix, get_dspy_source_code
+from dspy.teleprompt.utils import get_signature, get_prompt_model
from .propose_base import Proposer
@@ -186,21 +186,25 @@ class GenerateModuleInstruction(dspy.Module):
program_description = ""
module_code = ""
if self.program_aware:
- program_description = strip_prefix(
- self.describe_program(
- program_code=self.program_code_string, program_example=task_demos,
- ).program_description,
- )
- if self.verbose: print(f"PROGRAM DESCRIPTION: {program_description}")
-
- # Identify all modules
- init_pattern = r"def __init__\([\s\S]*?\):([\s\S]*?)(?=^\s*def|\Z)"
- init_content_match = re.search(init_pattern, self.program_code_string)
- init_content = init_content_match.group(0)
- pattern = r"^(.*dspy\.(ChainOfThought|Predict).*)$" # TODO: make it so that this extends out to any dspy Module
- matches = re.findall(pattern, init_content, re.MULTILINE)
- modules = [match[0].strip() for match in matches]
- module_code = modules[pred_i]
+ try:
+ program_description = strip_prefix(
+ self.describe_program(
+ program_code=self.program_code_string, program_example=task_demos,
+ ).program_description,
+ )
+ if self.verbose: print(f"PROGRAM DESCRIPTION: {program_description}")
+
+ # Identify all modules
+ init_pattern = r"def __init__\([\s\S]*?\):([\s\S]*?)(?=^\s*def|\Z)"
+ init_content_match = re.search(init_pattern, self.program_code_string)
+ init_content = init_content_match.group(0)
+ pattern = r"^(.*dspy\.(ChainOfThought|Predict).*)$" # TODO: make it so that this extends out to any dspy Module
+ matches = re.findall(pattern, init_content, re.MULTILINE)
+ modules = [match[0].strip() for match in matches]
+ module_code = modules[pred_i]
+ except:
+ if self.verbose: print("Error getting program description. Running without program aware proposer.")
+ self.program_aware = False
module_description = self.describe_module(
program_code=self.program_code_string,
@@ -236,8 +240,8 @@ class GroundedProposer(Proposer):
def __init__(
self,
prompt_model,
+ program,
trainset,
- program_code_string=None,
view_data_batch_size=10,
use_dataset_summary=True,
program_aware=True,
@@ -258,8 +262,17 @@ class GroundedProposer(Proposer):
self.set_history_randomly=set_history_randomly
self.verbose = verbose
- self.prompt_model = prompt_model
- self.program_code_string = program_code_string
+ self.prompt_model = get_prompt_model(prompt_model)
+ if self.program_aware:
+ try:
+ self.program_code_string = get_dspy_source_code(program)
+ if self.verbose: print("SOURCE CODE:",self.program_code_string)
+ except Exception as e:
+ print(f"Error getting source code: {e}.\n\nRunning without program aware proposer.")
+ self.program_code_string = None
+ self.program_aware = False
+ else:
+ self.program_code_string = None
self.data_summary = create_dataset_summary(
trainset=trainset, view_data_batch_size=view_data_batch_size, prompt_model=prompt_model,
)
@@ -270,7 +283,6 @@ class GroundedProposer(Proposer):
trainset,
program,
demo_candidates,
- prompt_model,
trial_logs,
N,
T,
@@ -306,7 +318,7 @@ class GroundedProposer(Proposer):
program=program,
predictor=predictor,
pred_i=pred_i,
- prompt_model=prompt_model,
+ # prompt_model=prompt_model,
T=T,
demo_candidates=demo_candidates,
demo_set_i=demo_set_i,
@@ -321,7 +333,7 @@ class GroundedProposer(Proposer):
program,
predictor,
pred_i,
- prompt_model,
+ # prompt_model,
T,
demo_candidates,
demo_set_i,
@@ -347,9 +359,10 @@ class GroundedProposer(Proposer):
)
# Generate a new instruction for our predictor, using the temperature specified for this round
- original_temp = prompt_model.kwargs["temperature"]
- with dspy.settings.context(lm=prompt_model):
- prompt_model.kwargs["temperature"] = T
+ original_temp = self.prompt_model.kwargs["temperature"]
+
+ with dspy.settings.context(lm=self.prompt_model):
+ self.prompt_model.kwargs["temperature"] = T
proposed_instruction = instruction_generator.forward(
demo_candidates=demo_candidates,
pred_i=pred_i,
@@ -359,10 +372,10 @@ class GroundedProposer(Proposer):
previous_instructions=instruction_history,
tip=tip,
).proposed_instruction
- prompt_model.kwargs["temperature"] = original_temp
+ self.prompt_model.kwargs["temperature"] = original_temp
# Log the trace used to generate the new instruction, along with the new instruction itself
- prompt_model.inspect_history(n=1)
+ if self.verbose: self.prompt_model.inspect_history(n=1)
if self.verbose: print(f"PROPOSED INSTRUCTION: {proposed_instruction}")
return strip_prefix(proposed_instruction)
diff --git a/dspy/propose/utils.py b/dspy/propose/utils.py
index 736e1b925..99e146f71 100644
--- a/dspy/propose/utils.py
+++ b/dspy/propose/utils.py
@@ -1,8 +1,16 @@
import json
import re
+import dspy
+import inspect
+try:
+ from IPython.core.magics.code import extract_symbols
+except ImportError:
+ # Won't be able to read code from juptyer notebooks
+ extract_symbols = None
-from dspy.teleprompt.utils import get_signature
+from dspy.predict.parameter import Parameter
+from dspy.teleprompt.utils import get_signature
def strip_prefix(text):
pattern = r'^[\*\s]*(([\w\'\-]+\s+){0,4}[\w\'\-]+):\s*'
@@ -131,4 +139,45 @@ def create_example_string(fields, example):
output.append(field_str)
# Joining all the field strings
- return '\n'.join(output)
\ No newline at end of file
+ return '\n'.join(output)
+
+def get_dspy_source_code(module):
+ header = []
+ base_code = ""
+
+ # Don't get source code for Predict or ChainOfThought modules (NOTE we will need to extend this list as more DSPy.modules are added)
+ if not type(module).__name__ == "Predict" and not type(module).__name__ == "ChainOfThought":
+ try:
+ base_code = inspect.getsource(type(module))
+ except TypeError:
+ obj = type(module)
+ cell_code = "".join(inspect.linecache.getlines(new_getfile(obj)))
+ class_code = extract_symbols(cell_code, obj.__name__)[0][0]
+ base_code = str(class_code)
+
+ completed_set = set()
+ for attribute in module.__dict__.keys():
+ try:
+ iterable = iter(getattr(module, attribute))
+ except TypeError:
+ iterable = [getattr(module, attribute)]
+
+ for item in iterable:
+ if item in completed_set:
+ continue
+ if isinstance(item, Parameter):
+ if hasattr(item, 'signature') and item.signature is not None and item.signature.__pydantic_parent_namespace__['signature_name'] + "_sig" not in completed_set:
+ try:
+ header.append(inspect.getsource(item.signature))
+ print(inspect.getsource(item.signature))
+ except (TypeError, OSError):
+ header.append(str(item.signature))
+ completed_set.add(item.signature.__pydantic_parent_namespace__['signature_name'] + "_sig")
+ if isinstance(item, dspy.Module):
+ code = get_dspy_source_code(item).strip()
+ if code not in completed_set:
+ header.append(code)
+ completed_set.add(code)
+ completed_set.add(item)
+
+ return '\n\n'.join(header) + '\n\n' + base_code
\ No newline at end of file
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.py
index 18239d98f..95e04bc47 100644
--- a/dspy/teleprompt/mipro_optimizer_v2.py
+++ b/dspy/teleprompt/mipro_optimizer_v2.py
@@ -8,6 +8,7 @@ from collections import defaultdict
import numpy as np
import optuna
+import dspy
from dspy.evaluate.evaluate import Evaluate
from dspy.propose import GroundedProposer
@@ -15,7 +16,6 @@ from dspy.teleprompt.teleprompt import Teleprompter
from dspy.teleprompt.utils import (
create_n_fewshot_demo_sets,
eval_candidate_program,
- get_dspy_source_code,
get_program_with_highest_avg_score,
get_signature,
get_task_model_history_for_full_example,
@@ -23,7 +23,7 @@ from dspy.teleprompt.utils import (
save_candidate_program,
save_file_to_log_dir,
set_signature,
- setup_logging,
+ # setup_logging,
)
try:
@@ -41,7 +41,7 @@ from dspy.teleprompt import MIPROv2
teleprompter = MIPROv2(prompt_model=prompt_model, task_model=task_model, metric=metric, num_candidates=10, init_temperature=1.0)
kwargs = dict(num_threads=NUM_THREADS, display_progress=True, display_table=0)
-compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_batches=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)
+compiled_prompt_opt = teleprompter.compile(program, trainset=trainset[:TRAIN_NUM], num_trials=100, max_bootstrapped_demos=3, max_labeled_demos=5, eval_kwargs=kwargs)
eval_score = evaluate(compiled_prompt_opt, devset=evalset[:EVAL_NUM], **kwargs)
```
@@ -73,100 +73,86 @@ Note that this teleprompter takes in the following parameters:
BOOTSTRAPPED_FEWSHOT_EXAMPLES_IN_CONTEXT = 3
LABELED_FEWSHOT_EXAMPLES_IN_CONTEXT = 0
-MB_FULL_EVAL_STEPS = 10
-MINIBATCH_SIZE = 25#50
-
class MIPROv2(Teleprompter):
def __init__(
self,
+ metric,
prompt_model=None,
task_model=None,
teacher_settings={},
num_candidates=10,
- metric=None,
- init_temperature=1.4,
+ num_threads=6,
+ max_errors=10,
+ init_temperature=0.5,
verbose=False,
track_stats=True,
log_dir=None,
- view_data_batch_size=10,
- minibatch_size=MINIBATCH_SIZE,
- minibatch_full_eval_steps=MB_FULL_EVAL_STEPS,
metric_threshold=None,
):
- self.n = num_candidates
+ self.num_candidates = num_candidates
self.metric = metric
self.init_temperature = init_temperature
- self.prompt_model = prompt_model
- self.task_model = task_model
+ self.prompt_model = prompt_model if prompt_model else dspy.settings.lm
+ self.task_model = task_model if task_model else dspy.settings.lm
self.verbose = verbose
self.track_stats = track_stats
self.log_dir = log_dir
- self.view_data_batch_size = view_data_batch_size
self.teacher_settings = teacher_settings
self.prompt_model_total_calls = 0
self.total_calls = 0
- self.minibatch_size = minibatch_size
- self.minibatch_full_eval_steps = minibatch_full_eval_steps
- self.metric_threshold = None
+ self.num_threads = num_threads
+ self.max_errors = max_errors
+ self.metric_threshold = metric_threshold
# Check if WANDB_RUN_ID is set in the environment
self.wandb_run_id = None
-
- def _get_batch_size(
- self,
- minibatch,
- trainset,
- ):
- if minibatch:
- return self.minibatch_size
- else:
- return len(trainset)
def compile(
self,
student,
*,
trainset,
- valset=None,
- num_batches=30,
- max_bootstrapped_demos=5,
- max_labeled_demos=2,
- eval_kwargs={},
+ num_trials=30,
+ max_bootstrapped_demos=4,
+ max_labeled_demos=16,
seed=9,
minibatch=True,
program_aware_proposer=True,
requires_permission_to_run=True,
+ view_data_batch_size=10,
+ minibatch_size=25, # TODO: more intelligently set these defaults
+ minibatch_full_eval_steps=10,
):
# Define ANSI escape codes for colors
YELLOW = "\033[93m"
+ GREEN = "\033[92m"
BLUE = "\033[94m"
BOLD = "\033[1m"
ENDC = "\033[0m" # Resets the color to default
random.seed(seed)
- valset = valset or trainset
- estimated_prompt_model_calls = 10 + self.n * len(
+ estimated_prompt_model_calls = 10 + self.num_candidates * len(
student.predictors(),
) + (0 if not program_aware_proposer else len(student.predictors()) + 1) # num data summary calls + N * P + (P + 1)
prompt_model_line = ""
if not program_aware_proposer:
- prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.n}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""
+ prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.num_candidates}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""
else:
- prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.n}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program + ({BLUE}{BOLD}{len(student.predictors()) + 1}{ENDC}{YELLOW}) lm calls in program aware proposer = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""
+ prompt_model_line = f"""{YELLOW}- Prompt Model: {BLUE}{BOLD}10{ENDC}{YELLOW} data summarizer calls + {BLUE}{BOLD}{self.num_candidates}{ENDC}{YELLOW} * {BLUE}{BOLD}{len(student.predictors())}{ENDC}{YELLOW} lm calls in program + ({BLUE}{BOLD}{len(student.predictors()) + 1}{ENDC}{YELLOW}) lm calls in program aware proposer = {BLUE}{BOLD}{estimated_prompt_model_calls}{ENDC}{YELLOW} prompt model calls{ENDC}"""
estimated_task_model_calls_wo_module_calls = 0
task_model_line = ""
if not minibatch:
- estimated_task_model_calls_wo_module_calls = len(trainset) * num_batches # M * T * P
- task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in train set * {BLUE}{BOLD}{num_batches}{ENDC}{YELLOW} batches * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}"""
+ estimated_task_model_calls_wo_module_calls = len(trainset) * num_trials # M * T * P
+ task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in train set * {BLUE}{BOLD}{num_trials}{ENDC}{YELLOW} batches * {BLUE}{BOLD}# of LM calls in your program{ENDC}{YELLOW} = ({BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls} * # of LM calls in your program{ENDC}{YELLOW}) task model calls{ENDC}"""
else:
- estimated_task_model_calls_wo_module_calls = self.minibatch_size * num_batches + (len(trainset) * (num_batches // self.minibatch_full_eval_steps)) # B * T * P
- task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{self.minibatch_size}{ENDC}{YELLOW} examples in minibatch * {BLUE}{BOLD}{num_batches}{ENDC}{YELLOW} batches + {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in train set * {BLUE}{BOLD}{num_batches // self.minibatch_full_eval_steps}{ENDC}{YELLOW} full evals = {BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls}{ENDC}{YELLOW} task model calls{ENDC}"""
+ estimated_task_model_calls_wo_module_calls = minibatch_size * num_trials + (len(trainset) * (num_trials // minibatch_full_eval_steps)) # B * T * P
+ task_model_line = f"""{YELLOW}- Task Model: {BLUE}{BOLD}{minibatch_size}{ENDC}{YELLOW} examples in minibatch * {BLUE}{BOLD}{num_trials}{ENDC}{YELLOW} batches + {BLUE}{BOLD}{len(trainset)}{ENDC}{YELLOW} examples in train set * {BLUE}{BOLD}{num_trials // minibatch_full_eval_steps}{ENDC}{YELLOW} full evals = {BLUE}{BOLD}{estimated_task_model_calls_wo_module_calls}{ENDC}{YELLOW} task model calls{ENDC}"""
user_message = textwrap.dedent(f"""\
- {YELLOW}{BOLD}WARNING: Projected Language Model (LM) Calls{ENDC}
+ {YELLOW}{BOLD}Projected Language Model (LM) Calls{ENDC}
Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows:
@@ -182,22 +168,23 @@ class MIPROv2(Teleprompter):
For a preliminary estimate of potential costs, we recommend you perform your own calculations based on the task
and prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider:
- {YELLOW}- Reducing the number of trials (`num_batches`), the size of the trainset, or the number of LM calls in your program.{ENDC}
- {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}""")
+ {YELLOW}- Reducing the number of trials (`num_trials`), the size of the trainset, or the number of LM calls in your program.{ENDC}
+ {YELLOW}- Using a cheaper task model to optimize the prompt.{ENDC}\n""")
user_confirmation_message = textwrap.dedent(f"""\
To proceed with the execution of this program, please confirm by typing {BLUE}'y'{ENDC} for yes or {BLUE}'n'{ENDC} for no.
- If you would like to bypass this confirmation step in future executions, set the {YELLOW}`requires_permission_to_run`{ENDC} flag to {YELLOW}`False` when calling compile.{ENDC}
+ If you would like to bypass this confirmation step in future executions, set the {YELLOW}`requires_permission_to_run`{ENDC} flag to {YELLOW}`False`{ENDC} when calling compile.
{YELLOW}Awaiting your input...{ENDC}
""")
- print(user_message)
+ if requires_permission_to_run: print(user_message)
sys.stdout.flush() # Flush the output buffer to force the message to print
run = True
+ # TODO: make sure these estimates are good for mini-batching
if requires_permission_to_run:
print(user_confirmation_message)
user_input = input("Do you wish to continue? (y/n): ").strip().lower()
@@ -206,53 +193,20 @@ class MIPROv2(Teleprompter):
run = False
if run:
- if program_aware_proposer:
- try:
- self.program_code_string = get_dspy_source_code(student)
- if self.verbose:
- if self.verbose: print("SOURCE CODE:",self.program_code_string)
- except Exception as e:
- print(f"Error getting source code: {e}.\n\nRunning without program aware proposer.")
- self.program_code_string = None
- program_aware_proposer = False
- else:
- self.program_code_string = None
-
- # Setup our proposer
- proposer = GroundedProposer(
- trainset=trainset,
- prompt_model=self.prompt_model,
- program_code_string=self.program_code_string,
- view_data_batch_size=self.view_data_batch_size,
- program_aware=program_aware_proposer,
- )
-
- # Setup logging
- logging.getLogger('openai').setLevel(logging.WARNING)
- logging.basicConfig(level=logging.WARNING)
- setup_logging(self.log_dir)
-
- self.wandb_run_id = os.getenv("WANDB_RUN_ID", None)
-
- if self.wandb_run_id:
- # Initialize wandb with the same run ID
- wandb.init(
- project="prompt_optimizers",
- id=self.wandb_run_id,
- resume="must",
- )
-
# Setup random seeds
random.seed(seed)
np.random.seed(seed)
- # Log current file to log_dir
- curr_file = os.path.abspath(__file__)
- save_file_to_log_dir(curr_file, self.log_dir)
-
# Set up program and evaluation function
program = student.deepcopy()
- evaluate = Evaluate(devset=trainset, metric=self.metric, **eval_kwargs)
+ evaluate = Evaluate(
+ devset=trainset,
+ metric=self.metric,
+ num_threads=self.num_threads,
+ max_errors=self.max_errors,
+ display_table=False,
+ display_progress=True,
+ )
# Determine the number of fewshot examples to use to generate demos for prompt
if max_bootstrapped_demos == 0 and max_labeled_demos == 0:
@@ -261,20 +215,24 @@ class MIPROv2(Teleprompter):
else:
max_bootstrapped_demos_for_candidate_gen = max_bootstrapped_demos
max_labeled_demos_for_candidate_gen = max_labeled_demos
-
- # # Generate N few shot example sets
- # if not demo_candidates and not instruction_candidates:
- # demo_candidates = create_n_fewshot_example_sets(program=program, trainset=trainset, n=self.n, hard_fewshot=hard_fewshot, metric=self.metric, teacher_settings=self.teacher_settings, max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen, max_labeled_demos=max_labeled_demos_for_candidate_gen)
-
- # Generate N few shot example sets
+
+ # Generate N few shot example sets (these will inform instruction creation, and be used as few-shot examples in our prompt)
+ print(f"Beginning MIPROv2 optimization process...")
+ print(f"\n==> STEP 1: BOOTSTRAP FEWSHOT EXAMPLES <==")
+ if max_bootstrapped_demos > 0:
+ print(f"These will be used for as few-shot examples candidates for our program and for creating instructions.\n")
+ else:
+ print(f"These will be used for informing instruction proposal.\n")
+ print(f"Bootstrapping N={self.num_candidates} sets of demonstrations...")
try:
demo_candidates = create_n_fewshot_demo_sets(
student=program,
- num_candidate_sets=self.n,
+ num_candidate_sets=self.num_candidates,
trainset=trainset,
max_labeled_demos=max_labeled_demos_for_candidate_gen,
max_bootstrapped_demos=max_bootstrapped_demos_for_candidate_gen,
metric=self.metric,
+ max_errors=self.max_errors,
teacher_settings=self.teacher_settings,
seed=seed,
metric_threshold=self.metric_threshold,
@@ -283,38 +241,48 @@ class MIPROv2(Teleprompter):
print(f"Error generating fewshot examples: {e}")
print("Running without fewshot examples.")
demo_candidates = None
+
+ # Generate N candidate instructions
- # Generate N candidate prompts
- proposer.program_aware = program_aware_proposer
- proposer.use_tip = True
- proposer.use_instruct_history = False
- proposer.set_history_randomly = False
+ # Setup our proposer
+ print(f"\n==> STEP 2: PROPOSE INSTRUCTION CANDIDATES <==")
+ print("In this step, by default we will use the few-shot examples from the previous step, a generated dataset summary, and a summary of the program code, and a randomly selected prompting tip to propose instructions.")
+
+ proposer = GroundedProposer(
+ program=program,
+ trainset=trainset,
+ prompt_model=self.prompt_model,
+ # program_code_string=self.program_code_string,
+ view_data_batch_size=view_data_batch_size,
+ program_aware=program_aware_proposer,
+ use_instruct_history=False,
+ set_history_randomly=False,
+ verbose = self.verbose,
+ )
+
+ print(f"\nProposing instructions...\n")
instruction_candidates = proposer.propose_instructions_for_program(
trainset=trainset,
program=program,
demo_candidates=demo_candidates,
- N=self.n,
- prompt_model=self.prompt_model,
+ N=self.num_candidates,
T=self.init_temperature,
trial_logs={},
)
for i, pred in enumerate(program.predictors()):
+ print(f"Proposed Instructions for Predictor {i}:\n")
instruction_candidates[i][0] = get_signature(pred).instructions
+ for j, instruction in enumerate(instruction_candidates[i]):
+ print(f"{j}: {instruction}\n")
+ print("\n")
- # instruction_candidates[1][0] = "Given the question, and context, respond with the number of the document that is most relevant to answering the question in the field 'Answer' (ex. Answer: '3')."
-
- # Save the candidate instructions generated
- if self.log_dir:
- fp = os.path.join(self.log_dir, "instructions_to_save.pickle")
- with open(fp, "wb") as file:
- pickle.dump(instruction_candidates, file)
-
- # If we're doing zero-shot, reset demo_candidates to none
+ # If we're doing zero-shot, reset demo_candidates to none now that we've used them for instruction proposal
if max_bootstrapped_demos == 0 and max_labeled_demos == 0:
demo_candidates = None
- # Initialize variables to store the best program and its score
+ # Initialize variables to track during the optimization process
best_score = float("-inf")
+ best_scoring_trial = 0
best_program = None
trial_logs = {}
total_eval_calls = 0
@@ -330,11 +298,14 @@ class MIPROv2(Teleprompter):
trainset,
):
def objective(trial):
- nonlocal best_program, best_score, trial_logs, total_eval_calls # Allow access to the outer variables
+ nonlocal best_program, best_score, best_scoring_trial, trial_logs, total_eval_calls # Allow access to the outer variables
# Kick off trial
- logging.info(f"Starting trial num: {trial.number}")
- trial_logs[trial.number] = {}
+ if minibatch:
+ print(f"== Minibatch Trial {trial.number+1} / {num_trials}==")
+ else:
+ print(f"===== Trial {trial.number+1} / {num_trials} =====")
+ trial_logs[trial.number+1] = {}
# Create a new candidate program
candidate_program = baseline_program.deepcopy()
@@ -353,23 +324,24 @@ class MIPROv2(Teleprompter):
f"{i}_predictor_instruction",
range(len(p_instruction_candidates)),
)
- chosen_params.append(instruction_idx)
+ # chosen_params.append(instruction_idx)
+ chosen_params.append(f"Predictor {i+1}: Instruction {instruction_idx}")
if demo_candidates:
demos_idx = trial.suggest_categorical(
f"{i}_predictor_demos", range(len(p_demo_candidates)),
)
- chosen_params.append(demo_candidates)
+ chosen_params.append(f"Predictor {i+1}: Few-Shot Set {demos_idx}")
# Log the selected instruction / demo candidate
- trial_logs[trial.number][
+ trial_logs[trial.number+1][
f"{i}_predictor_instruction"
] = instruction_idx
if demo_candidates:
- trial_logs[trial.number][f"{i}_predictor_demos"] = demos_idx
+ trial_logs[trial.number+1][f"{i}_predictor_demos"] = demos_idx
- logging.info(f"instruction_idx {instruction_idx}")
+ dspy.logger.debug(f"instruction_idx {instruction_idx}")
if demo_candidates:
- logging.info(f"demos_idx {demos_idx}")
+ dspy.logger.debug(f"demos_idx {demos_idx}")
# Set the instruction
selected_instruction = p_instruction_candidates[instruction_idx]
@@ -383,119 +355,115 @@ class MIPROv2(Teleprompter):
p_new.demos = p_demo_candidates[demos_idx]
# Log assembled program
- if self.verbose: print("CANDIDATE PROGRAM:")
+ if self.verbose: print("Evaluating the following candidate program...\n")
if self.verbose: print_full_program(candidate_program)
- if self.verbose: print("...")
# Save the candidate program
- trial_logs[trial.number]["program_path"] = save_candidate_program(
- candidate_program, self.log_dir, trial.number,
+ trial_logs[trial.number+1]["program_path"] = save_candidate_program(
+ candidate_program, self.log_dir, trial.number+1,
)
- trial_logs[trial.number]["num_eval_calls"] = 0
+ trial_logs[trial.number+1]["num_eval_calls"] = 0
# Evaluate the candidate program with relevant batch size
- batch_size = self._get_batch_size(minibatch, trainset)
+ batch_size = minibatch_size if minibatch else len(trainset)
score = eval_candidate_program(
batch_size, trainset, candidate_program,
evaluate,
)
- if self.verbose: print(f"SCORE {score}, BATCH SIZE {batch_size}")
# Print out a full trace of the program in use
if self.verbose:
- print("FULL TRACE")
+ print("Full trace of prompts in use on an example...")
get_task_model_history_for_full_example(
candidate_program, self.task_model, trainset, evaluate,
)
- print("...")
# Log relevant information
- if self.verbose: print(f"Score {score}")
categorical_key = ",".join(map(str, chosen_params))
param_score_dict[categorical_key].append(
(score, candidate_program),
)
- trial_logs[trial.number]["num_eval_calls"] = batch_size
- trial_logs[trial.number]["full_eval"] = batch_size >= len(trainset)
- trial_logs[trial.number]["score"] = score
- trial_logs[trial.number]["pruned"] = False
- total_eval_calls += trial_logs[trial.number]["num_eval_calls"]
- trial_logs[trial.number]["total_eval_calls_so_far"] = total_eval_calls
- trial_logs[trial.number]["program"] = candidate_program.deepcopy()
- if self.wandb_run_id:
- wandb.log(
- {
- "score": score,
- "num_eval_calls": trial_logs[trial.number]["num_eval_calls"],
- "total_eval_calls": total_eval_calls,
- },
- )
-
- # Update the best program if the current score is better, and if we're not using minibatching
+ trial_logs[trial.number+1]["num_eval_calls"] = batch_size
+ trial_logs[trial.number+1]["full_eval"] = batch_size >= len(trainset)
+ trial_logs[trial.number+1]["score"] = score
+ trial_logs[trial.number+1]["pruned"] = False
+ total_eval_calls += trial_logs[trial.number+1]["num_eval_calls"]
+ trial_logs[trial.number+1]["total_eval_calls_so_far"] = total_eval_calls
+ trial_logs[trial.number+1]["program"] = candidate_program.deepcopy()
+
+ # If this score was from a full evaluation, update the best program if the new score is better
best_score_updated = False
- if score > best_score and trial_logs[trial.number]["full_eval"] and not minibatch:
- if self.verbose: print("Updating best score")
+ if score > best_score and trial_logs[trial.number+1]["full_eval"] and not minibatch:
best_score = score
+ best_scoring_trial = trial.number+1
best_program = candidate_program.deepcopy()
best_score_updated = True
+
+ if minibatch:
+ print(f"Score: {score} on minibatch of size {batch_size} with parameters {chosen_params}.\n\n")
+ else:
+ print(f"Score: {score} with parameters {chosen_params}.")
+ if best_score_updated:
+ print(f"{GREEN}New best score updated!{ENDC} Score: {best_score} on trial {best_scoring_trial}.\n\n")
+ else:
+ print(f"Best score so far: {best_score} on trial {best_scoring_trial}.\n\n")
# If we're doing minibatching, check to see if it's time to do a full eval
- if minibatch and trial.number % self.minibatch_full_eval_steps == 0:
+ if minibatch and trial.number+1 % minibatch_full_eval_steps == 0:
+ print(f"===== Full Eval {trial.number+1 // minibatch_full_eval_steps} =====")
# Save old information as the minibatch version
- trial_logs[trial.number]["mb_score"] = score
- trial_logs[trial.number]["mb_program_path"] = trial_logs[trial.number]["program_path"]
+ trial_logs[trial.number+1]["mb_score"] = score
+ trial_logs[trial.number+1]["mb_program_path"] = trial_logs[trial.number+1]["program_path"]
# Identify our best program (based on mean of scores so far, and do a full eval on it)
- highest_mean_program, combo_key = get_program_with_highest_avg_score(param_score_dict, fully_evaled_param_combos)
+ highest_mean_program, mean, combo_key = get_program_with_highest_avg_score(param_score_dict, fully_evaled_param_combos)
full_train_score = eval_candidate_program(
len(trainset), trainset, highest_mean_program, evaluate,
)
+ if trial.number+1 // minibatch_full_eval_steps > 0:
+ print(f"Doing full eval on next top averaging program (Avg Score: {mean}) so far from mini-batch trials...")
+ else:
+ print(f"Doing full eval on top averaging program (Avg Score: {mean}) so far from mini-batch trials...")
+
# Log relevant information
fully_evaled_param_combos[combo_key] = {"program":highest_mean_program, "score": full_train_score}
total_eval_calls += len(trainset)
- trial_logs[trial.number]["total_eval_calls_so_far"] = total_eval_calls
- trial_logs[trial.number]["full_eval"] = True
- trial_logs[trial.number]["program_path"] = save_candidate_program(
- program=highest_mean_program, log_dir=self.log_dir, trial_num=trial.number, note="full_eval",
+ trial_logs[trial.number+1]["total_eval_calls_so_far"] = total_eval_calls
+ trial_logs[trial.number+1]["full_eval"] = True
+ trial_logs[trial.number+1]["program_path"] = save_candidate_program(
+ program=highest_mean_program, log_dir=self.log_dir, trial_num=trial.number+1, note="full_eval",
)
- trial_logs[trial.number]["score"] = full_train_score
+ trial_logs[trial.number+1]["score"] = full_train_score
if full_train_score > best_score:
- if self.verbose: print(f"UPDATING BEST SCORE WITH {full_train_score}")
+ print(f"{GREEN}Best full eval score so far!{ENDC} Score: {full_train_score}")
best_score = full_train_score
+ best_scoring_trial = trial.number+1
best_program = highest_mean_program.deepcopy()
best_score_updated = True
+ else:
+ print(f"Full eval score: {full_train_score}")
+ print(f"Best full eval score so far: {best_score}")
+ print("=======================\n\n")
- # If the best score was updated, do a full eval on the dev set
- if best_score_updated:
- full_dev_score = evaluate(
- best_program,
- devset=valset,
- display_table=0,
- )
- if self.wandb_run_id:
- wandb.log(
- {
- "best_prog_so_far_train_score": best_score,
- "best_prog_so_far_dev_score": full_dev_score,
- },
- )
-
return score
return objective
# Run the trial
+ optuna.logging.set_verbosity(optuna.logging.WARNING)
+ print(f"\n==> STEP 3: FINDING OPTIMAL PROMPT PARAMETERS <==")
+ print("In this step, we will evaluate the program over a series of trials with different combinations of instructions and few-shot examples to find the optimal combination. Bayesian Optimization will be used for this search process.\n")
objective_function = create_objective(
program, instruction_candidates, demo_candidates, evaluate, trainset,
)
sampler = optuna.samplers.TPESampler(seed=seed, multivariate=True)
study = optuna.create_study(direction="maximize", sampler=sampler)
- _ = study.optimize(objective_function, n_trials=num_batches)
+ _ = study.optimize(objective_function, n_trials=num_trials)
if best_program is not None and self.track_stats:
best_program.trial_logs = trial_logs
@@ -503,13 +471,6 @@ class MIPROv2(Teleprompter):
best_program.prompt_model_total_calls = self.prompt_model_total_calls
best_program.total_calls = self.total_calls
- if self.log_dir:
- program_file_path = os.path.join(self.log_dir, "best_program")
- best_program.save(program_file_path)
-
- optuna_study_file_path = os.path.join(self.log_dir, "optuna_study.pickle")
- with open(optuna_study_file_path, "wb") as file:
- pickle.dump(study, file)
-
return best_program
+
return student
diff --git a/dspy/teleprompt/utils.py b/dspy/teleprompt/utils.py
index 0d7000293..7973f9b89 100644
--- a/dspy/teleprompt/utils.py
+++ b/dspy/teleprompt/utils.py
@@ -44,13 +44,12 @@ def eval_candidate_program(batch_size, trainset, candidate_program, evaluate):
"""Evaluate a candidate program on the trainset, using the specified batch size."""
# Evaluate on the full trainset
if batch_size >= len(trainset):
- score = evaluate(candidate_program, devset=trainset, display_table=0)
+ score = evaluate(candidate_program, devset=trainset)
# Or evaluate on a minibatch
else:
score = evaluate(
candidate_program,
devset=create_minibatch(trainset, batch_size),
- display_table=0,
)
return score
@@ -120,12 +119,10 @@ def get_program_with_highest_avg_score(param_score_dict, fully_evaled_param_comb
if key in fully_evaled_param_combos:
continue
- print(f"Best Combination: {key} with Mean = {mean}")
-
- return program, key
+ return program, mean, key
# If no valid program is found, we return the last valid one that we found
- return program, key
+ return program, mean, key
def calculate_last_n_proposed_quality(
@@ -184,7 +181,7 @@ def print_full_program(program):
print(f"i: {get_signature(predictor).instructions}")
*_, last_field = get_signature(predictor).fields.values()
print(f"p: {last_field.json_schema_extra['prefix']}")
- print("\n")
+ print("\n")
def save_candidate_program(program, log_dir, trial_num, note=None):
@@ -246,6 +243,12 @@ def setup_logging(log_dir):
### OTHER UTILS ###
+def get_prompt_model(prompt_model):
+ if prompt_model:
+ return prompt_model
+ else:
+ return dspy.settings.lm
+
def get_signature(predictor):
if hasattr(predictor, "extended_signature"):
return predictor.extended_signature
@@ -269,6 +272,7 @@ def create_n_fewshot_demo_sets(
max_bootstrapped_demos,
metric,
teacher_settings,
+ max_errors=10,
max_rounds=1,
labeled_sample=True,
min_num_samples=1,
@@ -297,6 +301,8 @@ def create_n_fewshot_demo_sets(
# Go through and create each candidate set
for seed in range(-3, num_candidate_sets):
+ print(f"Bootstrapping set {seed+4}/{num_candidate_sets+3}")
+
trainset2 = list(trainset)
if seed == -3 and include_non_bootstrapped:
@@ -318,6 +324,7 @@ def create_n_fewshot_demo_sets(
# unshuffled few-shot
program = BootstrapFewShot(
metric=metric,
+ max_errors=max_errors,
max_bootstrapped_demos=max_bootstrapped_demos,
max_labeled_demos=max_labeled_demos,
teacher_settings=teacher_settings,
@@ -332,6 +339,7 @@ def create_n_fewshot_demo_sets(
teleprompter = BootstrapFewShot(
metric=metric,
+ max_errors=max_errors,
metric_threshold=metric_threshold,
max_bootstrapped_demos=size,
max_labeled_demos=max_labeled_demos,
@@ -392,45 +400,4 @@ def new_getfile(object):
return inspect.getfile(member)
raise TypeError(f'Source for {object!r} not found')
-inspect.getfile = new_getfile
-
-def get_dspy_source_code(module):
- header = []
- base_code = ""
-
- # Don't get source code for Predict or ChainOfThought modules (NOTE we will need to extend this list as more DSPy.modules are added)
- if not type(module).__name__ == "Predict" and not type(module).__name__ == "ChainOfThought":
- try:
- base_code = inspect.getsource(type(module))
- except TypeError:
- obj = type(module)
- cell_code = "".join(inspect.linecache.getlines(new_getfile(obj)))
- class_code = extract_symbols(cell_code, obj.__name__)[0][0]
- base_code = str(class_code)
-
- completed_set = set()
- for attribute in module.__dict__.keys():
- try:
- iterable = iter(getattr(module, attribute))
- except TypeError:
- iterable = [getattr(module, attribute)]
-
- for item in iterable:
- if item in completed_set:
- continue
- if isinstance(item, Parameter):
- if hasattr(item, 'signature') and item.signature is not None and item.signature.__pydantic_parent_namespace__['signature_name'] + "_sig" not in completed_set:
- try:
- header.append(inspect.getsource(item.signature))
- print(inspect.getsource(item.signature))
- except (TypeError, OSError):
- header.append(str(item.signature))
- completed_set.add(item.signature.__pydantic_parent_namespace__['signature_name'] + "_sig")
- if isinstance(item, dspy.Module):
- code = get_dspy_source_code(item).strip()
- if code not in completed_set:
- header.append(code)
- completed_set.add(code)
- completed_set.add(item)
-
- return '\n\n'.join(header) + '\n\n' + base_code
\ No newline at end of file
+inspect.getfile = new_getfile
\ No newline at end of file
diff --git a/examples/optimizers/mipro_test.py b/examples/optimizers/mipro_test.py
new file mode 100644
index 000000000..bcca24f9a
--- /dev/null
+++ b/examples/optimizers/mipro_test.py
@@ -0,0 +1,118 @@
+import dspy
+import os;
+os.environ['LITELLM_LOG'] = 'INFO'
+from datetime import datetime
+
+# lm = dspy.LM("gpt-4o-mini-2024-07-18", api_base=os.getenv('OPENAI_API_BASE'), api_key=os.getenv('OPENAI_API_KEY'))
+# lm = dspy.HFClientVLLM(
+# model="meta-llama/Meta-Llama-3-8B",
+# port=7410,
+# url=["http://future-hgx-1:7501"],
+# max_tokens=250,
+# stop=["\n\n", "\n---", "assistant"],
+# )
+
+# lm = dspy.LM("openai/meta-llama/Meta-Llama-3-8B", api_base="http://future-hgx-1:7501/v1", api_key="EMPTY", temperature=0.4, max_tokens=500)
+lm = dspy.LM("openai/meta-llama/Meta-Llama-3-8B", api_base="http://future-hgx-1:7501/v1", api_key="EMPTY", max_tokens=700)
+# lm = dspy.LM("openai/meta-llama/Meta-Llama-3-8B", api_base="http://future-hgx-1:7501/v1", api_key="EMPTY")
+
+
+dspy.configure(lm=lm)
+
+# dspy.ChainOfThought('question -> answer: int')(question="What is two plus two?")
+# breakpoint()
+
+from dspy.datasets.gsm8k import GSM8K, gsm8k_metric
+
+gms8k = GSM8K()
+
+trainset, devset = gms8k.train, gms8k.dev
+
+program = dspy.ChainOfThought('question -> answer: int')
+
+from dspy.evaluate import Evaluate
+
+evaluate = Evaluate(devset=devset[:100], metric=gsm8k_metric, num_threads=1, display_progress=True, display_table=False, max_errors=10000)
+
+# class GSM8Reflection(dspy.Module):
+# def __init__(self, n = 3):
+# super().__init__()
+# self.gen_candidate_answer = dspy.ChainOfThought("question -> answer:int", n=n)
+# self.gen_final_answer = dspy.ChainOfThought("candidate_answers -> final_answer:int")
+
+# def get_candidate_answers_string(self, candidate_answers):
+# return "\n".join([f"Potential Answer {i+1}:\n\nReasoning: Let's think step by step to {candidate_answers.completions.rationale[i]}\n\nAnswer: {candidate_answers.completions.answer[i]}\n\n" for i in range(len(candidate_answers.completions.answer))])
+
+# def forward(self, question):
+# candidate_answers = self.gen_candidate_answer(question=question)
+# create_candidate_answer_string = self.get_candidate_answers_string(candidate_answers=candidate_answers)
+# final_answer = self.gen_final_answer(candidate_answers=create_candidate_answer_string).final_answer
+# return dspy.Prediction(answer=final_answer)
+
+# program = GSM8Reflection()
+print(f"Evaluating baseline program...")
+evaluate(program, devset=devset[:300])
+breakpoint()
+# Apply 0-shot MIPRO
+from dspy.teleprompt import MIPROv2
+
+# teleprompter = MIPROv2(
+# metric=gsm8k_metric,
+# num_candidates=10,
+# max_errors = 10000,
+# init_temperature=0.5,
+# verbose=False,
+# num_threads=4,
+# # log_dir="/lfs/0/kristaoo/dspy/examples/functional/logs",
+# )
+
+# zeroshot_optimized_program = teleprompter.compile(
+# program.deepcopy(),
+# trainset=trainset[:200],
+# max_bootstrapped_demos=0, # 0-shot optimization
+# max_labeled_demos=0,
+# num_batches=10,
+# # minibatch_size=25,
+# # minibatch_full_eval_steps=10, # TODO: should these be params in the compile step instead?
+# minibatch=False,
+# seed=9,
+# requires_permission_to_run=False,
+# )
+
+# now = datetime.now()
+# date_time = now.strftime("%Y%m%d_%H%M%S")
+# zeroshot_optimized_program.save(f"zeroshot_mipro_optimized_{date_time}")
+
+# print(f"Evluate optimized program...")
+# evaluate(zeroshot_optimized_program, devset=devset[:300])
+
+
+teleprompter = MIPROv2(
+ metric=gsm8k_metric,
+ num_candidates=10,
+ max_errors = 10000,
+ init_temperature=0.5,
+ verbose=False,
+ num_threads=4,
+ # log_dir="/lfs/0/kristaoo/dspy/examples/functional/logs",
+)
+
+optimized_program = teleprompter.compile(
+ program.deepcopy(),
+ trainset=trainset[:200],
+ max_bootstrapped_demos=3, # 0-shot optimization
+ max_labeled_demos=4,
+ num_trials=15,
+ # minibatch_size=25,
+ # minibatch_full_eval_steps=10, # TODO: should these be params in the compile step instead?
+ minibatch=False,
+ seed=9,
+ requires_permission_to_run=False,
+)
+
+now = datetime.now()
+date_time = now.strftime("%Y%m%d_%H%M%S")
+optimized_program.save(f"mipro_optimized_{date_time}")
+
+print(f"Evaluate optimized program...")
+evaluate(optimized_program, devset=devset[:300])
\ No newline at end of file