DSPy: The framework for programming—not prompting—language models
Python
Completing ExperimentalAdapter, improving Bootstrap logging (ad-hoc)
Omar Khattabb793d7b2d023
diff --git a/dsp/adapters/experimental_adapter.py b/dsp/adapters/experimental_adapter.pyindex f923a5e36..4dd6fb561 100644--- a/dsp/adapters/experimental_adapter.py+++ b/dsp/adapters/experimental_adapter.py@@ -21,36 +21,28 @@ class ExperimentalAdapter(BaseTemplate):for field in self.fields]- # If there are no inputs, set the first field to ""if not any(has_value):- example[self.fields[0].input_variable] = ""- # Otherwise find the first field without a value.- else:- for i in range(1, len(has_value)):- if has_value[i - 1] and not any(has_value[i:]):- example[self.fields[i].input_variable] = ""- break+ assert False, "No input variables found in the example"++ for i in range(1, len(has_value)):+ if has_value[i - 1] and not any(has_value[i:]):+ example[self.fields[i].input_variable] = ""+ breakfor field in self.fields:if field.input_variable in example and example[field.input_variable] is not None:if field.input_variable in self.format_handlers:format_handler = self.format_handlers[field.input_variable]else:-def format_handler(x):- assert type(x) == str, f"Need format_handler for {field.input_variable} of type {type(x)}"- return " ".join(x.split())+ return str(x).strip()formatted_value = format_handler(example[field.input_variable])separator = "\n" if field.separator == " " and "\n" in formatted_value else field.separator- result.append(- f"{field.name}{separator}{formatted_value}",- )+ result.append(f"{field.name}{separator}{formatted_value}",)- if self._has_augmented_guidelines() and (example.get("augmented", False)):- return "\n\n".join([r for r in result if r])- return "\n".join([r for r in result if r])+ return "\n\n".join([r for r in result if r])def guidelines(self, show_guidelines=True) -> str:"""Returns the task guidelines as described in the lm prompt"""@@ -62,16 +54,11 @@ class ExperimentalAdapter(BaseTemplate):example = dsp.Example()for field in self.fields:example[field.input_variable] = field.description- example.augmented = self._has_augmented_guidelines()+ example.augmented = Trueresult += self.query(example)return result- def _has_augmented_guidelines(self):- return len(self.fields) > 3 or any(- ("\n" in field.separator) or ("\n" in field.description) for field in self.fields- )-def extract(self,example: Union[Example, dict[str, Any]],@@ -132,10 +119,7 @@ class ExperimentalAdapter(BaseTemplate):raw_pred = raw_pred[offset + len(next_field_name) :].strip()idx += 1else:- if dspy.settings.release >= 20231003:- example[self.fields[idx].output_variable] = raw_pred.strip().rstrip("---").strip()- else:- example[self.fields[idx].output_variable] = raw_pred.strip()+ example[self.fields[idx].output_variable] = raw_pred.strip().rstrip("---").strip()raw_pred = ""idx += 1@@ -174,8 +158,8 @@ class ExperimentalAdapter(BaseTemplate):return self.query(example)# The training data should not contain the output variable- if self.fields[-1].input_variable in example:- del example[self.fields[-1].input_variable]+ assert self.fields[-1].input_variable not in example, f"Output variable {self.fields[-1].input_variable} should not be supplied for querying the LM."+ # del example[self.fields[-1].input_variable]rdemos = [self.query(demo, is_demo=True)@@ -195,59 +179,24 @@ class ExperimentalAdapter(BaseTemplate):new_ademos = []for rdemo in rdemos:if all((field.name in rdemo) for field in self.fields if field.input_variable in example):- import dspy-- if dspy.settings.release >= 20230928:- new_ademos.append(rdemo)- else:- ademos.append(rdemo)+ new_ademos.append(rdemo)else:rdemos_.append(rdemo)ademos = new_ademos + ademosrdemos = rdemos_- long_query = self._has_augmented_guidelines()-- if long_query:- example["augmented"] = True+ example["augmented"] = Truequery = self.query(example)-- # if it has more lines than fields- if len(query.split("\n")) > len(self.fields):- long_query = True-- if not example.get("augmented", False):- example["augmented"] = True- query = self.query(example)-- rdemos = "\n\n".join(rdemos)-- if len(rdemos) >= 1 and len(ademos) == 0 and not long_query:- rdemos_and_query = "\n\n".join([rdemos, query])- parts = [- self.instructions,- self.guidelines(show_guidelines),- rdemos_and_query,- ]- elif len(rdemos) == 0:- parts = [- self.instructions,- self.guidelines(show_guidelines),- *ademos,- query,- ]- else:- parts = [- self.instructions,- rdemos,- self.guidelines(show_guidelines),- *ademos,- query,- ]+ parts = [self.instructions, *rdemos, self.guidelines(show_guidelines), *ademos, query,]prompt = "\n\n---\n\n".join([p.strip() for p in parts if p])prompt_ = prompt[: prompt.rfind("\n")].strip()- prompt_ += f"\n\nPlease provide the output fields {', '.join(output_fields[:-1]) + (', then ' if len(output_fields) > 2 else ' then ') + output_fields[-1] if len(output_fields) > 1 else output_fields[0]}. Do so immediately, without additional content before or after, and precisely as the format above shows. Always separate the output fields by only 2 newlines. Begin with the field {output_fields[0]}."++ s_or_not = "s" if len(output_fields) > 1 else ""+ only_or_not = "only " if len(output_fields) == 1 else ""++ prompt_ += f"\n\nPlease provide the output field{s_or_not} {', '.join(output_fields[:-1]) + (', then ' if len(output_fields) > 2 else ' then ') + output_fields[-1] if len(output_fields) > 1 else output_fields[0]}. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with {only_or_not}the field {output_fields[0]}."return prompt_.strip()+diff --git a/dsp/modules/lm.py b/dsp/modules/lm.pyindex eb5ebb54e..e2b67b9cf 100644--- a/dsp/modules/lm.py+++ b/dsp/modules/lm.py@@ -26,7 +26,7 @@ class LM(ABC):return self.basic_request(prompt, **kwargs)def print_green(self, text: str, end: str = "\n"):- return "\n" + "\x1b[32m" + str(text) + "\x1b[0m" + end+ return "\n\n" + "\x1b[32m" + str(text) + "\x1b[0m" + enddef print_red(self, text: str, end: str = "\n"):return "\x1b[31m" + str(text) + "\x1b[0m" + enddiff --git a/dspy/predict/predict.py b/dspy/predict/predict.pyindex 8ce7741a9..086009e5d 100644--- a/dspy/predict/predict.py+++ b/dspy/predict/predict.py@@ -146,6 +146,8 @@ def old_generate(demos, signature, kwargs, config, lm, stage):def new_generate(lm, signature, example, max_depth=6, **kwargs):+ kwargs['stop'] = tuple(kwargs.get('stop', [])) or ('\n---', )+# Generate and extract the fields.template = signature_to_template(signature, adapter=dsp.ExperimentalAdapter)prompt = template(example)diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.pyindex 91f0de186..69e13d4b3 100644--- a/dspy/teleprompt/bootstrap.py+++ b/dspy/teleprompt/bootstrap.py@@ -156,7 +156,7 @@ class BootstrapFewShot(Teleprompter):if success:bootstrapped[example_idx] = True- dspy.logger.info(+ print(f"Bootstrapped {len(bootstrapped)} full traces after {example_idx + 1} examples in round {round_idx}.",)diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.pyindex 2b9c4e592..587f96b66 100644--- a/dspy/teleprompt/random_search.py+++ b/dspy/teleprompt/random_search.py@@ -55,10 +55,10 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):# self.max_bootstrapped_demos = self.max_num_tracesself.max_labeled_demos = max_labeled_demos- dspy.logger.info(+ print("Going to sample between", self.min_num_samples, "and", self.max_num_samples, "traces per predictor.",)- dspy.logger.info("Will attempt to train", self.num_candidate_sets, "candidate sets.")+ print("Will attempt to bootstrap", self.num_candidate_sets, "candidate sets.")def compile(self, student, *, teacher=None, trainset, valset=None, restrict=None, labeled_sample=True):self.trainset = trainset@@ -132,16 +132,16 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):score = 0 if program2._assert_failures > 0 else score######################################################- dspy.logger.info("Score:", score, "for set:", [len(predictor.demos) for predictor in program2.predictors()])+ print("Score:", score, "for set:", [len(predictor.demos) for predictor in program2.predictors()])if len(scores) == 0 or score > max(scores):- dspy.logger.info("New best sscore:", score, "for seed", seed)+ print("New best sscore:", score, "for seed", seed)best_program = program2scores.append(score)- dspy.logger.info(f"Scores so far: {scores}")+ print(f"Scores so far: {scores}")- dspy.logger.info("Best score:", max(scores))+ print("Best score:", max(scores))score_data.append((score, subscores, seed, program2))@@ -153,17 +153,17 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):transposed_subscores = zip(*[subscores for _, subscores, *_ in top_3_scores if subscores])avg_of_max_per_entry = sum(max(entry) for entry in transposed_subscores) / len(top_3_scores[0][1])- dspy.logger.info(f"Average of max per entry across top {k} scores: {avg_of_max_per_entry}")+ print(f"Average of max per entry across top {k} scores: {avg_of_max_per_entry}")if self.stop_at_score is not None and score >= self.stop_at_score:- dspy.logger.info(f"Stopping early because score {score} is >= stop_at_score {self.stop_at_score}")+ print(f"Stopping early because score {score} is >= stop_at_score {self.stop_at_score}")break# To best program, attach all program candidates in decreasing average scorebest_program.candidate_programs = score_databest_program.candidate_programs = sorted(best_program.candidate_programs, key=lambda x: x[0], reverse=True)- dspy.logger.info(f"{len(best_program.candidate_programs)} candidate programs found.")+ print(f"{len(best_program.candidate_programs)} candidate programs found.")return best_program