Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Completing ExperimentalAdapter, improving Bootstrap logging (ad-hoc)

Omar Khattabb793d7b2d023
diff --git a/dsp/adapters/experimental_adapter.py b/dsp/adapters/experimental_adapter.py
index f923a5e36..4dd6fb561 100644
--- a/dsp/adapters/experimental_adapter.py
+++ b/dsp/adapters/experimental_adapter.py
@@ -21,36 +21,28 @@ class ExperimentalAdapter(BaseTemplate):
for field in self.fields
]
- # If there are no inputs, set the first field to ""
if not any(has_value):
- example[self.fields[0].input_variable] = ""
- # Otherwise find the first field without a value.
- else:
- for i in range(1, len(has_value)):
- if has_value[i - 1] and not any(has_value[i:]):
- example[self.fields[i].input_variable] = ""
- break
+ assert False, "No input variables found in the example"
+
+ for i in range(1, len(has_value)):
+ if has_value[i - 1] and not any(has_value[i:]):
+ example[self.fields[i].input_variable] = ""
+ break
for field in self.fields:
if field.input_variable in example and example[field.input_variable] is not None:
if field.input_variable in self.format_handlers:
format_handler = self.format_handlers[field.input_variable]
else:
-
def format_handler(x):
- assert type(x) == str, f"Need format_handler for {field.input_variable} of type {type(x)}"
- return " ".join(x.split())
+ return str(x).strip()
formatted_value = format_handler(example[field.input_variable])
separator = "\n" if field.separator == " " and "\n" in formatted_value else field.separator
- result.append(
- f"{field.name}{separator}{formatted_value}",
- )
+ result.append(f"{field.name}{separator}{formatted_value}",)
- if self._has_augmented_guidelines() and (example.get("augmented", False)):
- return "\n\n".join([r for r in result if r])
- return "\n".join([r for r in result if r])
+ return "\n\n".join([r for r in result if r])
def guidelines(self, show_guidelines=True) -> str:
"""Returns the task guidelines as described in the lm prompt"""
@@ -62,16 +54,11 @@ class ExperimentalAdapter(BaseTemplate):
example = dsp.Example()
for field in self.fields:
example[field.input_variable] = field.description
- example.augmented = self._has_augmented_guidelines()
+ example.augmented = True
result += self.query(example)
return result
- def _has_augmented_guidelines(self):
- return len(self.fields) > 3 or any(
- ("\n" in field.separator) or ("\n" in field.description) for field in self.fields
- )
-
def extract(
self,
example: Union[Example, dict[str, Any]],
@@ -132,10 +119,7 @@ class ExperimentalAdapter(BaseTemplate):
raw_pred = raw_pred[offset + len(next_field_name) :].strip()
idx += 1
else:
- if dspy.settings.release >= 20231003:
- example[self.fields[idx].output_variable] = raw_pred.strip().rstrip("---").strip()
- else:
- example[self.fields[idx].output_variable] = raw_pred.strip()
+ example[self.fields[idx].output_variable] = raw_pred.strip().rstrip("---").strip()
raw_pred = ""
idx += 1
@@ -174,8 +158,8 @@ class ExperimentalAdapter(BaseTemplate):
return self.query(example)
# The training data should not contain the output variable
- if self.fields[-1].input_variable in example:
- del example[self.fields[-1].input_variable]
+ assert self.fields[-1].input_variable not in example, f"Output variable {self.fields[-1].input_variable} should not be supplied for querying the LM."
+ # del example[self.fields[-1].input_variable]
rdemos = [
self.query(demo, is_demo=True)
@@ -195,59 +179,24 @@ class ExperimentalAdapter(BaseTemplate):
new_ademos = []
for rdemo in rdemos:
if all((field.name in rdemo) for field in self.fields if field.input_variable in example):
- import dspy
-
- if dspy.settings.release >= 20230928:
- new_ademos.append(rdemo)
- else:
- ademos.append(rdemo)
+ new_ademos.append(rdemo)
else:
rdemos_.append(rdemo)
ademos = new_ademos + ademos
rdemos = rdemos_
- long_query = self._has_augmented_guidelines()
-
- if long_query:
- example["augmented"] = True
+ example["augmented"] = True
query = self.query(example)
-
- # if it has more lines than fields
- if len(query.split("\n")) > len(self.fields):
- long_query = True
-
- if not example.get("augmented", False):
- example["augmented"] = True
- query = self.query(example)
-
- rdemos = "\n\n".join(rdemos)
-
- if len(rdemos) >= 1 and len(ademos) == 0 and not long_query:
- rdemos_and_query = "\n\n".join([rdemos, query])
- parts = [
- self.instructions,
- self.guidelines(show_guidelines),
- rdemos_and_query,
- ]
- elif len(rdemos) == 0:
- parts = [
- self.instructions,
- self.guidelines(show_guidelines),
- *ademos,
- query,
- ]
- else:
- parts = [
- self.instructions,
- rdemos,
- self.guidelines(show_guidelines),
- *ademos,
- query,
- ]
+ parts = [self.instructions, *rdemos, self.guidelines(show_guidelines), *ademos, query,]
prompt = "\n\n---\n\n".join([p.strip() for p in parts if p])
prompt_ = prompt[: prompt.rfind("\n")].strip()
- prompt_ += f"\n\nPlease provide the output fields {', '.join(output_fields[:-1]) + (', then ' if len(output_fields) > 2 else ' then ') + output_fields[-1] if len(output_fields) > 1 else output_fields[0]}. Do so immediately, without additional content before or after, and precisely as the format above shows. Always separate the output fields by only 2 newlines. Begin with the field {output_fields[0]}."
+
+ s_or_not = "s" if len(output_fields) > 1 else ""
+ only_or_not = "only " if len(output_fields) == 1 else ""
+
+ prompt_ += f"\n\nPlease provide the output field{s_or_not} {', '.join(output_fields[:-1]) + (', then ' if len(output_fields) > 2 else ' then ') + output_fields[-1] if len(output_fields) > 1 else output_fields[0]}. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with {only_or_not}the field {output_fields[0]}."
return prompt_.strip()
+
diff --git a/dsp/modules/lm.py b/dsp/modules/lm.py
index eb5ebb54e..e2b67b9cf 100644
--- a/dsp/modules/lm.py
+++ b/dsp/modules/lm.py
@@ -26,7 +26,7 @@ class LM(ABC):
return self.basic_request(prompt, **kwargs)
def print_green(self, text: str, end: str = "\n"):
- return "\n" + "\x1b[32m" + str(text) + "\x1b[0m" + end
+ return "\n\n" + "\x1b[32m" + str(text) + "\x1b[0m" + end
def print_red(self, text: str, end: str = "\n"):
return "\x1b[31m" + str(text) + "\x1b[0m" + end
diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py
index 8ce7741a9..086009e5d 100644
--- a/dspy/predict/predict.py
+++ b/dspy/predict/predict.py
@@ -146,6 +146,8 @@ def old_generate(demos, signature, kwargs, config, lm, stage):
def new_generate(lm, signature, example, max_depth=6, **kwargs):
+ kwargs['stop'] = tuple(kwargs.get('stop', [])) or ('\n---', )
+
# Generate and extract the fields.
template = signature_to_template(signature, adapter=dsp.ExperimentalAdapter)
prompt = template(example)
diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.py
index 91f0de186..69e13d4b3 100644
--- a/dspy/teleprompt/bootstrap.py
+++ b/dspy/teleprompt/bootstrap.py
@@ -156,7 +156,7 @@ class BootstrapFewShot(Teleprompter):
if success:
bootstrapped[example_idx] = True
- dspy.logger.info(
+ print(
f"Bootstrapped {len(bootstrapped)} full traces after {example_idx + 1} examples in round {round_idx}.",
)
diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.py
index 2b9c4e592..587f96b66 100644
--- a/dspy/teleprompt/random_search.py
+++ b/dspy/teleprompt/random_search.py
@@ -55,10 +55,10 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
# self.max_bootstrapped_demos = self.max_num_traces
self.max_labeled_demos = max_labeled_demos
- dspy.logger.info(
+ print(
"Going to sample between", self.min_num_samples, "and", self.max_num_samples, "traces per predictor.",
)
- dspy.logger.info("Will attempt to train", self.num_candidate_sets, "candidate sets.")
+ print("Will attempt to bootstrap", self.num_candidate_sets, "candidate sets.")
def compile(self, student, *, teacher=None, trainset, valset=None, restrict=None, labeled_sample=True):
self.trainset = trainset
@@ -132,16 +132,16 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
score = 0 if program2._assert_failures > 0 else score
######################################################
- dspy.logger.info("Score:", score, "for set:", [len(predictor.demos) for predictor in program2.predictors()])
+ print("Score:", score, "for set:", [len(predictor.demos) for predictor in program2.predictors()])
if len(scores) == 0 or score > max(scores):
- dspy.logger.info("New best sscore:", score, "for seed", seed)
+ print("New best sscore:", score, "for seed", seed)
best_program = program2
scores.append(score)
- dspy.logger.info(f"Scores so far: {scores}")
+ print(f"Scores so far: {scores}")
- dspy.logger.info("Best score:", max(scores))
+ print("Best score:", max(scores))
score_data.append((score, subscores, seed, program2))
@@ -153,17 +153,17 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
transposed_subscores = zip(*[subscores for _, subscores, *_ in top_3_scores if subscores])
avg_of_max_per_entry = sum(max(entry) for entry in transposed_subscores) / len(top_3_scores[0][1])
- dspy.logger.info(f"Average of max per entry across top {k} scores: {avg_of_max_per_entry}")
+ print(f"Average of max per entry across top {k} scores: {avg_of_max_per_entry}")
if self.stop_at_score is not None and score >= self.stop_at_score:
- dspy.logger.info(f"Stopping early because score {score} is >= stop_at_score {self.stop_at_score}")
+ print(f"Stopping early because score {score} is >= stop_at_score {self.stop_at_score}")
break
# To best program, attach all program candidates in decreasing average score
best_program.candidate_programs = score_data
best_program.candidate_programs = sorted(best_program.candidate_programs, key=lambda x: x[0], reverse=True)
- dspy.logger.info(f"{len(best_program.candidate_programs)} candidate programs found.")
+ print(f"{len(best_program.candidate_programs)} candidate programs found.")
return best_program