Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Fix BootstrapFinetune example in index doc and add basic tests for `bootstrap_finetune`. (#8435)

Hangzhi44ba438432a5
diff --git a/docs/docs/index.md b/docs/docs/index.md
index 9979d7ad5..7c8b0ebb9 100644
--- a/docs/docs/index.md
+++ b/docs/docs/index.md
@@ -403,17 +403,20 @@ Given a few tens or hundreds of representative _inputs_ of your task and a _metr
```python linenums="1"
import dspy
- dspy.configure(lm=dspy.LM("openai/gpt-4o-mini-2024-07-18"))
+ lm=dspy.LM('openai/gpt-4o-mini-2024-07-18')
# Define the DSPy module for classification. It will use the hint at training time, if available.
signature = dspy.Signature("text, hint -> label").with_updated_fields("label", type_=Literal[tuple(CLASSES)])
classify = dspy.ChainOfThought(signature)
+ classify.set_lm(lm)
# Optimize via BootstrapFinetune.
optimizer = dspy.BootstrapFinetune(metric=(lambda x, y, trace=None: x.label == y.label), num_threads=24)
optimized = optimizer.compile(classify, trainset=trainset)
optimized(text="What does a pending cash withdrawal mean?")
+
+ # For a complete fine-tuning tutorial, see: https://dspy.ai/tutorials/classification_finetuning/
```
**Possible Output (from the last line):**
diff --git a/docs/docs/learn/optimization/optimizers.md b/docs/docs/learn/optimization/optimizers.md
index 6cf9ffe6c..8323ac4b1 100644
--- a/docs/docs/learn/optimization/optimizers.md
+++ b/docs/docs/learn/optimization/optimizers.md
@@ -58,17 +58,18 @@ These optimizers produce optimal instructions for the prompt and, in the case of
6. [**`MIPROv2`**](../../api/optimizers/MIPROv2.md): Generates instructions *and* few-shot examples in each step. The instruction generation is data-aware and demonstration-aware. Uses Bayesian Optimization to effectively search over the space of generation instructions/demonstrations across your modules.
+7. [**`SIMBA`**](../../api/optimizers/SIMBA.md)
### Automatic Finetuning
This optimizer is used to fine-tune the underlying LLM(s).
-7. [**`BootstrapFinetune`**](../../api/optimizers/BootstrapFinetune.md): Distills a prompt-based DSPy program into weight updates. The output is a DSPy program that has the same steps, but where each step is conducted by a finetuned model instead of a prompted LM.
+8. [**`BootstrapFinetune`**](/api/optimizers/BootstrapFinetune): Distills a prompt-based DSPy program into weight updates. The output is a DSPy program that has the same steps, but where each step is conducted by a finetuned model instead of a prompted LM. [See the classification fine-tuning tutorial](https://dspy.ai/tutorials/classification_finetuning/) for a complete example.
### Program Transformations
-8. [**`Ensemble`**](../../api/optimizers/Ensemble.md): Ensembles a set of DSPy programs and either uses the full set or randomly samples a subset into a single program.
+9. [**`Ensemble`**](../../api/optimizers/Ensemble.md): Ensembles a set of DSPy programs and either uses the full set or randomly samples a subset into a single program.
## Which optimizer should I use?
@@ -176,17 +177,20 @@ optimized_program = teleprompter.compile(YOUR_PROGRAM_HERE, trainset=YOUR_TRAINS
```python linenums="1"
import dspy
- dspy.configure(lm=dspy.LM('openai/gpt-4o-mini-2024-07-18'))
+ lm=dspy.LM('openai/gpt-4o-mini-2024-07-18')
# Define the DSPy module for classification. It will use the hint at training time, if available.
signature = dspy.Signature("text, hint -> label").with_updated_fields('label', type_=Literal[tuple(CLASSES)])
classify = dspy.ChainOfThought(signature)
+ classify.set_lm(lm)
# Optimize via BootstrapFinetune.
optimizer = dspy.BootstrapFinetune(metric=(lambda x, y, trace=None: x.label == y.label), num_threads=24)
optimized = optimizer.compile(classify, trainset=trainset)
optimized(text="What does a pending cash withdrawal mean?")
+
+ # For a complete fine-tuning tutorial, see: https://dspy.ai/tutorials/classification_finetuning/
```
**Possible Output (from the last line):**
diff --git a/dspy/teleprompt/bootstrap_finetune.py b/dspy/teleprompt/bootstrap_finetune.py
index b8d531f79..4aa0f71fa 100644
--- a/dspy/teleprompt/bootstrap_finetune.py
+++ b/dspy/teleprompt/bootstrap_finetune.py
@@ -81,7 +81,14 @@ class BootstrapFinetune(FinetuneTeleprompter):
key_to_data = {}
for pred_ind, pred in enumerate(student.predictors()):
data_pred_ind = None if self.multitask else pred_ind
+ if pred.lm is None:
+ raise ValueError(
+ f"Predictor {pred_ind} does not have an LM assigned. "
+ f"Please ensure the module's predictors have their LM set before fine-tuning. "
+ f"You can set it using: your_module.set_lm(your_lm)"
+ )
training_key = (pred.lm, data_pred_ind)
+
if training_key not in key_to_data:
train_data, data_format = self._prepare_finetune_data(
trace_data=trace_data, lm=pred.lm, pred_ind=data_pred_ind
diff --git a/tests/teleprompt/test_bootstrap_finetune.py b/tests/teleprompt/test_bootstrap_finetune.py
new file mode 100644
index 000000000..59c685902
--- /dev/null
+++ b/tests/teleprompt/test_bootstrap_finetune.py
@@ -0,0 +1,81 @@
+from unittest.mock import patch
+
+import dspy
+from dspy import Example
+from dspy.predict import Predict
+from dspy.teleprompt import BootstrapFinetune
+from dspy.utils.dummies import DummyLM
+
+
+# Define a simple metric function for testing
+def simple_metric(example, prediction, trace=None):
+ return example.output == prediction.output
+
+
+examples = [
+ Example(input="What is the color of the sky?", output="blue").with_inputs("input"),
+ Example(input="What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!").with_inputs("input"),
+]
+trainset = [examples[0]]
+
+
+def test_bootstrap_finetune_initialization():
+ """Test BootstrapFinetune initialization with various parameters."""
+ bootstrap = BootstrapFinetune(metric=simple_metric)
+ assert bootstrap.metric == simple_metric, "Metric not correctly initialized"
+ assert bootstrap.multitask == True, "Multitask should default to True"
+
+
+class SimpleModule(dspy.Module):
+ def __init__(self, signature):
+ super().__init__()
+ self.predictor = Predict(signature)
+
+ def forward(self, **kwargs):
+ return self.predictor(**kwargs)
+
+
+def test_compile_with_predict_instances():
+ """Test BootstrapFinetune compilation with Predict instances."""
+ # Create SimpleModule instances for student and teacher
+ student = SimpleModule("input -> output")
+ teacher = SimpleModule("input -> output")
+
+ lm = DummyLM([{"output": "blue"}, {"output": "Ring-ding-ding-ding-dingeringeding!"}])
+ dspy.settings.configure(lm=lm)
+
+ # Set LM for both student and teacher
+ student.set_lm(lm)
+ teacher.set_lm(lm)
+
+ bootstrap = BootstrapFinetune(metric=simple_metric)
+
+ # Mock the fine-tuning process since DummyLM doesn't support it
+ with patch.object(bootstrap, "finetune_lms") as mock_finetune:
+ mock_finetune.return_value = {(lm, None): lm}
+ compiled_student = bootstrap.compile(student, teacher=teacher, trainset=trainset)
+
+ assert compiled_student is not None, "Failed to compile student"
+ assert hasattr(compiled_student, "_compiled") and compiled_student._compiled, "Student compilation flag not set"
+
+ mock_finetune.assert_called_once()
+
+
+def test_error_handling_missing_lm():
+ """Test error handling when predictor doesn't have an LM assigned."""
+
+ lm = DummyLM([{"output": "test"}])
+ dspy.settings.configure(lm=lm)
+
+ student = SimpleModule("input -> output")
+ # Intentionally NOT setting LM for the student module
+
+ bootstrap = BootstrapFinetune(metric=simple_metric)
+
+ # This should raise ValueError about missing LM and hint to use set_lm
+ try:
+ bootstrap.compile(student, trainset=trainset)
+ assert False, "Should have raised ValueError for missing LM"
+ except ValueError as e:
+ assert "does not have an LM assigned" in str(e)
+ assert "set_lm" in str(e)