Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Update finetune methods

dilara484bc6f732ef
diff --git a/dsp/modules/trainable_openai.py b/dsp/modules/trainable_openai.py
index 869558277..0d6ae203b 100644
--- a/dsp/modules/trainable_openai.py
+++ b/dsp/modules/trainable_openai.py
@@ -59,7 +59,6 @@ def openai_data_validation(dataset: List[dict[str, Any]]) -> Union[dict[str, Any
Either a list of errors and their counts or None if no errors are found
"""
# TODO: Move the import outside the function
- from dsp import logger
# TODO: Counting the number of errors is not very useful, we can consider
# raising an error as we run into issues.
@@ -136,7 +135,6 @@ def num_assistant_tokens_from_messages(messages):
def check_message_lengths(dataset: List[dict[str, Any]]) -> list[int]:
# TODO: Move the import outside the function
- from dsp import logger
n_missing_system = 0
n_missing_user = 0
@@ -173,7 +171,6 @@ def check_message_lengths(dataset: List[dict[str, Any]]) -> list[int]:
def estimate_cost(dataset: dict[str, Any], tokens_per_message=3, tokens_per_name=1, convo_lens=None):
# TODO: Move the import outside the function
- from dsp import logger
MAX_TOKENS_PER_EXAMPLE = 16385
@@ -308,7 +305,6 @@ class TrainableOpenAI(GPT3, TrainableLM):
str: The file id of the data to be used for fine-tuning.
"""
# TODO: Move the import outside the function
- from dsp import logger
datasets = {"train": train_path}
if eval_path:
@@ -327,7 +323,6 @@ class TrainableOpenAI(GPT3, TrainableLM):
# TODO: support OAI wandb integration
def _start_remote_training(self, **kwargs) -> str:
# TODO: Move the import outside the function
- from dsp import logger
assert self.active_finetuning_file_ids and self.active_finetuning_file_ids["train"] is not None, "You must load data before starting training"
hyperparameters = kwargs.get("hyperparameters", {})
diff --git a/dspy/teleprompt/finetune_teleprompter.py b/dspy/teleprompt/finetune_teleprompter.py
index 44b302e52..421053073 100644
--- a/dspy/teleprompt/finetune_teleprompter.py
+++ b/dspy/teleprompt/finetune_teleprompter.py
@@ -250,12 +250,14 @@ def bootstrap_data(
Data as a list of dictionaries with the keys `example`, `prediction`,
`trace`, and optionally, `score` fields. For a given example:
- The `example` field corresponds to the example itself.
+ - The `example_ind` field corresponds to the index of the example in the
+ `dataset`.
- The `prediction` field corresponds to the prediction made by the
- program on the example.
+ program on the example.
- The `trace` field corresponds to the trace generated by the program
- on the example.
+ on the example.
- The `score` field corresponds to the metric score of the example, if
- the metric is provided. Otherwise, it is not included in the data.
+ the metric is provided. Otherwise, it is not included in the data.
"""
data = []
@@ -269,7 +271,7 @@ def bootstrap_data(
evaluator(program, metric=metric)
# Re-iterate over the dataset to build the cached prompt-completion data
- for example in dataset:
+ for example_ind, example in enumerate(dataset):
# Run the program on the example
with dspy.context(trace=[]):
@@ -279,6 +281,7 @@ def bootstrap_data(
# Build the data dictionary and extend the data list
data_dict = dict(example=example, prediction=prediction, trace=trace)
+ data_dict['example_ind'] = example_ind
if metric:
data_dict['score'] = score
data.append(data_dict)
@@ -367,16 +370,10 @@ def bootstrap_data_for_round(
`0.001`.
Returns:
- Data as a list of dictionaries with the keys `example`, `prediction`,
- `trace`, and optionally, `score` fields, along with the `round` field.
- For a given example:
- - The `example` field corresponds to the example itself.
- - The `prediction` field corresponds to the prediction made by the
- program on the example.
- - The `trace` field corresponds to the trace generated by the program on
- the example.
- - The `score` field corresponds to the metric score of the example, if
- the metric is provided. Otherwise, it is not included in the data.
+ Data as a list of dictionaries with the keys returned by
+ the `bootstrap_data` function, descriptions for which are shared in the
+ function documentation. This function adds the following extra field
+ to the dictionaries:
- The `round` field corresponds to the `sampling_round` argument.
"""
# Helper function to adjust the temperature of the LM. If a None temperature