Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Add data functions

dilaraf9ae0bb8cf21
diff --git a/dspy/teleprompt/finetune_teleprompter.py b/dspy/teleprompt/finetune_teleprompter.py
new file mode 100644
index 000000000..14f42474c
--- /dev/null
+++ b/dspy/teleprompt/finetune_teleprompter.py
@@ -0,0 +1,322 @@
+from collections import OrderedDict
+from typing import Any, Callable, Dict, List, Optional, Union
+
+import dspy
+import dspy.logger as logger
+from dspy.evaluate.evaluate import Evaluate
+from dspy.primitives.example import Example
+from dspy.primitives.program import Program
+from dspy.primitives.prediction import Prediction
+from dspy.signatures.signature import signature_to_template
+
+
+#-------------------------------------------------------------------------------
+# Templates for the user-facing strings used by this module
+#-------------------------------------------------------------------------------
+
+_INFO_DEFAULT_TEACHER = """No teacher provided. Using a copy of the student \
+program as the teacher."""
+
+_INFO_STRUCTURAL_EQUIVALENCY = """Ensuring that the student and teacher are \
+are structurally equivalent."""
+
+_INFO_SHARED_PREDICTOR = """Ensuring that the student and teacher programs do \
+not share predictors."""
+
+_INFO_LM_CONSISTENCY = """Ensuring that the teacher program satisfies the LM \
+consistency property."""
+
+_INFO_BOOTSTRAP_DATA = """Bootstrapping data on {} examples with the program \
+{}, with {} threads"""
+
+#-------------------------------------------------------------------------------
+# Helper functions
+#-------------------------------------------------------------------------------
+
+# TODO: Shared below are useful functions. Similar procedures are implemented
+# separately and used by other DSPy teleprompters. These can be moved to shared
+# locations.
+def prepare_teacher(
+ student: Program,
+ teacher: Program = None
+ ) -> Union[Program, AssertionError]:
+ """Prepare the teacher program with respect to the student program.
+
+ Args:
+ student: The student program.
+ teacher: The teacher program. If `None`, a copy of the student program
+ is used as the teacher. Defaults to `None`.
+
+ Returns:
+ The copied teacher program.
+
+ Raises:
+ AssertionError: If the teacher is not an instance of the Program class.
+ """
+ # If teacher is None, use a copy of the student program as the teacher
+ if teacher is None:
+ logger.info(_INFO_DEFAULT_TEACHER)
+ teacher = student.deepcopy()
+ else:
+ teacher = teacher.deepcopy()
+
+ # Ensure that the student and teacher programs have the same structure
+ logger.info(_INFO_STRUCTURAL_EQUIVALENCY)
+ assert student._assert_structural_equivalency(teacher)
+
+ # Ensure that the predictors of the programs point to different objects
+ logger.info(_INFO_SHARED_PREDICTOR)
+ assert not student._assert_no_shared_predictor(teacher)
+
+ # Ensure that the LM consistency property is satisfied
+ logger.info(_INFO_LM_CONSISTENCY)
+ teacher._assert_lm_consistency()
+
+ # If the global LM is being used, set it to the LMs of the copied teacher
+ # program predictors to to avoid handling the same edge cases later
+ if dspy.settings.lm:
+ teacher._set_all_predictor_lms(dspy.settings.lm)
+
+ return teacher
+
+
+def build_prompt_completion_data_from_trace(
+ trace: List[Dict],
+ program: Optional[Program] = None,
+ exclude_demos: bool=False,
+ record_lm_kwargs: bool = False,
+ ) -> Dict[str, List[Dict[str, Any]]]:
+ """Build prompt completion data from a given trace.
+
+ Args:
+ trace: The trace from which the prompt-completion data will be built.
+ program: Optional argument corresponding to the program that generated
+ the trace, used to augment the prompt-completion pairs with the
+ names of the predictors that generated them. Defaults to `None`.
+ exclude_demos: Exclude the demos from the prompts even if they are
+ present in the trace. Defaults to `False`.
+ record_lm_kwargs: Whether to record the LM kwargs in the data. Defaults
+ to `False`.
+
+ Returns:
+ Data as a list of dictionaries with the keys `prompt`, `completion` and
+ optionally with the keys `predictor_name` and `lm_kwargs`. For a given
+ prompt-completion pair:
+ - The `prompt` field corresponds to the prompt.
+ - The `completion` field corresponds to the completion.
+ - The `predictor_name` field corresponds to the name of the predictor
+ that generated the prompt-completion pair.
+ - The `lm_kwargs` field corresponds to the LM kwargs that generated the
+ prompt-completion pair.
+ """
+ # If program is provided, build a mapping from predictor ids to names
+ pred_id_to_name = None
+ if program:
+ pred_id_to_name = OrderedDict(
+ (id(pred), name) for name, pred in program.named_predictors()
+ )
+
+ # Build the prompt-completion data
+ data = []
+ for pred, inputs, outputs in trace:
+ # Get the demos from the predictor if exclude_demos is False
+ demos = [] if exclude_demos else pred.demos
+
+ # Build prompt and completion strings
+ template = signature_to_template(pred.signature)
+ prompt = template(Example(demos=demos, **inputs))
+ completion = template.query(Example(**outputs))
+
+ # TODO: This part of the code could be improved.
+ # The method we use to build the completion (template.query) is meant to
+ # be used for creating, well, queries, and hence contains field prefixes
+ # (e.g. "Reasoning: Let's think step by step in order to"), which are
+ # also contained in the last piece of the prompt (separated with a new
+ # line) We remove this piece from the completion. This is a hacky
+ # solution since it assumes a particular template format.
+ prompt_last = prompt.split("\n")[-1]
+ completion = completion[len(prompt_last):]
+
+ # Create prompt-completion dictionary and add it to the data; optionally
+ # add the predictor_name key as well as the lm_kwargs.
+ data_dict = dict(prompt=prompt, completion=completion)
+
+ if program:
+ pred_name = pred_id_to_name[id(pred)]
+ data_dict['predictor_name'] = pred_name
+ if record_lm_kwargs:
+ data_dict['lm_kwargs'] = pred.lm.kwargs
+ data.append(data_dict)
+
+ return data
+
+
+def bootstrap_data(
+ program: Program,
+ dataset: List[Example],
+ metric: Optional[Callable[
+ [Example, Prediction, Optional[List]], Union[bool, int, float]
+ ]] = None,
+ num_threads = 1,
+ ) -> List[Dict[str, Any]]:
+ """Bootstrap prediction and trace data for the program using the dataset.
+
+ Args:
+ program: The program that will be used to generate the traces for data
+ collection.
+ dataset: The dataset to be used for data collection.
+ metric: The optional metric to be used to get a score for the example,
+ recorded in a `score` field in the data. If the metric is not
+ provided, the `score` field is not included in the data. Defaults
+ to `None`.
+ num_threads: The number of threads to be used for data collection.
+ Defaults to `1`.
+
+ Returns:
+ Data as a list of dictionaries with the keys `example`, `prediction`,
+ `trace`, and optionally, `score` fields. For a given example:
+ - The `example` field corresponds to the example itself.
+ - The `prediction` field corresponds to the prediction made by the
+ program on the example.
+ - The `trace` field corresponds to the trace generated by the program
+ on the example.
+ - The `score` field corresponds to the metric score of the example, if
+ the metric is provided. Otherwise, it is not included in the data.
+ """
+ data = []
+
+ # Use Evaluate to call the program have the responses cached
+ cname = program.__class__.__name__
+ info = _INFO_BOOTSTRAP_DATA.format(len(dataset), cname, num_threads)
+ logger.info(info)
+ evaluator = Evaluate(
+ devset=dataset, num_threads=num_threads, display_progress=True
+ )
+ evaluator(program, metric=metric)
+
+ # Re-iterate over the dataset to build the cached prompt-completion data
+ for example in dataset:
+
+ # Run the program on the example
+ with dspy.context(trace=[]):
+ prediction = program(**example.inputs())
+ trace = dspy.settings.trace
+ score = metric(example, prediction, trace) if metric else None
+
+ # Build the data dictionary and extend the data list
+ data_dict = dict(example=example, prediction=prediction, trace=trace)
+ if metric:
+ data_dict['score'] = score
+ data.append(data_dict)
+
+ return data
+
+
+# TODO: If we can ensure to pass the "round" information every time a call is
+# issued to an LM, we can make repetitive un-cached calls to the same LM without
+# modifying it's temperature. The `temperature_delta` argument can be removed
+# then.
+def bootstrap_data_multiple_rounds(
+ program: Program,
+ dataset: List[Example],
+ metric: Optional[Callable[
+ [Example, Prediction, Optional[List]], Union[bool, int, float]
+ ]] = None,
+ num_threads = 1,
+ num_rounds: int = 1,
+ sampling_temperature_base: Optional[float] = 0.9,
+ sampling_temperature_delta: float = 0.001,
+ next_round_dataset_callback: Optional[Callable[
+ [List[Example], List[Dict[str, Any]]], List[Example]
+ ]] = lambda x, y: x,
+ ) -> List[Dict]:
+ """ Bootstrap multiple rounds of data using the dataset and program.
+
+ Args:
+ program: The program that will be used to generate the traces for data
+ collection.
+ dataset: The dataset to be used for data collection.
+ metric: The optional metric to be used to get a score for the example,
+ recorded in a `score` field in the data. If the metric is not
+ provided, the `score` field is not included in the data. Defaults to
+ `None`.
+ num_threads: The number of threads to be used for data collection.
+ Defaults to `1`.
+ num_rounds: The number of rounds of data collection to be performed.
+ Defaults to `1`.
+ sampling_temperature_base: The sampling temperature to be used in the
+ first round. If a value of `None` is passed, the already set
+ temperature of the LM is used as the base sampling temperature
+ instead. Defaults to a high temperature of `0.9`.
+ sampling_temperature_delta: The small temperature difference utilized to
+ generate different bootstrapped traces for the different rounds to
+ circummvent caching. Defaults to `0.001`.
+ next_round_dataset_callback: The callback function to be used to
+ generate the dataset for the next round of data collection. The
+ function should take the current dataset and the data from the
+ current round as arguments and return the dataset for the next
+ round, an example signature for which provided below. Defaults to a
+ function that returns the current dataset, meaning that the default
+ behavior of this function is to sample `num_rounds` bootstrapped
+ examples for each example in the original dataset.
+
+ ```
+ def next_round_dataset_callback(
+ dataset: List[Example],
+ data: List[Dict[str, Any]]
+ ) -> List[Example]:
+ # Your code modifying the dataset here
+ return dataset
+ ```
+
+ Returns:
+ Data as a list of dictionaries with the keys `example`, `prediction`,
+ `trace`, and optionally, `score` fields, along with the `round` field.
+ For a given example:
+ - The `example` field corresponds to the example itself.
+ - The `prediction` field corresponds to the prediction made by the
+ program on the example.
+ - The `trace` field corresponds to the trace generated by the program on
+ the example.
+ - The `score` field corresponds to the metric score of the example, if
+ the metric is provided. Otherwise, it is not included in the data.
+ - The `round` field corresponds to the index of the round the particular
+ data dictionary is collected in.
+ """
+ # Create a copy of the program and dataset so that they are not modified
+ program = program.deepcopy()
+ dataset = dataset.copy()
+
+ # Ensure that the LM consistency is satisfied, push the global LM to the
+ # predictors if dspy.settings.lm is set
+ program.assert_lm_consistency()
+ if dspy.settings.lm:
+ program.set_all_predictor_lms(dspy.settings.lm)
+
+ # Collect rounds of data collection with different sampling temperatures
+ data = []
+ for round_ind in range(num_rounds):
+ # Adjust the LM temperatures of the program predictors
+ for pred in program.predictors:
+ # If a None temperature is passed, keep the predictor's temperature
+ # as is for the first round
+ temp = sampling_temperature_base
+ temp = pred.lm.kwargs["temperature"] if temp is None else temp
+ temp += sampling_temperature_delta * round_ind
+ pred.lm = pred.lm.copy(temperature=temp)
+
+ # Collect the data for the round, add the round index to the data
+ # dictionaries, and extend the data list
+ info = f"Round {round_ind + 1}/{num_rounds}"
+ logger.info(info)
+ round_data = bootstrap_data(
+ program, dataset, metric=metric, num_threads=num_threads
+ )
+ for data_dict in round_data:
+ data_dict["round"] = round_ind
+ data += round_data
+
+ # Update the dataset for the next round
+ dataset = next_round_dataset_callback(dataset, round_data)
+
+ return data