Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

WIP Converting to 2.5 LM FT

Isaac Miller00014ef8868d
diff --git a/dsp/modules/trainable_anyscale.py b/dsp/modules/trainable_anyscale.py
index 325ad3f9c..4c4f47411 100644
--- a/dsp/modules/trainable_anyscale.py
+++ b/dsp/modules/trainable_anyscale.py
@@ -116,6 +116,7 @@ class TrainableAnyscale(MultiOpenAI, TrainableLM):
model_config_data.update(kwargs.get("hyperparameters", {}))
model_config_data["model_id"] = self.kwargs["model"]
+
custom_modifications = {
"model_id": self.kwargs["model"],
"train_path": train_path,
@@ -123,7 +124,6 @@ class TrainableAnyscale(MultiOpenAI, TrainableLM):
"provider": "wandb",
},
"num_checkpoints_to_keep": 10
- # "output_dir": kwargs.get("output_dir", "/mnt/cluster_storage/dspy/finetuning/artifacts/")
}
if kwargs.get("output_dir", None):
custom_modifications["output_dir"] = kwargs["output_dir"]
@@ -235,7 +235,6 @@ class TrainableAnyscale(MultiOpenAI, TrainableLM):
s3_path = os.path.join(storage, path.split("/")[-1])
print(f"Uploading {name} data to S3 at {s3_path}")
- # ray.data.read_json(path).write_json(s3_path)
# NOTE: trying a local copy for now
if s3_path[:2] == "s3":
os.system(f"aws s3 cp {path} {s3_path}")
@@ -255,7 +254,7 @@ class TrainableAnyscale(MultiOpenAI, TrainableLM):
os.system(f"anyscale job submit --config-file {finetuning_job_path}")
return "job.id"
- # TODO
+ # TODO: Straight up out of scope lmao
def validate_hyperparameters(self, hyperparameters: dict[str, Any]) -> bool:
"""Validate the hyperparameters before starting training. Only checks the hyperparameters that are allowed in the OpenAI API.
More information on hyperparameter validation can be found here: https://platform.openai.com/docs/api-reference/fine-tuning/create#fine-tuning-create-hyperparameters
@@ -265,24 +264,6 @@ class TrainableAnyscale(MultiOpenAI, TrainableLM):
Returns:
bool: Whether the hyperparameters are valid."""
- def is_positive_number(value, convert_func):
- try:
- return convert_func(value) > 0
- except (ValueError, TypeError):
- return False
-
- parameters = {
- "batch_size": int,
- "n_epochs": int,
- "learning_rate_multiplier": float,
- }
-
- for param, convert_func in parameters.items():
- value = hyperparameters.get(param, None)
- if value and not is_positive_number(value, convert_func):
- print(
- f"Invalid {param}: Must be a positive {convert_func.__name__}.")
- return False
return True
@@ -322,7 +303,8 @@ class TrainableAnyscale(MultiOpenAI, TrainableLM):
else:
raise RuntimeError("Job not completed yet, cannot retrieve model")
- # TODO
+ # TODO
+ # Note: working here
def start_training(self, future: Future['TrainableOpenAI'], train_path: str, eval_path: Optional[str], method: TrainingMethod, **kwargs):
"""
Handles the fine-tuning process for an OpenAIModel instance.
@@ -337,11 +319,12 @@ class TrainableAnyscale(MultiOpenAI, TrainableLM):
raise NotImplementedError(f"TrainableOpenAI can only support {TrainingMethod.SFT} for the time being")
# Convert the data from prompt completion to OAI compatible messages
+ # This can be slightly flexible to accept json or jsonl and to check if the key starts with "user/assistant/system" or if first key is "messages"
train_dataset = self._format_data_for_vanilla_finetuning(train_path)
val_dataset = self._format_data_for_vanilla_finetuning(eval_path) if eval_path else None
# This is where we validate the yaml + kwargs combo
- if not self._verify_training_arguments(train_dataset, val_dataset, **kwargs):
+ if not self._verify_datasets(train_dataset, val_dataset, **kwargs):
print("Unable to verify arguments")
raise RuntimeError("Unable to verify argument")
@@ -383,7 +366,7 @@ class TrainableAnyscale(MultiOpenAI, TrainableLM):
This includes:
- Convert the data to the required format
- Validate the data
- - Load the data
+ - Submit the data to the cloud storage
- Start the remote training
- Wait for the training to complete
- Retrieve the trained model
@@ -398,3 +381,72 @@ class TrainableAnyscale(MultiOpenAI, TrainableLM):
Future[TrainableLM]: A Future object that will hold the fine-tuned model
"""
return super().get_finetune(train_path=train_path, eval_path=eval_path, method=method, **kwargs)
+
+
+# TODO: This should be moved into the TrainableAnyscaleLM class
+
+def copy_lora_weights(storage_uri, model_info, job_id):
+ """
+ Copies LoRA weights from a source GCS bucket to a destination folder.
+
+ Args:
+ storage_uri (str): The GCS URI of the source bucket and folder.
+ model_info (dict): A dictionary containing model information, including 'base_model_id'.
+ job_id (str): The ID of the job associated with the LoRA weights.
+
+ Returns:
+ list: A list of copied model names.
+ """
+ try:
+ from google.cloud import storage
+
+ # Initialize the Google Cloud Storage client
+ storage_client = storage.Client()
+
+ # Parse the storage_uri to get bucket name and source folder
+ bucket_name = storage_uri.split('/')[2]
+ source_folder = '/'.join(storage_uri.split('/')[3:-1])
+ print(f"Source folder: {source_folder}")
+
+ # Get the bucket
+ bucket = storage_client.bucket(bucket_name)
+
+ # List all subfolders in the source folder
+ blobs = bucket.list_blobs(prefix=source_folder)
+
+ subfolders = set()
+ for blob in blobs:
+ if '/' in blob.name[len(source_folder):]:
+ subfolder = blob.name.split('/')[:-1]
+ subfolders.add('/'.join(subfolder))
+
+ # Construct the destination folder path
+ base_model_id = model_info["base_model_id"]
+ lora_dynamic_path = f"dspy/lora_weights/{job_id}/{base_model_id}"
+
+ model_names = []
+ # Copy each subfolder to the main storage folder
+ for subfolder in subfolders:
+ subfolder_name = subfolder.split('/')[-1]
+ destination_folder = f"{lora_dynamic_path}:{subfolder_name}"
+ if subfolder_name.startswith("epoch"):
+ model_names.append("/".join(destination_folder.split("/")[-2:]))
+ else:
+ continue
+
+ # List all blobs in the subfolder
+ subfolder_blobs = bucket.list_blobs(prefix=subfolder)
+
+ # Copy each blob to the destination folder
+ for blob in subfolder_blobs:
+ source_blob = bucket.blob(blob.name)
+ destination_blob_name = f"{destination_folder}/{blob.name.split('/')[-1]}"
+ bucket.copy_blob(source_blob, bucket, destination_blob_name)
+ print(f"Copied {source_blob.name} to {destination_blob_name}")
+
+ print(f"All subfolders copied to: gs://{bucket_name}/{lora_dynamic_path}")
+ return model_names
+
+ except Exception as e:
+ print(f"An error occurred: {str(e)}")
+ raise e
\ No newline at end of file
diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py
index ddb0ee3e8..0969bd6a1 100644
--- a/dspy/adapters/base.py
+++ b/dspy/adapters/base.py
@@ -3,12 +3,17 @@ class Adapter:
inputs = self.format(signature, demos, inputs)
inputs = dict(prompt=inputs) if isinstance(inputs, str) else dict(messages=inputs)
+ # print(lm.model)
outputs = lm(**inputs, **lm_kwargs)
values = []
for output in outputs:
- value = self.parse(signature, output, _parse_values=_parse_values)
- assert set(value.keys()) == set(signature.output_fields.keys()), f"Expected {signature.output_fields.keys()} but got {value.keys()}"
- values.append(value)
+ try:
+ value = self.parse(signature, output, _parse_values=_parse_values)
+ assert set(value.keys()) == set(signature.output_fields.keys()), f"Expected {signature.output_fields.keys()} but got {value.keys()}"
+ values.append(value)
+ except Exception as e:
+ # print(f"Error parsing output: {output} for input: {inputs}")
+ raise e
return values
diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.py
index e2c3abeba..df5be77b0 100644
--- a/dspy/adapters/chat_adapter.py
+++ b/dspy/adapters/chat_adapter.py
@@ -54,11 +54,31 @@ class ChatAdapter(Adapter):
except Exception as e:
raise ValueError(f"Error parsing field {k}: {e}.\n\n\t\tOn attempting to parse the value\n```\n{v}\n```")
- if fields.keys() != signature.output_fields.keys():
- raise ValueError(f"Expected {signature.output_fields.keys()} but got {fields.keys()}")
+ if list(fields.keys()) != list(signature.output_fields.keys()):
+ # print("Expected", list(signature.output_fields.keys()), "but got", fields.keys(), "from", completion)
+ raise ValueError(f"Expected {list(signature.output_fields.keys())} but got {fields.keys()}")
return fields
+ def format_completion(self, signature, outputs):
+ reconstructed = []
+
+ fields_dict = signature.output_fields
+
+ field_name_output_map = {field: outputs[field] for field in fields_dict.keys()}
+
+ for field, value in field_name_output_map.items():
+ reconstructed.append(f"[[ ## {field} ## ]]")
+ reconstructed.append(str(value))
+ reconstructed.append("") # Add an empty line for separation
+
+ reconstructed.append("[[ ## completed ## ]]")
+
+ result = "\n".join(reconstructed).strip()
+
+ return result
+
+
def format_blob(blob):
if '\n' not in blob and "«" not in blob and "»" not in blob: return f"«{blob}»"
@@ -110,9 +130,10 @@ def format_turn(signature, values, role, incomplete=False):
content.append(format_fields({k: values.get(k, "Not supplied for this particular example.") for k in field_names}))
if role == "user":
- content.append("Respond with the corresponding output fields, starting with the field " +
- ", then ".join(f"`{f}`" for f in signature.output_fields) +
- ", and then ending with the marker for `completed`.")
+ content.append(f"Respond with the corresponding output fields using the proper format of "
+ f"[[ ## <field_name> ## ]] followed by the field value. Start with the field "
+ f"{', then '.join(f'`[[ ## {f} ## ]]`' for f in signature.output_fields)}, "
+ f"and then ending with the marker for `completed`.")
return {"role": role, "content": '\n\n'.join(content).strip()}
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index 1c611ca34..45a14f706 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -60,6 +60,14 @@ class LM:
def inspect_history(self, n: int = 1):
_inspect_history(self, n)
+ def copy(self, **kwargs):
+ """Returns a copy of the language model with the same parameters."""
+ kwargs = {**self.kwargs, **kwargs}
+ # model = kwargs.pop("model") or self.model
+ init_kwargs = dict(model=self.model, model_type=self.model_type, cache=self.cache, temperature=self.kwargs["temperature"], max_tokens=self.kwargs["max_tokens"])
+ init_kwargs = {**init_kwargs, **kwargs}
+ return self.__class__(**init_kwargs)
+
@functools.lru_cache(maxsize=None)
def cached_litellm_completion(request):
diff --git a/dspy/predict/chain_of_thought.py b/dspy/predict/chain_of_thought.py
index 74656697c..b6b9cb575 100644
--- a/dspy/predict/chain_of_thought.py
+++ b/dspy/predict/chain_of_thought.py
@@ -29,7 +29,7 @@ class ChainOfThought(Module):
rationale_type = rationale_type or dspy.OutputField(prefix=prefix, desc=desc)
# Add "rationale" field to the output signature.
- if isinstance(dspy.settings.lm, dspy.LM):
+ if isinstance(dspy.settings.lm, dspy.LM) or dspy.settings.experimental:
extended_signature = signature.prepend("reasoning", rationale_type, type_=str)
else:
extended_signature = signature.prepend("rationale", rationale_type, type_=str)
diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py
index 8bd42699d..906ea76e8 100644
--- a/dspy/predict/predict.py
+++ b/dspy/predict/predict.py
@@ -109,6 +109,7 @@ class Predict(Module, Parameter):
# Get the right LM to use.
lm = kwargs.pop("lm", self.lm) or dsp.settings.lm
+ # print(dsp.settings)
assert lm is not None, "No LM is loaded."
# If temperature is 0.0 but its n > 1, set temperature to 0.7.
diff --git a/dspy/teleprompt/ensemble.py b/dspy/teleprompt/ensemble.py
index f7cfb0160..d56d34f1d 100644
--- a/dspy/teleprompt/ensemble.py
+++ b/dspy/teleprompt/ensemble.py
@@ -30,9 +30,18 @@ class Ensemble(Teleprompter):
def forward(self, *args, **kwargs):
programs = random.sample(self.programs, size) if size else self.programs
- outputs = [prog(*args, **kwargs) for prog in programs]
+ outputs = []
+ for prog in programs:
+ try:
+ outputs.append(prog(*args, **kwargs))
+ except Exception as e:
+ print(f"Error for example in dev set: \t\t {e}")
+ pass
if reduce_fn:
+ # print(f"outputs: {outputs}")
+ if len(outputs) == 0:
+ return dspy.Prediction(label="none")
return reduce_fn(outputs)
return outputs
diff --git a/dspy/teleprompt/finetune_teleprompter.py b/dspy/teleprompt/finetune_teleprompter.py
index 7053bcbba..9c48feedf 100644
--- a/dspy/teleprompt/finetune_teleprompter.py
+++ b/dspy/teleprompt/finetune_teleprompter.py
@@ -78,102 +78,8 @@ def prepare_teacher(
return teacher
-
-def build_prompt_completion_data_from_trace(
- trace: List[Dict],
- exclude_demos: bool=False,
- try_to_record_lm_kwargs: bool = False,
- program: Program = None,
- ) -> Dict[str, List[Dict[str, Any]]]:
- """Build prompt completion data from a given trace.
-
- Args:
- trace: The trace from which the prompt-completion data will be built.
- exclude_demos: Exclude the demos from the prompts even if they are
- present in the trace. Defaults to `False`.
- try_to_record_lm_kwargs: Whether to record the LM kwargs in the data.
- Defaults to `False`. If set, the `lm_kwargs` field of the LM used to
- generate the prompt-completion pair is included in the data. To
- find the LM, we first check if the predictor that generated the
- prompt-completion pair has an LM field set (`predictor.lm`). If it
- does, we record it's kwargs. If it doesn't, we get the kwargs from
- `dspy.settings.lm`. If `dspy.settings.lm` is not set either, this
- function will not record the LM kwargs.
- program: Optional argument used to infer the name of the predictor that
- generated the prompt-completion pair. If provided, the returned data
- will include the `predictor_name` field. If not provided, the
- `predictor_name` field is not included in the data, but the caller
- of this function can recover this information by using the
- `predictor_ind` field that' included in the data by default, by
- building the following dictionary:
-
- {ind: n for ind, (n, _) in enumerate(program.named_predictors())}
-
- where `ind` is the index of the predictor in the list returned by
- the `named_predictors()` method of the program. Defaults to `None`.
-
- Returns:
- Data as a list of dictionaries with the keys `prompt`, `completion` and
- optionally with the keys `predictor_name` and `lm_kwargs`. For a given
- prompt-completion pair:
- - The `prompt` field corresponds to the prompt.
- - The `completion` field corresponds to the completion.
- - The `predictor_ind` field corresponds to the index of the predictor
- in the predictor list
- - The `predictor_name` field corresponds to the index of the predictor
- in the list returned by the the named_predictor() method of the
- program used to generate the trace. This field is included only if
- the `pred_ind_to_name` argument is provided.
- - The `lm_kwargs` field corresponds to the LM kwargs that generated the
- prompt-completion pair. Included only if the `record_lm_kwargs` is
- set and there is an active LM.
- """
- # If the program is provided, build the predictor index to name mapping
- if program:
- pred_ind_to_name = {
- ind: name for ind, (name, _) in enumerate(program.named_predictors())
- }
-
- # Build the prompt-completion data
- data = []
- for pred_ind, (pred, inputs, outputs) in enumerate(trace):
- # Get the demos from the predictor if exclude_demos is False
- demos = [] if exclude_demos else pred.demos
-
- # Build prompt and completion strings
- template = signature_to_template(pred.signature)
- prompt = template(Example(demos=demos, **inputs))
- completion = template.query(Example(**outputs))
-
- # TODO: This part of the code could be improved.
- # The method we use to build the completion (template.query) is meant to
- # be used for creating, well, queries, and hence contains field prefixes
- # (e.g. "Reasoning: Let's think step by step in order to"), which are
- # also contained in the last piece of the prompt (separated with a new
- # line) We remove this piece from the completion. This is a hacky
- # solution since it assumes a particular template format.
- prompt_last = prompt.split("\n")[-1]
- completion = completion[len(prompt_last):]
-
- # Create prompt-completion dictionary and add it to the data; optionally
- # add the predictor_name key as well as the lm_kwargs.
- data_dict = dict(prompt=prompt, completion=completion)
-
- # Record the predictor index and optionally, name
- data_dict['predictor_ind'] = pred_ind
- if program:
- data_dict['predictor_name'] = pred_ind_to_name[pred_ind]
-
- # Optionally, record the LM kwargs
- lm = pred.lm or dspy.settings.lm
- if try_to_record_lm_kwargs and lm:
- data_dict['lm_kwargs'] = lm.kwargs
- data.append(data_dict)
-
- return data
-
-
-def convert_to_module_level_prompt_completion_data(
+# TODO: fix docstring
+def convert_to_module_level_message_data(
data: List[Dict],
keep_data_keys: bool = False,
exclude_demos: bool = False,
@@ -214,7 +120,7 @@ def convert_to_module_level_prompt_completion_data(
prompt_completion_data = []
for data_dict in data:
trace = data_dict["trace"]
- trace_prompt_comletion_data = build_prompt_completion_data_from_trace(
+ trace_prompt_comletion_data = build_messages_from_trace(
trace=trace, exclude_demos=exclude_demos,
try_to_record_lm_kwargs=try_to_record_lm_kwargs, program=program
)
@@ -224,6 +130,39 @@ def convert_to_module_level_prompt_completion_data(
prompt_completion_data.append(prompt_completion_dict)
return prompt_completion_data
+# TODO: fix docstring
+def build_messages_from_trace(
+ trace: List[Dict],
+ exclude_demos: bool=False,
+ try_to_record_lm_kwargs: bool = False,
+ program: Program = None,
+ ) -> Dict[str, List[Dict[str, Any]]]:
+ """Build messages from a given trace.
+ """
+ messages = []
+ # If the program is provided, build the predictor index to name mapping
+ if program:
+ pred_ind_to_name = {
+ ind: name for ind, (name, _) in enumerate(program.named_predictors())
+ }
+
+ # Build the prompt-completion data
+
+ adapter = dspy.settings.adapter or dspy.ChatAdapter()
+ data = []
+ for pred_ind, (pred, inputs, outputs) in enumerate(trace):
+ # Get the demos from the predictor if exclude_demos is False
+ demos = [] if exclude_demos else pred.demos
+
+ messages = adapter.format(pred.signature, demos, inputs)
+
+ formatted_completion = adapter.format_completion(pred.signature, outputs)
+ messages.append({"role": "assistant", "content": formatted_completion})
+ data.append(messages)
+
+ return data
+
+
def bootstrap_data(
program: Program,
@@ -232,6 +171,7 @@ def bootstrap_data(
[Example, Prediction, Optional[List]], Union[bool, int, float]
]] = None,
num_threads = 1,
+ max_errors: int = 0
) -> List[Dict[str, Any]]:
"""Bootstrap prediction and trace data for the program using the dataset.
@@ -266,24 +206,25 @@ def bootstrap_data(
info = _INFO_BOOTSTRAP_DATA.format(len(dataset), cname, num_threads)
logger.info(info)
evaluator = Evaluate(
- devset=dataset, num_threads=num_threads, display_progress=True
+ devset=dataset, num_threads=num_threads, display_progress=True, max_errors=max_errors, provide_traceback=True
)
x = evaluator(program, metric=metric)
- print(x)
- data = process_dataset_threaded(dataset, program, metric, num_threads)
+ # print(x)
+ data = process_dataset_threaded(dataset, program, metric, num_threads, max_errors)
return data
-def process_dataset_threaded(dataset: List[Any], program: Callable, metric: Optional[Callable] = None, max_workers: int = None) -> List[Dict[str, Any]]:
+def process_dataset_threaded(dataset: List[Any], program: Callable, metric: Optional[Callable] = None, max_workers: int = None, max_errors: int = 0) -> List[Dict[str, Any]]:
data = []
- num_threads = max_workers if max_workers else min(80, len(dataset)) #TODO: Dont limit to 80, just a temp fix
+ num_threads = max_workers if max_workers else 10
with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
future_to_example = {executor.submit(process_example, example, i, program, metric): i
for i, example in enumerate(dataset)}
for future in concurrent.futures.as_completed(future_to_example):
data_dict = future.result()
- data.append(data_dict)
+ if data_dict is not None:
+ data.append(data_dict)
# Sort the results based on example_ind to maintain original order
data.sort(key=lambda x: x['example_ind'])
@@ -294,7 +235,11 @@ def process_example(example: Any, example_ind: int, program: Callable, metric: O
# print("Processing example:", example_ind)
with dspy.context(trace=[]):
# print("Running program...", example_ind)
- prediction = program(**example.inputs())
+ try:
+ prediction = program(**example.inputs())
+ except Exception as e:
+ print(f"Error processing example {example_ind}: {e}")
+ return None
# print("Getting trace...", example_ind)
trace = dspy.settings.trace
# print("Getting score...", example_ind)
@@ -324,6 +269,7 @@ def bootstrap_data_for_round(
sampling_round: int = 0,
sampling_temperature: Optional[float] = 0.9,
sampling_temperature_delta: float = 0.001,
+ max_errors: int = 0
) -> Union[List[Dict], AssertionError]:
""" Bootstrap data for the given sampling round.
@@ -425,9 +371,9 @@ def bootstrap_data_for_round(
# Collect the data for the given round
with dspy.context(lm=context_lm):
- print(context_lm.kwargs)
+ # print(context_lm.kwargs)
data = bootstrap_data(
- program, dataset, metric=metric, num_threads=num_threads
+ program, dataset, metric=metric, num_threads=num_threads, max_errors=max_errors
)
# Add the round information to the data
diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.py
index 57bd38150..4fa8ba23f 100644
--- a/dspy/teleprompt/random_search.py
+++ b/dspy/teleprompt/random_search.py
@@ -85,6 +85,7 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
max_labeled_demos=self.max_labeled_demos,
teacher_settings=self.teacher_settings,
max_rounds=self.max_rounds,
+ max_errors=self.max_errors,
)
program = optimizer.compile(student, teacher=teacher, trainset=trainset_copy)
@@ -101,6 +102,7 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
max_labeled_demos=self.max_labeled_demos,
teacher_settings=self.teacher_settings,
max_rounds=self.max_rounds,
+ max_errors=self.max_errors,
)
program = optimizer.compile(student, teacher=teacher, trainset=trainset_copy)