Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Remove eval_set

Isaac Millercbdcbafd81fb
diff --git a/dspy/clients/anyscale.py b/dspy/clients/anyscale.py
index c097d5f1f..f2b938a77 100644
--- a/dspy/clients/anyscale.py
+++ b/dspy/clients/anyscale.py
@@ -83,7 +83,6 @@ def finetune_anyscale(
model: str,
method: TrainingMethod,
train_data: List[Dict[str, Any]],
- eval_data: List[Dict[str, Any]] = None,
train_kwargs: Optional[Dict[str, Any]]=None,
) -> List[str]:
"""Start the finetune job."""
@@ -103,21 +102,14 @@ def finetune_anyscale(
logger.error(err)
raise RuntimeError(err)
- if eval_data and not verify_dataset(eval_data):
- err = "[Finetune] Error: Unable to verify that the eval dataset is in the correct format."
- logger.error(err)
- raise RuntimeError(err)
-
logger.info("[Finetune] Converting data to JSONL format...")
train_data_path = save_data(train_data, provider_name=PROVIDER_ANYSCALE)
- eval_data_path = save_data(eval_data, provider_name=PROVIDER_ANYSCALE) if eval_data else None
-
logger.info("[Finetune] Submitting data to remote storage...")
- remote_train_path, _ = submit_data(train_path=train_data_path, eval_path=eval_data_path)
+ remote_train_path, _ = submit_data(train_path=train_data_path)
logger.info(f"[Finetune] Data submitted. Remote train path: {remote_train_path}")
logger.info("[Finetune] Generating configuration files...")
- _, compute_config = generate_config_files(train_path=remote_train_path, eval_path=None, **train_kwargs_copy)
+ _, compute_config = generate_config_files(train_path=remote_train_path, **train_kwargs_copy)
logger.info("[Finetune] Starting remote training...")
job_id = start_remote_training(compute_config=compute_config, **train_kwargs_copy)
@@ -190,13 +182,11 @@ def verify_dataset(dataset: List[dict[str, Any]]) -> bool:
return True
-def submit_data(train_path: str, eval_path: Optional[str]):
+def submit_data(train_path: str):
"""Upload the data to the Workspace cloud storage."""
storage = os.environ['ANYSCALE_ARTIFACT_STORAGE']
datasets = {"train": train_path}
- if eval_path:
- datasets["val"] = eval_path
fine_tuning_file_ids = {}
for name, path in datasets.items():
@@ -217,7 +207,7 @@ def submit_data(train_path: str, eval_path: Optional[str]):
return fine_tuning_file_ids["train"], fine_tuning_file_ids.get("val", None)
-def generate_config_files(train_path: str, eval_path: Optional[str], **kwargs):
+def generate_config_files(train_path: str, **kwargs):
base_model_yaml_path = kwargs.get("train_config_yaml", None)
assert kwargs["model"] is not None, "Model is required to generate the config files"
@@ -255,7 +245,6 @@ def generate_config_files(train_path: str, eval_path: Optional[str], **kwargs):
custom_modifications = {
"model_id": kwargs["model"],
"train_path": train_path,
- "valid_path": eval_path,
"logger": {
"provider": "wandb",
},
diff --git a/dspy/clients/finetune.py b/dspy/clients/finetune.py
index 8cedf5c75..c91b6b557 100644
--- a/dspy/clients/finetune.py
+++ b/dspy/clients/finetune.py
@@ -77,7 +77,7 @@ class FinetuneJob(Future):
)
def __str__(self):
- return f"FinetuningJob({self.model})"
+ return f"FinetuningJob({self.to_dict()})"
def __repr__(self):
return str(self)
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index e5b6d84f8..a7012a874 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -125,7 +125,6 @@ class LM:
def finetune(self,
train_method: TrainingMethod,
train_data: List[Dict[str, Any]],
- eval_data: List[Dict[str, Any]] = None,
train_kwargs: Optional[Dict[str, Any]]=None,
cache_finetune: bool = True,
provider: str = "openai",