DSPy: The framework for programming—not prompting—language models
Python
[WIP] Arbor Interface Update (#8837)
Noah Ziems5359996d9355
diff --git a/docs/docs/tutorials/rl_multihop/index.ipynb b/docs/docs/tutorials/rl_multihop/index.ipynbindex 24f19cfc0..b2757b0a9 100644--- a/docs/docs/tutorials/rl_multihop/index.ipynb+++ b/docs/docs/tutorials/rl_multihop/index.ipynb@@ -11,21 +11,8 @@"For this tutorial, you will also need DSPy's Arbor RL server.\n","\n","```bash\n",- "> pip install arbor-ai\n",- "> python -m arbor.cli serve --arbor-config arbor.yaml\n",- "```\n",- "\n",- "where you create `arbor.yaml` in your directory, containing a plan like:\n",- "\n",- "```text\n",- "inference:\n",- " gpu_ids: '0'\n",- "\n",- "training:\n",- " gpu_ids: '1, 2'\n",- "```\n",- "\n",- "which assigns GPU 0 for inference and GPUs 1 and 2 for training."+ "> pip install -U arbor-ai\n",+ "```"]},{@@ -37,14 +24,16 @@"import dspy\n","from dspy.clients.lm_local_arbor import ArborProvider\n","\n",+ "import arbor\n",+ "arbor_server_info = arbor.init() # Initialize the Arbor server in the background\n",+ "\n","port = 7453\n","local_lm_name = \"Qwen/Qwen2.5-7B-Instruct\"\n","local_lm = dspy.LM(\n"," model=f\"openai/arbor:{local_lm_name}\",\n"," provider=ArborProvider(),\n"," temperature=0.7,\n",- " api_base=f\"http://localhost:{port}/v1/\",\n",- " api_key=\"arbor\",\n",+ " api_base=arbor_server_info[\"api_base\"],\n",")\n","\n","dspy.configure(lm=local_lm)\n",@@ -238,6 +227,7 @@"outputs": [],"source": ["from dspy.teleprompt.grpo import GRPO\n",+ "from dspy.clients.utils_finetune import MultiGPUConfig\n","\n","program = ResearchHop(num_docs=4, num_hops=2)\n","program.set_lm(local_lm)\n",@@ -245,10 +235,10 @@"# NOTE: Training on 6 GPUs.\n","train_kwargs = {\n"," \"per_device_train_batch_size\": 2,\n",- " \"gradient_accumulation_steps\": 4,\n",- " \"temperature\": 0.7,\n",+ " \"gradient_accumulation_steps\": 8,\n",+ " \"temperature\": 1.0,\n"," \"beta\": 0.04,\n",- " \"learning_rate\": 2e-5,\n",+ " \"learning_rate\": 1e-5,\n"," \"gradient_checkpointing\": True,\n"," \"gradient_checkpointing_kwargs\": {\"use_reentrant\": False},\n"," \"bf16\": True,\n",@@ -262,16 +252,16 @@"\n","compiler = GRPO(\n"," metric=recall,\n",- " multitask=True,\n"," num_dspy_examples_per_grpo_step=6,\n",- " num_samples_per_input=8,\n",+ " num_rollouts_per_grpo_step=4,\n"," exclude_demos=True,\n",- " num_train_steps=500,\n",- " num_threads=24,\n",+ " num_train_steps=100,\n",+ " num_threads=16,\n"," use_train_as_val=False,\n"," num_steps_for_val=10,\n"," train_kwargs=train_kwargs,\n"," report_train_scores=False,\n",+ " gpu_config=MultiGPUConfig(num_inference_gpus=1, num_training_gpus=1),\n",")\n","\n","optimized_program = compiler.compile(\n",@@ -304,11 +294,6 @@"source": ["In our preliminary experiments, training above for about 18 hours boosts the recall (devset) from 61.8% to 66.2%. This is _typically_ worse on cost/quality basis than you'd get from running prompt optimizers dspy.MIPROv2 or dspy.SIMBA, but it's still a very solid start for online RL over arbitrary LM programs for small LMs."]- },- {- "cell_type": "markdown",- "metadata": {},- "source": []}],"metadata": {diff --git a/docs/docs/tutorials/rl_papillon/index.ipynb b/docs/docs/tutorials/rl_papillon/index.ipynbindex 247f685b5..5049924dc 100644--- a/docs/docs/tutorials/rl_papillon/index.ipynb+++ b/docs/docs/tutorials/rl_papillon/index.ipynb@@ -15,21 +15,8 @@"For this tutorial, you will also need the Arbor RL server.\n","\n","```bash\n",- "> pip install arbor-ai\n",- "> python -m arbor.cli serve --arbor-config arbor.yaml\n",- "```\n",- "\n",- "where you create `arbor.yaml` in your directory, containing a plan like:\n",- "\n",- "```text\n",- "inference:\n",- " gpu_ids: '0'\n",- "\n",- "training:\n",- " gpu_ids: '1, 2'\n",- "```\n",- "\n",- "which assigns GPU 0 for inference and GPUs 1 and 2 for training."+ "> pip install -U arbor-ai\n",+ "```"]},{@@ -41,14 +28,16 @@"import dspy\n","from dspy.clients.lm_local_arbor import ArborProvider\n","\n",+ "import arbor\n",+ "arbor_server_info = arbor.init() # Initialize the Arbor server in the background\n",+ "\n","port = 7453\n",- "local_lm_name = \"Qwen/Qwen3-1.7B\"\n",+ "local_lm_name = \"Qwen/Qwen2.5-7B-Instruct\"\n","local_lm = dspy.LM(\n"," model=f\"openai/arbor:{local_lm_name}\",\n"," provider=ArborProvider(),\n"," temperature=0.7,\n",- " api_base=f\"http://localhost:{port}/v1/\",\n",- " api_key=\"arbor\",\n",+ " api_base=arbor_server_info[\"api_base\"],\n",")\n","\n","dspy.configure(lm=local_lm)\n",@@ -267,6 +256,7 @@"outputs": [],"source": ["from dspy.teleprompt.grpo import GRPO\n",+ "from dspy.clients.utils_finetune import MultiGPUConfig\n","\n","papillon = PAPILLON(untrusted_model=openai_lm)\n","papillon.set_lm(local_lm)\n",@@ -275,7 +265,7 @@"train_kwargs = {\n"," \"per_device_train_batch_size\": 8,\n"," \"gradient_accumulation_steps\": 4,\n",- " \"temperature\": 0.7,\n",+ " \"temperature\": 1.0,\n"," \"beta\": 0.04,\n"," \"learning_rate\": 2e-6,\n"," \"gradient_checkpointing\": True,\n",@@ -301,6 +291,7 @@" num_steps_for_val=10,\n"," train_kwargs=train_kwargs,\n"," report_train_scores=False,\n",+ " gpu_config=MultiGPUConfig(num_inference_gpus=2, num_training_gpus=2),\n",")\n","\n","optimized_papillon = compiler.compile(\n",diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex 3c133a7c0..76f5b90c7 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -13,7 +13,7 @@ import dspyfrom dspy.clients.cache import request_cachefrom dspy.clients.openai import OpenAIProviderfrom dspy.clients.provider import Provider, ReinforceJob, TrainingJob-from dspy.clients.utils_finetune import TrainDataFormat+from dspy.clients.utils_finetune import MultiGPUConfig, TrainDataFormatfrom dspy.dsp.utils.settings import settingsfrom dspy.utils.callback import BaseCallback@@ -237,14 +237,14 @@ class LM(BaseLM):return job- def reinforce(self, train_kwargs) -> ReinforceJob:+ def reinforce(self, train_kwargs, gpu_config: MultiGPUConfig = MultiGPUConfig(num_inference_gpus=1, num_training_gpus=1)) -> ReinforceJob:# TODO(GRPO Team): Should we return an initialized job here?from dspy import settings as settingserr = f"Provider {self.provider} does not implement the reinforcement learning interface."assert self.provider.reinforceable, err- job = self.provider.ReinforceJob(lm=self, train_kwargs=train_kwargs)+ job = self.provider.ReinforceJob(lm=self, train_kwargs=train_kwargs, gpu_config=gpu_config)job.initialize()return jobdiff --git a/dspy/clients/lm_local_arbor.py b/dspy/clients/lm_local_arbor.pyindex 9b07bdefb..c4c6fd992 100644--- a/dspy/clients/lm_local_arbor.py+++ b/dspy/clients/lm_local_arbor.py@@ -1,13 +1,14 @@import timefrom datetime import datetimefrom typing import TYPE_CHECKING, Any, TypedDict+from urllib.parse import urljoinimport openaiimport requestsimport dspyfrom dspy.clients.provider import Provider, ReinforceJob, TrainingJob-from dspy.clients.utils_finetune import GRPOGroup, TrainDataFormat, TrainingStatus, save_data+from dspy.clients.utils_finetune import GRPOGroup, MultiGPUConfig, TrainDataFormat, TrainingStatus, save_dataif TYPE_CHECKING:from dspy.clients.lm import LM@@ -70,7 +71,7 @@ class ArborReinforceJob(ReinforceJob):"lora": False,}- def __init__(self, lm: "LM", train_kwargs: GRPOTrainKwargs):+ def __init__(self, lm: "LM", train_kwargs: GRPOTrainKwargs, gpu_config: MultiGPUConfig = MultiGPUConfig(num_inference_gpus=1, num_training_gpus=1)):# The teleprompter must ensure that this is setif "num_generations" not in train_kwargs:raise ValueError("num_generations must be set in the training kwargs")@@ -80,6 +81,7 @@ class ArborReinforceJob(ReinforceJob):self.provider_job_id = Noneself.checkpoints = {}self.last_checkpoint = None+ self.gpu_config = gpu_configdef initialize(self):# TODO(GRPO Team): Set provider job ID@@ -118,6 +120,8 @@ class ArborReinforceJob(ReinforceJob):api_base = self.lm.kwargs["api_base"]finetune_model = ArborProvider._remove_provider_prefix(self.lm.model)+ # Only multi-GPU is supported for now+ gpu_config_type = "multi"data = {"model": finetune_model,"num_generations": num_generations,@@ -140,8 +144,12 @@ class ArborReinforceJob(ReinforceJob):"logging_steps": logging_steps,"max_context_length": max_context_length,"lora": lora,+ "gpu_config": {+ "type": gpu_config_type,+ gpu_config_type: self.gpu_config,+ },}- url = f"{api_base}fine_tuning/grpo/initialize"+ url = urljoin(api_base, "fine_tuning/grpo/initialize")headers = {"Content-Type": "application/json"}response = requests.post(url=url, headers=headers, json=data)assert response.status_code == 200, f"Failed to initialize GRPO: {response}"@@ -158,7 +166,7 @@ class ArborReinforceJob(ReinforceJob):finetune_model = ArborProvider._remove_provider_prefix(self.lm.model)data = {"job_id": self.provider_job_id, "model": finetune_model, "batch": train_group}- url = f"{api_base}fine_tuning/grpo/step"+ url = urljoin(api_base, f"fine_tuning/grpo/{self.provider_job_id}/step")headers = {"Content-Type": "application/json"}response = requests.post(url, headers=headers, json=data)assert response.status_code == 200, f"Failed to run a GRPO step: {response.text}"@@ -184,7 +192,7 @@ class ArborReinforceJob(ReinforceJob):def save_checkpoint(self, checkpoint_name: str, score: float | None = None):api_base = self.lm.kwargs["api_base"]- url = f"{api_base}fine_tuning/grpo/checkpoint"+ url = urljoin(api_base, f"fine_tuning/grpo/{self.provider_job_id}/checkpoint")headers = {"Content-Type": "application/json"}body = {"job_id": self.provider_job_id, "checkpoint_name": checkpoint_name}response = requests.post(url, headers=headers, json=body)@@ -203,7 +211,7 @@ class ArborReinforceJob(ReinforceJob):def terminate(self):api_base = self.lm.kwargs["api_base"]- url = f"{api_base}fine_tuning/grpo/terminate"+ url = urljoin(api_base, f"fine_tuning/grpo/{self.provider_job_id}/terminate")headers = {"Content-Type": "application/json"}body = {"job_id": self.provider_job_id}response = requests.post(url, headers=headers, json=body)@@ -214,14 +222,15 @@ class ArborReinforceJob(ReinforceJob):self.lm.model = ArborProvider._add_provider_prefix(current_model)def cancel(self):- if ArborProvider.does_job_exist(self.provider_job_id):- status = self.status()- if ArborProvider.is_terminal_training_status(status):- err_msg = "Jobs that are complete cannot be canceled."- err_msg += f" Job with ID {self.provider_job_id} is done."- raise Exception(err_msg)- openai.fine_tuning.jobs.cancel(self.provider_job_id)- self.provider_job_id = None+ if self.provider_job_id:+ api_base = self.lm.kwargs["api_base"]+ url = urljoin(api_base, f"fine_tuning/grpo/{self.provider_job_id}/cancel")+ headers = {"Content-Type": "application/json"}+ response = requests.post(url, headers=headers)+ if response.status_code == 200:+ self.provider_job_id = None+ else:+ raise Exception(f"Failed to cancel GRPO job: {response.text}")def status(self) -> TrainingStatus:status = ArborProvider.get_training_status(self.provider_job_id)@@ -245,7 +254,7 @@ class ArborProvider(Provider):launch_kwargs = launch_kwargs or lm.launch_kwargs# Make request to launch endpoint- response = requests.post(f"{api_base}chat/launch", json={"model": model, "launch_kwargs": launch_kwargs})+ response = requests.post(urljoin(api_base, "chat/launch"), json={"model": model, "launch_kwargs": launch_kwargs})if response.status_code != 200:raise Exception(f"Failed to launch model. Status code: {response.status_code}, Response: {response.text}")@@ -257,7 +266,7 @@ class ArborProvider(Provider):api_base = lm.kwargs["api_base"]response = requests.post(- f"{api_base}chat/kill",+ urljoin(api_base, "chat/kill"),)if response.status_code != 200:diff --git a/dspy/clients/provider.py b/dspy/clients/provider.pyindex 5425136c2..1e7e47316 100644--- a/dspy/clients/provider.py+++ b/dspy/clients/provider.py@@ -3,7 +3,7 @@ from concurrent.futures import Futurefrom threading import Threadfrom typing import TYPE_CHECKING, Any-from dspy.clients.utils_finetune import TrainDataFormat+from dspy.clients.utils_finetune import MultiGPUConfig, TrainDataFormatif TYPE_CHECKING:from dspy.clients.lm import LM@@ -36,11 +36,14 @@ class TrainingJob(Future):class ReinforceJob:- def __init__(self, lm: "LM", train_kwargs: dict[str, Any] | None = None):+ def __init__(self, lm: "LM", train_kwargs: dict[str, Any] | None = None, gpu_config: MultiGPUConfig = MultiGPUConfig(num_inference_gpus=1, num_training_gpus=1)):self.lm = lmself.train_kwargs = train_kwargs or {}+ self.gpu_config = gpu_configself.checkpoints = {}self.last_checkpoint = None+ self.gpu_config = gpu_config+@abstractmethoddef initialize(self):diff --git a/dspy/clients/utils_finetune.py b/dspy/clients/utils_finetune.pyindex 7ec3380cb..8e6a49518 100644--- a/dspy/clients/utils_finetune.py+++ b/dspy/clients/utils_finetune.py@@ -43,6 +43,13 @@ class GRPOChatData(TypedDict):GRPOGroup = list[GRPOChatData]+class MultiGPUConfig(TypedDict):+ # Number of GPUs to use for inference+ num_inference_gpus: int+ # Number of GPUs to use for training+ num_training_gpus: int++def infer_data_format(adapter: Adapter) -> str:if isinstance(adapter, dspy.ChatAdapter):return TrainDataFormat.CHATdiff --git a/dspy/teleprompt/grpo.py b/dspy/teleprompt/grpo.pyindex 9b58aac3f..d3f80935e 100644--- a/dspy/teleprompt/grpo.py+++ b/dspy/teleprompt/grpo.py@@ -6,7 +6,7 @@ from typing import Any, Callable, Literalfrom dspy.adapters.base import Adapterfrom dspy.adapters.chat_adapter import ChatAdapterfrom dspy.clients.lm import LM-from dspy.clients.utils_finetune import GRPOGroup, TrainDataFormat+from dspy.clients.utils_finetune import GRPOGroup, MultiGPUConfig, TrainDataFormatfrom dspy.dsp.utils.settings import settingsfrom dspy.evaluate.evaluate import Evaluatefrom dspy.primitives.example import Example@@ -41,6 +41,7 @@ class GRPO(FinetuneTeleprompter):format_failure_score: float = -1,variably_invoked_predictor_grouping_mode: Literal["truncate"] | Literal["fill"] | Literal["ragged"] = "truncate",variably_invoked_predictor_fill_strategy: Literal["randint"] | Literal["max"] | None = None,+ gpu_config: MultiGPUConfig = MultiGPUConfig(num_inference_gpus=1, num_training_gpus=1),):super().__init__(train_kwargs=train_kwargs)self.metric = metric@@ -57,6 +58,7 @@ class GRPO(FinetuneTeleprompter):self.report_train_scores = report_train_scoresself.failure_score = failure_scoreself.format_failure_score = format_failure_score+ self.gpu_config = gpu_configassert failure_score > format_failure_score, "failure_score must be greater than format_failure_score since the range [format_failure_score, failure_score] is used to provide dspy formatting rewards"@@ -332,7 +334,7 @@ class GRPO(FinetuneTeleprompter):job_key = (pred.lm, data_key)if job_key not in grpo_training_jobs:train_kwargs = self.train_kwargs[pred.lm]- job = pred.lm.reinforce(train_kwargs=train_kwargs)+ job = pred.lm.reinforce(train_kwargs=train_kwargs, gpu_config=self.gpu_config)grpo_training_jobs[job_key] = jobself.report_validation_metrics(