DSPy: The framework for programming—not prompting—language models
Python
Make `dspy.BaseLM` extensible (#7950)
Chen Qianfe8c5b751f11
diff --git a/dspy/adapters/base.py b/dspy/adapters/base.pyindex 017b8db34..f5d948ee5 100644--- a/dspy/adapters/base.py+++ b/dspy/adapters/base.py@@ -1,9 +1,10 @@from abc import ABC, abstractmethod-from typing import Type, Any, Optional, TYPE_CHECKING+from typing import TYPE_CHECKING, Any, Optional, Typefrom dspy.adapters.types import History-from dspy.utils.callback import BaseCallback, with_callbacksfrom dspy.signatures.signature import Signature+from dspy.utils.callback import BaseCallback, with_callbacks+if TYPE_CHECKING:from dspy.clients.lm import LMdiff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.pyindex 885270813..71b56ccc3 100644--- a/dspy/adapters/chat_adapter.py+++ b/dspy/adapters/chat_adapter.py@@ -7,18 +7,18 @@ from collections.abc import Mappingfrom typing import Any, Dict, Literal, NamedTuple, Optional, Typeimport pydantic-from pydantic.fields import FieldInfofrom litellm import ContextWindowExceededError+from pydantic.fields import FieldInfofrom dspy.adapters.base import Adapter-from dspy.adapters.types.image import try_expand_image_tags+from dspy.adapters.json_adapter import JSONAdapterfrom dspy.adapters.types.history import History+from dspy.adapters.types.image import try_expand_image_tagsfrom dspy.adapters.utils import format_field_value, get_annotation_name, parse_value+from dspy.clients.lm import LMfrom dspy.signatures.field import OutputFieldfrom dspy.signatures.signature import Signature, SignatureMetafrom dspy.signatures.utils import get_dspy_field_type-from dspy.adapters.json_adapter import JSONAdapter-from dspy.clients.lm import LMfrom dspy.utils.callback import BaseCallbackfield_header_pattern = re.compile(r"\[\[ ## (\w+) ## \]\]")diff --git a/dspy/clients/base_lm.py b/dspy/clients/base_lm.pyindex fe1fc16ab..c4005d44f 100644--- a/dspy/clients/base_lm.py+++ b/dspy/clients/base_lm.py@@ -1,20 +1,98 @@-from abc import ABC, abstractmethod+import datetime+import uuid+from abc import ABC-import dspy+from dspy.dsp.utils import settings+from dspy.utils.callback import with_callbacksGLOBAL_HISTORY = []+class BaseLM(ABC):- def __init__(self, model, model_type='chat', temperature=0.0, max_tokens=1000, cache=True, **kwargs):+ """Base class for handling LLM calls.++ Most users can directly use the `dspy.LM` class, which is a subclass of `BaseLM`. Users can also implement their+ own subclasses of `BaseLM` to support custom LLM providers and inject custom logic. To do so, simply override the+ `forward` method and make sure the return format is identical to the+ [OpenAI response format](https://platform.openai.com/docs/api-reference/responses/object).++ Example:++ ```python+ from openai import OpenAI++ import dspy+++ class MyLM(dspy.BaseLM):+ def forward(self, prompt, messages=None, **kwargs):+ client = OpenAI()+ return client.chat.completions.create(+ model=self.model,+ messages=messages or [{"role": "user", "content": prompt}],+ **self.kwargs,+ )+++ lm = MyLM(model="gpt-4o-mini")+ dspy.configure(lm=lm)+ print(dspy.Predict("q->a")(q="Why did the chicken cross the kitchen?"))+ ```+ """++ def __init__(self, model, model_type="chat", temperature=0.0, max_tokens=1000, cache=True, **kwargs):self.model = modelself.model_type = model_typeself.cache = cacheself.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)self.history = []- @abstractmethod+ @with_callbacksdef __call__(self, prompt=None, messages=None, **kwargs):- pass+ response = self.forward(prompt=prompt, messages=messages, **kwargs)++ if kwargs.get("logprobs"):+ outputs = [+ {+ "text": c.message.content if hasattr(c, "message") else c["text"],+ "logprobs": c.logprobs if hasattr(c, "logprobs") else c["logprobs"],+ }+ for c in response.choices+ ]+ else:+ outputs = [c.message.content if hasattr(c, "message") else c["text"] for c in response.choices]++ if settings.disable_history:+ return outputs++ # Logging, with removed api key & where `cost` is None on cache hit.+ kwargs = {k: v for k, v in kwargs.items() if not k.startswith("api_")}+ entry = {+ "prompt": prompt,+ "messages": messages,+ "kwargs": kwargs,+ "response": response,+ "outputs": outputs,+ "usage": dict(response.usage),+ "cost": getattr(response, "_hidden_params", {}).get("response_cost"),+ "timestamp": datetime.datetime.now().isoformat(),+ "uuid": str(uuid.uuid4()),+ "model": self.model,+ "response_model": response.model,+ "model_type": self.model_type,+ }+ self.history.append(entry)+ self.update_global_history(entry)++ return outputs++ def forward(self, prompt=None, messages=None, **kwargs):+ """Forward pass for the language model.++ Subclasses must implement this method, and the response should be identical to+ [OpenAI response format](https://platform.openai.com/docs/api-reference/responses/object).+ """+ raise NotImplementedError("Subclasses must implement this method.")def copy(self, **kwargs):"""Returns a copy of the language model with possibly updated parameters."""@@ -36,7 +114,7 @@ class BaseLM(ABC):_inspect_history(self.history, n)def update_global_history(self, entry):- if dspy.settings.disable_history:+ if settings.disable_history:returnGLOBAL_HISTORY.append(entry)@@ -49,6 +127,7 @@ def _green(text: str, end: str = "\n"):def _red(text: str, end: str = "\n"):return "\x1b[31m" + str(text) + "\x1b[0m" + end+def _blue(text: str, end: str = "\n"):return "\x1b[34m" + str(text) + "\x1b[0m" + end@@ -92,6 +171,7 @@ def _inspect_history(history, n: int = 1):print("\n\n\n")+def inspect_history(n: int = 1):"""The global history shared across all LMs."""return _inspect_history(GLOBAL_HISTORY, n)diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex 0a56d7da3..adaba55e7 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -3,8 +3,6 @@ import loggingimport osimport reimport threading-import uuid-from datetime import datetimefrom hashlib import sha256from typing import Any, Dict, List, Literal, Optional, cast@@ -98,7 +96,7 @@ class LM(BaseLM):self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)@with_callbacks- def __call__(self, prompt=None, messages=None, **kwargs):+ def forward(self, prompt=None, messages=None, **kwargs):# Build the request.cache = kwargs.pop("cache", self.cache)# disable cache will also disable in memory cache@@ -110,55 +108,20 @@ class LM(BaseLM):if cache_in_memory:completion = cached_litellm_completion if self.model_type == "chat" else cached_litellm_text_completion- response = completion(+ return completion(request=dict(model=self.model, messages=messages, **kwargs),num_retries=self.num_retries,)else:completion = litellm_completion if self.model_type == "chat" else litellm_text_completion- response = completion(+ return completion(request=dict(model=self.model, messages=messages, **kwargs),num_retries=self.num_retries,# only leverage LiteLLM cache in this casecache={"no-cache": not cache, "no-store": not cache},)- if kwargs.get("logprobs"):- outputs = [- {- "text": c.message.content if hasattr(c, "message") else c["text"],- "logprobs": c.logprobs if hasattr(c, "logprobs") else c["logprobs"],- }- for c in response["choices"]- ]- else:- outputs = [c.message.content if hasattr(c, "message") else c["text"] for c in response["choices"]]-- if dspy.settings.disable_history:- return outputs-- # Logging, with removed api key & where `cost` is None on cache hit.- kwargs = {k: v for k, v in kwargs.items() if not k.startswith("api_")}- entry = {- "prompt": prompt,- "messages": messages,- "kwargs": kwargs,- "response": response,- "outputs": outputs,- "usage": dict(response["usage"]),- "cost": response.get("_hidden_params", {}).get("response_cost"),- "timestamp": datetime.now().isoformat(),- "uuid": str(uuid.uuid4()),- "model": self.model,- "response_model": response["model"],- "model_type": self.model_type,- }- self.history.append(entry)- self.update_global_history(entry)-- return outputs-def launch(self, launch_kwargs: Optional[Dict[str, Any]] = None):self.provider.launch(self, launch_kwargs)@@ -185,7 +148,7 @@ class LM(BaseLM):thread = threading.Thread(target=thread_function_wrapper)train_kwargs = train_kwargs or self.train_kwargs- model_to_finetune = self.finetuning_model or self.model+ model_to_finetune = self.finetuning_model or self.modeljob = self.provider.TrainingJob(thread=thread,model=model_to_finetune,@@ -220,8 +183,17 @@ class LM(BaseLM):return Provider()def dump_state(self):- state_keys = ["model", "model_type", "cache", "cache_in_memory", "num_retries", "finetuning_model", "launch_kwargs", "train_kwargs"]- return { key: getattr(self, key) for key in state_keys } | self.kwargs+ state_keys = [+ "model",+ "model_type",+ "cache",+ "cache_in_memory",+ "num_retries",+ "finetuning_model",+ "launch_kwargs",+ "train_kwargs",+ ]+ return {key: getattr(self, key) for key in state_keys} | self.kwargsdef request_cache(maxsize: Optional[int] = None):