Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Make `dspy.BaseLM` extensible (#7950)

Chen Qianfe8c5b751f11
diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py
index 017b8db34..f5d948ee5 100644
--- a/dspy/adapters/base.py
+++ b/dspy/adapters/base.py
@@ -1,9 +1,10 @@
from abc import ABC, abstractmethod
-from typing import Type, Any, Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Optional, Type
from dspy.adapters.types import History
-from dspy.utils.callback import BaseCallback, with_callbacks
from dspy.signatures.signature import Signature
+from dspy.utils.callback import BaseCallback, with_callbacks
+
if TYPE_CHECKING:
from dspy.clients.lm import LM
diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.py
index 885270813..71b56ccc3 100644
--- a/dspy/adapters/chat_adapter.py
+++ b/dspy/adapters/chat_adapter.py
@@ -7,18 +7,18 @@ from collections.abc import Mapping
from typing import Any, Dict, Literal, NamedTuple, Optional, Type
import pydantic
-from pydantic.fields import FieldInfo
from litellm import ContextWindowExceededError
+from pydantic.fields import FieldInfo
from dspy.adapters.base import Adapter
-from dspy.adapters.types.image import try_expand_image_tags
+from dspy.adapters.json_adapter import JSONAdapter
from dspy.adapters.types.history import History
+from dspy.adapters.types.image import try_expand_image_tags
from dspy.adapters.utils import format_field_value, get_annotation_name, parse_value
+from dspy.clients.lm import LM
from dspy.signatures.field import OutputField
from dspy.signatures.signature import Signature, SignatureMeta
from dspy.signatures.utils import get_dspy_field_type
-from dspy.adapters.json_adapter import JSONAdapter
-from dspy.clients.lm import LM
from dspy.utils.callback import BaseCallback
field_header_pattern = re.compile(r"\[\[ ## (\w+) ## \]\]")
diff --git a/dspy/clients/base_lm.py b/dspy/clients/base_lm.py
index fe1fc16ab..c4005d44f 100644
--- a/dspy/clients/base_lm.py
+++ b/dspy/clients/base_lm.py
@@ -1,20 +1,98 @@
-from abc import ABC, abstractmethod
+import datetime
+import uuid
+from abc import ABC
-import dspy
+from dspy.dsp.utils import settings
+from dspy.utils.callback import with_callbacks
GLOBAL_HISTORY = []
+
class BaseLM(ABC):
- def __init__(self, model, model_type='chat', temperature=0.0, max_tokens=1000, cache=True, **kwargs):
+ """Base class for handling LLM calls.
+
+ Most users can directly use the `dspy.LM` class, which is a subclass of `BaseLM`. Users can also implement their
+ own subclasses of `BaseLM` to support custom LLM providers and inject custom logic. To do so, simply override the
+ `forward` method and make sure the return format is identical to the
+ [OpenAI response format](https://platform.openai.com/docs/api-reference/responses/object).
+
+ Example:
+
+ ```python
+ from openai import OpenAI
+
+ import dspy
+
+
+ class MyLM(dspy.BaseLM):
+ def forward(self, prompt, messages=None, **kwargs):
+ client = OpenAI()
+ return client.chat.completions.create(
+ model=self.model,
+ messages=messages or [{"role": "user", "content": prompt}],
+ **self.kwargs,
+ )
+
+
+ lm = MyLM(model="gpt-4o-mini")
+ dspy.configure(lm=lm)
+ print(dspy.Predict("q->a")(q="Why did the chicken cross the kitchen?"))
+ ```
+ """
+
+ def __init__(self, model, model_type="chat", temperature=0.0, max_tokens=1000, cache=True, **kwargs):
self.model = model
self.model_type = model_type
self.cache = cache
self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)
self.history = []
- @abstractmethod
+ @with_callbacks
def __call__(self, prompt=None, messages=None, **kwargs):
- pass
+ response = self.forward(prompt=prompt, messages=messages, **kwargs)
+
+ if kwargs.get("logprobs"):
+ outputs = [
+ {
+ "text": c.message.content if hasattr(c, "message") else c["text"],
+ "logprobs": c.logprobs if hasattr(c, "logprobs") else c["logprobs"],
+ }
+ for c in response.choices
+ ]
+ else:
+ outputs = [c.message.content if hasattr(c, "message") else c["text"] for c in response.choices]
+
+ if settings.disable_history:
+ return outputs
+
+ # Logging, with removed api key & where `cost` is None on cache hit.
+ kwargs = {k: v for k, v in kwargs.items() if not k.startswith("api_")}
+ entry = {
+ "prompt": prompt,
+ "messages": messages,
+ "kwargs": kwargs,
+ "response": response,
+ "outputs": outputs,
+ "usage": dict(response.usage),
+ "cost": getattr(response, "_hidden_params", {}).get("response_cost"),
+ "timestamp": datetime.datetime.now().isoformat(),
+ "uuid": str(uuid.uuid4()),
+ "model": self.model,
+ "response_model": response.model,
+ "model_type": self.model_type,
+ }
+ self.history.append(entry)
+ self.update_global_history(entry)
+
+ return outputs
+
+ def forward(self, prompt=None, messages=None, **kwargs):
+ """Forward pass for the language model.
+
+ Subclasses must implement this method, and the response should be identical to
+ [OpenAI response format](https://platform.openai.com/docs/api-reference/responses/object).
+ """
+ raise NotImplementedError("Subclasses must implement this method.")
def copy(self, **kwargs):
"""Returns a copy of the language model with possibly updated parameters."""
@@ -36,7 +114,7 @@ class BaseLM(ABC):
_inspect_history(self.history, n)
def update_global_history(self, entry):
- if dspy.settings.disable_history:
+ if settings.disable_history:
return
GLOBAL_HISTORY.append(entry)
@@ -49,6 +127,7 @@ def _green(text: str, end: str = "\n"):
def _red(text: str, end: str = "\n"):
return "\x1b[31m" + str(text) + "\x1b[0m" + end
+
def _blue(text: str, end: str = "\n"):
return "\x1b[34m" + str(text) + "\x1b[0m" + end
@@ -92,6 +171,7 @@ def _inspect_history(history, n: int = 1):
print("\n\n\n")
+
def inspect_history(n: int = 1):
"""The global history shared across all LMs."""
return _inspect_history(GLOBAL_HISTORY, n)
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index 0a56d7da3..adaba55e7 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -3,8 +3,6 @@ import logging
import os
import re
import threading
-import uuid
-from datetime import datetime
from hashlib import sha256
from typing import Any, Dict, List, Literal, Optional, cast
@@ -98,7 +96,7 @@ class LM(BaseLM):
self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)
@with_callbacks
- def __call__(self, prompt=None, messages=None, **kwargs):
+ def forward(self, prompt=None, messages=None, **kwargs):
# Build the request.
cache = kwargs.pop("cache", self.cache)
# disable cache will also disable in memory cache
@@ -110,55 +108,20 @@ class LM(BaseLM):
if cache_in_memory:
completion = cached_litellm_completion if self.model_type == "chat" else cached_litellm_text_completion
- response = completion(
+ return completion(
request=dict(model=self.model, messages=messages, **kwargs),
num_retries=self.num_retries,
)
else:
completion = litellm_completion if self.model_type == "chat" else litellm_text_completion
- response = completion(
+ return completion(
request=dict(model=self.model, messages=messages, **kwargs),
num_retries=self.num_retries,
# only leverage LiteLLM cache in this case
cache={"no-cache": not cache, "no-store": not cache},
)
- if kwargs.get("logprobs"):
- outputs = [
- {
- "text": c.message.content if hasattr(c, "message") else c["text"],
- "logprobs": c.logprobs if hasattr(c, "logprobs") else c["logprobs"],
- }
- for c in response["choices"]
- ]
- else:
- outputs = [c.message.content if hasattr(c, "message") else c["text"] for c in response["choices"]]
-
- if dspy.settings.disable_history:
- return outputs
-
- # Logging, with removed api key & where `cost` is None on cache hit.
- kwargs = {k: v for k, v in kwargs.items() if not k.startswith("api_")}
- entry = {
- "prompt": prompt,
- "messages": messages,
- "kwargs": kwargs,
- "response": response,
- "outputs": outputs,
- "usage": dict(response["usage"]),
- "cost": response.get("_hidden_params", {}).get("response_cost"),
- "timestamp": datetime.now().isoformat(),
- "uuid": str(uuid.uuid4()),
- "model": self.model,
- "response_model": response["model"],
- "model_type": self.model_type,
- }
- self.history.append(entry)
- self.update_global_history(entry)
-
- return outputs
-
def launch(self, launch_kwargs: Optional[Dict[str, Any]] = None):
self.provider.launch(self, launch_kwargs)
@@ -185,7 +148,7 @@ class LM(BaseLM):
thread = threading.Thread(target=thread_function_wrapper)
train_kwargs = train_kwargs or self.train_kwargs
- model_to_finetune = self.finetuning_model or self.model
+ model_to_finetune = self.finetuning_model or self.model
job = self.provider.TrainingJob(
thread=thread,
model=model_to_finetune,
@@ -220,8 +183,17 @@ class LM(BaseLM):
return Provider()
def dump_state(self):
- state_keys = ["model", "model_type", "cache", "cache_in_memory", "num_retries", "finetuning_model", "launch_kwargs", "train_kwargs"]
- return { key: getattr(self, key) for key in state_keys } | self.kwargs
+ state_keys = [
+ "model",
+ "model_type",
+ "cache",
+ "cache_in_memory",
+ "num_retries",
+ "finetuning_model",
+ "launch_kwargs",
+ "train_kwargs",
+ ]
+ return {key: getattr(self, key) for key in state_keys} | self.kwargs
def request_cache(maxsize: Optional[int] = None):