Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Support async in DSPy critical paths (#8080)

Chen Qian4f3b9f35c8ab
diff --git a/dspy/adapters/base.py b/dspy/adapters/base.py
index bc69a5af6..0fa3b5d65 100644
--- a/dspy/adapters/base.py
+++ b/dspy/adapters/base.py
@@ -20,17 +20,7 @@ class Adapter:
cls.format = with_callbacks(cls.format)
cls.parse = with_callbacks(cls.parse)
- def __call__(
- self,
- lm: "LM",
- lm_kwargs: dict[str, Any],
- signature: Type[Signature],
- demos: list[dict[str, Any]],
- inputs: dict[str, Any],
- ) -> list[dict[str, Any]]:
- inputs = self.format(signature, demos, inputs)
-
- outputs = lm(messages=inputs, **lm_kwargs)
+ def _call_post_process(self, outputs: list[dict[str, Any]], signature: Type[Signature]) -> list[dict[str, Any]]:
values = []
for output in outputs:
@@ -48,6 +38,32 @@ class Adapter:
return values
+ def __call__(
+ self,
+ lm: "LM",
+ lm_kwargs: dict[str, Any],
+ signature: Type[Signature],
+ demos: list[dict[str, Any]],
+ inputs: dict[str, Any],
+ ) -> list[dict[str, Any]]:
+ inputs = self.format(signature, demos, inputs)
+
+ outputs = lm(messages=inputs, **lm_kwargs)
+ return self._call_post_process(outputs, signature)
+
+ async def acall(
+ self,
+ lm: "LM",
+ lm_kwargs: dict[str, Any],
+ signature: Type[Signature],
+ demos: list[dict[str, Any]],
+ inputs: dict[str, Any],
+ ) -> list[dict[str, Any]]:
+ inputs = self.format(signature, demos, inputs)
+
+ outputs = await lm.acall(messages=inputs, **lm_kwargs)
+ return self._call_post_process(outputs, signature)
+
def format(
self,
signature: Type[Signature],
diff --git a/dspy/adapters/two_step_adapter.py b/dspy/adapters/two_step_adapter.py
index 1004b0ff7..02cb7750e 100644
--- a/dspy/adapters/two_step_adapter.py
+++ b/dspy/adapters/two_step_adapter.py
@@ -1,13 +1,11 @@
from typing import Any, Type
-from dspy.signatures.signature import Signature
from dspy.adapters.base import Adapter
from dspy.adapters.chat_adapter import ChatAdapter
from dspy.adapters.utils import get_field_description_string
-from dspy.signatures.field import InputField
-from dspy.signatures.signature import make_signature
from dspy.clients import LM
-
+from dspy.signatures.field import InputField
+from dspy.signatures.signature import Signature, make_signature
"""
NOTE/TODO/FIMXE:
@@ -101,6 +99,48 @@ class TwoStepAdapter(Adapter):
except Exception as e:
raise ValueError(f"Failed to parse response from the original completion: {completion}") from e
+ async def acall(
+ self,
+ lm: "LM",
+ lm_kwargs: dict[str, Any],
+ signature: Type[Signature],
+ demos: list[dict[str, Any]],
+ inputs: dict[str, Any],
+ ) -> list[dict[str, Any]]:
+ inputs = self.format(signature, demos, inputs)
+
+ outputs = await lm.acall(messages=inputs, **lm_kwargs)
+ # The signature is supposed to be "text -> {original output fields}"
+ extractor_signature = self._create_extractor_signature(signature)
+
+ values = []
+
+ for output in outputs:
+ output_logprobs = None
+
+ if isinstance(output, dict):
+ output, output_logprobs = output["text"], output["logprobs"]
+
+ try:
+ # Call the smaller LM to extract structured data from the raw completion text with ChatAdapter
+ value = await ChatAdapter().acall(
+ lm=self.extraction_model,
+ lm_kwargs={},
+ signature=extractor_signature,
+ demos=[],
+ inputs={"text": output},
+ )
+ value = value[0]
+
+ except Exception as e:
+ raise ValueError(f"Failed to parse response from the original completion: {output}") from e
+
+ if output_logprobs is not None:
+ value["logprobs"] = output_logprobs
+
+ values.append(value)
+ return values
+
def format_task_description(self, signature: Signature) -> str:
"""Create a description of the task based on the signature"""
parts = []
diff --git a/dspy/clients/base_lm.py b/dspy/clients/base_lm.py
index 016f1e3bf..c0a181232 100644
--- a/dspy/clients/base_lm.py
+++ b/dspy/clients/base_lm.py
@@ -48,10 +48,7 @@ class BaseLM(ABC):
self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)
self.history = []
- @with_callbacks
- def __call__(self, prompt=None, messages=None, **kwargs):
- response = self.forward(prompt=prompt, messages=messages, **kwargs)
-
+ def _process_lm_response(self, response, prompt, messages, **kwargs):
if kwargs.get("logprobs"):
outputs = [
{
@@ -84,7 +81,18 @@ class BaseLM(ABC):
}
self.history.append(entry)
self.update_global_history(entry)
+ return outputs
+
+ @with_callbacks
+ def __call__(self, prompt=None, messages=None, **kwargs):
+ response = self.forward(prompt=prompt, messages=messages, **kwargs)
+ outputs = self._process_lm_response(response, prompt, messages, **kwargs)
+
+ return outputs
+ async def acall(self, prompt=None, messages=None, **kwargs):
+ response = await self.aforward(prompt=prompt, messages=messages, **kwargs)
+ outputs = self._process_lm_response(response, prompt, messages, **kwargs)
return outputs
def forward(self, prompt=None, messages=None, **kwargs):
@@ -95,6 +103,14 @@ class BaseLM(ABC):
"""
raise NotImplementedError("Subclasses must implement this method.")
+ async def aforward(self, prompt=None, messages=None, **kwargs):
+ """Async forward pass for the language model.
+
+ Subclasses that support async should implement this method, and the response should be identical to
+ [OpenAI response format](https://platform.openai.com/docs/api-reference/responses/object).
+ """
+ raise NotImplementedError("Subclasses must implement this method.")
+
def copy(self, **kwargs):
"""Returns a copy of the language model with possibly updated parameters."""
@@ -170,7 +186,7 @@ def _inspect_history(history, n: int = 1):
print(_green(outputs[0].strip()))
if len(outputs) > 1:
- choices_text = f" \t (and {len(outputs)-1} other completions)"
+ choices_text = f" \t (and {len(outputs) - 1} other completions)"
print(_red(choices_text, end=""))
print("\n\n\n")
diff --git a/dspy/clients/embedding.py b/dspy/clients/embedding.py
index 9dee002a9..0c08f188d 100644
--- a/dspy/clients/embedding.py
+++ b/dspy/clients/embedding.py
@@ -78,60 +78,83 @@ class Embedder:
self.caching = caching
self.default_kwargs = kwargs
+ def _preprocess(self, inputs, batch_size=None, caching=None, **kwargs):
+ if isinstance(inputs, str):
+ is_single_input = True
+ inputs = [inputs]
+ else:
+ is_single_input = False
+
+ if not all(isinstance(inp, str) for inp in inputs):
+ raise ValueError("All inputs must be strings.")
+
+ batch_size = batch_size or self.batch_size
+ caching = caching or self.caching
+ merged_kwargs = self.default_kwargs.copy()
+ merged_kwargs.update(kwargs)
+
+ input_batches = []
+ for i in range(0, len(inputs), batch_size):
+ input_batches.append(inputs[i : i + batch_size])
+
+ return input_batches, caching, merged_kwargs, is_single_input
+
+ def _postprocess(self, embeddings_list, is_single_input):
+ embeddings = np.array(embeddings_list, dtype=np.float32)
+ if is_single_input:
+ return embeddings[0]
+ else:
+ return np.array(embeddings, dtype=np.float32)
+
def __call__(self, inputs, batch_size=None, caching=None, **kwargs):
"""Compute embeddings for the given inputs.
Args:
inputs: The inputs to compute embeddings for, can be a single string or a list of strings.
- batch_size (int, optional): The batch size for processing inputs. If None, defaults to the batch_size set during initialization.
- caching (bool, optional): Whether to cache the embedding response when using a hosted model. If None, defaults to the caching setting from initialization.
- **kwargs: Additional keyword arguments to pass to the embedding model. These will override the default kwargs provided during initialization.
+ batch_size (int, optional): The batch size for processing inputs. If None, defaults to the batch_size set
+ during initialization.
+ caching (bool, optional): Whether to cache the embedding response when using a hosted model. If None,
+ defaults to the caching setting from initialization.
+ **kwargs: Additional keyword arguments to pass to the embedding model. These will override the default
+ kwargs provided during initialization.
Returns:
numpy.ndarray: If the input is a single string, returns a 1D numpy array representing the embedding.
If the input is a list of strings, returns a 2D numpy array of embeddings, one embedding per row.
"""
+ input_batches, caching, kwargs, is_single_input = self._preprocess(inputs, batch_size, caching, **kwargs)
- if isinstance(inputs, str):
- is_single_input = True
- inputs = [inputs]
- else:
- is_single_input = False
+ embeddings_list = []
- assert all(isinstance(inp, str) for inp in inputs), "All inputs must be strings."
+ for batch in input_batches:
+ if isinstance(self.model, str):
+ embedding_response = litellm.embedding(model=self.model, input=batch, caching=caching, **kwargs)
+ batch_embeddings = [data["embedding"] for data in embedding_response.data]
+ elif callable(self.model):
+ batch_embeddings = self.model(batch, **kwargs)
+ else:
+ raise ValueError(
+ f"`model` in `dspy.Embedder` must be a string or a callable, but got {type(self.model)}."
+ )
- if batch_size is None:
- batch_size = self.batch_size
- if caching is None:
- caching = self.caching
+ embeddings_list.extend(batch_embeddings)
+ return self._postprocess(embeddings_list, is_single_input)
- merged_kwargs = self.default_kwargs.copy()
- merged_kwargs.update(kwargs)
+ async def acall(self, inputs, batch_size=None, caching=None, **kwargs):
+ input_batches, caching, kwargs, is_single_input = self._preprocess(inputs, batch_size, caching, **kwargs)
embeddings_list = []
- def chunk(inputs_list, size):
- for i in range(0, len(inputs_list), size):
- yield inputs_list[i : i + size]
-
- for batch_inputs in chunk(inputs, batch_size):
+ for batch in input_batches:
if isinstance(self.model, str):
- embedding_response = litellm.embedding(
- model=self.model, input=batch_inputs, caching=caching, **merged_kwargs
- )
+ embedding_response = await litellm.aembedding(model=self.model, input=batch, caching=caching, **kwargs)
batch_embeddings = [data["embedding"] for data in embedding_response.data]
elif callable(self.model):
- batch_embeddings = self.model(batch_inputs, **merged_kwargs)
+ batch_embeddings = self.model(batch, **kwargs)
else:
raise ValueError(
f"`model` in `dspy.Embedder` must be a string or a callable, but got {type(self.model)}."
)
embeddings_list.extend(batch_embeddings)
-
- embeddings = np.array(embeddings_list, dtype=np.float32)
-
- if is_single_input:
- return embeddings[0]
- else:
- return embeddings
+ return self._postprocess(embeddings_list, is_single_input)
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index e70aa8652..fe9f6d1f8 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -89,9 +89,9 @@ class LM(BaseLM):
if model_pattern:
# Handle OpenAI reasoning models (o1, o3)
- assert (
- max_tokens >= 20_000 and temperature == 1.0
- ), "OpenAI's reasoning models require passing temperature=1.0 and max_tokens >= 20_000 to `dspy.LM(...)`"
+ assert max_tokens >= 20_000 and temperature == 1.0, (
+ "OpenAI's reasoning models require passing temperature=1.0 and max_tokens >= 20_000 to `dspy.LM(...)`"
+ )
self.kwargs = dict(temperature=temperature, max_completion_tokens=max_tokens, **kwargs)
else:
self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)
@@ -127,6 +127,15 @@ class LM(BaseLM):
settings.usage_tracker.add_usage(self.model, dict(results.usage))
return results
+ async def aforward(self, prompt=None, messages=None, **kwargs):
+ completion = alitellm_completion if self.model_type == "chat" else alitellm_text_completion
+
+ results = await completion(
+ request=dict(model=self.model, messages=messages, **kwargs),
+ num_retries=self.num_retries,
+ )
+ return results
+
def launch(self, launch_kwargs: Optional[Dict[str, Any]] = None):
self.provider.launch(self, launch_kwargs)
@@ -368,6 +377,51 @@ def litellm_text_completion(request: Dict[str, Any], num_retries: int, cache={"n
)
+async def alitellm_completion(request: Dict[str, Any], num_retries: int, cache={"no-cache": True, "no-store": True}):
+ retry_kwargs = dict(
+ retry_policy=_get_litellm_retry_policy(num_retries),
+ retry_strategy="exponential_backoff_retry",
+ # In LiteLLM version 1.55.3 (the first version that supports retry_policy as an argument
+ # to completion()), the default value of max_retries is non-zero for certain providers, and
+ # max_retries is stacked on top of the retry_policy. To avoid this, we set max_retries=0
+ max_retries=0,
+ )
+
+ return await litellm.acompletion(
+ cache=cache,
+ **retry_kwargs,
+ **request,
+ )
+
+
+async def alitellm_text_completion(
+ request: Dict[str, Any], num_retries: int, cache={"no-cache": True, "no-store": True}
+):
+ model = request.pop("model").split("/", 1)
+ provider, model = model[0] if len(model) > 1 else "openai", model[-1]
+
+ # Use the API key and base from the request, or from the environment.
+ api_key = request.pop("api_key", None) or os.getenv(f"{provider}_API_KEY")
+ api_base = request.pop("api_base", None) or os.getenv(f"{provider}_API_BASE")
+
+ # Build the prompt from the messages.
+ prompt = "\n\n".join([x["content"] for x in request.pop("messages")] + ["BEGIN RESPONSE:"])
+
+ return await litellm.atext_completion(
+ cache=cache,
+ model=f"text-completion-openai/{model}",
+ api_key=api_key,
+ api_base=api_base,
+ prompt=prompt,
+ retry_policy=_get_litellm_retry_policy(num_retries),
+ # In LiteLLM version 1.55.3 (the first version that supports retry_policy as an argument
+ # to completion()), the default value of max_retries is non-zero for certain providers, and
+ # max_retries is stacked on top of the retry_policy. To avoid this, we set max_retries=0
+ max_retries=0,
+ **request,
+ )
+
+
def _get_litellm_retry_policy(num_retries: int) -> RetryPolicy:
"""
Get a LiteLLM retry policy for retrying requests when transient API errors occur.
diff --git a/dspy/predict/chain_of_thought.py b/dspy/predict/chain_of_thought.py
index d7b56448b..3d980942c 100644
--- a/dspy/predict/chain_of_thought.py
+++ b/dspy/predict/chain_of_thought.py
@@ -1,23 +1,24 @@
+from typing import Optional, Type, Union
+
+from pydantic.fields import FieldInfo
+
import dspy
from dspy.primitives.program import Module
from dspy.signatures.field import OutputField
-from dspy.signatures.signature import ensure_signature, Signature
-from pydantic.fields import FieldInfo
-from typing import Optional, Union, Type
+from dspy.signatures.signature import Signature, ensure_signature
class ChainOfThought(Module):
-
def __init__(
- self,
- signature: Type[Signature],
- rationale_field: Optional[Union[OutputField, FieldInfo]] = None,
+ self,
+ signature: Type[Signature],
+ rationale_field: Optional[Union[OutputField, FieldInfo]] = None,
rationale_field_type: Type = str,
- **config
+ **config,
):
"""
A module that reasons step by step in order to predict the output of a task.
-
+
Args:
signature (Type[dspy.Signature]): The signature of the module.
rationale_field (Optional[Union[dspy.OutputField, pydantic.fields.FieldInfo]]): The field that will contain the reasoning.
@@ -35,3 +36,6 @@ class ChainOfThought(Module):
def forward(self, **kwargs):
return self.predict(**kwargs)
+
+ async def aforward(self, **kwargs):
+ return await self.predict.acall(**kwargs)
diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py
index 618d086cc..e6c800dd1 100644
--- a/dspy/predict/predict.py
+++ b/dspy/predict/predict.py
@@ -75,7 +75,10 @@ class Predict(Module, Parameter):
def __call__(self, **kwargs):
return self.forward(**kwargs)
- def forward(self, **kwargs):
+ async def acall(self, **kwargs):
+ return await self.aforward(**kwargs)
+
+ def _forward_preprocess(self, **kwargs):
# Extract the three privileged keyword arguments.
assert "new_signature" not in kwargs, "new_signature is no longer a valid keyword argument."
signature = ensure_signature(kwargs.pop("signature", self.signature))
@@ -101,6 +104,17 @@ class Predict(Module, Parameter):
present,
missing,
)
+ return lm, config, signature, demos, kwargs
+
+ def _forward_postprocess(self, completions, signature, **kwargs):
+ pred = Prediction.from_completions(completions, signature=signature)
+ if kwargs.pop("_trace", True) and settings.trace is not None:
+ trace = settings.trace
+ trace.append((self, {**kwargs}, pred))
+ return pred
+
+ def forward(self, **kwargs):
+ lm, config, signature, demos, kwargs = self._forward_preprocess(**kwargs)
adapter = settings.adapter or ChatAdapter()
@@ -116,13 +130,14 @@ class Predict(Module, Parameter):
with settings.context(send_stream=None):
completions = adapter(lm, lm_kwargs=config, signature=signature, demos=demos, inputs=kwargs)
- pred = Prediction.from_completions(completions, signature=signature)
+ return self._forward_postprocess(completions, signature, **kwargs)
- if kwargs.pop("_trace", True) and settings.trace is not None:
- trace = settings.trace
- trace.append((self, {**kwargs}, pred))
+ async def aforward(self, **kwargs):
+ lm, config, signature, demos, kwargs = self._forward_preprocess(**kwargs)
- return pred
+ adapter = settings.adapter or ChatAdapter()
+ completions = await adapter.acall(lm, lm_kwargs=config, signature=signature, demos=demos, inputs=kwargs)
+ return self._forward_postprocess(completions, signature, **kwargs)
def update_config(self, **kwargs):
self.config = {**self.config, **kwargs}
diff --git a/dspy/primitives/program.py b/dspy/primitives/program.py
index a5f88f17a..23ccdec1f 100644
--- a/dspy/primitives/program.py
+++ b/dspy/primitives/program.py
@@ -1,6 +1,7 @@
-import magicattr
from typing import Optional
+import magicattr
+
from dspy.dsp.utils.settings import settings
from dspy.predict.parallel import Parallel
from dspy.primitives.module import BaseModule
@@ -30,6 +31,15 @@ class Module(BaseModule, metaclass=ProgramMeta):
return self.forward(*args, **kwargs)
+ async def acall(self, *args, **kwargs):
+ if settings.track_usage and settings.usage_tracker is None:
+ with track_usage() as usage_tracker:
+ output = await self.aforward(*args, **kwargs)
+ output.set_lm_usage(usage_tracker.get_total_tokens())
+ return output
+
+ return await self.aforward(*args, **kwargs)
+
def named_predictors(self):
from dspy.predict.predict import Predict
diff --git a/dspy/utils/dummies.py b/dspy/utils/dummies.py
index 9b96012b9..99a028fd0 100644
--- a/dspy/utils/dummies.py
+++ b/dspy/utils/dummies.py
@@ -129,6 +129,9 @@ class DummyLM(LM):
return outputs
+ async def acall(self, prompt=None, messages=None, **kwargs):
+ return self.__call__(prompt=prompt, messages=messages, **kwargs)
+
def get_convo(self, index):
"""Get the prompt + answer from the ith message."""
return self.history[index]["messages"], self.history[index]["outputs"]
diff --git a/tests/adapters/test_chat_adapter.py b/tests/adapters/test_chat_adapter.py
index 4866ac013..ce22e3fcf 100644
--- a/tests/adapters/test_chat_adapter.py
+++ b/tests/adapters/test_chat_adapter.py
@@ -77,3 +77,20 @@ def test_chat_adapter_quotes_literals_as_expected(
assert expected_input_str in content
assert expected_output_str in content
+
+
+def test_chat_adapter_sync_call():
+ signature = dspy.make_signature("question->answer")
+ adapter = dspy.ChatAdapter()
+ lm = dspy.utils.DummyLM([{"answer": "Paris"}])
+ result = adapter(lm, {}, signature, [], {"question": "What is the capital of France?"})
+ assert result == [{"answer": "Paris"}]
+
+
+async def test_chat_adapter_async_call():
+ signature = dspy.make_signature("question->answer")
+ adapter = dspy.ChatAdapter()
+ lm = dspy.utils.DummyLM([{"answer": "Paris"}])
+ result = await adapter.acall(lm, {}, signature, [], {"question": "What is the capital of France?"})
+ assert result == [{"answer": "Paris"}]
diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.py
index 68de08a7f..d2e79c63f 100644
--- a/tests/adapters/test_json_adapter.py
+++ b/tests/adapters/test_json_adapter.py
@@ -93,3 +93,20 @@ def test_json_adapter_with_structured_outputs_does_not_mutate_original_signature
program(input1="Test input")
assert program.signature.output_fields == TestSignature.output_fields
+
+
+def test_json_adapter_sync_call():
+ signature = dspy.make_signature("question->answer")
+ adapter = dspy.ChatAdapter()
+ lm = dspy.utils.DummyLM([{"answer": "Paris"}])
+ result = adapter(lm, {}, signature, [], {"question": "What is the capital of France?"})
+ assert result == [{"answer": "Paris"}]
+
+
+async def test_json_adapter_async_call():
+ signature = dspy.make_signature("question->answer")
+ adapter = dspy.ChatAdapter()
+ lm = dspy.utils.DummyLM([{"answer": "Paris"}])
+ result = await adapter.acall(lm, {}, signature, [], {"question": "What is the capital of France?"})
+ assert result == [{"answer": "Paris"}]
diff --git a/tests/adapters/test_two_step_adapter.py b/tests/adapters/test_two_step_adapter.py
index 304d4651e..cf83c3dbd 100644
--- a/tests/adapters/test_two_step_adapter.py
+++ b/tests/adapters/test_two_step_adapter.py
@@ -68,6 +68,70 @@ def test_two_step_adapter_call():
content = call_kwargs["messages"][1]["content"]
assert "text from main LM" in content
+async def test_two_step_adapter_async_call():
+ class TestSignature(dspy.Signature):
+ question: str = dspy.InputField(desc="The math question to solve")
+ solution: str = dspy.OutputField(desc="Step by step solution")
+ answer: float = dspy.OutputField(desc="The final numerical answer")
+
+ program = dspy.Predict(TestSignature)
+
+ mock_main_lm = mock.MagicMock(spec=dspy.LM)
+ mock_main_lm.acall.return_value = ["text from main LM"]
+ mock_main_lm.kwargs = {"temperature": 1.0}
+
+ mock_extraction_lm = mock.MagicMock(spec=dspy.LM)
+ mock_extraction_lm.acall.return_value = [
+ """
+[[ ## solution ## ]] result
+[[ ## answer ## ]] 12
+[[ ## completed ## ]]
+"""
+ ]
+ mock_extraction_lm.kwargs = {"temperature": 1.0}
+ mock_extraction_lm.model = "openai/gpt-4o"
+
+ dspy.configure(lm=mock_main_lm, adapter=dspy.TwoStepAdapter(extraction_model=mock_extraction_lm))
+
+ result = await program.acall(question="What is 5 + 7?")
+
+ assert result.answer == 12
+
+ # main LM call
+ mock_main_lm.acall.assert_called_once()
+ _, call_kwargs = mock_main_lm.acall.call_args
+ assert len(call_kwargs["messages"]) == 2
+
+ # assert first message
+ assert call_kwargs["messages"][0]["role"] == "system"
+ content = call_kwargs["messages"][0]["content"]
+ assert "1. `question` (str)" in content
+ assert "1. `solution` (str)" in content
+ assert "2. `answer` (float)" in content
+
+ # assert second message
+ assert call_kwargs["messages"][1]["role"] == "user"
+ content = call_kwargs["messages"][1]["content"]
+ assert "question:" in content.lower()
+ assert "What is 5 + 7?" in content
+
+ # extraction LM call
+ mock_extraction_lm.acall.assert_called_once()
+ _, call_kwargs = mock_extraction_lm.acall.call_args
+ assert len(call_kwargs["messages"]) == 2
+
+ # assert first message
+ assert call_kwargs["messages"][0]["role"] == "system"
+ content = call_kwargs["messages"][0]["content"]
+ assert "`text` (str)" in content
+ assert "`solution` (str)" in content
+ assert "`answer` (float)" in content
+
+ # assert second message
+ assert call_kwargs["messages"][1]["role"] == "user"
+ content = call_kwargs["messages"][1]["content"]
+ assert "text from main LM" in content
def test_two_step_adapter_parse():
class ComplexSignature(dspy.Signature):
diff --git a/tests/clients/test_embedding.py b/tests/clients/test_embedding.py
index 0ac9e24ba..38425cef1 100644
--- a/tests/clients/test_embedding.py
+++ b/tests/clients/test_embedding.py
@@ -62,3 +62,27 @@ def test_invalid_model_type():
with pytest.raises(ValueError):
embedding = Embedder(123) # Invalid model type
embedding(["test"])
+
+
+async def test_async_embedding():
+ model = "text-embedding-ada-002"
+ inputs = ["hello", "world"]
+ mock_embeddings = [
+ [0.1, 0.2, 0.3], # embedding for "hello"
+ [0.4, 0.5, 0.6], # embedding for "world"
+ ]
+
+ with patch("litellm.aembedding") as mock_litellm:
+ # Configure mock to return proper response format.
+ mock_litellm.return_value = MockEmbeddingResponse(mock_embeddings)
+
+ # Create embedding instance and call it.
+ embedding = Embedder(model)
+ result = await embedding.acall(inputs)
+
+ # Verify litellm was called with correct parameters.
+ mock_litellm.assert_called_once_with(model=model, input=inputs, caching=True)
+
+ assert len(result) == len(inputs)
+ np.testing.assert_allclose(result, mock_embeddings)
diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.py
index a68e20d72..637b087ea 100644
--- a/tests/clients/test_lm.py
+++ b/tests/clients/test_lm.py
@@ -274,3 +274,12 @@ def test_exponential_backoff_retry():
# The first retry happens immediately regardless of the configuration
for i in range(1, len(time_counter) - 1):
assert time_counter[i + 1] - time_counter[i] >= 2 ** (i - 1)
+
+
+async def test_async_lm_call(litellm_test_server):
+ api_base, _ = litellm_test_server
+
+ lm = dspy.LM(model="openai/gpt-4o-mini", api_base=api_base, api_key="fakekey", model_type="chat")
+ result = await lm.acall("question")
+ assert result == "answer"
diff --git a/tests/predict/test_chain_of_thought.py b/tests/predict/test_chain_of_thought.py
index c050819ef..5b6198368 100644
--- a/tests/predict/test_chain_of_thought.py
+++ b/tests/predict/test_chain_of_thought.py
@@ -2,6 +2,8 @@ import dspy
from dspy import ChainOfThought
from dspy.utils import DummyLM
+import pytest
+
def test_initialization_with_string_signature():
lm = DummyLM([{"reasoning": "find the number after 1", "answer": "2"}])
@@ -12,3 +14,12 @@ def test_initialization_with_string_signature():
"answer",
]
assert predict(question="What is 1+1?").answer == "2"
+
+
+async def test_async_chain_of_thought():
+ lm = DummyLM([{"reasoning": "find the number after 1", "answer": "2"}])
+ dspy.settings.configure(lm=lm)
+ program = ChainOfThought("question -> answer")
+ result = await program.acall(question="What is 1+1?")
+ assert result.answer == "2"
diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py
index dcd7b9421..9384667b1 100644
--- a/tests/predict/test_predict.py
+++ b/tests/predict/test_predict.py
@@ -569,3 +569,11 @@ def test_litellm_cache_initialization_failure():
# No exception should be raised when litellm.cache is None even if we try to use the cache.
assert dspy.settings.lm.cache == True
predictor(question="test")
+
+
+async def test_async_predict():
+ program = Predict("question -> answer")
+ dspy.settings.configure(lm=DummyLM([{"answer": "Paris"}]))
+ result = await program.acall(question="What is the capital of France?")
+ assert result.answer == "Paris"