DSPy: The framework for programming—not prompting—language models
Python
perf(dspy): defer anyio, openai and jiter to their use sites (#10381)
Adriaan Moorsb04a07335bb8
diff --git a/dspy/clients/engines/lifecycle.py b/dspy/clients/engines/lifecycle.pyindex 45c543692..c966850cb 100644--- a/dspy/clients/engines/lifecycle.py+++ b/dspy/clients/engines/lifecycle.py@@ -3,7 +3,9 @@import inspectfrom contextlib import asynccontextmanager, contextmanager-import anyio+from dspy.utils.lazy_import import require++anyio = require("anyio")def _secondary(primary, cleanup):diff --git a/dspy/clients/execution.py b/dspy/clients/execution.pyindex b1dc8c342..2b8759b8f 100644--- a/dspy/clients/execution.py+++ b/dspy/clients/execution.py@@ -7,7 +7,6 @@ from contextlib import nullcontextfrom dataclasses import dataclass, replacefrom typing import Any-import anyioimport pydanticfrom dspy._vendor.lm15.result import StreamAccumulator@@ -36,6 +35,9 @@ from dspy.lm15 import (request_from_openai_chat,)from dspy.utils.exceptions import LMUnsupportedFeatureError, is_retryable_lm_error+from dspy.utils.lazy_import import require++anyio = require("anyio")IGNORED_CACHE_KEYS = ["api_key", "api_base", "base_url"]diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex 0d0487b08..a3b30b1d2 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -4,10 +4,7 @@ import osimport reimport threadingimport warnings-from typing import Any, Literal, cast--import anyio.from_thread-from anyio.streams.memory import MemoryObjectSendStream+from typing import TYPE_CHECKING, Any, Literal, castimport dspyfrom dspy.clients._litellm import get_litellm@@ -21,9 +18,15 @@ from dspy.clients.utils_finetune import TrainDataFormatfrom dspy.lm15 import CacheConfigfrom dspy.utils.callback import BaseCallbackfrom dspy.utils.exceptions import LMConfigurationError, LMError, LMUnsupportedFeatureError+from dspy.utils.lazy_import import requirefrom .base_lm import BaseLM+anyio = require("anyio")++if TYPE_CHECKING:+ from anyio.streams.memory import MemoryObjectSendStream+logger = logging.getLogger(__name__)@@ -452,7 +455,7 @@ def _get_stream_completion_fn(return None# The stream is already opened, and will be closed by the caller.- stream = cast(MemoryObjectSendStream, stream)+ stream = cast("MemoryObjectSendStream", stream)caller_predict_id = id(caller_predict) if caller_predict else Noneif dspy.settings.track_usage:diff --git a/dspy/clients/openai.py b/dspy/clients/openai.pyindex f687249d7..cd7c195c5 100644--- a/dspy/clients/openai.py+++ b/dspy/clients/openai.py@@ -2,10 +2,11 @@ import timefrom datetime import datetimefrom typing import Any-import openai-from dspy.clients.provider import Provider, TrainingJobfrom dspy.clients.utils_finetune import TrainDataFormat, TrainingStatus, save_data+from dspy.utils.lazy_import import require++openai = require("openai")class TrainingJobOpenAI(TrainingJob):diff --git a/dspy/streaming/messages.py b/dspy/streaming/messages.pyindex 0c58e19ec..19e1f915d 100644--- a/dspy/streaming/messages.py+++ b/dspy/streaming/messages.py@@ -3,10 +3,11 @@ import concurrent.futuresfrom dataclasses import dataclassfrom typing import Any-import anyio.from_thread-from dspy.dsp.utils.settings import settingsfrom dspy.utils.callback import BaseCallback+from dspy.utils.lazy_import import require++anyio = require("anyio")@dataclassdiff --git a/dspy/streaming/streamify.py b/dspy/streaming/streamify.pyindex 5fd3ea436..b3a64e517 100644--- a/dspy/streaming/streamify.py+++ b/dspy/streaming/streamify.py@@ -7,14 +7,15 @@ from queue import Queuefrom typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, Generatorimport orjson-from anyio import create_memory_object_stream, create_task_group-from anyio.streams.memory import MemoryObjectSendStreamfrom dspy.dsp.utils.settings import settingsfrom dspy.primitives.prediction import Predictionfrom dspy.streaming.messages import StatusMessage, StatusMessageProvider, StatusStreamingCallbackfrom dspy.streaming.streaming_listener import StreamListener, find_predictor_for_stream_listenersfrom dspy.utils.asyncify import asyncify+from dspy.utils.lazy_import import require++anyio = require("anyio")logger = logging.getLogger(__name__)@@ -29,6 +30,8 @@ def _is_litellm_model_response_stream(value: Any) -> bool:if TYPE_CHECKING:+ from anyio.streams.memory import MemoryObjectSendStream+from dspy.primitives.module import Module@@ -174,15 +177,15 @@ def streamify(if not any(isinstance(c, StatusStreamingCallback) for c in callbacks):callbacks.append(status_streaming_callback)- async def generator(args, kwargs, stream: MemoryObjectSendStream):+ async def generator(args, kwargs, stream: "MemoryObjectSendStream"):with settings.context(send_stream=stream, callbacks=callbacks, stream_listeners=stream_listeners):prediction = await program(*args, **kwargs)await stream.send(prediction)async def async_streamer(*args, **kwargs):- send_stream, receive_stream = create_memory_object_stream(16)- async with create_task_group() as tg, send_stream, receive_stream:+ send_stream, receive_stream = anyio.create_memory_object_stream(16)+ async with anyio.create_task_group() as tg, send_stream, receive_stream:tg.start_soon(generator, args, kwargs, send_stream)async for value in receive_stream:diff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.pyindex d38112ab6..32e9f9eb5 100644--- a/dspy/streaming/streaming_listener.py+++ b/dspy/streaming/streaming_listener.py@@ -6,14 +6,15 @@ from collections import defaultdictfrom queue import Queuefrom typing import TYPE_CHECKING, Any-import jiter-from dspy.adapters.chat_adapter import ChatAdapterfrom dspy.adapters.json_adapter import JSONAdapterfrom dspy.adapters.types import Typefrom dspy.adapters.xml_adapter import XMLAdapterfrom dspy.dsp.utils.settings import settingsfrom dspy.streaming.messages import StreamResponse+from dspy.utils.lazy_import import require++jiter = require("jiter")if TYPE_CHECKING:from litellm import ModelResponseStreamdiff --git a/dspy/utils/asyncify.py b/dspy/utils/asyncify.pyindex 7f9259af6..9bc3ae435 100644--- a/dspy/utils/asyncify.py+++ b/dspy/utils/asyncify.py@@ -1,8 +1,9 @@import functoolsfrom typing import TYPE_CHECKING, Any, Awaitable, Callable-import anyio-from anyio import CapacityLimiter+from dspy.utils.lazy_import import require++anyio = require("anyio")if TYPE_CHECKING:from dspy.primitives.module import Module@@ -21,7 +22,7 @@ def get_limiter():global _limiterif _limiter is None:- _limiter = CapacityLimiter(async_max_workers)+ _limiter = anyio.CapacityLimiter(async_max_workers)elif _limiter.total_tokens != async_max_workers:_limiter.total_tokens = async_max_workers