Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python
19.9 KB
Raw Blame
"""Failed streams, cancellation, and post-completion work cannot replay a generation."""

import asyncio

import anyio
import pytest

import dspy
from dspy import lm15
from dspy.clients.engines.lifecycle import aclosing_stream, closing_stream
from dspy.utils.callback import BaseCallback
from dspy.utils.usage_tracker import track_usage
from tests.clients.test_engine_errors import AsyncEngine, Engine, Sink, invoke


def events(case):
    start = lm15.StreamStartEvent()  # a missing model/id is legitimate
    text = lm15.StreamDeltaEvent(lm15.TextDelta("partial"))
    end = lm15.StreamEndEvent(finish_reason="stop", usage=lm15.Usage(input_tokens=2, output_tokens=1))
    error = lm15.StreamErrorEvent(lm15.ErrorDetail(code="auth", message="denied"))
    return {
        "empty": [], "missing-end": [start, text], "missing-start": [text, end],
        "duplicate-start": [start, start], "duplicate-end": [start, text, end, end],
        "after-end": [start, text, end, text], "unknown-event": [start, object()],
        "error-first": [error], "error-midstream": [start, text, error],
    }[case]


class StreamEngine(Engine):
    def __init__(self, case):
        super().__init__()
        self.case = case
        self.closed = 0

    def stream(self, request):
        self.calls += 1
        try:
            yield from events(self.case)
        finally:
            self.closed += 1


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("case", ["empty", "missing-end", "missing-start", "duplicate-start", "duplicate-end", "after-end", "unknown-event", "error-first", "error-midstream"])
async def test_all_streams_are_validated_and_failures_are_not_cached(case, asynchronous):
    engine = StreamEngine(case)
    lm = dspy.LM("custom", engine=engine, async_engine=AsyncEngine(engine), cache=True, num_retries=2)
    expected = dspy.LMAuthError if case.startswith("error-") else dspy.LMStreamAssemblyError
    with dspy.context(send_stream=Sink()), track_usage() as tracker:
        for _ in range(2):
            with pytest.raises(expected) as caught:
                await invoke(lm, asynchronous, "hello")
            if case == "missing-end":
                assert caught.value.partial.text == "partial"
    assert engine.calls == 2  # no retries and no partial-success cache hits
    assert engine.closed == 2
    assert not lm.history
    assert not dspy.cache.memory_cache
    usage = tracker.get_total_tokens()
    if case in ("duplicate-end", "after-end"):
        assert usage[lm.model]["total_tokens"] == 6
    else:
        assert usage == {}


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_transient_stream_open_failure_retries_but_failure_after_output_does_not(asynchronous):
    class Transient(Engine):
        def stream(self, request):
            self.calls += 1
            if self.calls == 1:
                raise lm15.RateLimitError("wait", retry_after=0)
            yield lm15.StreamStartEvent()
            yield lm15.StreamDeltaEvent(lm15.TextDelta("ok"))
            yield lm15.StreamEndEvent(finish_reason="stop")

    engine = Transient()
    lm = dspy.LM("custom", engine=engine, async_engine=AsyncEngine(engine), cache=False, num_retries=1)
    with dspy.context(send_stream=Sink()):
        assert await invoke(lm, asynchronous, "hello") == ["ok"]
    assert engine.calls == 2

    class Partial(Engine):
        def stream(self, request):
            self.calls += 1
            yield lm15.StreamStartEvent()
            yield lm15.StreamDeltaEvent(lm15.TextDelta("partial"))
            raise lm15.RateLimitError("wait", retry_after=0)

    engine = Partial()
    lm = dspy.LM("custom", engine=engine, async_engine=AsyncEngine(engine), cache=False, num_retries=1)
    with dspy.context(send_stream=Sink()), pytest.raises(dspy.LMRateLimitError):
        await invoke(lm, asynchronous, "hello")
    assert engine.calls == 1


class BrokenClose:
    def __init__(self, primary):
        self.primary = primary
        self.cleanup = RuntimeError("cleanup failure")
        self.closed = 0

    def __iter__(self):
        return self

    def __next__(self):
        raise self.primary

    def __aiter__(self):
        return self

    async def __anext__(self):
        raise self.primary

    def close(self):
        self.closed += 1
        raise self.cleanup

    async def aclose(self):
        self.close()


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("cancelled", [False, True])
async def test_primary_error_and_cancellation_survive_cleanup(asynchronous, cancelled):
    primary = asyncio.CancelledError("cancelled") if cancelled else lm15.AuthError("denied")
    source = BrokenClose(primary)
    if asynchronous:
        with pytest.raises(type(primary)) as caught:
            async with aclosing_stream(source):
                await source.__anext__()
    else:
        with pytest.raises(type(primary)) as caught:
            with closing_stream(source):
                next(source)
    assert caught.value is primary
    assert primary.cleanup_errors == (source.cleanup,)
    assert source.closed == 1


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_cleanup_diagnostics_survive_public_projection(asynchronous):
    source = BrokenClose(lm15.AuthError("denied"))

    class Backend(Engine):
        def stream(self, request):
            return source

    class AsyncBackend(AsyncEngine):
        def stream(self, request):
            return source

    lm = dspy.LM("custom", engine=Backend(), async_engine=AsyncBackend(None), cache=False, num_retries=0)
    with dspy.context(send_stream=Sink()), pytest.raises(dspy.LMAuthError) as caught:
        await invoke(lm, asynchronous, "hello")
    assert caught.value.__cause__ is source.primary
    assert caught.value.cleanup_errors == (source.cleanup,)


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("phase", ["pricing", "cache", "output-conversion"])
async def test_post_completion_failure_does_not_retry_and_keeps_usage(phase, asynchronous, monkeypatch):
    import dspy.clients.execution as execution

    engine = Engine()
    errors = []

    class Callback(BaseCallback):
        def on_lm_end(self, call_id, outputs, exception):
            errors.append(exception)

    lm = dspy.LM("custom", engine=engine, async_engine=AsyncEngine(engine), cache=True,
                 num_retries=3, callbacks=[Callback()])
    failure = dspy.LMRateLimitError("injected post-completion failure", retry_after=0)

    def fail(*args, **kwargs):
        raise failure

    monkeypatch.setattr(execution, {"pricing": "_price_result", "cache": "_store", "output-conversion": "_result"}[phase], fail)
    with track_usage() as tracker, pytest.raises(dspy.LMRateLimitError) as caught:
        await invoke(lm, asynchronous, "hello")
    assert caught.value is failure
    assert engine.calls == 1
    assert tracker.get_total_tokens()[lm.model]["total_tokens"] == 3
    assert errors == [failure]
    assert not lm.history
    assert not dspy.cache.memory_cache


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_completed_stream_cleanup_failure_does_not_retry_and_keeps_usage(asynchronous):
    failure = lm15.TransportError("close failed")

    class Source:
        def __init__(self):
            self.iterator = iter([lm15.StreamStartEvent(), lm15.StreamDeltaEvent(lm15.TextDelta("ok")),
                                  lm15.StreamEndEvent(finish_reason="stop", usage=lm15.Usage(input_tokens=2, output_tokens=1))])

        def __iter__(self):
            return self

        def __next__(self):
            return next(self.iterator)

        def __aiter__(self):
            return self

        async def __anext__(self):
            try:
                return next(self.iterator)
            except StopIteration:
                raise StopAsyncIteration from None

        def close(self):
            raise failure

        async def aclose(self):
            self.close()

    class Backend(Engine):
        def stream(self, request):
            self.calls += 1
            return Source()

    class AsyncBackend(AsyncEngine):
        def stream(self, request):
            return self.sync.stream(request)

    engine = Backend()
    lm = dspy.LM("custom", engine=engine, async_engine=AsyncBackend(engine), cache=True, num_retries=2)
    with dspy.context(send_stream=Sink()), track_usage() as tracker, pytest.raises(dspy.LMTransportError):
        await invoke(lm, asynchronous, "hello")
    assert engine.calls == 1
    assert tracker.get_total_tokens()[lm.model]["total_tokens"] == 3
    assert not dspy.cache.memory_cache


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_unnamed_tool_call_preserves_partial_usage(asynchronous):
    class Backend(Engine):
        def stream(self, request):
            yield lm15.StreamStartEvent()
            yield lm15.StreamDeltaEvent(lm15.ToolCallDelta(input="{}", id="call-1"))
            yield lm15.StreamEndEvent(finish_reason="tool_call", usage=lm15.Usage(input_tokens=2, output_tokens=1))

    engine = Backend()
    lm = dspy.LM("custom", engine=engine, async_engine=AsyncEngine(engine), cache=True, num_retries=2)
    with dspy.context(send_stream=Sink()), track_usage() as tracker, pytest.raises(dspy.LMStreamAssemblyError) as caught:
        await invoke(lm, asynchronous, "hello")
    assert caught.value.partial is not None
    assert tracker.get_total_tokens()[lm.model]["total_tokens"] == 3
    assert not dspy.cache.memory_cache


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_parser_bug_is_not_hidden_by_adapter_fallback(asynchronous):
    class Broken(dspy.ChatAdapter):
        def parse(self, signature, completion):
            raise RuntimeError("parser implementation bug")

    engine = Engine()
    lm = dspy.LM("custom", engine=engine, async_engine=AsyncEngine(engine), cache=False)
    args = (lm, {}, dspy.Signature("question -> answer"), [], {"question": "hi"})
    with pytest.raises(RuntimeError, match="parser implementation bug"):
        if asynchronous:
            await Broken().acall(*args)
        else:
            Broken()(*args)
    assert engine.calls == 1


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_parse_failure_after_visible_stream_never_falls_back(asynchronous):
    # Engine returns plain 'ok', which is not a valid ChatAdapter answer.
    engine = Engine()
    lm = dspy.LM("custom", engine=engine, async_engine=AsyncEngine(engine), cache=False)
    args = (lm, {}, dspy.Signature("question -> answer"), [], {"question": "hi"})
    adapter = dspy.ChatAdapter()
    with dspy.context(send_stream=Sink()), pytest.raises(dspy.AdapterParseError):
        if asynchronous:
            await adapter.acall(*args)
        else:
            await anyio.to_thread.run_sync(lambda: adapter(*args))
    assert engine.calls == 1


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_json_schema_fallback_is_only_for_setup(asynchronous, monkeypatch):
    import dspy.adapters.json_adapter as json_adapter

    class Backend(Engine):
        supports_response_schema = True
        supported_params = frozenset({"response_format"})

        def complete(self, request):
            response = super().complete(request)
            self.format = request.config.response_format
            return lm15.Response(None, request.model, lm15.Message.assistant('{"answer":"ok"}'), "stop", response.usage)

    engine = Backend()
    lm = dspy.LM("custom", engine=engine, async_engine=AsyncEngine(engine), cache=False)
    adapter = dspy.JSONAdapter()
    args = (lm, {}, dspy.Signature("question -> answer"), [], {"question": "hi"})

    def bad_schema(*args):
        raise ValueError("schema cannot be represented")

    monkeypatch.setattr(json_adapter, "_get_structured_outputs_response_format", bad_schema)
    result = await adapter.acall(*args) if asynchronous else adapter(*args)
    assert result == [{"answer": "ok"}]
    assert engine.calls == 1
    assert engine.format == {"type": "json_object"}

    def bug(*args):
        raise RuntimeError("schema builder bug")

    monkeypatch.setattr(json_adapter, "_get_structured_outputs_response_format", bug)
    with pytest.raises(RuntimeError, match="schema builder bug"):
        if asynchronous:
            await adapter.acall(*args)
        else:
            adapter(*args)
    assert engine.calls == 1


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_materializing_backend_keeps_completed_usage_on_cleanup_failure(asynchronous):
    from dspy._vendor.lm15.result import amaterialize_response, materialize_response

    class Source(BrokenClose):
        def __init__(self):
            super().__init__(None)
            response = lm15.Response(None, "custom", lm15.Message.assistant("ok"), "stop", lm15.Usage(input_tokens=2, output_tokens=1))
            self.iterator = iter(lm15.response_to_events(response))

        def __next__(self):
            return next(self.iterator)

        async def __anext__(self):
            try:
                return next(self.iterator)
            except StopIteration:
                raise StopAsyncIteration from None

    class Backend(Engine):
        def source(self):
            self.calls += 1
            return Source()

        def complete(self, request):
            return materialize_response(self.source(), request)

    class AsyncBackend(AsyncEngine):
        async def complete(self, request):
            return await amaterialize_response(self.sync.source(), request)

    engine = Backend()
    lm = dspy.LM("custom", engine=engine, async_engine=AsyncBackend(engine), cache=True, num_retries=2)
    # lm15 1.0.0rc1 (contract 2026-09-11-stream-completion): a failure after
    # the end event never withholds the complete Response — it is returned,
    # the failure is a StreamCleanupWarning.  No retry, usage kept, cached.
    with track_usage() as tracker, pytest.warns(lm15.StreamCleanupWarning):
        outputs = await invoke(lm, asynchronous, "hello")
    assert outputs == ["ok"]
    assert tracker.get_total_tokens()[lm.model]["total_tokens"] == 3
    assert engine.calls == 1
    assert dspy.cache.memory_cache


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_compatibility_stream_keeps_usage_if_sdk_close_fails(asynchronous, monkeypatch):
    import litellm

    class Source:
        def __init__(self):
            self.sent = False

        def __aiter__(self):
            return self

        async def __anext__(self):
            if self.sent:
                raise StopAsyncIteration
            self.sent = True
            return {"choices": [{"delta": {"content": "ok"}, "finish_reason": "stop"}]}

        async def aclose(self):
            raise lm15.TransportError("cleanup failed")

    calls = []

    async def complete(**kwargs):
        calls.append(kwargs)
        return Source()

    raw = litellm.ModelResponse(model="fake", choices=[{"message": {"role": "assistant", "content": "ok"}}],
                               usage={"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3})
    monkeypatch.setattr(litellm, "acompletion", complete)
    monkeypatch.setattr(litellm, "stream_chunk_builder", lambda chunks: raw)
    lm = dspy.LM("openai/fake", engine="litellm", cache=True, num_retries=2)
    with dspy.context(send_stream=Sink()), track_usage() as tracker, pytest.raises(dspy.LMTransportError):
        await invoke(lm, asynchronous, "hello")
    assert len(calls) == 1
    assert tracker.get_total_tokens()[lm.model]["total_tokens"] == 3
    assert not dspy.cache.memory_cache


@pytest.mark.parametrize("adapter,module,text", [
    (dspy.ChatAdapter(), "dspy.adapters.chat_adapter", '[[ ## answer ## ]]\n{"value": 1}'),
    (dspy.JSONAdapter(), "dspy.adapters.json_adapter", '{"answer":{"value":1}}'),
    (dspy.XMLAdapter(), "dspy.adapters.xml_adapter", '<answer>{"value":1}</answer>'),
])
def test_unexpected_field_parser_errors_are_not_labelled_as_model_output_errors(adapter, module, text, monkeypatch):
    def fail(*args):
        raise RuntimeError("validator implementation bug")

    monkeypatch.setattr(module + ".parse_value", fail)
    with pytest.raises(RuntimeError, match="validator implementation bug"):
        adapter.parse(dspy.Signature("question -> answer"), text)


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_usage_reporting_failure_does_not_replace_primary_error(asynchronous):
    reporting_error = RuntimeError("tracker failed")

    class Tracker:
        def add_usage(self, model, usage):
            raise reporting_error

    class Backend(Engine):
        def complete(self, request):
            if self.calls:
                self.error = lm15.AuthError("denied")
            return super().complete(request)

    engine = Backend()
    lm = dspy.LM("custom", engine=engine, async_engine=AsyncEngine(engine), cache=False, num_retries=0)
    with dspy.context(usage_tracker=Tracker()), pytest.raises(dspy.LMAuthError) as caught:
        await invoke(lm, asynchronous, "hello", n=2)
    assert caught.value.usage_errors == (reporting_error,)
    assert isinstance(caught.value.__cause__, lm15.AuthError)
    assert engine.calls == 2


@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_direct_native_engine_stream_is_also_guarded(asynchronous, monkeypatch):
    from types import SimpleNamespace

    from dspy.clients.engines import AsyncLM15Engine, LM15Engine

    class Provider:
        provider = "fake"

        def stream(self, request):
            return iter([lm15.StreamStartEvent(), lm15.StreamDeltaEvent(lm15.TextDelta("partial"))])

    request = lm15.Request(model="custom", messages=(lm15.Message.user("hi"),))
    if asynchronous:
        async def source(request):
            for event in Provider().stream(request):
                yield event

        engine = AsyncLM15Engine()
        provider = SimpleNamespace(provider="fake", stream=source)
        monkeypatch.setattr(engine, "_target", lambda *args, **kwargs: (provider, request))
        with pytest.raises(lm15.StreamAssemblyError):
            _ = [event async for event in engine.stream(request)]
    else:
        engine = LM15Engine()
        monkeypatch.setattr(engine, "_target", lambda *args, **kwargs: (Provider(), request))
        with pytest.raises(lm15.StreamAssemblyError):
            list(engine.stream(request))


@pytest.mark.asyncio
async def test_stream_fallback_progress_is_isolated_across_concurrent_calls():
    emitted, release = asyncio.Event(), asyncio.Event()

    class Backend(AsyncEngine):
        calls = 0

        async def complete(self, request):
            self.calls += 1
            return lm15.Response(None, request.model, lm15.Message.assistant('{"answer":"ok"}'), "stop", lm15.Usage())

        async def stream(self, request):
            yield lm15.StreamStartEvent()
            yield lm15.StreamDeltaEvent(lm15.TextDelta('{"answer":"ok"}'))
            await release.wait()
            yield lm15.StreamEndEvent(finish_reason="stop")

    class SignallingSink(Sink):
        async def send(self, chunk):
            emitted.set()

    backend = Backend(None)
    lm = dspy.LM("custom", engine=Engine(), async_engine=backend, cache=False)
    adapter = dspy.ChatAdapter()
    args = (lm, {}, dspy.Signature("question -> answer"), [], {"question": "hi"})

    async def streaming():
        with dspy.context(send_stream=SignallingSink()):
            with pytest.raises(dspy.AdapterParseError):
                await adapter.acall(*args)

    task = asyncio.create_task(streaming())
    try:
        await asyncio.wait_for(emitted.wait(), 5)
        assert await adapter.acall(*args) == [{"answer": "ok"}]
        assert backend.calls == 2  # non-streaming Chat parse failure still gets one JSON fallback
    finally:
        release.set()
        await task