Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python
5.9 KB
Raw Blame
import dspy
from dspy.utils.exceptions import (
    AdapterParseError,
    ContextWindowExceededError,
    DSPyError,
    LMError,
    LMInvalidRequestError,
    format_error_for_lm,
)


def test_lm_errors_are_exported_from_dspy():
    assert dspy.DSPyError is not None
    assert dspy.LMError is LMError
    assert dspy.LMUnexpectedError is not None
    assert dspy.AdapterParseError is AdapterParseError
    assert dspy.is_retryable_lm_error is not None


def test_retryable_lm_errors_classification():
    assert dspy.is_retryable_lm_error(dspy.LMRateLimitError())
    assert dspy.is_retryable_lm_error(dspy.LMTimeoutError())
    assert dspy.is_retryable_lm_error(dspy.LMServerError())
    assert dspy.is_retryable_lm_error(dspy.LMTransportError())
    assert not dspy.is_retryable_lm_error(dspy.LMAuthError())
    assert not dspy.is_retryable_lm_error(dspy.LMInvalidRequestError())
    assert not dspy.is_retryable_lm_error(dspy.LMUnexpectedError())
    assert not dspy.is_retryable_lm_error(ValueError("not an LM error"))


def test_lm_error_metadata():
    error = dspy.LMRateLimitError(
        "rate limited",
        model="openai/gpt-4o",
        provider="openai",
        status=429,
        request_id="req-123",
        retry_after=2.5,
    )

    assert error.code == "rate_limit"
    assert error.model == "openai/gpt-4o"
    assert error.provider == "openai"
    assert error.status == 429
    assert error.request_id == "req-123"
    assert error.retry_after == 2.5
    assert str(error) == "[openai/gpt-4o] rate limited"


def test_context_window_exceeded_error_defaults():
    error = ContextWindowExceededError()
    assert isinstance(error, LMInvalidRequestError)
    assert isinstance(error, LMError)
    assert error.code == "context_window_exceeded"
    assert error.model is None
    assert str(error) == "Context window exceeded"


def test_context_window_exceeded_error_with_model():
    error = ContextWindowExceededError(model="openai/gpt-4o")
    assert error.model == "openai/gpt-4o"
    assert str(error) == "[openai/gpt-4o] Context window exceeded"


def test_context_window_exceeded_error_with_message():
    error = ContextWindowExceededError(model="openai/gpt-4o", message="Input is 200k tokens, limit is 128k")
    assert error.model == "openai/gpt-4o"
    assert str(error) == "[openai/gpt-4o] Input is 200k tokens, limit is 128k"


def test_context_window_exceeded_error_message_without_model():
    error = ContextWindowExceededError(message="Too many tokens")
    assert error.model is None
    assert str(error) == "Too many tokens"


def test_adapter_parse_error_basic():
    adapter_name = "ChatAdapter"
    signature = dspy.make_signature("question->answer1, answer2")
    lm_response = "[[ ## answer1 ## ]]\nanswer1"

    error = AdapterParseError(adapter_name=adapter_name, signature=signature, lm_response=lm_response)

    assert isinstance(error, DSPyError)
    assert error.code == "adapter_parse_error"
    assert error.adapter_name == adapter_name
    assert error.signature == signature
    assert error.lm_response == lm_response

    error_message = str(error)
    assert error_message == (
        "Adapter ChatAdapter failed to parse the LM response. \n\n"
        "LM Response: [[ ## answer1 ## ]]\nanswer1 \n\n"
        "Expected to find output fields in the LM response: [answer1, answer2] \n\n"
    )


def test_adapter_parse_error_with_message():
    adapter_name = "ChatAdapter"
    signature = dspy.make_signature("question->answer1, answer2")
    lm_response = "[[ ## answer1 ## ]]\nanswer1"
    message = "Critical error, please fix!"

    error = AdapterParseError(adapter_name=adapter_name, signature=signature, lm_response=lm_response, message=message)

    assert error.adapter_name == adapter_name
    assert error.signature == signature
    assert error.lm_response == lm_response

    error_message = str(error)
    assert error_message == (
        "Critical error, please fix!\n\n"
        "Adapter ChatAdapter failed to parse the LM response. \n\n"
        "LM Response: [[ ## answer1 ## ]]\nanswer1 \n\n"
        "Expected to find output fields in the LM response: [answer1, answer2] \n\n"
    )


def test_adapter_parse_error_with_parsed_result():
    adapter_name = "ChatAdapter"
    signature = dspy.make_signature("question->answer1, answer2")
    lm_response = "[[ ## answer1 ## ]]\nanswer1"
    parsed_result = {"answer1": "value1"}

    error = AdapterParseError(
        adapter_name=adapter_name, signature=signature, lm_response=lm_response, parsed_result=parsed_result
    )

    error_message = str(error)
    assert error_message == (
        "Adapter ChatAdapter failed to parse the LM response. \n\n"
        "LM Response: [[ ## answer1 ## ]]\nanswer1 \n\n"
        "Expected to find output fields in the LM response: [answer1, answer2] \n\n"
        "Actual output fields parsed from the LM response: [answer1] \n\n"
    )


def test_format_error_for_lm_without_traceback():
    error = ValueError("bad value")
    assert format_error_for_lm(error) == "bad value"
    assert format_error_for_lm(error, traceback_frames=0) == str(error)


def test_format_error_for_lm_with_traceback():
    try:
        raise ValueError("boom")
    except ValueError as error:
        formatted = format_error_for_lm(error, traceback_frames=5)

    assert formatted.startswith("\nTraceback (most recent call last):")
    assert formatted.endswith("ValueError: boom")
    assert "test_exceptions.py" in formatted


def test_format_error_for_lm_limits_traceback_frames():
    def level_one():
        raise RuntimeError("deep failure")

    def level_two():
        level_one()

    def level_three():
        level_two()

    try:
        level_three()
    except RuntimeError as error:
        limited = format_error_for_lm(error, traceback_frames=1)
        full = format_error_for_lm(error, traceback_frames=5)

    assert limited.count('File "') == 1
    assert full.count('File "') == 4
    assert limited.endswith("RuntimeError: deep failure")
    assert full.endswith("RuntimeError: deep failure")