Skip to content

A Simple and Universal Swarm Intelligence Engine, Predicting Anything. 简洁通用的群体智能引擎,预测万物

7.5 KB
Raw Blame
from types import SimpleNamespace

import pytest

from app.utils.llm_client import LLMClient, LLMResponseError


class CompletionSequence:
    def __init__(self, *results):
        self.results = list(results)
        self.calls = []

    def create(self, **kwargs):
        self.calls.append(kwargs)
        result = self.results.pop(0)
        if isinstance(result, Exception):
            raise result
        return result


class ProviderError(RuntimeError):
    def __init__(self, *, status_code, body):
        super().__init__(body.get("error", {}).get("message", "provider error"))
        self.status_code = status_code
        self.body = body


def _response(content, *, finish_reason="stop", include_choice=True):
    choices = []
    if include_choice:
        choices.append(
            SimpleNamespace(
                finish_reason=finish_reason,
                message=SimpleNamespace(content=content),
            )
        )
    return SimpleNamespace(choices=choices)


def _client_for(sequence):
    client = object.__new__(LLMClient)
    client.model = "compatible-model"
    client.client = SimpleNamespace(
        chat=SimpleNamespace(completions=sequence)
    )
    return client


def test_chat_json_retries_truncated_completion_without_token_cap():
    sequence = CompletionSequence(
        _response('{"items": [', finish_reason="length"),
        _response('{"items": [1, 2]}'),
    )
    client = _client_for(sequence)

    result = client.chat_json(
        messages=[{"role": "user", "content": "Return JSON"}],
        max_tokens=4096,
        max_attempts=2,
    )

    assert result == {"items": [1, 2]}
    assert sequence.calls[0]["max_tokens"] == 4096
    assert "max_tokens" not in sequence.calls[1]


def test_chat_json_omits_token_cap_when_requested():
    sequence = CompletionSequence(_response('{"ok": true}'))
    client = _client_for(sequence)

    assert client.chat_json(
        messages=[{"role": "user", "content": "Return JSON"}],
        max_tokens=None,
    ) == {"ok": True}
    assert "max_tokens" not in sequence.calls[0]


def test_chat_json_retries_empty_content_once():
    sequence = CompletionSequence(
        _response(None),
        _response('{"ok": true}'),
    )
    client = _client_for(sequence)

    result = client.chat_json(
        messages=[{"role": "user", "content": "Return JSON"}],
        max_attempts=2,
    )

    assert result == {"ok": True}
    assert len(sequence.calls) == 2


def test_chat_json_accepts_complete_object_before_trailing_text():
    sequence = CompletionSequence(
        _response('{"ok": true}\nThis object is ready to use.')
    )
    client = _client_for(sequence)

    assert client.chat_json(
        messages=[{"role": "user", "content": "Return JSON"}],
    ) == {"ok": True}


@pytest.mark.parametrize(
    "content",
    [
        '{"status": "draft"}\n{"status": "complete"}',
        '{"status": "draft"}\n```json\n{"status": "complete"}\n```',
        '{"status": "draft"}\nExplanation first.\n{"status": "complete"}',
    ],
)
def test_chat_json_rejects_a_second_json_document(content):
    sequence = CompletionSequence(_response(content))
    client = _client_for(sequence)

    with pytest.raises(LLMResponseError, match="multiple JSON"):
        client.chat_json(
            messages=[{"role": "user", "content": "Return JSON"}],
        )


def test_chat_json_rejects_top_level_array():
    sequence = CompletionSequence(_response('[{"ok": true}]'))
    client = _client_for(sequence)

    with pytest.raises(LLMResponseError, match="JSON object"):
        client.chat_json(
            messages=[{"role": "user", "content": "Return JSON"}],
            max_attempts=1,
        )


def test_chat_json_error_does_not_echo_partial_model_output():
    partial = '{"private_source_text": "SENTINEL-SHOULD-NOT-LEAK"'
    sequence = CompletionSequence(
        _response(partial, finish_reason="length"),
        _response(partial, finish_reason="length"),
    )
    client = _client_for(sequence)

    with pytest.raises(LLMResponseError) as captured:
        client.chat_json(
            messages=[{"role": "user", "content": "Return JSON"}],
            max_attempts=2,
        )

    assert "SENTINEL-SHOULD-NOT-LEAK" not in str(captured.value)
    assert captured.value.finish_reason == "length"


def test_chat_json_falls_back_only_for_explicit_response_format_rejection():
    unsupported = ProviderError(
        status_code=400,
        body={
            "error": {
                "param": "response_format",
                "code": "unsupported_parameter",
                "message": "response_format is not supported by this model",
            }
        },
    )
    sequence = CompletionSequence(unsupported, _response('{"ok": true}'))
    client = _client_for(sequence)

    result = client.chat_json(
        messages=[{"role": "user", "content": "Return JSON"}],
        max_attempts=1,
    )

    assert result == {"ok": True}
    assert sequence.calls[0]["response_format"] == {"type": "json_object"}
    assert "response_format" not in sequence.calls[1]


def test_response_format_fallback_keeps_content_retry_available():
    unsupported = ProviderError(
        status_code=400,
        body={
            "error": {
                "param": "response_format",
                "code": "unsupported_parameter",
                "message": "response_format is not supported by this model",
            }
        },
    )
    sequence = CompletionSequence(
        unsupported,
        _response('{"items": [', finish_reason="length"),
        _response('{"items": [1]}'),
    )
    client = _client_for(sequence)

    result = client.chat_json(
        messages=[{"role": "user", "content": "Return JSON"}],
        max_attempts=2,
    )

    assert result == {"items": [1]}
    assert sequence.calls[0]["response_format"] == {"type": "json_object"}
    assert "response_format" not in sequence.calls[1]
    assert "response_format" not in sequence.calls[2]
    assert "max_tokens" not in sequence.calls[2]


def test_chat_json_defaults_to_one_content_attempt():
    sequence = CompletionSequence(
        _response(None),
        _response('{"ok": true}'),
    )
    client = _client_for(sequence)

    with pytest.raises(LLMResponseError, match="empty JSON"):
        client.chat_json(
            messages=[{"role": "user", "content": "Return JSON"}],
        )

    assert len(sequence.calls) == 1


@pytest.mark.parametrize("status_code", [400, 401, 429, 500])
def test_chat_json_does_not_retry_unrelated_provider_errors(status_code):
    provider_error = ProviderError(
        status_code=status_code,
        body={
            "error": {
                "param": "messages",
                "code": "invalid_request",
                "message": "request failed for an unrelated reason",
            }
        },
    )
    sequence = CompletionSequence(provider_error, _response('{"ok": true}'))
    client = _client_for(sequence)

    with pytest.raises(ProviderError) as captured:
        client.chat_json(
            messages=[{"role": "user", "content": "Return JSON"}],
            max_attempts=2,
        )

    assert captured.value is provider_error
    assert len(sequence.calls) == 1


def test_chat_json_reports_missing_choices_without_retrying_forever():
    sequence = CompletionSequence(
        _response(None, include_choice=False),
        _response(None, include_choice=False),
    )
    client = _client_for(sequence)

    with pytest.raises(LLMResponseError, match="no choices"):
        client.chat_json(
            messages=[{"role": "user", "content": "Return JSON"}],
            max_attempts=2,
        )

    assert len(sequence.calls) == 2