Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

4.2 KB
Raw Blame
"""The project's one rule for a structured call a reasoning model starved.

A reasoning model pays for hidden tokens out of the same ``max_tokens`` as its
answer, so a one-shot structured call can come back empty with no error —
Book's spine collapsing to a single "Overview" chapter (#1316), the quiz plan
emitting zero templates (#1318), a research topic decomposing to one
sub-topic. The remedy is always the same: ask again with thinking turned down.

These tests pin the two things that make it safe to share: the retry actually
happens, and deciding "the model answered" stays with the caller.
"""

from __future__ import annotations

from typing import Any

import pytest

from deeptutor.services.llm.reasoning_params import RETRY_REASONING_EFFORT
from deeptutor.services.llm.structured_retry import (
    json_payload_is_usable,
    json_with_reasoning_retry,
    payload_with_reasoning_retry,
)


def _recorder(*responses: str):
    """A ``run`` that returns the given bodies in order, logging the effort."""
    efforts: list[str | None] = []
    remaining = list(responses)

    async def run(reasoning_effort: str | None) -> str:
        efforts.append(reasoning_effort)
        return remaining.pop(0) if remaining else ""

    return run, efforts


@pytest.mark.asyncio
async def test_a_usable_first_answer_is_not_paid_for_twice() -> None:
    run, efforts = _recorder('{"spine": [1]}')
    result = await json_with_reasoning_retry(run, expected_key="spine")
    assert result == {"spine": [1]}
    assert efforts == [None], "a good answer must not trigger a second call"


@pytest.mark.asyncio
async def test_a_starved_answer_is_retried_with_thinking_turned_down() -> None:
    run, efforts = _recorder("", '{"spine": [1]}')
    result = await json_with_reasoning_retry(run, expected_key="spine")
    assert result == {"spine": [1]}
    assert efforts == [None, RETRY_REASONING_EFFORT]


@pytest.mark.asyncio
async def test_the_expected_key_is_what_counts_as_answered() -> None:
    """A well-formed object missing the key is still a starved answer.

    This is the #1316 shape exactly: valid JSON, no content, and a caller that
    would have degraded silently.
    """
    run, efforts = _recorder('{"notes": "thinking..."}', '{"spine": [1]}')
    result = await json_with_reasoning_retry(run, expected_key="spine")
    assert result == {"spine": [1]}
    assert efforts == [None, RETRY_REASONING_EFFORT]


@pytest.mark.asyncio
async def test_two_starved_attempts_return_the_callers_empty_fallback() -> None:
    run, _ = _recorder("", "")
    assert await json_with_reasoning_retry(run, expected_key="spine") == {}


@pytest.mark.asyncio
async def test_the_object_shaped_helper_never_hands_back_a_list() -> None:
    """Callers of the object form index the result, so it must be a dict."""
    run, _ = _recorder("[1, 2, 3]", "[4]")
    assert await json_with_reasoning_retry(run, expected_key=None) == {}


@pytest.mark.asyncio
async def test_a_caller_that_accepts_an_array_keeps_its_array() -> None:
    """The generic form must not impose the object shape on everyone.

    Research's decompose accepts a bare array of sub-topics. Judging that
    "unusable" would retry a perfectly good answer and then discard it — a
    regression the object-shaped helper would have caused if it were the only
    entry point.
    """

    def accepts_anything_non_empty(payload: Any) -> bool:
        return bool(payload)

    run, efforts = _recorder("[1, 2, 3]")
    result = await payload_with_reasoning_retry(run, is_usable=accepts_anything_non_empty)
    assert result == [1, 2, 3]
    assert efforts == [None]


@pytest.mark.asyncio
async def test_nothing_usable_and_nothing_non_empty_is_none() -> None:
    run, _ = _recorder("", "")
    assert await payload_with_reasoning_retry(run, is_usable=bool) is None


@pytest.mark.parametrize(
    ("payload", "expected_key", "usable"),
    [
        ({"spine": [1]}, "spine", True),
        ({"spine": []}, "spine", False),
        ({}, None, False),
        ({"anything": 1}, None, True),
        ([1], None, False),
        ("not json", None, False),
    ],
)
def test_object_usability_rule(payload: Any, expected_key: str | None, usable: bool) -> None:
    assert json_payload_is_usable(payload, expected_key) is usable