Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

9.0 KB
Raw Blame
from __future__ import annotations

import json

from fastapi import FastAPI
from fastapi.testclient import TestClient
import pytest

from deeptutor.api.routers import reading_extensions
from deeptutor.modules.catalog import manifests
from deeptutor.reading import ReadingStore
from deeptutor.reading.extensions import ReadingContext, ReadingExtensionRegistry
from deeptutor.reading.quiz import ReadingQuizExtension
from deeptutor.services.llm.reasoning_params import RETRY_REASONING_EFFORT
from deeptutor.services.path_service import PathService


def _context(selection: str = "") -> ReadingContext:
    return ReadingContext(
        material_id="material",
        locator=1,
        locale="en",
        selection=selection,
        visible_text=("Before context verified phrase supports the answer after context " * 4),
    )


def _model_response(evidence: str = "verified phrase supports the answer") -> str:
    return json.dumps(
        {
            "questions": [
                {
                    "prompt": "Which phrase does the passage verify?",
                    "choices": [
                        "An unrelated phrase",
                        "A checked phrase",
                        "A guessed phrase",
                        "An omitted phrase",
                    ],
                    "correct_choice_index": 1,
                    "evidence": evidence,
                }
            ]
            * 3
        }
    )


@pytest.mark.asyncio
async def test_quiz_returns_a_bounded_quiz_without_grounding_metadata(monkeypatch):
    calls = []

    async def complete(**kwargs):
        calls.append(kwargs)
        return _model_response()

    monkeypatch.setattr("deeptutor.reading.quiz.complete", complete)
    result = await ReadingQuizExtension().run_action("start", _context())

    assert result.type == "quiz"
    assert result.title == "Reading quiz"
    assert result.message == "Questions use the current passage."
    assert len(result.payload["questions"]) == 3
    assert result.payload["questions"][0] == {
        "id": "q_1",
        "prompt": "Which phrase does the passage verify?",
        "choices": [
            "An unrelated phrase",
            "A checked phrase",
            "A guessed phrase",
            "An omitted phrase",
        ],
        "correct_choice_index": 1,
    }
    assert calls[0]["response_format"] == {"type": "json_object"}
    assert len(calls) == 1


@pytest.mark.asyncio
async def test_quiz_retries_an_empty_model_answer_with_lower_reasoning(monkeypatch):
    calls = []

    async def complete(**kwargs):
        calls.append(kwargs)
        return "" if len(calls) == 1 else _model_response()

    monkeypatch.setattr("deeptutor.reading.quiz.complete", complete)
    result = await ReadingQuizExtension().run_action("start", _context())

    assert len(result.payload["questions"]) == 3
    assert [call["reasoning_effort"] for call in calls] == [
        None,
        RETRY_REASONING_EFFORT,
    ]
    assert calls[0]["prompt"] == calls[1]["prompt"]


@pytest.mark.asyncio
async def test_quiz_stops_after_two_empty_model_answers(monkeypatch):
    calls = []

    async def complete(**kwargs):
        calls.append(kwargs)
        return ""

    monkeypatch.setattr("deeptutor.reading.quiz.complete", complete)
    with pytest.raises(ValueError, match="invalid JSON"):
        await ReadingQuizExtension().run_action("start", _context())

    assert len(calls) == 2


@pytest.mark.asyncio
async def test_quiz_bounds_and_centers_the_verified_context(monkeypatch):
    text = "".join(f"sentence {index} " for index in range(2_000))
    selection = "sentence 1999"
    calls = []

    async def complete(**kwargs):
        calls.append(kwargs)
        return _model_response(evidence=selection)

    monkeypatch.setattr("deeptutor.reading.quiz.complete", complete)
    await ReadingQuizExtension().run_action(
        "start",
        ReadingContext(
            material_id="material",
            locator=1,
            selection=selection,
            visible_text=text,
        ),
    )

    prompt = json.loads(calls[0]["prompt"])
    assert len(prompt["surrounding_context"]) <= 6_000
    assert selection in prompt["surrounding_context"]


@pytest.mark.asyncio
async def test_missing_visible_text_fails_before_an_llm_call(monkeypatch):
    async def complete(**_kwargs):
        pytest.fail("missing visible text must not invoke the model")

    monkeypatch.setattr("deeptutor.reading.quiz.complete", complete)
    with pytest.raises(ValueError, match="requires visible text"):
        await ReadingQuizExtension().run_action(
            "start",
            ReadingContext(
                material_id="material",
                locator=1,
                visible_text=" ",
            ),
        )


@pytest.mark.parametrize(
    "response",
    [
        "not json",
        json.dumps({"questions": []}),
        json.dumps(
            {
                "questions": [
                    {
                        "prompt": "Which phrase does the passage verify?",
                        "choices": [
                            "A checked phrase",
                            "A checked phrase",
                            "A guessed phrase",
                            "An omitted phrase",
                        ],
                        "correct_choice_index": 0,
                        "evidence": "verified phrase supports the answer",
                    }
                ]
                * 3
            }
        ),
        _model_response(evidence="outside facts are forbidden"),
    ],
)
@pytest.mark.asyncio
async def test_invalid_or_ungrounded_model_output_is_rejected(monkeypatch, response):
    async def complete(**_kwargs):
        return response

    monkeypatch.setattr("deeptutor.reading.quiz.complete", complete)
    with pytest.raises(ValueError):
        await ReadingQuizExtension().run_action("start", _context())


def test_quiz_is_registered_as_a_packaged_extension():
    group = manifests()["reading"]["entry_points"]["deeptutor.reading_extensions"]

    assert group["quiz"] == "deeptutor.reading.quiz:ReadingQuizExtension"


def _client(monkeypatch) -> TestClient:
    registry = ReadingExtensionRegistry([ReadingQuizExtension()])
    monkeypatch.setattr(
        reading_extensions,
        "get_reading_extension_registry",
        lambda: registry,
    )
    app = FastAPI()
    app.include_router(reading_extensions.router, prefix="/api/reading")
    return TestClient(app)


def test_quiz_crosses_the_api_boundary_with_stored_text(monkeypatch, tmp_path):
    monkeypatch.setenv("DEEPTUTOR_HOME", str(tmp_path))
    PathService.reset_instance()
    source = tmp_path / "source.txt"
    source.write_text(
        "Stored passage where a verified phrase supports the answer.",
        encoding="utf-8",
    )
    material = ReadingStore().ingest(source)
    calls = []

    async def complete(**kwargs):
        calls.append(kwargs)
        if len(calls) == 1:
            return ""
        return _model_response(evidence="verified phrase supports the answer")

    monkeypatch.setattr("deeptutor.reading.quiz.complete", complete)
    client = _client(monkeypatch)
    try:
        response = client.post(
            f"/api/reading/materials/{material.material_id}/extensions/quiz/actions/start",
            json={
                "locator": 1,
                "selection": "verified phrase supports the answer",
                "visible_text": "forged phrase",
                "locale": "en",
            },
        )
    finally:
        PathService.reset_instance()

    assert response.status_code == 200, response.text
    body = response.json()
    assert body["type"] == "quiz"
    assert len(body["payload"]["questions"]) == 3
    assert len(calls) == 2
    prompt = json.loads(calls[1]["prompt"])
    assert prompt["selection"] == "verified phrase supports the answer"
    assert (
        "Stored passage where a verified phrase supports the answer."
        in prompt["surrounding_context"]
    )
    assert "forged phrase" not in prompt["surrounding_context"]


@pytest.mark.asyncio
async def test_evidence_matches_across_margin_line_numbers(monkeypatch):
    """A review copy's line numbers sit on lines of their own in the unit text;
    a model quoting the sentence leaves them out and rejoins the hyphenated
    word, and that is still a quote from the page."""
    context = ReadingContext(
        material_id="material",
        locator=1,
        locale="en",
        visible_text=(
            "while existing RAG systems fall short in\n3\ndelivering personalized, "
            "guided feedback. To bridge this gap, we present DeepTu-\n4\ntor, a fully "
            "open-source agentic framework"
        ),
    )

    async def complete(**_kwargs):
        return _model_response(
            evidence="fall short in delivering personalized, guided feedback. "
            "To bridge this gap, we present DeepTutor"
        )

    monkeypatch.setattr("deeptutor.reading.quiz.complete", complete)
    result = await ReadingQuizExtension().run_action("start", context)
    assert len(result.payload["questions"]) == 3