Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

22.8 KB
Raw Blame
"""Tests for Mastery Path mastery updates from graded answers.

The unified post-answer pipeline is ``LearningService.grade_and_record``: it
grades one answer, recomputes mastery (recency-weighted with a low-confidence
cap), advances the spaced-repetition state, rebuilds the review queue, and
persists. The mastery tools (``mastery_grade``) fold every answer through
exactly this pipeline, so mastery is updated deterministically with the
expected answer held server-side. These tests assert that contract end to end.
"""

import pytest

from deeptutor.learning.mastery import compute_mastery
from deeptutor.learning.models import (
    KnowledgePoint,
    KnowledgeType,
    LearningModule,
    LearningProgress,
)
from deeptutor.learning.scheduler import SpacedRepetitionScheduler
from deeptutor.learning.service import LearningService
from deeptutor.learning.storage import LearningStore

# ── Helpers ──────────────────────────────────────────────────────────────


def _make_progress(book_id="book1") -> LearningProgress:
    progress = LearningProgress(book_id=book_id)
    progress.modules = [
        LearningModule(
            id="m1",
            name="Module 1",
            order=0,
            knowledge_points=[
                KnowledgePoint(id="kp1", name="KP1", type=KnowledgeType.CONCEPT, module_id="m1")
            ],
        )
    ]
    progress.current_module_id = "m1"
    progress.knowledge_types["kp1"] = KnowledgeType.CONCEPT
    return progress


# ── compute_mastery policy: low-confidence cap ─────────────────────────────


def test_compute_mastery_single_correct_is_capped_at_half():
    """A single lucky answer cannot 'master' a point: 1 attempt caps at 0.5."""
    assert compute_mastery([True]) == 0.5


def test_compute_mastery_cap_progression():
    """Cap relaxes as evidence accumulates: 1->0.5, 2->0.8, 3+->up to 1.0."""
    assert compute_mastery([True]) == 0.5
    assert compute_mastery([True, True]) == 0.8
    assert compute_mastery([True, True, True]) == pytest.approx(1.0)


def test_compute_mastery_empty_is_zero():
    assert compute_mastery([]) == 0.0


def test_compute_mastery_partial_correctness_scales():
    """More correct answers within the recency window yield a higher score
    (below the cap, where attempt count no longer clamps the result)."""
    one_of_five = compute_mastery([True, False, False, False, False])
    two_of_five = compute_mastery([True, True, False, False, False])
    three_of_five = compute_mastery([True, True, True, False, False])
    assert one_of_five < two_of_five < three_of_five


def test_compute_mastery_weighs_recent_attempts_more_heavily():
    """Regression test for #618: recency weighting must actually distinguish
    a recovering history from a declining one, even with the same correct
    count. A miss-then-two-hits history should score higher than a
    two-hits-then-miss history."""
    recovering = compute_mastery([False, True, True])
    declining = compute_mastery([True, True, False])
    assert recovering > declining
    assert recovering == pytest.approx(0.696, abs=1e-3)
    assert declining == pytest.approx(0.643, abs=1e-3)


# ── grade_and_record: the unified post-answer pipeline ─────────────────────


def test_grade_and_record_correct_updates_capped_mastery_and_persists(tmp_path):
    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    progress = _make_progress()
    service.save(progress)

    result = service.grade_and_record(
        progress,
        question_id="q1",
        knowledge_point_id="kp1",
        module_id="m1",
        user_answer="paris",
        expected_answer="paris",
    )

    assert result is True
    # Single correct attempt -> capped at 0.5, NOT 1.0.
    assert progress.mastery_levels["kp1"] == 0.5
    assert len(progress.quiz_attempts) == 1
    assert progress.quiz_attempts[0].is_correct is True

    loaded = store.load("book1")
    assert loaded is not None
    assert len(loaded.quiz_attempts) == 1
    assert loaded.mastery_levels["kp1"] == 0.5


def test_grade_and_record_fail_closed_without_expected_answer(tmp_path):
    """Fail-closed: with no stored expected answer the attempt is recorded
    wrong, never right — even when the user answer matches an empty string."""
    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    progress = _make_progress()

    result = service.grade_and_record(
        progress,
        question_id="q1",
        knowledge_point_id="kp1",
        module_id="m1",
        user_answer="",
        expected_answer="",
    )

    assert result is False
    assert progress.quiz_attempts[0].is_correct is False
    # Wrong answer creates an active error record.
    assert len(progress.error_records) == 1
    assert progress.error_records[0].status == "active"


def test_grade_and_record_advances_sr_state_and_builds_queue(tmp_path):
    """When a scheduler is supplied, a graded answer advances the spaced-
    repetition state and rebuilds the review queue."""
    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    scheduler = SpacedRepetitionScheduler()
    progress = _make_progress()

    service.grade_and_record(
        progress,
        question_id="q1",
        knowledge_point_id="kp1",
        module_id="m1",
        user_answer="paris",
        expected_answer="paris",
        scheduler=scheduler,
    )

    assert "kp1" in progress.repetition_states
    # A correct answer advanced the interval index past the initial 0.
    assert progress.repetition_states["kp1"].interval_index >= 1
    assert progress.repetition_states["kp1"].stability > 0
    assert progress.repetition_states["kp1"].review_count == 1
    assert len(progress.review_queue) == 1
    assert progress.review_queue[0].knowledge_point_id == "kp1"
    assert len(progress.learning_evidence) == 1
    assert progress.learning_evidence[0].result == "correct"
    assert progress.learning_evidence[0].quality == 1.0


def test_grade_and_record_no_scheduler_skips_sr_state(tmp_path):
    """Without a scheduler, mastery still updates but no SR state/queue is built."""
    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    progress = _make_progress()

    service.grade_and_record(
        progress,
        question_id="q1",
        knowledge_point_id="kp1",
        module_id="m1",
        user_answer="paris",
        expected_answer="paris",
    )

    assert progress.mastery_levels["kp1"] == 0.5
    assert progress.repetition_states == {}
    assert progress.review_queue == []
    assert len(progress.learning_evidence) == 1
    assert progress.learning_evidence[0].assessment_type == "quiz"


def test_grade_and_record_blank_wrong_is_metacognitive(tmp_path):
    """A blank wrong answer is classified metacognitive at record time."""
    from deeptutor.learning.models import ErrorType

    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    progress = _make_progress()

    service.grade_and_record(
        progress,
        question_id="q1",
        knowledge_point_id="kp1",
        module_id="m1",
        user_answer="   ",
        expected_answer="paris",
    )

    assert progress.quiz_attempts[0].is_correct is False
    assert progress.quiz_attempts[0].error_type == ErrorType.METACOGNITIVE


# ── Error-record graduation across attempts ────────────────────────────────


def test_error_record_graduates_on_later_correct_answer(tmp_path):
    """A wrong answer opens an active error record; a later correct answer for
    the same question + KP graduates it, and mastery climbs out of the cap."""
    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    progress = _make_progress()

    # First attempt wrong.
    service.grade_and_record(
        progress,
        question_id="q1",
        knowledge_point_id="kp1",
        module_id="m1",
        user_answer="london",
        expected_answer="paris",
    )
    assert len(progress.error_records) == 1
    assert progress.error_records[0].status == "active"

    # Two correct attempts on the same question + KP.
    for _ in range(2):
        service.grade_and_record(
            progress,
            question_id="q1",
            knowledge_point_id="kp1",
            module_id="m1",
            user_answer="paris",
            expected_answer="paris",
        )

    # The error record graduated on the first correct answer.
    assert progress.error_records[0].status == "graduated"
    # 3 attempts total (1 wrong + 2 right) lifts mastery above the 1-attempt cap.
    assert len(progress.quiz_attempts) == 3
    assert progress.mastery_levels["kp1"] > 0.5


# ── Pending-question lifecycle + qualitative recording (loop-driven path) ──


def test_pending_question_set_and_clear_round_trip(tmp_path):
    from deeptutor.learning.models import PendingQuestion

    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    progress = _make_progress()

    service.set_pending_question(
        progress,
        PendingQuestion(
            question_id="q1",
            knowledge_point_id="kp1",
            module_id="m1",
            prompt="Capital of France?",
            expected_answer="paris",
        ),
    )
    assert store.load("book1").pending_question.expected_answer == "paris"

    service.clear_pending_question(progress)
    assert progress.pending_question is None
    assert store.load("book1").pending_question is None


def test_record_qualitative_pass_and_fail_drive_display_mastery(tmp_path):
    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    progress = _make_progress()

    service.record_qualitative(progress, "kp1", passed=True, evidence="clear explanation")
    assert progress.qualitative_mastery["kp1"] is True
    assert progress.mastery_levels["kp1"] == 1.0
    assert progress.feynman_explanations["kp1"] == "clear explanation"
    assert progress.learning_evidence[-1].result == "correct"
    assert progress.learning_evidence[-1].quality == 1.0

    service.record_qualitative(progress, "kp1", passed=False)
    assert progress.qualitative_mastery["kp1"] is False
    assert progress.mastery_levels["kp1"] <= 0.4
    assert progress.learning_evidence[-1].result == "partial"
    assert progress.learning_evidence[-1].quality == 0.2


@pytest.mark.parametrize(
    "knowledge_type",
    [
        KnowledgeType.CONCEPT,
        KnowledgeType.DESIGN,
    ],
)
def test_record_qualitative_starts_at_first_review_interval(tmp_path, monkeypatch, knowledge_type):
    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    scheduler = SpacedRepetitionScheduler()
    progress = _make_progress()
    progress.knowledge_types["kp1"] = knowledge_type
    now = 1_700_000_000.0
    monkeypatch.setattr("deeptutor.learning.scheduler.time.time", lambda: now)
    monkeypatch.setattr("deeptutor.learning.service.time.time", lambda: now)

    service.record_qualitative(progress, "kp1", passed=True, scheduler=scheduler)

    state = progress.repetition_states["kp1"]
    assert state.review_count == 1
    assert state.last_review_at == now
    assert state.next_review_at > now
    assert state.stability > 0
    assert [task.knowledge_point_id for task in progress.review_queue] == ["kp1"]
    assert progress.review_queue[0].due_at == state.next_review_at
    assert progress.learning_evidence[-1].assessment_type == "qualitative"


def test_record_qualitative_updates_existing_review_state(tmp_path, monkeypatch):
    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    scheduler = SpacedRepetitionScheduler()
    progress = _make_progress()
    now = [1_700_000_000.0]
    monkeypatch.setattr("deeptutor.learning.scheduler.time.time", lambda: now[0])
    monkeypatch.setattr("deeptutor.learning.service.time.time", lambda: now[0])

    service.record_qualitative(progress, "kp1", passed=True, scheduler=scheduler)
    first_due = progress.repetition_states["kp1"].next_review_at
    first_stability = progress.repetition_states["kp1"].stability
    service.record_qualitative(progress, "kp1", passed=True, scheduler=scheduler)
    immediate = progress.repetition_states["kp1"]
    assert immediate.review_count == 2
    assert immediate.stability == first_stability
    assert immediate.next_review_at == first_due
    immediate_stability = immediate.stability

    now[0] = immediate.next_review_at
    service.record_qualitative(progress, "kp1", passed=True, scheduler=scheduler)
    after_success = progress.repetition_states["kp1"]
    success_stability = after_success.stability
    success_due = after_success.next_review_at
    assert success_stability > immediate_stability
    assert success_due > first_due
    assert after_success.review_count == 3

    now[0] = success_due
    service.record_qualitative(progress, "kp1", passed=False, scheduler=scheduler)
    after_fail = progress.repetition_states["kp1"]
    assert after_fail.stability < success_stability
    assert after_fail.lapse_count == 1
    success_interval = success_due - first_due
    fail_interval = after_fail.next_review_at - now[0]
    assert fail_interval < success_interval


def test_record_qualitative_initial_failure_schedules_repair_review(tmp_path):
    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    scheduler = SpacedRepetitionScheduler()
    progress = _make_progress()

    service.record_qualitative(progress, "kp1", passed=False, scheduler=scheduler)

    state = progress.repetition_states["kp1"]
    assert state.review_count == 1
    assert state.lapse_count == 1
    assert [task.knowledge_point_id for task in progress.review_queue] == ["kp1"]
    assert len(progress.learning_evidence) == 1
    assert progress.learning_evidence[0].result == "partial"


def test_qualitative_live_state_matches_evidence_replay(tmp_path, monkeypatch):
    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    scheduler = SpacedRepetitionScheduler()
    progress = _make_progress()
    moment = [1_700_000_000.0]
    monkeypatch.setattr("deeptutor.learning.service.time.time", lambda: moment[0])

    service.record_qualitative_in_memory(
        progress, "kp1", passed=False, evidence="partial", scheduler=scheduler
    )
    moment[0] += 3 * 86400
    service.record_qualitative_in_memory(
        progress, "kp1", passed=True, evidence="clear", scheduler=scheduler
    )

    replayed = scheduler.replay(KnowledgeType.CONCEPT, progress.learning_evidence)
    live = progress.repetition_states["kp1"]
    assert live.model_dump() == pytest.approx(replayed.model_dump())


def test_record_qualitative_multi_turn_repair_does_not_compound_lapses(tmp_path, monkeypatch):
    """Regression test for #1781 through LearningService.record_qualitative_in_memory."""
    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    scheduler = SpacedRepetitionScheduler()
    progress = _make_progress()
    moment = [1_700_000_000.0]
    monkeypatch.setattr("deeptutor.learning.service.time.time", lambda: moment[0])

    # Turn 1: initial incomplete explanation
    service.record_qualitative_in_memory(
        progress, "kp1", passed=False, evidence="incomplete concept", scheduler=scheduler
    )
    state = progress.repetition_states["kp1"]
    assert state.lapse_count == 1
    assert state.review_count == 1
    stability_after_fail = state.stability
    due_after_fail = state.next_review_at

    # Turn 2: 3 minutes later, formative clarification still incomplete
    moment[0] += 3 * 60
    service.record_qualitative_in_memory(
        progress, "kp1", passed=False, evidence="partial clarification", scheduler=scheduler
    )
    assert state.lapse_count == 1
    assert state.review_count == 2
    assert state.stability == stability_after_fail
    assert state.next_review_at == due_after_fail

    # Turn 3: 7 minutes from start, completed explanation
    moment[0] += 4 * 60
    service.record_qualitative_in_memory(
        progress, "kp1", passed=True, evidence="complete explanation", scheduler=scheduler
    )
    assert progress.qualitative_mastery["kp1"] is True
    assert progress.mastery_levels["kp1"] == 1.0
    assert state.lapse_count == 1
    assert state.review_count == 3
    assert state.stability == stability_after_fail
    assert state.next_review_at == due_after_fail
    assert state.retrievability == 1.0

    # Ensure replay produces the exact same state
    replayed = scheduler.replay(KnowledgeType.CONCEPT, progress.learning_evidence)
    assert state.model_dump() == pytest.approx(replayed.model_dump())


def test_qualitative_repair_persists_distinct_turns_and_deduplicates_retries(tmp_path, monkeypatch):
    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    scheduler = SpacedRepetitionScheduler()
    service.save(_make_progress())
    start = 1_700_000_000.0
    moment = [start]
    monkeypatch.setattr("deeptutor.learning.service.time.time", lambda: moment[0])

    for offset, turn_id, passed in [
        (0, "turn-1", False),
        (4 * 60, "turn-2", False),
        (8 * 60, "turn-3", True),
    ]:
        moment[0] = start + offset
        progress = service.record_qualitative_for_path(
            "book1",
            "kp1",
            passed=passed,
            evidence=f"assessment {turn_id}",
            scheduler=scheduler,
            session_id="session-1",
            turn_id=turn_id,
        )
        if turn_id == "turn-1":
            first_due = progress.repetition_states["kp1"].next_review_at
            stability_after_failure = progress.repetition_states["kp1"].stability

    state = progress.repetition_states["kp1"]
    repair_stability = state.stability
    assert progress.qualitative_mastery["kp1"] is True
    assert [event.turn_id for event in progress.learning_evidence] == [
        "turn-1",
        "turn-2",
        "turn-3",
    ]
    assert state.review_count == 3
    assert state.lapse_count == 1
    assert state.scheduled_after_failure is True
    assert state.next_review_at == first_due
    assert repair_stability == stability_after_failure

    moment[0] += 60
    duplicate = service.record_qualitative_for_path(
        "book1",
        "kp1",
        passed=False,
        evidence="retried turn",
        scheduler=scheduler,
        session_id="session-1",
        turn_id="turn-2",
    )
    assert len(duplicate.learning_evidence) == 3
    assert duplicate.qualitative_mastery["kp1"] is True
    assert duplicate.repetition_states["kp1"].lapse_count == 1
    assert duplicate.repetition_states["kp1"].next_review_at == first_due
    assert duplicate.repetition_states["kp1"].stability == repair_stability

    moment[0] = first_due + 60
    after_due = service.record_qualitative_for_path(
        "book1",
        "kp1",
        passed=False,
        evidence="delayed retrieval",
        scheduler=scheduler,
        session_id="session-1",
        turn_id="turn-4",
    )
    state = after_due.repetition_states["kp1"]
    assert state.lapse_count == 2
    assert state.stability < repair_stability
    assert [event.turn_id for event in after_due.learning_evidence] == [
        "turn-1",
        "turn-2",
        "turn-3",
        "turn-4",
    ]
    persisted = store.load("book1")
    assert persisted is not None
    assert [event.turn_id for event in persisted.learning_evidence] == [
        "turn-1",
        "turn-2",
        "turn-3",
        "turn-4",
    ]
    replayed = scheduler.replay(KnowledgeType.CONCEPT, persisted.learning_evidence)
    assert persisted.repetition_states["kp1"].model_dump() == pytest.approx(replayed.model_dump())


def test_grade_and_record_retry_correct_uses_weaker_quality(tmp_path):
    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    progress = _make_progress()

    service.grade_and_record(
        progress,
        question_id="q1",
        knowledge_point_id="kp1",
        module_id="m1",
        user_answer="london",
        expected_answer="paris",
    )
    service.grade_and_record(
        progress,
        question_id="q1",
        knowledge_point_id="kp1",
        module_id="m1",
        user_answer="paris",
        expected_answer="paris",
    )

    assert [event.quality for event in progress.learning_evidence] == [0.0, 0.6]
    assert progress.learning_evidence[1].attempt_count == 2


def test_grade_interaction_persists_evidence_and_emits_event(tmp_path):
    from deeptutor.learning.models import PendingQuestion

    store = LearningStore(root=tmp_path)
    service = LearningService(store)
    service.save(_make_progress())
    service.register_question(
        "book1",
        PendingQuestion(
            question_id="q1",
            knowledge_point_id="kp1",
            module_id="m1",
            prompt="Capital of France?",
            expected_answer="paris",
        ),
        session_id="sess-1",
        turn_id="turn-1",
    )
    progress, _interaction, replayed = service.grade_interaction(
        "book1",
        answer="paris",
        question_id="q1",
        scheduler=SpacedRepetitionScheduler(),
        session_id="sess-1",
        turn_id="turn-1",
    )

    assert replayed is False
    assert progress.learning_evidence[-1].session_id == "sess-1"
    assert progress.learning_evidence[-1].turn_id == "turn-1"
    assert progress.learning_evidence[-1].quality == 1.0
    assert any(event.event_type == "evidence.recorded" for event in store.list_events("book1"))


@pytest.mark.parametrize("review_question", ["q1", "q2"])
def test_graduated_retry_does_not_weaken_later_independent_review(tmp_path, review_question):
    service = LearningService(LearningStore(root=tmp_path))
    progress = _make_progress()
    for question_id, answer in [("q1", "london"), ("q1", "paris"), (review_question, "paris")]:
        service.grade_and_record(
            progress,
            question_id=question_id,
            knowledge_point_id="kp1",
            module_id="m1",
            user_answer=answer,
            expected_answer="paris",
        )
    assert progress.error_records[0].status == "graduated"
    assert [event.quality for event in progress.learning_evidence] == [0.0, 0.6, 1.0]
    restored = service.store.load(progress.book_id)
    assert restored.learning_evidence[-1].quality == 1.0


def test_active_error_on_another_question_does_not_weaken_correct_answer(tmp_path):
    service = LearningService(LearningStore(root=tmp_path))
    progress = _make_progress()
    for question_id, answer in [("q1", "london"), ("q1", "london"), ("q2", "paris")]:
        service.grade_and_record(
            progress,
            question_id=question_id,
            knowledge_point_id="kp1",
            module_id="m1",
            user_answer=answer,
            expected_answer="paris",
        )
    assert progress.error_records[0].status == "retrying"
    assert [event.quality for event in progress.learning_evidence] == [0.0, 0.0, 1.0]