DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(mastery): shuffle choice options so the correct answer moves (#1691)
XZHcb4c25db26fa
diff --git a/deeptutor/capabilities/mastery/tools.py b/deeptutor/capabilities/mastery/tools.pyindex e756e9d16..97c01f578 100644--- a/deeptutor/capabilities/mastery/tools.py+++ b/deeptutor/capabilities/mastery/tools.py@@ -18,6 +18,7 @@ from __future__ import annotationsimport asyncioimport jsonimport logging+import randomimport refrom typing import TYPE_CHECKING, Anyimport uuid@@ -64,7 +65,7 @@ from deeptutor.learning.objective_relations import (RelationRefs,normalize_refs,)-from deeptutor.learning.pending import public_pending_question+from deeptutor.learning.pending import positional_label, public_pending_questionfrom deeptutor.learning.policy import (QUALITATIVE_TYPES,display_mastery,@@ -286,6 +287,33 @@ def _normalize_quiz_contract(return question_type, options, resolved_expected+def _shuffle_choice_options(+ options: list[dict[str, str]], expected: str, rng: random.Random+) -> tuple[list[dict[str, str]], str]:+ """Reorder choice options so the correct answer is not pinned to one label.++ Models put the correct option first the overwhelming majority of the time,+ and the registration order is exactly what the card renders — so an unmixed+ session reads A = correct on quiz after quiz (HKUDS/DeepTutor#1691). The+ shuffle runs after validation and echo stripping, so every contract+ invariant still holds on the payload that gets persisted: bodies stay+ unique, labels are re-issued positionally, and ``expected`` follows its+ body to the new label — the interactive card and the deterministic grader+ keep comparing the same representation.+ """+ if len(options) < 2:+ return options, expected+ expected_body = next(option["body"] for option in options if option["label"] == expected)+ shuffled = list(options)+ rng.shuffle(shuffled)+ remapped = [+ {"label": positional_label(index), "body": option["body"]}+ for index, option in enumerate(shuffled)+ ]+ new_expected = next(option["label"] for option in remapped if option["body"] == expected_body)+ return remapped, new_expected++async def _resolve_pending_choice(pending: PendingQuestion, turn_id: str) -> tuple[dict[str, str], str]:@@ -915,6 +943,9 @@ class MasteryQuizTool(BaseTool):question, {option["label"]: option["body"] for option in options})+ if q_type == "choice":+ options, expected = _shuffle_choice_options(options, expected, random.Random())+service = _new_service()progress = _load_path(service, path_id)if progress is None:diff --git a/deeptutor/learning/tests/test_mastery_tools.py b/deeptutor/learning/tests/test_mastery_tools.pyindex f330ba370..837184dda 100644--- a/deeptutor/learning/tests/test_mastery_tools.py+++ b/deeptutor/learning/tests/test_mastery_tools.py@@ -559,8 +559,10 @@ async def test_quiz_infers_choice_and_normalizes_expected_answer(pending = LearningStore().load(path_id).pending_questionassert pending is not Noneassert pending.question_type == "choice"- assert pending.expected_answer == "B"- assert pending.choice_map == {"A": "red", "B": "blue"}+ # Registration shuffles the options (#1691), so labels are asserted through+ # the bodies they carry instead of the order the model registered.+ assert set(pending.choice_map.values()) == {"red", "blue"}+ assert pending.choice_map[pending.expected_answer] == "blue"@pytest.mark.asyncio@@ -668,17 +670,20 @@ async def test_choice_quiz_accepts_ask_user_key_names(path_id):assert result.success is Truepending = LearningStore().load(path_id).pending_questionassert pending is not None- assert pending.choice_map == {- "A": "overwrite the old value",- "B": "concatenate both lists",- "C": "raise, the reducer rejects a list",- }- assert pending.expected_answer == "B"- assert [option["body"] for option in posed_card(result)["options"]] == [+ # Shuffled registration: assert through bodies, not registration order.+ assert sorted(pending.choice_map.values()) == [+ "concatenate both lists","overwrite the old value",+ "raise, the reducer rejects a list",+ ]+ assert pending.choice_map[pending.expected_answer] == "concatenate both lists"+ card_options = posed_card(result)["options"]+ assert sorted(option["body"] for option in card_options) == ["concatenate both lists",+ "overwrite the old value","raise, the reducer rejects a list",]+ assert [option["label"] for option in card_options] == ["A", "B", "C"]@pytest.mark.asyncio@@ -699,8 +704,8 @@ async def test_choice_quiz_rejoins_bare_labels_sent_with_descriptions(path_id):pending = LearningStore().load(path_id).pending_questionassert pending is not Noneassert pending.question_type == "choice"- assert pending.choice_map == {"A": "red", "B": "blue"}- assert pending.expected_answer == "B"+ assert set(pending.choice_map.values()) == {"red", "blue"}+ assert pending.choice_map[pending.expected_answer] == "blue"@pytest.mark.asyncio@@ -765,7 +770,10 @@ async def test_choice_grade_reads_an_answer_typed_in_the_composer(path_id):],)- grade = await MasteryGradeTool().execute(_mastery_path_id=path_id, answer="选C")+ # The shuffle re-labels options, so the typed answer targets whatever+ # label the correct body carries after registration.+ expected_label = LearningStore().load(path_id).pending_question.expected_answer+ grade = await MasteryGradeTool().execute(_mastery_path_id=path_id, answer=f"选{expected_label}")assert grade.success is Trueassert json.loads(grade.content)["is_correct"] is True@@ -817,13 +825,18 @@ async def test_choice_quiz_preserves_bodies_and_normalizes_answer(path_id, sessi)assert quiz.success is True+ # Registration shuffles, so the submission targets the correct body's+ # post-shuffle label and the notebook assertions follow the stored map.+ pending_after_quiz = LearningStore().load(path_id).pending_question+ choice_map = pending_after_quiz.choice_map+ expected_label = pending_after_quiz.expected_answergrade = json.loads((await MasteryGradeTool().execute(_mastery_path_id=path_id,_session_id=session["id"],_turn_id="turn_choice_1",- answer="C",+ answer=expected_label,)).content)@@ -831,14 +844,9 @@ async def test_choice_quiz_preserves_bodies_and_normalizes_answer(path_id, sessientries = await session_store.list_notebook_entries()entry = entries["items"][0]- assert entry["options"] == {- "A": "Step 2 — write the first tool",- "B": "Step 4 — test one call",- "C": "Step 6 — add the stop condition",- "D": "Step 7 — add another tool",- }- assert entry["correct_answer"] == "C"- assert entry["user_answer"] == "C"+ assert entry["options"] == choice_map+ assert entry["correct_answer"] == expected_label+ assert entry["user_answer"] == expected_labelassert entry["is_correct"] is True@@ -871,10 +879,9 @@ async def test_pending_choice_status_reuses_public_contract_without_answer(path_assert card["question_id"] == pending["question_id"]assert card["prompt"] == "Pick a colour"assert card["allow_free_text"] is True- assert card["options"] == [- {"label": "A", "body": "red"},- {"label": "B", "body": "blue"},- ]+ card_options = card["options"]+ assert [option["label"] for option in card_options] == ["A", "B"]+ assert sorted(option["body"] for option in card_options) == ["blue", "red"]assert card["objective"]["id"] == kp_idassert card["attempt"] == 1# The answer key never travels to the learner.@@ -1221,12 +1228,15 @@ async def test_grade_recovers_unreadable_choice_answer(path_id):assert blocked.success is Falseassert "NOT graded" in blocked.content+ # The shuffle re-labels options, so the recovery answers with whatever+ # label the correct body ("-8") carries after registration.+ expected_label = LearningStore().load(path_id).pending_question.expected_answerrecovered = json.loads((await MasteryGradeTool().execute(_mastery_path_id=path_id,question_id=quiz["question_id"],- answer="A",+ answer=expected_label,)).content)@@ -1234,7 +1244,7 @@ async def test_grade_recovers_unreadable_choice_answer(path_id):graded = LearningStore().get_interaction(path_id, quiz["question_id"])assert graded is not Noneassert graded.status == InteractionStatus.GRADED- assert graded.user_answer == "A"+ assert graded.user_answer == expected_label# ── assess: the qualitative gate ─────────────────────────────────────────────diff --git a/tests/capabilities/test_mastery_choice_shuffle.py b/tests/capabilities/test_mastery_choice_shuffle.pynew file mode 100644index 000000000..a50a2f591--- /dev/null+++ b/tests/capabilities/test_mastery_choice_shuffle.py@@ -0,0 +1,136 @@+"""Choice options are shuffled on registration so the correct answer moves.++HKUDS/DeepTutor#1691: models put the correct option first nearly always, and+the registration order is exactly what the card renders, so quiz after quiz+read A = correct. The shuffle lives in the tool layer, after validation and+echo stripping; these tests pin the label remapping, its determinism under a+fixed rng, and the persisted end-to-end contract the grader relies on.+"""++from __future__ import annotations++from pathlib import Path+import random++import pytest++from deeptutor.capabilities.mastery.tools import MasteryQuizTool, _shuffle_choice_options+from deeptutor.learning.models import (+ KnowledgePoint,+ KnowledgeType,+ LearningModule,+ LearningProgress,+)+from deeptutor.learning.storage import LearningStore+++def _use_store_root(monkeypatch, root: Path) -> None:+ def _init(self, root_arg=None):+ self._root = root / "learning"+ self._root.mkdir(parents=True, exist_ok=True)++ monkeypatch.setattr(LearningStore, "__init__", _init)+++def _options() -> list[dict[str, str]]:+ return [+ {"label": "A", "body": "right"},+ {"label": "B", "body": "wrong-1"},+ {"label": "C", "body": "wrong-2"},+ {"label": "D", "body": "wrong-3"},+ ]+++def test_expected_follows_its_body_to_the_new_label() -> None:+ shuffled, expected = _shuffle_choice_options(_options(), "A", random.Random(7))++ assert sorted(option["body"] for option in shuffled) == sorted(+ option["body"] for option in _options()+ )+ assert [option["label"] for option in shuffled] == ["A", "B", "C", "D"]+ assert next(option for option in shuffled if option["label"] == expected)["body"] == "right"+++def test_a_fixed_rng_is_deterministic() -> None:+ once = _shuffle_choice_options(_options(), "A", random.Random(11))+ twice = _shuffle_choice_options(_options(), "A", random.Random(11))++ assert once == twice+++def test_the_correct_answer_does_not_stay_on_one_label() -> None:+ labels = {+ _shuffle_choice_options(_options(), "A", random.Random(seed))[1] for seed in range(40)+ }++ assert len(labels) >= 2+++def test_fewer_than_two_options_is_returned_unchanged() -> None:+ options = [{"label": "A", "body": "only"}]++ assert _shuffle_choice_options(options, "A", random.Random(3)) == (options, "A")+++def _built_path(path_id: str = "path-1") -> LearningProgress:+ return LearningProgress(+ book_id=path_id,+ modules=[+ LearningModule(+ id="m1",+ name="Algebra",+ order=0,+ knowledge_points=[+ KnowledgePoint(+ id=f"{path_id}-kp1",+ name="slope",+ type=KnowledgeType.CONCEPT,+ module_id="m1",+ )+ ],+ )+ ],+ )+++async def test_registration_shuffles_the_card_the_learner_sees(tmp_path, monkeypatch) -> None:+ """The persisted pair must stay consistent: expected points at its body.++ The grader compares the learner's picked label against ``expected_answer``+ on the persisted options, so the shuffle may never decouple them — even+ though the model registered the correct option as A.+ """+ _use_store_root(monkeypatch, tmp_path)+ real_random_class = random.Random+ monkeypatch.setattr(+ "deeptutor.capabilities.mastery.tools.random.Random",+ lambda *args, **kwargs: real_random_class(7),+ )+ LearningStore().save(_built_path())++ result = await MasteryQuizTool().execute(+ _mastery_path_id="path-1",+ knowledge_point_id="path-1-kp1",+ question="Which option is correct?",+ question_type="choice",+ options=[+ {"label": "A", "body": "correct body"},+ {"label": "B", "body": "distractor one"},+ {"label": "C", "body": "distractor two"},+ ],+ expected_answer="A",+ explanation="because.",+ )++ assert result.success is True, result.content+ pending = LearningStore().load("path-1").pending_question+ assert pending is not None+ assert pending.question_type == "choice"+ assert sorted(option.body for option in pending.options) == [+ "correct body",+ "distractor one",+ "distractor two",+ ]+ assert [option.label for option in pending.options] == ["A", "B", "C"]+ assert pending.choice_map[pending.expected_answer] == "correct body"