Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(mastery): shuffle choice options so the correct answer moves (#1691)

XZHcb4c25db26fa
diff --git a/deeptutor/capabilities/mastery/tools.py b/deeptutor/capabilities/mastery/tools.py
index e756e9d16..97c01f578 100644
--- a/deeptutor/capabilities/mastery/tools.py
+++ b/deeptutor/capabilities/mastery/tools.py
@@ -18,6 +18,7 @@ from __future__ import annotations
import asyncio
import json
import logging
+import random
import re
from typing import TYPE_CHECKING, Any
import uuid
@@ -64,7 +65,7 @@ from deeptutor.learning.objective_relations import (
RelationRefs,
normalize_refs,
)
-from deeptutor.learning.pending import public_pending_question
+from deeptutor.learning.pending import positional_label, public_pending_question
from deeptutor.learning.policy import (
QUALITATIVE_TYPES,
display_mastery,
@@ -286,6 +287,33 @@ def _normalize_quiz_contract(
return question_type, options, resolved_expected
+def _shuffle_choice_options(
+ options: list[dict[str, str]], expected: str, rng: random.Random
+) -> tuple[list[dict[str, str]], str]:
+ """Reorder choice options so the correct answer is not pinned to one label.
+
+ Models put the correct option first the overwhelming majority of the time,
+ and the registration order is exactly what the card renders — so an unmixed
+ session reads A = correct on quiz after quiz (HKUDS/DeepTutor#1691). The
+ shuffle runs after validation and echo stripping, so every contract
+ invariant still holds on the payload that gets persisted: bodies stay
+ unique, labels are re-issued positionally, and ``expected`` follows its
+ body to the new label — the interactive card and the deterministic grader
+ keep comparing the same representation.
+ """
+ if len(options) < 2:
+ return options, expected
+ expected_body = next(option["body"] for option in options if option["label"] == expected)
+ shuffled = list(options)
+ rng.shuffle(shuffled)
+ remapped = [
+ {"label": positional_label(index), "body": option["body"]}
+ for index, option in enumerate(shuffled)
+ ]
+ new_expected = next(option["label"] for option in remapped if option["body"] == expected_body)
+ return remapped, new_expected
+
+
async def _resolve_pending_choice(
pending: PendingQuestion, turn_id: str
) -> tuple[dict[str, str], str]:
@@ -915,6 +943,9 @@ class MasteryQuizTool(BaseTool):
question, {option["label"]: option["body"] for option in options}
)
+ if q_type == "choice":
+ options, expected = _shuffle_choice_options(options, expected, random.Random())
+
service = _new_service()
progress = _load_path(service, path_id)
if progress is None:
diff --git a/deeptutor/learning/tests/test_mastery_tools.py b/deeptutor/learning/tests/test_mastery_tools.py
index f330ba370..837184dda 100644
--- a/deeptutor/learning/tests/test_mastery_tools.py
+++ b/deeptutor/learning/tests/test_mastery_tools.py
@@ -559,8 +559,10 @@ async def test_quiz_infers_choice_and_normalizes_expected_answer(
pending = LearningStore().load(path_id).pending_question
assert pending is not None
assert pending.question_type == "choice"
- assert pending.expected_answer == "B"
- assert pending.choice_map == {"A": "red", "B": "blue"}
+ # Registration shuffles the options (#1691), so labels are asserted through
+ # the bodies they carry instead of the order the model registered.
+ assert set(pending.choice_map.values()) == {"red", "blue"}
+ assert pending.choice_map[pending.expected_answer] == "blue"
@pytest.mark.asyncio
@@ -668,17 +670,20 @@ async def test_choice_quiz_accepts_ask_user_key_names(path_id):
assert result.success is True
pending = LearningStore().load(path_id).pending_question
assert pending is not None
- assert pending.choice_map == {
- "A": "overwrite the old value",
- "B": "concatenate both lists",
- "C": "raise, the reducer rejects a list",
- }
- assert pending.expected_answer == "B"
- assert [option["body"] for option in posed_card(result)["options"]] == [
+ # Shuffled registration: assert through bodies, not registration order.
+ assert sorted(pending.choice_map.values()) == [
+ "concatenate both lists",
"overwrite the old value",
+ "raise, the reducer rejects a list",
+ ]
+ assert pending.choice_map[pending.expected_answer] == "concatenate both lists"
+ card_options = posed_card(result)["options"]
+ assert sorted(option["body"] for option in card_options) == [
"concatenate both lists",
+ "overwrite the old value",
"raise, the reducer rejects a list",
]
+ assert [option["label"] for option in card_options] == ["A", "B", "C"]
@pytest.mark.asyncio
@@ -699,8 +704,8 @@ async def test_choice_quiz_rejoins_bare_labels_sent_with_descriptions(path_id):
pending = LearningStore().load(path_id).pending_question
assert pending is not None
assert pending.question_type == "choice"
- assert pending.choice_map == {"A": "red", "B": "blue"}
- assert pending.expected_answer == "B"
+ assert set(pending.choice_map.values()) == {"red", "blue"}
+ assert pending.choice_map[pending.expected_answer] == "blue"
@pytest.mark.asyncio
@@ -765,7 +770,10 @@ async def test_choice_grade_reads_an_answer_typed_in_the_composer(path_id):
],
)
- grade = await MasteryGradeTool().execute(_mastery_path_id=path_id, answer="选C")
+ # The shuffle re-labels options, so the typed answer targets whatever
+ # label the correct body carries after registration.
+ expected_label = LearningStore().load(path_id).pending_question.expected_answer
+ grade = await MasteryGradeTool().execute(_mastery_path_id=path_id, answer=f"选{expected_label}")
assert grade.success is True
assert json.loads(grade.content)["is_correct"] is True
@@ -817,13 +825,18 @@ async def test_choice_quiz_preserves_bodies_and_normalizes_answer(path_id, sessi
)
assert quiz.success is True
+ # Registration shuffles, so the submission targets the correct body's
+ # post-shuffle label and the notebook assertions follow the stored map.
+ pending_after_quiz = LearningStore().load(path_id).pending_question
+ choice_map = pending_after_quiz.choice_map
+ expected_label = pending_after_quiz.expected_answer
grade = json.loads(
(
await MasteryGradeTool().execute(
_mastery_path_id=path_id,
_session_id=session["id"],
_turn_id="turn_choice_1",
- answer="C",
+ answer=expected_label,
)
).content
)
@@ -831,14 +844,9 @@ async def test_choice_quiz_preserves_bodies_and_normalizes_answer(path_id, sessi
entries = await session_store.list_notebook_entries()
entry = entries["items"][0]
- assert entry["options"] == {
- "A": "Step 2 — write the first tool",
- "B": "Step 4 — test one call",
- "C": "Step 6 — add the stop condition",
- "D": "Step 7 — add another tool",
- }
- assert entry["correct_answer"] == "C"
- assert entry["user_answer"] == "C"
+ assert entry["options"] == choice_map
+ assert entry["correct_answer"] == expected_label
+ assert entry["user_answer"] == expected_label
assert entry["is_correct"] is True
@@ -871,10 +879,9 @@ async def test_pending_choice_status_reuses_public_contract_without_answer(path_
assert card["question_id"] == pending["question_id"]
assert card["prompt"] == "Pick a colour"
assert card["allow_free_text"] is True
- assert card["options"] == [
- {"label": "A", "body": "red"},
- {"label": "B", "body": "blue"},
- ]
+ card_options = card["options"]
+ assert [option["label"] for option in card_options] == ["A", "B"]
+ assert sorted(option["body"] for option in card_options) == ["blue", "red"]
assert card["objective"]["id"] == kp_id
assert card["attempt"] == 1
# The answer key never travels to the learner.
@@ -1221,12 +1228,15 @@ async def test_grade_recovers_unreadable_choice_answer(path_id):
assert blocked.success is False
assert "NOT graded" in blocked.content
+ # The shuffle re-labels options, so the recovery answers with whatever
+ # label the correct body ("-8") carries after registration.
+ expected_label = LearningStore().load(path_id).pending_question.expected_answer
recovered = json.loads(
(
await MasteryGradeTool().execute(
_mastery_path_id=path_id,
question_id=quiz["question_id"],
- answer="A",
+ answer=expected_label,
)
).content
)
@@ -1234,7 +1244,7 @@ async def test_grade_recovers_unreadable_choice_answer(path_id):
graded = LearningStore().get_interaction(path_id, quiz["question_id"])
assert graded is not None
assert graded.status == InteractionStatus.GRADED
- assert graded.user_answer == "A"
+ assert graded.user_answer == expected_label
# ── assess: the qualitative gate ─────────────────────────────────────────────
diff --git a/tests/capabilities/test_mastery_choice_shuffle.py b/tests/capabilities/test_mastery_choice_shuffle.py
new file mode 100644
index 000000000..a50a2f591
--- /dev/null
+++ b/tests/capabilities/test_mastery_choice_shuffle.py
@@ -0,0 +1,136 @@
+"""Choice options are shuffled on registration so the correct answer moves.
+
+HKUDS/DeepTutor#1691: models put the correct option first nearly always, and
+the registration order is exactly what the card renders, so quiz after quiz
+read A = correct. The shuffle lives in the tool layer, after validation and
+echo stripping; these tests pin the label remapping, its determinism under a
+fixed rng, and the persisted end-to-end contract the grader relies on.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+import random
+
+import pytest
+
+from deeptutor.capabilities.mastery.tools import MasteryQuizTool, _shuffle_choice_options
+from deeptutor.learning.models import (
+ KnowledgePoint,
+ KnowledgeType,
+ LearningModule,
+ LearningProgress,
+)
+from deeptutor.learning.storage import LearningStore
+
+
+def _use_store_root(monkeypatch, root: Path) -> None:
+ def _init(self, root_arg=None):
+ self._root = root / "learning"
+ self._root.mkdir(parents=True, exist_ok=True)
+
+ monkeypatch.setattr(LearningStore, "__init__", _init)
+
+
+def _options() -> list[dict[str, str]]:
+ return [
+ {"label": "A", "body": "right"},
+ {"label": "B", "body": "wrong-1"},
+ {"label": "C", "body": "wrong-2"},
+ {"label": "D", "body": "wrong-3"},
+ ]
+
+
+def test_expected_follows_its_body_to_the_new_label() -> None:
+ shuffled, expected = _shuffle_choice_options(_options(), "A", random.Random(7))
+
+ assert sorted(option["body"] for option in shuffled) == sorted(
+ option["body"] for option in _options()
+ )
+ assert [option["label"] for option in shuffled] == ["A", "B", "C", "D"]
+ assert next(option for option in shuffled if option["label"] == expected)["body"] == "right"
+
+
+def test_a_fixed_rng_is_deterministic() -> None:
+ once = _shuffle_choice_options(_options(), "A", random.Random(11))
+ twice = _shuffle_choice_options(_options(), "A", random.Random(11))
+
+ assert once == twice
+
+
+def test_the_correct_answer_does_not_stay_on_one_label() -> None:
+ labels = {
+ _shuffle_choice_options(_options(), "A", random.Random(seed))[1] for seed in range(40)
+ }
+
+ assert len(labels) >= 2
+
+
+def test_fewer_than_two_options_is_returned_unchanged() -> None:
+ options = [{"label": "A", "body": "only"}]
+
+ assert _shuffle_choice_options(options, "A", random.Random(3)) == (options, "A")
+
+
+def _built_path(path_id: str = "path-1") -> LearningProgress:
+ return LearningProgress(
+ book_id=path_id,
+ modules=[
+ LearningModule(
+ id="m1",
+ name="Algebra",
+ order=0,
+ knowledge_points=[
+ KnowledgePoint(
+ id=f"{path_id}-kp1",
+ name="slope",
+ type=KnowledgeType.CONCEPT,
+ module_id="m1",
+ )
+ ],
+ )
+ ],
+ )
+
+
+async def test_registration_shuffles_the_card_the_learner_sees(tmp_path, monkeypatch) -> None:
+ """The persisted pair must stay consistent: expected points at its body.
+
+ The grader compares the learner's picked label against ``expected_answer``
+ on the persisted options, so the shuffle may never decouple them — even
+ though the model registered the correct option as A.
+ """
+ _use_store_root(monkeypatch, tmp_path)
+ real_random_class = random.Random
+ monkeypatch.setattr(
+ "deeptutor.capabilities.mastery.tools.random.Random",
+ lambda *args, **kwargs: real_random_class(7),
+ )
+ LearningStore().save(_built_path())
+
+ result = await MasteryQuizTool().execute(
+ _mastery_path_id="path-1",
+ knowledge_point_id="path-1-kp1",
+ question="Which option is correct?",
+ question_type="choice",
+ options=[
+ {"label": "A", "body": "correct body"},
+ {"label": "B", "body": "distractor one"},
+ {"label": "C", "body": "distractor two"},
+ ],
+ expected_answer="A",
+ explanation="because.",
+ )
+
+ assert result.success is True, result.content
+ pending = LearningStore().load("path-1").pending_question
+ assert pending is not None
+ assert pending.question_type == "choice"
+ assert sorted(option.body for option in pending.options) == [
+ "correct body",
+ "distractor one",
+ "distractor two",
+ ]
+ assert [option.label for option in pending.options] == ["A", "B", "C"]
+ assert pending.choice_map[pending.expected_answer] == "correct body"