Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(mastery): don't pre-search the KB with a quiz card answer (#1624)

Maxi3f76b561ff32
diff --git a/deeptutor/agents/loop/pipeline.py b/deeptutor/agents/loop/pipeline.py
index 3211cb44b..ad2318acd 100644
--- a/deeptutor/agents/loop/pipeline.py
+++ b/deeptutor/agents/loop/pipeline.py
@@ -848,6 +848,29 @@ class AgenticLoopPipeline:
]
return "\n\n".join(seed for seed in seeds if seed)
+ def _capability_skips_kb_seed(self, context: UnifiedContext) -> bool:
+ """Let an active capability say this turn's message is not a query.
+
+ The KB seed searches the raw user message. That is right for a
+ question, and wrong for a quiz pick like "A": the search has no
+ context and short-query backends reject it (#1624). The KBs stay
+ mounted, so the model can still retrieve with a real query.
+ """
+ for cap in self._active_loop_capabilities(context):
+ hook = getattr(cap, "skip_kb_seed", None)
+ if not callable(hook):
+ continue
+ try:
+ if hook(context):
+ return True
+ except Exception:
+ logger.warning(
+ "kb seed hook failed for capability %s",
+ getattr(cap, "name", "?"),
+ exc_info=True,
+ )
+ return False
+
def _capability_finish_instruction(self, context: UnifiedContext, final_text: str) -> str:
"""Let an active capability reject a narrow tool-less finish once.
@@ -1458,6 +1481,8 @@ class AgenticLoopPipeline:
query = (context.user_message or "").strip()
if not kbs or not query:
return ""
+ if self._capability_skips_kb_seed(context):
+ return ""
if len(kbs) > KB_SEED_MAX_KBS:
kbs = kbs[:KB_SEED_MAX_KBS]
results = await asyncio.gather(*(self._seed_search_one_kb(kb, query, stream) for kb in kbs))
diff --git a/deeptutor/capabilities/mastery/loop.py b/deeptutor/capabilities/mastery/loop.py
index faee3f6cf..ef8047baa 100644
--- a/deeptutor/capabilities/mastery/loop.py
+++ b/deeptutor/capabilities/mastery/loop.py
@@ -425,6 +425,25 @@ class MasteryLoopCapability:
graded = self._grade_seed(context)
return "\n\n".join(part for part in (skipped, graded) if part)
+ def skip_kb_seed(self, context: UnifiedContext) -> bool:
+ """Don't pre-search the knowledge base with a card answer.
+
+ The message of a card turn is the learner's pick ("A", "True") or a
+ declined question, not a question to look up. The engine grades it
+ against the stored key, and the tutor can still call ``rag`` with the
+ actual question if it needs the material.
+ """
+ if not self.is_active(context):
+ return False
+ meta = context.metadata
+ grade = meta.get("mastery_card_grade")
+ skip = meta.get("mastery_card_skip")
+ return bool(
+ meta.get("mastery_card_answered")
+ or (isinstance(grade, dict) and grade)
+ or (isinstance(skip, dict) and skip.get("skipped"))
+ )
+
def _skip_seed(self, context: UnifiedContext) -> str:
"""State that the learner's declined question is already gone.
diff --git a/deeptutor/services/session/turns/executor.py b/deeptutor/services/session/turns/executor.py
index 16d7f2884..a12d73029 100644
--- a/deeptutor/services/session/turns/executor.py
+++ b/deeptutor/services/session/turns/executor.py
@@ -944,6 +944,10 @@ class TurnExecutor:
"mastery_card_grade": mastery_card_grade or {},
# The question this turn opened by dropping, if it did.
"mastery_card_skip": mastery_card_skip or {},
+ # Whether the message is a pick on the open card at all,
+ # graded or not: "A" is an answer, not something to search.
+ "mastery_card_answered": workspace_mode == WORKSPACE_MODE_MASTERY
+ and bool(payload.get("mastery_answer")),
"mastery_path_lease_managed": mastery_lease_managed,
# Immersive reading: the open material activates the reading
# capability and binds its tools; the viewport tells the
diff --git a/tests/agents/chat/test_kb_seed_context.py b/tests/agents/chat/test_kb_seed_context.py
index 31c628eeb..828d54267 100644
--- a/tests/agents/chat/test_kb_seed_context.py
+++ b/tests/agents/chat/test_kb_seed_context.py
@@ -348,3 +348,65 @@ async def test_run_clips_oversized_seed_passages(monkeypatch: pytest.MonkeyPatch
# section title, truncation marker, and the trailing template line.
seed_block = turn_context.split("[Knowledge Base Context]", 1)[1]
assert len(seed_block) < KB_SEED_CHARS_PER_KB + 400
+
+
+class _SeedPolicyCapability:
+ """A loop capability that only has an opinion on the KB seed."""
+
+ name = "seed_policy"
+
+ def __init__(self, skip: bool | None = None, raise_exc: bool = False) -> None:
+ self._skip = skip
+ self._raise = raise_exc
+
+ def skip_kb_seed(self, _context: UnifiedContext) -> bool:
+ if self._raise:
+ raise RuntimeError("hook broke")
+ return bool(self._skip)
+
+
+async def test_capability_can_skip_the_kb_seed(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A quiz pick like "A" is not a query (#1624): the capability says so."""
+ registry = _SeedRegistry()
+ client = _ScriptedChatClient([[_llm_chunk(content="Correct.")]])
+ pipeline = _make_pipeline(monkeypatch, registry, client)
+ monkeypatch.setattr(
+ pipeline, "_active_loop_capabilities", lambda _ctx: (_SeedPolicyCapability(skip=True),)
+ )
+
+ context = UnifiedContext(
+ session_id="s1",
+ user_message="A",
+ knowledge_bases=["uc_berkeley"],
+ language="en",
+ metadata={"turn_id": "t1"},
+ )
+ seed = await pipeline._retrieve_kb_seed_block(context, StreamBus())
+
+ assert seed == ""
+ assert registry.executed == []
+
+
+async def test_kb_seed_still_runs_when_the_hook_fails(monkeypatch: pytest.MonkeyPatch) -> None:
+ registry = _SeedRegistry()
+ client = _ScriptedChatClient([[_llm_chunk(content="Done.")]])
+ pipeline = _make_pipeline(monkeypatch, registry, client)
+ monkeypatch.setattr(
+ pipeline,
+ "_active_loop_capabilities",
+ lambda _ctx: (_SeedPolicyCapability(raise_exc=True), _SeedPolicyCapability(skip=False)),
+ )
+
+ context = UnifiedContext(
+ session_id="s1",
+ user_message="Intended Audience",
+ knowledge_bases=["uc_berkeley"],
+ language="en",
+ metadata={"turn_id": "t1"},
+ )
+ seed = await pipeline._retrieve_kb_seed_block(context, StreamBus())
+
+ assert "[Knowledge Base Context]" in seed
+ assert [e["kwargs"]["query"] for e in registry.executed] == ["Intended Audience"]
diff --git a/tests/capabilities/test_mastery_capability.py b/tests/capabilities/test_mastery_capability.py
index 3c731f37d..9c5cfb6ab 100644
--- a/tests/capabilities/test_mastery_capability.py
+++ b/tests/capabilities/test_mastery_capability.py
@@ -728,3 +728,30 @@ async def test_a_mode_switch_and_a_quiz_in_one_round_use_the_new_mode(tmp_path,
quiz = next(message for message in results.tool_messages if message.get("tool_call_id") == "c1")
assert "belongs to the" not in str(quiz.get("content", "")), quiz
assert context.metadata["mastery_session_mode"] == "study"
+
+
+ "extra, expected",
+ [
+ ({}, False),
+ ({"mastery_card_answered": True}, True),
+ ({"mastery_card_grade": {"is_correct": True, "result": {"learner_answer": "A"}}}, True),
+ ({"mastery_card_grade": {}}, False),
+ ({"mastery_card_skip": {"skipped": True}}, True),
+ ({"mastery_card_skip": {}}, False),
+ ],
+)
+def test_card_turns_skip_the_kb_seed(extra: dict, expected: bool) -> None:
+ """#1624: a card pick ("A") must not be sent to the KB as a search query."""
+ context = _context()
+ context.metadata.update(extra)
+ assert MasteryLoopCapability().skip_kb_seed(context) is expected
+
+
+def test_kb_seed_is_left_alone_outside_mastery_mode() -> None:
+ context = UnifiedContext(
+ user_message="A",
+ session_id="session-1",
+ metadata={"mastery_card_answered": True},
+ )
+ assert MasteryLoopCapability().skip_kb_seed(context) is False