Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(mastery): validate visual hints and select original source excerpts

yepyhunb8fccb92b20f
diff --git a/deeptutor/capabilities/mastery/tools.py b/deeptutor/capabilities/mastery/tools.py
index 0410168db..4ee8214ac 100644
--- a/deeptutor/capabilities/mastery/tools.py
+++ b/deeptutor/capabilities/mastery/tools.py
@@ -1032,6 +1032,9 @@ class MasteryQuizTool(BaseTool):
],
)
leaf_parameters = {p.name: p.to_schema() for p in definition.parameters}
+ from deeptutor.learning.visual_practice import visual_request_schema
+
+ leaf_parameters["visual"].update(visual_request_schema())
for parameter in definition.parameters:
if parameter.name in {"knowledge_point_id", "question", "expected_answer"}:
parameter.required = False
@@ -1084,6 +1087,9 @@ class MasteryQuizTool(BaseTool):
),
]
)
+ schema = definition.to_openai_schema()["function"]["parameters"]
+ schema["properties"]["visual"].update(visual_request_schema())
+ definition.raw_parameters = schema
return definition
async def execute(self, **kwargs: Any) -> ToolResult:
@@ -1153,6 +1159,16 @@ class MasteryQuizTool(BaseTool):
}
leaves = []
echoed_options = False
+ # Reject malformed assistance anywhere in the batch before the
+ # first leaf performs source/image inspection.
+ from deeptutor.learning.visual_practice import visual_hint_count
+
+ try:
+ for raw in raw_questions:
+ if isinstance(raw, dict) and isinstance(raw.get("visual"), dict):
+ visual_hint_count(raw["visual"].get("hints_used"))
+ except ValueError as exc:
+ return ToolResult(content=str(exc), success=False)
for raw in raw_questions:
if (
not isinstance(raw, dict)
diff --git a/deeptutor/core/tool_protocol.py b/deeptutor/core/tool_protocol.py
index e36d63c78..6b854b129 100644
--- a/deeptutor/core/tool_protocol.py
+++ b/deeptutor/core/tool_protocol.py
@@ -46,7 +46,7 @@ class ToolParameter:
"""Convert to JSON Schema property dict."""
schema: dict[str, Any] = {"type": self.type, "description": self.description}
if self.enum:
- schema["enum"] = self.enum
+ schema["enum"] = list(self.enum)
if self.type == "array":
schema["items"] = self.items if self.items is not None else {"type": "string"}
return schema
diff --git a/deeptutor/learning/lesson_evidence.py b/deeptutor/learning/lesson_evidence.py
index c160e3196..b3ffa5955 100644
--- a/deeptutor/learning/lesson_evidence.py
+++ b/deeptutor/learning/lesson_evidence.py
@@ -13,6 +13,7 @@ import json
import math
from pathlib import Path
import time
+import unicodedata
from typing import Any
from deeptutor.learning.policy import find_knowledge_point
@@ -35,6 +36,46 @@ class LessonUnavailable(ValueError):
"""Stop source-grounded tutoring, rather than silently fall back to guesses."""
+def _selection_excerpt(text: str, chosen: dict[str, Any]) -> str:
+ """Selection uses source IDs, not model-authored quotations as evidence."""
+ if "quote" not in chosen:
+ # A preview of the original, not the model's claimed supporting passage.
+ # The complete candidate text and original pixels still go to inspection.
+ return text[:300]
+ quote = chosen["quote"]
+ error = "The model's source quotation does not match the retrieved text."
+ if not isinstance(quote, str) or not 8 <= len(quote) <= 300:
+ raise LessonUnavailable(error)
+ if quote in text:
+ return quote
+
+ # Legacy responses may change only Unicode composition or whitespace.
+ # Keep offsets into the original so the stored excerpt is never rewritten.
+ clusters: list[tuple[int, str]] = []
+ for offset, char in enumerate(text):
+ if unicodedata.combining(char) and clusters:
+ start, value = clusters[-1]
+ clusters[-1] = (start, value + char)
+ else:
+ clusters.append((offset, char))
+ normalized, spans = [], []
+ for start, value in clusters:
+ for char in unicodedata.normalize("NFD", value):
+ char = " " if char.isspace() else char
+ if char == " " and normalized and normalized[-1] == " ":
+ spans[-1] = (spans[-1][0], start + len(value))
+ else:
+ normalized.append(char)
+ spans.append((start, start + len(value)))
+ target = " ".join(unicodedata.normalize("NFD", quote).split())
+ offset = "".join(normalized).find(target) if target else -1
+ if offset >= 0:
+ original = text[spans[offset][0]:spans[offset + len(target) - 1][1]]
+ if " ".join(unicodedata.normalize("NFD", original).split()) == target:
+ return original
+ raise LessonUnavailable(error)
+
+
def _page_identity(candidate: dict[str, Any]) -> tuple[str, str, int]:
ref = candidate["ref"]
return ref["kb_ref"], ref["document_key"], ref["page_index"]
@@ -511,11 +552,12 @@ async def prepare_lesson(store: Any, path_id: str, kp_id: str, config: Any, stre
'or expand an introductory comparison into an exhaustive mechanistic proof. '
'This is preparation for teaching, '
'not a claim of learner mastery. '
- 'Return JSON {"sufficient":true|false,"selected":[{"id":"c0","quote":"exact substring",'
+ 'Return JSON {"sufficient":true|false,"selected":[{"id":"c0",'
'"reason":"what it supports","neighbor":0|1|-1}],"missing":"..."}. '
f'Select the smallest sufficient set, at most {MAX_LESSON_PAGES} distinct pages. '
'Prefer 1-3 pages; use more only when separate parts of the objective require them. '
- 'Use only supplied IDs, quotes 8-300 characters copied exactly. '
+ 'Use only supplied IDs; do not return quotes. The server retains original source text '
+ 'and a deterministic excerpt, and inspects the original pixels before accepting evidence. '
'Use one selected entry per physical page, combining its supported concepts in the reason; '
'do not repeat the same page once for every fact it supports. '
'Use a neighbor only when an adjacent figure/table is explicitly referenced. '
@@ -542,9 +584,7 @@ async def prepare_lesson(store: Any, path_id: str, kp_id: str, config: Any, stre
candidate = by_id.get(chosen.get("id")) if isinstance(chosen, dict) else None
if candidate is None:
raise LessonUnavailable("The model selected a source ID that was not retrieved.")
- quote = str(chosen.get("quote") or "")
- if not 8 <= len(quote) <= 300 or quote not in candidate["text"]:
- raise LessonUnavailable("The model's source quotation does not match the retrieved text.")
+ quote = _selection_excerpt(candidate["text"], chosen)
ref = SourceRef.model_validate(candidate["ref"])
resolve_source(ref)
page_id = (ref.kb_ref, ref.document_key, ref.page_index)
diff --git a/deeptutor/learning/visual_practice.py b/deeptutor/learning/visual_practice.py
index 7292712f4..234fa5840 100644
--- a/deeptutor/learning/visual_practice.py
+++ b/deeptutor/learning/visual_practice.py
@@ -22,6 +22,49 @@ def _answer_in_quote(answer: str, quote: str) -> bool:
return needle in quote
+def visual_hint_count(value: Any) -> int:
+ """Normalize explicit assistance before resolving any source or image."""
+ if value is None:
+ return 0
+ if type(value) is int and value >= 0:
+ return value
+ if isinstance(value, list) and value and all(
+ isinstance(hint, str) and hint.strip() for hint in value
+ ):
+ return len(value)
+ raise ValueError(
+ "visual.hints_used must be a nonnegative integer or a nonempty list of "
+ "nonempty hint descriptions; omit it or use null when no hints were used."
+ )
+
+
+def visual_request_schema() -> dict[str, Any]:
+ """The same typed visual contract for single cards and grouped leaves."""
+ return {
+ "type": "object",
+ "properties": {
+ "task": {"type": "string", "enum": ["identification", "relationship", "table_graph", "comparison"]},
+ "sources": {"type": "array", "minItems": 1, "maxItems": 2, "items": {
+ "type": "object", "properties": {
+ "kb_name": {"type": "string"}, "asset_id": {"type": "string"},
+ "source_path": {"type": "string"}, "page": {"type": "integer", "minimum": 1},
+ "source_hash": {"type": "string"}, "figure": {"type": "string"},
+ "region": {"type": "array", "minItems": 4, "maxItems": 4,
+ "items": {"type": "number", "minimum": 0, "maximum": 1}},
+ }, "required": ["kb_name"],
+ }},
+ "reference_quote": {"type": "string"},
+ "accepted_answers": {"type": "array", "maxItems": 12, "items": {"type": "string"}},
+ "key_status": {"type": "string", "enum": ["verified", "unverified"]},
+ "answer_cues": {"type": "string", "enum": ["none", "visible", "unverified"]},
+ "hints_used": {"description": "Assistance count, or descriptions of hints already given. Descriptions count as assistance, not independent recall.",
+ "anyOf": [{"type": "integer", "minimum": 0},
+ {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}},
+ {"type": "null"}]},
+ }, "required": ["sources"],
+ }
+
+
def prepare_visual(
request: dict[str, Any],
*,
@@ -30,6 +73,7 @@ def prepare_visual(
attached_kbs: list[str] | None = None,
inspected_image_hashes: list[str] | None = None,
) -> tuple[dict[str, Any], list[str]]:
+ hints_used = visual_hint_count(request.get("hints_used"))
from deeptutor.multi_user.knowledge_access import resolve_for_rag
from deeptutor.services.rag.kb_paths import resolve_kb_dir
from deeptutor.services.rag.source_visuals import retrieve_visual
@@ -141,7 +185,7 @@ def prepare_visual(
"pixels_inspected": inspected,
"reference_quote": quote,
"reference_answer": correct_body,
- "hints_used": max(int(request.get("hints_used") or 0), int(cues != "none")),
+ "hints_used": max(hints_used, int(cues != "none")),
}, aliases
diff --git a/tests/api/test_tools_router.py b/tests/api/test_tools_router.py
index b4e0703a4..e6023223a 100644
--- a/tests/api/test_tools_router.py
+++ b/tests/api/test_tools_router.py
@@ -14,6 +14,31 @@ from deeptutor.tools.builtin import (
)
+def test_nullable_request_schema_does_not_corrupt_public_tool_metadata():
+ from deeptutor.core.tool_protocol import ToolDefinition, ToolParameter
+
+ definition = ToolDefinition(
+ name="source_view", description="Synthetic source presentation",
+ parameters=[ToolParameter(name="presentation", type="string", required=False,
+ enum=["inspect", "teaching", "reveal"])],
+ )
+ request = definition.to_openai_schema()["function"]["parameters"]
+ request["properties"]["presentation"]["enum"].append(None)
+ _, _, metadata = tools_router._serialise_definition(definition)
+ assert metadata[0].enum == ["inspect", "teaching", "reveal"]
+ assert definition.to_openai_schema()["function"]["parameters"]["properties"]["presentation"]["enum"] == metadata[0].enum
+
+
+def test_rag_definition_can_be_serialised_for_tools_ui():
+ from deeptutor.tools.builtin import RAGTool
+
+ definition = RAGTool().get_definition()
+ name, _, parameters = tools_router._serialise_definition(definition)
+ assert name == "rag"
+ presentation = next(parameter for parameter in parameters if parameter.name == "presentation")
+ assert presentation.enum == ["inspect", "teaching", "reveal"]
+
+
@pytest.mark.asyncio
async def test_list_builtin_tools_marks_toggleable_set(
monkeypatch: pytest.MonkeyPatch, tmp_path
diff --git a/tests/capabilities/test_mastery_visual_hint_contract.py b/tests/capabilities/test_mastery_visual_hint_contract.py
new file mode 100644
index 000000000..55144d5d3
--- /dev/null
+++ b/tests/capabilities/test_mastery_visual_hint_contract.py
@@ -0,0 +1,91 @@
+"""Hint descriptions retain assistance without failing after source inspection."""
+
+from types import SimpleNamespace
+
+import pytest
+
+from deeptutor.capabilities.mastery.tools import MasteryQuizTool
+from deeptutor.learning.models import LearningProgress
+from deeptutor.learning.service import LearningService
+from deeptutor.learning.storage import LearningStore
+from deeptutor.learning.visual_practice import prepare_visual
+
+
+def visual_source(monkeypatch):
+ calls = []
+ def resolve(_name):
+ calls.append("resolve")
+ return SimpleNamespace(base_dir="synthetic", name="kb")
+ def retrieve(*_args, **_kwargs):
+ calls.append("retrieve")
+ return SimpleNamespace(error="", images=[({
+ "asset_id": "synthetic-figure", "source_path": "synthetic.pdf",
+ "source_document_id": "a" * 64, "page_number": 1,
+ "text": "The target is alpha.",
+ }, b"synthetic-pixels")], sources=[{
+ "visual_asset_url": "/synthetic-image", "url": "/synthetic-document",
+ }])
+ monkeypatch.setattr("deeptutor.multi_user.knowledge_access.resolve_for_rag", resolve)
+ monkeypatch.setattr("deeptutor.services.rag.source_visuals.retrieve_visual", retrieve)
+ return calls
+
+
+def request(hints):
+ return {"sources": [{"kb_name": "kb", "asset_id": "synthetic-figure"}],
+ "reference_quote": "The target is alpha.", "key_status": "verified",
+ "answer_cues": "none", "pixels_inspected": True, "hints_used": hints}
+
+
[email protected]("hints,count", [
+ (["A solution was previously visible"], 1), (["First hint", "Second hint"], 2),
+ (3, 3), (0, 0), (None, 0),
+])
+def test_valid_hints_preserve_count_and_independent_credit(visual_source, tmp_path, hints, count):
+ context, _ = prepare_visual(request(hints), expected_answer="alpha", options={})
+ assert context["hints_used"] == count
+ progress = LearningProgress(book_id="synthetic")
+ LearningService(LearningStore(tmp_path))._apply_grade(progress, question_id="q", knowledge_point_id="kp",
+ module_id="m", user_answer="alpha", expected_answer="alpha", question_type="short",
+ visual_context=context)
+ assert progress.quiz_attempts[0].independent is (count == 0)
+ assert progress.learning_evidence[0].hints_used == count
+ assert progress.learning_evidence[0].quality == (.6 if count else 1)
+
+
[email protected]("invalid", [True, False, 1.5, -1, "", "1", [], [""], [" "], [1], {}])
+def test_invalid_hint_shapes_fail_before_source_io(visual_source, invalid):
+ with pytest.raises(ValueError, match="hints_used"):
+ prepare_visual(request(invalid), expected_answer="alpha", options={})
+ assert visual_source == []
+
+
+def test_single_and_group_visual_schema_type_hint_count_or_descriptions():
+ definition = MasteryQuizTool().get_definition()
+ props = definition.to_openai_schema()["function"]["parameters"]["properties"]
+ single = props["visual"]["properties"]["hints_used"]
+ grouped = props["questions"]["items"]["properties"]["visual"]["properties"]["hints_used"]
+ assert grouped == single
+ assert {branch["type"] for branch in single["anyOf"]} == {"integer", "array", "null"}
+ assert next(branch for branch in single["anyOf"] if branch["type"] == "integer")["minimum"] == 0
+ descriptions = next(branch for branch in single["anyOf"] if branch["type"] == "array")
+ assert descriptions["minItems"] == 1 and descriptions["items"]["type"] == "string"
+ assert all(p.sensitive for p in definition.parameters if p.name in {"visual", "questions"})
+
+
+async def test_invalid_second_leaf_hints_reject_group_before_building_first_leaf(monkeypatch):
+ from unittest.mock import AsyncMock
+
+ from deeptutor.capabilities.mastery import tools
+
+ monkeypatch.setattr(tools, "_new_service", lambda: object())
+ monkeypatch.setattr(tools, "_load_path", lambda *_: LearningProgress(book_id="synthetic"))
+ builder = AsyncMock(side_effect=AssertionError("No source-bearing leaf should be built"))
+ monkeypatch.setattr(MasteryQuizTool, "_build_question", builder)
+ leaf = {"knowledge_point_id": "kp", "question": "Identify the target", "expected_answer": "alpha"}
+ result = await MasteryQuizTool().execute(_mastery_path_id="synthetic", questions=[
+ {**leaf, "visual": request(1)}, {**leaf, "visual": request([""])},
+ ])
+ assert not result.success and "hints_used" in result.content
+ builder.assert_not_called()
diff --git a/tests/services/rag/test_mastery_source_pages.py b/tests/services/rag/test_mastery_source_pages.py
index 764b734e7..ff23d5729 100644
--- a/tests/services/rag/test_mastery_source_pages.py
+++ b/tests/services/rag/test_mastery_source_pages.py
@@ -691,6 +691,43 @@ def test_repair_checks_original_with_detail_even_for_second_source(material):
assert result[2] is sources[0]
+def test_selection_id_without_quote_preserves_original_evidence(material, monkeypatch):
+ calls = _install_preparation(material, monkeypatch, rankings=[{
+ "sufficient": True, "selected": [{"id": "c0", "reason": "Membrane structure"}]}])
+ evidence = asyncio.run(lessons.prepare_lesson(material.store, "path", "kp", material.config))
+ source = evidence["sources"][0]
+ candidate = calls[1][0]["candidates"][0]
+ assert source["text"] == candidate["text"]
+ assert source["quote"] == candidate["text"][:300]
+ assert calls[2][0]["sources"][0]["text"] == candidate["text"]
+ assert calls[2][1] and len(calls) == 3
+
+
+def test_selection_unknown_id_without_quote_is_rejected(material, monkeypatch):
+ _install_preparation(material, monkeypatch, rankings=[{
+ "sufficient": True, "selected": [{"id": "not-retrieved", "reason": "Membrane structure"}]}])
+ with pytest.raises(lessons.LessonUnavailable, match="source ID"):
+ asyncio.run(lessons.prepare_lesson(material.store, "path", "kp", material.config))
+ assert not material.store.load("path").lesson_evidence
+
+
[email protected]("original,quote", [
+ ("prefix lipids\n\tform a\u00a0bilayer suffix", "lipids form a bilayer"),
+ ("prefix cafe\u0301 membrane suffix", "caf\u00e9 membrane"),
+ ("prefix caf\u00e9 membrane suffix", "cafe\u0301 membrane"),
+])
+def test_legacy_selection_quote_normalization_preserves_original(original, quote):
+ excerpt = lessons._selection_excerpt(original, {"quote": quote})
+ assert excerpt in original
+ assert excerpt == original[len("prefix "):-len(" suffix")]
+
+
[email protected]("quote", ["membranes are made of lipids", "LIPIDS form a bilayer", "", "bilayer"])
+def test_legacy_selection_quote_never_accepts_paraphrase_or_invalid_length(quote):
+ with pytest.raises(lessons.LessonUnavailable, match="quotation"):
+ lessons._selection_excerpt("Membrane lipids form a bilayer.", {"quote": quote})
+
+
def test_unknown_quote_is_not_ready_and_not_mastery(material, monkeypatch):
_install_preparation(material, monkeypatch, bad_quote=True)
with pytest.raises(lessons.LessonUnavailable, match="quotation"):