DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(mastery): validate visual hints and select original source excerpts
yepyhunb8fccb92b20f
diff --git a/deeptutor/capabilities/mastery/tools.py b/deeptutor/capabilities/mastery/tools.pyindex 0410168db..4ee8214ac 100644--- a/deeptutor/capabilities/mastery/tools.py+++ b/deeptutor/capabilities/mastery/tools.py@@ -1032,6 +1032,9 @@ class MasteryQuizTool(BaseTool):],)leaf_parameters = {p.name: p.to_schema() for p in definition.parameters}+ from deeptutor.learning.visual_practice import visual_request_schema++ leaf_parameters["visual"].update(visual_request_schema())for parameter in definition.parameters:if parameter.name in {"knowledge_point_id", "question", "expected_answer"}:parameter.required = False@@ -1084,6 +1087,9 @@ class MasteryQuizTool(BaseTool):),])+ schema = definition.to_openai_schema()["function"]["parameters"]+ schema["properties"]["visual"].update(visual_request_schema())+ definition.raw_parameters = schemareturn definitionasync def execute(self, **kwargs: Any) -> ToolResult:@@ -1153,6 +1159,16 @@ class MasteryQuizTool(BaseTool):}leaves = []echoed_options = False+ # Reject malformed assistance anywhere in the batch before the+ # first leaf performs source/image inspection.+ from deeptutor.learning.visual_practice import visual_hint_count++ try:+ for raw in raw_questions:+ if isinstance(raw, dict) and isinstance(raw.get("visual"), dict):+ visual_hint_count(raw["visual"].get("hints_used"))+ except ValueError as exc:+ return ToolResult(content=str(exc), success=False)for raw in raw_questions:if (not isinstance(raw, dict)diff --git a/deeptutor/core/tool_protocol.py b/deeptutor/core/tool_protocol.pyindex e36d63c78..6b854b129 100644--- a/deeptutor/core/tool_protocol.py+++ b/deeptutor/core/tool_protocol.py@@ -46,7 +46,7 @@ class ToolParameter:"""Convert to JSON Schema property dict."""schema: dict[str, Any] = {"type": self.type, "description": self.description}if self.enum:- schema["enum"] = self.enum+ schema["enum"] = list(self.enum)if self.type == "array":schema["items"] = self.items if self.items is not None else {"type": "string"}return schemadiff --git a/deeptutor/learning/lesson_evidence.py b/deeptutor/learning/lesson_evidence.pyindex c160e3196..b3ffa5955 100644--- a/deeptutor/learning/lesson_evidence.py+++ b/deeptutor/learning/lesson_evidence.py@@ -13,6 +13,7 @@ import jsonimport mathfrom pathlib import Pathimport time+import unicodedatafrom typing import Anyfrom deeptutor.learning.policy import find_knowledge_point@@ -35,6 +36,46 @@ class LessonUnavailable(ValueError):"""Stop source-grounded tutoring, rather than silently fall back to guesses."""+def _selection_excerpt(text: str, chosen: dict[str, Any]) -> str:+ """Selection uses source IDs, not model-authored quotations as evidence."""+ if "quote" not in chosen:+ # A preview of the original, not the model's claimed supporting passage.+ # The complete candidate text and original pixels still go to inspection.+ return text[:300]+ quote = chosen["quote"]+ error = "The model's source quotation does not match the retrieved text."+ if not isinstance(quote, str) or not 8 <= len(quote) <= 300:+ raise LessonUnavailable(error)+ if quote in text:+ return quote++ # Legacy responses may change only Unicode composition or whitespace.+ # Keep offsets into the original so the stored excerpt is never rewritten.+ clusters: list[tuple[int, str]] = []+ for offset, char in enumerate(text):+ if unicodedata.combining(char) and clusters:+ start, value = clusters[-1]+ clusters[-1] = (start, value + char)+ else:+ clusters.append((offset, char))+ normalized, spans = [], []+ for start, value in clusters:+ for char in unicodedata.normalize("NFD", value):+ char = " " if char.isspace() else char+ if char == " " and normalized and normalized[-1] == " ":+ spans[-1] = (spans[-1][0], start + len(value))+ else:+ normalized.append(char)+ spans.append((start, start + len(value)))+ target = " ".join(unicodedata.normalize("NFD", quote).split())+ offset = "".join(normalized).find(target) if target else -1+ if offset >= 0:+ original = text[spans[offset][0]:spans[offset + len(target) - 1][1]]+ if " ".join(unicodedata.normalize("NFD", original).split()) == target:+ return original+ raise LessonUnavailable(error)++def _page_identity(candidate: dict[str, Any]) -> tuple[str, str, int]:ref = candidate["ref"]return ref["kb_ref"], ref["document_key"], ref["page_index"]@@ -511,11 +552,12 @@ async def prepare_lesson(store: Any, path_id: str, kp_id: str, config: Any, stre'or expand an introductory comparison into an exhaustive mechanistic proof. ''This is preparation for teaching, ''not a claim of learner mastery. '- 'Return JSON {"sufficient":true|false,"selected":[{"id":"c0","quote":"exact substring",'+ 'Return JSON {"sufficient":true|false,"selected":[{"id":"c0",''"reason":"what it supports","neighbor":0|1|-1}],"missing":"..."}. 'f'Select the smallest sufficient set, at most {MAX_LESSON_PAGES} distinct pages. ''Prefer 1-3 pages; use more only when separate parts of the objective require them. '- 'Use only supplied IDs, quotes 8-300 characters copied exactly. '+ 'Use only supplied IDs; do not return quotes. The server retains original source text '+ 'and a deterministic excerpt, and inspects the original pixels before accepting evidence. ''Use one selected entry per physical page, combining its supported concepts in the reason; ''do not repeat the same page once for every fact it supports. ''Use a neighbor only when an adjacent figure/table is explicitly referenced. '@@ -542,9 +584,7 @@ async def prepare_lesson(store: Any, path_id: str, kp_id: str, config: Any, strecandidate = by_id.get(chosen.get("id")) if isinstance(chosen, dict) else Noneif candidate is None:raise LessonUnavailable("The model selected a source ID that was not retrieved.")- quote = str(chosen.get("quote") or "")- if not 8 <= len(quote) <= 300 or quote not in candidate["text"]:- raise LessonUnavailable("The model's source quotation does not match the retrieved text.")+ quote = _selection_excerpt(candidate["text"], chosen)ref = SourceRef.model_validate(candidate["ref"])resolve_source(ref)page_id = (ref.kb_ref, ref.document_key, ref.page_index)diff --git a/deeptutor/learning/visual_practice.py b/deeptutor/learning/visual_practice.pyindex 7292712f4..234fa5840 100644--- a/deeptutor/learning/visual_practice.py+++ b/deeptutor/learning/visual_practice.py@@ -22,6 +22,49 @@ def _answer_in_quote(answer: str, quote: str) -> bool:return needle in quote+def visual_hint_count(value: Any) -> int:+ """Normalize explicit assistance before resolving any source or image."""+ if value is None:+ return 0+ if type(value) is int and value >= 0:+ return value+ if isinstance(value, list) and value and all(+ isinstance(hint, str) and hint.strip() for hint in value+ ):+ return len(value)+ raise ValueError(+ "visual.hints_used must be a nonnegative integer or a nonempty list of "+ "nonempty hint descriptions; omit it or use null when no hints were used."+ )+++def visual_request_schema() -> dict[str, Any]:+ """The same typed visual contract for single cards and grouped leaves."""+ return {+ "type": "object",+ "properties": {+ "task": {"type": "string", "enum": ["identification", "relationship", "table_graph", "comparison"]},+ "sources": {"type": "array", "minItems": 1, "maxItems": 2, "items": {+ "type": "object", "properties": {+ "kb_name": {"type": "string"}, "asset_id": {"type": "string"},+ "source_path": {"type": "string"}, "page": {"type": "integer", "minimum": 1},+ "source_hash": {"type": "string"}, "figure": {"type": "string"},+ "region": {"type": "array", "minItems": 4, "maxItems": 4,+ "items": {"type": "number", "minimum": 0, "maximum": 1}},+ }, "required": ["kb_name"],+ }},+ "reference_quote": {"type": "string"},+ "accepted_answers": {"type": "array", "maxItems": 12, "items": {"type": "string"}},+ "key_status": {"type": "string", "enum": ["verified", "unverified"]},+ "answer_cues": {"type": "string", "enum": ["none", "visible", "unverified"]},+ "hints_used": {"description": "Assistance count, or descriptions of hints already given. Descriptions count as assistance, not independent recall.",+ "anyOf": [{"type": "integer", "minimum": 0},+ {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}},+ {"type": "null"}]},+ }, "required": ["sources"],+ }++def prepare_visual(request: dict[str, Any],*,@@ -30,6 +73,7 @@ def prepare_visual(attached_kbs: list[str] | None = None,inspected_image_hashes: list[str] | None = None,) -> tuple[dict[str, Any], list[str]]:+ hints_used = visual_hint_count(request.get("hints_used"))from deeptutor.multi_user.knowledge_access import resolve_for_ragfrom deeptutor.services.rag.kb_paths import resolve_kb_dirfrom deeptutor.services.rag.source_visuals import retrieve_visual@@ -141,7 +185,7 @@ def prepare_visual("pixels_inspected": inspected,"reference_quote": quote,"reference_answer": correct_body,- "hints_used": max(int(request.get("hints_used") or 0), int(cues != "none")),+ "hints_used": max(hints_used, int(cues != "none")),}, aliasesdiff --git a/tests/api/test_tools_router.py b/tests/api/test_tools_router.pyindex b4e0703a4..e6023223a 100644--- a/tests/api/test_tools_router.py+++ b/tests/api/test_tools_router.py@@ -14,6 +14,31 @@ from deeptutor.tools.builtin import ()+def test_nullable_request_schema_does_not_corrupt_public_tool_metadata():+ from deeptutor.core.tool_protocol import ToolDefinition, ToolParameter++ definition = ToolDefinition(+ name="source_view", description="Synthetic source presentation",+ parameters=[ToolParameter(name="presentation", type="string", required=False,+ enum=["inspect", "teaching", "reveal"])],+ )+ request = definition.to_openai_schema()["function"]["parameters"]+ request["properties"]["presentation"]["enum"].append(None)+ _, _, metadata = tools_router._serialise_definition(definition)+ assert metadata[0].enum == ["inspect", "teaching", "reveal"]+ assert definition.to_openai_schema()["function"]["parameters"]["properties"]["presentation"]["enum"] == metadata[0].enum+++def test_rag_definition_can_be_serialised_for_tools_ui():+ from deeptutor.tools.builtin import RAGTool++ definition = RAGTool().get_definition()+ name, _, parameters = tools_router._serialise_definition(definition)+ assert name == "rag"+ presentation = next(parameter for parameter in parameters if parameter.name == "presentation")+ assert presentation.enum == ["inspect", "teaching", "reveal"]++@pytest.mark.asyncioasync def test_list_builtin_tools_marks_toggleable_set(monkeypatch: pytest.MonkeyPatch, tmp_pathdiff --git a/tests/capabilities/test_mastery_visual_hint_contract.py b/tests/capabilities/test_mastery_visual_hint_contract.pynew file mode 100644index 000000000..55144d5d3--- /dev/null+++ b/tests/capabilities/test_mastery_visual_hint_contract.py@@ -0,0 +1,91 @@+"""Hint descriptions retain assistance without failing after source inspection."""++from types import SimpleNamespace++import pytest++from deeptutor.capabilities.mastery.tools import MasteryQuizTool+from deeptutor.learning.models import LearningProgress+from deeptutor.learning.service import LearningService+from deeptutor.learning.storage import LearningStore+from deeptutor.learning.visual_practice import prepare_visual+++def visual_source(monkeypatch):+ calls = []+ def resolve(_name):+ calls.append("resolve")+ return SimpleNamespace(base_dir="synthetic", name="kb")+ def retrieve(*_args, **_kwargs):+ calls.append("retrieve")+ return SimpleNamespace(error="", images=[({+ "asset_id": "synthetic-figure", "source_path": "synthetic.pdf",+ "source_document_id": "a" * 64, "page_number": 1,+ "text": "The target is alpha.",+ }, b"synthetic-pixels")], sources=[{+ "visual_asset_url": "/synthetic-image", "url": "/synthetic-document",+ }])+ monkeypatch.setattr("deeptutor.multi_user.knowledge_access.resolve_for_rag", resolve)+ monkeypatch.setattr("deeptutor.services.rag.source_visuals.retrieve_visual", retrieve)+ return calls+++def request(hints):+ return {"sources": [{"kb_name": "kb", "asset_id": "synthetic-figure"}],+ "reference_quote": "The target is alpha.", "key_status": "verified",+ "answer_cues": "none", "pixels_inspected": True, "hints_used": hints}++[email protected]("hints,count", [+ (["A solution was previously visible"], 1), (["First hint", "Second hint"], 2),+ (3, 3), (0, 0), (None, 0),+])+def test_valid_hints_preserve_count_and_independent_credit(visual_source, tmp_path, hints, count):+ context, _ = prepare_visual(request(hints), expected_answer="alpha", options={})+ assert context["hints_used"] == count+ progress = LearningProgress(book_id="synthetic")+ LearningService(LearningStore(tmp_path))._apply_grade(progress, question_id="q", knowledge_point_id="kp",+ module_id="m", user_answer="alpha", expected_answer="alpha", question_type="short",+ visual_context=context)+ assert progress.quiz_attempts[0].independent is (count == 0)+ assert progress.learning_evidence[0].hints_used == count+ assert progress.learning_evidence[0].quality == (.6 if count else 1)++[email protected]("invalid", [True, False, 1.5, -1, "", "1", [], [""], [" "], [1], {}])+def test_invalid_hint_shapes_fail_before_source_io(visual_source, invalid):+ with pytest.raises(ValueError, match="hints_used"):+ prepare_visual(request(invalid), expected_answer="alpha", options={})+ assert visual_source == []+++def test_single_and_group_visual_schema_type_hint_count_or_descriptions():+ definition = MasteryQuizTool().get_definition()+ props = definition.to_openai_schema()["function"]["parameters"]["properties"]+ single = props["visual"]["properties"]["hints_used"]+ grouped = props["questions"]["items"]["properties"]["visual"]["properties"]["hints_used"]+ assert grouped == single+ assert {branch["type"] for branch in single["anyOf"]} == {"integer", "array", "null"}+ assert next(branch for branch in single["anyOf"] if branch["type"] == "integer")["minimum"] == 0+ descriptions = next(branch for branch in single["anyOf"] if branch["type"] == "array")+ assert descriptions["minItems"] == 1 and descriptions["items"]["type"] == "string"+ assert all(p.sensitive for p in definition.parameters if p.name in {"visual", "questions"})+++async def test_invalid_second_leaf_hints_reject_group_before_building_first_leaf(monkeypatch):+ from unittest.mock import AsyncMock++ from deeptutor.capabilities.mastery import tools++ monkeypatch.setattr(tools, "_new_service", lambda: object())+ monkeypatch.setattr(tools, "_load_path", lambda *_: LearningProgress(book_id="synthetic"))+ builder = AsyncMock(side_effect=AssertionError("No source-bearing leaf should be built"))+ monkeypatch.setattr(MasteryQuizTool, "_build_question", builder)+ leaf = {"knowledge_point_id": "kp", "question": "Identify the target", "expected_answer": "alpha"}+ result = await MasteryQuizTool().execute(_mastery_path_id="synthetic", questions=[+ {**leaf, "visual": request(1)}, {**leaf, "visual": request([""])},+ ])+ assert not result.success and "hints_used" in result.content+ builder.assert_not_called()diff --git a/tests/services/rag/test_mastery_source_pages.py b/tests/services/rag/test_mastery_source_pages.pyindex 764b734e7..ff23d5729 100644--- a/tests/services/rag/test_mastery_source_pages.py+++ b/tests/services/rag/test_mastery_source_pages.py@@ -691,6 +691,43 @@ def test_repair_checks_original_with_detail_even_for_second_source(material):assert result[2] is sources[0]+def test_selection_id_without_quote_preserves_original_evidence(material, monkeypatch):+ calls = _install_preparation(material, monkeypatch, rankings=[{+ "sufficient": True, "selected": [{"id": "c0", "reason": "Membrane structure"}]}])+ evidence = asyncio.run(lessons.prepare_lesson(material.store, "path", "kp", material.config))+ source = evidence["sources"][0]+ candidate = calls[1][0]["candidates"][0]+ assert source["text"] == candidate["text"]+ assert source["quote"] == candidate["text"][:300]+ assert calls[2][0]["sources"][0]["text"] == candidate["text"]+ assert calls[2][1] and len(calls) == 3+++def test_selection_unknown_id_without_quote_is_rejected(material, monkeypatch):+ _install_preparation(material, monkeypatch, rankings=[{+ "sufficient": True, "selected": [{"id": "not-retrieved", "reason": "Membrane structure"}]}])+ with pytest.raises(lessons.LessonUnavailable, match="source ID"):+ asyncio.run(lessons.prepare_lesson(material.store, "path", "kp", material.config))+ assert not material.store.load("path").lesson_evidence++[email protected]("original,quote", [+ ("prefix lipids\n\tform a\u00a0bilayer suffix", "lipids form a bilayer"),+ ("prefix cafe\u0301 membrane suffix", "caf\u00e9 membrane"),+ ("prefix caf\u00e9 membrane suffix", "cafe\u0301 membrane"),+])+def test_legacy_selection_quote_normalization_preserves_original(original, quote):+ excerpt = lessons._selection_excerpt(original, {"quote": quote})+ assert excerpt in original+ assert excerpt == original[len("prefix "):-len(" suffix")]++[email protected]("quote", ["membranes are made of lipids", "LIPIDS form a bilayer", "", "bilayer"])+def test_legacy_selection_quote_never_accepts_paraphrase_or_invalid_length(quote):+ with pytest.raises(lessons.LessonUnavailable, match="quotation"):+ lessons._selection_excerpt("Membrane lipids form a bilayer.", {"quote": quote})++def test_unknown_quote_is_not_ready_and_not_mastery(material, monkeypatch):_install_preparation(material, monkeypatch, bad_quote=True)with pytest.raises(lessons.LessonUnavailable, match="quotation"):