Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(mastery): budget distinct pixels in grouped assessments (#1901)

Bingxi Zhao (Frank)e63bb09e6c40
diff --git a/deeptutor/learning/semantic_assessment.py b/deeptutor/learning/semantic_assessment.py
index 56af77fba..04255086e 100644
--- a/deeptutor/learning/semantic_assessment.py
+++ b/deeptutor/learning/semantic_assessment.py
@@ -94,6 +94,7 @@ async def _request(config, instruction, payload, sources=None, native_images=Non
from deeptutor.learning.lesson_evidence import _json_response
from deeptutor.services.llm.config import get_llm_config
from deeptutor.services.llm.factory import complete_with_config
+ from deeptutor.services.llm.image_replay import deduplicate_user_images
try:
messages = [
@@ -113,6 +114,9 @@ async def _request(config, instruction, payload, sources=None, native_images=Non
messages.append(await asyncio.to_thread(image_message, list(unique.values())))
if native_images:
messages.append({"role": "user", "content": native_images})
+ # A group can assess several relationships on the same pages (#1901).
+ # Count actual unique pixels; retain each source label via references.
+ messages = deduplicate_user_images(messages)
image_parts = [
part
for message in messages
@@ -137,6 +141,8 @@ async def _request(config, instruction, payload, sources=None, native_images=Non
allow_image_fallback=False,
)
return _json_response(response)
+ except AssessmentUnavailable:
+ raise
except Exception as exc:
raise AssessmentUnavailable(
"Assessment is unavailable; retry. No grade was saved."
diff --git a/tests/capabilities/test_mastery_semantic_assessment.py b/tests/capabilities/test_mastery_semantic_assessment.py
index 603f4f16f..4b90f4143 100644
--- a/tests/capabilities/test_mastery_semantic_assessment.py
+++ b/tests/capabilities/test_mastery_semantic_assessment.py
@@ -464,7 +464,10 @@ async def test_source_assessment_sends_pixels_with_no_silent_fallback(monkeypatc
async def complete(**kwargs):
assert kwargs["allow_image_fallback"] is False
- assert kwargs["messages"][-1] == image
+ assert kwargs["messages"][-1]["role"] == "user"
+ assert [
+ part for part in kwargs["messages"][-1]["content"] if part["type"] == "image_url"
+ ] == image["content"]
return '{"verdicts":[{"question_id":"q","result":"correct","confidence":0.99,"feedback":"The visual relationship matches."}]}'
monkeypatch.setattr(factory, "complete_with_config", complete)
@@ -474,6 +477,77 @@ async def test_source_assessment_sends_pixels_with_no_silent_fallback(monkeypatc
assert result["q"].result == "correct"
[email protected]("shared", [True, False])
+async def test_group_visual_assessment_budgets_unique_pixels_and_retains_source_labels(
+ monkeypatch, shared
+):
+ from copy import deepcopy
+ import json
+
+ from deeptutor.learning import semantic_assessment as semantic
+ from deeptutor.services.llm import factory
+
+ items = [
+ {
+ "question_id": f"q{i}",
+ "answer": "The membrane separates the two compartments.",
+ "_visual_images": [
+ part
+ for page in range(2)
+ for part in (
+ {"type": "text", "text": f"Question q{i}, source page {page}"},
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": f"data:image/png;base64,page-{page if shared else i * 2 + page}"
+ },
+ },
+ )
+ ],
+ }
+ for i in range(4)
+ ]
+ before = deepcopy(items)
+ requests = []
+
+ async def complete(**kwargs):
+ requests.append(kwargs)
+ parts = [
+ part
+ for message in kwargs["messages"]
+ if isinstance(message["content"], list)
+ for part in message["content"]
+ ]
+ assert sum(part["type"] == "image_url" for part in parts) == 2
+ for i in range(4):
+ assert any(f"Question q{i}, source page" in part.get("text", "") for part in parts)
+ assert kwargs["allow_image_fallback"] is False
+ return json.dumps(
+ {
+ "verdicts": [
+ {
+ "question_id": f"q{i}",
+ "result": "correct",
+ "confidence": 0.99,
+ "feedback": "The relationship matches.",
+ }
+ for i in range(4)
+ ]
+ }
+ )
+
+ monkeypatch.setattr(factory, "complete_with_config", complete)
+ if shared:
+ verdicts = await semantic.judge_answers(object(), items)
+ assert len(verdicts) == 4 and len(requests) == 1
+ else:
+ with pytest.raises(semantic.AssessmentUnavailable, match="split the group"):
+ await semantic.judge_answers(object(), items)
+ assert not requests
+ assert items == before
+
+
@pytest.mark.asyncio
async def test_scoped_group_clarification_resume_and_exactly_once_credit(service, monkeypatch):
from deeptutor.learning import lesson_evidence