Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(mastery): finish bounded memory and count preparation usage (#1920, #1926)

Bingxi Zhao (Frank)4fe1b5b24a5f
diff --git a/deeptutor/capabilities/mastery/memory.py b/deeptutor/capabilities/mastery/memory.py
index b61bdfda3..bf7c9bddd 100644
--- a/deeptutor/capabilities/mastery/memory.py
+++ b/deeptutor/capabilities/mastery/memory.py
@@ -102,6 +102,12 @@ class MasteryContextBuilder(ContextBuilder):
)
async def build(self, *, session_id, leaf_message_id=None, **kwargs):
+ # Retained memory and generation are different budgets: reasoning
+ # providers count hidden tokens in max_tokens too (#1926).
+ configured_limit = getattr(kwargs.get("llm_config"), "max_tokens", None)
+ self._generation_limit = (
+ configured_limit if type(configured_limit) is int and configured_limit > 0 else 8192
+ )
# The store enforces the active user/session and the selected ancestor
# chain. No filesystem path, arbitrary session selector, or global search.
rows = await self.store.get_messages_for_context(
@@ -187,7 +193,13 @@ class MasteryContextBuilder(ContextBuilder):
# A summary task needs learning evidence, not the full tutor persona,
# 47 tool schemas, encrypted provider reasoning or current page pixels.
summary, events = await super()._summarize(
- source_text=material, summary_budget=summary_budget, replay_request=None, **kwargs
+ source_text=material,
+ summary_budget=summary_budget,
+ generation_budget=min(
+ summary_budget * 2, 8192, getattr(self, "_generation_limit", 8192)
+ ),
+ replay_request=None,
+ **kwargs,
)
try:
data = json.loads(summary)
diff --git a/deeptutor/runtime/orchestrator.py b/deeptutor/runtime/orchestrator.py
index 3253a79ec..e4d5bb30d 100644
--- a/deeptutor/runtime/orchestrator.py
+++ b/deeptutor/runtime/orchestrator.py
@@ -135,8 +135,18 @@ class ChatOrchestrator:
async def _run() -> None:
from deeptutor.services.llm.metrics import TurnUsage, current_usage
- turn_usage = TurnUsage(session_id=context.session_id, turn_id=_turn_id, source=cap_name)
- usage_token = current_usage.set(turn_usage)
+ turn_usage = current_usage.get()
+ usage_token = None
+ if (
+ turn_usage is None
+ or not _turn_id
+ or turn_usage.turn_id != _turn_id
+ or turn_usage.session_id != context.session_id
+ ):
+ turn_usage = TurnUsage(
+ session_id=context.session_id, turn_id=_turn_id, source=cap_name
+ )
+ usage_token = current_usage.set(turn_usage)
status = "completed"
terminal_error_metadata: dict[str, Any] = {}
try:
@@ -197,7 +207,8 @@ class ChatOrchestrator:
},
)
)
- current_usage.reset(usage_token)
+ if usage_token is not None:
+ current_usage.reset(usage_token)
await bus.close()
if _turn_id:
unregister_bus(_turn_id)
diff --git a/deeptutor/services/session/context_builder.py b/deeptutor/services/session/context_builder.py
index ef7b417f7..e7c4139af 100644
--- a/deeptutor/services/session/context_builder.py
+++ b/deeptutor/services/session/context_builder.py
@@ -417,6 +417,7 @@ class ContextBuilder:
language: str,
source_text: str,
summary_budget: int,
+ generation_budget: int | None = None,
on_event: Callable[[StreamEvent], Awaitable[None]] | None = None,
replay_request: dict[str, Any] | None = None,
) -> tuple[str, list[StreamEvent]]:
@@ -552,7 +553,7 @@ class ContextBuilder:
async for _c in agent.stream_llm(
user_prompt=user_prompt,
system_prompt=system_prompt,
- max_tokens=summary_budget,
+ max_tokens=generation_budget or summary_budget,
stage="summarize_context",
trace_meta=trace_meta,
outcome=outcome,
diff --git a/deeptutor/services/session/turns/executor.py b/deeptutor/services/session/turns/executor.py
index 639a0a34d..7578e08b1 100644
--- a/deeptutor/services/session/turns/executor.py
+++ b/deeptutor/services/session/turns/executor.py
@@ -218,8 +218,14 @@ class TurnExecutor:
# body so prompt building, the deferred tool view and ``read_skill``
# all resolve against the same selection; ContextVars are task-local,
# so a concurrent turn in another conversation is unaffected.
+ # #1920/#1926: context condensation and pre-capability grading are
+ # part of this turn too; the orchestrator inherits this collector.
+ from deeptutor.services.llm.metrics import TurnUsage, current_usage
from deeptutor.services.workspace.resources import turn_resource_selection
+ usage_token = current_usage.set(
+ TurnUsage(session_id=session_id, turn_id=turn_id, source=capability_name)
+ )
resource_scope = contextlib.ExitStack()
resource_scope.enter_context(
turn_resource_selection(
@@ -1220,6 +1226,8 @@ class TurnExecutor:
# open briefly so the later ``session_meta`` title update can
# still arrive.
try:
+ current_usage.reset(usage_token)
+ usage_token = None
await self._maybe_generate_session_title(
execution=execution,
session_id=session_id,
@@ -1404,6 +1412,8 @@ class TurnExecutor:
retryable=resolved_retryable,
)
finally:
+ if usage_token is not None:
+ current_usage.reset(usage_token)
if execution.flush_task is not None:
execution.flush_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
diff --git a/deeptutor/services/session/turns/lifecycle.py b/deeptutor/services/session/turns/lifecycle.py
index 419f5d518..0d81e8971 100644
--- a/deeptutor/services/session/turns/lifecycle.py
+++ b/deeptutor/services/session/turns/lifecycle.py
@@ -657,8 +657,15 @@ class TurnLifecycle:
execution: _TurnExecution,
event: StreamEvent,
) -> dict[str, Any]:
- if event.type == StreamEventType.DONE and not event.metadata.get("status"):
- event.metadata = {**event.metadata, "status": "completed"}
+ if event.type == StreamEventType.DONE:
+ from deeptutor.services.llm.metrics import current_usage
+
+ collector = current_usage.get()
+ if collector is not None and collector.turn_id == execution.turn_id:
+ if summary := collector.summary():
+ event.metadata = {**event.metadata, "usage_summary": summary}
+ if not event.metadata.get("status"):
+ event.metadata = {**event.metadata, "status": "completed"}
event.session_id = execution.session_id
event.turn_id = execution.turn_id
payload = event.to_dict()
diff --git a/tests/capabilities/test_mastery_bounded_memory.py b/tests/capabilities/test_mastery_bounded_memory.py
index a201533c2..7a872fa43 100644
--- a/tests/capabilities/test_mastery_bounded_memory.py
+++ b/tests/capabilities/test_mastery_bounded_memory.py
@@ -375,3 +375,31 @@ async def test_trailing_memory_snapshot_counts_toward_tutoring_budget():
assert builder._model_tokens([], summary) == _count_model_context_tokens(
[]
) + _count_model_context_tokens(summary_snapshot(summary))
+
+
[email protected]("configured_limit, saved", [(8192, True), (3000, False)])
+async def test_reasoning_allowance_does_not_replace_the_retained_memory_budget(
+ monkeypatch, configured_limit, saved
+):
+ """A 4k combined reasoning/output cap repeatedly cut off live JSON memory."""
+ from deeptutor.capabilities.mastery.memory import archive_id
+ from deeptutor.services.session.model_history import history_groups
+
+ async def stream(_self, **kwargs):
+ kwargs["outcome"].finish_reason = "stop" if kwargs["max_tokens"] >= 5000 else "max_tokens"
+ assert kwargs["max_tokens"] <= configured_limit
+ yield memory([archive_id(g) for g in history_groups(builder._summary_rows)])
+
+ monkeypatch.setattr(_ContextSummaryAgent, "stream_llm", stream)
+ store = store_for(records())
+ builder = MasteryContextBuilder(store)
+ result = await builder.build(
+ session_id="s",
+ llm_config=SimpleNamespace(
+ model="reasoning-model", context_window=272000, max_tokens=configured_limit
+ ),
+ )
+ assert store.update_summary.await_count == int(saved)
+ assert result.token_count <= 20000
+ assert "example 0 failed" in json.dumps(result.history_sources)
diff --git a/tests/services/session/test_turn_runtime_subscribe.py b/tests/services/session/test_turn_runtime_subscribe.py
index f64a90705..0537b4b7d 100644
--- a/tests/services/session/test_turn_runtime_subscribe.py
+++ b/tests/services/session/test_turn_runtime_subscribe.py
@@ -1122,3 +1122,73 @@ async def test_no_path_push_when_the_turn_never_moved(tmp_path) -> None:
)
assert [event for event in execution.events if event["type"] == "session_meta"] == []
+
+
[email protected]("preparation_fails", [False, True])
+async def test_context_preparation_usage_is_included_in_durable_turn_total(
+ monkeypatch, tmp_path, preparation_fails
+):
+ """#1920/#1926: summary work is paid before the capability starts."""
+ from unittest.mock import AsyncMock, MagicMock
+
+ from deeptutor.core.capability_protocol import CapabilityManifest, TurnCapability
+ from deeptutor.runtime.orchestrator import ChatOrchestrator
+ from deeptutor.services.llm.metrics import CallMeasurement, current_usage
+
+ store = SQLiteSessionStore(tmp_path / "usage.db")
+ runtime = TurnRuntimeManager(store)
+
+ def measure(tokens):
+ meter = CallMeasurement(model="test", provider="test")
+ meter.finish(
+ SimpleNamespace(
+ usage={"prompt_tokens": tokens, "completion_tokens": 1, "total_tokens": tokens + 1}
+ )
+ )
+
+ class Builder:
+ async def build(self, **kwargs):
+ measure(100)
+ if preparation_fails:
+ raise RuntimeError("Preparation unavailable")
+ return SimpleNamespace(
+ conversation_history=[],
+ conversation_summary="",
+ context_text="",
+ token_count=0,
+ budget=0,
+ )
+
+ class Capability(TurnCapability):
+ manifest = CapabilityManifest(name="chat", description="Usage regression")
+
+ async def run(self, context, stream):
+ measure(20)
+ await stream.content("The answer", source="chat")
+
+ orchestrator = ChatOrchestrator.__new__(ChatOrchestrator)
+ orchestrator._cap_registry = MagicMock()
+ orchestrator._cap_registry.get.return_value = Capability()
+ orchestrator._tool_registry = MagicMock()
+ bus = MagicMock()
+ bus.publish = AsyncMock()
+ monkeypatch.setattr("deeptutor.runtime.orchestrator.get_event_bus", lambda: bus)
+ monkeypatch.setattr("deeptutor.runtime.orchestrator.ChatOrchestrator", lambda: orchestrator)
+ monkeypatch.setattr(runtime, "_create_context_builder", lambda **kwargs: Builder())
+ monkeypatch.setattr(runtime, "_maybe_generate_session_title", AsyncMock())
+ session, turn = await runtime.start_turn(
+ {"content": "Continue", "capability": "chat", "tools": []}
+ )
+ events = [e async for e in runtime.subscribe_turn(turn["id"])]
+ done = next(e for e in events if e["type"] == "done")
+ assert done["metadata"]["usage_summary"]["prompt_tokens"] == (100 if preparation_fails else 120)
+ assert done["metadata"]["usage_summary"]["total_calls"] == (1 if preparation_fails else 2)
+ messages = await store.get_messages(session["id"])
+ if preparation_fails:
+ assert done["metadata"]["status"] == "failed"
+ assert current_usage.get() is None
+ return
+ stored = next(e for e in messages[-1]["events"] if e["type"] == "done")
+ assert stored["metadata"]["usage_summary"] == done["metadata"]["usage_summary"]
+ assert current_usage.get() is None