DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(mastery): finish bounded memory and count preparation usage (#1920, #1926)
Bingxi Zhao (Frank)4fe1b5b24a5f
diff --git a/deeptutor/capabilities/mastery/memory.py b/deeptutor/capabilities/mastery/memory.pyindex b61bdfda3..bf7c9bddd 100644--- a/deeptutor/capabilities/mastery/memory.py+++ b/deeptutor/capabilities/mastery/memory.py@@ -102,6 +102,12 @@ class MasteryContextBuilder(ContextBuilder):)async def build(self, *, session_id, leaf_message_id=None, **kwargs):+ # Retained memory and generation are different budgets: reasoning+ # providers count hidden tokens in max_tokens too (#1926).+ configured_limit = getattr(kwargs.get("llm_config"), "max_tokens", None)+ self._generation_limit = (+ configured_limit if type(configured_limit) is int and configured_limit > 0 else 8192+ )# The store enforces the active user/session and the selected ancestor# chain. No filesystem path, arbitrary session selector, or global search.rows = await self.store.get_messages_for_context(@@ -187,7 +193,13 @@ class MasteryContextBuilder(ContextBuilder):# A summary task needs learning evidence, not the full tutor persona,# 47 tool schemas, encrypted provider reasoning or current page pixels.summary, events = await super()._summarize(- source_text=material, summary_budget=summary_budget, replay_request=None, **kwargs+ source_text=material,+ summary_budget=summary_budget,+ generation_budget=min(+ summary_budget * 2, 8192, getattr(self, "_generation_limit", 8192)+ ),+ replay_request=None,+ **kwargs,)try:data = json.loads(summary)diff --git a/deeptutor/runtime/orchestrator.py b/deeptutor/runtime/orchestrator.pyindex 3253a79ec..e4d5bb30d 100644--- a/deeptutor/runtime/orchestrator.py+++ b/deeptutor/runtime/orchestrator.py@@ -135,8 +135,18 @@ class ChatOrchestrator:async def _run() -> None:from deeptutor.services.llm.metrics import TurnUsage, current_usage- turn_usage = TurnUsage(session_id=context.session_id, turn_id=_turn_id, source=cap_name)- usage_token = current_usage.set(turn_usage)+ turn_usage = current_usage.get()+ usage_token = None+ if (+ turn_usage is None+ or not _turn_id+ or turn_usage.turn_id != _turn_id+ or turn_usage.session_id != context.session_id+ ):+ turn_usage = TurnUsage(+ session_id=context.session_id, turn_id=_turn_id, source=cap_name+ )+ usage_token = current_usage.set(turn_usage)status = "completed"terminal_error_metadata: dict[str, Any] = {}try:@@ -197,7 +207,8 @@ class ChatOrchestrator:},))- current_usage.reset(usage_token)+ if usage_token is not None:+ current_usage.reset(usage_token)await bus.close()if _turn_id:unregister_bus(_turn_id)diff --git a/deeptutor/services/session/context_builder.py b/deeptutor/services/session/context_builder.pyindex ef7b417f7..e7c4139af 100644--- a/deeptutor/services/session/context_builder.py+++ b/deeptutor/services/session/context_builder.py@@ -417,6 +417,7 @@ class ContextBuilder:language: str,source_text: str,summary_budget: int,+ generation_budget: int | None = None,on_event: Callable[[StreamEvent], Awaitable[None]] | None = None,replay_request: dict[str, Any] | None = None,) -> tuple[str, list[StreamEvent]]:@@ -552,7 +553,7 @@ class ContextBuilder:async for _c in agent.stream_llm(user_prompt=user_prompt,system_prompt=system_prompt,- max_tokens=summary_budget,+ max_tokens=generation_budget or summary_budget,stage="summarize_context",trace_meta=trace_meta,outcome=outcome,diff --git a/deeptutor/services/session/turns/executor.py b/deeptutor/services/session/turns/executor.pyindex 639a0a34d..7578e08b1 100644--- a/deeptutor/services/session/turns/executor.py+++ b/deeptutor/services/session/turns/executor.py@@ -218,8 +218,14 @@ class TurnExecutor:# body so prompt building, the deferred tool view and ``read_skill``# all resolve against the same selection; ContextVars are task-local,# so a concurrent turn in another conversation is unaffected.+ # #1920/#1926: context condensation and pre-capability grading are+ # part of this turn too; the orchestrator inherits this collector.+ from deeptutor.services.llm.metrics import TurnUsage, current_usagefrom deeptutor.services.workspace.resources import turn_resource_selection+ usage_token = current_usage.set(+ TurnUsage(session_id=session_id, turn_id=turn_id, source=capability_name)+ )resource_scope = contextlib.ExitStack()resource_scope.enter_context(turn_resource_selection(@@ -1220,6 +1226,8 @@ class TurnExecutor:# open briefly so the later ``session_meta`` title update can# still arrive.try:+ current_usage.reset(usage_token)+ usage_token = Noneawait self._maybe_generate_session_title(execution=execution,session_id=session_id,@@ -1404,6 +1412,8 @@ class TurnExecutor:retryable=resolved_retryable,)finally:+ if usage_token is not None:+ current_usage.reset(usage_token)if execution.flush_task is not None:execution.flush_task.cancel()with contextlib.suppress(asyncio.CancelledError):diff --git a/deeptutor/services/session/turns/lifecycle.py b/deeptutor/services/session/turns/lifecycle.pyindex 419f5d518..0d81e8971 100644--- a/deeptutor/services/session/turns/lifecycle.py+++ b/deeptutor/services/session/turns/lifecycle.py@@ -657,8 +657,15 @@ class TurnLifecycle:execution: _TurnExecution,event: StreamEvent,) -> dict[str, Any]:- if event.type == StreamEventType.DONE and not event.metadata.get("status"):- event.metadata = {**event.metadata, "status": "completed"}+ if event.type == StreamEventType.DONE:+ from deeptutor.services.llm.metrics import current_usage++ collector = current_usage.get()+ if collector is not None and collector.turn_id == execution.turn_id:+ if summary := collector.summary():+ event.metadata = {**event.metadata, "usage_summary": summary}+ if not event.metadata.get("status"):+ event.metadata = {**event.metadata, "status": "completed"}event.session_id = execution.session_idevent.turn_id = execution.turn_idpayload = event.to_dict()diff --git a/tests/capabilities/test_mastery_bounded_memory.py b/tests/capabilities/test_mastery_bounded_memory.pyindex a201533c2..7a872fa43 100644--- a/tests/capabilities/test_mastery_bounded_memory.py+++ b/tests/capabilities/test_mastery_bounded_memory.py@@ -375,3 +375,31 @@ async def test_trailing_memory_snapshot_counts_toward_tutoring_budget():assert builder._model_tokens([], summary) == _count_model_context_tokens([]) + _count_model_context_tokens(summary_snapshot(summary))++[email protected]("configured_limit, saved", [(8192, True), (3000, False)])+async def test_reasoning_allowance_does_not_replace_the_retained_memory_budget(+ monkeypatch, configured_limit, saved+):+ """A 4k combined reasoning/output cap repeatedly cut off live JSON memory."""+ from deeptutor.capabilities.mastery.memory import archive_id+ from deeptutor.services.session.model_history import history_groups++ async def stream(_self, **kwargs):+ kwargs["outcome"].finish_reason = "stop" if kwargs["max_tokens"] >= 5000 else "max_tokens"+ assert kwargs["max_tokens"] <= configured_limit+ yield memory([archive_id(g) for g in history_groups(builder._summary_rows)])++ monkeypatch.setattr(_ContextSummaryAgent, "stream_llm", stream)+ store = store_for(records())+ builder = MasteryContextBuilder(store)+ result = await builder.build(+ session_id="s",+ llm_config=SimpleNamespace(+ model="reasoning-model", context_window=272000, max_tokens=configured_limit+ ),+ )+ assert store.update_summary.await_count == int(saved)+ assert result.token_count <= 20000+ assert "example 0 failed" in json.dumps(result.history_sources)diff --git a/tests/services/session/test_turn_runtime_subscribe.py b/tests/services/session/test_turn_runtime_subscribe.pyindex f64a90705..0537b4b7d 100644--- a/tests/services/session/test_turn_runtime_subscribe.py+++ b/tests/services/session/test_turn_runtime_subscribe.py@@ -1122,3 +1122,73 @@ async def test_no_path_push_when_the_turn_never_moved(tmp_path) -> None:)assert [event for event in execution.events if event["type"] == "session_meta"] == []++[email protected]("preparation_fails", [False, True])+async def test_context_preparation_usage_is_included_in_durable_turn_total(+ monkeypatch, tmp_path, preparation_fails+):+ """#1920/#1926: summary work is paid before the capability starts."""+ from unittest.mock import AsyncMock, MagicMock++ from deeptutor.core.capability_protocol import CapabilityManifest, TurnCapability+ from deeptutor.runtime.orchestrator import ChatOrchestrator+ from deeptutor.services.llm.metrics import CallMeasurement, current_usage++ store = SQLiteSessionStore(tmp_path / "usage.db")+ runtime = TurnRuntimeManager(store)++ def measure(tokens):+ meter = CallMeasurement(model="test", provider="test")+ meter.finish(+ SimpleNamespace(+ usage={"prompt_tokens": tokens, "completion_tokens": 1, "total_tokens": tokens + 1}+ )+ )++ class Builder:+ async def build(self, **kwargs):+ measure(100)+ if preparation_fails:+ raise RuntimeError("Preparation unavailable")+ return SimpleNamespace(+ conversation_history=[],+ conversation_summary="",+ context_text="",+ token_count=0,+ budget=0,+ )++ class Capability(TurnCapability):+ manifest = CapabilityManifest(name="chat", description="Usage regression")++ async def run(self, context, stream):+ measure(20)+ await stream.content("The answer", source="chat")++ orchestrator = ChatOrchestrator.__new__(ChatOrchestrator)+ orchestrator._cap_registry = MagicMock()+ orchestrator._cap_registry.get.return_value = Capability()+ orchestrator._tool_registry = MagicMock()+ bus = MagicMock()+ bus.publish = AsyncMock()+ monkeypatch.setattr("deeptutor.runtime.orchestrator.get_event_bus", lambda: bus)+ monkeypatch.setattr("deeptutor.runtime.orchestrator.ChatOrchestrator", lambda: orchestrator)+ monkeypatch.setattr(runtime, "_create_context_builder", lambda **kwargs: Builder())+ monkeypatch.setattr(runtime, "_maybe_generate_session_title", AsyncMock())+ session, turn = await runtime.start_turn(+ {"content": "Continue", "capability": "chat", "tools": []}+ )+ events = [e async for e in runtime.subscribe_turn(turn["id"])]+ done = next(e for e in events if e["type"] == "done")+ assert done["metadata"]["usage_summary"]["prompt_tokens"] == (100 if preparation_fails else 120)+ assert done["metadata"]["usage_summary"]["total_calls"] == (1 if preparation_fails else 2)+ messages = await store.get_messages(session["id"])+ if preparation_fails:+ assert done["metadata"]["status"] == "failed"+ assert current_usage.get() is None+ return+ stored = next(e for e in messages[-1]["events"] if e["type"] == "done")+ assert stored["metadata"]["usage_summary"] == done["metadata"]["usage_summary"]+ assert current_usage.get() is None