Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

feat(video_lesson): register the video lesson capability

SeleiXi6b7897a32fd6
diff --git a/deeptutor/agents/video_lesson/capability.py b/deeptutor/agents/video_lesson/capability.py
new file mode 100644
index 000000000..b57324e1d
--- /dev/null
+++ b/deeptutor/agents/video_lesson/capability.py
@@ -0,0 +1,218 @@
+"""Video lesson capability: a Remotion explainer the learner plays in the browser."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from typing import Any
+
+from deeptutor.agents._shared.capability_result import emit_capability_result
+from deeptutor.core.capability_protocol import CapabilityManifest, TurnCapability
+from deeptutor.core.context import UnifiedContext
+from deeptutor.core.trace import merge_trace_metadata
+from deeptutor.i18n import StatusI18n
+from deeptutor.runtime.agentic.usage import UsageTracker
+from deeptutor.runtime.request_contracts import get_capability_request_schema
+from deeptutor.runtime.stream_bus import StreamBus
+
+MAX_ATTEMPTS = 3
+
+
+class VideoLessonCapability(TurnCapability):
+ manifest = CapabilityManifest(
+ name="video_lesson",
+ description="Generate an animated video lesson (Remotion) that plays in the browser.",
+ stages=["code_generation", "render_check", "render_output"],
+ tools_used=[],
+ cli_aliases=["lesson"],
+ request_schema=get_capability_request_schema("video_lesson"),
+ config_defaults={},
+ )
+
+ async def run(self, context: UnifiedContext, stream: StreamBus) -> None:
+ from deeptutor.agents.video_lesson.lesson_writer import LessonWriterAgent
+ from deeptutor.agents.video_lesson.runtime import (
+ build_lesson,
+ ensure_runtime,
+ prepare_workdir,
+ write_page,
+ )
+ from deeptutor.services.llm.config import get_llm_config
+ from deeptutor.services.path_service import get_path_service
+
+ llm_config = get_llm_config()
+ usage = UsageTracker(model=getattr(llm_config, "model", None))
+ i18n = StatusI18n(self.name, context.language, module="video_lesson")
+ paths = get_path_service()
+ turn_id = str(context.metadata.get("turn_id", "") or context.session_id or "video-lesson")
+ base_dir = paths.get_agent_dir("video_lesson") / turn_id
+ history = str(context.metadata.get("conversation_context_text", "") or "")
+
+ writer = LessonWriterAgent(
+ api_key=llm_config.api_key,
+ base_url=llm_config.base_url,
+ api_version=llm_config.api_version,
+ language=context.language,
+ )
+ writer.set_trace_callback(self._build_trace_bridge(stream, i18n))
+ writer.start(user_input=context.user_message, history=history)
+
+ # npm install (first use only) overlaps the first, much longer, LLM call.
+ runtime_ready = asyncio.create_task(ensure_runtime())
+ timings: dict[str, float] = {"code_generation": 0.0, "render_check": 0.0}
+ attempts: list[dict[str, Any]] = []
+ draft = None
+ build = None
+ try:
+ for attempt in range(1, MAX_ATTEMPTS + 1):
+ started = time.perf_counter()
+ async with stream.stage("code_generation", source=self.name):
+ await stream.progress(
+ message=i18n.t("writing", "Writing the explainer animation."),
+ source=self.name,
+ stage="code_generation",
+ )
+ draft = await writer.write(attempt=attempt)
+ timings["code_generation"] += round(time.perf_counter() - started, 3)
+ if not draft.code:
+ attempts.append(
+ {"attempt": attempt, "stage": "parse", "errors": ["no ```tsx block"]}
+ )
+ writer.add_feedback(
+ previous=draft.raw,
+ stage="parse",
+ errors=["The reply had no ```tsx code block."],
+ )
+ continue
+
+ started = time.perf_counter()
+ async with stream.stage("render_check", source=self.name):
+ if not runtime_ready.done():
+ await stream.progress(
+ message=i18n.t("preparing_runtime", "Preparing the Remotion runtime."),
+ source=self.name,
+ stage="render_check",
+ )
+ await runtime_ready
+ await stream.progress(
+ message=i18n.t(
+ "checking", "Compiling the animation and checking its frames."
+ ),
+ source=self.name,
+ stage="render_check",
+ )
+ lesson = prepare_workdir(base_dir / "source", draft.code)
+ build = await build_lesson(lesson, base_dir / "source" / "player.js")
+ timings["render_check"] += round(time.perf_counter() - started, 3)
+ attempts.append({"attempt": attempt, "stage": build.stage, "errors": build.errors})
+ if build.ok:
+ break
+ await stream.progress(
+ message=i18n.t(
+ "retry",
+ f"Check {attempt} failed: {build.errors[0] if build.errors else build.stage}",
+ attempt=attempt,
+ error=build.errors[0] if build.errors else build.stage,
+ ),
+ source=self.name,
+ stage="render_check",
+ metadata={"trace_layer": "raw"},
+ )
+ writer.add_feedback(previous=draft.raw, stage=build.stage, errors=build.errors)
+ finally:
+ if not runtime_ready.done():
+ runtime_ready.cancel()
+
+ if draft is None or build is None or not build.ok:
+ last = attempts[-1]["errors"] if attempts else []
+ raise RuntimeError(
+ f"The video lesson did not pass its checks after {MAX_ATTEMPTS} attempts: "
+ + "; ".join(last[:3])
+ )
+
+ async with stream.stage("render_output", source=self.name):
+ title = draft.title or context.user_message.strip()[:60]
+ page = write_page(
+ base_dir / "source" / "player.js",
+ base_dir / "artifacts" / "lesson.html",
+ title=title,
+ language=context.language,
+ )
+ (base_dir / "source" / "player.js").unlink(missing_ok=True)
+ fps, frames = 30, max((f.get("frame", 0) for f in build.frames), default=0) + 1
+ seconds = round(frames / fps)
+ await stream.progress(
+ message=i18n.t("ready", f"Video lesson ready ({seconds} s).", seconds=seconds),
+ source=self.name,
+ stage="render_output",
+ )
+ if draft.summary:
+ await stream.content(draft.summary, source=self.name, stage="render_output")
+
+ rel = page.resolve().relative_to(paths.user_data_dir.resolve()).as_posix()
+ await emit_capability_result(
+ stream,
+ {
+ "response": draft.summary,
+ "title": title,
+ "artifacts": [
+ {
+ "type": "html",
+ "url": f"/files/outputs/{rel}",
+ "filename": page.name,
+ "content_type": "text/html",
+ "label": title,
+ }
+ ],
+ "code": {"language": "tsx", "content": draft.code},
+ "render": {"attempts": attempts, "duration_seconds": seconds},
+ "timings": timings,
+ },
+ source=self.name,
+ usage=usage,
+ )
+
+ def _build_trace_bridge(self, stream: StreamBus, i18n: StatusI18n):
+ async def _trace_bridge(update: dict[str, Any]) -> None:
+ if str(update.get("event", "") or "") != "llm_call":
+ return
+ stage = str(update.get("phase") or update.get("stage") or "code_generation")
+ base_metadata = {
+ key: value
+ for key, value in update.items()
+ if key
+ not in {"event", "state", "response", "chunk", "result", "tool_name", "tool_args"}
+ }
+ state = str(update.get("state", "running"))
+ label = str(base_metadata.get("label", "") or stage.replace("_", " ").title())
+ if state in {"running", "complete"}:
+ await stream.progress(
+ message=label,
+ source=self.name,
+ stage=stage,
+ metadata=merge_trace_metadata(
+ base_metadata, {"trace_kind": "call_status", "call_state": state}
+ ),
+ )
+ elif state == "streaming":
+ chunk = str(update.get("chunk", "") or "")
+ if chunk:
+ await stream.thinking(
+ chunk,
+ source=self.name,
+ stage=stage,
+ metadata=merge_trace_metadata(base_metadata, {"trace_kind": "llm_chunk"}),
+ )
+ elif state == "error":
+ await stream.error(
+ str(
+ update.get("response", "") or i18n.t("llm_call_failed", "LLM call failed.")
+ ),
+ source=self.name,
+ stage=stage,
+ metadata=merge_trace_metadata(
+ base_metadata, {"trace_kind": "call_status", "call_state": "error"}
+ ),
+ )
+
+ return _trace_bridge
diff --git a/deeptutor/agents/video_lesson/prompts/en/video_lesson.yaml b/deeptutor/agents/video_lesson/prompts/en/video_lesson.yaml
new file mode 100644
index 000000000..3f3fe8f72
--- /dev/null
+++ b/deeptutor/agents/video_lesson/prompts/en/video_lesson.yaml
@@ -0,0 +1,7 @@
+status:
+ preparing_runtime: "Preparing the Remotion runtime (first use installs its packages)."
+ writing: "Writing the explainer animation."
+ checking: "Compiling the animation and checking its frames."
+ retry: "Check {attempt} failed, fixing: {error}"
+ ready: "Video lesson ready ({seconds} s)."
+ llm_call_failed: "LLM call failed."
diff --git a/deeptutor/agents/video_lesson/prompts/zh/video_lesson.yaml b/deeptutor/agents/video_lesson/prompts/zh/video_lesson.yaml
new file mode 100644
index 000000000..e2bc1ba66
--- /dev/null
+++ b/deeptutor/agents/video_lesson/prompts/zh/video_lesson.yaml
@@ -0,0 +1,7 @@
+status:
+ preparing_runtime: "正在准备 Remotion 运行环境(首次使用需要安装依赖)。"
+ writing: "正在编写讲解动画。"
+ checking: "正在编译并逐帧检查动画。"
+ retry: "第 {attempt} 次检查未通过,正在修复:{error}"
+ ready: "讲解动画已生成({seconds} 秒)。"
+ llm_call_failed: "LLM 调用失败。"
diff --git a/deeptutor/app/facade.py b/deeptutor/app/facade.py
index ef69bc9e3..e628aefe0 100644
--- a/deeptutor/app/facade.py
+++ b/deeptutor/app/facade.py
@@ -7,6 +7,7 @@ from functools import wraps
import importlib.util
import inspect
import json
+import shutil
from typing import Any, AsyncIterator
from deeptutor.services.notebook import get_notebook_manager
@@ -126,6 +127,17 @@ class DeepTutorApp:
"or `pip install -r requirements/math-animator.txt`."
),
)
+ if resolved == "video_lesson":
+ available = shutil.which("node") is not None and shutil.which("npm") is not None
+ return CapabilityAvailability(
+ name=resolved,
+ available=available,
+ install_hint=(
+ ""
+ if available
+ else "Install Node.js 22+ (with npm); Remotion is installed on first use."
+ ),
+ )
return CapabilityAvailability(name=resolved, available=True)
async def start_turn(
diff --git a/deeptutor/i18n/metadata_i18n.py b/deeptutor/i18n/metadata_i18n.py
index dae452920..55316ef83 100644
--- a/deeptutor/i18n/metadata_i18n.py
+++ b/deeptutor/i18n/metadata_i18n.py
@@ -23,6 +23,10 @@ _CAPABILITY_DESCRIPTIONS: dict[str, dict[str, str]] = {
"en": "Generate math animations or storyboard images with Manim.",
"zh": "使用 Manim 生成数学动画或分镜图。",
},
+ "video_lesson": {
+ "en": "Generate an animated video lesson (Remotion) that plays in the browser.",
+ "zh": "生成可在浏览器中播放的讲解动画视频(Remotion)。",
+ },
"mastery_path": {
"en": "Structured mastery-based learning with spaced repetition.",
"zh": "结构化掌握式学习,结合间隔复习。",
diff --git a/deeptutor/runtime/bootstrap/builtin_capabilities.py b/deeptutor/runtime/bootstrap/builtin_capabilities.py
index 4a9444f0a..c23f09962 100644
--- a/deeptutor/runtime/bootstrap/builtin_capabilities.py
+++ b/deeptutor/runtime/bootstrap/builtin_capabilities.py
@@ -39,6 +39,7 @@ BUILTIN_CAPABILITY_CLASSES: dict[str, str] = {
"deep_question": "deeptutor.agents.question.capability:DeepQuestionCapability",
"deep_research": "deeptutor.agents.research.capability:DeepResearchCapability",
"math_animator": "deeptutor.agents.math_animator.capability:MathAnimatorCapability",
+ "video_lesson": "deeptutor.agents.video_lesson.capability:VideoLessonCapability",
"visualize": "deeptutor.agents.visualize.capability:VisualizeCapability",
"mastery_path": "deeptutor.capabilities.mastery.capability:MasteryPathCapability",
"immersive_reading": "deeptutor.capabilities.reading.mode:ImmersiveReadingCapability",
@@ -145,6 +146,16 @@ BUILTIN_CAPABILITY_SPECS: dict[str, BuiltinCapabilitySpec] = {
},
),
),
+ "video_lesson": BuiltinCapabilitySpec(
+ BUILTIN_CAPABILITY_CLASSES["video_lesson"],
+ _manifest(
+ "video_lesson",
+ "Generate an animated video lesson (Remotion) that plays in the browser.",
+ stages=["code_generation", "render_check", "render_output"],
+ tools_used=[],
+ cli_aliases=["lesson"],
+ ),
+ ),
"visualize": BuiltinCapabilitySpec(
BUILTIN_CAPABILITY_CLASSES["visualize"],
_manifest(
diff --git a/deeptutor/runtime/request_contracts.py b/deeptutor/runtime/request_contracts.py
index 9906b96e8..b3a8af2b0 100644
--- a/deeptutor/runtime/request_contracts.py
+++ b/deeptutor/runtime/request_contracts.py
@@ -44,6 +44,10 @@ class CourseStudyRequestConfig(EmptyConfig):
pass
+class VideoLessonRequestConfig(EmptyConfig):
+ pass
+
+
class DeepQuestionRequestConfig(BaseModel):
model_config = ConfigDict(extra="forbid")
@@ -159,6 +163,7 @@ CAPABILITY_CONFIG_MODELS: dict[str, type[BaseModel]] = {
"mastery_path": MasteryPathRequestConfig,
"immersive_reading": ImmersiveReadingRequestConfig,
"course_study": CourseStudyRequestConfig,
+ "video_lesson": VideoLessonRequestConfig,
}
@@ -216,6 +221,7 @@ __all__ = [
"DeepSolveRequestConfig",
"ImmersiveReadingRequestConfig",
"MasteryPathRequestConfig",
+ "VideoLessonRequestConfig",
"VisualizeRequestConfig",
"build_request_schema",
"get_capability_request_schema",
diff --git a/deeptutor/services/config/loader.py b/deeptutor/services/config/loader.py
index c2a14718b..ddd36bd2b 100644
--- a/deeptutor/services/config/loader.py
+++ b/deeptutor/services/config/loader.py
@@ -246,6 +246,7 @@ def get_agent_params(module_name: str) -> dict:
"brainstorm": ("tools", "brainstorm"),
"vision_solver": ("plugins", "vision_solver"),
"math_animator": ("plugins", "math_animator"),
+ "video_lesson": ("plugins", "video_lesson"),
"llm_probe": ("diagnostics", "llm_probe"),
}
path = get_runtime_settings_dir(PROJECT_ROOT) / "agents.yaml"
diff --git a/deeptutor/services/setup/init.py b/deeptutor/services/setup/init.py
index 661489094..e24fb5068 100644
--- a/deeptutor/services/setup/init.py
+++ b/deeptutor/services/setup/init.py
@@ -105,6 +105,8 @@ DEFAULT_AGENTS_SETTINGS: dict[str, Any] = {
"plugins": {
"vision_solver": {"temperature": 0.3, "max_tokens": 12000},
"math_animator": {"temperature": 0.4, "max_tokens": 12000},
+ # A whole Remotion lesson in one reply, plus a reasoning model's thinking.
+ "video_lesson": {"temperature": 0.5, "max_tokens": 48000},
},
# Settings' "test this model" probe. A reasoning model spends its budget
# thinking before it answers, so the 1024 that sufficed for a chat model
diff --git a/tests/core/test_capabilities_runtime.py b/tests/core/test_capabilities_runtime.py
index 3435bf4d6..b3a00d7cb 100644
--- a/tests/core/test_capabilities_runtime.py
+++ b/tests/core/test_capabilities_runtime.py
@@ -82,6 +82,7 @@ def test_builtin_capability_registry_covers_documented_capabilities() -> None:
# Course Study orchestrates across the surfaces above rather than
# teaching itself, so it registers here like any other mode.
"course_study",
+ "video_lesson",
}