Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

feat(video_lesson): add narrated scenes to the lesson kit

SeleiXi40f50641a96f
diff --git a/deeptutor/agents/video_lesson/remotion/build.mjs b/deeptutor/agents/video_lesson/remotion/build.mjs
index 33db66d0f..1c2c13f0b 100644
--- a/deeptutor/agents/video_lesson/remotion/build.mjs
+++ b/deeptutor/agents/video_lesson/remotion/build.mjs
@@ -115,7 +115,7 @@ const still = query.has("frame") ? Math.max(0, Math.min(LESSON.durationInFrames
createRoot(document.getElementById("player")).render(
<Player component={Lesson} durationInFrames={LESSON.durationInFrames} fps={LESSON.fps}
compositionWidth={LESSON.width} compositionHeight={LESSON.height} initialFrame={still ?? 0}
- controls autoPlay={false} loop clickToPlay showVolumeControls={false} allowFullscreen
+ controls autoPlay={false} loop clickToPlay showVolumeControls allowFullscreen
style={{width: "100%", aspectRatio: LESSON.width + " / " + LESSON.height}} />);
`);
try {
diff --git a/deeptutor/agents/video_lesson/remotion/kit.tsx b/deeptutor/agents/video_lesson/remotion/kit.tsx
index 740b25aa4..6c2b315c9 100644
--- a/deeptutor/agents/video_lesson/remotion/kit.tsx
+++ b/deeptutor/agents/video_lesson/remotion/kit.tsx
@@ -2,8 +2,9 @@
// Lora + Geist type, and restrained motion live here, so the model composes content instead of
// inventing styling. Every component reads the frame from its enclosing <Sequence>.
import React, {createContext, useContext} from "react";
-import {AbsoluteFill, Easing, interpolate, useCurrentFrame} from "remotion";
+import {AbsoluteFill, Easing, Html5Audio, Sequence, Series, interpolate, useCurrentFrame} from "remotion";
import outline from "./china_outline.json";
+import {VOICE} from "./voice";
export const T = {
paper: "#fdfcf9", paper2: "#f5f2ea", ink: "#1c1816", soft: "#3b3430", muted: "#6d645a",
@@ -404,3 +405,37 @@ export const ChinaMap: React.FC<{x?: number; y?: number; h?: number; delay?: num
</svg>
);
};
+
+// ------------------------------------------------------------------ narrated scenes
+
+/** One scene: `voice` names its narration in the json block, `frames` is the planned length. */
+export type Scene = {voice: string; frames: number; content: React.ReactNode};
+type Clip = {text: string; frames: number; src?: string; captions: {at: number; text: string}[]};
+const LEAD = 10; // frames of picture before the voice starts
+
+const clip = (id: string): Clip => {
+ const c = (VOICE as Record<string, Clip>)[id];
+ if (!c) throw new Error(`scene voice "${id}" has no narration entry in the json block`);
+ return c;
+};
+// A scene lasts as long as planned, or longer when its narration needs the time.
+const sceneLength = (s: Scene) => Math.max(s.frames, LEAD + clip(s.voice).frames + 20);
+
+/** Total length of a lesson built from `scenes`; use it for LESSON.durationInFrames. */
+export const lessonLength = (scenes: Scene[]) => scenes.reduce((n, s) => n + sceneLength(s), 0);
+
+/** Plays the scenes back to back, each with its voice-over and captions timed to the voice. */
+export const Scenes: React.FC<{scenes: Scene[]}> = ({scenes}) => (
+ <Series>
+ {scenes.map((s, i) => {
+ const c = clip(s.voice);
+ return (
+ <Series.Sequence key={i} durationInFrames={sceneLength(s)}>
+ {s.content}
+ {c.src ? <Sequence from={LEAD} layout="none"><Html5Audio src={c.src} /></Sequence> : null}
+ <Narration lines={c.captions.map((l) => ({at: LEAD + l.at, text: l.text}))} />
+ </Series.Sequence>
+ );
+ })}
+ </Series>
+);
diff --git a/deeptutor/agents/video_lesson/remotion/voice.ts b/deeptutor/agents/video_lesson/remotion/voice.ts
new file mode 100644
index 000000000..d10aef718
--- /dev/null
+++ b/deeptutor/agents/video_lesson/remotion/voice.ts
@@ -0,0 +1,3 @@
+// Narration for one lesson. The host replaces this file with the recorded voice-over
+// (see video_lesson/narration.py); this empty default keeps the kit importable on its own.
+export const VOICE: Record<string, unknown> = {};
diff --git a/deeptutor/agents/video_lesson/runtime.py b/deeptutor/agents/video_lesson/runtime.py
index ccc1136aa..b3b6e844e 100644
--- a/deeptutor/agents/video_lesson/runtime.py
+++ b/deeptutor/agents/video_lesson/runtime.py
@@ -20,7 +20,7 @@ import shutil
from deeptutor.services.path_service import get_path_service
ASSETS_DIR = Path(__file__).resolve().parent / "remotion"
-KIT_FILES = ("kit.tsx", "china_outline.json")
+KIT_FILES = ("kit.tsx", "china_outline.json", "voice.ts")
BUILD_TIMEOUT_S = 600
INSTALL_TIMEOUT_S = 900
@@ -104,11 +104,16 @@ async def ensure_runtime() -> Path:
return root
-def prepare_workdir(work_dir: Path, code: str) -> Path:
- """Write the lesson next to a fresh copy of the kit and return the lesson path."""
+def prepare_workdir(work_dir: Path, code: str, voice: str | None = None) -> Path:
+ """Write the lesson next to a fresh copy of the kit and return the lesson path.
+
+ ``voice`` is the generated ``voice.ts`` module; without it the kit's empty default is used.
+ """
work_dir.mkdir(parents=True, exist_ok=True)
for name in KIT_FILES:
shutil.copy(ASSETS_DIR / name, work_dir / name)
+ if voice is not None:
+ (work_dir / "voice.ts").write_text(voice, encoding="utf-8")
lesson = work_dir / "Lesson.tsx"
lesson.write_text(code, encoding="utf-8")
return lesson
diff --git a/pyproject.toml b/pyproject.toml
index 12f2ee6e5..60910ce6c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -337,6 +337,7 @@ deeptutor = [
"**/*.j2",
"**/*.jinja",
# video_lesson's Remotion kit and build script, compiled at run time
+ "**/*.ts",
"**/*.tsx",
"**/*.mjs",
]
diff --git a/tests/agents/video_lesson/test_runtime.py b/tests/agents/video_lesson/test_runtime.py
index 4498f95a0..0403592e2 100644
--- a/tests/agents/video_lesson/test_runtime.py
+++ b/tests/agents/video_lesson/test_runtime.py
@@ -32,3 +32,8 @@ def test_write_page_inlines_the_bundle_safely(tmp_path: Path) -> None:
assert "<\/script><b>x</b>" in html
assert "<title>a &lt; b</title>" in html
assert 'lang="zh-CN"' in html
+
+
+def test_prepare_workdir_writes_the_voice_module(tmp_path: Path) -> None:
+ prepare_workdir(tmp_path / "source", "", voice="export const VOICE = {s1: {}};")
+ assert "s1" in (tmp_path / "source" / "voice.ts").read_text(encoding="utf-8")