DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
feat(video_lesson): voice lessons with edge-tts
SeleiXi26c3a3917424
diff --git a/deeptutor/agents/video_lesson/capability.py b/deeptutor/agents/video_lesson/capability.pyindex b57324e1d..e0e3100b8 100644--- a/deeptutor/agents/video_lesson/capability.py+++ b/deeptutor/agents/video_lesson/capability.py@@ -31,6 +31,7 @@ class VideoLessonCapability(TurnCapability):async def run(self, context: UnifiedContext, stream: StreamBus) -> None:from deeptutor.agents.video_lesson.lesson_writer import LessonWriterAgent+ from deeptutor.agents.video_lesson.narration import narrate, voice_modulefrom deeptutor.agents.video_lesson.runtime import (build_lesson,ensure_runtime,@@ -60,6 +61,8 @@ class VideoLessonCapability(TurnCapability):# npm install (first use only) overlaps the first, much longer, LLM call.runtime_ready = asyncio.create_task(ensure_runtime())timings: dict[str, float] = {"code_generation": 0.0, "render_check": 0.0}+ voice_cache: dict = {}+ narrated = Falseattempts: list[dict[str, Any]] = []draft = Nonebuild = None@@ -93,7 +96,16 @@ class VideoLessonCapability(TurnCapability):source=self.name,stage="render_check",)- await runtime_ready+ if draft.narration:+ await stream.progress(+ message=i18n.t("narrating", "Recording the narration."),+ source=self.name,+ stage="render_check",+ )+ clips, _ = await asyncio.gather(+ narrate(draft.narration, context.language, voice_cache), runtime_ready+ )+ narrated = any(clip.audio for clip in clips.values())await stream.progress(message=i18n.t("checking", "Compiling the animation and checking its frames."@@ -101,7 +113,9 @@ class VideoLessonCapability(TurnCapability):source=self.name,stage="render_check",)- lesson = prepare_workdir(base_dir / "source", draft.code)+ lesson = prepare_workdir(+ base_dir / "source", draft.code, voice=voice_module(clips)+ )build = await build_lesson(lesson, base_dir / "source" / "player.js")timings["render_check"] += round(time.perf_counter() - started, 3)attempts.append({"attempt": attempt, "stage": build.stage, "errors": build.errors})@@ -165,7 +179,11 @@ class VideoLessonCapability(TurnCapability):}],"code": {"language": "tsx", "content": draft.code},- "render": {"attempts": attempts, "duration_seconds": seconds},+ "render": {+ "attempts": attempts,+ "duration_seconds": seconds,+ "narrated": narrated,+ },"timings": timings,},source=self.name,diff --git a/deeptutor/agents/video_lesson/lesson_writer.py b/deeptutor/agents/video_lesson/lesson_writer.pyindex 8562f2e5e..79cae6250 100644--- a/deeptutor/agents/video_lesson/lesson_writer.py+++ b/deeptutor/agents/video_lesson/lesson_writer.py@@ -2,7 +2,7 @@from __future__ import annotations-from dataclasses import dataclass+from dataclasses import dataclass, fieldimport jsonimport refrom typing import Any@@ -10,6 +10,7 @@ from typing import Anyfrom deeptutor.agents.base_agent import BaseAgentfrom deeptutor.core.trace import build_trace_metadata, new_call_id+from .narration import NarrationLinefrom .runtime import ASSETS_DIR_JSON_BLOCK = re.compile(r"```json[ \t]*\n(.*?)\n```", re.S)@@ -22,6 +23,7 @@ class LessonDraft:summary: strcode: strraw: str+ narration: list[NarrationLine] = field(default_factory=list)def parse_draft(reply: str) -> LessonDraft:@@ -36,11 +38,19 @@ def parse_draft(reply: str) -> LessonDraft:meta = loaded if isinstance(loaded, dict) else {}except json.JSONDecodeError:meta = {}+ narration = [+ NarrationLine(str(item["id"]).strip(), str(item["text"]).strip())+ for item in meta.get("narration") or []+ if isinstance(item, dict)+ and str(item.get("id") or "").strip()+ and str(item.get("text") or "").strip()+ ]return LessonDraft(title=str(meta.get("title") or "").strip(),summary=str(meta.get("summary") or "").strip(),code=code,raw=reply,+ narration=narration,)diff --git a/deeptutor/agents/video_lesson/narration.py b/deeptutor/agents/video_lesson/narration.pynew file mode 100644index 000000000..6b5807879--- /dev/null+++ b/deeptutor/agents/video_lesson/narration.py@@ -0,0 +1,116 @@+"""Voice-over for video lessons: speak each scene's narration and time its captions.++Speech comes from edge-tts (the free read-aloud voices behind Microsoft Edge; no API+key). When the package is missing or the service fails, a scene keeps its captions,+timed from an estimated speaking rate, and plays silently.+"""++from __future__ import annotations++import asyncio+import base64+from dataclasses import dataclass+import importlib.util+import json+import logging+import math+import re++logger = logging.getLogger(__name__)++FPS = 30+VOICES = {"zh": "zh-CN-XiaoxiaoNeural", "en": "en-US-AvaNeural"}+SPEAK_TIMEOUT_S = 60+_CHARS_PER_SECOND = {"zh": 4.2, "en": 15.0}+_MP3_BYTES_PER_SECOND = 48_000 / 8 # edge-tts streams 24 kHz, 48 kbit/s mono mp3+_TICKS_PER_SECOND = 10_000_000 # edge-tts boundary offsets are in 100 ns units+_PARALLEL = 4+_SENTENCE_END = re.compile(r"(?<=[。!?;!?])|(?<=[.;])\s+")+++@dataclass+class NarrationLine:+ id: str+ text: str+++@dataclass+class Clip:+ text: str+ frames: int+ captions: list[tuple[int, str]]+ audio: bytes | None = None+++def _lang(language: str) -> str:+ return "zh" if language.startswith("zh") else "en"+++def estimate(text: str, language: str) -> Clip:+ """Silent clip: captions spaced by an average speaking rate."""+ rate = _CHARS_PER_SECOND[_lang(language)]+ at, captions = 0.0, []+ for sentence in (s.strip() for s in _SENTENCE_END.split(text)):+ if sentence:+ captions.append((round(at * FPS), sentence))+ at += len(sentence) / rate+ return Clip(text, max(1, math.ceil(at * FPS)), captions or [(0, text)])+++async def speak(text: str, voice: str) -> Clip:+ import edge_tts++ audio = bytearray()+ captions: list[tuple[int, str]] = []+ async for chunk in edge_tts.Communicate(text, voice).stream():+ if chunk["type"] == "audio":+ audio += chunk["data"]+ elif chunk["type"] == "SentenceBoundary":+ captions.append((round(chunk["offset"] / _TICKS_PER_SECOND * FPS), chunk["text"]))+ if not audio:+ raise RuntimeError("edge-tts returned no audio")+ frames = math.ceil(len(audio) / _MP3_BYTES_PER_SECOND * FPS)+ return Clip(text, frames, captions or [(0, text)], bytes(audio))+++async def narrate(+ lines: list[NarrationLine], language: str, cache: dict[str, Clip] | None = None+) -> dict[str, Clip]:+ """Record every line; ``cache`` keeps clips across retries of the same lesson."""+ cache = {} if cache is None else cache+ voiced = importlib.util.find_spec("edge_tts") is not None+ voice = VOICES[_lang(language)]+ gate = asyncio.Semaphore(_PARALLEL)++ async def one(text: str) -> Clip:+ if text not in cache:+ clip = None+ if voiced:+ async with gate:+ try:+ clip = await asyncio.wait_for(speak(text, voice), SPEAK_TIMEOUT_S)+ except Exception as exc: # the service is best effort; captions remain+ logger.warning("video lesson narration failed: %s", exc)+ cache[text] = clip or estimate(text, language)+ return cache[text]++ clips = await asyncio.gather(*(one(line.text) for line in lines))+ return {line.id: clip for line, clip in zip(lines, clips)}+++def voice_module(clips: dict[str, Clip]) -> str:+ """The ``voice.ts`` the kit imports: text, length, captions and inline audio per scene."""+ data = {}+ for key, clip in clips.items():+ entry: dict = {+ "text": clip.text,+ "frames": clip.frames,+ "captions": [{"at": at, "text": text} for at, text in clip.captions],+ }+ if clip.audio:+ entry["src"] = "data:audio/mpeg;base64," + base64.b64encode(clip.audio).decode()+ data[key] = entry+ return "export const VOICE = " + json.dumps(data, ensure_ascii=False) + ";\n"+++__all__ = ["Clip", "NarrationLine", "VOICES", "estimate", "narrate", "speak", "voice_module"]diff --git a/deeptutor/agents/video_lesson/prompts/en/lesson_writer.yaml b/deeptutor/agents/video_lesson/prompts/en/lesson_writer.yamlindex 67cba59e3..a734d1ced 100644--- a/deeptutor/agents/video_lesson/prompts/en/lesson_writer.yaml+++ b/deeptutor/agents/video_lesson/prompts/en/lesson_writer.yaml@@ -1,23 +1,35 @@system: |You are DeepTutor's video lesson author: for the learner's question, write a 60-120 second explainer animationwith Remotion (a React video framework). The goal is real understanding: lead with the answer or the key idea,- then build the reasoning or the story step by step; the captions say what the picture shows.+ then build the reasoning or the story step by step; the narration says what the picture shows.## Deliverable- Write one file, Lesson.tsx, exporting:- export const LESSON = {durationInFrames: number, fps: 30, width: 1920, height: 1080};- export const Lesson: React.FC = () => ...;- The host plays it live in the browser with @remotion/player (seek bar, pause, fullscreen). There is no voice-over,- so every spoken line appears as a <Narration> caption.+ Write one file, Lesson.tsx, that organises the film into 5-8 scenes with the kit's <Scenes>, exporting:+ const SCENES: Scene[] = [+ {voice: "s1", frames: 420, content: <Intro />},+ {voice: "s2", frames: 540, content: <Steps />},+ ...+ ];+ export const LESSON = {durationInFrames: lessonLength(SCENES), fps: 30, width: 1920, height: 1080};+ export const Lesson: React.FC = () => <Scenes scenes={SCENES} />;+ The host plays it live in the browser with @remotion/player (seek bar, pause, volume, fullscreen).++ ## Narration and voice-over+ - Each scene's narration goes in the json block's narration list, its id matching the scene's voice. The host+ voices it with TTS and adds captions that follow the voice; do not write <Narration> yourself.+ - Narration is what the teacher says: 2-5 sentences per scene, each under 25 words, ending in punctuation. Write+ numbers and formulas the way they are read aloud ("a sub n plus one", "one half"), never LaTeX or symbol strings.+ - English speech runs about 2.5 words per second; estimate each scene's frames from it. A scene stretches when its+ voice runs longer, so place the picture's delays within the first 70% of the estimate.## Remotion essentials- Every animation is driven by the frame number: useCurrentFrame() + interpolate(). No CSS transitions oranimations, setTimeout, useEffect timers, Math.random() or Date. Give interpolate{extrapolateLeft: "clamp", extrapolateRight: "clamp"}.- - Cut the film into 5-8 scenes with <Sequence from={start} durationInFrames={length}>; inside a Sequence,- useCurrentFrame() starts at 0. The kit's delay / at props are frames relative to the enclosing Sequence.+ - Inside each scene of <Scenes>, useCurrentFrame() starts at 0; the kit's delay props are frames relative to the+ scene. Nest a <Sequence> when a scene needs its own beats.- Coordinates are 1920x1080 pixels. Keep 120px side margins and leave the bottom 200px to captions.- - At 30fps a caption should stay at least 90 frames (3 s); a scene is usually 300-600 frames.+ - At 30fps a scene is usually 300-600 frames.## Art direction (lose the "AI-generated" look)- Reference points: good textbook illustration, documentary infographics, newspaper graphics desks, a teacher's@@ -29,17 +41,19 @@ system: |- Colours come only from the kit's theme T; the accent marks the current focus only, and one thing keeps one colour.- Left-aligned, asymmetric layouts with generous space; at most 3 levels of text; every scene draws something(a plot, a timeline, a map, a diagram) rather than being a slide of text.- - Captions sound like a teacher talking: plain, concrete, short sentences — "look here, because, so".+ - Narration sounds like a teacher talking: plain, concrete, short sentences — "look here, because, so".- One focus on screen at a time; motion is limited to fades, small moves, lines drawing in, counters and pen marks.## The ./kit component library- Build every scene from it: Stage / Title / Body / Em / Narration / Card / Mark / Arrow / Plot (+ Curve / Dots /+ Build every scene from it: Scenes / Stage / Title / Body / Em / Card / Mark / Arrow / Plot (+ Curve / Dots /Cobweb / HLine / VLine / PlotLabel) / Timeline / ChinaMap / Count / Formula / Frac / Reveal. For a small graphicthe kit lacks you may write SVG, but colours and fonts must come from T. Only react, remotion and ./kit may beimported; any other import fails to compile. Facts and numbers must be accurate; leave out numbers you are unsure of.## Output format- First a ```json block: {"title": "lesson title", "summary": "2-4 sentences: what the lesson covers and in what order"}+ First a ```json block:+ {"title": "lesson title", "summary": "2-4 sentences: what the lesson covers and in what order",+ "narration": [{"id": "s1", "text": "what the teacher says in scene one"}, {"id": "s2", "text": "..."}]}Then a ```tsx block with the complete Lesson.tsx. Nothing outside the two code blocks.## ./kit.tsx sourcediff --git a/deeptutor/agents/video_lesson/prompts/en/video_lesson.yaml b/deeptutor/agents/video_lesson/prompts/en/video_lesson.yamlindex 3f3fe8f72..e6691e63a 100644--- a/deeptutor/agents/video_lesson/prompts/en/video_lesson.yaml+++ b/deeptutor/agents/video_lesson/prompts/en/video_lesson.yaml@@ -1,6 +1,7 @@status:preparing_runtime: "Preparing the Remotion runtime (first use installs its packages)."writing: "Writing the explainer animation."+ narrating: "Recording the narration."checking: "Compiling the animation and checking its frames."retry: "Check {attempt} failed, fixing: {error}"ready: "Video lesson ready ({seconds} s)."diff --git a/deeptutor/agents/video_lesson/prompts/zh/lesson_writer.yaml b/deeptutor/agents/video_lesson/prompts/zh/lesson_writer.yamlindex 75a8a690a..f667ad151 100644--- a/deeptutor/agents/video_lesson/prompts/zh/lesson_writer.yaml+++ b/deeptutor/agents/video_lesson/prompts/zh/lesson_writer.yaml@@ -1,20 +1,32 @@system: |你是 DeepTutor 的「视频讲解」作者:针对学习者的问题,用 Remotion(React 视频框架)写一段 60~120 秒的讲解动画。- 目标是让学习者真正看懂:先给结论或要点,再一步步展开推理或脉络;画面讲什么,字幕就讲什么。+ 目标是让学习者真正看懂:先给结论或要点,再一步步展开推理或脉络;画面讲什么,旁白就讲什么。## 产物- 只写一个文件 Lesson.tsx,必须导出:- export const LESSON = {durationInFrames: 数字, fps: 30, width: 1920, height: 1080};- export const Lesson: React.FC = () => ...;- 宿主用 @remotion/player 在浏览器里实时播放(带进度条、暂停、全屏),没有配音,所以讲解词全部用 <Narration> 字幕呈现。+ 只写一个文件 Lesson.tsx,用 kit 的 <Scenes> 把全片组织成 5~8 幕,必须导出:+ const SCENES: Scene[] = [+ {voice: "s1", frames: 420, content: <Intro />},+ {voice: "s2", frames: 540, content: <Steps />},+ ...+ ];+ export const LESSON = {durationInFrames: lessonLength(SCENES), fps: 30, width: 1920, height: 1080};+ export const Lesson: React.FC = () => <Scenes scenes={SCENES} />;+ 宿主用 @remotion/player 在浏览器里实时播放(带进度条、暂停、音量、全屏)。++ ## 旁白与配音+ - 每一幕的旁白写在 json 的 narration 里,id 与该幕的 voice 对应。宿主会用 TTS 配音,并生成跟随配音的字幕;+ 不要再自己写 <Narration>。+ - 旁白就是老师要说的话:每幕 2~5 句、每句不超过 30 个字,句末用标点。数字、公式写成念得出来的样子+ (「a n 加一」「二分之一」),不要写 LaTeX 或符号串。+ - 中文语速约每秒 4 个字,用它估算每幕的 frames。配音更长时这一幕会自动延长,所以画面的 delay 安排在估算时长的前 70% 里。## Remotion 要点- 一切动画都由帧号驱动:useCurrentFrame() + interpolate();不要用 CSS transition/animation、setTimeout、useEffect 计时、Math.random()、Date。interpolate 记得 {extrapolateLeft: "clamp", extrapolateRight: "clamp"}。- - 用 <Sequence from={起始帧} durationInFrames={时长}> 把全片切成 5~8 幕;Sequence 里的 useCurrentFrame() 从 0 开始计。- kit 组件的 delay / at 参数都是相对所在 Sequence 的帧数。+ - <Scenes> 里每一幕的 useCurrentFrame() 从 0 开始计;kit 组件的 delay 参数都是相对所在幕的帧数。+ 幕内需要分段时可以再套 <Sequence>。- 坐标系是 1920×1080 像素。正文区左右各留 120px,底部 200px 留给字幕,不要把内容放进字幕区。- - 30fps:一句字幕至少停留 90 帧(3 秒),一幕通常 300~600 帧。+ - 30fps:一幕通常 300~600 帧。## 美术指导(去掉「AI 生成感」)- 参考对象:优秀教科书插图、纪录片里的信息图、报纸图表版、老师的板书。不要科技发布会、SaaS 官网、赛博朋克风格。@@ -22,17 +34,19 @@ system: |三张等宽卡片并排、「关键要点」式项目符号列表、大圆角与投影、打字机效果、「让我们一起探索…之旅」之类的口号。- 配色只用 kit 导出的主题 T;强调色 accent 只给当前焦点,同一事物全片同色。- 版式左对齐、不对称、留白大;文字层级不超过 3 级;每一幕都要有一个「画出来」的东西(坐标图、时间轴、地图、示意图)。- - 字幕像老师在说话:口语、具体、短句,说清「看这里、因为、所以」,不喊口号。+ - 旁白像老师在说话:口语、具体、短句,说清「看这里、因为、所以」,不喊口号。- 同一时刻画面只有一个焦点;动效只用淡入、小位移、线条描出、数字计数、圈注。## 组件库 ./kit- 所有画面都用它搭:Stage / Title / Body / Em / Narration / Card / Mark / Arrow / Plot(+ Curve / Dots / Cobweb / HLine /+ 所有画面都用它搭:Scenes / Stage / Title / Body / Em / Card / Mark / Arrow / Plot(+ Curve / Dots / Cobweb / HLine /VLine / PlotLabel)/ Timeline / ChinaMap / Count / Formula / Frac / Reveal。需要组件库没有的小图形时可以自己写 SVG,但颜色和字体只能取 T。import 只允许 react、remotion 和 ./kit,其它 import 会编译失败。事实与数字必须准确;没有把握的数字不要写。## 输出格式- 先输出一个 ```json 代码块:{"title": "讲解标题", "summary": "2~4 句:这段讲解讲了什么、按什么顺序讲"}+ 先输出一个 ```json 代码块:+ {"title": "讲解标题", "summary": "2~4 句:这段讲解讲了什么、按什么顺序讲",+ "narration": [{"id": "s1", "text": "第一幕的旁白"}, {"id": "s2", "text": "…"}]}再输出一个 ```tsx 代码块:完整的 Lesson.tsx。代码块之外不要写任何内容。## ./kit.tsx 源码diff --git a/deeptutor/agents/video_lesson/prompts/zh/video_lesson.yaml b/deeptutor/agents/video_lesson/prompts/zh/video_lesson.yamlindex e2bc1ba66..f23d4dd67 100644--- a/deeptutor/agents/video_lesson/prompts/zh/video_lesson.yaml+++ b/deeptutor/agents/video_lesson/prompts/zh/video_lesson.yaml@@ -1,6 +1,7 @@status:preparing_runtime: "正在准备 Remotion 运行环境(首次使用需要安装依赖)。"writing: "正在编写讲解动画。"+ narrating: "正在录制旁白配音。"checking: "正在编译并逐帧检查动画。"retry: "第 {attempt} 次检查未通过,正在修复:{error}"ready: "讲解动画已生成({seconds} 秒)。"diff --git a/pyproject.toml b/pyproject.tomlindex 60910ce6c..5e0255208 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -165,6 +165,10 @@ codebuddy = ["codebuddy-agent-sdk>=0.3,<0.4"]# adapter enables transcript-grounded learning when public captions exist.video-learning = ["youtube-transcript-api>=1.0.0,<2.0.0"]+# Voice-over for video lessons via Edge's free read-aloud voices. Without it+# lessons still play, with captions only.+video-lesson = ["edge-tts>=7.0,<8.0"]+# Compatibility extra for source installs and older docs. These packages are# already included in the public `deeptutor` wheel.server = [@@ -310,6 +314,7 @@ all = ["deeptutor[math-animator]","deeptutor[dev]","deeptutor[video-learning]",+ "deeptutor[video-lesson]",][tool.setuptools]diff --git a/requirements/video-lesson.txt b/requirements/video-lesson.txtnew file mode 100644index 000000000..973c40b82--- /dev/null+++ b/requirements/video-lesson.txt@@ -0,0 +1,2 @@+# Optional voice-over for video lessons (Edge read-aloud voices, no API key).+edge-tts>=7.0,<8.0diff --git a/tests/agents/video_lesson/test_lesson_writer.py b/tests/agents/video_lesson/test_lesson_writer.pyindex 898fb096f..b782e7410 100644--- a/tests/agents/video_lesson/test_lesson_writer.py+++ b/tests/agents/video_lesson/test_lesson_writer.py@@ -40,3 +40,12 @@ def test_writer_agent_starts_a_conversation_with_the_kit() -> None:system, user = agent._messagesassert "export const Stage" in system["content"]assert "Explain a cobweb plot" in user["content"]+++def test_parse_draft_reads_the_narration() -> None:+ reply = (+ '```json\n{"title": "t", "narration": [{"id": "s1", "text": "第一句。"}, {"id": "", "text": "x"}]}\n```\n'+ "```tsx\nexport const Lesson = () => null;\n```"+ )+ draft = parse_draft(reply)+ assert [(n.id, n.text) for n in draft.narration] == [("s1", "第一句。")]diff --git a/tests/agents/video_lesson/test_narration.py b/tests/agents/video_lesson/test_narration.pynew file mode 100644index 000000000..55ce2caf2--- /dev/null+++ b/tests/agents/video_lesson/test_narration.py@@ -0,0 +1,35 @@+"""Narration timing and the generated voice module, without calling the TTS service."""++from __future__ import annotations++import asyncio++from deeptutor.agents.video_lesson import narration+from deeptutor.agents.video_lesson.narration import Clip, NarrationLine, estimate, voice_module+++def test_estimate_spaces_captions_by_sentence() -> None:+ clip = estimate("先看结论。再看为什么!", "zh")+ assert [text for _, text in clip.captions] == ["先看结论。", "再看为什么!"]+ assert clip.captions[0][0] == 0 < clip.captions[1][0] < clip.frames+ assert clip.audio is None+++def test_narrate_falls_back_to_captions_when_speech_fails(monkeypatch) -> None:+ async def broken(text: str, voice: str) -> Clip:+ raise RuntimeError("offline")++ monkeypatch.setattr(narration, "speak", broken)+ cache: dict = {}+ clips = asyncio.run(narration.narrate([NarrationLine("s1", "一句话。")], "zh", cache))+ assert clips["s1"].audio is None and clips["s1"].captions+ assert "一句话。" in cache+++def test_voice_module_inlines_audio() -> None:+ module = voice_module(+ {"s1": Clip("a。", 30, [(0, "a。")], b"ID3"), "s2": estimate("b。", "zh")}+ )+ assert module.startswith("export const VOICE = ")+ assert '"src": "data:audio/mpeg;base64,SUQz"' in module+ assert module.count('"src"') == 1