DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
Decode timed captions once across parser and storage
Bingxi Zhao (Frank)304022624ecd
diff --git a/deeptutor/video_learning/service.py b/deeptutor/video_learning/service.pyindex 3451db195..9dea9be07 100644--- a/deeptutor/video_learning/service.py+++ b/deeptutor/video_learning/service.py@@ -283,7 +283,9 @@ def parse_webvtt(text: str) -> list[dict[str, Any]]:body_lines.append(line)index += 1- body = clean_transcript_text(re.sub(r"<[^>]+>", "", "\n".join(body_lines)))+ # Entity decoding belongs to normalize_cues. Parsing and normalizing+ # the same source must not turn nested &lt; into a literal tag.+ body = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", "", "\n".join(body_lines))).strip()if body:result.append({@@ -328,10 +330,11 @@ class TimedMediaStore:@staticmethoddef _repair_transcript_text(material: dict[str, Any]) -> bool:"""Repair legacy machine-generated captions without touching user content."""+ if material.get("_caption_text_version") == 1:+ return Falsetranscript = material.get("transcript")if not isinstance(transcript, dict):return False- changed = Falsefor key in ("cues", "segments"):rows = material.get(key) if key == "segments" else transcript.get(key)if not isinstance(rows, list):@@ -342,8 +345,8 @@ class TimedMediaStore:text = clean_transcript_text(row["text"])if text != row["text"]:row["text"] = text- changed = True- return changed+ material["_caption_text_version"] = 1+ return Truedef get(self, material_id: str, *, lock_held: bool = False) -> dict[str, Any]:payload = self._load(material_id)@@ -672,6 +675,7 @@ async def resolve_material("segments": build_segments(cues),"learning": learning,"provider_cache": {"invidious_formats": formats} if formats else {},+ "_caption_text_version": 1,}with store.lock(material_id):# Network resolution happens outside the file lock. Re-read only the@@ -773,7 +777,11 @@ async def refresh_invidious_transcript(material_id: str) -> dict[str, Any]:def public_material(material: dict[str, Any], *, provider: str) -> dict[str, Any]:- payload = {key: value for key, value in material.items() if key != "provider_cache"}+ payload = {+ key: value+ for key, value in material.items()+ if key not in {"provider_cache", "_caption_text_version"}+ }source = payload.get("source") if isinstance(payload.get("source"), dict) else {}learning = payload.get("learning") if isinstance(payload.get("learning"), dict) else {}start = float(learning.get("last_position") or source.get("entry_time_seconds") or 0)diff --git a/tests/video_learning/test_service.py b/tests/video_learning/test_service.pyindex 36bf94b66..51f216655 100644--- a/tests/video_learning/test_service.py+++ b/tests/video_learning/test_service.py@@ -95,9 +95,19 @@ def test_transcript_normalization_decodes_entities_and_normalizes_whitespace() -assert cues == [{"start": 0.0, "end": 1.0, "text": "Learn from A & B"}]+def test_nested_caption_entities_decode_only_once() -> None:+ parsed = service.parse_webvtt(+ "WEBVTT\n\n00:00:00.000 --> 00:00:01.000\nUse &lt;tag&gt; literally\n"+ )++ assert service.normalize_cues(parsed)[0]["text"] == "Use <tag> literally"++def test_webvtt_preserves_caption_after_leading_blank_and_inline_tags() -> None:- cues = service.parse_webvtt(- "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\n \nOpening <c>idea</c> &\nand continuation\n"+ cues = service.normalize_cues(+ service.parse_webvtt(+ "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\n \nOpening <c>idea</c> &\nand continuation\n"+ ))assert cues == [@@ -157,6 +167,26 @@ def test_store_repairs_legacy_caption_entities(isolated: Path) -> None:assert store.get(material_id)["segments"][0]["text"] == "Learn & apply"+def test_store_legacy_repair_does_not_decode_nested_entities_twice(isolated: Path) -> None:+ store = service.TimedMediaStore()+ material_id = service.material_id_for("dQw4w9WgXcQ")+ store.save(+ {+ "version": 1,+ "type": "timed_media",+ "material_id": material_id,+ "transcript": {+ "status": "ready",+ "cues": [{"start": 0, "end": 1, "text": "Use &lt;tag&gt;"}],+ },+ "segments": [{"locator": 1, "start": 0, "end": 1, "text": "Use &lt;tag&gt;"}],+ }+ )++ assert store.get(material_id)["segments"][0]["text"] == "Use <tag>"+ assert store.get(material_id)["segments"][0]["text"] == "Use <tag>"++def test_invidious_caption_choice_accepts_the_real_snake_case_schema() -> None:captions = [{"label": "English (auto-generated)", "language_code": "en", "auto_generated": True},