Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

Decode timed captions once across parser and storage

Bingxi Zhao (Frank)304022624ecd
diff --git a/deeptutor/video_learning/service.py b/deeptutor/video_learning/service.py
index 3451db195..9dea9be07 100644
--- a/deeptutor/video_learning/service.py
+++ b/deeptutor/video_learning/service.py
@@ -283,7 +283,9 @@ def parse_webvtt(text: str) -> list[dict[str, Any]]:
body_lines.append(line)
index += 1
- body = clean_transcript_text(re.sub(r"<[^>]+>", "", "\n".join(body_lines)))
+ # Entity decoding belongs to normalize_cues. Parsing and normalizing
+ # the same source must not turn nested &amp;lt; into a literal tag.
+ body = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", "", "\n".join(body_lines))).strip()
if body:
result.append(
{
@@ -328,10 +330,11 @@ class TimedMediaStore:
@staticmethod
def _repair_transcript_text(material: dict[str, Any]) -> bool:
"""Repair legacy machine-generated captions without touching user content."""
+ if material.get("_caption_text_version") == 1:
+ return False
transcript = material.get("transcript")
if not isinstance(transcript, dict):
return False
- changed = False
for key in ("cues", "segments"):
rows = material.get(key) if key == "segments" else transcript.get(key)
if not isinstance(rows, list):
@@ -342,8 +345,8 @@ class TimedMediaStore:
text = clean_transcript_text(row["text"])
if text != row["text"]:
row["text"] = text
- changed = True
- return changed
+ material["_caption_text_version"] = 1
+ return True
def get(self, material_id: str, *, lock_held: bool = False) -> dict[str, Any]:
payload = self._load(material_id)
@@ -672,6 +675,7 @@ async def resolve_material(
"segments": build_segments(cues),
"learning": learning,
"provider_cache": {"invidious_formats": formats} if formats else {},
+ "_caption_text_version": 1,
}
with store.lock(material_id):
# Network resolution happens outside the file lock. Re-read only the
@@ -773,7 +777,11 @@ async def refresh_invidious_transcript(material_id: str) -> dict[str, Any]:
def public_material(material: dict[str, Any], *, provider: str) -> dict[str, Any]:
- payload = {key: value for key, value in material.items() if key != "provider_cache"}
+ payload = {
+ key: value
+ for key, value in material.items()
+ if key not in {"provider_cache", "_caption_text_version"}
+ }
source = payload.get("source") if isinstance(payload.get("source"), dict) else {}
learning = payload.get("learning") if isinstance(payload.get("learning"), dict) else {}
start = float(learning.get("last_position") or source.get("entry_time_seconds") or 0)
diff --git a/tests/video_learning/test_service.py b/tests/video_learning/test_service.py
index 36bf94b66..51f216655 100644
--- a/tests/video_learning/test_service.py
+++ b/tests/video_learning/test_service.py
@@ -95,9 +95,19 @@ def test_transcript_normalization_decodes_entities_and_normalizes_whitespace() -
assert cues == [{"start": 0.0, "end": 1.0, "text": "Learn from A & B"}]
+def test_nested_caption_entities_decode_only_once() -> None:
+ parsed = service.parse_webvtt(
+ "WEBVTT\n\n00:00:00.000 --> 00:00:01.000\nUse &amp;lt;tag&amp;gt; literally\n"
+ )
+
+ assert service.normalize_cues(parsed)[0]["text"] == "Use &lt;tag&gt; literally"
+
+
def test_webvtt_preserves_caption_after_leading_blank_and_inline_tags() -> None:
- cues = service.parse_webvtt(
- "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\n \nOpening <c>idea</c>&nbsp;&amp;\nand continuation\n"
+ cues = service.normalize_cues(
+ service.parse_webvtt(
+ "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\n \nOpening <c>idea</c>&nbsp;&amp;\nand continuation\n"
+ )
)
assert cues == [
@@ -157,6 +167,26 @@ def test_store_repairs_legacy_caption_entities(isolated: Path) -> None:
assert store.get(material_id)["segments"][0]["text"] == "Learn & apply"
+def test_store_legacy_repair_does_not_decode_nested_entities_twice(isolated: Path) -> None:
+ store = service.TimedMediaStore()
+ material_id = service.material_id_for("dQw4w9WgXcQ")
+ store.save(
+ {
+ "version": 1,
+ "type": "timed_media",
+ "material_id": material_id,
+ "transcript": {
+ "status": "ready",
+ "cues": [{"start": 0, "end": 1, "text": "Use &amp;lt;tag&amp;gt;"}],
+ },
+ "segments": [{"locator": 1, "start": 0, "end": 1, "text": "Use &amp;lt;tag&amp;gt;"}],
+ }
+ )
+
+ assert store.get(material_id)["segments"][0]["text"] == "Use &lt;tag&gt;"
+ assert store.get(material_id)["segments"][0]["text"] == "Use &lt;tag&gt;"
+
+
def test_invidious_caption_choice_accepts_the_real_snake_case_schema() -> None:
captions = [
{"label": "English (auto-generated)", "language_code": "en", "auto_generated": True},