Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

Preserve reading source fidelity and resilient activity tracking

Bingxi Zhao (Frank)ce590620d7c4
diff --git a/deeptutor/api/routers/reading.py b/deeptutor/api/routers/reading.py
index f19abfff3..c615b6ed3 100644
--- a/deeptutor/api/routers/reading.py
+++ b/deeptutor/api/routers/reading.py
@@ -1178,11 +1178,21 @@ async def get_revision_unit(material_id: str, revision: int, locator: int) -> Un
@router.get("/materials/{material_id}/raw")
async def get_raw(material_id: str) -> FileResponse:
"""The original bytes, for the faithful viewer. Serves Range requests."""
+ return _material_file_response(material_id, browser_ready=False)
+
+
[email protected]("/materials/{material_id}/render")
+async def get_render(material_id: str) -> FileResponse:
+ """Serve a browser-ready EPUB archive, or the original for other formats."""
+ return _material_file_response(material_id, browser_ready=True)
+
+
+def _material_file_response(material_id: str, *, browser_ready: bool) -> FileResponse:
store = _store()
try:
assert_learning_material(material_id)
manifest = store.manifest(material_id)
- path = store.raw_path(material_id)
+ path = store.render_path(material_id) if browser_ready else store.raw_path(material_id)
except Exception as exc:
raise _http_error(exc) from exc
if path is None or not path.is_file():
@@ -1262,12 +1272,15 @@ async def save_position(material_id: str, payload: PositionPayload) -> PositionI
percentage=payload.percentage,
),
)
- await asyncio.to_thread(
- _record_reading_position,
- material_id,
- locator=saved.locator,
- percentage=saved.percentage,
- )
+ try:
+ await asyncio.to_thread(
+ _record_reading_position,
+ material_id,
+ locator=saved.locator,
+ percentage=saved.percentage,
+ )
+ except Exception:
+ logger.exception("Reading position saved, but learning activity recording failed")
return PositionInfo(**saved.to_dict())
except Exception as exc:
raise _http_error(exc) from exc
diff --git a/deeptutor/api/routers/reading_extensions.py b/deeptutor/api/routers/reading_extensions.py
index 3c0b1c1d0..aded1ee62 100644
--- a/deeptutor/api/routers/reading_extensions.py
+++ b/deeptutor/api/routers/reading_extensions.py
@@ -237,14 +237,17 @@ async def run_extension_action(
worker.add_done_callback(_discard_late_worker_result)
registry.finish_action(extension_id)
- await asyncio.to_thread(
- _record_reading_activity,
- material_id,
- extension_id=extension_id,
- action=action,
- locator=payload.locator,
- result_type=result.type,
- )
+ try:
+ await asyncio.to_thread(
+ _record_reading_activity,
+ material_id,
+ extension_id=extension_id,
+ action=action,
+ locator=payload.locator,
+ result_type=result.type,
+ )
+ except Exception:
+ logger.exception("Reading action succeeded, but learning activity recording failed")
return dumped
diff --git a/deeptutor/reading/ingestion.py b/deeptutor/reading/ingestion.py
index af6cf4246..f6ba8ee2b 100644
--- a/deeptutor/reading/ingestion.py
+++ b/deeptutor/reading/ingestion.py
@@ -21,6 +21,8 @@ import tempfile
from typing import Any
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
+import httpx
+
from deeptutor.reading.catalog_models import (
IngestionStatus,
MaterialRecord,
@@ -797,11 +799,18 @@ def build_transcript_segments(cues: Sequence[TranscriptSegment]) -> list[Transcr
async def _load_youtube_captions(
url: str, languages: Sequence[str]
) -> tuple[str, str, list[TranscriptSegment]]:
- from deeptutor.video_learning.service import resolve_youtube_captions
+ from deeptutor.video_learning.service import TimedMediaError, resolve_youtube_captions
- resolution = await resolve_youtube_captions(url, list(languages))
- metadata = resolution.metadata
video_id = parse_youtube_url(url).video_id
+ try:
+ resolution = await resolve_youtube_captions(url, list(languages))
+ except (TimedMediaError, httpx.HTTPError) as exc:
+ # Caption metadata is optional for native playback. A configured
+ # provider outage must not turn an otherwise playable link into a
+ # failed reading material.
+ logger.warning("YouTube captions unavailable for %s: %s", video_id, exc)
+ return "YouTube video", f"https://i.ytimg.com/vi/{video_id}/hqdefault.jpg", []
+ metadata = resolution.metadata
segments = build_transcript_segments(normalize_transcript_segments(resolution.cues))
title = str(metadata.get("title") or "YouTube video")
cover = str(metadata.get("thumbnail_url") or f"https://i.ytimg.com/vi/{video_id}/hqdefault.jpg")
diff --git a/deeptutor/reading/store.py b/deeptutor/reading/store.py
index c83c1d8a0..40494f3ca 100644
--- a/deeptutor/reading/store.py
+++ b/deeptutor/reading/store.py
@@ -74,6 +74,7 @@ MAX_BOOKMARKS = 200
UNIT_REFS_NAME = "unit_refs.json"
UNITS_DIR = "units"
RAW_DIR = "raw"
+RENDER_DIR = "render"
ASSETS_DIR = "assets"
REVISIONS_DIR = "revisions"
@@ -260,6 +261,7 @@ class ReadingStore:
if not data:
raise ReadingError(f"{path.name} is empty")
+ source_data = data
is_epub = path.suffix.lower() == ".epub"
if is_epub:
from deeptutor.utils.document_extractor import (
@@ -268,11 +270,13 @@ class ReadingStore:
)
try:
- data = normalize_epub_archive(data, path.name)
+ data = normalize_epub_archive(source_data, path.name)
except DocumentExtractionError as exc:
raise ReadingError(f"{path.name}: failed to read EPUB ({exc})") from exc
- material_id = content_hash(data)
+ # Content identity and /raw describe the uploaded bytes. The repaired
+ # archive is a separate browser view so an older import keeps its ID.
+ material_id = content_hash(source_data)
display_name = (filename or path.name).strip() or path.name
with self._locked(material_id):
@@ -281,7 +285,12 @@ class ReadingStore:
wants_epub_upgrade = (
path.suffix.lower() == ".epub" and existing.render_mode != "epub"
)
- if not wants_epub_upgrade:
+ needs_render_archive = (
+ is_epub
+ and data != source_data
+ and not (self._dir(material_id) / RENDER_DIR).is_dir()
+ )
+ if not wants_epub_upgrade and not needs_render_archive:
return existing
if self.annotations(material_id):
raise ReadingUpgradeConflict(
@@ -303,7 +312,11 @@ class ReadingStore:
raw_dir = stage_dir / RAW_DIR
raw_dir.mkdir(parents=True, exist_ok=True)
raw_path = raw_dir / _safe_filename(display_name, fallback=path.name)
- raw_path.write_bytes(data)
+ raw_path.write_bytes(source_data)
+ if is_epub and data != source_data:
+ render_dir = stage_dir / RENDER_DIR
+ render_dir.mkdir(parents=True, exist_ok=True)
+ (render_dir / raw_path.name).write_bytes(data)
# A PDF page's first text line is not a table of contents. It is
# often a figure caption, running header, or reference entry, so
@@ -333,7 +346,7 @@ class ReadingStore:
title=extraction.title or Path(display_name).stem,
source_hash=material_id,
extractor=extraction.extractor,
- byte_size=len(data),
+ byte_size=len(source_data),
char_count=extraction.char_count,
created_at=time.time(),
# Compatibility: old clients route this boolean directly to
@@ -753,6 +766,14 @@ class ReadingStore:
return None
return self._find_raw(self._dir(material_id))
+ def render_path(self, material_id: str) -> Path | None:
+ """Browser-ready EPUB archive; original bytes for all other formats."""
+ manifest = self.manifest(material_id)
+ if manifest.render_mode == "text":
+ return None
+ material_dir = self._dir(material_id)
+ return self._find_file_in_dir(material_dir / RENDER_DIR) or self._find_raw(material_dir)
+
def _has_raw(self, material_id: str) -> bool:
"""Whether original bytes are already on disk, without loading them."""
try:
@@ -802,10 +823,13 @@ class ReadingStore:
@staticmethod
def _find_raw(material_dir: Path) -> Path | None:
- raw_dir = material_dir / RAW_DIR
- if not raw_dir.is_dir():
+ return ReadingStore._find_file_in_dir(material_dir / RAW_DIR)
+
+ @staticmethod
+ def _find_file_in_dir(directory: Path) -> Path | None:
+ if not directory.is_dir():
return None
- for candidate in sorted(raw_dir.iterdir()):
+ for candidate in sorted(directory.iterdir()):
if candidate.is_file():
return candidate
return None
@@ -912,20 +936,9 @@ class ReadingStore:
marked so the reader can show an explicit review state instead of
painting the wrong passage.
"""
- annotation_files = list((stage_dir / ANNOTATIONS_DIR).glob("*.json"))
+ annotation_files = sorted((stage_dir / ANNOTATIONS_DIR).glob("*.json"))
if not annotation_files:
return
- annotations_path = annotation_files[0]
- rows_data = _read_json(annotations_path)
- if not isinstance(rows_data, list) or not rows_data:
- return
- rows = [
- Annotation.from_dict(row)
- for row in rows_data
- if isinstance(row, dict) and row.get("annotation_id")
- ]
- if not rows:
- return
new_unit_texts: dict[int, str] = {}
units_dir = stage_dir / UNITS_DIR
@@ -935,58 +948,73 @@ class ReadingStore:
locator = int(unit_file.stem)
new_unit_texts[locator] = unit_file.read_text(encoding="utf-8")
- changed = False
- migrated: list[Annotation] = []
- for row in rows:
- quote_selectors = [s for s in row.selectors if isinstance(s, TextQuoteSelector)]
- if not quote_selectors:
- migrated.append(row)
+ for annotations_path in annotation_files:
+ rows_data = _read_json(annotations_path)
+ if not isinstance(rows_data, list) or not rows_data:
+ continue
+ rows = [
+ Annotation.from_dict(row)
+ for row in rows_data
+ if isinstance(row, dict) and row.get("annotation_id")
+ ]
+ if not rows:
continue
- quote_selector = quote_selectors[0]
- matches: list[tuple[int, tuple[int, int]]] = []
- for locator, text in sorted(new_unit_texts.items()):
- for span in _find_all_quote_spans(text, quote_selector):
- matches.append((locator, span))
- if len(matches) == 1:
- locator, (start, end) = matches[0]
- unit_text = new_unit_texts[locator]
- canonical_exact = unit_text[start:end]
- new_quote = dataclass_replace(quote_selector, exact=canonical_exact)
- new_selectors = []
- for s in row.selectors:
- if s is quote_selector:
- new_selectors.append(new_quote)
- elif isinstance(s, TextPositionSelector):
- new_selectors.append(TextPositionSelector(start=start, end=end))
- else:
- new_selectors.append(s)
- migrated.append(
- dataclass_replace(
- row,
- locator=locator,
- quote=canonical_exact,
- selectors=tuple(new_selectors),
- material_revision=new_revision,
- resolution="resolved",
+
+ changed = False
+ migrated: list[Annotation] = []
+ for row in rows:
+ quote_selectors = [s for s in row.selectors if isinstance(s, TextQuoteSelector)]
+ if not quote_selectors:
+ # A numeric position alone cannot be trusted after the text
+ # changes. Leave it pinned to the archived revision for review.
+ migrated.append(dataclass_replace(row, resolution="unresolved"))
+ changed = True
+ continue
+ quote_selector = quote_selectors[0]
+ matches: list[tuple[int, tuple[int, int]]] = []
+ for locator, text in sorted(new_unit_texts.items()):
+ for span in _find_all_quote_spans(text, quote_selector):
+ matches.append((locator, span))
+ if len(matches) == 1:
+ locator, (start, end) = matches[0]
+ unit_text = new_unit_texts[locator]
+ canonical_exact = unit_text[start:end]
+ new_quote = dataclass_replace(quote_selector, exact=canonical_exact)
+ new_selectors = []
+ for s in row.selectors:
+ if s is quote_selector:
+ new_selectors.append(new_quote)
+ elif isinstance(s, TextPositionSelector):
+ new_selectors.append(TextPositionSelector(start=start, end=end))
+ else:
+ new_selectors.append(s)
+ migrated.append(
+ dataclass_replace(
+ row,
+ locator=locator,
+ quote=canonical_exact,
+ selectors=tuple(new_selectors),
+ material_revision=new_revision,
+ resolution="resolved",
+ )
)
+ changed = True
+ elif len(matches) > 1:
+ migrated.append(dataclass_replace(row, resolution="ambiguous"))
+ changed = True
+ else:
+ migrated.append(dataclass_replace(row, resolution="unresolved"))
+ changed = True
+
+ if changed:
+ _atomic_write(
+ annotations_path,
+ json.dumps(
+ [row.to_dict() for row in migrated],
+ ensure_ascii=False,
+ indent=2,
+ ),
)
- changed = True
- elif len(matches) > 1:
- migrated.append(dataclass_replace(row, resolution="ambiguous"))
- changed = True
- else:
- migrated.append(dataclass_replace(row, resolution="unresolved"))
- changed = True
-
- if changed:
- _atomic_write(
- annotations_path,
- json.dumps(
- [row.to_dict() for row in migrated],
- ensure_ascii=False,
- indent=2,
- ),
- )
def _write_annotations(self, material_id: str, rows: Sequence[Annotation]) -> None:
_atomic_write(
diff --git a/tests/reading/test_engine.py b/tests/reading/test_engine.py
index 91cb72d80..271ff833f 100644
--- a/tests/reading/test_engine.py
+++ b/tests/reading/test_engine.py
@@ -284,10 +284,13 @@ def test_epub_store_normalizes_finder_packages_for_browser_readers(
manifest = store.ingest(path)
raw = store.raw_path(manifest.material_id)
- assert raw is not None
- normalized = raw.read_bytes()
- assert manifest.byte_size == len(normalized)
- assert manifest.source_hash == content_hash(normalized)
+ render = store.render_path(manifest.material_id)
+ assert raw is not None and render is not None
+ assert raw.read_bytes() == path.read_bytes()
+ normalized = render.read_bytes()
+ assert manifest.byte_size == len(path.read_bytes())
+ assert manifest.source_hash == content_hash(path.read_bytes())
+ assert store.ingest(path).material_id == manifest.material_id
assert [ref.source_href for ref in store.unit_references(manifest.material_id)] == [
"OEBPS/chapters/one.xhtml",
"OEBPS/chapters/two.xhtml",
diff --git a/tests/reading/test_extension_router.py b/tests/reading/test_extension_router.py
index aec32458e..3057aa7ec 100644
--- a/tests/reading/test_extension_router.py
+++ b/tests/reading/test_extension_router.py
@@ -89,6 +89,24 @@ def test_action_receives_only_server_verified_visible_text(material, monkeypatch
assert "visible_text" not in records.activities[0].model_dump()
+def test_action_succeeds_when_activity_store_fails(material, monkeypatch):
+ def fail_activity(*_args, **_kwargs):
+ raise OSError("activity database unavailable")
+
+ monkeypatch.setattr(reading_extensions, "_record_reading_activity", fail_activity)
+ client = _client(
+ monkeypatch,
+ _extension(lambda *_: ReadingExtensionResult(type="card", payload={"body": "ok"})),
+ )
+
+ response = client.post(
+ f"/api/reading/materials/{material.material_id}/extensions/sample/actions/open",
+ json={"locator": 1},
+ )
+ assert response.status_code == 200
+ assert response.json()["payload"]["body"] == "ok"
+
+
def test_source_anchor_is_loaded_from_server_position(material, monkeypatch):
ReadingStore().save_position(
material.material_id,
diff --git a/tests/reading/test_ingestion.py b/tests/reading/test_ingestion.py
index 6f674013d..86d55f373 100644
--- a/tests/reading/test_ingestion.py
+++ b/tests/reading/test_ingestion.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import json
import logging
from pathlib import Path
@@ -170,6 +171,59 @@ def test_revision_migration_reanchors_reliable_quote(stores) -> None:
assert unit_text[position_selector.start : position_selector.end] == quote_selector.exact
+def test_revision_migration_checks_each_material_annotation_file(stores) -> None:
+ reading, _catalog = stores
+ material_id = "abcdef0123456789"
+ reading.ingest_units(
+ material_id,
+ filename="article.md",
+ units=["The quick brown fox jumps. A second claim stays here."],
+ source_type="url_snapshot",
+ )
+ reading.save_annotation(
+ material_id,
+ Annotation(
+ annotation_id="first",
+ locator=1,
+ quote="quick brown fox",
+ selectors=(TextQuoteSelector(exact="quick brown fox"),),
+ ),
+ )
+ second_path = reading._dir(material_id) / "annotations" / "another-material.json"
+ second_path.write_text(
+ json.dumps(
+ [
+ Annotation(
+ annotation_id="second",
+ locator=1,
+ quote="second claim",
+ selectors=(TextQuoteSelector(exact="second claim"),),
+ ).to_dict(),
+ Annotation(
+ annotation_id="position-only",
+ locator=1,
+ selectors=(TextPositionSelector(start=5, end=10),),
+ ).to_dict(),
+ ]
+ ),
+ encoding="utf-8",
+ )
+
+ reading.ingest_units(
+ material_id,
+ filename="article-v2.md",
+ units=["An introduction. The quick brown fox jumps. A second claim stays here."],
+ source_type="url_snapshot",
+ )
+
+ assert reading.annotations(material_id)[0].material_revision == 2
+ second_rows = [Annotation.from_dict(row) for row in json.loads(second_path.read_text())]
+ assert second_rows[0].material_revision == 2
+ assert second_rows[0].resolution == "resolved"
+ assert second_rows[1].material_revision == 1
+ assert second_rows[1].resolution == "unresolved"
+
+
def test_revision_migration_marks_gone_quote_unresolved(stores) -> None:
reading, _catalog = stores
material_id = "abcdef0123456789"
@@ -457,10 +511,10 @@ async def test_youtube_ingestion_uses_configured_invidious_captions(
@pytest.mark.asyncio
-async def test_youtube_ingestion_reports_invidious_provider_failure(
+async def test_youtube_ingestion_keeps_playback_when_invidious_provider_fails(
stores, monkeypatch: pytest.MonkeyPatch
) -> None:
- _reading, catalog = stores
+ reading, catalog = stores
monkeypatch.setattr(
video_learning_service,
"load_video_learning_settings",
@@ -476,13 +530,14 @@ async def test_youtube_ingestion_reports_invidious_provider_failure(
raise video_learning_service.TimedMediaError("Invidious request failed with HTTP 503.")
monkeypatch.setattr(video_learning_service, "_invidious_metadata", metadata)
- service = ReadingIngestionService(_reading, catalog)
+ service = ReadingIngestionService(reading, catalog)
queued = service.queue_url("https://youtu.be/abc123xyz00")
- failed = await service.process_url(queued.material_id)
+ ready = await service.process_url(queued.material_id)
- assert failed.status is IngestionStatus.FAILED
- assert failed.error_code == "youtube_transcript_failed"
- assert "HTTP 503" in failed.error_detail
+ assert ready.status is IngestionStatus.READY
+ assert ready.cover_url.endswith("hqdefault.jpg")
+ assert reading.manifest(ready.material_id).extractor == "youtube-no-captions"
+ assert reading.unit_text(ready.material_id, 1) == TRANSCRIPT_UNAVAILABLE_TEXT
@pytest.mark.parametrize(
diff --git a/tests/reading/test_router.py b/tests/reading/test_router.py
index 2fd475c35..8a5f21419 100644
--- a/tests/reading/test_router.py
+++ b/tests/reading/test_router.py
@@ -388,17 +388,38 @@ def test_saved_reading_position_updates_account_learning_record(client: TestClie
assert "source_anchor" not in progress.model_dump()
-def test_epub_raw_response_is_normalized_for_browser_readers(client: TestClient) -> None:
+def test_position_save_succeeds_when_activity_store_fails(client, monkeypatch) -> None:
+ material = _upload(client)
+
+ def fail_activity(*_args, **_kwargs):
+ raise OSError("activity database unavailable")
+
+ monkeypatch.setattr(reading, "_record_reading_position", fail_activity)
+ response = client.put(
+ f"/api/reading/materials/{material['material_id']}/position",
+ json={"locator": 2, "percentage": 0.6},
+ )
+
+ assert response.status_code == 200
+ assert response.json()["locator"] == 2
+ assert ReadingStore().position(material["material_id"]).locator == 2
+
+
+def test_epub_render_response_is_normalized_and_raw_preserves_upload(client: TestClient) -> None:
+ uploaded = _epub_bytes(finder_package=True)
material = _upload(
client,
name="finder-book.epub",
- data=_epub_bytes(finder_package=True),
+ data=uploaded,
)
raw = client.get(f"/api/reading/materials/{material['material_id']}/raw")
+ render = client.get(f"/api/reading/materials/{material['material_id']}/render")
assert raw.status_code == 200
- with zipfile.ZipFile(io.BytesIO(raw.content)) as archive:
+ assert raw.content == uploaded
+ assert render.status_code == 200
+ with zipfile.ZipFile(io.BytesIO(render.content)) as archive:
infos = archive.infolist()
assert archive.read("mimetype") == b"application/epub+zip"
assert infos[0].filename == "mimetype"
diff --git a/web/components/reading/EpubDocumentView.tsx b/web/components/reading/EpubDocumentView.tsx
index 2b72eb730..ff6276993 100644
--- a/web/components/reading/EpubDocumentView.tsx
+++ b/web/components/reading/EpubDocumentView.tsx
@@ -7,7 +7,7 @@ import { useTranslation } from "react-i18next";
import { apiFetch } from "@/lib/api";
import {
getReadingPosition,
- rawMaterialUrl,
+ renderMaterialUrl,
saveReadingPosition,
type AnnotationItem,
type UnitReference,
@@ -315,7 +315,7 @@ export function EpubDocumentView({
setLoadError("");
const [module, response, position] = await Promise.all([
import("epubjs"),
- apiFetch(rawMaterialUrl(materialId), { cache: "no-store" }),
+ apiFetch(renderMaterialUrl(materialId), { cache: "no-store" }),
getReadingPosition(materialId).catch(() => null),
]);
if (!response.ok) throw new Error(t("Could not load this section."));
diff --git a/web/lib/reading-api.ts b/web/lib/reading-api.ts
index 48c97dbb7..0c8186a62 100644
--- a/web/lib/reading-api.ts
+++ b/web/lib/reading-api.ts
@@ -408,6 +408,10 @@ export function rawMaterialUrl(materialId: string): string {
return apiUrl(`${BASE}/materials/${materialId}/raw`);
}
+export function renderMaterialUrl(materialId: string): string {
+ return apiUrl(`${BASE}/materials/${materialId}/render`);
+}
+
export async function getReadingPosition(
materialId: string,
): Promise<ReadingPosition> {