Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

test(reading): cover the bilingual EPUB pairing pipeline edge cases (#1738)

evan188199-techd81e1b2ea07f
diff --git a/tests/reading/test_epub_bilingual_units.py b/tests/reading/test_epub_bilingual_units.py
new file mode 100644
index 000000000..a823cee58
--- /dev/null
+++ b/tests/reading/test_epub_bilingual_units.py
@@ -0,0 +1,512 @@
+"""Direct unit tests for the bilingual EPUB pairing pipeline.
+
+Covers the failure-facing surface the pairing engine relies on: OPF
+entity decoding, whitespace normalisation, degraded reads of corrupt
+archives and ledgers, recommendation scoring and its skip paths, and
+pairing CRUD edge behaviour.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+import shutil
+import zipfile
+
+import pytest
+
+import deeptutor.reading.epub_bilingual as bilingual
+from deeptutor.reading.epub_bilingual import (
+ PAIRINGS_NAME,
+ create_epub_pairing,
+ delete_epub_pairing,
+ delete_epub_pairings_for_material,
+ list_epub_pairings,
+ recommend_epub_candidates,
+)
+from deeptutor.reading.models import ReadingError
+from deeptutor.reading.store import ReadingStore
+
+_CONTAINER = (
+ "<container xmlns='urn:oasis:names:tc:opendocument:xmlns:container'>"
+ "<rootfiles><rootfile full-path='{opf}'/></rootfiles></container>"
+)
+
+_OPF_TEMPLATE = (
+ "<package xmlns='http://www.idpf.org/2007/opf' "
+ "xmlns:dc='http://purl.org/dc/elements/1.1/' version='3.0'>"
+ "<metadata>{metadata}</metadata>"
+ "<manifest><item id='one' href='one.xhtml' "
+ "media-type='application/xhtml+xml'/></manifest>"
+ "<spine><itemref idref='one'/></spine></package>"
+)
+
+
+def _write_epub(
+ path: Path,
+ *,
+ metadata: str = "<dc:identifier>urn:uuid:unit-tests</dc:identifier>"
+ "<dc:title>Unit Test Book</dc:title><dc:language>en</dc:language>"
+ "<dc:creator>Fixture Author</dc:creator>",
+ chapter: str = "Chapter One",
+ body: str = "<p>Some story text.</p>",
+ container: str | None = None,
+ opf_name: str = "OPS/book.opf",
+ include_opf: bool = True,
+) -> Path:
+ with zipfile.ZipFile(path, "w") as archive:
+ archive.writestr("mimetype", "application/epub+zip", zipfile.ZIP_STORED)
+ archive.writestr(
+ "META-INF/container.xml",
+ container if container is not None else _CONTAINER.format(opf=opf_name),
+ )
+ if include_opf:
+ archive.writestr(opf_name, _OPF_TEMPLATE.format(metadata=metadata))
+ archive.writestr(
+ "OPS/one.xhtml",
+ "<html xmlns='http://www.w3.org/1999/xhtml'><head><title>"
+ f"{chapter}</title></head><body><h1>{chapter}</h1>{body}</body></html>",
+ )
+ return path
+
+
+def _drop_raw(store: ReadingStore, material_id: str) -> None:
+ raw_file = store.raw_path(material_id)
+ assert raw_file is not None
+ shutil.rmtree(raw_file.parent)
+
+
+# ---------------------------------------------------------------------------
+# OPF metadata: encoding, entities, normalisation
+# ---------------------------------------------------------------------------
+
+
+def test_metadata_decodes_entities_and_normalises_whitespace(tmp_path: Path) -> None:
+ epub = _write_epub(
+ tmp_path / "entities.epub",
+ metadata=(
+ "<dc:title>A &amp; B &#x4e2d;&#25991;</dc:title>"
+ "<dc:creator>\n\tFixture&#160;Author\n</dc:creator>"
+ ),
+ )
+
+ values = bilingual._metadata(epub)
+
+ assert values["title"] == "A & B 中文"
+ assert values["creator"] == "Fixture Author"
+
+
+def test_metadata_keeps_the_first_tag_and_records_empty_values(tmp_path: Path) -> None:
+ epub = _write_epub(
+ tmp_path / "dupes.epub",
+ metadata=(
+ "<dc:title>First title</dc:title><dc:title>Second title</dc:title>"
+ "<dc:language></dc:language>"
+ ),
+ )
+
+ values = bilingual._metadata(epub)
+
+ assert values["title"] == "First title"
+ assert values["language"] == ""
+ assert "creator" not in values
+ assert "identifier" not in values
+
+
+def test_metadata_on_an_empty_metadata_block_is_empty(tmp_path: Path) -> None:
+ epub = _write_epub(tmp_path / "bare.epub", metadata="")
+
+ assert bilingual._metadata(epub) == {}
+
+
+def test_local_name_strips_the_namespace_and_casefolds() -> None:
+ assert bilingual._local_name("{http://www.idpf.org/2007/opf}Title") == "title"
+ assert bilingual._local_name("TITLE") == "title"
+
+
+ ("value", "expected"),
+ [
+ ("en-US", "en"),
+ ("ZH-Hans-CN", "zh"),
+ (" fr ", "fr"),
+ ("", ""),
+ ("ja", "ja"),
+ ],
+)
+def test_language_normalises_tags(value: str, expected: str) -> None:
+ assert bilingual._language(value) == expected
+
+
+def test_tokens_casefold_and_keep_cjk_and_accents() -> None:
+ tokens = bilingual._tokens("Hello CAFÉ 中文")
+
+ assert tokens == {"hello", "café", "中文"}
+ assert bilingual._tokens("CAFÉ") & bilingual._tokens("café") == {"café"}
+
+
+# ---------------------------------------------------------------------------
+# Corrupt archives degrade to empty metadata
+# ---------------------------------------------------------------------------
+
+
+def test_metadata_returns_empty_for_a_non_archive(tmp_path: Path) -> None:
+ bad = tmp_path / "bad.epub"
+ bad.write_bytes(b"this is not a zip archive")
+
+ assert bilingual._metadata(bad) == {}
+
+
+def test_metadata_returns_empty_for_malformed_container(tmp_path: Path) -> None:
+ epub = _write_epub(tmp_path / "broken.epub", container="<container><unclosed")
+
+ assert bilingual._metadata(epub) == {}
+
+
+def test_metadata_returns_empty_without_a_rootfile(tmp_path: Path) -> None:
+ epub = _write_epub(
+ tmp_path / "noroot.epub",
+ container=(
+ "<container xmlns='urn:oasis:names:tc:opendocument:xmlns:container'>"
+ "<rootfiles/></container>"
+ ),
+ )
+
+ assert bilingual._metadata(epub) == {}
+
+
+ "full_path",
+ ["../evil.opf", "/abs/book.opf", "OPS/../book.opf"],
+)
+def test_metadata_rejects_opf_path_traversal(tmp_path: Path, full_path: str) -> None:
+ epub = _write_epub(
+ tmp_path / "traversal.epub",
+ container=(
+ "<container xmlns='urn:oasis:names:tc:opendocument:xmlns:container'>"
+ f"<rootfiles><rootfile full-path='{full_path}'/></rootfiles></container>"
+ ),
+ )
+
+ assert bilingual._metadata(epub) == {}
+
+
[email protected]("missing", ["META-INF/container.xml", "OPS/book.opf"])
+def test_metadata_returns_empty_for_missing_archive_entries(tmp_path: Path, missing: str) -> None:
+ path = tmp_path / "incomplete.epub"
+ with zipfile.ZipFile(path, "w") as archive:
+ archive.writestr("mimetype", "application/epub+zip", zipfile.ZIP_STORED)
+ if missing != "META-INF/container.xml":
+ archive.writestr("META-INF/container.xml", _CONTAINER.format(opf="OPS/book.opf"))
+ if missing != "OPS/book.opf":
+ archive.writestr("OPS/book.opf", _OPF_TEMPLATE.format(metadata=""))
+
+ assert bilingual._metadata(path) == {}
+
+
+# ---------------------------------------------------------------------------
+# Recommendation scoring: aligning two editions
+# ---------------------------------------------------------------------------
+
+
+def test_ranking_prefers_identifier_author_and_language_bonus(tmp_path: Path) -> None:
+ store = ReadingStore(root=tmp_path / "materials")
+ english = store.ingest(
+ _write_epub(
+ tmp_path / "english.epub",
+ metadata=(
+ "<dc:identifier>urn:uuid:twin</dc:identifier>"
+ "<dc:title>Twin Study</dc:title><dc:language>en</dc:language>"
+ "<dc:creator>A. Author</dc:creator>"
+ ),
+ chapter="Twin Study",
+ )
+ )
+ chinese = store.ingest(
+ _write_epub(
+ tmp_path / "chinese.epub",
+ metadata=(
+ "<dc:identifier>urn:uuid:twin</dc:identifier>"
+ "<dc:title>Twin Study</dc:title><dc:language>zh</dc:language>"
+ "<dc:creator>A. Author</dc:creator>"
+ ),
+ chapter="别的章节",
+ body="<p>别的故事。</p>",
+ )
+ )
+ english_twin = store.ingest(
+ _write_epub(
+ tmp_path / "english-twin.epub",
+ metadata=(
+ "<dc:identifier>urn:uuid:twin</dc:identifier>"
+ "<dc:title>Twin Study</dc:title><dc:language>en</dc:language>"
+ "<dc:creator>A. Author</dc:creator>"
+ ),
+ chapter="Other Chapter",
+ body="<p>Another telling.</p>",
+ )
+ )
+
+ candidates = recommend_epub_candidates(store, english.material_id)
+
+ assert [row["material_id"] for row in candidates] == [
+ chinese.material_id,
+ english_twin.material_id,
+ ]
+ assert candidates[0]["reasons"] == {
+ "title": 1.0,
+ "toc": 0.0,
+ "identifier": True,
+ "author": True,
+ "different_language": True,
+ }
+ assert candidates[1]["reasons"]["different_language"] is False
+ assert candidates[0]["score"] - candidates[1]["score"] == pytest.approx(0.1)
+
+
+def test_outline_title_overlap_scores_without_metadata(tmp_path: Path) -> None:
+ store = ReadingStore(root=tmp_path / "materials")
+ english = store.ingest(
+ _write_epub(
+ tmp_path / "english.epub",
+ metadata="<dc:language>en</dc:language>",
+ chapter="Shared Heading",
+ )
+ )
+ chinese = store.ingest(
+ _write_epub(
+ tmp_path / "chinese.epub",
+ metadata="<dc:language>zh</dc:language>",
+ chapter="Shared Heading",
+ body="<p>中文正文。</p>",
+ )
+ )
+
+ candidates = recommend_epub_candidates(store, english.material_id)
+
+ assert [row["material_id"] for row in candidates] == [chinese.material_id]
+ assert candidates[0]["reasons"]["toc"] == 1.0
+ assert candidates[0]["reasons"]["different_language"] is True
+ assert candidates[0]["score"] == pytest.approx(0.3)
+
+
+def test_a_bare_candidate_is_listed_with_a_zero_score(tmp_path: Path) -> None:
+ store = ReadingStore(root=tmp_path / "materials")
+ english = store.ingest(_write_epub(tmp_path / "english.epub", chapter="Chapter One"))
+ bare = store.ingest(
+ _write_epub(
+ tmp_path / "bare.epub",
+ metadata="",
+ chapter="完全不同",
+ body="<p>无元数据。</p>",
+ )
+ )
+
+ candidates = recommend_epub_candidates(store, english.material_id)
+
+ assert [row["material_id"] for row in candidates] == [bare.material_id]
+ row = candidates[0]
+ assert row["score"] == 0.0
+ assert row["language"] == ""
+ assert row["author"] == ""
+ assert row["reasons"] == {
+ "title": 0.0,
+ "toc": 0.0,
+ "identifier": False,
+ "author": False,
+ "different_language": False,
+ }
+
+
+# ---------------------------------------------------------------------------
+# Degraded recommendation paths
+# ---------------------------------------------------------------------------
+
+
+def test_candidates_skip_text_materials_and_unreadable_epubs(tmp_path: Path) -> None:
+ store = ReadingStore(root=tmp_path / "materials")
+ english = store.ingest(_write_epub(tmp_path / "english.epub", chapter="Chapter One"))
+ chinese = store.ingest(
+ _write_epub(
+ tmp_path / "chinese.epub",
+ metadata="<dc:identifier>urn:uuid:twin</dc:identifier><dc:language>zh</dc:language>",
+ chapter="别的章节",
+ )
+ )
+ broken = store.ingest(
+ _write_epub(
+ tmp_path / "broken.epub",
+ metadata="<dc:identifier>urn:uuid:twin</dc:identifier><dc:language>zh</dc:language>",
+ chapter="损坏章节",
+ body="<p>原文丢失。</p>",
+ )
+ )
+ notes = store.ingest(_write_text(tmp_path / "notes.md", "# Notes\n\nSome text."))
+ _drop_raw(store, broken.material_id)
+
+ candidates = recommend_epub_candidates(store, english.material_id)
+
+ assert notes.render_mode == "text"
+ assert [row["material_id"] for row in candidates] == [chinese.material_id]
+
+
+def test_recommendation_requires_an_epub_source(tmp_path: Path) -> None:
+ store = ReadingStore(root=tmp_path / "materials")
+ notes = store.ingest(_write_text(tmp_path / "notes.md", "# Notes\n\nSome text."))
+
+ with pytest.raises(ReadingError, match="only available for EPUB materials"):
+ recommend_epub_candidates(store, notes.material_id)
+
+
+def test_recommendation_fails_when_the_source_epub_is_unreadable(
+ tmp_path: Path,
+) -> None:
+ store = ReadingStore(root=tmp_path / "materials")
+ english = store.ingest(_write_epub(tmp_path / "english.epub", chapter="Chapter One"))
+ _drop_raw(store, english.material_id)
+
+ with pytest.raises(ReadingError, match="The source EPUB is unavailable."):
+ recommend_epub_candidates(store, english.material_id)
+
+
+# ---------------------------------------------------------------------------
+# Pairing CRUD edges
+# ---------------------------------------------------------------------------
+
+
+def test_pairing_rejects_pairing_a_material_with_itself(tmp_path: Path) -> None:
+ store = ReadingStore(root=tmp_path / "materials")
+ english = store.ingest(_write_epub(tmp_path / "english.epub", chapter="Chapter One"))
+
+ with pytest.raises(ReadingError, match="two different"):
+ create_epub_pairing(store, english.material_id, english.material_id)
+ assert list_epub_pairings(store) == []
+
+
+def test_pairing_requires_available_raw_files(tmp_path: Path) -> None:
+ english_store = ReadingStore(root=tmp_path / "english-side")
+ english = english_store.ingest(_write_epub(tmp_path / "english.epub", chapter="Chapter One"))
+ chinese = english_store.ingest(
+ _write_epub(
+ tmp_path / "chinese.epub",
+ metadata="<dc:language>zh</dc:language>",
+ chapter="第一章",
+ )
+ )
+ _drop_raw(english_store, english.material_id)
+
+ with pytest.raises(ReadingError, match="The English EPUB is unavailable."):
+ create_epub_pairing(english_store, english.material_id, chinese.material_id)
+
+ chinese_store = ReadingStore(root=tmp_path / "chinese-side")
+ english_again = chinese_store.ingest(
+ _write_epub(tmp_path / "english-2.epub", chapter="Chapter Two")
+ )
+ chinese_again = chinese_store.ingest(
+ _write_epub(
+ tmp_path / "chinese-2.epub",
+ metadata="<dc:language>zh</dc:language>",
+ chapter="第二章",
+ )
+ )
+ _drop_raw(chinese_store, chinese_again.material_id)
+
+ with pytest.raises(ReadingError, match="The Chinese EPUB is unavailable."):
+ create_epub_pairing(chinese_store, english_again.material_id, chinese_again.material_id)
+ assert list_epub_pairings(english_store) == []
+ assert list_epub_pairings(chinese_store) == []
+
+
+def test_creating_the_same_pair_twice_keeps_one_row(tmp_path: Path) -> None:
+ store = ReadingStore(root=tmp_path / "materials")
+ english = store.ingest(_write_epub(tmp_path / "english.epub", chapter="Chapter One"))
+ chinese = store.ingest(
+ _write_epub(
+ tmp_path / "chinese.epub",
+ metadata="<dc:language>zh</dc:language>",
+ chapter="第一章",
+ )
+ )
+
+ first = create_epub_pairing(store, english.material_id, chinese.material_id)
+ second = create_epub_pairing(store, english.material_id, chinese.material_id)
+
+ assert first["pairing_id"] == second["pairing_id"]
+ assert list_epub_pairings(store) == [second]
+
+
+def test_deleting_an_unknown_pairing_is_a_noop(tmp_path: Path) -> None:
+ store = ReadingStore(root=tmp_path / "materials")
+ pairing_path = store.root / PAIRINGS_NAME
+
+ assert delete_epub_pairing(store, "missing-id") is False
+ assert not pairing_path.exists()
+
+ english = store.ingest(_write_epub(tmp_path / "english.epub", chapter="Chapter One"))
+ chinese = store.ingest(
+ _write_epub(
+ tmp_path / "chinese.epub",
+ metadata="<dc:language>zh</dc:language>",
+ chapter="第一章",
+ )
+ )
+ pairing = create_epub_pairing(store, english.material_id, chinese.material_id)
+
+ assert delete_epub_pairing(store, "missing-id") is False
+ assert list_epub_pairings(store) == [pairing]
+
+
+def test_deleting_pairings_for_a_material_removes_both_roles(tmp_path: Path) -> None:
+ store = ReadingStore(root=tmp_path / "materials")
+ english = store.ingest(_write_epub(tmp_path / "english.epub", chapter="Chapter One"))
+ first_chinese = store.ingest(
+ _write_epub(
+ tmp_path / "first-chinese.epub",
+ metadata="<dc:language>zh</dc:language>",
+ chapter="第一章",
+ )
+ )
+ second_chinese = store.ingest(
+ _write_epub(
+ tmp_path / "second-chinese.epub",
+ metadata="<dc:language>zh</dc:language>",
+ chapter="第二章",
+ )
+ )
+ create_epub_pairing(store, english.material_id, first_chinese.material_id)
+ create_epub_pairing(store, english.material_id, second_chinese.material_id)
+
+ assert delete_epub_pairings_for_material(store, first_chinese.material_id) == 1
+ remaining = list_epub_pairings(store)
+ assert [row["chinese_material_id"] for row in remaining] == [second_chinese.material_id]
+
+ assert delete_epub_pairings_for_material(store, english.material_id) == 1
+ assert list_epub_pairings(store) == []
+
+
+def test_deleting_pairings_for_an_unknown_material_writes_nothing(
+ tmp_path: Path,
+) -> None:
+ store = ReadingStore(root=tmp_path / "materials")
+
+ assert delete_epub_pairings_for_material(store, "unknown-material") == 0
+ assert bilingual._pairing_path(store) == store.root / PAIRINGS_NAME
+ assert not (store.root / PAIRINGS_NAME).exists()
+ assert list_epub_pairings(store) == []
+
+
[email protected]("payload", [None, "not json", '{"pairing_id": "x"}'])
+def test_pairing_ledger_degrades_to_empty_on_unreadable_files(
+ tmp_path: Path, payload: str | None
+) -> None:
+ store = ReadingStore(root=tmp_path / "materials")
+ pairing_path = bilingual._pairing_path(store)
+ pairing_path.parent.mkdir(parents=True, exist_ok=True)
+ if payload is not None:
+ pairing_path.write_text(payload, encoding="utf-8")
+
+ assert list_epub_pairings(store) == []
+
+
+def _write_text(path: Path, content: str) -> Path:
+ path.write_text(content, encoding="utf-8")
+ return path