Skip to content

知らない分野を、原文にたどりつくまで芋づる式に掘る Claude Code のスキル

Python
33.4 KB
Raw Blame
"""再取得が失敗したときの原文保全と、成功時の保存先を検査する。"""

import contextlib
import importlib.util
import io
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch


SCRIPT = Path(__file__).resolve().parents[1] / "skills/imozuru/scripts/fetch.py"
SPEC = importlib.util.spec_from_file_location("imozuru_fetch", SCRIPT)
fetch = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(fetch)
# テストの宛先(example.test)は名前を引けない。インターネット上の宛先に引けたことにする。
# 宛先の検査そのものは UnsafeUrlTests が本物に近い引き先を渡して確かめる
PUBLIC_IP = "93.184.215.14"
fetch.resolve = lambda host: {PUBLIC_IP}


class SourceNameTests(unittest.TestCase):
    def test_rejects_all_bidi_controls_in_filename_pattern(self):
        controls = [0x200E, 0x200F, *range(0x202A, 0x202F), *range(0x2066, 0x206A)]
        for codepoint in controls:
            with self.subTest(codepoint=f"U+{codepoint:04X}"):
                self.assertFalse(fetch.valid_source_name(f"paper{chr(codepoint)}.md"))

    def test_accepts_characters_adjacent_to_bidi_ranges(self):
        for codepoint in (0x200D, 0x2010, 0x2029, 0x202F, 0x2065, 0x206A):
            with self.subTest(codepoint=f"U+{codepoint:04X}"):
                self.assertTrue(fetch.valid_source_name(f"paper{chr(codepoint)}.md"))

    def test_fetch_source_contains_no_literal_bidi_controls(self):
        source = SCRIPT.read_text(encoding="utf-8")
        controls = [0x200E, 0x200F, *range(0x202A, 0x202F), *range(0x2066, 0x206A)]
        for codepoint in controls:
            with self.subTest(codepoint=f"U+{codepoint:04X}"):
                self.assertNotIn(chr(codepoint), source)


class FetchPreservationTests(unittest.TestCase):
    def setUp(self):
        tmp = tempfile.TemporaryDirectory()
        self.addCleanup(tmp.cleanup)
        self.sources = Path(tmp.name) / "sources"
        self.sources.mkdir()
        self.stdout = io.StringIO()
        self.stack = contextlib.ExitStack()
        self.addCleanup(self.stack.close)
        self.stack.enter_context(contextlib.redirect_stdout(self.stdout))
        self.stack.enter_context(patch.object(fetch, "HAS_AX", False))
        # 外部コマンドの有無で結果が変わらないように、既定では全部あることにする
        self.stack.enter_context(patch.object(fetch, "missing", return_value=[]))

    def seed(self, *suffixes):
        for suffix in suffixes:
            (self.sources / f"paper{suffix}").write_bytes(
                f"Original verified {suffix} source\n".encode() * 150)
        (self.sources / "unrelated.md").write_bytes(b"Unrelated source\n")

    def snapshot(self):
        return {p.name: p.read_bytes() for p in self.sources.iterdir() if p.is_file()}

    def assert_preserved(self, before):
        self.assertEqual(self.snapshot(), before)
        self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))

    def run_fetch(self, url="https://example.test/source"):
        with patch.object(sys, "argv", [str(SCRIPT), str(self.sources), "paper", url]):
            return fetch.main()

    @staticmethod
    def pdf_text(text):
        def convert(pdf):
            pdf.with_suffix(".txt").write_text(text, encoding="utf-8")
            return text
        return convert

    @staticmethod
    def markdown(text):
        def convert(url, out, body=None, ctype=""):
            out.write_text(text, encoding="utf-8")
            return text
        return convert

    def test_short_text_retry_preserves_existing_source(self):
        for ctype, suffix in (("text/plain", ".md"), ("text/markdown", ".md"),
                              ("application/json", ".json"), ("text/csv", ".csv")):
            with self.subTest(ctype=ctype):
                self.seed(suffix)
                before = self.snapshot()
                with patch.object(fetch, "download", return_value=(b"Short response", ctype)):
                    self.assertEqual(self.run_fetch(), 1)
                self.assert_preserved(before)
        self.assertIn("FAIL\tpaper\t", self.stdout.getvalue())

    def test_html_wall_retry_preserves_existing_source(self):
        self.seed(".md")
        before = self.snapshot()
        body = b"<article>captcha " + b"Wall text " * 150 + b"</article>"
        with patch.object(fetch, "download", return_value=(body, "text/html")):
            self.assertEqual(self.run_fetch(), 1)
        self.assert_preserved(before)

    def test_short_pdf_retry_preserves_both_files(self):
        self.seed(".pdf", ".txt")
        before = self.snapshot()
        with patch.object(fetch, "download", return_value=(b"%PDF-1.4\nNew PDF", "application/pdf")), \
                patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text("Short text " * 30)):
            self.assertEqual(self.run_fetch(), 1)
        self.assert_preserved(before)

    def test_short_youtube_retry_preserves_existing_transcript(self):
        self.seed(".md")
        before = self.snapshot()
        with patch.object(fetch, "youtube_to_md", side_effect=self.markdown("Short transcript")):
            self.assertEqual(self.run_fetch("https://www.youtube.com/watch?v=example"), 1)
        self.assert_preserved(before)

    def test_download_error_preserves_existing_source(self):
        self.seed(".md")
        before = self.snapshot()
        with patch.object(fetch, "download", side_effect=OSError("Connection failed")):
            self.assertEqual(self.run_fetch(), 1)
        self.assert_preserved(before)

    def test_conversion_exception_preserves_both_files(self):
        self.seed(".pdf", ".txt")
        before = self.snapshot()
        with patch.object(fetch, "download", return_value=(b"%PDF-1.4\nNew PDF", "application/pdf")), \
                patch.object(fetch, "pdf_to_text", side_effect=FileNotFoundError("pdftotext")):
            with self.assertRaises(FileNotFoundError):
                self.run_fetch()
        self.assert_preserved(before)

    def test_failed_arxiv_fallback_preserves_existing_files(self):
        self.seed(".pdf", ".txt", ".md")
        before = self.snapshot()
        with patch.object(fetch, "download", return_value=(b"%PDF-1.4\nNew PDF", "application/pdf")), \
                patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text("")), \
                patch.object(fetch, "html_to_md", side_effect=self.markdown("Short HTML")) as html:
            self.assertEqual(self.run_fetch("https://arxiv.org/pdf/2601.12345v2"), 1)
        self.assertEqual(html.call_args.args[0], "https://arxiv.org/html/2601.12345v2")
        self.assert_preserved(before)

    def test_failed_redirected_pdf_preserves_existing_files(self):
        self.seed(".md", ".pdf", ".txt")
        before = self.snapshot()

        def invalid_pdf(url, out):
            out.write_bytes(b"Not a PDF")
            return True

        with patch.object(fetch, "download", return_value=(b"<html>Redirect</html>", "text/html")), \
                patch.object(fetch, "html_to_md", side_effect=self.markdown("%PDF")), \
                patch.object(fetch, "pdf_save", side_effect=invalid_pdf):
            self.assertEqual(self.run_fetch(), 1)
        self.assert_preserved(before)

    def test_hal_style_pdf_is_fetched_with_plain_user_agent(self):
        """HAL はブラウザの名乗りに HTML を、それ以外に PDF を返す。名乗りを変えて頼み直し、PDF を残す。"""
        pdf = b"%PDF-1.4\nHAL paper"

        def download(url, user_agent=fetch.BROWSER_UA):
            return (pdf, "application/pdf") if user_agent == fetch.PLAIN_UA else (b"<html>viewer</html>", "text/html")

        with patch.object(fetch, "download", side_effect=download), \
                patch.object(fetch, "html_to_md", side_effect=self.markdown("Viewer page " * 200)), \
                patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text("Verified paper text. " * 200)):
            self.assertEqual(self.run_fetch(), 0)
        self.assertEqual((self.sources / "paper.pdf").read_bytes(), pdf)
        self.assertFalse((self.sources / "paper.md").exists())

    def test_image_only_pdf_keeps_the_html_text(self):
        """名乗りを変えて取れた PDF が文字の層の無い画像なら、HTML の本文を残す。"""
        def download(url, user_agent=fetch.BROWSER_UA):
            return (b"%PDF-1.4\nscan", "application/pdf") if user_agent == fetch.PLAIN_UA \
                else (b"<html>page</html>", "text/html")

        with patch.object(fetch, "download", side_effect=download), \
                patch.object(fetch, "html_to_md", side_effect=self.markdown("Article body. " * 200)), \
                patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text("")), \
                patch.object(fetch, "ocr_pdf", return_value=("", "")):
            self.assertEqual(self.run_fetch(), 0)
        self.assertIn("Article body.", (self.sources / "paper.md").read_text(encoding="utf-8"))
        self.assertFalse((self.sources / "paper.pdf").exists())

    def test_arxiv_html_error_keeps_the_pdf_text(self):
        """arXiv の HTML 版が取れなくても、先に取れた PDF の本文は残す。"""
        text = "Short but real PDF text. " * 80  # 2000字。下限の3000字には届かない
        with patch.object(fetch, "download", return_value=(b"%PDF-1.4\npaper", "application/pdf")), \
                patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text(text)), \
                patch.object(fetch, "html_to_md", side_effect=OSError("404")):
            self.assertEqual(self.run_fetch("https://arxiv.org/pdf/2601.12345"), 1)
        self.assertIn("本文が短すぎる(2000文字)", self.stdout.getvalue())
        self.assertFalse(any(p.name.startswith(".") for p in self.sources.iterdir()))

    def test_media_type_is_case_insensitive(self):
        body = b"Plain text line\n" * 100
        with patch.object(fetch, "download", return_value=(body, "Text/Plain; Charset=UTF-8")):
            self.assertEqual(self.run_fetch(), 0)
        self.assertEqual((self.sources / "paper.md").read_bytes(), body)

    def test_successful_text_retry_replaces_source_and_reports_final_path(self):
        for ctype, suffix in (("text/plain", ".md"), ("text/markdown", ".md"),
                              ("application/json", ".json"), ("text/csv", ".csv")):
            with self.subTest(ctype=ctype):
                self.seed(suffix)
                body = b"New verified text\n" * 100
                with patch.object(fetch, "download", return_value=(body, ctype)):
                    self.assertEqual(self.run_fetch(), 0)
                target = self.sources / f"paper{suffix}"
                self.assertEqual(target.read_text(encoding="utf-8"), body.decode())
                self.assertIn(f"OK\tpaper\t{target}\t", self.stdout.getvalue())
                self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))
                self.assertEqual((self.sources / "unrelated.md").read_bytes(), b"Unrelated source\n")

    def test_successful_html_retry_saves_verified_text(self):
        self.seed(".md")
        body = b"<article>" + b"New article text " * 100 + b"</article>"
        with patch.object(fetch, "download", return_value=(body, "text/html")):
            self.assertEqual(self.run_fetch(), 0)
        target = self.sources / "paper.md"
        self.assertIn("New article text", target.read_text(encoding="utf-8"))
        self.assertIn(f"OK\tpaper\t{target}\t", self.stdout.getvalue())
        self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))

    def test_successful_pdf_retry_replaces_both_files(self):
        self.seed(".pdf", ".txt")
        pdf = b"%PDF-1.4\nNew verified PDF"
        text = "New verified PDF text\n" * 200
        with patch.object(fetch, "download", return_value=(pdf, "application/pdf")), \
                patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text(text)):
            self.assertEqual(self.run_fetch(), 0)
        self.assertEqual((self.sources / "paper.pdf").read_bytes(), pdf)
        target = self.sources / "paper.txt"
        self.assertEqual(target.read_text(encoding="utf-8"), text)
        self.assertIn(f"OK\tpaper\t{target}\t", self.stdout.getvalue())
        self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))
        self.assertEqual(set(self.snapshot()), {"paper.pdf", "paper.txt", "unrelated.md"})

    def test_failed_second_replace_restores_both_files(self):
        for existing in (True, False):
            with self.subTest(existing=existing):
                if existing:
                    self.seed(".pdf", ".txt")
                before = self.snapshot()
                replace = Path.replace

                def fail_text(path, target):
                    if path.name == "paper.txt" and path.parent != self.sources:
                        raise OSError("Cannot replace text")
                    return replace(path, target)

                with patch.object(fetch, "download", return_value=(b"%PDF-1.4\nNew PDF", "application/pdf")), \
                        patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text("New PDF text\n" * 300)), \
                        patch.object(Path, "replace", fail_text):
                    with self.assertRaisesRegex(OSError, "Cannot replace text"):
                        self.run_fetch()
                self.assert_preserved(before)
                for suffix in (".pdf", ".txt"):
                    (self.sources / f"paper{suffix}").unlink(missing_ok=True)

    def test_failed_backup_preserves_pdf_and_destination_directory(self):
        self.seed(".pdf")
        before = self.snapshot()
        target = self.sources / "paper.txt"
        target.mkdir()
        (target / "original.txt").write_bytes(b"Keep directory contents")
        with patch.object(fetch, "download", return_value=(b"%PDF-1.4\nNew PDF", "application/pdf")), \
                patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text("New PDF text\n" * 300)):
            with self.assertRaises(OSError):
                self.run_fetch()
        self.assertEqual((target / "original.txt").read_bytes(), b"Keep directory contents")
        self.assertEqual(self.snapshot(), before)
        self.assertEqual({p.name for p in self.sources.iterdir()}, {*before, "paper.txt"})

    def test_successful_youtube_retry_replaces_transcript(self):
        self.seed(".md")
        text = "[00:00] New transcript\n" * 30
        with patch.object(fetch, "youtube_to_md", side_effect=self.markdown(text)):
            self.assertEqual(self.run_fetch("https://www.youtube.com/watch?v=example"), 0)
        target = self.sources / "paper.md"
        self.assertEqual(target.read_text(encoding="utf-8"), text)
        self.assertIn(f"OK\tpaper\t{target}\t", self.stdout.getvalue())
        self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))

    def test_bot_check_page_is_a_wall(self):
        # HAL(hal.science)の Anubis が返した 1,297 字のページ。壁の文に掛かることを確かめる
        self.seed(".md")
        before = self.snapshot()
        body = (b"<html><head><title>Making sure you&#39;re not a bot!</title></head><body>"
                b"<h1>Making sure you're not a bot!</h1><p>Loading...</p><p>You are seeing this because "
                b"the administrator of this website has set up Anubis to protect the server.</p>"
                + b"<p>Anubis is a compromise. Anubis uses a Proof-of-Work scheme.</p>" * 20
                + b"</body></html>")
        with patch.object(fetch, "download", return_value=(body, "text/html")):
            self.assertEqual(self.run_fetch("https://hal.science/hal-01577896"), 1)
        self.assertIn("FAIL\tpaper\t壁に当たった(you're not a bot)", self.stdout.getvalue())
        self.assert_preserved(before)

    JATS = """<?xml version="1.0"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD JATS (Z39.96) Journal Archiving and Interchange DTD v1.4//EN" "JATS.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink"><front>
<journal-meta><journal-title-group><journal-title>Mol Ecol</journal-title></journal-title-group></journal-meta>
<article-meta><article-id pub-id-type="pmcid">PMC1234567</article-id>
<article-id pub-id-type="doi">10.1111/mec.1</article-id>
<title-group><article-title>Environmental <italic>DNA</italic> for biomonitoring</article-title></title-group>
<contrib-group><contrib contrib-type="author"><name><surname>Pawlowski</surname><given-names>Jan</given-names></name></contrib></contrib-group>
<pub-date><year>2021</year></pub-date>
<abstract><p>Short abstract text.</p></abstract></article-meta></front>
<body><sec><title>Introduction</title><p>First paragraph (<xref ref-type="bibr" rid="r1">Smith, 2020</xref>).</p>
<p>Second paragraph with a <ext-link xlink:href="https://example.org/data">dataset</ext-link>.</p>
<sec><title>Sub section</title><p>Nested paragraph.</p></sec></sec>
<sec><title>Results</title><p>{filler}</p>
<fig><label>Figure 1</label><caption><p>Caption of the figure.</p></caption><graphic xlink:href="f1.jpg"/></fig></sec></body>
<back><ref-list><title>References</title><ref><label>1</label><element-citation><person-group>
<name><surname>Smith</surname><given-names>J</given-names></name></person-group>
<article-title>A paper</article-title><source>Nature</source><year>2020</year>
<pub-id pub-id-type="doi">10.1038/x</pub-id></element-citation></ref></ref-list></back></article>
""".replace("{filler}", "Results text. " * 100)

    def test_pmc_url_is_fetched_from_europe_pmc_as_lines(self):
        self.seed(".md")
        with patch.object(fetch, "download", return_value=(self.JATS.encode(), "application/xml")) as dl:
            self.assertEqual(self.run_fetch("https://pmc.ncbi.nlm.nih.gov/articles/PMC1234567/"), 0)
        self.assertEqual(dl.call_args.args[0],
                         "https://www.ebi.ac.uk/europepmc/webservices/rest/PMC1234567/fullTextXML")
        lines = (self.sources / "paper.md").read_text(encoding="utf-8").splitlines()
        # 1行目は題名だけ。雑誌名や ID がつながらない
        self.assertEqual(lines[0], "# Environmental DNA for biomonitoring")
        self.assertIn("Mol Ecol / 2021 / https://doi.org/10.1111/mec.1 / PMC1234567", lines)
        self.assertIn("著者: Jan Pawlowski", lines)
        for line in ("## Abstract", "Short abstract text.", "## Introduction",
                     "First paragraph (Smith, 2020).",
                     "Second paragraph with a [dataset](https://example.org/data).",
                     "### Sub section", "Nested paragraph.", "## Results",
                     "[Figure 1] Caption of the figure.", "## References",
                     "[1] Smith J A paper Nature 2020 10.1038/x https://doi.org/10.1038/x"):
            self.assertIn(line, lines)
        self.assertLess(lines.index("## Introduction"), lines.index("### Sub section"))
        self.assertIn("OK\tpaper\t", self.stdout.getvalue())

    def test_pmc_without_europe_pmc_full_text_fails_and_preserves(self):
        self.seed(".md")
        before = self.snapshot()
        error = fetch.urllib.error.HTTPError("https://www.ebi.ac.uk/", 404, "Not Found", {}, None)
        with patch.object(fetch, "download", side_effect=error):
            self.assertEqual(self.run_fetch("https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7654321/"), 1)
        self.assertIn("FAIL\tpaper\tEurope PMC に全文 XML が無い", self.stdout.getvalue())
        self.assert_preserved(before)

    def test_pmc_is_recognized_by_host_not_anywhere_in_url(self):
        for base in ("https://pmc.ncbi.nlm.nih.gov/articles/",
                     "https://www.ncbi.nlm.nih.gov/pmc/articles/",
                     "https://europepmc.org/article/PMC/"):
            for suffix in ("PMC1234567", "pmc1234567/"):
                with self.subTest(url=base + suffix):
                    self.assertEqual(fetch.pmc_id(base + suffix), "PMC1234567")
            for suffix in ("PMC1234567fake", "PMC1234567/../../articles/PMC7654321/",
                           "../PMC1234567/", "PMC1234567/./", "PMC1234567//"):
                with self.subTest(url=base + suffix):
                    self.assertIsNone(fetch.pmc_id(base + suffix))
        # 他人のページの URL に書き込まれた PMCID の本文を、そのページの原文として保存しない
        self.assertIsNone(fetch.pmc_id("https://evil.example/?next=pmc.ncbi.nlm.nih.gov/articles/PMC1234567/"))
        self.assertIsNone(fetch.pmc_id("https://pmc.ncbi.nlm.nih.gov.evil.example/articles/PMC1234567/"))

    def test_jats_with_entity_declaration_is_refused(self):
        self.seed(".md")
        before = self.snapshot()
        for encoding, bom in (("utf-8", b""), ("utf-16le", b"\xff\xfe"), ("utf-16be", b"\xfe\xff")):
            for declaration in ('<!ENTITY x "xx">', '<!ENTITY x SYSTEM "file:///etc/passwd">'):
                with self.subTest(encoding=encoding, declaration=declaration):
                    xml = bom + (f'<?xml version="1.0"?><!DOCTYPE article [{declaration}]>'
                                 '<article>&x;</article>').encode(encoding)
                    with self.assertRaisesRegex(ValueError, "ENTITY"):
                        fetch.jats_to_md(xml, "https://example.org/", "https://example.org/fullTextXML")
                    with patch.object(fetch, "download", return_value=(xml, "application/xml")):
                        self.assertEqual(self.run_fetch("https://pmc.ncbi.nlm.nih.gov/articles/PMC1234567/"), 1)
                    self.assert_preserved(before)
        self.assertIn("ENTITY", self.stdout.getvalue())

    def test_jats_pmcid_mismatch_fails_and_preserves(self):
        self.seed(".md")
        before = self.snapshot()
        xml = self.JATS.replace('>PMC1234567</article-id>', '>PMC7654321</article-id>').encode()
        with patch.object(fetch, "download", return_value=(xml, "application/xml")):
            self.assertEqual(self.run_fetch("https://pmc.ncbi.nlm.nih.gov/articles/PMC1234567/"), 1)
        self.assertIn("PMCID", self.stdout.getvalue())
        self.assert_preserved(before)

    def test_jats_pmcid_accepts_optional_prefix_and_missing_id(self):
        for article_id in ('<article-id pub-id-type="pmcid">1234567</article-id>',
                           '<article-id pub-id-type="pmcid">pMc1234567</article-id>', ""):
            with self.subTest(article_id=article_id):
                xml = self.JATS.replace('<article-id pub-id-type="pmcid">PMC1234567</article-id>', article_id)
                with patch.object(fetch, "download", return_value=(xml.encode(), "application/xml")):
                    self.assertEqual(self.run_fetch("https://pmc.ncbi.nlm.nih.gov/articles/PMC1234567/"), 0)

    def test_jats_links_follow_html_safety_rules(self):
        for tag in ("ext-link", "uri"):
            for href, expected in (("javascript:alert(1)", "label [x]\\y"),
                                   ("data:text/html,x", "label [x]\\y"),
                                   ("https://example.org/a (1)",
                                    r"[label \[x\]\\y](https://example.org/a%20%281%29)")):
                with self.subTest(tag=tag, href=href):
                    xml = (f'<article xmlns:xlink="http://www.w3.org/1999/xlink"><body><p>'
                           f'<{tag} xlink:href="{href}">label [x]\\y</{tag}>'
                           '</p></body></article>').encode()
                    text = fetch.jats_to_md(xml, "https://example.org/", "https://example.org/fullTextXML")
                    self.assertEqual(text.splitlines()[-1], expected)

    def test_malformed_hrefs_keep_text_without_failing(self):
        # 全角の「#」や閉じていない「[」は urlsplit が ValueError を出す。リンク1本で変換全体を落とさない
        for href in ("https://example.org#frag", "https://[x"):
            with self.subTest(href=href):
                self.assertEqual(fetch.md_link("label", href), "label")
                parser = fetch._TextExtractor("https://example.org/")
                parser.feed(f'<p><a href="{href}">bad</a> <a href="https://ok.example/">ok</a></p>')
                self.assertIn("[ok](https://ok.example/)", parser.text())

    def test_jats_tree_limits_fail_and_preserve(self):
        self.seed(".md")
        before = self.snapshot()
        for limit in ("JATS_MAX_ELEMENTS", "JATS_MAX_DEPTH"):
            with self.subTest(limit=limit), patch.object(fetch, limit, 3):
                with patch.object(fetch, "download", return_value=(self.JATS.encode(), "application/xml")):
                    self.assertEqual(self.run_fetch("https://pmc.ncbi.nlm.nih.gov/articles/PMC1234567/"), 1)
                self.assert_preserved(before)
                self.assertIn("ValueError", self.stdout.getvalue())

    def test_jats_tree_limits_allow_boundary(self):
        xml = b'<article><body><p>Body</p><p>Text</p></body></article>'
        with patch.object(fetch, "JATS_MAX_ELEMENTS", 4), patch.object(fetch, "JATS_MAX_DEPTH", 3):
            text = fetch.jats_to_md(xml, "https://example.org/", "https://example.org/fullTextXML")
        self.assertTrue(text.endswith("Body\nText\n"))

    def test_failed_first_fetch_leaves_no_partial_source(self):
        with patch.object(fetch, "download", return_value=(b"Short response", "text/plain")):
            self.assertEqual(self.run_fetch(), 1)
        self.assertEqual(list(self.sources.iterdir()), [])

    def test_non_http_url_is_rejected_without_reading(self):
        secret = Path(self.sources.parent) / "secret.txt"
        secret.write_text("local secret\n" * 200, encoding="utf-8")
        for url in (secret.as_uri(), "FILE://" + secret.as_posix(),
                    "ftp://example.test/paper.txt", "-o/tmp/x"):
            with self.subTest(url=url):
                with patch.object(fetch, "download") as download:
                    self.assertEqual(self.run_fetch(url), 1)
                download.assert_not_called()
                self.assertEqual(list(self.sources.iterdir()), [])
        self.assertIn("http(s) の URL だけを取りに行く", self.stdout.getvalue())


class OptionInjectionTests(unittest.TestCase):
    """外のコマンドへ渡す URL は「--」の直後に置く。「-」で始まる URL をオプションとして読ませない。"""

    URL = "--exec=touch /tmp/pwned"

    def setUp(self):
        tmp = tempfile.TemporaryDirectory()
        self.addCleanup(tmp.cleanup)
        self.out = Path(tmp.name) / "page.md"

    def capture(self, call):
        with patch.object(fetch, "run_tool",
                          return_value=subprocess.CompletedProcess([], 0, "", "")) as run:
            call()
        return [c.args[0] for c in run.call_args_list]

    def assert_url_after_separator(self, cmds):
        self.assertTrue(cmds)
        for cmd in cmds:
            with self.subTest(cmd=cmd):
                self.assertEqual(cmd[-2:], ["--", self.URL])

    def test_ax_gets_a_file_never_the_url(self):
        """ax には取ってきた HTML のファイルだけを渡す。URL を渡すと ax が自分で通信する。"""
        with patch.object(fetch, "HAS_AX", True):
            cmds = self.capture(lambda: fetch.html_to_md(self.URL, self.out, b"<p>body</p>", "text/html"))
        self.assertEqual(len(cmds), 1)
        self.assertNotIn(self.URL, cmds[0])
        self.assertEqual(cmds[0][-2], "--")
        self.assertTrue(Path(cmds[0][-1]).is_absolute())

    def test_yt_dlp(self):
        self.assert_url_after_separator(self.capture(lambda: fetch.youtube_to_md(self.URL, self.out)))


class YoutubeCaptionTests(unittest.TestCase):
    """yt-dlp の呼び出しを差し替えて、どの字幕を指定するかを検査する。"""

    def setUp(self):
        tmp = tempfile.TemporaryDirectory()
        self.addCleanup(tmp.cleanup)
        self.out = Path(tmp.name) / "talk.md"

    def run_youtube(self, language, manual=None, auto=None):
        """manual と auto は動画が持つ字幕(言語 → 文)。指定されて、かつ有るものだけを .vtt に書く。"""
        asked = []

        def yt_dlp(cmd, **kwargs):
            flag = next(c for c in cmd if c.startswith("--write-"))
            langs = cmd[cmd.index("--sub-langs") + 1]
            asked.append((flag, langs))
            tracks = (auto if flag == "--write-auto-subs" else manual) or {}
            folder = Path(cmd[cmd.index("-o") + 1]).parent
            for lang in langs.split(","):
                if lang in tracks:
                    (folder / f"v.{lang}.vtt").write_text(
                        f"WEBVTT\n\n00:00:01.000 --> 00:00:02.000\n{tracks[lang]}\n", encoding="utf-8")
            return subprocess.CompletedProcess(cmd, 0, f"Talk\tSpeaker\t20220820\t{language}\n", "")

        with patch.object(fetch, "run_tool", side_effect=yt_dlp):
            body = fetch.youtube_to_md("https://www.youtube.com/watch?v=example", self.out)
        return asked, body

    def test_auto_captions_ask_only_the_spoken_language(self):
        # 吹き替えの音声にも ja-orig が付く。中身は翻訳なので取らない
        asked, body = self.run_youtube("en-US", auto={"en-orig": "English speech", "ja-orig": "吹き替え"})
        self.assertEqual(asked, [("--write-subs", "ja,en"), ("--write-auto-subs", "en-orig")])
        self.assertEqual(body, "[00:01] English speech")
        text = self.out.read_text(encoding="utf-8")
        self.assertIn("\nSpeaker / 20220820\n字幕: 自動", text)
        self.assertIn("v.en-orig.vtt", text)

    def test_japanese_auto_captions(self):
        asked, body = self.run_youtube("ja", auto={"ja-orig": "日本語の音声"})
        self.assertEqual(asked[-1], ("--write-auto-subs", "ja-orig"))
        self.assertEqual(body, "[00:01] 日本語の音声")

    def test_unknown_language_asks_original_japanese_or_english(self):
        asked, body = self.run_youtube("", auto={"en-orig": "English speech"})
        self.assertEqual(asked[-1], ("--write-auto-subs", "ja-orig,en-orig"))
        self.assertEqual(body, "[00:01] English speech")

    def test_manual_captions_skip_auto_captions(self):
        asked, body = self.run_youtube("en", manual={"ja": "手動の日本語"}, auto={"en-orig": "English"})
        self.assertEqual(asked, [("--write-subs", "ja,en")])
        self.assertEqual(body, "[00:01] 手動の日本語")
        self.assertIn("字幕: 手動 v.ja.vtt", self.out.read_text(encoding="utf-8"))


class CjkPdfFloorTests(unittest.TestCase):
    """和文の PDF は 3,000 字に届かなくても、HTML と同じ下限を超えていれば残す。"""

    def setUp(self):
        tmp = tempfile.TemporaryDirectory()
        self.addCleanup(tmp.cleanup)
        self.sources = Path(tmp.name) / "sources"
        self.sources.mkdir()
        stack = contextlib.ExitStack()
        self.addCleanup(stack.close)
        stack.enter_context(contextlib.redirect_stdout(io.StringIO()))
        stack.enter_context(patch.object(fetch, "missing", return_value=[]))

    def fetch_pdf(self, text):
        def convert(pdf):
            pdf.with_suffix(".txt").write_text(text, encoding="utf-8")
            return text
        argv = [str(SCRIPT), str(self.sources), "paper", "https://example.test/paper.pdf"]
        with patch.object(sys, "argv", argv), \
                patch.object(fetch, "download", return_value=(b"%PDF-1.4\nshort", "application/pdf")), \
                patch.object(fetch, "pdf_to_text", side_effect=convert):
            return fetch.main()

    def test_short_japanese_pdf_is_kept(self):
        text = "日本語の本文です。\n" * 150  # 1,500字
        self.assertLess(len(text), fetch.MIN_TEXT_CHARS)
        self.assertEqual(self.fetch_pdf(text), 0)
        self.assertEqual((self.sources / "paper.txt").read_text(encoding="utf-8"), text)

    def test_short_english_pdf_still_fails(self):
        text = "English body text.\n" * 120  # 2,280字
        self.assertGreater(len(text), fetch.MIN_HTML_CHARS)
        self.assertEqual(self.fetch_pdf(text), 1)
        self.assertEqual(list(self.sources.iterdir()), [])

    def test_very_short_japanese_pdf_still_fails(self):
        text = "日本語。\n" * 150  # 750字
        self.assertEqual(self.fetch_pdf(text), 1)
        self.assertEqual(list(self.sources.iterdir()), [])

    def test_english_pdf_with_japanese_heading_is_not_cjk(self):
        text = "日本語の見出し\n" + "English body text.\n" * 100
        self.assertFalse(fetch.mostly_cjk(text))
        self.assertTrue(fetch.mostly_cjk("全角の123"))


class NonAsciiUrlTests(unittest.TestCase):
    """日本語の字をそのまま含む URL も、符号化して取りに行く(イシュー #9)。"""

    def requested(self, url):
        seen = []

        def open_(req, timeout):
            seen.append(req.full_url)
            res = unittest.mock.MagicMock()
            res.__enter__.return_value.read.return_value = b"body"
            return res

        with patch.object(fetch._OPENER, "open", side_effect=open_):
            fetch.download(url)
        return seen[0]

    def test_japanese_path_and_query_are_encoded(self):
        self.assertEqual(self.requested("https://kotobank.jp/word/漆工芸"),
                         "https://kotobank.jp/word/%E6%BC%86%E5%B7%A5%E8%8A%B8")
        self.assertEqual(self.requested("https://example.test/search?q=和竿&page=2"),
                         "https://example.test/search?q=%E5%92%8C%E7%AB%BF&page=2")

    def test_encoded_url_is_unchanged(self):
        for url in ("https://kotobank.jp/word/%E6%BC%86%E5%B7%A5%E8%8A%B8",
                    "https://www.youtube.com/watch?v=jPCV4GKX9Dw&t=30s",
                    "https://example.test/a%20b/c?x=1+2&y=%2F"):
            with self.subTest(url=url):
                self.assertEqual(self.requested(url), url)


if __name__ == "__main__":
    unittest.main()