知らない分野を、原文にたどりつくまで芋づる式に掘る Claude Code のスキル
Python
再取得失敗時に保存済みの原文を保持する
Yebisu0b54834c545d
diff --git a/skills/imozuru/scripts/fetch.py b/skills/imozuru/scripts/fetch.pyindex c08b98d..7cec206 100755--- a/skills/imozuru/scripts/fetch.py+++ b/skills/imozuru/scripts/fetch.py@@ -20,6 +20,7 @@ import reimport shutilimport subprocessimport sys+import tempfileimport urllib.parseimport urllib.requestfrom html.parser import HTMLParser@@ -269,12 +270,17 @@ def first_line(text: str) -> str:return ""-def main() -> int:- if len(sys.argv) != 4:- print(__doc__, file=sys.stderr)- return 2- sources, name, url = Path(sys.argv[1]), sys.argv[2], sys.argv[3]- sources.mkdir(parents=True, exist_ok=True)+def save_verified(saved: Path, kept: Path, destination: Path) -> Path:+ """検査を通ったファイルだけを、同じファイルシステム内で保存先に移す。"""+ if saved != kept:+ saved.replace(destination / saved.name)+ target = destination / kept.name+ kept.replace(target)+ return target+++def fetch_source(sources: Path, name: str, url: str, destination: Path) -> int:+ """sources は今回の取得専用の一時置き場。取得・変換・検査中は保存済みの原文に触れない。"""if X_POST.match(url):print(f"FAIL\t{name}\tX はログインの壁で本文が取れない(URL を map.md に残す)")@@ -286,6 +292,7 @@ def main() -> int:kept.unlink(missing_ok=True)print(f"FAIL\t{name}\t字幕が無いか短すぎる({len(body)}文字)")return 1+ kept = save_verified(kept, kept, destination)print(f"OK\t{name}\t{kept}\t{len(body)}\t{first_line(body)}")return 0@@ -356,9 +363,22 @@ def main() -> int:print(f"FAIL\t{name}\t{reason}")return 1+ kept = save_verified(saved, kept, destination)print(f"OK\t{name}\t{kept}\t{len(text)}\t{first_line(text)}")return 0+def main() -> int:+ if len(sys.argv) != 4:+ print(__doc__, file=sys.stderr)+ return 2+ sources, name, url = Path(sys.argv[1]), sys.argv[2], sys.argv[3]+ sources.mkdir(parents=True, exist_ok=True)+ # 変換や FAIL の後始末は一時置き場だけで行う。同名の原文を検査前に上書きしない。+ # 保存先の中に作るので、成功時の replace はファイルシステムをまたがない。+ with tempfile.TemporaryDirectory(prefix=".imozuru-fetch-", dir=sources) as tmp:+ return fetch_source(Path(tmp), name, url, sources)++if __name__ == "__main__":sys.exit(main())diff --git a/tests/test_fetch.py b/tests/test_fetch.pynew file mode 100644index 0000000..59e05d4--- /dev/null+++ b/tests/test_fetch.py@@ -0,0 +1,190 @@+"""再取得が失敗したときの原文保全と、成功時の保存先を検査する。"""++import contextlib+import importlib.util+import io+import sys+import tempfile+import unittest+from pathlib import Path+from unittest.mock import patch+++SCRIPT = Path(__file__).resolve().parents[1] / "skills/imozuru/scripts/fetch.py"+SPEC = importlib.util.spec_from_file_location("imozuru_fetch", SCRIPT)+fetch = importlib.util.module_from_spec(SPEC)+SPEC.loader.exec_module(fetch)+++class FetchPreservationTests(unittest.TestCase):+ def setUp(self):+ tmp = tempfile.TemporaryDirectory()+ self.addCleanup(tmp.cleanup)+ self.sources = Path(tmp.name) / "sources"+ self.sources.mkdir()+ self.stdout = io.StringIO()+ self.stack = contextlib.ExitStack()+ self.addCleanup(self.stack.close)+ self.stack.enter_context(contextlib.redirect_stdout(self.stdout))+ self.stack.enter_context(patch.object(fetch, "HAS_AX", False))++ def seed(self, *suffixes):+ for suffix in suffixes:+ (self.sources / f"paper{suffix}").write_bytes(+ f"Original verified {suffix} source\n".encode() * 150)+ (self.sources / "unrelated.md").write_bytes(b"Unrelated source\n")++ def snapshot(self):+ return {p.name: p.read_bytes() for p in self.sources.iterdir() if p.is_file()}++ def assert_preserved(self, before):+ self.assertEqual(self.snapshot(), before)+ self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))++ def run_fetch(self, url="https://example.test/source"):+ with patch.object(sys, "argv", [str(SCRIPT), str(self.sources), "paper", url]):+ return fetch.main()++ @staticmethod+ def pdf_text(text):+ def convert(pdf):+ pdf.with_suffix(".txt").write_text(text, encoding="utf-8")+ return text+ return convert++ @staticmethod+ def markdown(text):+ def convert(url, out, body=None):+ out.write_text(text, encoding="utf-8")+ return text+ return convert++ def test_short_text_retry_preserves_existing_source(self):+ for ctype, suffix in (("text/plain", ".md"), ("text/markdown", ".md"),+ ("application/json", ".json"), ("text/csv", ".csv")):+ with self.subTest(ctype=ctype):+ self.seed(suffix)+ before = self.snapshot()+ with patch.object(fetch, "download", return_value=(b"Short response", ctype)):+ self.assertEqual(self.run_fetch(), 1)+ self.assert_preserved(before)+ self.assertIn("FAIL\tpaper\t", self.stdout.getvalue())++ def test_html_wall_retry_preserves_existing_source(self):+ self.seed(".md")+ before = self.snapshot()+ body = b"<article>captcha " + b"Wall text " * 150 + b"</article>"+ with patch.object(fetch, "download", return_value=(body, "text/html")):+ self.assertEqual(self.run_fetch(), 1)+ self.assert_preserved(before)++ def test_short_pdf_retry_preserves_both_files(self):+ self.seed(".pdf", ".txt")+ before = self.snapshot()+ with patch.object(fetch, "download", return_value=(b"%PDF-1.4\nNew PDF", "application/pdf")), \+ patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text("Short text " * 30)):+ self.assertEqual(self.run_fetch(), 1)+ self.assert_preserved(before)++ def test_short_youtube_retry_preserves_existing_transcript(self):+ self.seed(".md")+ before = self.snapshot()+ with patch.object(fetch, "youtube_to_md", side_effect=self.markdown("Short transcript")):+ self.assertEqual(self.run_fetch("https://www.youtube.com/watch?v=example"), 1)+ self.assert_preserved(before)++ def test_download_error_preserves_existing_source(self):+ self.seed(".md")+ before = self.snapshot()+ with patch.object(fetch, "download", side_effect=OSError("Connection failed")):+ self.assertEqual(self.run_fetch(), 1)+ self.assert_preserved(before)++ def test_conversion_exception_preserves_both_files(self):+ self.seed(".pdf", ".txt")+ before = self.snapshot()+ with patch.object(fetch, "download", return_value=(b"%PDF-1.4\nNew PDF", "application/pdf")), \+ patch.object(fetch, "pdf_to_text", side_effect=FileNotFoundError("pdftotext")):+ with self.assertRaises(FileNotFoundError):+ self.run_fetch()+ self.assert_preserved(before)++ def test_failed_arxiv_fallback_preserves_existing_files(self):+ self.seed(".pdf", ".txt", ".md")+ before = self.snapshot()+ with patch.object(fetch, "download", return_value=(b"%PDF-1.4\nNew PDF", "application/pdf")), \+ patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text("")), \+ patch.object(fetch, "html_to_md", side_effect=self.markdown("Short HTML")) as html:+ self.assertEqual(self.run_fetch("https://arxiv.org/pdf/2601.12345v2"), 1)+ self.assertEqual(html.call_args.args[0], "https://arxiv.org/html/2601.12345v2")+ self.assert_preserved(before)++ def test_failed_redirected_pdf_preserves_existing_files(self):+ self.seed(".md", ".pdf", ".txt")+ before = self.snapshot()++ def invalid_pdf(url, out):+ out.write_bytes(b"Not a PDF")+ return True++ with patch.object(fetch, "download", return_value=(b"<html>Redirect</html>", "text/html")), \+ patch.object(fetch, "html_to_md", side_effect=self.markdown("%PDF")), \+ patch.object(fetch, "ax_save", side_effect=invalid_pdf):+ self.assertEqual(self.run_fetch(), 1)+ self.assert_preserved(before)++ def test_successful_text_retry_replaces_source_and_reports_final_path(self):+ for ctype, suffix in (("text/plain", ".md"), ("text/markdown", ".md"),+ ("application/json", ".json"), ("text/csv", ".csv")):+ with self.subTest(ctype=ctype):+ self.seed(suffix)+ body = b"New verified text\n" * 100+ with patch.object(fetch, "download", return_value=(body, ctype)):+ self.assertEqual(self.run_fetch(), 0)+ target = self.sources / f"paper{suffix}"+ self.assertEqual(target.read_text(encoding="utf-8"), body.decode())+ self.assertIn(f"OK\tpaper\t{target}\t", self.stdout.getvalue())+ self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))+ self.assertEqual((self.sources / "unrelated.md").read_bytes(), b"Unrelated source\n")++ def test_successful_html_retry_saves_verified_text(self):+ self.seed(".md")+ body = b"<article>" + b"New article text " * 100 + b"</article>"+ with patch.object(fetch, "download", return_value=(body, "text/html")):+ self.assertEqual(self.run_fetch(), 0)+ target = self.sources / "paper.md"+ self.assertIn("New article text", target.read_text(encoding="utf-8"))+ self.assertIn(f"OK\tpaper\t{target}\t", self.stdout.getvalue())+ self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))++ def test_successful_pdf_retry_replaces_both_files(self):+ self.seed(".pdf", ".txt")+ pdf = b"%PDF-1.4\nNew verified PDF"+ text = "New verified PDF text\n" * 200+ with patch.object(fetch, "download", return_value=(pdf, "application/pdf")), \+ patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text(text)):+ self.assertEqual(self.run_fetch(), 0)+ self.assertEqual((self.sources / "paper.pdf").read_bytes(), pdf)+ target = self.sources / "paper.txt"+ self.assertEqual(target.read_text(encoding="utf-8"), text)+ self.assertIn(f"OK\tpaper\t{target}\t", self.stdout.getvalue())+ self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))++ def test_successful_youtube_retry_replaces_transcript(self):+ self.seed(".md")+ text = "[00:00] New transcript\n" * 30+ with patch.object(fetch, "youtube_to_md", side_effect=self.markdown(text)):+ self.assertEqual(self.run_fetch("https://www.youtube.com/watch?v=example"), 0)+ target = self.sources / "paper.md"+ self.assertEqual(target.read_text(encoding="utf-8"), text)+ self.assertIn(f"OK\tpaper\t{target}\t", self.stdout.getvalue())+ self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))++ def test_failed_first_fetch_leaves_no_partial_source(self):+ with patch.object(fetch, "download", return_value=(b"Short response", "text/plain")):+ self.assertEqual(self.run_fetch(), 1)+ self.assertEqual(list(self.sources.iterdir()), [])+++if __name__ == "__main__":+ unittest.main()