Skip to content

知らない分野を、原文にたどりつくまで芋づる式に掘る Claude Code のスキル

Python

再取得失敗時に保存済みの原文を保持する

Yebisu0b54834c545d
diff --git a/skills/imozuru/scripts/fetch.py b/skills/imozuru/scripts/fetch.py
index c08b98d..7cec206 100755
--- a/skills/imozuru/scripts/fetch.py
+++ b/skills/imozuru/scripts/fetch.py
@@ -20,6 +20,7 @@ import re
import shutil
import subprocess
import sys
+import tempfile
import urllib.parse
import urllib.request
from html.parser import HTMLParser
@@ -269,12 +270,17 @@ def first_line(text: str) -> str:
return ""
-def main() -> int:
- if len(sys.argv) != 4:
- print(__doc__, file=sys.stderr)
- return 2
- sources, name, url = Path(sys.argv[1]), sys.argv[2], sys.argv[3]
- sources.mkdir(parents=True, exist_ok=True)
+def save_verified(saved: Path, kept: Path, destination: Path) -> Path:
+ """検査を通ったファイルだけを、同じファイルシステム内で保存先に移す。"""
+ if saved != kept:
+ saved.replace(destination / saved.name)
+ target = destination / kept.name
+ kept.replace(target)
+ return target
+
+
+def fetch_source(sources: Path, name: str, url: str, destination: Path) -> int:
+ """sources は今回の取得専用の一時置き場。取得・変換・検査中は保存済みの原文に触れない。"""
if X_POST.match(url):
print(f"FAIL\t{name}\tX はログインの壁で本文が取れない(URL を map.md に残す)")
@@ -286,6 +292,7 @@ def main() -> int:
kept.unlink(missing_ok=True)
print(f"FAIL\t{name}\t字幕が無いか短すぎる({len(body)}文字)")
return 1
+ kept = save_verified(kept, kept, destination)
print(f"OK\t{name}\t{kept}\t{len(body)}\t{first_line(body)}")
return 0
@@ -356,9 +363,22 @@ def main() -> int:
print(f"FAIL\t{name}\t{reason}")
return 1
+ kept = save_verified(saved, kept, destination)
print(f"OK\t{name}\t{kept}\t{len(text)}\t{first_line(text)}")
return 0
+def main() -> int:
+ if len(sys.argv) != 4:
+ print(__doc__, file=sys.stderr)
+ return 2
+ sources, name, url = Path(sys.argv[1]), sys.argv[2], sys.argv[3]
+ sources.mkdir(parents=True, exist_ok=True)
+ # 変換や FAIL の後始末は一時置き場だけで行う。同名の原文を検査前に上書きしない。
+ # 保存先の中に作るので、成功時の replace はファイルシステムをまたがない。
+ with tempfile.TemporaryDirectory(prefix=".imozuru-fetch-", dir=sources) as tmp:
+ return fetch_source(Path(tmp), name, url, sources)
+
+
if __name__ == "__main__":
sys.exit(main())
diff --git a/tests/test_fetch.py b/tests/test_fetch.py
new file mode 100644
index 0000000..59e05d4
--- /dev/null
+++ b/tests/test_fetch.py
@@ -0,0 +1,190 @@
+"""再取得が失敗したときの原文保全と、成功時の保存先を検査する。"""
+
+import contextlib
+import importlib.util
+import io
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+
+SCRIPT = Path(__file__).resolve().parents[1] / "skills/imozuru/scripts/fetch.py"
+SPEC = importlib.util.spec_from_file_location("imozuru_fetch", SCRIPT)
+fetch = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(fetch)
+
+
+class FetchPreservationTests(unittest.TestCase):
+ def setUp(self):
+ tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(tmp.cleanup)
+ self.sources = Path(tmp.name) / "sources"
+ self.sources.mkdir()
+ self.stdout = io.StringIO()
+ self.stack = contextlib.ExitStack()
+ self.addCleanup(self.stack.close)
+ self.stack.enter_context(contextlib.redirect_stdout(self.stdout))
+ self.stack.enter_context(patch.object(fetch, "HAS_AX", False))
+
+ def seed(self, *suffixes):
+ for suffix in suffixes:
+ (self.sources / f"paper{suffix}").write_bytes(
+ f"Original verified {suffix} source\n".encode() * 150)
+ (self.sources / "unrelated.md").write_bytes(b"Unrelated source\n")
+
+ def snapshot(self):
+ return {p.name: p.read_bytes() for p in self.sources.iterdir() if p.is_file()}
+
+ def assert_preserved(self, before):
+ self.assertEqual(self.snapshot(), before)
+ self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))
+
+ def run_fetch(self, url="https://example.test/source"):
+ with patch.object(sys, "argv", [str(SCRIPT), str(self.sources), "paper", url]):
+ return fetch.main()
+
+ @staticmethod
+ def pdf_text(text):
+ def convert(pdf):
+ pdf.with_suffix(".txt").write_text(text, encoding="utf-8")
+ return text
+ return convert
+
+ @staticmethod
+ def markdown(text):
+ def convert(url, out, body=None):
+ out.write_text(text, encoding="utf-8")
+ return text
+ return convert
+
+ def test_short_text_retry_preserves_existing_source(self):
+ for ctype, suffix in (("text/plain", ".md"), ("text/markdown", ".md"),
+ ("application/json", ".json"), ("text/csv", ".csv")):
+ with self.subTest(ctype=ctype):
+ self.seed(suffix)
+ before = self.snapshot()
+ with patch.object(fetch, "download", return_value=(b"Short response", ctype)):
+ self.assertEqual(self.run_fetch(), 1)
+ self.assert_preserved(before)
+ self.assertIn("FAIL\tpaper\t", self.stdout.getvalue())
+
+ def test_html_wall_retry_preserves_existing_source(self):
+ self.seed(".md")
+ before = self.snapshot()
+ body = b"<article>captcha " + b"Wall text " * 150 + b"</article>"
+ with patch.object(fetch, "download", return_value=(body, "text/html")):
+ self.assertEqual(self.run_fetch(), 1)
+ self.assert_preserved(before)
+
+ def test_short_pdf_retry_preserves_both_files(self):
+ self.seed(".pdf", ".txt")
+ before = self.snapshot()
+ with patch.object(fetch, "download", return_value=(b"%PDF-1.4\nNew PDF", "application/pdf")), \
+ patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text("Short text " * 30)):
+ self.assertEqual(self.run_fetch(), 1)
+ self.assert_preserved(before)
+
+ def test_short_youtube_retry_preserves_existing_transcript(self):
+ self.seed(".md")
+ before = self.snapshot()
+ with patch.object(fetch, "youtube_to_md", side_effect=self.markdown("Short transcript")):
+ self.assertEqual(self.run_fetch("https://www.youtube.com/watch?v=example"), 1)
+ self.assert_preserved(before)
+
+ def test_download_error_preserves_existing_source(self):
+ self.seed(".md")
+ before = self.snapshot()
+ with patch.object(fetch, "download", side_effect=OSError("Connection failed")):
+ self.assertEqual(self.run_fetch(), 1)
+ self.assert_preserved(before)
+
+ def test_conversion_exception_preserves_both_files(self):
+ self.seed(".pdf", ".txt")
+ before = self.snapshot()
+ with patch.object(fetch, "download", return_value=(b"%PDF-1.4\nNew PDF", "application/pdf")), \
+ patch.object(fetch, "pdf_to_text", side_effect=FileNotFoundError("pdftotext")):
+ with self.assertRaises(FileNotFoundError):
+ self.run_fetch()
+ self.assert_preserved(before)
+
+ def test_failed_arxiv_fallback_preserves_existing_files(self):
+ self.seed(".pdf", ".txt", ".md")
+ before = self.snapshot()
+ with patch.object(fetch, "download", return_value=(b"%PDF-1.4\nNew PDF", "application/pdf")), \
+ patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text("")), \
+ patch.object(fetch, "html_to_md", side_effect=self.markdown("Short HTML")) as html:
+ self.assertEqual(self.run_fetch("https://arxiv.org/pdf/2601.12345v2"), 1)
+ self.assertEqual(html.call_args.args[0], "https://arxiv.org/html/2601.12345v2")
+ self.assert_preserved(before)
+
+ def test_failed_redirected_pdf_preserves_existing_files(self):
+ self.seed(".md", ".pdf", ".txt")
+ before = self.snapshot()
+
+ def invalid_pdf(url, out):
+ out.write_bytes(b"Not a PDF")
+ return True
+
+ with patch.object(fetch, "download", return_value=(b"<html>Redirect</html>", "text/html")), \
+ patch.object(fetch, "html_to_md", side_effect=self.markdown("%PDF")), \
+ patch.object(fetch, "ax_save", side_effect=invalid_pdf):
+ self.assertEqual(self.run_fetch(), 1)
+ self.assert_preserved(before)
+
+ def test_successful_text_retry_replaces_source_and_reports_final_path(self):
+ for ctype, suffix in (("text/plain", ".md"), ("text/markdown", ".md"),
+ ("application/json", ".json"), ("text/csv", ".csv")):
+ with self.subTest(ctype=ctype):
+ self.seed(suffix)
+ body = b"New verified text\n" * 100
+ with patch.object(fetch, "download", return_value=(body, ctype)):
+ self.assertEqual(self.run_fetch(), 0)
+ target = self.sources / f"paper{suffix}"
+ self.assertEqual(target.read_text(encoding="utf-8"), body.decode())
+ self.assertIn(f"OK\tpaper\t{target}\t", self.stdout.getvalue())
+ self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))
+ self.assertEqual((self.sources / "unrelated.md").read_bytes(), b"Unrelated source\n")
+
+ def test_successful_html_retry_saves_verified_text(self):
+ self.seed(".md")
+ body = b"<article>" + b"New article text " * 100 + b"</article>"
+ with patch.object(fetch, "download", return_value=(body, "text/html")):
+ self.assertEqual(self.run_fetch(), 0)
+ target = self.sources / "paper.md"
+ self.assertIn("New article text", target.read_text(encoding="utf-8"))
+ self.assertIn(f"OK\tpaper\t{target}\t", self.stdout.getvalue())
+ self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))
+
+ def test_successful_pdf_retry_replaces_both_files(self):
+ self.seed(".pdf", ".txt")
+ pdf = b"%PDF-1.4\nNew verified PDF"
+ text = "New verified PDF text\n" * 200
+ with patch.object(fetch, "download", return_value=(pdf, "application/pdf")), \
+ patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text(text)):
+ self.assertEqual(self.run_fetch(), 0)
+ self.assertEqual((self.sources / "paper.pdf").read_bytes(), pdf)
+ target = self.sources / "paper.txt"
+ self.assertEqual(target.read_text(encoding="utf-8"), text)
+ self.assertIn(f"OK\tpaper\t{target}\t", self.stdout.getvalue())
+ self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))
+
+ def test_successful_youtube_retry_replaces_transcript(self):
+ self.seed(".md")
+ text = "[00:00] New transcript\n" * 30
+ with patch.object(fetch, "youtube_to_md", side_effect=self.markdown(text)):
+ self.assertEqual(self.run_fetch("https://www.youtube.com/watch?v=example"), 0)
+ target = self.sources / "paper.md"
+ self.assertEqual(target.read_text(encoding="utf-8"), text)
+ self.assertIn(f"OK\tpaper\t{target}\t", self.stdout.getvalue())
+ self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))
+
+ def test_failed_first_fetch_leaves_no_partial_source(self):
+ with patch.object(fetch, "download", return_value=(b"Short response", "text/plain")):
+ self.assertEqual(self.run_fetch(), 1)
+ self.assertEqual(list(self.sources.iterdir()), [])
+
+
+if __name__ == "__main__":
+ unittest.main()