知らない分野を、原文にたどりつくまで芋づる式に掘る Claude Code のスキル
Python
Revert "fetch: 取り直しに失敗しても保存済みの原文を残す (#1)"
MaryCache56270be4791b
diff --git a/skills/imozuru/scripts/fetch.py b/skills/imozuru/scripts/fetch.pyindex bfeb3b7..c08b98d 100755--- a/skills/imozuru/scripts/fetch.py+++ b/skills/imozuru/scripts/fetch.py@@ -16,12 +16,10 @@FAIL <name> <理由>"""-import osimport reimport shutilimport subprocessimport sys-import tempfileimport urllib.parseimport urllib.requestfrom html.parser import HTMLParser@@ -272,60 +270,38 @@ def first_line(text: str) -> str:def main() -> int:- """取得は作業用のフォルダで行い、検査を通ったファイルだけを sources/ へ移す。-- 同じ名前で取り直して失敗したとき、前に取った原文を消さないため。sources/ に直接- 書いていたころは、上書きしたあと検査で落ちて消していた(issue #1)。原文が消えると- map.md の行番号と抜き書きの照合先が無くなる。作業用のフォルダを sources/ の中に作るのは、- 同じファイルシステムの中なら os.replace で一度に置き換えられるため。- """if len(sys.argv) != 4:print(__doc__, file=sys.stderr)return 2sources, name, url = Path(sys.argv[1]), sys.argv[2], sys.argv[3]sources.mkdir(parents=True, exist_ok=True)- work = Path(tempfile.mkdtemp(prefix=f".{name}-fetch-", dir=sources))- try:- result = fetch_into(work, name, url)- if isinstance(result, str):- print(f"FAIL\t{name}\t{result}")- return 1- files, text = result- for f in files:- os.replace(f, sources / f.name)- print(f"OK\t{name}\t{sources / files[-1].name}\t{len(text)}\t{first_line(text)}")- return 0- finally:- shutil.rmtree(work, ignore_errors=True)---def fetch_into(work: Path, name: str, url: str) -> tuple[list[Path], str] | str:- """work に原文を落として検査する。-- 通ったら(sources/ へ移すファイル, 本文)を返す。ファイルは最後が報告に出す本文。- 落ちたら理由の文字列を返す。work は呼び手が丸ごと消すので、ここでは後始末しない。- """if X_POST.match(url):- return "X はログインの壁で本文が取れない(URL を map.md に残す)"+ print(f"FAIL\t{name}\tX はログインの壁で本文が取れない(URL を map.md に残す)")+ return 1if YOUTUBE.search(url):- kept = work / f"{name}.md"+ kept = sources / f"{name}.md"body = youtube_to_md(url, kept)if len(body) < MIN_TRANSCRIPT_CHARS:- return f"字幕が無いか短すぎる({len(body)}文字)"- return [kept], body+ kept.unlink(missing_ok=True)+ print(f"FAIL\t{name}\t字幕が無いか短すぎる({len(body)}文字)")+ return 1+ print(f"OK\t{name}\t{kept}\t{len(body)}\t{first_line(body)}")+ return 0try:body, ctype = download(url)except Exception as e: # HTTP エラーも接続失敗も、呼び手にとっては同じ「取れない」- return f"{type(e).__name__}: {e}"+ print(f"FAIL\t{name}\t{type(e).__name__}: {e}")+ return 1if not body.strip():# 中身の無い 200/202 を返すサーバーがある(Cornell の論文置き場・2026-10-10)。# ブラウザでないと中身を渡さないボット対策のことが多い=ユーザーのブラウザで開いてもらう候補- return "空の応答(ボット対策の可能性・ブラウザで開けば読めるかも)"+ print(f"FAIL\t{name}\t空の応答(ボット対策の可能性・ブラウザで開けば読めるかも)")+ return 1if body[:5] == b"%PDF-":- saved = work / f"{name}.pdf"+ saved = sources / f"{name}.pdf"saved.write_bytes(body)text = pdf_to_text(saved)kept = saved.with_suffix(".txt")@@ -337,41 +313,51 @@ def fetch_into(work: Path, name: str, url: str) -> tuple[list[Path], str] | str:# HTML の判定より先に見る。先頭が `<!-- AUTO GENERATED -->` の markdown を# HTML と見なして ax に回し、改行が全部消えた(chrome-devtools-mcp の tool-reference・2026-10-11)。ext = ".json" if "json" in ctype else ".csv" if "csv" in ctype else ".md"- kept = saved = work / f"{name}{ext}"+ kept = saved = sources / f"{name}{ext}"text = body.decode("utf-8", errors="replace")kept.write_text(text)elif "html" in ctype or body.lstrip()[:1] == b"<":- kept = work / f"{name}.md"+ kept = sources / f"{name}.md"text = html_to_md(url, kept, body)saved = kept# HTML のつもりが、ax は転送をたどって PDF 本体を受け取っていた(HAL)。PDF として取り直すif text.lstrip().startswith("%PDF"):- saved = work / f"{name}.pdf"+ kept.unlink(missing_ok=True)+ saved = sources / f"{name}.pdf"if ax_save(url, saved) and saved.read_bytes()[:5] == b"%PDF-":text = pdf_to_text(saved)kept = saved.with_suffix(".txt")if len(text.strip()) < 200:text = ocr_pdf(saved)else:- return "転送先の PDF を保存できなかった"+ saved.unlink(missing_ok=True)+ print(f"FAIL\t{name}\t転送先の PDF を保存できなかった")+ return 1else:- return f"扱えない形式: {ctype or '不明'}"+ print(f"FAIL\t{name}\t扱えない形式: {ctype or '不明'}")+ return 1# arXiv の PDF は pdftotext で本文が0文字になることがある(2026-10-09 の試しで# COLE・Accordion)。同じ論文の HTML 版なら取れたので、短いときだけ切り替える。arxiv_id = re.search(r"arxiv\.org/(?:pdf|abs)/(\d{4}\.\d{4,5}(?:v\d+)?)", url)if arxiv_id and len(text) < MIN_TEXT_CHARS:- kept = saved = work / f"{name}.md"+ for p in (saved, kept):+ p.unlink(missing_ok=True)+ kept = saved = sources / f"{name}.md"text = html_to_md(f"https://arxiv.org/html/{arxiv_id.group(1)}", kept)floor = MIN_TEXT_CHARS if saved.suffix == ".pdf" else MIN_HTML_CHARShead = text[:4000].lower()wall = next((p for p in WALL_PATTERNS if p in head), None)if len(text) < floor or (wall and len(text) < MIN_TEXT_CHARS * 4):- return f"壁に当たった({wall})" if wall else f"本文が短すぎる({len(text)}文字)"-- # PDF なら .pdf と .txt の両方を移す。同じファイルを2回移さないように1つにまとめる- return list(dict.fromkeys((saved, kept))), text+ for p in (saved, kept):+ p.unlink(missing_ok=True)+ reason = f"壁に当たった({wall})" if wall else f"本文が短すぎる({len(text)}文字)"+ print(f"FAIL\t{name}\t{reason}")+ return 1++ print(f"OK\t{name}\t{kept}\t{len(text)}\t{first_line(text)}")+ return 0if __name__ == "__main__":diff --git a/skills/imozuru/tests/test_fetch.py b/skills/imozuru/tests/test_fetch.pydeleted file mode 100644index 3b9770a..0000000--- a/skills/imozuru/tests/test_fetch.py+++ /dev/null@@ -1,121 +0,0 @@-"""fetch.py が、取り直しに失敗しても保存済みの原文を残すかを確かめる(issue #1)。--ネットワークと外部コマンドは使わない。download・pdf_to_text・youtube_to_md などを差し替える。-実行: python3 -m unittest discover -s skills/imozuru/tests-"""--import importlib.util-import sys-import tempfile-import unittest-from pathlib import Path-from unittest.mock import patch--SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "fetch.py"-spec = importlib.util.spec_from_file_location("imozuru_fetch", SCRIPT)-fetch = importlib.util.module_from_spec(spec)-spec.loader.exec_module(fetch)--OLD = b"Original verified source\n" * 100-LONG = "Long enough body line.\n" * 200 # 下限(PDF 3,000 字・HTML 1,000 字)を超える長さ---def fake_pdf_to_text(text):- """pdftotext の代わりに、決まった本文を .txt に書く。"""- def run(pdf):- out = pdf.with_suffix(".txt")- out.write_text(text)- return text- return run---class FetchKeepsOriginal(unittest.TestCase):- def setUp(self):- self._tmp = tempfile.TemporaryDirectory()- self.sources = Path(self._tmp.name)-- def tearDown(self):- self._tmp.cleanup()-- def run_fetch(self, url="https://example.test/source", **patches):- argv = [str(SCRIPT), str(self.sources), "paper", url]- with patch.object(sys, "argv", argv), patch.multiple(fetch, **patches), \- patch("builtins.print"):- return fetch.main()-- def assert_no_work_dir(self):- self.assertEqual([p.name for p in self.sources.iterdir() if p.name.startswith(".")], [])-- def test_short_text_keeps_old_md(self):- (self.sources / "paper.md").write_bytes(OLD)- code = self.run_fetch(download=lambda url: (b"Short response", "text/plain"))- self.assertEqual(code, 1)- self.assertEqual((self.sources / "paper.md").read_bytes(), OLD)- self.assert_no_work_dir()-- def test_success_replaces_old_md(self):- (self.sources / "paper.md").write_bytes(OLD)- code = self.run_fetch(download=lambda url: (LONG.encode(), "text/plain"))- self.assertEqual(code, 0)- self.assertEqual((self.sources / "paper.md").read_text(), LONG)- self.assert_no_work_dir()-- def test_short_pdf_keeps_old_pdf_and_txt(self):- (self.sources / "paper.pdf").write_bytes(b"%PDF-old")- (self.sources / "paper.txt").write_bytes(OLD)- code = self.run_fetch(download=lambda url: (b"%PDF-new", "application/pdf"),- pdf_to_text=fake_pdf_to_text("x" * 500))- self.assertEqual(code, 1)- self.assertEqual((self.sources / "paper.pdf").read_bytes(), b"%PDF-old")- self.assertEqual((self.sources / "paper.txt").read_bytes(), OLD)- self.assert_no_work_dir()-- def test_pdf_success_moves_pdf_and_txt(self):- code = self.run_fetch(download=lambda url: (b"%PDF-new", "application/pdf"),- pdf_to_text=fake_pdf_to_text(LONG))- self.assertEqual(code, 0)- self.assertEqual((self.sources / "paper.pdf").read_bytes(), b"%PDF-new")- self.assertEqual((self.sources / "paper.txt").read_text(), LONG)- self.assert_no_work_dir()-- def test_short_transcript_keeps_old_md(self):- (self.sources / "paper.md").write_bytes(OLD)-- def short_subs(url, out):- out.write_text("# title\n\nshort")- return "short"- code = self.run_fetch(url="https://www.youtube.com/watch?v=abc", youtube_to_md=short_subs)- self.assertEqual(code, 1)- self.assertEqual((self.sources / "paper.md").read_bytes(), OLD)- self.assert_no_work_dir()-- def test_failed_arxiv_fallback_keeps_old_files(self):- (self.sources / "paper.pdf").write_bytes(b"%PDF-old")- (self.sources / "paper.txt").write_bytes(OLD)- (self.sources / "paper.md").write_bytes(OLD)-- def short_html(url, out, body=None):- out.write_text("short")- return "short"- code = self.run_fetch(url="https://arxiv.org/pdf/2401.00001",- download=lambda url: (b"%PDF-new", "application/pdf"),- pdf_to_text=fake_pdf_to_text(""), ocr_pdf=lambda pdf: "",- html_to_md=short_html)- self.assertEqual(code, 1)- for f in ("paper.txt", "paper.md"):- self.assertEqual((self.sources / f).read_bytes(), OLD)- self.assertEqual((self.sources / "paper.pdf").read_bytes(), b"%PDF-old")- self.assert_no_work_dir()-- def test_failed_pdf_redirect_keeps_old_md(self):- (self.sources / "paper.md").write_bytes(OLD)- code = self.run_fetch(download=lambda url: (b"<html></html>", "text/html"),- html_to_md=lambda url, out, body=None: (out.write_text("%PDF"), "%PDF")[1],- ax_save=lambda url, out: False)- self.assertEqual(code, 1)- self.assertEqual((self.sources / "paper.md").read_bytes(), OLD)- self.assert_no_work_dir()---if __name__ == "__main__":- unittest.main()