Skip to content

知らない分野を、原文にたどりつくまで芋づる式に掘る Claude Code のスキル

Python

fetch: PR #12 のレビューで見つけた点を直す

MaryCache5e7f1e7cba5f
diff --git a/skills/imozuru/scripts/fetch.py b/skills/imozuru/scripts/fetch.py
index 46b9f1a..0785dd4 100755
--- a/skills/imozuru/scripts/fetch.py
+++ b/skills/imozuru/scripts/fetch.py
@@ -35,6 +35,7 @@ import urllib.request
import xml.etree.ElementTree as ET
from html.parser import HTMLParser
from pathlib import Path
+from xml.parsers import expat
# 本文として読める最低の文字数。404 ページや認証の壁は数百〜数千文字に収まる。
# PDF は英語の論文が多いので高め。HTML は日本語の短い記事を通すため低め
@@ -458,6 +459,24 @@ def ocr_if_longer(pdf: Path, text: str) -> tuple[str, str]:
HAS_AX = shutil.which("ax") is not None
+def md_link(text: str, href: str) -> str:
+ """原文のリンクを、開いても危険な処理が動かず markdown の形が崩れないように残す。"""
+ href = href.strip()
+ # JavaScript: や data: のリンクは、.md を開いた先で押すとページの中で動く。http(s) だけ残す。
+ # 全角の「#」などを含む URL は urlsplit が ValueError を出す。リンク1本でページ全体を FAIL にしない
+ try:
+ scheme = urllib.parse.urlsplit(href).scheme.lower()
+ except ValueError:
+ return text
+ if scheme not in {"http", "https"}:
+ return text
+ # 括弧と空白が URL の中にあると、[文字](URL) の形がそこで切れる
+ href = re.sub(r"[()\s]", lambda m: urllib.parse.quote(m.group(0), safe=""), href)
+ # 「\」を先に符号化する。後にすると、元からある「\]」が「\\]」になって「]」が効き直す
+ text = text.replace("\\", "\\\\").replace("[", r"\[").replace("]", r"\]")
+ return f"[{text}]({href})"
+
+
class _TextExtractor(HTMLParser):
"""ax が無い環境で、HTML から本文の文字を取り出す。
@@ -503,12 +522,11 @@ class _TextExtractor(HTMLParser):
elif tag == "a":
href = (dict(attrs).get("href") or "").strip()
if href and not href.startswith("#"):
- full = urllib.parse.urljoin(self.base, href)
- # JavaScript: や data: のリンクは、.md を開いた先で押すとページの中で動く。http(s) だけ残す
- if urllib.parse.urlsplit(full).scheme.lower() in {"http", "https"}:
- # 括弧と空白が URL の中にあると、[文字](URL) の形がそこで切れる
- self.href = re.sub(r"[()\s]", lambda m: f"%{ord(m.group(0)):02X}", full)
- self.link_text = []
+ try:
+ self.href = urllib.parse.urljoin(self.base, href)
+ except ValueError: # 閉じていない「[」などの壊れた href。リンクにせず文字だけ残す
+ return
+ self.link_text = []
def handle_endtag(self, tag):
if tag in self.SKIP:
@@ -521,9 +539,7 @@ class _TextExtractor(HTMLParser):
self._emit("\n")
elif tag == "a" and self.href is not None:
text = " ".join("".join(self.link_text).split())
- # 「\」を先に符号化する。後にすると、元からある「\]」が「\\]」になって「]」が効き直す
- text = text.replace("\\", "\\\\").replace("[", r"\[").replace("]", r"\]")
- self._emit(f"[{text}]({self.href})" if text else "")
+ self._emit(md_link(text, self.href) if text else "")
self.href = None
def handle_data(self, data):
@@ -686,11 +702,14 @@ def pmc_id(url: str) -> str | None:
"""PMC の記事の URL なら PMCID を返す。arxiv_id と同じく、URL のどこかではなくホスト名とパスで見る。"""
p = urllib.parse.urlsplit(url)
pattern = PMC_PATHS.get((p.hostname or "").lower().rstrip("."))
- m = re.match(pattern, p.path, re.I) if pattern else None
+ m = re.fullmatch(pattern + r"/?", p.path, re.I) if pattern else None
return m.group(1).upper() if m else None
EUROPE_PMC_XML = "https://www.ebi.ac.uk/europepmc/webservices/rest/{}/fullTextXML"
+# 長い論文にも余裕を持たせつつ、小さな XML でメモリや再帰の上限を使い切らせない。
+JATS_MAX_ELEMENTS = 200_000
+JATS_MAX_DEPTH = 200
_XLINK_HREF = "{http://www.w3.org/1999/xlink}href"
# 段落の中の飾りや参照。前後に空白を足すと「Smith et al. ( 2020 )」のように崩れるので、そのままつなぐ
_JATS_INLINE = {"italic", "bold", "sup", "sub", "sc", "underline", "monospace", "roman", "strike",
@@ -722,7 +741,7 @@ def _jats_raw(el) -> str:
inner = _jats_raw(child).strip()
href = child.get(_XLINK_HREF)
if tag in ("ext-link", "uri") and href and href != inner:
- inner = f"[{inner}]({href})" # HTML と同じく、後ろへたどれるようにリンクを残す
+ inner = md_link(inner, href)
parts.append(inner if tag in _JATS_INLINE else f" {inner} ")
parts.append(child.tail or "")
return "".join(parts)
@@ -782,10 +801,28 @@ def jats_to_md(xml: bytes, url: str, xml_url: str) -> str:
XML をそのまま文字にすると、雑誌名・ID・著者が区切りなく1行目につながって読めなかった
(2026-10-19 の試しで Dully 2021 など)。題名と書誌は頭の数行にまとめ、本文は行番号で指せる形にする。
"""
- # JATS は DOCTYPE を必ず持つので watch.py のように DOCTYPE ごとは断れない。外部の DTD は expat が
- # 取りに行かないので害は無い。膨らむ実体(billion laughs)だけは中で宣言されるので、それを断る
- if b"<!ENTITY" in xml:
+ def refuse_entity(*args):
raise ValueError("XML が ENTITY を宣言している")
+
+ count = depth = 0
+
+ def start(tag, attrs):
+ nonlocal count, depth
+ count += 1
+ depth += 1
+ if count > JATS_MAX_ELEMENTS or depth > JATS_MAX_DEPTH:
+ raise ValueError("XML の要素数か深さが上限を超えた")
+
+ def end(tag):
+ nonlocal depth
+ depth -= 1
+
+ # JATS の DOCTYPE は許すが、実体の宣言・外部参照は文字コードに関係なく断る。
+ # 木を作る前に上限も検査し、細工した XML でメモリを使い切らせない。外部 DTD は読まない。
+ parser = expat.ParserCreate()
+ parser.EntityDeclHandler = parser.ExternalEntityRefHandler = refuse_entity
+ parser.StartElementHandler, parser.EndElementHandler = start, end
+ parser.Parse(xml, True)
root = ET.fromstring(xml)
meta = root.find("front/article-meta")
if meta is None:
@@ -793,6 +830,10 @@ def jats_to_md(xml: bytes, url: str, xml_url: str) -> str:
journal = _jats_flat(root.find("front/journal-meta/journal-title-group/journal-title"))
year = next((y.text for y in meta.iter("year") if y.text), "")
ids = {a.get("pub-id-type"): (a.text or "").strip() for a in meta.findall("article-id")}
+ requested = pmc_id(url)
+ if requested and any(not re.fullmatch(r"(?:PMC)?" + requested[3:], (a.text or "").strip(), re.I)
+ for a in meta.findall("article-id") if a.get("pub-id-type") == "pmcid"):
+ raise ValueError("XML の PMCID が要求した記事と違う")
authors = []
for contrib in meta.iter("contrib"):
if contrib.get("contrib-type") != "author":
@@ -837,12 +878,37 @@ def first_line(text: str) -> str:
def save_verified(saved: Path, kept: Path, destination: Path) -> Path:
- """検査を通ったファイルだけを、同じファイルシステム内で保存先に移す。"""
- if saved != kept:
- saved.replace(destination / saved.name)
- target = destination / kept.name
- kept.replace(target)
- return target
+ """検査済みの原文を移し、途中で失敗したら保存済みの原文を戻す。"""
+ files = [saved, kept] if saved != kept else [kept]
+ files = [p for p in files if p.parent != destination]
+ backups, moved = {}, []
+ # 強制終了や電源断のときは退避した写しが残るだけで、自動では戻らない(その備えが要るなら復旧の記録を残す形にする)
+ try:
+ # PDF と本文の片方だけ新しくなると引用元が食い違うので、両方を退避してから移す。
+ for source in files:
+ target = destination / source.name
+ if target.exists() or target.is_symlink():
+ with tempfile.NamedTemporaryFile(prefix=f".{target.name}-", dir=destination, delete=False) as tmp:
+ backup = Path(tmp.name)
+ try:
+ target.replace(backup)
+ except BaseException:
+ backup.unlink()
+ raise
+ backups[target] = backup
+ for source in files:
+ target = destination / source.name
+ source.replace(target)
+ moved.append(target)
+ except BaseException:
+ for target in moved:
+ target.unlink()
+ for target, backup in backups.items():
+ backup.replace(target)
+ raise
+ for backup in backups.values():
+ backup.unlink()
+ return destination / kept.name
def valid_source_name(name: str) -> bool:
diff --git a/tests/test_fetch.py b/tests/test_fetch.py
index 1e8343f..d10bc55 100644
--- a/tests/test_fetch.py
+++ b/tests/test_fetch.py
@@ -220,6 +220,43 @@ class FetchPreservationTests(unittest.TestCase):
self.assertEqual(target.read_text(encoding="utf-8"), text)
self.assertIn(f"OK\tpaper\t{target}\t", self.stdout.getvalue())
self.assertFalse(any(p.is_dir() for p in self.sources.iterdir()))
+ self.assertEqual(set(self.snapshot()), {"paper.pdf", "paper.txt", "unrelated.md"})
+
+ def test_failed_second_replace_restores_both_files(self):
+ for existing in (True, False):
+ with self.subTest(existing=existing):
+ if existing:
+ self.seed(".pdf", ".txt")
+ before = self.snapshot()
+ replace = Path.replace
+
+ def fail_text(path, target):
+ if path.name == "paper.txt" and path.parent != self.sources:
+ raise OSError("Cannot replace text")
+ return replace(path, target)
+
+ with patch.object(fetch, "download", return_value=(b"%PDF-1.4\nNew PDF", "application/pdf")), \
+ patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text("New PDF text\n" * 300)), \
+ patch.object(Path, "replace", fail_text):
+ with self.assertRaisesRegex(OSError, "Cannot replace text"):
+ self.run_fetch()
+ self.assert_preserved(before)
+ for suffix in (".pdf", ".txt"):
+ (self.sources / f"paper{suffix}").unlink(missing_ok=True)
+
+ def test_failed_backup_preserves_pdf_and_destination_directory(self):
+ self.seed(".pdf")
+ before = self.snapshot()
+ target = self.sources / "paper.txt"
+ target.mkdir()
+ (target / "original.txt").write_bytes(b"Keep directory contents")
+ with patch.object(fetch, "download", return_value=(b"%PDF-1.4\nNew PDF", "application/pdf")), \
+ patch.object(fetch, "pdf_to_text", side_effect=self.pdf_text("New PDF text\n" * 300)):
+ with self.assertRaises(OSError):
+ self.run_fetch()
+ self.assertEqual((target / "original.txt").read_bytes(), b"Keep directory contents")
+ self.assertEqual(self.snapshot(), before)
+ self.assertEqual({p.name for p in self.sources.iterdir()}, {*before, "paper.txt"})
def test_successful_youtube_retry_replaces_transcript(self):
self.seed(".md")
@@ -297,18 +334,89 @@ class FetchPreservationTests(unittest.TestCase):
self.assert_preserved(before)
def test_pmc_is_recognized_by_host_not_anywhere_in_url(self):
- self.assertEqual(fetch.pmc_id("https://pmc.ncbi.nlm.nih.gov/articles/PMC1234567/"), "PMC1234567")
- self.assertEqual(fetch.pmc_id("https://www.ncbi.nlm.nih.gov/pmc/articles/pmc1234567"), "PMC1234567")
- self.assertEqual(fetch.pmc_id("https://europepmc.org/article/PMC/PMC1234567"), "PMC1234567")
+ for base in ("https://pmc.ncbi.nlm.nih.gov/articles/",
+ "https://www.ncbi.nlm.nih.gov/pmc/articles/",
+ "https://europepmc.org/article/PMC/"):
+ for suffix in ("PMC1234567", "pmc1234567/"):
+ with self.subTest(url=base + suffix):
+ self.assertEqual(fetch.pmc_id(base + suffix), "PMC1234567")
+ for suffix in ("PMC1234567fake", "PMC1234567/../../articles/PMC7654321/",
+ "../PMC1234567/", "PMC1234567/./", "PMC1234567//"):
+ with self.subTest(url=base + suffix):
+ self.assertIsNone(fetch.pmc_id(base + suffix))
# 他人のページの URL に書き込まれた PMCID の本文を、そのページの原文として保存しない
self.assertIsNone(fetch.pmc_id("https://evil.example/?next=pmc.ncbi.nlm.nih.gov/articles/PMC1234567/"))
self.assertIsNone(fetch.pmc_id("https://pmc.ncbi.nlm.nih.gov.evil.example/articles/PMC1234567/"))
def test_jats_with_entity_declaration_is_refused(self):
- xml = b'<?xml version="1.0"?><!DOCTYPE a [<!ENTITY x "xx">]><article>&x;</article>'
+ self.seed(".md")
+ before = self.snapshot()
+ for encoding, bom in (("utf-8", b""), ("utf-16le", b"\xff\xfe"), ("utf-16be", b"\xfe\xff")):
+ for declaration in ('<!ENTITY x "xx">', '<!ENTITY x SYSTEM "file:///etc/passwd">'):
+ with self.subTest(encoding=encoding, declaration=declaration):
+ xml = bom + (f'<?xml version="1.0"?><!DOCTYPE article [{declaration}]>'
+ '<article>&x;</article>').encode(encoding)
+ with self.assertRaisesRegex(ValueError, "ENTITY"):
+ fetch.jats_to_md(xml, "https://example.org/", "https://example.org/fullTextXML")
+ with patch.object(fetch, "download", return_value=(xml, "application/xml")):
+ self.assertEqual(self.run_fetch("https://pmc.ncbi.nlm.nih.gov/articles/PMC1234567/"), 1)
+ self.assert_preserved(before)
+ self.assertIn("ENTITY", self.stdout.getvalue())
+
+ def test_jats_pmcid_mismatch_fails_and_preserves(self):
+ self.seed(".md")
+ before = self.snapshot()
+ xml = self.JATS.replace('>PMC1234567</article-id>', '>PMC7654321</article-id>').encode()
with patch.object(fetch, "download", return_value=(xml, "application/xml")):
self.assertEqual(self.run_fetch("https://pmc.ncbi.nlm.nih.gov/articles/PMC1234567/"), 1)
- self.assertIn("ENTITY", self.stdout.getvalue())
+ self.assertIn("PMCID", self.stdout.getvalue())
+ self.assert_preserved(before)
+
+ def test_jats_pmcid_accepts_optional_prefix_and_missing_id(self):
+ for article_id in ('<article-id pub-id-type="pmcid">1234567</article-id>',
+ '<article-id pub-id-type="pmcid">pMc1234567</article-id>', ""):
+ with self.subTest(article_id=article_id):
+ xml = self.JATS.replace('<article-id pub-id-type="pmcid">PMC1234567</article-id>', article_id)
+ with patch.object(fetch, "download", return_value=(xml.encode(), "application/xml")):
+ self.assertEqual(self.run_fetch("https://pmc.ncbi.nlm.nih.gov/articles/PMC1234567/"), 0)
+
+ def test_jats_links_follow_html_safety_rules(self):
+ for tag in ("ext-link", "uri"):
+ for href, expected in (("javascript:alert(1)", "label [x]\\y"),
+ ("data:text/html,x", "label [x]\\y"),
+ ("https://example.org/a (1)",
+ r"[label \[x\]\\y](https://example.org/a%20%281%29)")):
+ with self.subTest(tag=tag, href=href):
+ xml = (f'<article xmlns:xlink="http://www.w3.org/1999/xlink"><body><p>'
+ f'<{tag} xlink:href="{href}">label [x]\\y</{tag}>'
+ '</p></body></article>').encode()
+ text = fetch.jats_to_md(xml, "https://example.org/", "https://example.org/fullTextXML")
+ self.assertEqual(text.splitlines()[-1], expected)
+
+ def test_malformed_hrefs_keep_text_without_failing(self):
+ # 全角の「#」や閉じていない「[」は urlsplit が ValueError を出す。リンク1本で変換全体を落とさない
+ for href in ("https://example.org#frag", "https://[x"):
+ with self.subTest(href=href):
+ self.assertEqual(fetch.md_link("label", href), "label")
+ parser = fetch._TextExtractor("https://example.org/")
+ parser.feed(f'<p><a href="{href}">bad</a> <a href="https://ok.example/">ok</a></p>')
+ self.assertIn("[ok](https://ok.example/)", parser.text())
+
+ def test_jats_tree_limits_fail_and_preserve(self):
+ self.seed(".md")
+ before = self.snapshot()
+ for limit in ("JATS_MAX_ELEMENTS", "JATS_MAX_DEPTH"):
+ with self.subTest(limit=limit), patch.object(fetch, limit, 3):
+ with patch.object(fetch, "download", return_value=(self.JATS.encode(), "application/xml")):
+ self.assertEqual(self.run_fetch("https://pmc.ncbi.nlm.nih.gov/articles/PMC1234567/"), 1)
+ self.assert_preserved(before)
+ self.assertIn("ValueError", self.stdout.getvalue())
+
+ def test_jats_tree_limits_allow_boundary(self):
+ xml = b'<article><body><p>Body</p><p>Text</p></body></article>'
+ with patch.object(fetch, "JATS_MAX_ELEMENTS", 4), patch.object(fetch, "JATS_MAX_DEPTH", 3):
+ text = fetch.jats_to_md(xml, "https://example.org/", "https://example.org/fullTextXML")
+ self.assertTrue(text.endswith("Body\nText\n"))
def test_failed_first_fetch_leaves_no_partial_source(self):
with patch.object(fetch, "download", return_value=(b"Short response", "text/plain")):