Skip to content

知らない分野を、原文にたどりつくまで芋づる式に掘る Claude Code のスキル

Python

fix: 日本語の字を含む URL と、W で始まる題名で落ちないようにする

MaryCache4f8fec7b6867
diff --git a/skills/imozuru/scripts/chain.py b/skills/imozuru/scripts/chain.py
index 6741482..f47ea37 100755
--- a/skills/imozuru/scripts/chain.py
+++ b/skills/imozuru/scripts/chain.py
@@ -16,11 +16,14 @@ OpenAlex は1日 1,000 回まで無料の枠(x-ratelimit-limit-usd 0.1)。
"""
import json
+import re
import sys
import urllib.parse
import urllib.request
API = "https://api.openalex.org"
+# OpenAlex の work ID(W と数字だけ)。「Weak …」のような題名と取り違えないよう、形で見分ける
+WORK_ID = re.compile(r"W\d+")
FIELDS = "id,display_name,publication_year,cited_by_count,doi,best_oa_location"
@@ -54,7 +57,7 @@ def search_title(key: str) -> list[dict]:
def resolve(key: str) -> dict:
"""DOI・OpenAlex ID・題名のどれからでも論文を1本引く。題名なら被引用の最も多い1本。"""
- if key.startswith("W") or "openalex.org/" in key:
+ if WORK_ID.fullmatch(key) or "openalex.org/" in key:
return get(f"/works/{key.rsplit('/', 1)[-1]}", select=FIELDS)
if key.startswith("10.") or "doi.org/" in key:
doi = key.split("doi.org/")[-1]
@@ -87,7 +90,7 @@ def main() -> int:
if cmd == "find":
# 題名のときは候補を全部見せる。どれが本体かを読み手が確かめられるように。
- hits = [resolve(key)] if key.startswith(("W", "10.")) or "/" in key else search_title(key)
+ hits = [resolve(key)] if WORK_ID.fullmatch(key) or key.startswith("10.") or "/" in key else search_title(key)
if not hits:
print(f"候補なし: {key}(DOI で引き直すか、題名の語を減らす)")
for w in hits:
diff --git a/skills/imozuru/scripts/fetch.py b/skills/imozuru/scripts/fetch.py
index b521ef9..fbd68ed 100755
--- a/skills/imozuru/scripts/fetch.py
+++ b/skills/imozuru/scripts/fetch.py
@@ -69,9 +69,22 @@ class _Redirect308(urllib.request.HTTPRedirectHandler):
_OPENER = urllib.request.build_opener(_Redirect308)
+def ascii_url(url: str) -> str:
+ """パスとクエリにそのまま入った日本語の字などを % で符号化する。
+
+ urllib は要求の行を ASCII で組むので、辞書の項目名や自治体のファイル名を符号化せずに
+ 書いたリンクは UnicodeEncodeError で止まる(和竿の漆塗りの試しで 36 本中 2 本)。
+ safe に % を入れるので、符号化済みの URL は変わらない。国際化ドメイン名は扱わない。
+ """
+ p = urllib.parse.urlsplit(url)
+ path = urllib.parse.quote(p.path, safe="/%:@!$&'()*+,;=~")
+ query = urllib.parse.quote(p.query, safe="=&%/:+?@!$'()*,;~")
+ return urllib.parse.urlunsplit((p.scheme, p.netloc, path, query, p.fragment))
+
+
def download(url: str) -> tuple[bytes, str]:
"""URL を取りに行き、本体と Content-Type を返す。"""
- req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 imozuru"})
+ req = urllib.request.Request(ascii_url(url), headers={"User-Agent": "Mozilla/5.0 imozuru"})
with _OPENER.open(req, timeout=90) as res:
return res.read(MAX_BYTES), res.headers.get("Content-Type", "")
diff --git a/tests/test_chain.py b/tests/test_chain.py
new file mode 100644
index 0000000..9a6b0b8
--- /dev/null
+++ b/tests/test_chain.py
@@ -0,0 +1,53 @@
+"""chain.py が、キーを OpenAlex の ID・DOI・題名のどれと見なすかを検査する。"""
+
+import contextlib
+import importlib.util
+import io
+import sys
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+
+SCRIPT = Path(__file__).resolve().parents[1] / "skills/imozuru/scripts/chain.py"
+SPEC = importlib.util.spec_from_file_location("imozuru_chain", SCRIPT)
+chain = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(chain)
+
+WORK = {"id": "https://openalex.org/W2073651707", "display_name": "Weak Bourbaki unmixed rings",
+ "publication_year": 2004, "cited_by_count": 10, "doi": None, "best_oa_location": None}
+
+
+class KeyKindTests(unittest.TestCase):
+ """「W」で始まる題名を ID と取り違えない(イシュー #7)。"""
+
+ def paths(self, *argv):
+ seen = []
+
+ def get(path, **params):
+ seen.append(path)
+ return WORK if path.startswith("/works/") else {"results": [WORK]}
+
+ with patch.object(chain, "get", side_effect=get), \
+ patch.object(sys, "argv", ["chain.py", *argv]), \
+ contextlib.redirect_stdout(io.StringIO()):
+ chain.main()
+ return seen
+
+ def test_title_starting_with_w_is_searched(self):
+ self.assertEqual(self.paths("find", "Weak", "Bourbaki", "unmixed", "rings"), ["/works"])
+ with patch.object(chain, "get", return_value={"results": [WORK]}) as get:
+ self.assertEqual(chain.resolve("What is exploratory search"), WORK)
+ self.assertEqual(get.call_args.args[0], "/works")
+
+ def test_work_id_is_resolved_directly(self):
+ self.assertEqual(self.paths("find", "W2073651707"), ["/works/W2073651707"])
+ self.assertEqual(self.paths("find", "https://openalex.org/W2073651707"), ["/works/W2073651707"])
+
+ def test_doi_is_resolved_directly(self):
+ self.assertEqual(self.paths("find", "10.1016/j.jalgebra.2004.01.001"),
+ ["/works/doi:10.1016/j.jalgebra.2004.01.001"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_fetch.py b/tests/test_fetch.py
index b860716..a833662 100644
--- a/tests/test_fetch.py
+++ b/tests/test_fetch.py
@@ -301,5 +301,35 @@ class CjkPdfFloorTests(unittest.TestCase):
self.assertTrue(fetch.mostly_cjk("全角の123"))
+class NonAsciiUrlTests(unittest.TestCase):
+ """日本語の字をそのまま含む URL も、符号化して取りに行く(イシュー #9)。"""
+
+ def requested(self, url):
+ seen = []
+
+ def open_(req, timeout):
+ seen.append(req.full_url)
+ res = unittest.mock.MagicMock()
+ res.__enter__.return_value.read.return_value = b"body"
+ return res
+
+ with patch.object(fetch._OPENER, "open", side_effect=open_):
+ fetch.download(url)
+ return seen[0]
+
+ def test_japanese_path_and_query_are_encoded(self):
+ self.assertEqual(self.requested("https://kotobank.jp/word/漆工芸"),
+ "https://kotobank.jp/word/%E6%BC%86%E5%B7%A5%E8%8A%B8")
+ self.assertEqual(self.requested("https://example.test/search?q=和竿&page=2"),
+ "https://example.test/search?q=%E5%92%8C%E7%AB%BF&page=2")
+
+ def test_encoded_url_is_unchanged(self):
+ for url in ("https://kotobank.jp/word/%E6%BC%86%E5%B7%A5%E8%8A%B8",
+ "https://www.youtube.com/watch?v=jPCV4GKX9Dw&t=30s",
+ "https://example.test/a%20b/c?x=1+2&y=%2F"):
+ with self.subTest(url=url):
+ self.assertEqual(self.requested(url), url)
+
+
if __name__ == "__main__":
unittest.main()