Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

refactor(metadata): 共享领域层的资料源连接检查

changjing455ecea7cc02
diff --git a/src/peach/source_checks.py b/src/peach/source_checks.py
new file mode 100644
index 000000000..0dc4a774e
--- /dev/null
+++ b/src/peach/source_checks.py
@@ -0,0 +1,103 @@
+"""资料源页面与封面 CDN 的有界连接检查;Web 与安装诊断共用。"""
+import io
+import time
+import httpx
+from PIL import Image
+from .http import HttpRequest
+from .scraping_access import SOURCES, SourcePaused, SourceTransport, record_source_check
+
+MOBILE_BROADBAND_HINT = "中国移动宽带直连 DMM 图片主机常在握手后被断开,请把 DMM / FANZA 的连接方式设成 Peach 代理。"
+
+
+def _connection_failure(exc, label):
+ """来源连接异常对应可操作的诊断说明。"""
+ if isinstance(exc, SourcePaused):
+ detail, message = "cooldown", "来源正在冷却,请稍后重试。"
+ elif isinstance(exc, httpx.ProxyError):
+ detail, message = "proxy", "代理连接失败,请检查 Peach 代理。"
+ elif isinstance(exc, httpx.TimeoutException):
+ detail, message = "timeout", "来源连接超时,请稍后重试或检查连接方式。"
+ else:
+ detail = "network"
+ message = (f"连接未取得;{MOBILE_BROADBAND_HINT}" if label == "高清图片 CDN"
+ else "连接未取得;此来源可能需要代理,请检查来源连接方式。")
+ return {"ok": False, "kind": "unavailable", "reason": detail,
+ "message": message, "error_type": type(exc).__name__}
+
+
+def _connection_message(detail, reason):
+ """来源响应的失败原因决定用户下一步操作。"""
+ from .metadata import reason_blames_credentials
+
+ if detail == "cloudflare_challenge":
+ return "来源返回人机验证页,请检查来源验证状态或更换出口。"
+ if detail == "rate_limited":
+ return "来源正在限流,请稍后重试。"
+ if detail == "not_found":
+ return "来源检查地址不存在,请核对站点地址。"
+ if reason_blames_credentials(reason):
+ return "来源要求登录或验证,请在官网完成登录后重试。"
+ return f"{reason}。" if reason else "来源暂不可用,请稍后重试。"
+
+
+def check_source(secrets_root, source, *, preflight=False):
+ """复用来源连接与图片 CDN 判据;安装检查只使用有界 HTTP 请求。"""
+ from .metadata import auth_wall_reason
+ if source not in SOURCES:
+ raise ValueError("未知采集来源")
+ if not SOURCES[source]["login"]:
+ return {"ok": True, "results": [{"label": "来源页面", "ok": False, "kind": "unavailable",
+ "reason": "needs_code", "message": "片商官网按番号选择,请输入馆藏番号检查高清封面。"}],
+ "session_verified": False}
+ targets = [("来源页面", SOURCES[source]["login"])]
+ if source == "dmm":
+ targets.append(("高清图片 CDN", "https://awsimgsrc.dmm.co.jp/pics_dig/digital/video/gyan00017/gyan00017pl.jpg"))
+ results = []
+ transport = (SourceTransport(secrets_root, max_requests=12, max_bytes=12 * 65536,
+ max_seconds=15, allow_browser=False)
+ if preflight else SourceTransport(secrets_root))
+ try:
+ for label, url in targets:
+ result = {"label": label, "ok": False}
+ started = time.monotonic()
+ try:
+ response = transport(HttpRequest("GET", url, {"Range": "bytes=0-65535"}), 10, 65536)
+ # 鉴权失败与「站点挂了」是两件事,页面上要分得开:前者要人去官网登录
+ # 或换一份 Cookie,后者只要等。判据和元数据来源共用一份;重定向终点
+ # 只在真的跳转过时才算数,否则配置里本身就指向登录页的来源会自判失败。
+ reason = auth_wall_reason(
+ status_code=response.status, body=response.body,
+ final_url=response.url if response.url and response.url != url else "")
+ result.update(status=response.status, bytes=len(response.body),
+ ok=response.status in {200, 206} and not reason,
+ kind="ok" if response.status in {200, 206} and not reason
+ else "auth" if reason else "unavailable")
+ from .browser_transport import CHALLENGE_MARKS
+ challenge = any(mark.lower() in response.body.decode("utf-8", errors="replace").lower()
+ for mark in CHALLENGE_MARKS)
+ detail = ("cloudflare_challenge" if challenge else "auth_required" if reason
+ else "ok" if result["ok"] else "rate_limited" if response.status == 429
+ else "server_error" if response.status >= 500
+ else "not_found" if response.status == 404 else "http_error")
+ result["reason"] = detail
+ if challenge:
+ result.update(ok=False, kind="auth")
+ if label == "高清图片 CDN" and result["ok"]:
+ with Image.open(io.BytesIO(response.body)) as image:
+ result["width"], result["height"] = image.size
+ if not result["ok"]:
+ # 成因锁死在凭据上才让人去登录。只剩状态码可看的 403 直接把
+ # `auth_wall_reason` 那句原样给用户:出口 IP 被封时登录是白做,
+ # 页面上再手写一份「请去登录」等于把人引去做错事。
+ result["message"] = _connection_message(detail, reason)
+ except Exception as exc:
+ result.update(_connection_failure(exc, label))
+ result.update(checked_at=time.time(), elapsed_s=round(time.monotonic() - started, 2))
+ if label == "来源页面":
+ record_source_check(secrets_root, source, "connection", result)
+ results.append(result)
+ finally:
+ transport.close()
+ return {"ok": True, "results": results, "session_verified": False}
+
+
diff --git a/src/peach/source_preflight.py b/src/peach/source_preflight.py
index e06edfba7..c74c5a47f 100644
--- a/src/peach/source_preflight.py
+++ b/src/peach/source_preflight.py
@@ -6,7 +6,7 @@ from pathlib import Path
import time
from .scraping_access import SOURCES
-from .web_scraping import check_source
+from .source_checks import check_source
def check_sources(secrets_root: Path, sources: list[str] | None = None) -> dict:
diff --git a/src/peach/web_scraping.py b/src/peach/web_scraping.py
index 337cb27dc..cabb52eb2 100644
--- a/src/peach/web_scraping.py
+++ b/src/peach/web_scraping.py
@@ -1,5 +1,4 @@
"""采集设置与有界来源连接检查;凭据只写本机。"""
-import io
import hashlib
import json
import time
@@ -7,12 +6,9 @@ import httpx
from PIL import Image
-from .http import HttpRequest
-from .scraping_access import SOURCES, SourcePaused, SourceTransport, describe, save, record_source_check
+from .scraping_access import SOURCES, SourcePaused, SourceTransport, describe, save
-#: 连不上 DMM 图片主机时给的那句去处:2026-09 实测中国移动宽带直连大多在握手后被断开,
-#: 电信、联通直连正常,所以不是站点挂了,换条线路就好。
-MOBILE_BROADBAND_HINT = "中国移动宽带直连 DMM 图片主机常在握手后被断开,请把 DMM / FANZA 的连接方式设成 Peach 代理。"
+from .source_checks import MOBILE_BROADBAND_HINT, check_source
def _kept_cover_size(target, incoming, incoming_size, installed, previous):
@@ -158,102 +154,10 @@ def w_scraping_settings(contract, body):
return {"ok": True, "saved": save(contract.follow_secrets_root, str(body.get("source", "")), body)}
-def _connection_failure(exc, label):
- """来源连接异常对应可操作的诊断说明。"""
- if isinstance(exc, SourcePaused):
- detail, message = "cooldown", "来源正在冷却,请稍后重试。"
- elif isinstance(exc, httpx.ProxyError):
- detail, message = "proxy", "代理连接失败,请检查 Peach 代理。"
- elif isinstance(exc, httpx.TimeoutException):
- detail, message = "timeout", "来源连接超时,请稍后重试或检查连接方式。"
- else:
- detail = "network"
- message = (f"连接未取得;{MOBILE_BROADBAND_HINT}" if label == "高清图片 CDN"
- else "连接未取得;此来源可能需要代理,请检查来源连接方式。")
- return {"ok": False, "kind": "unavailable", "reason": detail,
- "message": message, "error_type": type(exc).__name__}
-
-
-def _connection_message(detail, reason):
- """来源响应的失败原因决定用户下一步操作。"""
- from .metadata import reason_blames_credentials
-
- if detail == "cloudflare_challenge":
- return "来源返回人机验证页,请检查来源验证状态或更换出口。"
- if detail == "rate_limited":
- return "来源正在限流,请稍后重试。"
- if detail == "not_found":
- return "来源检查地址不存在,请核对站点地址。"
- if reason_blames_credentials(reason):
- return "来源要求登录或验证,请在官网完成登录后重试。"
- return f"{reason}。" if reason else "来源暂不可用,请稍后重试。"
-
-
def w_scraping_check(contract, body):
return check_source(contract.follow_secrets_root, str(body.get("source", "")))
-def check_source(secrets_root, source, *, preflight=False):
- """复用来源连接与图片 CDN 判据;安装检查只使用有界 HTTP 请求。"""
- from .metadata import auth_wall_reason
- if source not in SOURCES:
- raise ValueError("未知采集来源")
- if not SOURCES[source]["login"]:
- return {"ok": True, "results": [{"label": "来源页面", "ok": False, "kind": "unavailable",
- "reason": "needs_code", "message": "片商官网按番号选择,请输入馆藏番号检查高清封面。"}],
- "session_verified": False}
- targets = [("来源页面", SOURCES[source]["login"])]
- if source == "dmm":
- targets.append(("高清图片 CDN", "https://awsimgsrc.dmm.co.jp/pics_dig/digital/video/gyan00017/gyan00017pl.jpg"))
- results = []
- transport = (SourceTransport(secrets_root, max_requests=12, max_bytes=12 * 65536,
- max_seconds=15, allow_browser=False)
- if preflight else SourceTransport(secrets_root))
- try:
- for label, url in targets:
- result = {"label": label, "ok": False}
- started = time.monotonic()
- try:
- response = transport(HttpRequest("GET", url, {"Range": "bytes=0-65535"}), 10, 65536)
- # 鉴权失败与「站点挂了」是两件事,页面上要分得开:前者要人去官网登录
- # 或换一份 Cookie,后者只要等。判据和元数据来源共用一份;重定向终点
- # 只在真的跳转过时才算数,否则配置里本身就指向登录页的来源会自判失败。
- reason = auth_wall_reason(
- status_code=response.status, body=response.body,
- final_url=response.url if response.url and response.url != url else "")
- result.update(status=response.status, bytes=len(response.body),
- ok=response.status in {200, 206} and not reason,
- kind="ok" if response.status in {200, 206} and not reason
- else "auth" if reason else "unavailable")
- from .browser_transport import CHALLENGE_MARKS
- challenge = any(mark.lower() in response.body.decode("utf-8", errors="replace").lower()
- for mark in CHALLENGE_MARKS)
- detail = ("cloudflare_challenge" if challenge else "auth_required" if reason
- else "ok" if result["ok"] else "rate_limited" if response.status == 429
- else "server_error" if response.status >= 500
- else "not_found" if response.status == 404 else "http_error")
- result["reason"] = detail
- if challenge:
- result.update(ok=False, kind="auth")
- if label == "高清图片 CDN" and result["ok"]:
- with Image.open(io.BytesIO(response.body)) as image:
- result["width"], result["height"] = image.size
- if not result["ok"]:
- # 成因锁死在凭据上才让人去登录。只剩状态码可看的 403 直接把
- # `auth_wall_reason` 那句原样给用户:出口 IP 被封时登录是白做,
- # 页面上再手写一份「请去登录」等于把人引去做错事。
- result["message"] = _connection_message(detail, reason)
- except Exception as exc:
- result.update(_connection_failure(exc, label))
- result.update(checked_at=time.time(), elapsed_s=round(time.monotonic() - started, 2))
- if label == "来源页面":
- record_source_check(secrets_root, source, "connection", result)
- results.append(result)
- finally:
- transport.close()
- return {"ok": True, "results": results, "session_verified": False}
-
-
def q_scraping_amane_bridge(contract, args):
"""amane 桥那张卡:钉的 sha、锁里的版本、venv 建没建、开了哪几站,外加重建任务的快照。"""
from . import metadata_amane
diff --git a/tests/test_scraping_access.py b/tests/test_scraping_access.py
index 27484e9e2..d6116de62 100644
--- a/tests/test_scraping_access.py
+++ b/tests/test_scraping_access.py
@@ -168,7 +168,7 @@ class ScrapingAccessTests(unittest.TestCase):
def test_connections_do_not_claim_session_validity_or_echo_exception_urls(self):
contract = SimpleNamespace(follow_secrets_root=self.root)
- with patch("peach.web_scraping.SourceTransport") as factory:
+ with patch("peach.source_checks.SourceTransport") as factory:
factory.return_value.side_effect = httpx.ConnectError("https://secret:[email protected]/")
result = w_scraping_check(contract, {"source": "fc2cmadb"})
self.assertFalse(result["session_verified"])
@@ -193,7 +193,7 @@ class ScrapingAccessTests(unittest.TestCase):
(HttpResponse(200, {}, b"<html>ok</html>", login), "ok", True, "", "登录"))
for response, kind, ok, says, avoids in cases:
with self.subTest(status=response.status, url=response.url), \
- patch("peach.web_scraping.SourceTransport") as factory:
+ patch("peach.source_checks.SourceTransport") as factory:
factory.return_value.return_value = response
result = w_scraping_check(contract, {"source": "fc2cmadb"})["results"][0]
self.assertEqual((result["kind"], result["ok"]), (kind, ok))
@@ -218,7 +218,7 @@ class ScrapingAccessTests(unittest.TestCase):
(HttpResponse(200, {}, b"<title>Just a moment</title>", "https://airav.io/"), "cloudflare_challenge"),
]
for outcome, reason in cases:
- with self.subTest(reason=reason), patch("peach.web_scraping.SourceTransport") as factory:
+ with self.subTest(reason=reason), patch("peach.source_checks.SourceTransport") as factory:
if isinstance(outcome, Exception):
factory.return_value.side_effect = outcome
else:
@@ -602,7 +602,7 @@ class ScrapingAccessTests(unittest.TestCase):
result = _fetch_cover(contract, "ABW-232")
self.assertEqual(result["reason"], "network")
self.assertIn(MOBILE_BROADBAND_HINT, result["error"])
- with patch("peach.web_scraping.SourceTransport") as factory:
+ with patch("peach.source_checks.SourceTransport") as factory:
factory.return_value.side_effect = httpx.ConnectError("EOF occurred in violation of protocol")
checks = {item["label"]: item for item in
w_scraping_check(SimpleNamespace(follow_secrets_root=self.root), {"source": "dmm"})["results"]}
@@ -817,10 +817,10 @@ class SourcePreflightTests(unittest.TestCase):
checked.assert_not_called()
def test_preflight_bounds_requests_and_uses_no_browser(self):
- from peach.web_scraping import check_source
+ from peach.source_checks import check_source
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
- with patch('peach.web_scraping.SourceTransport') as factory:
+ with patch('peach.source_checks.SourceTransport') as factory:
factory.return_value.return_value = HttpResponse(200, {}, b'<html></html>')
result = check_source(root, 'javdb', preflight=True)
self.assertTrue(result['results'][0]['ok'])