Peach: Local-first video metadata scraper and media manager.
feat(metadata): 提供安装后的资料源连接检测
changjing622cdeebe393
diff --git a/docs/DOCKER.md b/docs/DOCKER.mdindex a757a7d0d..a50c8062f 100644--- a/docs/DOCKER.md+++ b/docs/DOCKER.md@@ -169,3 +169,16 @@ Linux 容器不读取用户电脑上的浏览历史,也不包含交互式浏GitHub Actions 的 `Docker` 工作流先构建并运行临时卷验收,通过后发布。仓库变量 `DOCKERHUB_USERNAME` 和机密 `DOCKERHUB_TOKEN` 用于 Docker Hub 发布;机密只保存在 GitHub,不进入项目文件。未配置发布身份时仍执行构建验收。ARM 镜像需通过同等运行验收后再加入发布。Docker Hub 页面以英文为主,概览正文取自 [docs/dockerhub/README.md](dockerhub/README.md),链接回 GitHub 的源码、本文和中文 README。简介填写 `Local-first video metadata scraper and media manager. Self-hosted web UI for VPS and NAS.`。发布流水线只推送镜像,概览与简介需要在 Docker Hub 仓库设置中手动更新;镜像、端口、目录或 Compose 示例变化时同批修改该文件。++## 安装后检查资料源与封面连接++完成首次设置、保存代理或来源 Cookie 后,在容器管理器的终端执行:++```bash+python -m peach check-sources+python -m peach check-sources --source dmm --source javdb --json+```++检查在 NAS 容器内发起,使用该实例已有的来源网络设置、Cookie 与冷却记录。最多四个来源并发,每个来源限制请求次数、读取量与时间预算;只读取来源页面和 DMM 图片 CDN 样本,不读取影片、不启动桌面浏览器。结果保留到现有采集设置的来源连接记录;限流或拒绝访问沿用现有冷却机制,不自动关闭来源或修改代理。++报告逐项区分可连接、登录或人机验证、HTTP 错误、冷却与网络失败。页面能连接仅说明检查地址可达,不代表全部作品资料、封面或登录会话可用;片商官网需要番号时标记为未检查,不计为连接失败。其余目标未通过时退出码为 1。更换 NAS 出口或代理后可再次运行;`--source` 可重复指定,仅重查关心的来源。不要在短时间内反复检查正在冷却的来源。diff --git a/src/peach/cli.py b/src/peach/cli.pyindex 77802fe39..a2bb43781 100644--- a/src/peach/cli.py+++ b/src/peach/cli.py@@ -754,6 +754,23 @@ def _doctor(args: argparse.Namespace) -> int:return 1 if result["status"] == "failed" else 0++def _check_sources(args: argparse.Namespace) -> int:+ import json+ from .source_preflight import check_sources++ result = check_sources(settings_file.active().directory("secrets"), args.source)+ if args.json:+ print(json.dumps(result, ensure_ascii=False))+ else:+ print("检查来源页面与图片 CDN;连通不代表作品资料或登录会话已验证。")+ for source in result["sources"]:+ for target in source["results"]:+ state = "可连接" if target["ok"] else target.get("reason", "未取得")+ print(f"{source['label']} / {target['label']} [{state}] {target.get('message', '')}")+ return 0 if result["ok"] else 1++def build_parser() -> argparse.ArgumentParser:parser = argparse.ArgumentParser(prog="peach")commands = parser.add_subparsers(dest="command", required=True)@@ -853,6 +870,11 @@ def build_parser() -> argparse.ArgumentParser:status.add_argument("--shared-db", type=Path, default=SHARED_DATABASE_PATH)status.set_defaults(handler=_status)+ sources = commands.add_parser("check-sources", help="安装后有界检测外部资料源与图片 CDN,不读取影片")+ sources.add_argument("--source", action="append", help="只检查指定来源 ID;可重复指定,默认检查全部资料源")+ sources.add_argument("--json", action="store_true", help="输出不含凭据的 JSON 结果")+ sources.set_defaults(handler=_check_sources)+doctor = commands.add_parser("doctor", help="逐项诊断本机配置、数据库、工具与挂载,不执行修复")doctor.add_argument("--json", action="store_true", help="输出脱敏 JSON 报告")doctor.add_argument("--db", type=Path, default=DEFAULT_DB)diff --git a/src/peach/scraping_access.py b/src/peach/scraping_access.pyindex 567525495..cc51a66a3 100644--- a/src/peach/scraping_access.py+++ b/src/peach/scraping_access.py@@ -404,8 +404,9 @@ class SourceTransport:"""公开采集按来源复用连接池;登录会话由特定来源消费者显式建立。"""def __init__(self, secrets_root: Path, *, max_requests: int = 0,- max_bytes: int = 0, max_seconds: float = 0):+ max_bytes: int = 0, max_seconds: float = 0, allow_browser: bool = True):self.root = secrets_root+ self.allow_browser = allow_browserself.transports: dict[str | None, HttpxTransport] = {}self.max_requests, self.max_bytes = max_requests, max_bytesfrom .processing_control import deadline_after@@ -482,7 +483,7 @@ class SourceTransport:SOURCES[source]["blocked_pause"]), blocks, via)raise SourcePaused("来源的人机验证没有在限时内通过,暂停向它请求一段时间;""浏览器窗口再弹出时点一下验证即可;已有图片保留")- if via == VIA_HTTP and source and SOURCES[source].get("browser_fallback") and _refused(response):+ if self.allow_browser and via == VIA_HTTP and source and SOURCES[source].get("browser_fallback") and _refused(response):verified = self._verify_in_browser(source, request, timeout, max_bytes)if verified is not None:# HTTP 客户端那条路攒下的账不压到浏览器上:清掉,之后按浏览器那条路重新数。@@ -527,7 +528,7 @@ class SourceTransport:def _via(self, source: str | None) -> str:"""这个来源这次走哪条路。该走浏览器的来源要先选出传输才知道(不发请求、不起进程)。"""- if source and source not in self.transports and wants_browser(self.root, source):+ if self.allow_browser and source and source not in self.transports and wants_browser(self.root, source):self.transports[source] = self._transport_for(source)transport = self.transports.get(source)return VIA_BROWSER if isinstance(transport, browser_transport.BrowserTransport) else VIA_HTTP@@ -572,7 +573,7 @@ class SourceTransport:def _transport_for(self, source: str | None):"""这个来源的请求走哪条路:该走浏览器(`wants_browser`)且本机有浏览器就走浏览器页面,否则 httpx。"""- if source and wants_browser(self.root, source):+ if self.allow_browser and source and wants_browser(self.root, source):browser = self._browser_for(source)if browser is not None:return browserdiff --git a/src/peach/source_preflight.py b/src/peach/source_preflight.pynew file mode 100644index 000000000..e06edfba7--- /dev/null+++ b/src/peach/source_preflight.py@@ -0,0 +1,28 @@+"""安装后的资料源检查;不扫描影片,不改变来源配置。"""+from __future__ import annotations++from concurrent.futures import ThreadPoolExecutor+from pathlib import Path+import time++from .scraping_access import SOURCES+from .web_scraping import check_source+++def check_sources(secrets_root: Path, sources: list[str] | None = None) -> dict:+ selected = list(dict.fromkeys(sources if sources is not None else+ (source for source in SOURCES if source != "instagram")))+ if not selected or any(source not in SOURCES for source in selected):+ raise ValueError("请选择已登记的资料源 ID")++ def check(source):+ result = check_source(secrets_root, source, preflight=True)+ return {"source": source, "label": SOURCES[source]["label"],+ "results": result["results"], "session_verified": False}++ started = time.monotonic()+ with ThreadPoolExecutor(max_workers=4, thread_name_prefix="source-check") as pool:+ results = list(pool.map(check, selected))+ return {"ok": all(target["ok"] or target.get("reason") == "needs_code"+ for source in results for target in source["results"]),+ "sources": results, "elapsed_s": round(time.monotonic() - started, 2)}diff --git a/src/peach/web_scraping.py b/src/peach/web_scraping.pyindex 46933122c..337cb27dc 100644--- a/src/peach/web_scraping.py+++ b/src/peach/web_scraping.py@@ -190,8 +190,12 @@ def _connection_message(detail, reason):def w_scraping_check(contract, body):+ return check_source(contract.follow_secrets_root, str(body.get("source", "")))+++def check_source(secrets_root, source, *, preflight=False):+ """复用来源连接与图片 CDN 判据;安装检查只使用有界 HTTP 请求。"""from .metadata import auth_wall_reason- source = str(body.get("source", ""))if source not in SOURCES:raise ValueError("未知采集来源")if not SOURCES[source]["login"]:@@ -202,7 +206,9 @@ def w_scraping_check(contract, body):if source == "dmm":targets.append(("高清图片 CDN", "https://awsimgsrc.dmm.co.jp/pics_dig/digital/video/gyan00017/gyan00017pl.jpg"))results = []- transport = SourceTransport(contract.follow_secrets_root)+ transport = (SourceTransport(secrets_root, max_requests=12, max_bytes=12 * 65536,+ max_seconds=15, allow_browser=False)+ if preflight else SourceTransport(secrets_root))try:for label, url in targets:result = {"label": label, "ok": False}@@ -241,7 +247,7 @@ def w_scraping_check(contract, body):result.update(_connection_failure(exc, label))result.update(checked_at=time.time(), elapsed_s=round(time.monotonic() - started, 2))if label == "来源页面":- record_source_check(contract.follow_secrets_root, source, "connection", result)+ record_source_check(secrets_root, source, "connection", result)results.append(result)finally:transport.close()diff --git a/tests/test_cli.py b/tests/test_cli.pyindex f7cec3b4e..3192f02b7 100644--- a/tests/test_cli.py+++ b/tests/test_cli.py@@ -716,3 +716,13 @@ class PackagedEntryTests(unittest.TestCase):if __name__ == "__main__":unittest.main()+++class SourceChecksCliTests(unittest.TestCase):+ def test_source_check_selection_json_and_failed_exit_status(self):+ result = {'ok': False, 'sources': [], 'elapsed_s': 0}+ with mock.patch('peach.source_preflight.check_sources', return_value=result) as check, \+ mock.patch('peach.cli.settings_file.active') as active, redirect_stdout(io.StringIO()) as output:+ self.assertEqual(cli.main(['check-sources', '--source', 'dmm', '--source', 'javdb', '--json']), 1)+ check.assert_called_once_with(active.return_value.directory.return_value, ['dmm', 'javdb'])+ self.assertEqual(__import__('json').loads(output.getvalue()), result)diff --git a/tests/test_scraping_access.py b/tests/test_scraping_access.pyindex 28acb0e23..27484e9e2 100644--- a/tests/test_scraping_access.py+++ b/tests/test_scraping_access.py@@ -795,3 +795,50 @@ class BrowserFallbackTests(unittest.TestCase):with patch("peach.browser_transport.shared", side_effect=AssertionError("avwikidb 不走浏览器")), \self.assertRaises(SourcePaused):transport(HttpRequest("GET", "https://avwikidb.com/actor/x/", {}), 1, 100)+++class SourcePreflightTests(unittest.TestCase):+ def test_batch_deduplicates_and_preserves_target_failures(self):+ from peach.source_preflight import check_sources+ def check(root, source, **options):+ self.assertTrue(options['preflight'])+ return {'results': [{'label': '来源页面', 'ok': source == 'dmm'},+ {'label': '高清图片 CDN', 'ok': False, 'reason': 'network'}]}+ with patch('peach.source_preflight.check_source', side_effect=check) as checked:+ result = check_sources(Path('/unused'), ['dmm', 'javdb', 'dmm'])+ self.assertEqual(checked.call_count, 2)+ self.assertEqual([row['source'] for row in result['sources']], ['dmm', 'javdb'])+ self.assertFalse(result['ok'])+ self.assertTrue(all(not row['session_verified'] for row in result['sources']))+ with patch('peach.source_preflight.check_source') as checked:+ for selected in ([], ['unknown'], ['dmm', 'unknown']):+ with self.assertRaises(ValueError):+ check_sources(Path('/unused'), selected)+ checked.assert_not_called()++ def test_preflight_bounds_requests_and_uses_no_browser(self):+ from peach.web_scraping import check_source+ with tempfile.TemporaryDirectory() as temporary:+ root = Path(temporary)+ with patch('peach.web_scraping.SourceTransport') as factory:+ factory.return_value.return_value = HttpResponse(200, {}, b'<html></html>')+ result = check_source(root, 'javdb', preflight=True)+ self.assertTrue(result['results'][0]['ok'])+ factory.assert_called_once_with(root, max_requests=12, max_bytes=12 * 65536,+ max_seconds=15, allow_browser=False)+ factory.return_value.close.assert_called_once()++ def test_http_only_transport_does_not_launch_or_fix_browser_route(self):+ from peach.scraping_access import SourcePaused, fix_to_browser, fixed_to_browser+ with tempfile.TemporaryDirectory() as temporary:+ root = Path(temporary)+ fix_to_browser(root, 'minnano-av')+ transport = SourceTransport(root, allow_browser=False)+ with patch.object(transport, '_browser_for', side_effect=AssertionError('不能启动浏览器')), \+ patch('peach.scraping_access.client_for') as client:+ client.return_value = httpx.Client(transport=httpx.MockTransport(+ lambda request: httpx.Response(403, content=b'blocked')))+ with self.assertRaises(SourcePaused):+ transport(HttpRequest('GET', 'https://www.minnano-av.com/', {}), 1, 1024)+ transport.close()+ self.assertTrue(fixed_to_browser(root, 'minnano-av'))