Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

feat(metadata): 提供安装后的资料源连接检测

changjing622cdeebe393
diff --git a/docs/DOCKER.md b/docs/DOCKER.md
index a757a7d0d..a50c8062f 100644
--- a/docs/DOCKER.md
+++ b/docs/DOCKER.md
@@ -169,3 +169,16 @@ Linux 容器不读取用户电脑上的浏览历史,也不包含交互式浏
GitHub Actions 的 `Docker` 工作流先构建并运行临时卷验收,通过后发布。仓库变量 `DOCKERHUB_USERNAME` 和机密 `DOCKERHUB_TOKEN` 用于 Docker Hub 发布;机密只保存在 GitHub,不进入项目文件。未配置发布身份时仍执行构建验收。ARM 镜像需通过同等运行验收后再加入发布。
Docker Hub 页面以英文为主,概览正文取自 [docs/dockerhub/README.md](dockerhub/README.md),链接回 GitHub 的源码、本文和中文 README。简介填写 `Local-first video metadata scraper and media manager. Self-hosted web UI for VPS and NAS.`。发布流水线只推送镜像,概览与简介需要在 Docker Hub 仓库设置中手动更新;镜像、端口、目录或 Compose 示例变化时同批修改该文件。
+
+## 安装后检查资料源与封面连接
+
+完成首次设置、保存代理或来源 Cookie 后,在容器管理器的终端执行:
+
+```bash
+python -m peach check-sources
+python -m peach check-sources --source dmm --source javdb --json
+```
+
+检查在 NAS 容器内发起,使用该实例已有的来源网络设置、Cookie 与冷却记录。最多四个来源并发,每个来源限制请求次数、读取量与时间预算;只读取来源页面和 DMM 图片 CDN 样本,不读取影片、不启动桌面浏览器。结果保留到现有采集设置的来源连接记录;限流或拒绝访问沿用现有冷却机制,不自动关闭来源或修改代理。
+
+报告逐项区分可连接、登录或人机验证、HTTP 错误、冷却与网络失败。页面能连接仅说明检查地址可达,不代表全部作品资料、封面或登录会话可用;片商官网需要番号时标记为未检查,不计为连接失败。其余目标未通过时退出码为 1。更换 NAS 出口或代理后可再次运行;`--source` 可重复指定,仅重查关心的来源。不要在短时间内反复检查正在冷却的来源。
diff --git a/src/peach/cli.py b/src/peach/cli.py
index 77802fe39..a2bb43781 100644
--- a/src/peach/cli.py
+++ b/src/peach/cli.py
@@ -754,6 +754,23 @@ def _doctor(args: argparse.Namespace) -> int:
return 1 if result["status"] == "failed" else 0
+
+def _check_sources(args: argparse.Namespace) -> int:
+ import json
+ from .source_preflight import check_sources
+
+ result = check_sources(settings_file.active().directory("secrets"), args.source)
+ if args.json:
+ print(json.dumps(result, ensure_ascii=False))
+ else:
+ print("检查来源页面与图片 CDN;连通不代表作品资料或登录会话已验证。")
+ for source in result["sources"]:
+ for target in source["results"]:
+ state = "可连接" if target["ok"] else target.get("reason", "未取得")
+ print(f"{source['label']} / {target['label']} [{state}] {target.get('message', '')}")
+ return 0 if result["ok"] else 1
+
+
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="peach")
commands = parser.add_subparsers(dest="command", required=True)
@@ -853,6 +870,11 @@ def build_parser() -> argparse.ArgumentParser:
status.add_argument("--shared-db", type=Path, default=SHARED_DATABASE_PATH)
status.set_defaults(handler=_status)
+ sources = commands.add_parser("check-sources", help="安装后有界检测外部资料源与图片 CDN,不读取影片")
+ sources.add_argument("--source", action="append", help="只检查指定来源 ID;可重复指定,默认检查全部资料源")
+ sources.add_argument("--json", action="store_true", help="输出不含凭据的 JSON 结果")
+ sources.set_defaults(handler=_check_sources)
+
doctor = commands.add_parser("doctor", help="逐项诊断本机配置、数据库、工具与挂载,不执行修复")
doctor.add_argument("--json", action="store_true", help="输出脱敏 JSON 报告")
doctor.add_argument("--db", type=Path, default=DEFAULT_DB)
diff --git a/src/peach/scraping_access.py b/src/peach/scraping_access.py
index 567525495..cc51a66a3 100644
--- a/src/peach/scraping_access.py
+++ b/src/peach/scraping_access.py
@@ -404,8 +404,9 @@ class SourceTransport:
"""公开采集按来源复用连接池;登录会话由特定来源消费者显式建立。"""
def __init__(self, secrets_root: Path, *, max_requests: int = 0,
- max_bytes: int = 0, max_seconds: float = 0):
+ max_bytes: int = 0, max_seconds: float = 0, allow_browser: bool = True):
self.root = secrets_root
+ self.allow_browser = allow_browser
self.transports: dict[str | None, HttpxTransport] = {}
self.max_requests, self.max_bytes = max_requests, max_bytes
from .processing_control import deadline_after
@@ -482,7 +483,7 @@ class SourceTransport:
SOURCES[source]["blocked_pause"]), blocks, via)
raise SourcePaused("来源的人机验证没有在限时内通过,暂停向它请求一段时间;"
"浏览器窗口再弹出时点一下验证即可;已有图片保留")
- if via == VIA_HTTP and source and SOURCES[source].get("browser_fallback") and _refused(response):
+ if self.allow_browser and via == VIA_HTTP and source and SOURCES[source].get("browser_fallback") and _refused(response):
verified = self._verify_in_browser(source, request, timeout, max_bytes)
if verified is not None:
# HTTP 客户端那条路攒下的账不压到浏览器上:清掉,之后按浏览器那条路重新数。
@@ -527,7 +528,7 @@ class SourceTransport:
def _via(self, source: str | None) -> str:
"""这个来源这次走哪条路。该走浏览器的来源要先选出传输才知道(不发请求、不起进程)。"""
- if source and source not in self.transports and wants_browser(self.root, source):
+ if self.allow_browser and source and source not in self.transports and wants_browser(self.root, source):
self.transports[source] = self._transport_for(source)
transport = self.transports.get(source)
return VIA_BROWSER if isinstance(transport, browser_transport.BrowserTransport) else VIA_HTTP
@@ -572,7 +573,7 @@ class SourceTransport:
def _transport_for(self, source: str | None):
"""这个来源的请求走哪条路:该走浏览器(`wants_browser`)且本机有浏览器就走浏览器页面,否则 httpx。"""
- if source and wants_browser(self.root, source):
+ if self.allow_browser and source and wants_browser(self.root, source):
browser = self._browser_for(source)
if browser is not None:
return browser
diff --git a/src/peach/source_preflight.py b/src/peach/source_preflight.py
new file mode 100644
index 000000000..e06edfba7
--- /dev/null
+++ b/src/peach/source_preflight.py
@@ -0,0 +1,28 @@
+"""安装后的资料源检查;不扫描影片,不改变来源配置。"""
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+import time
+
+from .scraping_access import SOURCES
+from .web_scraping import check_source
+
+
+def check_sources(secrets_root: Path, sources: list[str] | None = None) -> dict:
+ selected = list(dict.fromkeys(sources if sources is not None else
+ (source for source in SOURCES if source != "instagram")))
+ if not selected or any(source not in SOURCES for source in selected):
+ raise ValueError("请选择已登记的资料源 ID")
+
+ def check(source):
+ result = check_source(secrets_root, source, preflight=True)
+ return {"source": source, "label": SOURCES[source]["label"],
+ "results": result["results"], "session_verified": False}
+
+ started = time.monotonic()
+ with ThreadPoolExecutor(max_workers=4, thread_name_prefix="source-check") as pool:
+ results = list(pool.map(check, selected))
+ return {"ok": all(target["ok"] or target.get("reason") == "needs_code"
+ for source in results for target in source["results"]),
+ "sources": results, "elapsed_s": round(time.monotonic() - started, 2)}
diff --git a/src/peach/web_scraping.py b/src/peach/web_scraping.py
index 46933122c..337cb27dc 100644
--- a/src/peach/web_scraping.py
+++ b/src/peach/web_scraping.py
@@ -190,8 +190,12 @@ def _connection_message(detail, reason):
def w_scraping_check(contract, body):
+ return check_source(contract.follow_secrets_root, str(body.get("source", "")))
+
+
+def check_source(secrets_root, source, *, preflight=False):
+ """复用来源连接与图片 CDN 判据;安装检查只使用有界 HTTP 请求。"""
from .metadata import auth_wall_reason
- source = str(body.get("source", ""))
if source not in SOURCES:
raise ValueError("未知采集来源")
if not SOURCES[source]["login"]:
@@ -202,7 +206,9 @@ def w_scraping_check(contract, body):
if source == "dmm":
targets.append(("高清图片 CDN", "https://awsimgsrc.dmm.co.jp/pics_dig/digital/video/gyan00017/gyan00017pl.jpg"))
results = []
- transport = SourceTransport(contract.follow_secrets_root)
+ transport = (SourceTransport(secrets_root, max_requests=12, max_bytes=12 * 65536,
+ max_seconds=15, allow_browser=False)
+ if preflight else SourceTransport(secrets_root))
try:
for label, url in targets:
result = {"label": label, "ok": False}
@@ -241,7 +247,7 @@ def w_scraping_check(contract, body):
result.update(_connection_failure(exc, label))
result.update(checked_at=time.time(), elapsed_s=round(time.monotonic() - started, 2))
if label == "来源页面":
- record_source_check(contract.follow_secrets_root, source, "connection", result)
+ record_source_check(secrets_root, source, "connection", result)
results.append(result)
finally:
transport.close()
diff --git a/tests/test_cli.py b/tests/test_cli.py
index f7cec3b4e..3192f02b7 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -716,3 +716,13 @@ class PackagedEntryTests(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+
+
+class SourceChecksCliTests(unittest.TestCase):
+ def test_source_check_selection_json_and_failed_exit_status(self):
+ result = {'ok': False, 'sources': [], 'elapsed_s': 0}
+ with mock.patch('peach.source_preflight.check_sources', return_value=result) as check, \
+ mock.patch('peach.cli.settings_file.active') as active, redirect_stdout(io.StringIO()) as output:
+ self.assertEqual(cli.main(['check-sources', '--source', 'dmm', '--source', 'javdb', '--json']), 1)
+ check.assert_called_once_with(active.return_value.directory.return_value, ['dmm', 'javdb'])
+ self.assertEqual(__import__('json').loads(output.getvalue()), result)
diff --git a/tests/test_scraping_access.py b/tests/test_scraping_access.py
index 28acb0e23..27484e9e2 100644
--- a/tests/test_scraping_access.py
+++ b/tests/test_scraping_access.py
@@ -795,3 +795,50 @@ class BrowserFallbackTests(unittest.TestCase):
with patch("peach.browser_transport.shared", side_effect=AssertionError("avwikidb 不走浏览器")), \
self.assertRaises(SourcePaused):
transport(HttpRequest("GET", "https://avwikidb.com/actor/x/", {}), 1, 100)
+
+
+class SourcePreflightTests(unittest.TestCase):
+ def test_batch_deduplicates_and_preserves_target_failures(self):
+ from peach.source_preflight import check_sources
+ def check(root, source, **options):
+ self.assertTrue(options['preflight'])
+ return {'results': [{'label': '来源页面', 'ok': source == 'dmm'},
+ {'label': '高清图片 CDN', 'ok': False, 'reason': 'network'}]}
+ with patch('peach.source_preflight.check_source', side_effect=check) as checked:
+ result = check_sources(Path('/unused'), ['dmm', 'javdb', 'dmm'])
+ self.assertEqual(checked.call_count, 2)
+ self.assertEqual([row['source'] for row in result['sources']], ['dmm', 'javdb'])
+ self.assertFalse(result['ok'])
+ self.assertTrue(all(not row['session_verified'] for row in result['sources']))
+ with patch('peach.source_preflight.check_source') as checked:
+ for selected in ([], ['unknown'], ['dmm', 'unknown']):
+ with self.assertRaises(ValueError):
+ check_sources(Path('/unused'), selected)
+ checked.assert_not_called()
+
+ def test_preflight_bounds_requests_and_uses_no_browser(self):
+ from peach.web_scraping import check_source
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ with patch('peach.web_scraping.SourceTransport') as factory:
+ factory.return_value.return_value = HttpResponse(200, {}, b'<html></html>')
+ result = check_source(root, 'javdb', preflight=True)
+ self.assertTrue(result['results'][0]['ok'])
+ factory.assert_called_once_with(root, max_requests=12, max_bytes=12 * 65536,
+ max_seconds=15, allow_browser=False)
+ factory.return_value.close.assert_called_once()
+
+ def test_http_only_transport_does_not_launch_or_fix_browser_route(self):
+ from peach.scraping_access import SourcePaused, fix_to_browser, fixed_to_browser
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ fix_to_browser(root, 'minnano-av')
+ transport = SourceTransport(root, allow_browser=False)
+ with patch.object(transport, '_browser_for', side_effect=AssertionError('不能启动浏览器')), \
+ patch('peach.scraping_access.client_for') as client:
+ client.return_value = httpx.Client(transport=httpx.MockTransport(
+ lambda request: httpx.Response(403, content=b'blocked')))
+ with self.assertRaises(SourcePaused):
+ transport(HttpRequest('GET', 'https://www.minnano-av.com/', {}), 1, 1024)
+ transport.close()
+ self.assertTrue(fixed_to_browser(root, 'minnano-av'))