Peach: Local-first video metadata scraper and media manager.
fix(covers): 补齐无快照作品的官方目录查询
longmeidaoe9cb34fd95ac
diff --git a/docs/REUSE.md b/docs/REUSE.mdindex 650436e40..e02ecba04 100644--- a/docs/REUSE.md+++ b/docs/REUSE.md@@ -215,6 +215,8 @@ JAV 默认封面(官方封面/预览图)与视频默认大小(大图/新作封面复用 `best_cover` 的官方候选与 `cover_artwork.install_cover`,保存本地封面、正封框、人脸坐标与来源证据。JavDB 图片仅作来源复核证据,不进入安装、封面索引、详情封面投影或缺图兜底;`scripts/quarantine_javdb_covers.py` 将有来源证据的历史图片及边车移入可恢复归档。未取得合格来源时显示未取得状态,详见 ADR-0107。+无快照的有码作品由封面入口复用现有 `DmmSource` 的 GraphQL 查询与精确身份核验;成功快照直接展开 DMM 图片候选,r18.dev 与 DMM 使用同一份目录证据。FC2 来源快照通过 `storage_original` 提取发行方存储原件,排除社区图床。目录查询沿用来源冷却,HTTP、网络与解析失败分别保留未取得状态。+取景由 `poster-crop-v12` 管理。Blu-ray 来源标题、明确的版次标签与图片比例共同决定 0.85 正封比例,标题随正封框保留,重算人脸与正封框均读取同一版次;有效宽正封框在没有版次标题的卡片上也完整显示。VR 标记、IENEE、IENFE、SDHS 数字发行拼图,以及可信主脸跨过 DVD 正封先验边界的大封保留整幅画面,由 `cover_frame.layout=still` 传给卡片。大封在竖版大图按主脸居中填满,横版小图完整显示,均不使用模糊补边;JPEG 软边的书脊识别结合突出度、左忙右静的方向与正封侧基线。手工框保持用户取景。重试队列按最早尝试时间轮转,收起的合集沿用设置跳过;逐番号结果保存在 `feed-cover-attempts/`。复用现有 HTTP、Pillow、OpenCV 与文件锁,无新增依赖。馆藏与新作取图共用 `cover_artwork.metadata_snapshots` 和 `best_cover`,片商官网、发行方、R18 与 DMM 的官方候选均参与尺寸比较。馆藏采集将长边 1600px 作为高清重探界限,社区转存图可由官方封面替换;同类官方图按像素面积升级,FC2 保留商品图的来源保护。`scripts/audit_cover_artwork.py` 逐张核对尺寸、来源、版次与取景框,输出 CSV;`scripts/upgrade_audited_covers.py` 按已有官方快照逐番号重探,`--live` 经同一官方链在线补查,保存原件备份与结果检查点。两者只处理可重建图片和边车,不写 ledger。diff --git a/docs/adr/0107-cover-quality-and-javdb-evidence.md b/docs/adr/0107-cover-quality-and-javdb-evidence.mdindex fab3fbbc8..bfc9c95f9 100644--- a/docs/adr/0107-cover-quality-and-javdb-evidence.md+++ b/docs/adr/0107-cover-quality-and-javdb-evidence.md@@ -13,6 +13,7 @@## 决策- 可信官方快照与在线官方候选共用身份校验、解码和尺寸比较。远程封面的高清重探界限为长边 1600px;本地自备图与 FC2 商品图沿用各自的来源保护。同类官方图只按像素面积升级,官方图可以替换社区转存图。+- 缺少快照的有码作品复用已有 `DmmSource` 查询官方目录,DMM 成功证据与其目录镜像共用图片候选;FC2 快照只提取发行方存储原件。目录连接失败、拒绝访问与解析失败属于未取得,不能写成来源确认没有。- JavDB 图片只作来源复核与图像一致性证据,不作为正式封面、下载兜底或页面显示兜底。安装、封面索引与详情投影均执行此边界。官方图片未取得时显示未取得状态。- 有 JavDB 来源证据的历史封面及其边车移入 `review/` 下的可恢复归档,保留逐番号清单。来源未登记不等于 JavDB,不凭缺少来源批量移走文件。整个流程不写 ledger。- `poster-crop-v12` 结合主脸是否跨越正封先验边界识别整幅大封,结合梯度突出度与正封侧基线收紧书脊切点;手工取景保持原样。diff --git a/scripts/upgrade_audited_covers.py b/scripts/upgrade_audited_covers.pyindex d780b1acf..0d1624171 100644--- a/scripts/upgrade_audited_covers.py+++ b/scripts/upgrade_audited_covers.py@@ -98,8 +98,9 @@ def main():result = upgrade(path, known, transport, args.report, live=args.live,metadata_root=args.data_root / 'sources/metadata/javinizer-go')except Exception as error:- from peach.library_processing import describe_failure- result = dict(code=path.stem, status='failed', error=describe_failure(error))+ from peach.library_processing import describe_failure, is_missing+ result = dict(code=path.stem, status='failed', error=describe_failure(error),+ error_type=type(error).__name__, retryable=not is_missing(error))results.append(result)atomic_write_text(checkpoint, json.dumps(result, ensure_ascii=False, indent=2))print(json.dumps({'index': index, **result}, ensure_ascii=False), flush=True)diff --git a/src/peach/jav_cover_fetch.py b/src/peach/jav_cover_fetch.pyindex 11712b39f..0be1fd4e6 100644--- a/src/peach/jav_cover_fetch.py+++ b/src/peach/jav_cover_fetch.py@@ -57,7 +57,7 @@ from peach.scripting import USER_AGENT, open_readonlyfrom peach.config import COVER_DIR, DATABASE_PATH, GENERATED_DIR, SOURCES_DIRfrom peach.http import HttpRequest, HttpTransport, HttpxTransportfrom peach.scripting import HostLimiter-from peach.scraping_access import SOURCES, SourceTransport, source_for+from peach.scraping_access import SOURCES, SourcePaused, SourceTransport, source_forfrom peach.config import SECRETS_DIRfrom peach.jobs import DiskGuard, JobPolicyErrorfrom peach.task_runs import TaskRunHandle, cli_run, inert_handle@@ -67,6 +67,7 @@ from peach.catalog_rules import (is_amateur_code,is_jav_code,is_korean_mib_code,+ is_uncensored_code,normalise_code_key,)from peach.metadata_policy import SOURCE_SPECS@@ -555,7 +556,7 @@ def _is_prestige(evidence: MetadataEvidence) -> bool:def r18_evidence(transport: HttpTransport, code: str, *,- deadline: float | None = None) -> MetadataEvidence:+ deadline: float | None = None, record=lambda key: None) -> MetadataEvidence:"""读取 r18 返回的官方封套,并保留旧数字版高清 URL 探测。`content_id` 不是稳定的数字版路径。Prestige 的 ABW 系列会返回@@ -570,7 +571,13 @@ def r18_evidence(transport: HttpTransport, code: str, *,referer="https://r18.dev/", limit=2 * 1024 * 1024,deadline=deadline,).decode("utf-8", "ignore"))+ except NotFound:+ continueexcept (Unavailable, ValueError, httpx.TransportError):+ record('source_unanswered')+ continue+ if not isinstance(payload, dict):+ record('source_unanswered')continuereturn r18_payload_evidence(payload)return MetadataEvidence()@@ -596,6 +603,23 @@ def r18_payload_evidence(payload: dict) -> MetadataEvidence:return MetadataEvidence(tuple(_unique_candidates(found)), makers, frozenset({"r18dev"}))+def dmm_evidence(transport: HttpTransport, code: str, *, deadline=None) -> MetadataEvidence:+ """复用已核对番号的 DMM 官方目录,取得图片与数字版候选。"""+ from peach.sources import Session+ from peach.sources.base import SourceFailure+ from peach.sources.dmm import DmmSource+ try:+ payload = DmmSource().query(code, session=Session(transport, deadline)).payload()+ except SourceFailure as error:+ if error.kind == 'not_found':+ return MetadataEvidence()+ raise Unavailable('DMM:' + str(error)) from error+ found = [candidate for url in payload.get('cover_urls') or () for candidate in dmm_cdn_images(url)]+ found += content_id_images(str(payload.get('content_id') or ''))+ makers = frozenset(str(payload.get(field) or '').strip().casefold() for field in ('maker', 'label'))+ return MetadataEvidence(tuple(_unique_candidates(found)), makers, frozenset({'dmm'}))++def mgstage_images(transport: HttpTransport, code: str, *,deadline: float | None = None) -> list[Candidate]:"""直取 MGS 商品页 EnlargeImage;年龄确认只用公开 cookie,不绕挑战。"""@@ -721,15 +745,23 @@ def _usable_candidates(code: str, candidates, known_sizes: dict[str, tuple[int,def _site_candidates(transport: HttpTransport, code: str, evidence: MetadataEvidence, *,- delay: float, deadline: float | None) -> list[Candidate]:- """现问官方站拿到的封面候选:r18.dev,再按厂牌证据问 Prestige 或按番号形状问 MGS。FC2 一处都不问。"""+ delay: float, deadline: float | None, record=lambda key: None) -> list[Candidate]:+ """复用官方目录并按发行面查询图片;FC2 由调用方提供商品图候选。"""if code.upper().startswith("FC2-PPV-"):return []candidates: list[Candidate] = []live_evidence = MetadataEvidence()+ if (not evidence.candidates and 'dmm' not in evidence.sources+ and not is_amateur_code(code) and not is_uncensored_code(code)):+ try:+ live_evidence = dmm_evidence(transport, code, deadline=deadline)+ candidates += list(live_evidence.candidates)+ except (Unavailable, SourcePaused):+ record('source_unanswered')# 有成功快照时不重复打 r18;失败快照不算证据,仍允许联网刷新。- if "r18dev" not in evidence.sources:- live_evidence = r18_evidence(transport, code, deadline=deadline)+ if ('r18dev' not in evidence.sources and not live_evidence.candidates+ and not ('dmm' in evidence.sources and evidence.candidates)):+ live_evidence = r18_evidence(transport, code, deadline=deadline, record=record)candidates += list(live_evidence.candidates)_sleep_within(delay, deadline)# MGS 与 Prestige 都是 Prestige 集团的官方供给面。只在本地厂牌证据命中时@@ -777,9 +809,25 @@ def best_cover(transport: HttpTransport, code: str, delay: float, *,def usable(found):return _usable_candidates(code, found, known_sizes or {}, minimum_quality, minimum_pixels)+ def source_transport(request, timeout, limit):+ try:+ response = transport(request, timeout, limit)+ except (httpx.TransportError, SourcePaused):+ record('source_unanswered')+ raise+ if response.status not in (200, 206, 404):+ record('source_unanswered')+ record('source_http_' + str(response.status))+ return response++ source_transport.inner = transport+first = usable([*prior_candidates, *evidence.candidates])- asked = usable([*first, *_site_candidates(transport, code, evidence, delay=delay, deadline=deadline)])+ asked = usable([*first, *_site_candidates(source_transport, code, evidence, delay=delay,+ deadline=deadline, record=record)])if not asked:+ if diagnostics.get('source_unanswered'):+ raise Unavailable('官方目录查询未完成,尚未取得封面候选')raise NotFound("所有渠道都没有候选")measured = _measure(transport, asked, record, minimum_width=minimum_width,delay=delay, deadline=deadline)diff --git a/src/peach/library_processing.py b/src/peach/library_processing.pyindex e5ccd083c..49fc46281 100644--- a/src/peach/library_processing.py+++ b/src/peach/library_processing.py@@ -157,6 +157,14 @@ def _official_evidence(snapshots, absent=()):found += content_id_images(str(payload.get('content_id') or ''))makers |= {str(payload[field]).strip().casefold() for field in ('maker', 'studio', 'label')if isinstance(payload.get(field), str) and payload[field].strip()}+ elif name in metadata_routes.FC2_STAGE:+ from .jav_cover_fetch import candidate_for+ from .scripting import host_under, hostname_of+ from .sources.fc2 import storage_original+ found += [candidate_for(original) for url in payload.get('cover_urls') or ()+ if isinstance(url, str) and (original := storage_original(url)) and IMAGE_URL.fullmatch(original)+ and (host_under(hostname_of(original), ('contents.fc2.com',))+ or re.fullmatch(r'contents-thumbnail\d*\.fc2\.com', hostname_of(original)))]else:continuesources.add(name)diff --git a/src/peach/routes_media.py b/src/peach/routes_media.pyindex b2225224a..31b83825f 100644--- a/src/peach/routes_media.py+++ b/src/peach/routes_media.py@@ -13,6 +13,7 @@ from __future__ import annotationsimport asyncioimport base64+import jsonimport loggingimport reimport time@@ -611,8 +612,20 @@ async def cover_crop(request: Request, args: dict[str, str] = Depends(require_auif payload.get("box") is None:# 恢复默认是当场按折痕判据重算一遍,不是删掉 sidecar:删掉之后没有人会# 再来算,页面拿到的是「这张图不该裁」,正封从此再也回不来。+ def read_evidence(suffix):+ try:+ value = json.loads(path.with_suffix(suffix).read_text(encoding='utf-8'))+ return value if isinstance(value, dict) else {}+ except (OSError, ValueError):+ return {}+ face = read_evidence('.face.json')+ current_face = face.get('px') == list(size)+ title = (jav_poster_crop.read_sidecar(path) or {}).get('title') or read_evidence('.scraping.json').get('title', '')record = jav_poster_crop.crop_record(- code, size[0], size[1], jav_poster_crop.file_gradient(path))+ code, size[0], size[1], jav_poster_crop.file_gradient(path), title=title,+ face=face.get('face') if current_face else None)+ if current_face and face.get('layout') == 'still':+ record['box'] = dict(x0=0, y0=0, x1=size[0], y1=size[1], method=jav_poster_crop.NONE)else:record = jav_poster_crop.manual_record(size[0], size[1], payload.get("box"))if record is None:diff --git a/tests/test_avatar_picker.py b/tests/test_avatar_picker.pyindex d7f4e5955..1014bc8d5 100644--- a/tests/test_avatar_picker.py+++ b/tests/test_avatar_picker.py@@ -1215,6 +1215,19 @@ class CoverCropRouteTests(unittest.TestCase):response = self.client.post("/api/cover-crop", json={"code": "ABW-232"})self.assertEqual(response.status_code, 401)+ def test_default_crop_uses_current_main_face_and_edition_evidence(self):+ self.cover.with_suffix('.face.json').write_text(json.dumps({+ 'px': [800, 540], 'layout': 'still',+ 'face': {'cx': .555, 'cy': .331, 'w': .257, 'h': .422, 'score': .918},+ }), encoding='utf-8')+ self.cover.with_suffix('.scraping.json').write_text(json.dumps({'title': 'VR 作品'}), encoding='utf-8')+ response = self.client.post('/api/cover-crop?t=secret', json={'code': 'ABW-232', 'box': None})+ self.assertEqual(response.status_code, 200)+ self.assertIsNone(response.json()['poster_box'])+ self.assertEqual(self.sidecar()['box'], {'x0': 0, 'y0': 0, 'x1': 800, 'y1': 540, 'method': 'none'})+ self.assertEqual(self.sidecar()['title'], 'VR 作品')+ self.assertEqual(self.cover.read_bytes(), self.before)+def test_a_missing_cover_or_an_impossible_box_says_which(self):missing = self.client.post("/api/cover-crop?t=secret",json={"code": "NOPE-001", "box": None})diff --git a/tests/test_jav_covers.py b/tests/test_jav_covers.pyindex 3249e6873..a2a0297ef 100644--- a/tests/test_jav_covers.py+++ b/tests/test_jav_covers.py@@ -32,6 +32,38 @@ class _Response:class FetchRetryTests(unittest.TestCase):+ def test_empty_snapshots_use_the_identified_dmm_catalog_and_largest_image(self):+ COVER = 'https://awsimgsrc.dmm.co.jp/pics_dig/digital/video/ssis00057/ssis00057pl.jpg'+ ENDPOINT = 'https://api.video.dmm.co.jp/graphql'+ item = {'id': 'ssis00057', 'title': '作品', 'makerContentId': 'SSIS-057',+ 'packageImage': {'largeUrl': COVER}}+ modern = COVER.replace('awsimgsrc.dmm.co.jp/pics_dig/', 'awsimgsrc.dmm.com/dig/')+ transport = transport_for({+ ENDPOINT: (200, json.dumps({'data': {'ppvContent': item}}).encode()),+ COVER: (200, jpeg(800, 540)), modern: (200, jpeg(2184, 1468)),+ })+ with patch.object(covers, 'r18_evidence') as mirror:+ winner, size, _ = covers.best_cover(transport, 'SSIS-057', 0)+ self.assertEqual(winner.url, modern)+ self.assertEqual(size, (2184, 1468))+ mirror.assert_not_called()++ def test_catalog_connection_failure_is_not_a_definitive_missing_cover(self):+ diagnostics = {}+ with patch.object(covers, 'dmm_evidence', return_value=covers.MetadataEvidence()):+ with self.assertRaises(covers.Unavailable) as raised:+ covers.best_cover(lambda *args: _Response(503, b''), 'MIDA-085', 0, diagnostics=diagnostics)+ self.assertNotIsInstance(raised.exception, covers.NotFound)+ self.assertGreater(diagnostics.get('source_http_503', 0), 0)++ def test_official_catalog_probe_preserves_source_cooldown_writes(self):+ from unittest.mock import Mock+ transport = Mock(return_value=_Response(429, b''))+ transport.hold = Mock()+ with self.assertRaises(covers.Unavailable):+ covers.best_cover(transport, 'MIDA-085', 0)+ self.assertTrue(any(call.args[0] == 'dmm' for call in transport.hold.call_args_list))+def test_cover_rejections_report_measured_sizes_and_decode_failures(self):candidate = covers.Candidate("test", "https://example.test/cover.jpg")for size, expected in [((600, 400), "too_small"), ((1600, 1000), "invalid_image")]:diff --git a/tests/test_metadata_library.py b/tests/test_metadata_library.pyindex 93098ed38..61dc46a1f 100644--- a/tests/test_metadata_library.py+++ b/tests/test_metadata_library.py@@ -1618,6 +1618,14 @@ class LibraryNfoTests(unittest.TestCase):self.assertIsNotNone(evidence)self.assertEqual([candidate.url for candidate in evidence.candidates], [url])+ def test_fc2_snapshots_supply_only_product_storage_originals(self):+ from peach.library_processing import _official_evidence+ original = 'https://storage12000.contents.fc2.com/file/342/34141418/1554539118.7.jpg'+ evidence = _official_evidence([('javten', {'cover_urls': [original]}),+ ('javarchive', {'cover_urls': ['https://img.javstore.net/a.jpg']}),+ ('javdb', {'cover_urls': ['https://c0.jdbstatic.com/a.jpg']})])+ self.assertEqual([candidate.url for candidate in evidence.candidates], [original])+def test_community_cover_is_eligible_for_official_replacement(self):from peach.library_processing import cover_settledtarget = self.root / 'MIDV-377.jpg'