Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(sourcing): 防止 JavDB 同番号串片并保留原标题

longmeidao98e5f5b45f63
diff --git a/docs/SOURCING.md b/docs/SOURCING.md
index 2d76b0432..8cd51bd9d 100644
--- a/docs/SOURCING.md
+++ b/docs/SOURCING.md
@@ -657,6 +657,11 @@ Instagram 的独立用户登录会话未取得,自动适配器不进正式依
### 确认来源返回的是同一部片
+- JavDB 的相同番号可能对应多个来源 ID,不能靠番号、时长或搜索顺序认作同一作品。网页与匿名 API
+ 都先汇总匹配的来源 ID;只有一个才取详情,多个时报告歧义。详情、图片及资源沿用这次确认的来源 ID,
+ 不在取资源时重新按番号选片。JavDB 旧的番号缓存、发现边车和原始快照须通过来源专属规则才可复用。
+- JavDB 的 `origin-title` / `origin_title` 是原标题,`current-title` 是本地化显示标题。有原题时优先原题,
+ 显示标题独立留证;缺原题时保留显示标题,不凭文字语言猜测或删改星号片段。
- 来源返回的番号必须和查询的番号比对过才算命中。javbus 一侧是拿番号做关键词搜索取首个结果,搜不到就返回
近似的别人(`SA-104 → AVSA-104`、`AR-301 → STAR-3016`)。不比对时大部分匹配番号根本不对,整个 MIB 目录
(韩国内容)曾因此被写上日本厂牌、系列和标题,还靠这些假证据升级成 JAV。
diff --git a/src/peach/jav_discovery_metadata.py b/src/peach/jav_discovery_metadata.py
index 112aff4e2..bb084416c 100644
--- a/src/peach/jav_discovery_metadata.py
+++ b/src/peach/jav_discovery_metadata.py
@@ -14,11 +14,12 @@ from .fsutil import atomic_write_text
from .javdb_public import (PUBLIC_API, PublicBudgetExceeded, PublicRequestBudget, PublicSourceUnavailable,
public_cache_scope, cache_checked_at, cache_error)
from .jav_cache import cache_fresh
+from .javdb_identity import RULE
from .jobs import BackgroundJob
MAX_ITEMS = 100
MAX_RECORDS = 2048
-FIELDS = ("id", "number", "title", "release_date", "maker_name", "series_name", "duration", "score", "tags", "actors")
+FIELDS = ("id", "number", "title", "origin_title", "release_date", "maker_name", "series_name", "duration", "score", "tags", "actors", "javdb_rule")
def actor_record(actor):
@@ -43,7 +44,9 @@ class DiscoveryMetadata:
"""缓存只承载来源资料;后台任务不写账本或 task_run。"""
def __init__(self, path: Path, *, api=PUBLIC_API, clock=time.time):
- self.path, self.api, self.clock = Path(path), api, clock
+ path = Path(path)
+ self.path = path.with_name(f'{path.stem}.javdb-{RULE}{path.suffix}')
+ self.api, self.clock = api, clock
self.lock = threading.RLock()
self.job = BackgroundJob("jav-discovery-metadata")
self.last_job = None
@@ -54,6 +57,8 @@ class DiscoveryMetadata:
saved = json.loads(self.job_path.read_text(encoding='utf-8'))
except (OSError, ValueError):
saved = {}
+ if saved.get('javdb_rule') != RULE:
+ saved = {}
if saved.get('status') == 'running':
self.ensure(saved.get('codes') or [], _resume=saved)
elif saved.get('job_id'):
@@ -63,14 +68,14 @@ class DiscoveryMetadata:
with self.lock:
state = self.job.snapshot()
if state:
- atomic_write_text(self.job_path, json.dumps(state, ensure_ascii=False))
+ atomic_write_text(self.job_path, json.dumps({**state, 'javdb_rule': RULE}, ensure_ascii=False))
def _load(self):
try:
if self.path.stat().st_size > 16 * 1024 * 1024:
return
payload = json.loads(self.path.read_text(encoding="utf-8"))
- rows = payload.get("records", {}) if payload.get("version") == 1 else {}
+ rows = payload.get("records", {}) if payload.get("version") == RULE else {}
if isinstance(rows, dict):
for code, record in list(rows.items())[-MAX_RECORDS:]:
if isinstance(record, dict) and isinstance(record.get("movie", {}), dict):
@@ -80,10 +85,10 @@ class DiscoveryMetadata:
def _save(self):
self.path.parent.mkdir(parents=True, exist_ok=True)
- encoded = json.dumps({"version": 1, "records": self.records}, ensure_ascii=False)
+ encoded = json.dumps({"version": RULE, "records": self.records}, ensure_ascii=False)
while len(encoded.encode("utf-8")) > 16 * 1024 * 1024 and self.records:
self.records.popitem(last=False)
- encoded = json.dumps({"version": 1, "records": self.records}, ensure_ascii=False)
+ encoded = json.dumps({"version": RULE, "records": self.records}, ensure_ascii=False)
atomic_write_text(self.path, encoded)
def remember(self, code, movie=None, *, error="", checked_at=None):
diff --git a/src/peach/javdb_identity.py b/src/peach/javdb_identity.py
new file mode 100644
index 000000000..1f341cc96
--- /dev/null
+++ b/src/peach/javdb_identity.py
@@ -0,0 +1,21 @@
+"""JavDB 原题与同番号来源身份的证据版本。"""
+from __future__ import annotations
+
+import re
+
+RULE = 2
+
+
+def current_evidence(payload: object) -> bool:
+ return (isinstance(payload, dict) and type(payload.get('javdb_rule')) is int
+ and payload['javdb_rule'] == RULE)
+
+
+def current_candidate(payload: object) -> bool:
+ return (current_evidence(payload) and isinstance(payload.get('source_id'), str)
+ and re.fullmatch(r'[A-Za-z0-9]+', payload['source_id']) is not None)
+
+
+def cache_key(key: str) -> str:
+ """只隔离旧的番号身份与作品详情,明确来源ID的资源缓存继续复用。"""
+ return f'javdb-{RULE}:{key}' if key.startswith(('movie-id:', 'movie:')) else key
diff --git a/src/peach/javdb_public.py b/src/peach/javdb_public.py
index 78037f7d4..db27958a6 100644
--- a/src/peach/javdb_public.py
+++ b/src/peach/javdb_public.py
@@ -21,6 +21,7 @@ from .catalog_rules import normalise_code_key
from .http import HttpRequest, HttpxTransport, response_text
from .scripting import HostLimiter
from .jav_cache import DAY, PublicSnapshots, cache_lifetime
+from .javdb_identity import RULE, cache_key as identity_cache_key, current_evidence
WEB = "https://catembylegacy.fastcdn.dpdns.org"
API = "https://jdforrepam.com/api"
@@ -84,7 +85,7 @@ def cache_checked_at(prefix, *, exact=False):
def cache_error(key):
return next((read['error'] for read in reversed(_CACHE_READS.get() or [])
- if read['key'] == key and read['error']), '')
+ if (read['key'] == key or read['key'].startswith(key + ':')) and read['error']), '')
def _record_read(key, snapshot, error=''):
@@ -121,6 +122,10 @@ class PublicSourceUnavailable(ValueError):
"""公开来源当前未取得。"""
+class PublicIdentityAmbiguous(PublicSourceUnavailable):
+ """同番号有多个来源身份,不能按番号绑定资料或媒体。"""
+
+
class PublicBudgetExceeded(PublicSourceUnavailable):
"""本轮匿名资料请求预算已用完。"""
@@ -264,6 +269,7 @@ class PublicAPI:
def _cache_key(self, key):
root = self._store().root
+ key = identity_cache_key(key)
return f'{root.resolve()}\0{key}' if root is not None else key
def _memory(self, cache_key, *, image, refresh):
@@ -303,10 +309,11 @@ class PublicAPI:
failed = self.failures.get(cache_key)
_record_read(key, self.snapshots.get(cache_key), str(failed[1]) if failed else '')
return result
- previous = snapshot if snapshot is not None else None if image else store.load(key)
+ stored_key = identity_cache_key(key)
+ previous = snapshot if snapshot is not None else None if image else store.load(stored_key)
try:
- with nullcontext() if image else store.acquire(key):
- disk = None if image else store.load(key)
+ with nullcontext() if image else store.acquire(stored_key):
+ disk = None if image else store.load(stored_key)
if disk is not None and disk.fresh(self.wallclock()) and (
not refresh or (previous is not None and disk.revision != previous.revision)):
snapshot = self._remember(key, disk.value, snapshot=disk)
@@ -347,7 +354,7 @@ class PublicAPI:
if isinstance(result, dict):
released = result.get('release_date') or released
released = released if isinstance(released, str) else None
- snapshot = self._store().save(key, result, released)
+ snapshot = self._store().save(identity_cache_key(key), result, released)
lifetime = 6 * 3600 if image else cache_lifetime(snapshot.released, datetime.fromtimestamp(self.wallclock()).date())
remaining = lifetime if image else None if lifetime is None else max(0, snapshot.saved + lifetime - self.wallclock())
with self.lock:
@@ -362,41 +369,57 @@ class PublicAPI:
self.failures.pop(removed, None)
return snapshot
- def movie(self, code: str, *, refresh=False) -> dict:
+ def movie(self, code: str, *, refresh=False, source_id=None) -> dict:
canonical = normalise_code_key(code)
def identify():
product = re.fullmatch(r'FC2-PPV-(\d+)', canonical)
query = product[1] if product else canonical
rows = self._rows("/v2/search", "movies", {"q": query, "type": "movie", "page": 1})
- match = next((row for row in rows if isinstance(row, dict)
- and normalise_code_key(row.get("number")) == canonical), None)
- if not match or not isinstance(match.get("id"), str) or not re.fullmatch(r"[A-Za-z0-9]+", match["id"]):
+ matches = [row for row in rows if isinstance(row, dict)
+ and normalise_code_key(row.get("number")) == canonical]
+ if not matches or any(not isinstance(row.get('id'), str)
+ or not re.fullmatch(r'[A-Za-z0-9]+', row['id']) for row in matches):
raise PublicSourceUnavailable("未取得这个番号的作品资料")
- return match["id"]
+ return list(dict.fromkeys(row['id'] for row in matches))
+
+ if source_id is None:
+ identities = self._cached("movie-id:" + canonical, identify, refresh=refresh)
+ if len(identities) != 1:
+ raise PublicIdentityAmbiguous("同番号对应多个作品,请核对来源身份:" + '、'.join(identities))
+ movie_id = identities[0]
+ elif isinstance(source_id, str) and re.fullmatch(r'[A-Za-z0-9]+', source_id):
+ movie_id = source_id
+ else:
+ raise PublicSourceUnavailable("来源作品身份不匹配")
def fetch():
- movie_id = self._cached("movie-id:" + canonical, identify, refresh=refresh)
movie = self._json("/v4/movies/" + movie_id).get("movie")
if (not isinstance(movie, dict) or normalise_code_key(movie.get("number")) != canonical
or movie.get("id") != movie_id):
raise PublicSourceUnavailable("来源作品身份不匹配")
- return movie
+ return {**movie, 'javdb_rule': RULE}
- return self._cached("movie:" + canonical, fetch, refresh=refresh)
+ return self._cached(f"movie:{canonical}:{movie_id}", fetch, refresh=refresh)
def remember_movie_identity(self, code: str, source: dict):
"""已取得的编目身份可复用,完整资料仍须通过来源的身份核对。"""
canonical = normalise_code_key(code)
identity = source.get("id")
- if not isinstance(identity, str) or normalise_code_key(source.get("number")) != canonical or not re.fullmatch(r"[A-Za-z0-9]+", identity):
+ if (not current_evidence(source) or not isinstance(identity, str)
+ or normalise_code_key(source.get("number")) != canonical
+ or not re.fullmatch(r"[A-Za-z0-9]+", identity)):
return
key = 'movie-id:' + canonical
- stored = self._store().load(key)
- if stored and stored.value == identity:
- self._remember(key, identity, snapshot=stored)
- else:
- self._remember(key, identity)
+ with self.lock:
+ # 已观察到的唯一身份或歧义不被较旧的发现边车反向覆盖。
+ if self._cache_key(key) in self.cache:
+ return
+ stored = self._store().load(identity_cache_key(key))
+ if stored:
+ self._remember(key, stored.value, snapshot=stored)
+ else:
+ self._remember(key, [identity])
def _rows(self, path, key, query=None):
rows = self._json(path, query).get(key)
@@ -408,52 +431,78 @@ class PublicAPI:
def peek_movie(self, code: str) -> dict | None:
"""只读已取得的资料,不触发网络;供发现排序投影。"""
with self.lock:
- key = 'movie:' + normalise_code_key(code)
+ canonical = normalise_code_key(code)
+ identity_key = 'movie-id:' + canonical
+ hit = self.cache.get(self._cache_key(identity_key))
+ identity = hit[1] if hit else None
+ if identity is None:
+ stored = self._store().load(identity_cache_key(identity_key))
+ identity = stored.value if stored else None
+ if not isinstance(identity, list) or len(identity) != 1:
+ return None
+ key = f'movie:{canonical}:{identity[0]}'
cache_key = self._cache_key(key)
hit = self.cache.get(cache_key)
- snapshot = self.snapshots.get(cache_key) or self._store().load(key)
+ snapshot = self.snapshots.get(cache_key) or self._store().load(identity_cache_key(key))
if snapshot and snapshot.fresh(self.wallclock()):
_record_read(key, snapshot)
return dict(snapshot.value) if isinstance(snapshot.value, dict) else None
return dict(hit[1]) if hit and hit[0] > self.clock() and snapshot is None else None
- def reviews(self, code: str, page: int, *, refresh=False) -> list[dict]:
+ def _bound_movie(self, code, movie):
+ if movie is None:
+ return self.movie(code)
+ if (not isinstance(movie, dict) or not isinstance(movie.get('id'), str)
+ or not re.fullmatch(r'[A-Za-z0-9]+', movie['id'])
+ or normalise_code_key(movie.get('number')) != normalise_code_key(code)):
+ raise PublicSourceUnavailable('来源作品身份不匹配')
+ return movie
+
+ def reviews(self, code: str, page: int, *, refresh=False, movie=None) -> list[dict]:
if not 1 <= page <= MAX_REVIEW_PAGES:
raise ValueError("评论页码必须在 1 至 8 之间")
- movie = self.movie(code)
+ movie = self._bound_movie(code, movie)
return self._cached(f"reviews:{movie['id']}:{page}", lambda:
self._rows(f"/v1/movies/{movie['id']}/reviews", "reviews", {"page": page}),
released=movie.get('release_date'), refresh=refresh)
- def media_movie(self, code: str) -> dict:
+ def media_movie(self, code: str, *, movie=None) -> dict:
"""未缓存媒体可补齐带查询参数的临时来源地址,公开编目仍按发行日期复用。"""
- movie = self.movie(code)
- key = 'movie:' + normalise_code_key(code)
- snapshot = self.snapshots.get(self._cache_key(key)) or self._store().load(key)
+ movie = self._bound_movie(code, movie)
+ key = f"movie:{normalise_code_key(code)}:{movie['id']}"
+ snapshot = self.snapshots.get(self._cache_key(key)) or self._store().load(identity_cache_key(key))
urls = [movie.get(name) for name in ('cover_url', 'thumb_url', 'preview_video_url')]
urls.extend(actor.get('avatar_url') for actor in movie.get('actors') or [] if isinstance(actor, dict))
urls.extend((image.get('large_url') or image.get('url')) if isinstance(image, dict) else image
for image in movie.get('preview_images') or [])
if snapshot and self.wallclock() - snapshot.saved >= DAY and any(
isinstance(url, str) and '?' in url for url in urls):
- return self.movie(code, refresh=True)
+ return self.movie(code, refresh=True, source_id=movie['id'])
return movie
- def magnets(self, code: str, *, refresh=False) -> list[dict]:
- movie = self.movie(code)
+ def magnets(self, code: str, *, refresh=False, movie=None) -> list[dict]:
+ movie = self._bound_movie(code, movie)
return self._cached("magnets:" + movie['id'], lambda:
self._rows(f"/v1/movies/{movie['id']}/magnets", "magnets"),
released=movie.get('release_date'), refresh=refresh)
- def official_preview(self, code, fetch, *, released=None, refresh=False, provider='prestige'):
+ def official_preview(self, code, fetch, *, released=None, refresh=False, provider='prestige', source_id=None):
"""预告按来源独立保存;DMM、JavDB 和未提供的预告每天可再检查。"""
def read():
url = fetch()
return {'url': url, **({'release_date': released} if url and provider == 'prestige' else {})}
- return self._cached(f'preview:{provider}:' + normalise_code_key(code), read, refresh=refresh)
+ key = f'preview:{provider}:' + normalise_code_key(code)
+ if provider == 'javdb':
+ if source_id is None:
+ key += f':search-{RULE}'
+ elif not isinstance(source_id, str) or not re.fullmatch(r'[A-Za-z0-9]+', source_id):
+ raise PublicSourceUnavailable('来源作品身份不匹配')
+ else:
+ key += ':' + source_id
+ return self._cached(key, read, refresh=refresh)
- def image(self, code: str, index: str, *, original=False) -> str:
- movie = self.media_movie(code)
+ def image(self, code: str, index: str, *, original=False, movie=None) -> str:
+ movie = self.media_movie(code, movie=movie)
if index == "cover":
url = movie.get("cover_url") or movie.get("thumb_url")
elif index.startswith("actor:"):
diff --git a/src/peach/library_processing.py b/src/peach/library_processing.py
index fde46da7b..e32122ca1 100644
--- a/src/peach/library_processing.py
+++ b/src/peach/library_processing.py
@@ -1078,6 +1078,8 @@ def _merge_candidates(groups, row, code, source, document, evidence_path, genre_
provider_id=str(document.get('id') or ''), content_id=str(document.get('content_id') or ''),
source_kind='local' if local else spec.kind if spec else 'community', official=official,
catalog_evidence=extract_catalog_evidence(document))
+ if source == 'javdb':
+ candidate.update(javdb_rule=document.get('javdb_rule'), source_id=document.get('source_id'))
if mode == 'review':
candidate['requires_review'] = True
# 只有 tags 字段有未收录原文;别的字段挂一个空列表只是让每条候选都胖一圈。
@@ -1374,7 +1376,7 @@ class _RemoteSession:
self.misses = misses
#: 用户对每种内容类型的来源链覆盖,形状见 `metadata_routes.parse_route_overrides`。
self._routes = routes or None
- # 「重试未完成项」按上一任务的失败集合强制重试,不看记忆;答复仍照记。
+ # 定向重试或明确重采不看旧记忆;本次答复仍照记。
self._consult = not retrying
self._task_evidence = {}
self._task_absent = set()
@@ -1500,6 +1502,10 @@ class _RemoteSession:
if time.time() - path.stat().st_mtime > MISS_TTL_SECONDS:
continue
payload = json.loads(path.read_text(encoding='utf-8'))
+ if name == 'javdb':
+ from .javdb_identity import current_evidence
+ if not current_evidence(payload):
+ continue
if not source_text_is_valid(payload):
continue
entry = (name, payload, path)
@@ -2699,7 +2705,7 @@ def process_library(config, db_path, candidate_root, cover_root, *, location='co
update()
remote = _RemoteSession(config, provider_factory, _MissCache(misses_path(config)),
- retrying=retrying, routes=routes)
+ retrying=retrying or recheck_existing, routes=routes)
flush_candidates = lambda force=False: None
try:
diff --git a/src/peach/metadata_auto_apply.py b/src/peach/metadata_auto_apply.py
index fcac081a0..0b08c1356 100644
--- a/src/peach/metadata_auto_apply.py
+++ b/src/peach/metadata_auto_apply.py
@@ -57,6 +57,7 @@ from .metadata_policy import (
blacklisted, chain_rank, preferred_tag_source, source_tier,
)
from .review_csv import read_candidates
+from .javdb_identity import current_candidate as current_javdb_candidate
from .sources import fc2
#: 一个写事务里最多落多少条。整批一个事务在本机实测是几千条候选压着写锁不放,
@@ -762,6 +763,7 @@ def _evidence_candidates(row: dict, field: str = "") -> list[dict]:
if (str(c.get("source") or "").strip() in SOURCE_SPECS
or str(c.get("source") or "").strip() == LOCAL_NFO_SOURCE)
and not blacklisted(field, str(c.get("source") or "").strip())
+ and (str(c.get('source') or '').strip() != 'javdb' or current_javdb_candidate(c))
and candidate_text_is_valid(c)
and _original_title_candidate(field, c, str(row.get('code') or ''))]
@@ -1516,6 +1518,9 @@ def auto_apply_metadata(database, candidate_root, *, batch_size=AUTO_APPLY_BATCH
if str(row.get("status") or "").strip() != "candidate":
continue
row["candidates"] = _row_candidates(row, genres)
+ row['candidates'] = [candidate for candidate in row['candidates']
+ if str(candidate.get('source') or '').strip() != 'javdb'
+ or current_javdb_candidate(candidate)]
prior = decided.get(item_key)
if overwrite_requires_review(connection, row):
skipped += 1
diff --git a/src/peach/sources/javdb.py b/src/peach/sources/javdb.py
index ad55e6471..7b5823b68 100644
--- a/src/peach/sources/javdb.py
+++ b/src/peach/sources/javdb.py
@@ -1,10 +1,11 @@
"""javdb 的作品页。
-搜索页 `/search?q=<番号>&f=all` 的结果卡片带番号,番号一致的那张进详情页;详情页
+搜索页 `/search?q=<番号>&f=all` 的结果卡片带番号,只在对应唯一来源 ID 时进详情页;详情页
`?locale=zh` 下面板字段是 `番號`、`日期`、`片商`、`發行`、`系列`、`導演`、`演員`,
演员里女优带 `actor-female`,类别保留链接里的原词。它按出口 IP 计配额,限速与封禁的处理在 `scraping_access`
和调用方的主机间隔里,这里只管取页与解析。几家的值常有出入:ABW-358 在 javdb 上发行日期是
MGS 的 5/23、标题带 MGS 附注、演员里有男优,各字段按 Peach 的来源策略结算。
+有隐藏 `origin-title` 时用作原标题,`current-title` 单独保存为显示标题;两者不互相改写。
一部分页面要登录才给,回的是登入页而不是 401/403(`peach.javdb.LOGIN`),搜索页与详情页都要认。
"""
@@ -17,6 +18,7 @@ from bs4 import BeautifulSoup
from ..catalog_rules import same_release_code
from ..javdb import LOGIN, clean
+from ..javdb_identity import RULE
from .base import FailureReason, Page, Session, SiteConfig, SiteRecord, SiteSource, SourceFailure
#: 主机间隔由用户定(2026-09-22 定为 3 秒,每分钟 20 页)。参照面:每分钟 40~50 页会招来 3~7 天
@@ -28,7 +30,6 @@ JAVDB = SiteConfig(name="javdb", label="javdb", provider="javdb-page",
_BOX = re.compile(r'<a href="(/v/[A-Za-z0-9]+)" class="box" title="[^"]*">.*?<strong>([^<]+)</strong>', re.S)
_PANEL = re.compile(r'<div class="panel-block[^"]*">\s*<strong>([^<::]+)[::]</strong>\s*(?:&nbsp;)?\s*'
r'<span class="value">(.*?)</span>', re.S)
-_TITLE = re.compile(r'<strong class="current-title">([^<]*)</strong>')
#: 详情页那张封面。认标签、再从标签里取 `src`,不把属性挨着写死:站上 `class` 与 `src` 的先后
#: 两种写法都出现过,中间还可能夹着 `width`、`height` 与 `fetchpriority`。
_COVER = re.compile(r'<img\b[^>]*\bclass="video-cover"[^>]*>')
@@ -124,10 +125,14 @@ class JavDBSource(SiteSource):
def fetch(self, code: str, *, session: Session) -> Page:
search = self._page(self.search_url(code), session=session)
- path = next((path for path, shown in _BOX.findall(search.text)
- if same_release_code(code, clean(shown))), None)
- if path is None:
+ paths = list(dict.fromkeys(path for path, shown in _BOX.findall(search.text)
+ if same_release_code(code, clean(shown))))
+ if not paths:
raise SourceFailure(FailureReason.NOT_FOUND, "javdb 没有这个番号")
+ if len(paths) > 1:
+ raise SourceFailure(FailureReason.PARSE_ERROR, "javdb 同番号对应多个作品,请核对来源身份:" + '、'.join(paths),
+ detail='identity_ambiguous')
+ path = paths[0]
detail = self._page(self.config.base_url + path + "?locale=zh", session=session)
# `source_url` 记不带界面语言参数的详情页地址。
return Page(self.config.base_url + path, detail.body)
@@ -139,7 +144,11 @@ class JavDBSource(SiteSource):
if not same_release_code(code, shown):
raise SourceFailure(FailureReason.PARSE_ERROR, "javdb 详情页的番号与搜索结果不一致")
runtime = re.search(r"\d+", clean(panel.get("時長", "")))
- title = _TITLE.search(text)
+ soup = BeautifulSoup(text, "html.parser")
+ title = soup.select_one('strong.current-title')
+ original = soup.select_one('.origin-title')
+ display_title = clean(title.get_text(' ', strip=True)) if title else ''
+ origin_title = clean(original.get_text(' ', strip=True)) if original else ''
tag = _COVER.search(text)
cover = _IMG_SRC.search(tag.group(0)) if tag else None
genres = tuple(dict.fromkeys(
@@ -148,13 +157,15 @@ class JavDBSource(SiteSource):
if clean(link.get_text(" ", strip=True))))
return SiteRecord(
source=self.config.name, provenance=self.config.provider, code=shown, source_url=page.url,
- title=clean(title.group(1)) if title else "",
+ title=origin_title or display_title,
performers=tuple(actresses(panel.get("演員", ""))),
studio=maker_writing(clean(panel.get("片商", ""))), label=clean(panel.get("發行", "")),
series=clean(panel.get("系列", "")), director=clean(panel.get("導演", "")),
release_date=clean(panel.get("日期", "")), runtime=int(runtime.group()) if runtime else None,
tags=genres,
- cover_urls=(cover.group(1),) if cover else ())
+ cover_urls=(cover.group(1),) if cover else (),
+ extra={'origin_title': origin_title, 'display_title': display_title,
+ 'source_id': urllib.parse.urlsplit(page.url).path.rsplit('/', 1)[-1], 'javdb_rule': RULE})
def parse_magnets(page: Page) -> list[dict]:
diff --git a/src/peach/web_jav_detail.py b/src/peach/web_jav_detail.py
index 459b0a85f..bdf015175 100644
--- a/src/peach/web_jav_detail.py
+++ b/src/peach/web_jav_detail.py
@@ -12,7 +12,8 @@ from .catalog_rules import normalise_code_key, tag_cat
from .genre_decisions import load_genre_decisions
from .genre_taxonomy import resolve_genre
from .javdb_public import (MAX_REVIEW_PAGES, PAGE_SIZE, PUBLIC_API, PublicSourceUnavailable,
- actor_gender, preview_video_url, public_image_url, public_cache_scope, cache_checked_at, cache_error)
+ PublicIdentityAmbiguous, actor_gender, preview_video_url, public_image_url,
+ public_cache_scope, cache_checked_at, cache_error)
from .jav_discovery_metadata import discovery_metadata
from .sources.javdb import maker_writing
from .remote_media_cache import AVATAR_REFRESH_SECONDS, cache_for
@@ -90,6 +91,7 @@ def q_jav_detail(contract, args) -> dict:
item = dict(row) if row else {}
source = None
message = ""
+ ambiguous = False
try:
stored, _ = discovery_metadata(contract).peek(code)
PUBLIC_API.remember_movie_identity(code, stored or {})
@@ -99,9 +101,10 @@ def q_jav_detail(contract, args) -> dict:
discovery_metadata(contract).remember(code, error=cache_error('movie:' + code))
except PublicSourceUnavailable as error:
message = str(error)
+ ambiguous = isinstance(error, PublicIdentityAmbiguous)
discovery_metadata(contract).remember(code, error=message)
if source:
- item.update(title=source.get("title") or source.get("origin_title") or item.get("title"),
+ item.update(title=source.get("origin_title") or source.get("title") or item.get("title"),
release_date=source.get("release_date") or item.get("release_date"),
studio=source.get("maker_name") or item.get("studio"))
payload = {"ok": True, "state": "ready" if source else "partial", "message": message,
@@ -110,7 +113,8 @@ def q_jav_detail(contract, args) -> dict:
with contract.database.read_connection() as connection:
decisions = load_genre_decisions(connection)
payload.update(_source_fields(source or {}, decisions))
- payload.update(_artwork_fields(contract, code, item, source or {}, refresh=args.get('refresh') == '1'))
+ payload.update(_artwork_fields(contract, code, item, source or {}, refresh=args.get('refresh') == '1',
+ ambiguous=ambiguous))
if payload.get('preview_message'):
payload['message'] = ';'.join(filter(None, (payload['message'], payload['preview_message'])))
payload.update(_identity_fields(contract, code, item, source or {}))
@@ -176,15 +180,16 @@ def source_performer_refs(contract, connection, actors):
return refs
-def _artwork_fields(contract, code, item, source, *, refresh=False):
+def _artwork_fields(contract, code, item, source, *, refresh=False, ambiguous=False):
from .jav_cover_fetch import is_javdb_image
source_cover = (source or {}).get('cover_url') or (source or {}).get('thumb_url')
- return {"has_cover": contract.has_cover(code), "cover_version": contract.cover_version(code),
- "cover_url": discovery_cover_url(item, source or {}),
+ return {"has_cover": not ambiguous and contract.has_cover(code), "cover_version": contract.cover_version(code),
+ "cover_url": '' if ambiguous else discovery_cover_url(item, source or {}),
"source_image": bool(source_cover and not is_javdb_image(source_cover)),
"preview_count": min(len(source.get("preview_images") or []), 30),
- **_preview_fields(contract, code, source, refresh=refresh),
- "source_url": "https://javdb.com/v/" + str(source["id"]) if source else item.get("link") or ""}
+ **({'preview_video_url': '', 'preview_state': 'unavailable'} if ambiguous else
+ _preview_fields(contract, code, source, refresh=refresh)),
+ "source_url": '' if ambiguous else "https://javdb.com/v/" + str(source["id"]) if source else item.get("link") or ""}
def _source_fields(source, decisions):
@@ -211,8 +216,10 @@ def q_jav_reviews(contract, args) -> dict:
page = int(args.get("page") or 1)
try:
if args.get('refresh') == '1':
- PUBLIC_API.movie(code, refresh=True)
- rows = PUBLIC_API.reviews(code, page, refresh=True) if args.get('refresh') == '1' else PUBLIC_API.reviews(code, page)
+ movie = PUBLIC_API.movie(code, refresh=True)
+ rows = PUBLIC_API.reviews(code, page, refresh=True, movie=movie)
+ else:
+ rows = PUBLIC_API.reviews(code, page)
except PublicSourceUnavailable as error:
return {"ok": False, "items": [], "more": False, "message": str(error)}
items = [{"id": str(row.get("id")), "username": str(row.get("username") or "匿名用户"),
@@ -241,15 +248,20 @@ def jav_image(contract, args) -> tuple[bytes, int]:
and (not index.isdigit() or not 0 <= int(index) < 30)):
raise ValueError("预览图不存在")
actor = index.startswith("actor:")
- key = f"jav:actor:{index.removeprefix('actor:')}" if actor else f"jav:{code}:image:{index}"
with public_cache_scope(contract.cover_root.parent / 'jav-public'):
+ movie = None
+ if not actor:
+ stored, _ = discovery_metadata(contract).peek(code)
+ PUBLIC_API.remember_movie_identity(code, stored or {})
+ movie = PUBLIC_API.movie(code)
+ key = f"jav:actor:{index.removeprefix('actor:')}" if actor else f"jav:{code}:{movie['id']}:image:{index}"
body = cache_for(contract).bytes(key,
- lambda: base64.b64decode(_fetch_image(contract, code, index).partition(',')[2], validate=True),
+ lambda: base64.b64decode(_fetch_image(contract, code, index, movie=movie).partition(',')[2], validate=True),
ttl=AVATAR_REFRESH_SECONDS if actor else None)
return body, JAV_ACTOR_IMAGE_MAX_AGE if actor else JAV_IMAGE_MAX_AGE
-def _fetch_image(contract, code, index):
+def _fetch_image(contract, code, index, *, movie=None):
stored, _ = discovery_metadata(contract).peek(code)
PUBLIC_API.remember_movie_identity(code, stored or {})
if index.startswith('actor:'):
@@ -257,11 +269,11 @@ def _fetch_image(contract, code, index):
if isinstance(person, dict) and str(person.get('id')) == index.removeprefix('actor:')), {})
if public_image_url(actor.get('avatar_image')):
return PUBLIC_API.image_from_url(actor['avatar_image'])
- image = PUBLIC_API.image(code, index)
+ image = PUBLIC_API.image(code, index, movie=movie)
if index.startswith('actor:'):
movie = PUBLIC_API.peek_movie(code)
if movie:
- discovery_metadata(contract).remember(code, movie, checked_at=cache_checked_at('movie:' + code, exact=True))
+ discovery_metadata(contract).remember(code, movie, checked_at=cache_checked_at('movie:' + code + ':'))
return image
@@ -278,7 +290,8 @@ def _preview_fields(contract, code, source, *, refresh=False):
for provider, fetch in sources:
try:
result = PUBLIC_API.official_preview(code, fetch, released=source.get('release_date'),
- refresh=refresh, provider=provider)
+ refresh=refresh, provider=provider,
+ **({'source_id': source.get('id')} if provider == 'javdb' else {}))
url = result['url']
failure = cache_error(f'preview:{provider}:' + code)
if failure and not url:
@@ -291,7 +304,7 @@ def _preview_fields(contract, code, source, *, refresh=False):
url = preview_video_url(source)
if url and '?' in url:
try:
- url = preview_video_url(PUBLIC_API.media_movie(code)) or url
+ url = preview_video_url(PUBLIC_API.media_movie(code, movie=source)) or url
except PublicSourceUnavailable as error:
errors.append(str(error))
if not url:
@@ -318,7 +331,7 @@ def q_jav_resources(contract, args) -> dict:
warnings = []
magnet_failed = False
try:
- rows = PUBLIC_API.magnets(code, refresh=args.get('refresh') == '1')
+ rows = PUBLIC_API.magnets(code, refresh=args.get('refresh') == '1', movie=movie)
except PublicSourceUnavailable as error:
rows = []
magnet_failed = True
@@ -342,7 +355,7 @@ def q_jav_resources(contract, args) -> dict:
comments_failed = False
try:
for page in range(1, MAX_REVIEW_PAGES + 1):
- reviews = PUBLIC_API.reviews(code, page, refresh=args.get('refresh') == '1')
+ reviews = PUBLIC_API.reviews(code, page, refresh=args.get('refresh') == '1', movie=movie)
for review in reviews:
origin = source_url + "#review-item-" + str(review.get("id") or "")
comments.extend(parse_comment_text(str(review.get("content") or ""), origin,
diff --git a/src/peach/web_jav_discovery.py b/src/peach/web_jav_discovery.py
index cdcb1e386..f2d5765ff 100644
--- a/src/peach/web_jav_discovery.py
+++ b/src/peach/web_jav_discovery.py
@@ -24,7 +24,7 @@ def _cached_movie(code, contract):
with public_cache_scope(contract.cover_root.parent / 'jav-public'):
source = PUBLIC_API.peek_movie(code)
if source:
- metadata.remember(code, source, checked_at=cache_checked_at('movie:' + code, exact=True))
+ metadata.remember(code, source, checked_at=cache_checked_at('movie:' + code + ':'))
return source or {}
diff --git a/tests/test_jav_cache.py b/tests/test_jav_cache.py
index 8c7619886..4fcd5d90a 100644
--- a/tests/test_jav_cache.py
+++ b/tests/test_jav_cache.py
@@ -37,20 +37,21 @@ class PublicationCacheTests(unittest.TestCase):
root = Path(directory).resolve()
now = [stamp('2026-10-13')]
api = self.api(root, now)
- movie = {'id': 'Exact', 'number': 'EXM-001', 'release_date': '2020-01-01'}
- api._cached('movie:EXM-001', lambda: movie)
+ movie = {'id': 'Exact', 'number': 'EXM-001', 'release_date': '2020-01-01', 'javdb_rule': 2}
+ api._cached('movie-id:EXM-001', lambda: ['Exact'])
+ api._cached('movie:EXM-001:Exact', lambda: movie)
api._rows = Mock(side_effect=[[{'id': 'review'}], [{'hash': 'resource'}]])
self.assertEqual(api.reviews('EXM-001', 1), [{'id': 'review'}])
self.assertEqual(api.magnets('EXM-001'), [{'hash': 'resource'}])
now[0] = stamp('2030-01-01')
reader = self.api(root, now)
- reader._rows = Mock(side_effect=AssertionError('缓存读取不得触发来源'))
+ reader._rows = Mock(return_value=[{'id': 'Exact', 'number': 'EXM-001'}])
with public_cache_scope(root):
self.assertEqual(reader.movie('EXM-001'), movie)
self.assertEqual(reader.reviews('EXM-001', 1), [{'id': 'review'}])
self.assertEqual(cache_checked_at('reviews:'), stamp('2026-10-13'))
self.assertEqual(reader.magnets('EXM-001'), [{'hash': 'resource'}])
- reader._rows.assert_not_called()
+ reader._rows.assert_called_once_with('/v2/search', 'movies', {'q': 'EXM-001', 'type': 'movie', 'page': 1})
def test_daily_refresh_and_failed_manual_refresh_keep_good_snapshot(self):
with tempfile.TemporaryDirectory() as directory:
@@ -100,9 +101,10 @@ class PublicationCacheTests(unittest.TestCase):
root = Path(directory).resolve()
now = [stamp('2026-10-13')]
api = self.api(root, now)
- movie = {'id': 'Exact', 'number': 'EXM-001', 'release_date': '2020-01-01',
+ movie = {'id': 'Exact', 'number': 'EXM-001', 'release_date': '2020-01-01', 'javdb_rule': 2,
'cover_url': 'https://tp.spfcas.com/cover.jpg?token=temporary'}
- api._cached('movie:EXM-001', lambda: movie)
+ api._cached('movie-id:EXM-001', lambda: ['Exact'])
+ api._cached('movie:EXM-001:Exact', lambda: movie)
api._rows = Mock(return_value=[{'id': 'Exact', 'number': 'EXM-001'}])
api._json = Mock(return_value={'movie': {**movie, 'cover_url': 'https://tp.spfcas.com/current.jpg'}})
now[0] += DAY
@@ -120,8 +122,11 @@ class PublicationCacheTests(unittest.TestCase):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory).resolve()
now = [stamp('2026-10-13')]
- movie = {'number': 'EXM-001', 'release_date': '2020-01-01', 'tags': [], 'score': 4, 'duration': 120}
- self.api(root / 'jav-public', now)._cached('movie:EXM-001', lambda: movie)
+ movie = {'id': 'Exact', 'number': 'EXM-001', 'release_date': '2020-01-01', 'tags': [],
+ 'score': 4, 'duration': 120, 'javdb_rule': 2}
+ writer = self.api(root / 'jav-public', now)
+ writer._cached('movie-id:EXM-001', lambda: ['Exact'])
+ writer._cached('movie:EXM-001:Exact', lambda: movie)
now[0] += 10 * DAY
reader = self.api(root / 'jav-public', now)
metadata = DiscoveryMetadata(root / 'metadata.json', api=reader, clock=lambda: now[0])
diff --git a/tests/test_jav_discovery_metadata.py b/tests/test_jav_discovery_metadata.py
index a873bd7e2..54cae1929 100644
--- a/tests/test_jav_discovery_metadata.py
+++ b/tests/test_jav_discovery_metadata.py
@@ -12,10 +12,29 @@ from peach.web_jav_discovery import _metadata_response
class DiscoveryMetadataTests(unittest.TestCase):
+ def test_legacy_sidecar_identity_is_not_trusted_and_new_original_title_survives_restart(self):
+ self.path.write_text(json.dumps({'version': 1, 'records': {'MIKR-023': {
+ 'movie': {'id': 'WrongFirst', 'number': 'MIKR-023'}, 'checked_at': self.now[0]}}}), encoding='utf8')
+ original = self.path.read_bytes()
+ old_job = self.path.with_suffix('.job.json')
+ old_job.write_text(json.dumps({'status': 'running', 'codes': ['MIKR-023'],
+ 'completed_codes': ['MIKR-023']}), encoding='utf8')
+ old = DiscoveryMetadata(self.path, api=self.api, clock=lambda: self.now[0])
+ self.addCleanup(old.stop)
+ self.assertEqual(old.peek('MIKR-023')[0], {})
+ self.assertIsNone(old.job.snapshot())
+ old.remember('MIKR-023', {'id': 'XeYkdb', 'number': 'MIKR-023',
+ 'title': '显示题', 'origin_title': 'Original 日本語', 'javdb_rule': 2})
+ restarted = DiscoveryMetadata(self.path, api=self.api, clock=lambda: self.now[0])
+ self.addCleanup(restarted.stop)
+ self.assertEqual(restarted.peek('MIKR-023')[0]['origin_title'], 'Original 日本語')
+ self.assertEqual(self.path.read_bytes(), original)
+ self.assertTrue(old_job.exists())
+
def test_restart_retains_batch_identity_budget_and_completed_codes(self):
self.cache.remember('EXM-001', {'number': 'EXM-001', 'tags': [], 'score': 4, 'duration': 120})
- self.path.with_suffix('.job.json').write_text(json.dumps(dict(
- job_id='original', status='running', codes=['EXM-001', 'EXM-002'], refresh=True,
+ self.cache.job_path.write_text(json.dumps(dict(
+ javdb_rule=2, job_id='original', status='running', codes=['EXM-001', 'EXM-002'], refresh=True,
completed=1, completed_codes=['EXM-001'], attempted_codes=['EXM-001'], requests=3, bytes=120,
active_seconds=10)), encoding='utf-8')
with patch.object(self.api, 'movie', return_value={'number': 'EXM-002', 'tags': [], 'score': 4, 'duration': 120}) as movie:
@@ -36,15 +55,15 @@ class DiscoveryMetadataTests(unittest.TestCase):
self.addCleanup(self.cache.stop)
def test_repeated_restart_keeps_checkpoint_before_the_next_item_finishes(self):
- self.path.with_suffix('.job.json').write_text(json.dumps(dict(
- job_id='original', status='running', codes=['EXM-001', 'EXM-002'], refresh=True,
+ self.cache.job_path.write_text(json.dumps(dict(
+ javdb_rule=2, job_id='original', status='running', codes=['EXM-001', 'EXM-002'], refresh=True,
started_at=100, completed=1, completed_codes=['EXM-001'], attempted_codes=['EXM-001'],
failed_codes=[], requests=3, bytes=120, active_seconds=10)), encoding='utf-8')
with patch.object(DiscoveryMetadata, '_run_batch'):
first = DiscoveryMetadata(self.path, api=self.api, clock=lambda: self.now[0])
first.job.thread.join(5)
first.stop()
- saved = json.loads(self.path.with_suffix('.job.json').read_text(encoding='utf-8'))
+ saved = json.loads(self.cache.job_path.read_text(encoding='utf-8'))
self.assertEqual(saved['completed_codes'], ['EXM-001'])
self.assertEqual(saved['started_at'], 100)
with patch.object(self.api, 'movie', return_value={'number': 'EXM-002', 'tags': [], 'score': 4, 'duration': 120}) as movie:
@@ -97,7 +116,7 @@ class DiscoveryMetadataTests(unittest.TestCase):
self.assertNotIn('avatar_image', actors[1])
self.assertTrue(actors[1]['has_avatar'])
self.assertFalse(actors[2]['has_avatar'])
- self.assertNotIn('signature', self.path.read_text(encoding='utf-8'))
+ self.assertNotIn('signature', self.cache.path.read_text(encoding='utf-8'))
def test_source_cooldown_stops_batch_and_preserves_failure(self):
def unavailable(code):
diff --git a/tests/test_javdb_public.py b/tests/test_javdb_public.py
index 8b1818728..46a4884b3 100644
--- a/tests/test_javdb_public.py
+++ b/tests/test_javdb_public.py
@@ -6,7 +6,7 @@ import threading
import unittest
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
-from unittest.mock import patch
+from unittest.mock import Mock, patch
from urllib.parse import parse_qs, urlsplit
from PIL import Image
@@ -22,6 +22,70 @@ from support.ledger import fresh_ledger
class PublicProtocolTests(unittest.TestCase):
+ def test_same_number_source_ids_are_ambiguous_in_both_orders(self):
+ rows = [{'id': 'XeYkdb', 'number': 'MIKR-023', 'maker_name': 'MOODYZ', 'release_date': '2025-06-17',
+ 'duration': 120, 'actors': [{'id': 'ActorA', 'name': '甲'}]},
+ {'id': 'MO50k', 'number': 'MIKR-023', 'maker_name': 'Mr.michiru', 'release_date': '2022-05-26',
+ 'duration': 120, 'actors': [{'id': 'ActorB', 'name': '乙'}]}]
+ for matches in (rows, rows[::-1]):
+ api = PublicAPI(sleeper=lambda _: None)
+ with patch.object(api, '_rows', return_value=matches), patch.object(api, '_json') as detail:
+ with self.assertRaisesRegex(PublicSourceUnavailable, '多个作品'):
+ api.movie('MIKR-023')
+ detail.assert_not_called()
+
+ def test_old_number_only_cache_and_unmarked_hint_do_not_bypass_ambiguity(self):
+ from peach.jav_cache import PublicSnapshots
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ old = PublicSnapshots(root)
+ old.save('movie-id:MIKR-023', 'XeYkdb', None)
+ old.save('movie:MIKR-023', {'id': 'XeYkdb', 'number': 'MIKR-023'}, '2025-06-17')
+ paths = {path: path.read_bytes() for path in root.glob('*.json')}
+ api = PublicAPI(cache_root=root, sleeper=lambda _: None)
+ api.remember_movie_identity('MIKR-023', {'id': 'XeYkdb', 'number': 'MIKR-023'})
+ with patch.object(api, '_rows', return_value=[{'id': identity, 'number': 'MIKR-023'}
+ for identity in ('XeYkdb', 'MO50k')]):
+ with self.assertRaisesRegex(PublicSourceUnavailable, '多个作品'):
+ api.movie('MIKR-023')
+ for path, body in paths.items():
+ self.assertEqual(path.read_bytes(), body)
+
+ def test_refresh_ambiguity_is_not_replaced_by_previous_success_snapshot(self):
+ api = self.api()
+ self.assertEqual(api.movie('EXM-001')['id'], 'Exact')
+ rows = [{'id': identity, 'number': 'EXM-001'} for identity in ('Exact', 'Another')]
+ with patch.object(api, '_rows', return_value=rows):
+ with self.assertRaisesRegex(PublicSourceUnavailable, '多个作品'):
+ api.movie('EXM-001', refresh=True)
+ with self.assertRaisesRegex(PublicSourceUnavailable, '多个作品'):
+ api.movie('EXM-001')
+
+ def test_transient_identity_refresh_failure_reuses_the_same_confirmed_work(self):
+ api = self.api()
+ first = api.movie('EXM-001')
+ with patch.object(api, '_rows', side_effect=PublicSourceUnavailable('临时网络失败')):
+ self.assertEqual(api.movie('EXM-001', refresh=True)['id'], first['id'])
+
+ def test_bound_resources_do_not_resolve_the_number_to_another_work(self):
+ api = PublicAPI(sleeper=lambda _: None)
+ first = {'id': 'XeYkdb', 'number': 'MIKR-023'}
+ with patch.object(api, 'movie', return_value={'id': 'MO50k', 'number': 'MIKR-023'}) as lookup, \
+ patch.object(api, '_rows', return_value=[]) as rows:
+ self.assertEqual(api.magnets('MIKR-023', movie=first), [])
+ self.assertEqual(api.reviews('MIKR-023', 1, movie=first), [])
+ lookup.assert_not_called()
+ self.assertEqual([call.args[0] for call in rows.call_args_list],
+ ['/v1/movies/XeYkdb/magnets', '/v1/movies/XeYkdb/reviews'])
+
+ def test_javdb_preview_cache_is_bound_to_source_id_and_ignores_old_number_only_result(self):
+ api = PublicAPI()
+ api._cached('preview:javdb:MIKR-023', lambda: {'url': 'old'})
+ for identity, url in (('XeYkdb', 'first'), ('MO50k', 'second'), (None, 'checked-search')):
+ fetch = Mock(return_value=url)
+ self.assertEqual(api.official_preview('MIKR-023', fetch, provider='javdb', source_id=identity)['url'], url)
+ fetch.assert_called_once()
+
def test_signature_rejects_multiple_recipes_unknown_time_and_control_literals(self):
script = ('const AA="sample-middle",BB="sample-suffix";'
'function makeHeader(){const now=Math.floor(Date.now()/1e3).toString();'
@@ -70,7 +134,8 @@ class PublicProtocolTests(unittest.TestCase):
started.set()
self.assertTrue(release.wait(3))
return 'pixels'
- api._cached('movie:EXM-001', lambda: {'id': 'Exact'})
+ api._cached('movie-id:EXM-001', lambda: ['Exact'])
+ api._cached('movie:EXM-001:Exact', lambda: {'id': 'Exact'})
with ThreadPoolExecutor(max_workers=3) as pool:
first = pool.submit(api._cached, 'image:Exact:0', fetch)
self.assertTrue(started.wait(1))
@@ -87,7 +152,8 @@ class PublicProtocolTests(unittest.TestCase):
def test_images_use_three_slots_and_leave_api_interval_intact(self):
api = PublicAPI(lambda *_: HttpResponse(200, {}, b'pixels'))
- api._cached('movie:EXM-001', lambda: {'id': 'Exact', 'number': 'EXM-001',
+ api._cached('movie-id:EXM-001', lambda: ['Exact'])
+ api._cached('movie:EXM-001:Exact', lambda: {'id': 'Exact', 'number': 'EXM-001',
'preview_images': [f'https://tp.spfcas.com/{index}.jpg' for index in range(6)]})
started, release = threading.Event(), threading.Event()
lock = threading.Lock()
@@ -136,11 +202,11 @@ class PublicProtocolTests(unittest.TestCase):
def test_known_movie_identity_skips_search_and_keeps_full_detail_validation(self):
api = self.api()
- api.remember_movie_identity('EXM-001', {'id': 'Exact', 'number': 'EXM-001'})
+ api.remember_movie_identity('EXM-001', {'id': 'Exact', 'number': 'EXM-001', 'javdb_rule': 2})
self.assertEqual(api.movie('EXM-001')['id'], 'Exact')
self.assertFalse(any('/v2/search' in request.url for request in self.requests))
wrong = self.api(wrong_code=True)
- wrong.remember_movie_identity('EXM-001', {'id': 'Exact', 'number': 'EXM-001'})
+ wrong.remember_movie_identity('EXM-001', {'id': 'Exact', 'number': 'EXM-001', 'javdb_rule': 2})
with self.assertRaises(PublicSourceUnavailable):
wrong.movie('EXM-001')
@@ -307,6 +373,58 @@ class PublicProtocolTests(unittest.TestCase):
class DetailProjectionTests(unittest.TestCase):
+ def test_ambiguous_detail_does_not_fall_back_to_number_bound_preview(self):
+ from peach.javdb_public import PublicIdentityAmbiguous
+ with patch('peach.web_jav_detail.PUBLIC_API.movie', side_effect=PublicIdentityAmbiguous('多个作品')), \
+ patch('peach.web_jav_detail._preview_fields') as preview:
+ result = q_jav_detail(self.contract, {'code': 'MIKR-023'})
+ self.assertEqual(result['state'], 'partial')
+ self.assertEqual(result['preview_state'], 'unavailable')
+ self.assertEqual(result['source_url'], '')
+ preview.assert_not_called()
+
+ def test_ambiguous_artwork_does_not_present_previous_number_bound_cover_or_link(self):
+ from peach.web_jav_detail import _artwork_fields
+ with patch.object(self.contract, 'has_cover', return_value=True):
+ result = _artwork_fields(self.contract, 'MIKR-023', {
+ 'link': 'https://javdb.com/v/OldFirst',
+ 'cover_url': 'https://pics.dmm.co.jp/digital/video/mikr00023/mikr00023pl.jpg'}, {}, ambiguous=True)
+ self.assertFalse(result['has_cover'])
+ self.assertEqual((result['cover_url'], result['source_url']), ('', ''))
+
+ def test_review_refresh_uses_the_identity_returned_by_that_refresh(self):
+ api = PublicAPI()
+ first = {'id': 'XeYkdb', 'number': 'MIKR-023'}
+ with patch.object(api, 'movie', side_effect=[first, {'id': 'MO50k', 'number': 'MIKR-023'}]) as movie, \
+ patch.object(api, '_rows', return_value=[]) as rows, \
+ patch('peach.web_jav_detail.PUBLIC_API', api):
+ result = q_jav_reviews(self.contract, {'code': 'MIKR-023', 'refresh': '1'})
+ self.assertTrue(result['ok'])
+ movie.assert_called_once_with('MIKR-023', refresh=True)
+ rows.assert_called_once_with('/v1/movies/XeYkdb/reviews', 'reviews', {'page': 1})
+
+ def test_detail_preserves_original_title_and_does_not_edit_display_title(self):
+ movie = {'id': 'XeYkdb', 'number': 'MIKR-023', 'title': '自动翻译标题',
+ 'origin_title': 'Original *A* **j*yqy*pq* 日本語'}
+ with patch('peach.web_jav_detail.PUBLIC_API.movie', return_value=movie):
+ result = q_jav_detail(self.contract, {'code': 'MIKR-023'})
+ self.assertEqual(result['title'], movie['origin_title'])
+ self.assertEqual(movie['title'], '自动翻译标题')
+
+ def test_legacy_image_bytes_cannot_bypass_ambiguous_source_and_new_ids_have_distinct_bytes(self):
+ from peach.remote_media_cache import cache_for
+ cache_for(self.contract).bytes('jav:MIKR-023:image:cover', lambda: b'old wrong work')
+ with patch('peach.web_jav_detail.PUBLIC_API.movie', side_effect=PublicSourceUnavailable('多个作品')):
+ with self.assertRaisesRegex(PublicSourceUnavailable, '多个作品'):
+ jav_image(self.contract, {'code': 'MIKR-023'})
+ for identity, data in (('XeYkdb', 'Zmlyc3Q='), ('MO50k', 'c2Vjb25k')):
+ source = {'id': identity, 'number': 'MIKR-023'}
+ with patch('peach.web_jav_detail.PUBLIC_API.movie', return_value=source), \
+ patch('peach.web_jav_detail.PUBLIC_API.image', return_value='data:image/jpeg;base64,' + data) as image:
+ self.assertEqual(jav_image(self.contract, {'code': 'MIKR-023'})[0],
+ b'first' if identity == 'XeYkdb' else b'second')
+ image.assert_called_once_with('MIKR-023', 'cover', movie=source)
+
def test_browser_people_projects_local_actor_rules_without_writes(self):
from peach.web_jav_browser_people import cached_people
source = {'actors': [{'name': '甲', 'gender': 0}, {'name': '乙', 'gender': 1},
@@ -337,10 +455,12 @@ class DetailProjectionTests(unittest.TestCase):
original = io.BytesIO()
Image.new('RGB', (800, 438), '#777').save(original, 'JPEG')
data_url = 'data:image/jpeg;base64,' + base64.b64encode(original.getvalue()).decode('ascii')
- with patch('peach.web_jav_detail.PUBLIC_API.image', return_value=data_url) as fetch:
- detail, _max_age = jav_image(self.contract, {'code': 'MIDA-085'})
- with patch('peach.web_jav_detail.PUBLIC_API.image', side_effect=AssertionError('缓存命中不请求来源')):
- self.assertEqual(jav_image(self.contract, {'code': 'MIDA-085'})[0], original.getvalue())
+ source = {'id': 'Exact', 'number': 'MIDA-085'}
+ with patch('peach.web_jav_detail.PUBLIC_API.movie', return_value=source):
+ with patch('peach.web_jav_detail.PUBLIC_API.image', return_value=data_url) as fetch:
+ detail, _max_age = jav_image(self.contract, {'code': 'MIDA-085'})
+ with patch('peach.web_jav_detail.PUBLIC_API.image', side_effect=AssertionError('缓存命中不请求图片')):
+ self.assertEqual(jav_image(self.contract, {'code': 'MIDA-085'})[0], original.getvalue())
self.assertEqual(detail, original.getvalue())
fetch.assert_called_once()
@@ -364,7 +484,7 @@ class DetailProjectionTests(unittest.TestCase):
def test_saved_identity_fetches_actor_portrait_without_code_search(self):
from peach.jav_discovery_metadata import discovery_metadata
metadata = discovery_metadata(self.contract)
- metadata.remember('EXM-001', {'id': 'Exact', 'number': 'EXM-001', 'actors': [
+ metadata.remember('EXM-001', {'id': 'Exact', 'number': 'EXM-001', 'javdb_rule': 2, 'actors': [
{'id': 'Person', 'name': '甲', 'gender': 0,
'avatar_url': 'https://tp.spfcas.com/actor.jpg?signature=expired'}]})
original = io.BytesIO()
@@ -509,7 +629,7 @@ class DetailProjectionTests(unittest.TestCase):
patch('peach.web_jav_detail.PUBLIC_API.magnets', return_value=rows) as magnets, \
patch('peach.web_jav_detail.PUBLIC_API.reviews', return_value=[]):
result = q_jav_resources(self.contract, {'code': 'EXM-001', 'refresh': '1'})
- magnets.assert_called_once_with('EXM-001', refresh=True)
+ magnets.assert_called_once_with('EXM-001', refresh=True, movie={'id': 'Exact'})
self.assertEqual(result['state'], 'ready')
self.assertEqual(len(result['items']), 1)
self.assertEqual(result['items'][0]['info_hash'], fingerprint)
@@ -579,13 +699,15 @@ class DetailProjectionTests(unittest.TestCase):
self.assertEqual(jav_image(self.contract, {'code': 'EXM-001', 'image': 'actor:Unknown'})[0], b'sample')
image.assert_called_once_with('https://tp.spfcas.com/b.jpg')
- def test_image_bytes_are_reused_without_querying_source_again(self):
- with patch('peach.web_jav_detail.PUBLIC_API.image', return_value='data:image/jpeg;base64,c2FtcGxl') as image:
- first = jav_image(self.contract, {'code': 'EXM-001'})
- image.assert_called_once()
+ def test_image_bytes_are_reused_for_the_same_confirmed_source_identity(self):
+ source = {'id': 'Exact', 'number': 'EXM-001'}
+ with patch('peach.web_jav_detail.PUBLIC_API.movie', return_value=source):
+ with patch('peach.web_jav_detail.PUBLIC_API.image', return_value='data:image/jpeg;base64,c2FtcGxl') as image:
+ first = jav_image(self.contract, {'code': 'EXM-001'})
+ image.assert_called_once_with('EXM-001', 'cover', movie=source)
+ with patch('peach.web_jav_detail.PUBLIC_API.image', side_effect=PublicSourceUnavailable('offline')) as offline:
+ replay = jav_image(self.contract, {'code': 'EXM-001'})
self.assertEqual(first, (b'sample', JAV_IMAGE_MAX_AGE))
- with patch('peach.web_jav_detail.PUBLIC_API.image', side_effect=PublicSourceUnavailable('offline')) as offline:
- replay = jav_image(self.contract, {'code': 'EXM-001'})
self.assertEqual(replay, first)
offline.assert_not_called()
diff --git a/tests/test_metadata_library.py b/tests/test_metadata_library.py
index 0056a9e5c..be4dc4a59 100644
--- a/tests/test_metadata_library.py
+++ b/tests/test_metadata_library.py
@@ -89,6 +89,50 @@ def _archive_pages(url):
class LibraryNfoTests(unittest.TestCase):
+ def test_javdb_candidate_keeps_its_verified_rule_and_source_identity(self):
+ from peach.library_processing import _merge_candidates
+ from peach.javdb_identity import current_candidate
+ groups = {}
+ document = {'id': 'MIKR-023', 'title': '原题', 'javdb_rule': 2,
+ 'source_id': 'XeYkdb', 'source_url': 'https://javdb.com/v/XeYkdb'}
+ row = {'id': 1, 'name': 'MIKR-023.mp4', 'path': 'R:/Films/MIKR-023.mp4', 'size': 1}
+ _merge_candidates(groups, row, 'MIKR-023', 'javdb', document, self.root / 'raw.json', {})
+ candidate = json.loads(groups['asset:1:title']['candidates_json'])[0]
+ self.assertTrue(current_candidate(candidate))
+ self.assertEqual((candidate['source_id'], candidate['source_url']),
+ ('XeYkdb', 'https://javdb.com/v/XeYkdb'))
+
+ @windows_ledger_roots
+ def test_targeted_recheck_asks_a_source_despite_old_miss_but_normal_collection_keeps_it(self):
+ from peach.library_processing import _MissCache, misses_path
+ for recheck in (False, True):
+ with self.subTest(recheck=recheck):
+ root = self.root / ('recheck' if recheck else 'normal')
+ media = root / 'media'
+ media.mkdir(parents=True)
+ video = media / 'DASS-468.mp4'
+ video.write_bytes(b'video')
+ db = fresh_ledger(root)
+ config = PeachConfig(root, root / 'config.toml', present=True,
+ locations={'local': (str(media),)})
+ with closing(sqlite3.connect(db)) as connection, connection:
+ connection.execute('INSERT INTO asset(id,location,path,name,medium,code) VALUES(1,?,?,?,?,?)',
+ ('local', str(video), video.name, 'video', 'DASS-468'))
+ misses = _MissCache(misses_path(config))
+ misses.record('javdb', 'DASS-468')
+ misses.record('javdb', 'DASS-469')
+ document = {'id': 'DASS-468', 'title': '原题', 'maker': 'Das', 'release_date': '2024-09-10',
+ 'actresses': [{'japanese_name': '胡桃さくら'}]}
+ provider = stub_provider()
+ provider.query.side_effect = lambda code, source='r18dev', **kwargs: (
+ {**document, 'genres': ['高跟鞋'], 'javdb_rule': 2} if source == 'javdb' else document)
+ provider.cover.return_value = False
+ process_library(config, db, root / 'generated', root / 'covers', asset_ids=[1],
+ stage='metadata', recheck_existing=recheck, provider_factory=lambda: provider)
+ javdb_calls = [call for call in provider.query.call_args_list if call.args[1] == 'javdb']
+ self.assertEqual(len(javdb_calls), 1 if recheck else 0)
+ self.assertTrue(_MissCache(misses_path(config)).fresh('javdb', 'DASS-469'))
+
def test_cover_failure_retains_redacted_exception_in_scraping_export_and_remains_retryable(self):
from peach.library_processing import _MissCache, _ProcessingIssues, _RemoteSession
from peach.scraping_diagnostics import _issues
@@ -2107,7 +2151,7 @@ class LibraryNfoTests(unittest.TestCase):
documents = {
'r18dev': {'id': 'DASS-468', 'title': '甘やかされて', 'maker': 'Das',
'release_date': '2024-09-10', 'actresses': [{'japanese_name': '胡桃さくら'}]},
- 'javdb': {'id': 'DASS-468', 'genres': ['高跟鞋', 'Anal Play'],
+ 'javdb': {'id': 'DASS-468', 'genres': ['高跟鞋', 'Anal Play'], 'javdb_rule': 2,
'source_url': 'https://javdb.com/v/example'},
}
for source, payload in documents.items():
@@ -2129,6 +2173,25 @@ class LibraryNfoTests(unittest.TestCase):
[('javdb', ['高跟', '屁眼'])])
self.assertEqual(Path(rows['tags'][0]['raw_snapshot']), directory / 'DASS-468-javdb.json')
+ def test_javdb_snapshot_requires_current_identity_evidence_without_removing_raw_json(self):
+ from peach.library_processing import _MissCache, _RemoteSession, misses_path
+ from peach.javdb_identity import current_evidence
+ config = PeachConfig(self.root, self.root / 'config.toml', present=True)
+ directory = config.directory('sources') / 'library-metadata'
+ directory.mkdir(parents=True)
+ for marker in (None, 1, True, '2', 2):
+ with self.subTest(marker=marker):
+ payload = {'id': 'DASS-468', 'title': '原题', 'javdb_rule': marker}
+ path = directory / 'DASS-468-javdb.json'
+ path.write_text(json.dumps(payload), encoding='utf8')
+ artifact_rules.remember(path, str(upgrade_rules.PROVIDER_EVIDENCE))
+ before = path.read_bytes()
+ session = _RemoteSession(config, None, _MissCache(misses_path(config)), retrying=False)
+ self.assertEqual(session._cached_evidence(['javdb'], 'DASS-468'),
+ [('javdb', payload, path)] if marker == 2 and type(marker) is int else [])
+ self.assertEqual(current_evidence(payload), marker == 2 and type(marker) is int)
+ self.assertEqual(path.read_bytes(), before)
+
def test_damaged_snapshot_is_not_reused_even_with_a_current_receipt(self):
from peach.library_processing import _MissCache, _RemoteSession, misses_path
config = PeachConfig(self.root, self.root / 'config.toml', present=True)
diff --git a/tests/test_metadata_sources.py b/tests/test_metadata_sources.py
index dedfbdc98..5babe269c 100644
--- a/tests/test_metadata_sources.py
+++ b/tests/test_metadata_sources.py
@@ -25,6 +25,49 @@ DEMO = SiteConfig(name="demo", label="Demo", provider="demo-page", base_url="htt
class JavDBResourceTests(unittest.TestCase):
+ def test_title_projection_accepts_original_only_display_only_and_english_original(self):
+ from peach.sources.javdb import JavDBSource
+ panel = '<div class="panel-block"><strong>番號:</strong><span class="value">MIKR-023</span></div>'
+ for html, expected in (('<span class="origin-title">日本語原題</span>', '日本語原題'),
+ ('<strong class="current-title">仅显示标题</strong>', '仅显示标题'),
+ ('<strong class="current-title">译文</strong><span class="origin-title">Original *Title*</span>',
+ 'Original *Title*')):
+ with self.subTest(html=html):
+ payload = JavDBSource().parse(Page('https://javdb.com/v/XeYkdb', (html + panel).encode()), 'MIKR-023').payload()
+ self.assertEqual(payload['title'], expected)
+
+ def test_original_title_is_separate_from_localized_display_and_keeps_source_identity(self):
+ from peach.sources.javdb import JavDBSource
+ html = '''<strong class="current-title">自动翻译标题</strong>
+ <span class="origin-title" style="display:none">Original *A* **j*yqy*pq* 日本語</span>
+ <div class="panel-block"><strong>番號:</strong><span class="value">MIKR-023</span></div>'''
+ payload = JavDBSource().parse(Page('https://javdb.com/v/XeYkdb', html.encode()), 'MIKR-023').payload()
+ self.assertEqual(payload['title'], 'Original *A* **j*yqy*pq* 日本語')
+ self.assertEqual(payload['origin_title'], payload['title'])
+ self.assertEqual(payload['display_title'], '自动翻译标题')
+ self.assertEqual(payload['source_id'], 'XeYkdb')
+ self.assertEqual(payload['javdb_rule'], 2)
+
+ def test_same_number_different_source_ids_are_not_selected_by_search_order(self):
+ from peach.sources.javdb import JavDBSource
+ source = JavDBSource()
+ for identities in [('XeYkdb', 'MO50k'), ('MO50k', 'XeYkdb')]:
+ html = ''.join(f'<a href="/v/{identity}" class="box" title="film"><strong>MIKR-023</strong></a>'
+ for identity in identities)
+ with self.subTest(order=identities), patch.object(source, '_page', return_value=Page('https://javdb.com/search', html.encode())) as get:
+ with self.assertRaises(SourceFailure) as caught:
+ source.fetch('MIKR-023', session=None)
+ self.assertEqual(caught.exception.detail, 'identity_ambiguous')
+ self.assertEqual(get.call_count, 1)
+
+ def test_repeated_card_for_one_source_id_is_not_a_second_work(self):
+ from peach.sources.javdb import JavDBSource
+ source = JavDBSource()
+ card = '<a href="/v/XeYkdb" class="box" title="film"><strong>MIKR-023</strong></a>'
+ detail = Page('https://javdb.com/v/XeYkdb', b'detail')
+ with patch.object(source, '_page', side_effect=[Page('https://javdb.com/search', (card * 2).encode()), detail]):
+ self.assertEqual(source.fetch('MIKR-023', session=None).url, detail.url)
+
def test_categories_keep_raw_words_and_deduplicate_links_after_identity_check(self):
from peach.sources.javdb import JavDBSource
from peach.metadata import extract_peach_fields
diff --git a/tests/test_web_review.py b/tests/test_web_review.py
index 67ab5fe6a..2062c8b7f 100644
--- a/tests/test_web_review.py
+++ b/tests/test_web_review.py
@@ -213,7 +213,8 @@ class ReviewQueueTests(unittest.TestCase):
extra = {key: item for key, item in value.items()
if key not in {"source", "display", "value"}}
value = value["value"]
- return {**extra, "candidate_key": f"{item['item_key']}:{index}", "source": source,
+ return {**({'javdb_rule': 2, 'source_id': 'FixtureWork'} if source == 'javdb' else {}),
+ **extra, "candidate_key": f"{item['item_key']}:{index}", "source": source,
"display_value": display, "value": value, "confidence": 0.9,
# 来源自报的番号。落库前要和这一行的番号对得上,所以默认取同一个值;
# 用例给 `provider_id` 就能造出「来源返回的是别的作品」那一半。
@@ -229,6 +230,28 @@ class ReviewQueueTests(unittest.TestCase):
def _auto(self):
return auto_apply_metadata(self.contract.database, self.candidates)
+ def test_old_javdb_candidate_replay_cannot_fill_fields_but_current_identity_can(self):
+ self._asset(801, 'MIKR-023', 'MIKR-023.mp4')
+ for marker, identity, expected in ((None, '', 0), (2, '', 0), (2, 'XeYkdb', 1)):
+ with self.subTest(marker=marker, identity=identity):
+ path = self.write_metadata_rows([{'item_key': 'old-javdb-title', 'code': 'MIKR-023',
+ 'field': 'title', 'current': '', 'candidates': [{'source': 'javdb',
+ 'value': '原题', 'javdb_rule': marker, 'source_id': identity}]}])
+ before = path.read_bytes()
+ self.assertEqual(self._auto()['applied'], expected)
+ self.assertEqual(path.read_bytes(), before)
+
+ def test_human_can_approve_old_javdb_evidence_explicitly(self):
+ self._asset(802, 'MIKR-024', 'MIKR-024.mp4')
+ self.write_metadata_rows([{'item_key': 'old-human-title', 'code': 'MIKR-024',
+ 'field': 'title', 'current': '', 'candidates': [{'source': 'javdb',
+ 'value': '人工核对标题', 'javdb_rule': None, 'source_id': ''}]}])
+ candidate = self.queue_row('metadata_fields', 'old-human-title')['candidates'][0]
+ rm_review.w_review_decision(self.contract, {'category': 'metadata_fields', 'item_key': 'old-human-title',
+ 'status': 'approved', 'candidate_key': candidate['candidate_key']})
+ with self.contract.database.read_connection() as connection:
+ self.assertEqual(connection.execute('SELECT catalog_title FROM asset WHERE id=802').fetchone()[0], '人工核对标题')
+
def test_auto_batch_reads_only_its_metadata_decisions_across_query_chunks(self):
keys = [f"batch-{index}" for index in range(205)]
with self.contract.database.write_transaction() as connection: