Peach: Local-first video metadata scraper and media manager.
feat(web): 头像框打开时在后台补取过期的图库索引
longmeidaobe46a9cd6c5d
diff --git a/docs/REUSE.md b/docs/REUSE.mdindex f73ab1295..dd054e992 100644--- a/docs/REUSE.md+++ b/docs/REUSE.md@@ -682,7 +682,7 @@ A 盘垃圾隔离使用短名称 `peach-purge-<32 位 UUID>.peach-quarantine`,| 相关推荐 | OpenAver `8cc17e50453d9f69a81f5fee1a072df80f7aab73` 的 Tag IDF + 系列/片商/出演者规则只作固定算法参考(MIT) | 独立实现规范实体评分、MMR 多样性、稳定 seed、解释原因与负反馈边界;不复制上游 UI/源码 || 女优姓名对照 | `li-peifeng/Jav-Actors-Mapping` 的固定 revision,仅作私有输入(仓库未声明许可证,不随 Peach 分发) | 精确匹配、冲突复核、别名、来源与真实 ledger 写入 || 女优头像候选 | Gfriends 的 GitHub raw 索引与单张媒体(只作外部 Provider,不克隆图库);r18.dev 人物对象给出的 DMM 官方 `actjpgs` 缩略图只作精确身份绑定的首次头像 | 名字链、质量档位、格式/尺寸/SHA-256 门槛、候选缓存、provenance、健康统计和人工复核;首次头像只在人物主名精确一致、实体已连到同一资产且没有在位头像时安装。自动档案缩略图未达标准尺寸时进入图库升级核验;没有可靠图库候选则保留档案照。换头像端点记录人工选择,自动后继保留该选择 |-| Gfriends 索引读法 | `src/peach/gfriends.py` | 页面与批处理共用:`Filetree.json` 解析、名字链匹配、`quality_key` 排序、raw 地址与本地缓存缓存有效期。**目录前缀是来源优先级,不是清晰度**(上游 README 写的「质量升序」不准):`0-` 网友投稿、`1-`~`8-` 写真机构与片商官方、往后是原图两三百像素的大型数据库,排第一的是最该先试的一张;`AI-Fix-` 是上游放大去水印版,去掉前缀取原件 |+| Gfriends 索引读法 | `src/peach/gfriends.py` | 页面与批处理共用:`Filetree.json` 解析、名字链匹配、`quality_key` 排序、raw 地址与本地缓存缓存有效期,`fetch_index` 校验后原子换入缓存(服务端由 `avatar_picker.IndexRefresher` 在后台调)。**目录前缀是来源优先级,不是清晰度**(上游 README 写的「质量升序」不准):`0-` 网友投稿、`1-`~`8-` 写真机构与片商官方、往后是原图两三百像素的大型数据库,排第一的是最该先试的一张;`AI-Fix-` 是上游放大去水印版,去掉前缀取原件 || 「这个地址能不能让 Peach 替人去取」 | `src/peach/http.py` 的 `public_https_url` + `resolves_publicly` | 追更图片代理与手填头像地址共用:必须 https 与公网域名,不收 IP 字面量与用户信息,解析出的每个地址都要 `is_global`;否则 Peach 就是能访问路由器、NAS 与本机端口的跳板 || 头像写入 | `src/peach/avatar_provider.install_entity_avatar` | 采集脚本、复核页与换头像共用:`.img` 经临时文件原子替换,`.ct`、`.provenance.json`、`.face.json` 一起换;检不出脸要删掉旧 `.face.json`,否则页面按上一张的脸框取景且看不出错 || 厂牌 Logo 候选 | 厂牌官网确认的社交 handle → unavatar URL 解析 → 平台 CDN 单图 | handle 归属、内容缓存、方形归一、精确/感知哈希、provenance、健康统计与变化复核 |diff --git a/docs/SOURCING.md b/docs/SOURCING.mdindex d255d9611..23c6415dd 100644--- a/docs/SOURCING.md+++ b/docs/SOURCING.md@@ -702,6 +702,9 @@ Instagram 的独立用户登录会话未取得,自动适配器不进正式依- Gfriends 只按 `Filetree.json` 和单张 raw 媒体当外部 Provider 用,不克隆图库,不把图片放进 Git。索引缓存按mtime 计龄(一天),取不到新索引就用旧缓存并在输出里告警;那一轮的「未收录」记 error 不记 no_match,否则`--resume` 会把一次网络失败固化成永久答案。Gfriends 每人中位 1 张、缓存图中位 500×600,不按作品分组。+- 服务进程在头像框打开时检查索引缓存:缺失或过期就在后台取一份(`avatar_picker.IndexRefresher`),出网走采集+ 统一出口与 Peach 代理设置。这一次照旧按旧缓存列候选;取不到只记日志、15 分钟内不再试,不进任务中心。+ 索引不随安装包分发:上游每天更新,带快照会让快照之后收录的人一直显示查不到。### 审计与补缺口diff --git a/src/peach/api.py b/src/peach/api.pyindex 530ab55f2..752c2b205 100644--- a/src/peach/api.py+++ b/src/peach/api.py@@ -45,6 +45,7 @@ from .follow_scheduler import FEED_JOB_ID, FollowScheduleConfig, FollowUpdateSchfrom .follow_covers import FollowCoverServicefrom .follow_faces import FollowFaceIndexfrom .follow_stream import FollowMediaResolver+from .avatar_picker import GFRIENDS_CACHE, IndexRefresherfrom .http import HttpxTransportfrom .health import inspect_databasefrom .media import (@@ -271,7 +272,15 @@ def create_app(network_reachability = NetworkReachability()resolver = FFmpegResolver(settings.ffmpeg_root)http_transport = HttpxTransport()- follow_media_resolver = FollowMediaResolver(http_transport).with_credential_loader(++ def gfriends_transport():+ # 图库索引走采集统一出口:Peach 代理设置与按站冷却都在那里。+ from .scraping_access import SourceTransport+ return SourceTransport(contract.follow_secrets_root)++ gfriends_index = IndexRefresher(+ routes_media._picker_roots(contract)[0] / GFRIENDS_CACHE, gfriends_transport)+ follow_media_resolver =FollowMediaResolver(http_transport).with_credential_loader(lambda provider: web_follow._credential_store(contract).load(provider))# 检查更新、浏览与画面签名的取图共用这一份按站冷却:同一个站被限流,几处都得停。contract.follow_media_resolver = follow_media_resolver@@ -475,6 +484,7 @@ def create_app(app.state.opencode_go = opencode_goapp.state.review_mirror = review_mirrorapp.state.http_transport = http_transport+ app.state.gfriends_index = gfriends_indexapp.state.follow_media_resolver = follow_media_resolverapp.state.follow_cover_service = follow_cover_serviceapp.state.follow_scheduler = follow_schedulerdiff --git a/src/peach/avatar_picker.py b/src/peach/avatar_picker.pyindex a6625686c..8402d860a 100644--- a/src/peach/avatar_picker.py+++ b/src/peach/avatar_picker.py@@ -20,7 +20,10 @@ from __future__ import annotationsimport hashlibimport json+import loggingimport sqlite3+import threading+import timeimport urllib.parsefrom collections.abc import Callablefrom dataclasses import dataclass, replace@@ -86,10 +89,67 @@ ADDRESS_CACHE = "url"STAND_IN_NOTE = "代表作画面,非本人"+#: 图库索引取失败后,隔这么久才再试。头像框一天能开几十次,离线时每开一次就出网+#: 一回只会白等超时;失败只记一行日志,不进任务中心——下次过期照样会再排。+INDEX_RETRY_SECONDS = 15 * 60++LOGGER = logging.getLogger(__name__)++class PickerError(RuntimeError):"""这一次换不成,原因可以直接给用户看。"""+class IndexRefresher:+ """图库索引缺失或过期时,在后台补一份。++ `request` 只排一轮、立刻返回,页面照旧读本地缓存(`gfriends.load_index`);+ 取到的索引下次打开头像框时生效。同一时刻最多一轮在跑。`transport` 每轮新建、+ 用完关掉:一天取一次,用不着常驻连接池。+ """++ def __init__(self, cache_dir: Path, transport: Callable[[], HttpTransport]):+ self.cache_dir = Path(cache_dir)+ self._transport = transport+ self._lock = threading.Lock()+ self._thread: threading.Thread | None = None+ self._failed_at: float | None = None++ def request(self) -> bool:+ """需要的话在后台排一轮,返回是否排上了。"""+ with self._lock:+ if not gfriends.index_stale(self.cache_dir):+ return False+ if self._thread is not None and self._thread.is_alive():+ return False+ if (self._failed_at is not None+ and time.monotonic() - self._failed_at < INDEX_RETRY_SECONDS):+ return False+ self._thread = threading.Thread(target=self.refresh, daemon=True,+ name="PeachGfriendsIndex")+ self._thread.start()+ return True++ def refresh(self) -> bool:+ """在当前线程里取一次。失败不抛,旧缓存原样留着。"""+ try:+ transport = self._transport()+ try:+ gfriends.fetch_index(transport, self.cache_dir)+ finally:+ close = getattr(transport, "close", None)+ if close is not None:+ close()+ except Exception as error:+ LOGGER.warning("图库索引未取得:%s", error)+ with self._lock:+ self._failed_at = time.monotonic()+ return False+ with self._lock:+ self._failed_at = None+ return True++@dataclass(frozen=True)class Choice:"""一个可选项。`ref` 是前端唯一回递的东西。"""@@ -447,8 +507,8 @@ def choices(connection: sqlite3.Connection, providers_root: Path,cover_root: Path | None = None, *, deduplicate: bool = True) -> dict:"""页面要展示的一切:图库同名候选、取过的历史、当前装着的是哪一张。- 索引只读本地缓存。联网补索引是批处理的事——为一次点击同步拉 6 MB,页面会卡在- 那里,而卡住的理由用户完全看不见。+ 索引只读本地缓存,过期了由 `IndexRefresher` 在后台补——为一次点击同步拉 6 MB,+ 页面会卡在那里,而卡住的理由用户完全看不见。"""index_dir = providers_root / GFRIENDS_CACHEnames = name_chain(connection, entity_id)@@ -496,7 +556,7 @@ def choices(connection: sqlite3.Connection, providers_root: Path,"names": names, "matched_names": list(match.names),"choices": [choice.as_dict() for choice in listed],"index_age_hours": round(age / 3600, 1) if age is not None else None,- "index_stale": age is None or age > gfriends.INDEX_MAX_AGE_SECONDS,+ "index_stale": gfriends.index_stale(index_dir),}diff --git a/src/peach/gfriends.py b/src/peach/gfriends.pyindex 32a581434..cfc9f7a0a 100644--- a/src/peach/gfriends.py+++ b/src/peach/gfriends.py@@ -1,4 +1,4 @@-"""Gfriends 头像图库的索引:目录约定、候选排序和本地缓存的读法。+"""Gfriends 头像图库的索引:目录约定、候选排序、本地缓存的读法与取法。图库把每个人的图分散在几十个来源目录下,一个人常有十几张。`Filetree.json` 是那棵文件树的快照,键是展示名(可能是别名),值才是实际文件——两者未必相同。@@ -21,8 +21,16 @@ import urllib.parsefrom dataclasses import dataclass, fieldfrom pathlib import Path+from .fsutil import atomic_write_bytes+from .http import HttpRequest, HttpTransport+#: 图库的 raw 根。索引和图片都挂在它下面。GFRIENDS_RAW = "https://raw.githubusercontent.com/gfriends/gfriends/master/"+#: 索引本身。上游不发版本也不给哈希,能判的只有「解析得出来」。+INDEX_URL = GFRIENDS_RAW + "Filetree.json"+#: 取一次索引的上限。实际 6.5 MB 上下,留足增长余量。+INDEX_MAX_BYTES = 32 * 1024 * 1024+INDEX_TIMEOUT = 60#: 目录名首字符的先后。未知前缀排在最后,不能因为 `find` 返回 -1 反而抢到最前。QUALITY_ORDER = "0123456789abcdefghijklmnopqrstuvwxyz"#: 索引缓存的保鲜期。图库持续增补,缓存不能永不过期:只要文件在就一直复用的话,@@ -82,11 +90,17 @@ def index_age(cache_dir: Path) -> float | None:return None+def index_stale(cache_dir: Path) -> bool:+ """没有缓存或缓存超过保鲜期。"""+ age = index_age(cache_dir)+ return age is None or age > INDEX_MAX_AGE_SECONDS++def load_index(cache_dir: Path) -> dict[str, list[tuple[str, str]]]:"""只读本地缓存的索引。取不到或坏了都返回空。- 联网补索引是长跑批处理的事(`scripts/audit_performer_portraits.py`)。页面这一侧- 宁可少列几个候选,也不该在一次点击里同步拉 6 MB。+ 这里从不联网:页面这一侧宁可少列几个候选,也不该在一次点击里同步拉 6 MB。+ 缓存由 `fetch_index` 补,服务在后台调它(`avatar_picker.IndexRefresher`)。"""try:return parse_filetree((cache_dir / INDEX_NAME).read_bytes())@@ -94,6 +108,23 @@ def load_index(cache_dir: Path) -> dict[str, list[tuple[str, str]]]:return {}+def fetch_index(transport: HttpTransport, cache_dir: Path) -> dict[str, list[tuple[str, str]]]:+ """联网取一份索引,解析得出来才换进缓存,返回解析结果。++ 任何一步失败都抛出,旧缓存原样留着:过期的索引照样能查,只是新收的人查不到,+ 比换成半个 JSON 强。写缓存用临时文件加 `os.replace`,读的一方只会看到完整的一份。+ """+ response = transport(HttpRequest("GET", INDEX_URL, {"Accept": "application/json"}),+ INDEX_TIMEOUT, INDEX_MAX_BYTES)+ if response.status != 200:+ raise ValueError(f"Gfriends 索引 HTTP {response.status}")+ if len(response.body) > INDEX_MAX_BYTES:+ raise ValueError("Gfriends 索引超过大小上限")+ index = parse_filetree(response.body)+ atomic_write_bytes(cache_dir / INDEX_NAME, response.body)+ return index++@dataclass(frozen=True)class Match:"""名字链在图库里的全部落点。diff --git a/src/peach/routes_media.py b/src/peach/routes_media.pyindex 24c2df2bc..7e99bd7d6 100644--- a/src/peach/routes_media.py+++ b/src/peach/routes_media.py@@ -1379,7 +1379,10 @@ def _picker_online(request: Request, connection, entity_id: int):@router.get("/api/avatar-choices")def avatar_choices(request: Request, kind: str = "performer", id: int = 0,args: dict[str, str] = Depends(require_auth)):- """这个人还能换成哪些图。只读,不联网,不写盘。"""+ """这个人还能换成哪些图。只读本地缓存,不等联网。++ 图库索引缺失或过期时顺手在后台排一轮刷新;这一次照旧按旧缓存列,下次打开生效。+ """state = request.app.state.web_contractproviders_root, avatar_root = _picker_roots(state)id = current_entity_id(state, id)@@ -1390,7 +1393,8 @@ def avatar_choices(request: Request, kind: str = "performer", id: int = 0,if kind == "creator":listing["choices"] += [choice.as_dict() for choice in avatar_online.choices(connection, id, project=_picker_online_projection)]- return JSONResponse(listing)+ request.app.state.gfriends_index.request()+ return JSONResponse(listing)@router.api_route("/avatar-choice", methods=["GET", "HEAD"])diff --git a/tests/test_avatar_picker.py b/tests/test_avatar_picker.pyindex 49284d060..5e3bad06e 100644--- a/tests/test_avatar_picker.py+++ b/tests/test_avatar_picker.py@@ -8,6 +8,7 @@ import osimport shutilimport sqlite3import tempfile+import timeimport unittestfrom pathlib import Pathfrom unittest import mock@@ -135,6 +136,99 @@ class IndexTests(unittest.TestCase):self.assertFalse(gfriends.candidates(index, ["谁都不是"]))+OLD_FILETREE = {"Content": {"7-S1": {"旧人.jpg": "旧人.jpg?t=0"}}}+++class IndexRefreshTests(unittest.TestCase):+ """图库索引缺失或过期时由服务在后台补;取不到时旧缓存原样留着,也不反复出网。"""++ def setUp(self):+ self.cache = Path(tempfile.mkdtemp()).resolve() / avatar_picker.GFRIENDS_CACHE+ self.addCleanup(shutil.rmtree, self.cache.parent, True)+ self.calls: list[str] = []+ self.reply = lambda: HttpResponse(200, {}, json.dumps(FILETREE).encode("utf-8"),+ gfriends.INDEX_URL)++ def transport(self):+ def send(request, timeout=None, max_bytes=None):+ self.calls.append(request.url)+ return self.reply()+ return send++ def refresher(self):+ return avatar_picker.IndexRefresher(self.cache, self.transport)++ def seed_old(self, age: float) -> bytes:+ body = json.dumps(OLD_FILETREE).encode("utf-8")+ self.cache.mkdir(parents=True, exist_ok=True)+ path = self.cache / gfriends.INDEX_NAME+ path.write_bytes(body)+ stamp = path.stat().st_mtime - age+ os.utime(path, (stamp, stamp))+ return body++ def run_queued(self, refresher) -> bool:+ queued = refresher.request()+ if queued:+ refresher._thread.join(10)+ return queued++ def test_a_missing_index_is_fetched_in_the_background(self):+ self.assertTrue(self.run_queued(self.refresher()))+ self.assertEqual(self.calls, [gfriends.INDEX_URL])+ self.assertIn("葵つかさ", gfriends.load_index(self.cache))+ self.assertFalse(gfriends.index_stale(self.cache))++ def test_a_stale_index_is_replaced(self):+ self.seed_old(gfriends.INDEX_MAX_AGE_SECONDS + 3600)+ self.assertTrue(self.run_queued(self.refresher()))+ index = gfriends.load_index(self.cache)+ self.assertIn("葵つかさ", index)+ self.assertNotIn("旧人", index)++ def test_a_fresh_index_is_left_alone(self):+ self.seed_old(60)+ self.assertFalse(self.run_queued(self.refresher()))+ self.assertEqual(self.calls, [])++ def test_a_failed_fetch_keeps_the_old_index_and_waits_before_retrying(self):+ """离线时每开一次头像框都出网一回,只会一次次白等超时。"""+ old = self.seed_old(gfriends.INDEX_MAX_AGE_SECONDS + 3600)+ mtime = (self.cache / gfriends.INDEX_NAME).stat().st_mtime+ failures = [+ lambda: HttpResponse(503, {}, b"", gfriends.INDEX_URL),+ lambda: HttpResponse(200, {}, b"<html>", gfriends.INDEX_URL),+ lambda: (_ for _ in ()).throw(OSError("连接被重置")),+ ]+ for reply in failures:+ with self.subTest(reply=reply):+ self.reply = reply+ refresher = self.refresher()+ with self.assertLogs(avatar_picker.LOGGER, "WARNING"):+ self.assertTrue(self.run_queued(refresher))+ path = self.cache / gfriends.INDEX_NAME+ self.assertEqual(path.read_bytes(), old)+ self.assertEqual(path.stat().st_mtime, mtime)+ self.assertTrue(gfriends.index_stale(self.cache))+ self.assertIn("旧人", gfriends.load_index(self.cache))+ calls = len(self.calls)+ self.assertFalse(self.run_queued(refresher))+ self.assertEqual(len(self.calls), calls)+ later = time.monotonic() + avatar_picker.INDEX_RETRY_SECONDS + 1+ self.reply = lambda: HttpResponse(200, {}, json.dumps(FILETREE).encode("utf-8"),+ gfriends.INDEX_URL)+ with mock.patch.object(avatar_picker.time, "monotonic", return_value=later):+ self.assertTrue(self.run_queued(refresher))+ self.assertIn("葵つかさ", gfriends.load_index(self.cache))++ def test_a_missing_index_that_cannot_be_fetched_stays_missing(self):+ self.reply = lambda: HttpResponse(404, {}, b"", gfriends.INDEX_URL)+ with self.assertLogs(avatar_picker.LOGGER, "WARNING"):+ self.assertFalse(self.refresher().refresh())+ self.assertFalse((self.cache / gfriends.INDEX_NAME).exists())+ self.assertEqual(gfriends.load_index(self.cache), {})++class SourceBoundaryTests(unittest.TestCase):"""手填地址的边界。Peach 跑在用户机器上,这道判据挡的是「替人去探内网」。"""@@ -318,8 +412,8 @@ class ChoiceTests(PickerFixture):listed = next(one for one in out if one["ref"] == LIBRARY_REF)self.assertEqual((listed["width"], listed["height"]), (40, 60))- def test_a_stale_index_is_reported_rather_than_refreshed(self):- """页面不为一次点击同步拉 6 MB;过期就说出来,补索引是批处理的事。"""+ def test_a_stale_index_is_reported_and_still_read(self):+ """页面不为一次点击同步拉 6 MB;过期就说出来,补索引交给后台(`IndexRefresher`)。"""self.assertFalse(self.listed()["index_stale"])old = gfriends.INDEX_MAX_AGE_SECONDS + 3600with mock.patch.object(gfriends, "index_age", return_value=old):@@ -902,6 +996,24 @@ class AvatarPickerRouteTests(unittest.TestCase):self.assertEqual(out["matched_names"], ["葵つかさ"])self.assertIn(LIBRARY_REF, [one["ref"] for one in out["choices"]])+ def test_opening_the_picker_on_a_stale_index_queues_a_refresh(self):+ """这一次照旧按旧缓存列,刷新只排进后台,不挡住这次回应。"""+ refresher = self.app.state.gfriends_index+ self.assertEqual(refresher.cache_dir, self.providers / avatar_picker.GFRIENDS_CACHE)+ # 出网走采集统一出口,Peach 代理设置在那里生效。+ from peach.scraping_access import SourceTransport+ transport = refresher._transport()+ self.addCleanup(transport.close)+ self.assertIsInstance(transport, SourceTransport)+ queued = mock.patch.object(refresher, "request").start()+ old = gfriends.INDEX_MAX_AGE_SECONDS + 3600+ with mock.patch.object(gfriends, "index_age", return_value=old):+ out = self.client.get(+ "/api/avatar-choices?kind=performer&id=7792&t=secret").json()+ self.assertTrue(out["index_stale"])+ self.assertIn(LIBRARY_REF, [one["ref"] for one in out["choices"]])+ queued.assert_called_once_with()+def _online_creator(self):from peach.follow_sources import FollowCandidate, SourceFetchfrom peach.follow_store import FollowStore