Peach: Local-first video metadata scraper and media manager.
perf(web): 后台任务写账本后的首次重算不再把文件名与名字全部重新解析
longmeidaod9bed413f621
diff --git a/src/peach/catalog_rules.py b/src/peach/catalog_rules.pyindex 25657025e..329edcffd 100644--- a/src/peach/catalog_rules.py+++ b/src/peach/catalog_rules.py@@ -73,6 +73,13 @@ def is_local_extra(name: str | None, path: str | None = None, *,return local_extra_type(name, path, library_roots=library_roots) is not None+@lru_cache(maxsize=65536)+def sql_is_local_extra(name: str | None, path: str | None, library_roots: tuple[str, ...]) -> bool:+ """SQL 里按行调用的那一份:目录、统计与形状名单每次重算都对每条资产判一遍,几条查询累计+ 几十万次。判定只看这三样,结果按它们记住。"""+ return is_local_extra(name, path, library_roots=library_roots)++LENGTH_TAGS = {"短片-2分内", "中片-10分内", "长片-30分内", "超长片-30分上"}TECH_TAGS = {"1080P", "720P", "4K", "2K", "2160P", "480P", "低画质", "高帧率",@@ -583,6 +590,7 @@ def western_release_identity(value: str | None) -> str:return ""+@lru_cache(maxsize=65536)def normalise_code_key(code: str | None) -> str:"""Normalize a release code into the stable cover-cache key.diff --git a/src/peach/entities.py b/src/peach/entities.pyindex 623e64471..5c94e2243 100644--- a/src/peach/entities.py+++ b/src/peach/entities.py@@ -3,9 +3,31 @@ from __future__ import annotationsimport jsonimport reimport unicodedata+from contextlib import contextmanager+from contextvars import ContextVarfrom datetime import datetime, timezonefrom sqlite3 import Connection+#: 只读请求里记住的名字解析结果,见 `memoized_resolution`。+_RESOLVED: ContextVar[dict | None] = ContextVar("peach_resolved_entities", default=None)+++@contextmanager+def memoized_resolution():+ """一次只读请求里同一个名字只解析一次。++ 新作与发现把订阅里的每部片逐个解析演员与厂牌,同一批名字要解析上千次,每次两条扫表查询。+ 请求内不写账本,解析结果不会变;块外与写路径照旧每次现查。+ """+ if _RESOLVED.get() is not None:+ yield+ return+ token = _RESOLVED.set({})+ try:+ yield+ finally:+ _RESOLVED.reset(token)+def normalize_entity_name(name: str) -> str:return strip_zero_width(name).strip().casefold()@@ -812,6 +834,16 @@ def resolve_entity(connection: Connection, kind: str, name: str):别名撞名返回 None 而不是随便挑一个:指错实体会把作品挂到另一个人名下,那是要人工复核才能发现的错误。"""+ memo = _RESOLVED.get()+ if memo is None:+ return _resolve_entity(connection, kind, name)+ key = (id(connection), kind, name)+ if key not in memo:+ memo[key] = _resolve_entity(connection, kind, name)+ return memo[key]+++def _resolve_entity(connection: Connection, kind: str, name: str):canonical = connection.execute("SELECT e.* FROM entity e WHERE e.kind=? AND e.canonical_name=? LIMIT 1",(kind, name),diff --git a/src/peach/repository.py b/src/peach/repository.pyindex 617887467..43800c993 100644--- a/src/peach/repository.py+++ b/src/peach/repository.py@@ -6,7 +6,7 @@ from contextlib import contextmanager, nullcontextfrom dataclasses import dataclassfrom pathlib import Path-from .catalog_rules import is_jav_code, is_local_extra, normalise_code_key+from .catalog_rules import is_jav_code, normalise_code_key, sql_is_local_extrafrom .entities import normalize_entity_namefrom .regions import infer_region@@ -33,7 +33,7 @@ class LedgerDatabase:connection.create_function("is_jav_code", 1, is_jav_code, deterministic=True)roots = self.media_rootsconnection.create_function("peach_is_local_extra", 2,- lambda name, path: is_local_extra(name, path, library_roots=roots), deterministic=True)+ lambda name, path: sql_is_local_extra(name, path, roots), deterministic=True)connection.create_function("normalise_code_key", 1, normalise_code_key, deterministic=True)# 产地推断要和 Python 侧同一份实现。SQL 里重写一遍前缀名单的话,账本筛出来的diff --git a/src/peach/web_catalog.py b/src/peach/web_catalog.pyindex c16d50a45..41675a81b 100644--- a/src/peach/web_catalog.py+++ b/src/peach/web_catalog.py@@ -14,6 +14,7 @@ import randomimport reimport timefrom collections.abc import Callable+from functools import lru_cachefrom . import entities, media_libraries, sample_images, settings_file@@ -146,7 +147,9 @@ def attach_jav_display_fields(row: dict, tags=(), entity_kinds=(), source_names=row.update(jav_display_metadata(row.get("name"), code, tags, source_names))+@lru_cache(maxsize=65536)def _filename_fc2_code(name) -> str:+ # 分组快照每次重算都要对全部文件名解析一遍;只看文件名,按它记住。code = release_code_from_filename(name) or ''return code if code.startswith('FC2-PPV-') else ''@@ -715,7 +718,13 @@ def _edition_label(row: dict, tags: tuple[str, ...] = (), source_names: tuple[st判据只有 `jav_edition_badges` 一份。自己按文件名再写一套「像不像无码」的判断,会和卡片上已经显示的徽章各说各话——同一个文件,角标写着无码、分组却当它是正片。"""- badges = jav_edition_badges(row.get('name'), row.get('code'), tags, source_names)+ return _edition_badge(row.get('name'), row.get('code'), tuple(tags), tuple(source_names))+++@lru_cache(maxsize=16384)+def _edition_badge(name, code, tags: tuple[str, ...], source_names: tuple[str, ...]) -> str:+ # 分组快照每次重算都要判一遍;判据只看这四样,按它们记住。+ badges = jav_edition_badges(name, code, tags, source_names)return str(badges[0]) if badges else "有码"diff --git a/src/peach/web_entity.py b/src/peach/web_entity.pyindex ebd7d8931..54ec9f88b 100644--- a/src/peach/web_entity.py+++ b/src/peach/web_entity.py@@ -402,10 +402,11 @@ def agency_mark_link(links: list[dict]) -> int | None:and not is_archive(link.get("url") or "")), None)-def _entity_shape_parts(contract, connection, entity_id: int | None = None) -> dict[int, list[str]]:- """实体页形状的判据;深链只查询地址上的实体。"""+def _entity_shape_parts(contract, connection, entity_id: int | None = None, *,+ feed: bool = True) -> dict[int, list[str]]:+ """实体页形状的判据;深链只查询地址上的实体。`feed=False` 不算新作那一行,见 `entity_shape_rest`。"""scope = (entity_id,) if entity_id is not None else ()- parts = {key: ["feed"] for key in web_feeds.feed_row_entity_ids(contract, connection)+ parts = {key: ["feed"] for key in (web_feeds.feed_row_entity_ids(contract, connection) if feed else ())if entity_id is None or key == entity_id}for row in connection.execute("SELECT DISTINCT scope.entity_id FROM asset_entity scope"@@ -478,22 +479,46 @@ def q_entity_shapes(contract, args) -> dict:跳。同台艺人的判据与 `q_entity` 那一份同一套;事务所那份名单走名册,不在卡底。名单放在服务端,几台设备看到的是同一份。页面按地址里的名字比对,所以别名一起给。"""+ return entity_shapes(entity_shape_rest(contract), entity_shape_feed(contract))+++#: 只有新作那一行读的表。后台采集每几秒写一次,名单其余部分不读它们,缓存不跟着它们失效。+FEED_SHAPE_TABLES = ("feed_source", "feed_item", "feed_discovery", "feed_discovery_entity")+++def entity_shape_feed(contract) -> dict:+ """新作那一行:首页有没有、哪些实体有。判据还读资产与设置,随整本账本失效;只有两条查询。"""+ with contract.database.read_connection() as connection:+ return {"home": web_feeds.home_has_feed(contract, connection),+ "ids": web_feeds.feed_row_entity_ids(contract, connection)}+++def entity_shape_rest(contract) -> dict:+ """新作那一行以外的形状,连同全部实体的名字与别名:只有新作那一行的实体也要按名字比对。"""with contract.database.read_connection() as connection:- home = {"feed": web_feeds.home_has_feed(contract, connection)}- parts = _entity_shape_parts(contract, connection)+ parts = _entity_shape_parts(contract, connection, feed=False)entities = {int(row["id"]): {"id": int(row["id"]), "kind": row["kind"],- "names": [row["canonical_name"]], "parts": parts[int(row["id"])]}- for row in connection.execute("SELECT id, kind, canonical_name FROM entity ORDER BY id")- if int(row["id"]) in parts}+ "names": [row["canonical_name"]], "parts": parts.get(int(row["id"]), [])}+ for row in connection.execute("SELECT id, kind, canonical_name FROM entity ORDER BY id")}for alias in connection.execute("SELECT entity_id, alias FROM entity_alias ORDER BY entity_id, alias"):names = entities.get(int(alias["entity_id"]), {}).get("names")if names is not None and alias["alias"] not in names:names.append(alias["alias"])from .performer_name_display import variants- from .kanji import JP_KANJI_TO_SIMPLIFIEDname_variants = variants(connection)- return {"ok": True, "home": home, "entities": list(entities.values()), "name_variants": name_variants,- "name_glyphs": JP_KANJI_TO_SIMPLIFIED}+ return {"entities": entities, "name_variants": name_variants}+++def entity_shapes(rest: dict, feed: dict) -> dict:+ """两份合成名单;只列至少有一块的实体,新作那一行排在最前。"""+ from .kanji import JP_KANJI_TO_SIMPLIFIED+ listed = []+ for entity_id, entity in rest["entities"].items():+ parts = (["feed"] if entity_id in feed["ids"] else []) + entity["parts"]+ if parts:+ listed.append({**entity, "parts": parts})+ return {"ok": True, "home": {"feed": feed["home"]}, "entities": listed,+ "name_variants": rest["name_variants"], "name_glyphs": JP_KANJI_TO_SIMPLIFIED}# ────────────────────────────── 照片 ──────────────────────────────# 图集就是目录:账本没有图集实体,一个目录下的图片本来就是一份图集,diff --git a/src/peach/web_jav_discovery.py b/src/peach/web_jav_discovery.pyindex b2a9358ad..ad4ac7bf6 100644--- a/src/peach/web_jav_discovery.py+++ b/src/peach/web_jav_discovery.py@@ -154,7 +154,7 @@ def q_jav_discoveries(contract, args):"""口味采用 IDF/MMR;资料补齐只处理一次请求确定的有限集合。"""scope = "library" if args.get("scope") == "library" else "external"limit = max(1, min(int(args.get("limit") or 60), 100))- with contract.read_connection() as connection:+ with contract.read_connection() as connection, entities.memoized_resolution():strengths, names, evidence = preference_profile(connection, contract.taste_history_store)profile = {"id": 0, "entities": {kind: set(values) for kind, values in strengths.items()}}candidates = _library(connection) if scope == "library" else _external(connection, contract)@@ -267,7 +267,7 @@ def q_jav_releases(contract, args):descending = args.get("direction") != "asc"matches = _release_matcher({facet: str(args.get(facet) or "").strip() for facet in RELEASE_FACETS},str(args.get("q") or "").strip().casefold())- with contract.read_connection() as connection:+ with contract.read_connection() as connection, entities.memoized_resolution():rows = _release_order(_release_rows(contract, connection, taste=sort == "taste"), sort, descending,str(args.get("seed") or ""))facets = {facet: _release_facet(contract, rows, facet, matches) for facet in RELEASE_FACETS}diff --git a/src/peach/web_router.py b/src/peach/web_router.pyindex 5c4dfcac4..cc742e739 100644--- a/src/peach/web_router.py+++ b/src/peach/web_router.py@@ -38,12 +38,15 @@ from .web_catalog import (w_item_tag,)from .web_entity import (+ FEED_SHAPE_TABLES,SUGGEST_GROUP_LIMIT,SUGGEST_KIND_LIMIT,+ entity_shape_feed,+ entity_shape_rest,+ entity_shapes,q_entity,q_entity_latest,q_entity_photos,- q_entity_shapes,q_index,q_photo_set,q_suggest,@@ -290,9 +293,13 @@ def _get_review(contract, args):def _get_entity_shapes(contract, args):- # 每一页开头都要这一份(骨架照它留位),全部来自账本,账本不变就一直用同一份。- return contract.cached_until_changed("entity-shapes",- lambda: q_entity_shapes(contract, args))+ # 每一页开头都要这一份(骨架照它留位),全部来自账本,账本不变就一直用同一份。后台采集每几秒+ # 写一次订阅表,只让新作那一小份跟着重算;其余部分按订阅表以外的表失效。+ rest = contract.cached_until_changed(+ "entity-shapes", lambda: entity_shape_rest(contract),+ tables=contract.ledger_tables_except(FEED_SHAPE_TABLES))+ feed = contract.cached_until_changed("entity-shapes-feed", lambda: entity_shape_feed(contract))+ return entity_shapes(rest, feed)#: 安装教程每一项只读这几个字段。载荷形状由 `test_post_setup_tutorial_only_carries_progress_counts`diff --git a/src/peach/web_search.py b/src/peach/web_search.pyindex d8b4c3b48..708dcd1bd 100644--- a/src/peach/web_search.py+++ b/src/peach/web_search.py@@ -22,6 +22,8 @@ POOL = 20AUTHOR_POOL = 24#: 关注页这一份里推荐栏用不上的键:筛选项与别名建议只有关注页自己画。FOLLOW_UNUSED = ("facets", "author_facets", "suggestions", "alias_suggestions", "counts")+#: 推荐池复用多久(秒)。+RECOMMENDATION_TTL = 300def _part(name, read):@@ -51,7 +53,29 @@ def _words(items: list[dict], jav: list[dict]) -> list[str]:def q_search_recommendations(contract, args) -> dict:+ """推荐池按 `RECOMMENDATION_TTL` 复用,每次请求在池里重新洗牌。++ 每开一页都要预取这一栏,而凑齐一池要好几秒;后台任务频繁写账本时,每页都重算一遍会和页面+ 自己的取数抢服务进程。推荐只是随手一翻的入口,晚几分钟换池无妨。+ """library = str(args.get("library") or "")+ pool = contract.cached(f"search-recommendations:{library}", lambda: _bundle(contract, library),+ ttl=RECOMMENDATION_TTL)+ return _shuffled(pool)+++def _shuffled(pool: dict) -> dict:+ """同一池换个顺序:「换一批」在池里翻页,打开时看到的第一页每次不同。缓存里那份不动。"""+ bundle = dict(pool)+ for key in ("items", "jav", "follow", "feeds", "authors"):+ part = bundle.get(key)+ if isinstance(part, dict) and isinstance(part.get("items"), list):+ bundle[key] = {**part, "items": random.sample(part["items"], len(part["items"]))}+ bundle["words"] = random.sample(pool["words"], len(pool["words"]))+ return bundle+++def _bundle(contract, library: str) -> dict:items_args = {"sort": "rand", "limit": str(POOL), **({"library": library} if library else {})}items = _part("作品推荐", lambda: q_items(contract, items_args))jav = _part("JAV 推荐", lambda: web_jav_discovery.q_jav_discoveries(diff --git a/src/peach/web_state.py b/src/peach/web_state.pyindex 13129ec17..b4de8a048 100644--- a/src/peach/web_state.py+++ b/src/peach/web_state.py@@ -325,6 +325,16 @@ class WebContract:return Nonereturn int(row[0])+ def ledger_tables_except(self, excluded) -> tuple[str, ...]:+ """登记了版本的账本表,去掉 `excluded`:聚合读的表太多、列不全时,按「除这几张以外」声明依赖。+ 读不到版本表时为空,`cached_until_changed` 随之退回整本账本的版本。"""+ try:+ with self.read_connection() as connection:+ tables = [row[0] for row in connection.execute("SELECT tbl FROM ledger_revision ORDER BY tbl")]+ except sqlite3.Error:+ return ()+ return tuple(table for table in tables if table not in excluded)+def cached_until_changed(self, key: str, fn, *inputs, tables=()):"""只读账本聚合的缓存:账本版本号与 `inputs` 都没变就一直命中,不按时间过期。diff --git a/tests/test_entity_redirect.py b/tests/test_entity_redirect.pyindex 1478f40a9..5445b4adc 100644--- a/tests/test_entity_redirect.py+++ b/tests/test_entity_redirect.py@@ -18,7 +18,7 @@ from urllib.parse import parse_qs, urlsplitfrom PIL import Image, ImageDraw-from peach.entities import merge_entity, resolve_entity_id+from peach.entities import memoized_resolution, merge_entity, resolve_entity, resolve_entity_idfrom peach.web_review import _use_canonical_entity_namesfrom support.ledger import fresh_ledger@@ -63,6 +63,20 @@ class EntityRedirectTests(unittest.TestCase):return self.con.execute("SELECT old_id,target_id FROM entity_redirect ORDER BY old_id").fetchall()+ def test_a_read_request_resolves_each_name_once(self):+ """只读请求里同一个名字只查一次库;块外照旧每次现查,写路径看得到刚写的实体。"""+ queries = []+ self.con.set_trace_callback(queries.append)+ with memoized_resolution():+ first = [resolve_entity(self.con, "performer", "橋本ありな")[0] for _ in range(3)]+ within = len(queries)+ resolve_entity(self.con, "performer", "橋本ありな")+ resolve_entity(self.con, "performer", "橋本ありな")+ self.con.set_trace_callback(None)+ self.assertEqual(first, [10, 10, 10])+ self.assertEqual(within, 1)+ self.assertEqual(len(queries) - within, 2)+def test_the_merged_id_resolves_to_the_target(self):merge(self.con, 11, 10, "新ありな")self.assertEqual(resolve_entity_id(self.con, 11), 10)diff --git a/tests/test_feeds_web.py b/tests/test_feeds_web.pyindex 6ed5e48a6..2e6e881e1 100644--- a/tests/test_feeds_web.py+++ b/tests/test_feeds_web.py@@ -18,7 +18,7 @@ from unittest.mock import patchsys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))sys.path.insert(0, str(Path(__file__).resolve().parent))-from peach import feed_followup, feeds, web_feeds, web_settings, web_state # noqa: E402+from peach import feed_followup, feeds, web_entity, web_feeds, web_settings, web_state # noqa: E402from peach import web_jav_browser_performer as performer # noqa: E402from peach.http import HttpResponse # noqa: E402from peach.web_contract import WebContract # noqa: E402@@ -344,6 +344,14 @@ class FeedWebTest(FeedWebFixture):{"action": "ignore", "ids": [item["id"] for item in items]})self.assertFalse(has_feed())+ def test_new_discoveries_only_recompute_the_feed_part_of_the_shapes(self):+ """采集每几秒写一次订阅表:形状名单只重算新作那一份,同台、资料表与版式那部分照旧命中缓存。"""+ self.assertFalse(dispatch_api_get(self.contract, "/api/entity/shapes", {})["home"]["feed"])+ with patch.object(web_entity, "_entity_shape_parts", side_effect=AssertionError("其余部分被重算了")):+ self._add()+ self._check()+ self.assertTrue(dispatch_api_get(self.contract, "/api/entity/shapes", {})["home"]["feed"])+def test_home_shape_applies_library_and_compilation_filters(self):self._compilation("波多野結衣、篠田ゆう")with self.contract.database.write_transaction() as connection:diff --git a/tests/test_rm_web.py b/tests/test_rm_web.pyindex 1e3696b57..12d94d935 100644--- a/tests/test_rm_web.py+++ b/tests/test_rm_web.py@@ -10,7 +10,7 @@ from datetime import datetimefrom pathlib import Pathfrom unittest import mock-from peach import catalog_rules, jav_poster_crop, web_batch, web_catalog, web_entity, web_playlists, web_stats+from peach import catalog_rules, jav_poster_crop, web_batch, web_catalog, web_entity, web_playlists, web_search, web_statsfrom peach import web_contract as rm_webfrom peach.previews import entity_image_key, logo_keyfrom peach.web_catalog import q_editions, q_items, q_parts@@ -174,6 +174,15 @@ class WebDataTests(unittest.TestCase):self.assertIn("ABC-001", bundle["words"])self.assertTrue(all(len(word) > 1 for word in bundle["words"]))+ def test_search_recommendations_reuse_one_pool_across_page_opens(self):+ """每开一页都预取推荐:池子复用,不在每次预取时重算;换的只是池里的顺序。"""+ first = rm_web.dispatch_api_get(self.contract, "/api/search/recommendations", {})+ with mock.patch.object(web_search, "q_items", side_effect=AssertionError("推荐池被重算了")):+ again = rm_web.dispatch_api_get(self.contract, "/api/search/recommendations", {})+ self.assertEqual({row["id"] for row in again["items"]["items"]},+ {row["id"] for row in first["items"]["items"]})+ self.assertEqual(sorted(again["words"]), sorted(first["words"]))+def setUp(self):self.tmp = tempfile.TemporaryDirectory()self.db_path = str(fresh_ledger(self.tmp.name))@@ -293,12 +302,12 @@ class WebDataTests(unittest.TestCase):calls = 0- def counted_extra(name, path, **kwargs):+ def counted_extra(name, path, roots):nonlocal callscalls += 1- return catalog_rules.is_local_extra(name, path, **kwargs)+ return catalog_rules.is_local_extra(name, path, library_roots=roots)- with mock.patch("peach.repository.is_local_extra", counted_extra):+ with mock.patch("peach.repository.sql_is_local_extra", counted_extra):actual = rm_web.q_facets(self.contract, filters=filters)# 工作量断言不依赖机器执行速度。self.assertLess(calls, asset_count * 40, f"asset predicate calls: {calls}")@@ -4067,14 +4076,14 @@ class TopsRotationTests(unittest.TestCase):with self.subTest(**scope):calls = 0- def counted_extra(name, path, **kwargs):+ def counted_extra(name, path, roots):nonlocal callscalls += 1- return catalog_rules.is_local_extra(name, path, **kwargs)+ return catalog_rules.is_local_extra(name, path, library_roots=roots)# 新契约使用冷缓存和新连接,确保执行的是修改统计后的查询。contract = rm_web.WebContract(Path(self.db_path), **roots)- with mock.patch('peach.repository.is_local_extra', counted_extra):+ with mock.patch('peach.repository.sql_is_local_extra', counted_extra):actual = rm_web.q_tops(contract, 8, seed='111', **scope)self.assertTrue(wanted['performers'])self.assertGreater(calls, 0)