Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

perf(web): 后台任务写账本后的首次重算不再把文件名与名字全部重新解析

longmeidaod9bed413f621
diff --git a/src/peach/catalog_rules.py b/src/peach/catalog_rules.py
index 25657025e..329edcffd 100644
--- a/src/peach/catalog_rules.py
+++ b/src/peach/catalog_rules.py
@@ -73,6 +73,13 @@ def is_local_extra(name: str | None, path: str | None = None, *,
return local_extra_type(name, path, library_roots=library_roots) is not None
+@lru_cache(maxsize=65536)
+def sql_is_local_extra(name: str | None, path: str | None, library_roots: tuple[str, ...]) -> bool:
+ """SQL 里按行调用的那一份:目录、统计与形状名单每次重算都对每条资产判一遍,几条查询累计
+ 几十万次。判定只看这三样,结果按它们记住。"""
+ return is_local_extra(name, path, library_roots=library_roots)
+
+
LENGTH_TAGS = {"短片-2分内", "中片-10分内", "长片-30分内", "超长片-30分上"}
TECH_TAGS = {
"1080P", "720P", "4K", "2K", "2160P", "480P", "低画质", "高帧率",
@@ -583,6 +590,7 @@ def western_release_identity(value: str | None) -> str:
return ""
+@lru_cache(maxsize=65536)
def normalise_code_key(code: str | None) -> str:
"""Normalize a release code into the stable cover-cache key.
diff --git a/src/peach/entities.py b/src/peach/entities.py
index 623e64471..5c94e2243 100644
--- a/src/peach/entities.py
+++ b/src/peach/entities.py
@@ -3,9 +3,31 @@ from __future__ import annotations
import json
import re
import unicodedata
+from contextlib import contextmanager
+from contextvars import ContextVar
from datetime import datetime, timezone
from sqlite3 import Connection
+#: 只读请求里记住的名字解析结果,见 `memoized_resolution`。
+_RESOLVED: ContextVar[dict | None] = ContextVar("peach_resolved_entities", default=None)
+
+
+@contextmanager
+def memoized_resolution():
+ """一次只读请求里同一个名字只解析一次。
+
+ 新作与发现把订阅里的每部片逐个解析演员与厂牌,同一批名字要解析上千次,每次两条扫表查询。
+ 请求内不写账本,解析结果不会变;块外与写路径照旧每次现查。
+ """
+ if _RESOLVED.get() is not None:
+ yield
+ return
+ token = _RESOLVED.set({})
+ try:
+ yield
+ finally:
+ _RESOLVED.reset(token)
+
def normalize_entity_name(name: str) -> str:
return strip_zero_width(name).strip().casefold()
@@ -812,6 +834,16 @@ def resolve_entity(connection: Connection, kind: str, name: str):
别名撞名返回 None 而不是随便挑一个:指错实体会把作品挂到另一个人名下,
那是要人工复核才能发现的错误。
"""
+ memo = _RESOLVED.get()
+ if memo is None:
+ return _resolve_entity(connection, kind, name)
+ key = (id(connection), kind, name)
+ if key not in memo:
+ memo[key] = _resolve_entity(connection, kind, name)
+ return memo[key]
+
+
+def _resolve_entity(connection: Connection, kind: str, name: str):
canonical = connection.execute(
"SELECT e.* FROM entity e WHERE e.kind=? AND e.canonical_name=? LIMIT 1",
(kind, name),
diff --git a/src/peach/repository.py b/src/peach/repository.py
index 617887467..43800c993 100644
--- a/src/peach/repository.py
+++ b/src/peach/repository.py
@@ -6,7 +6,7 @@ from contextlib import contextmanager, nullcontext
from dataclasses import dataclass
from pathlib import Path
-from .catalog_rules import is_jav_code, is_local_extra, normalise_code_key
+from .catalog_rules import is_jav_code, normalise_code_key, sql_is_local_extra
from .entities import normalize_entity_name
from .regions import infer_region
@@ -33,7 +33,7 @@ class LedgerDatabase:
connection.create_function("is_jav_code", 1, is_jav_code, deterministic=True)
roots = self.media_roots
connection.create_function("peach_is_local_extra", 2,
- lambda name, path: is_local_extra(name, path, library_roots=roots), deterministic=True)
+ lambda name, path: sql_is_local_extra(name, path, roots), deterministic=True)
connection.create_function(
"normalise_code_key", 1, normalise_code_key, deterministic=True)
# 产地推断要和 Python 侧同一份实现。SQL 里重写一遍前缀名单的话,账本筛出来的
diff --git a/src/peach/web_catalog.py b/src/peach/web_catalog.py
index c16d50a45..41675a81b 100644
--- a/src/peach/web_catalog.py
+++ b/src/peach/web_catalog.py
@@ -14,6 +14,7 @@ import random
import re
import time
from collections.abc import Callable
+from functools import lru_cache
from . import entities, media_libraries, sample_images, settings_file
@@ -146,7 +147,9 @@ def attach_jav_display_fields(row: dict, tags=(), entity_kinds=(), source_names=
row.update(jav_display_metadata(row.get("name"), code, tags, source_names))
+@lru_cache(maxsize=65536)
def _filename_fc2_code(name) -> str:
+ # 分组快照每次重算都要对全部文件名解析一遍;只看文件名,按它记住。
code = release_code_from_filename(name) or ''
return code if code.startswith('FC2-PPV-') else ''
@@ -715,7 +718,13 @@ def _edition_label(row: dict, tags: tuple[str, ...] = (), source_names: tuple[st
判据只有 `jav_edition_badges` 一份。自己按文件名再写一套「像不像无码」的判断,
会和卡片上已经显示的徽章各说各话——同一个文件,角标写着无码、分组却当它是正片。
"""
- badges = jav_edition_badges(row.get('name'), row.get('code'), tags, source_names)
+ return _edition_badge(row.get('name'), row.get('code'), tuple(tags), tuple(source_names))
+
+
+@lru_cache(maxsize=16384)
+def _edition_badge(name, code, tags: tuple[str, ...], source_names: tuple[str, ...]) -> str:
+ # 分组快照每次重算都要判一遍;判据只看这四样,按它们记住。
+ badges = jav_edition_badges(name, code, tags, source_names)
return str(badges[0]) if badges else "有码"
diff --git a/src/peach/web_entity.py b/src/peach/web_entity.py
index ebd7d8931..54ec9f88b 100644
--- a/src/peach/web_entity.py
+++ b/src/peach/web_entity.py
@@ -402,10 +402,11 @@ def agency_mark_link(links: list[dict]) -> int | None:
and not is_archive(link.get("url") or "")), None)
-def _entity_shape_parts(contract, connection, entity_id: int | None = None) -> dict[int, list[str]]:
- """实体页形状的判据;深链只查询地址上的实体。"""
+def _entity_shape_parts(contract, connection, entity_id: int | None = None, *,
+ feed: bool = True) -> dict[int, list[str]]:
+ """实体页形状的判据;深链只查询地址上的实体。`feed=False` 不算新作那一行,见 `entity_shape_rest`。"""
scope = (entity_id,) if entity_id is not None else ()
- parts = {key: ["feed"] for key in web_feeds.feed_row_entity_ids(contract, connection)
+ parts = {key: ["feed"] for key in (web_feeds.feed_row_entity_ids(contract, connection) if feed else ())
if entity_id is None or key == entity_id}
for row in connection.execute(
"SELECT DISTINCT scope.entity_id FROM asset_entity scope"
@@ -478,22 +479,46 @@ def q_entity_shapes(contract, args) -> dict:
跳。同台艺人的判据与 `q_entity` 那一份同一套;事务所那份名单走名册,不在卡底。
名单放在服务端,几台设备看到的是同一份。页面按地址里的名字比对,所以别名一起给。
"""
+ return entity_shapes(entity_shape_rest(contract), entity_shape_feed(contract))
+
+
+#: 只有新作那一行读的表。后台采集每几秒写一次,名单其余部分不读它们,缓存不跟着它们失效。
+FEED_SHAPE_TABLES = ("feed_source", "feed_item", "feed_discovery", "feed_discovery_entity")
+
+
+def entity_shape_feed(contract) -> dict:
+ """新作那一行:首页有没有、哪些实体有。判据还读资产与设置,随整本账本失效;只有两条查询。"""
+ with contract.database.read_connection() as connection:
+ return {"home": web_feeds.home_has_feed(contract, connection),
+ "ids": web_feeds.feed_row_entity_ids(contract, connection)}
+
+
+def entity_shape_rest(contract) -> dict:
+ """新作那一行以外的形状,连同全部实体的名字与别名:只有新作那一行的实体也要按名字比对。"""
with contract.database.read_connection() as connection:
- home = {"feed": web_feeds.home_has_feed(contract, connection)}
- parts = _entity_shape_parts(contract, connection)
+ parts = _entity_shape_parts(contract, connection, feed=False)
entities = {int(row["id"]): {"id": int(row["id"]), "kind": row["kind"],
- "names": [row["canonical_name"]], "parts": parts[int(row["id"])]}
- for row in connection.execute("SELECT id, kind, canonical_name FROM entity ORDER BY id")
- if int(row["id"]) in parts}
+ "names": [row["canonical_name"]], "parts": parts.get(int(row["id"]), [])}
+ for row in connection.execute("SELECT id, kind, canonical_name FROM entity ORDER BY id")}
for alias in connection.execute("SELECT entity_id, alias FROM entity_alias ORDER BY entity_id, alias"):
names = entities.get(int(alias["entity_id"]), {}).get("names")
if names is not None and alias["alias"] not in names:
names.append(alias["alias"])
from .performer_name_display import variants
- from .kanji import JP_KANJI_TO_SIMPLIFIED
name_variants = variants(connection)
- return {"ok": True, "home": home, "entities": list(entities.values()), "name_variants": name_variants,
- "name_glyphs": JP_KANJI_TO_SIMPLIFIED}
+ return {"entities": entities, "name_variants": name_variants}
+
+
+def entity_shapes(rest: dict, feed: dict) -> dict:
+ """两份合成名单;只列至少有一块的实体,新作那一行排在最前。"""
+ from .kanji import JP_KANJI_TO_SIMPLIFIED
+ listed = []
+ for entity_id, entity in rest["entities"].items():
+ parts = (["feed"] if entity_id in feed["ids"] else []) + entity["parts"]
+ if parts:
+ listed.append({**entity, "parts": parts})
+ return {"ok": True, "home": {"feed": feed["home"]}, "entities": listed,
+ "name_variants": rest["name_variants"], "name_glyphs": JP_KANJI_TO_SIMPLIFIED}
# ────────────────────────────── 照片 ──────────────────────────────
# 图集就是目录:账本没有图集实体,一个目录下的图片本来就是一份图集,
diff --git a/src/peach/web_jav_discovery.py b/src/peach/web_jav_discovery.py
index b2a9358ad..ad4ac7bf6 100644
--- a/src/peach/web_jav_discovery.py
+++ b/src/peach/web_jav_discovery.py
@@ -154,7 +154,7 @@ def q_jav_discoveries(contract, args):
"""口味采用 IDF/MMR;资料补齐只处理一次请求确定的有限集合。"""
scope = "library" if args.get("scope") == "library" else "external"
limit = max(1, min(int(args.get("limit") or 60), 100))
- with contract.read_connection() as connection:
+ with contract.read_connection() as connection, entities.memoized_resolution():
strengths, names, evidence = preference_profile(connection, contract.taste_history_store)
profile = {"id": 0, "entities": {kind: set(values) for kind, values in strengths.items()}}
candidates = _library(connection) if scope == "library" else _external(connection, contract)
@@ -267,7 +267,7 @@ def q_jav_releases(contract, args):
descending = args.get("direction") != "asc"
matches = _release_matcher({facet: str(args.get(facet) or "").strip() for facet in RELEASE_FACETS},
str(args.get("q") or "").strip().casefold())
- with contract.read_connection() as connection:
+ with contract.read_connection() as connection, entities.memoized_resolution():
rows = _release_order(_release_rows(contract, connection, taste=sort == "taste"), sort, descending,
str(args.get("seed") or ""))
facets = {facet: _release_facet(contract, rows, facet, matches) for facet in RELEASE_FACETS}
diff --git a/src/peach/web_router.py b/src/peach/web_router.py
index 5c4dfcac4..cc742e739 100644
--- a/src/peach/web_router.py
+++ b/src/peach/web_router.py
@@ -38,12 +38,15 @@ from .web_catalog import (
w_item_tag,
)
from .web_entity import (
+ FEED_SHAPE_TABLES,
SUGGEST_GROUP_LIMIT,
SUGGEST_KIND_LIMIT,
+ entity_shape_feed,
+ entity_shape_rest,
+ entity_shapes,
q_entity,
q_entity_latest,
q_entity_photos,
- q_entity_shapes,
q_index,
q_photo_set,
q_suggest,
@@ -290,9 +293,13 @@ def _get_review(contract, args):
def _get_entity_shapes(contract, args):
- # 每一页开头都要这一份(骨架照它留位),全部来自账本,账本不变就一直用同一份。
- return contract.cached_until_changed("entity-shapes",
- lambda: q_entity_shapes(contract, args))
+ # 每一页开头都要这一份(骨架照它留位),全部来自账本,账本不变就一直用同一份。后台采集每几秒
+ # 写一次订阅表,只让新作那一小份跟着重算;其余部分按订阅表以外的表失效。
+ rest = contract.cached_until_changed(
+ "entity-shapes", lambda: entity_shape_rest(contract),
+ tables=contract.ledger_tables_except(FEED_SHAPE_TABLES))
+ feed = contract.cached_until_changed("entity-shapes-feed", lambda: entity_shape_feed(contract))
+ return entity_shapes(rest, feed)
#: 安装教程每一项只读这几个字段。载荷形状由 `test_post_setup_tutorial_only_carries_progress_counts`
diff --git a/src/peach/web_search.py b/src/peach/web_search.py
index d8b4c3b48..708dcd1bd 100644
--- a/src/peach/web_search.py
+++ b/src/peach/web_search.py
@@ -22,6 +22,8 @@ POOL = 20
AUTHOR_POOL = 24
#: 关注页这一份里推荐栏用不上的键:筛选项与别名建议只有关注页自己画。
FOLLOW_UNUSED = ("facets", "author_facets", "suggestions", "alias_suggestions", "counts")
+#: 推荐池复用多久(秒)。
+RECOMMENDATION_TTL = 300
def _part(name, read):
@@ -51,7 +53,29 @@ def _words(items: list[dict], jav: list[dict]) -> list[str]:
def q_search_recommendations(contract, args) -> dict:
+ """推荐池按 `RECOMMENDATION_TTL` 复用,每次请求在池里重新洗牌。
+
+ 每开一页都要预取这一栏,而凑齐一池要好几秒;后台任务频繁写账本时,每页都重算一遍会和页面
+ 自己的取数抢服务进程。推荐只是随手一翻的入口,晚几分钟换池无妨。
+ """
library = str(args.get("library") or "")
+ pool = contract.cached(f"search-recommendations:{library}", lambda: _bundle(contract, library),
+ ttl=RECOMMENDATION_TTL)
+ return _shuffled(pool)
+
+
+def _shuffled(pool: dict) -> dict:
+ """同一池换个顺序:「换一批」在池里翻页,打开时看到的第一页每次不同。缓存里那份不动。"""
+ bundle = dict(pool)
+ for key in ("items", "jav", "follow", "feeds", "authors"):
+ part = bundle.get(key)
+ if isinstance(part, dict) and isinstance(part.get("items"), list):
+ bundle[key] = {**part, "items": random.sample(part["items"], len(part["items"]))}
+ bundle["words"] = random.sample(pool["words"], len(pool["words"]))
+ return bundle
+
+
+def _bundle(contract, library: str) -> dict:
items_args = {"sort": "rand", "limit": str(POOL), **({"library": library} if library else {})}
items = _part("作品推荐", lambda: q_items(contract, items_args))
jav = _part("JAV 推荐", lambda: web_jav_discovery.q_jav_discoveries(
diff --git a/src/peach/web_state.py b/src/peach/web_state.py
index 13129ec17..b4de8a048 100644
--- a/src/peach/web_state.py
+++ b/src/peach/web_state.py
@@ -325,6 +325,16 @@ class WebContract:
return None
return int(row[0])
+ def ledger_tables_except(self, excluded) -> tuple[str, ...]:
+ """登记了版本的账本表,去掉 `excluded`:聚合读的表太多、列不全时,按「除这几张以外」声明依赖。
+ 读不到版本表时为空,`cached_until_changed` 随之退回整本账本的版本。"""
+ try:
+ with self.read_connection() as connection:
+ tables = [row[0] for row in connection.execute("SELECT tbl FROM ledger_revision ORDER BY tbl")]
+ except sqlite3.Error:
+ return ()
+ return tuple(table for table in tables if table not in excluded)
+
def cached_until_changed(self, key: str, fn, *inputs, tables=()):
"""只读账本聚合的缓存:账本版本号与 `inputs` 都没变就一直命中,不按时间过期。
diff --git a/tests/test_entity_redirect.py b/tests/test_entity_redirect.py
index 1478f40a9..5445b4adc 100644
--- a/tests/test_entity_redirect.py
+++ b/tests/test_entity_redirect.py
@@ -18,7 +18,7 @@ from urllib.parse import parse_qs, urlsplit
from PIL import Image, ImageDraw
-from peach.entities import merge_entity, resolve_entity_id
+from peach.entities import memoized_resolution, merge_entity, resolve_entity, resolve_entity_id
from peach.web_review import _use_canonical_entity_names
from support.ledger import fresh_ledger
@@ -63,6 +63,20 @@ class EntityRedirectTests(unittest.TestCase):
return self.con.execute(
"SELECT old_id,target_id FROM entity_redirect ORDER BY old_id").fetchall()
+ def test_a_read_request_resolves_each_name_once(self):
+ """只读请求里同一个名字只查一次库;块外照旧每次现查,写路径看得到刚写的实体。"""
+ queries = []
+ self.con.set_trace_callback(queries.append)
+ with memoized_resolution():
+ first = [resolve_entity(self.con, "performer", "橋本ありな")[0] for _ in range(3)]
+ within = len(queries)
+ resolve_entity(self.con, "performer", "橋本ありな")
+ resolve_entity(self.con, "performer", "橋本ありな")
+ self.con.set_trace_callback(None)
+ self.assertEqual(first, [10, 10, 10])
+ self.assertEqual(within, 1)
+ self.assertEqual(len(queries) - within, 2)
+
def test_the_merged_id_resolves_to_the_target(self):
merge(self.con, 11, 10, "新ありな")
self.assertEqual(resolve_entity_id(self.con, 11), 10)
diff --git a/tests/test_feeds_web.py b/tests/test_feeds_web.py
index 6ed5e48a6..2e6e881e1 100644
--- a/tests/test_feeds_web.py
+++ b/tests/test_feeds_web.py
@@ -18,7 +18,7 @@ from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
sys.path.insert(0, str(Path(__file__).resolve().parent))
-from peach import feed_followup, feeds, web_feeds, web_settings, web_state # noqa: E402
+from peach import feed_followup, feeds, web_entity, web_feeds, web_settings, web_state # noqa: E402
from peach import web_jav_browser_performer as performer # noqa: E402
from peach.http import HttpResponse # noqa: E402
from peach.web_contract import WebContract # noqa: E402
@@ -344,6 +344,14 @@ class FeedWebTest(FeedWebFixture):
{"action": "ignore", "ids": [item["id"] for item in items]})
self.assertFalse(has_feed())
+ def test_new_discoveries_only_recompute_the_feed_part_of_the_shapes(self):
+ """采集每几秒写一次订阅表:形状名单只重算新作那一份,同台、资料表与版式那部分照旧命中缓存。"""
+ self.assertFalse(dispatch_api_get(self.contract, "/api/entity/shapes", {})["home"]["feed"])
+ with patch.object(web_entity, "_entity_shape_parts", side_effect=AssertionError("其余部分被重算了")):
+ self._add()
+ self._check()
+ self.assertTrue(dispatch_api_get(self.contract, "/api/entity/shapes", {})["home"]["feed"])
+
def test_home_shape_applies_library_and_compilation_filters(self):
self._compilation("波多野結衣、篠田ゆう")
with self.contract.database.write_transaction() as connection:
diff --git a/tests/test_rm_web.py b/tests/test_rm_web.py
index 1e3696b57..12d94d935 100644
--- a/tests/test_rm_web.py
+++ b/tests/test_rm_web.py
@@ -10,7 +10,7 @@ from datetime import datetime
from pathlib import Path
from unittest import mock
-from peach import catalog_rules, jav_poster_crop, web_batch, web_catalog, web_entity, web_playlists, web_stats
+from peach import catalog_rules, jav_poster_crop, web_batch, web_catalog, web_entity, web_playlists, web_search, web_stats
from peach import web_contract as rm_web
from peach.previews import entity_image_key, logo_key
from peach.web_catalog import q_editions, q_items, q_parts
@@ -174,6 +174,15 @@ class WebDataTests(unittest.TestCase):
self.assertIn("ABC-001", bundle["words"])
self.assertTrue(all(len(word) > 1 for word in bundle["words"]))
+ def test_search_recommendations_reuse_one_pool_across_page_opens(self):
+ """每开一页都预取推荐:池子复用,不在每次预取时重算;换的只是池里的顺序。"""
+ first = rm_web.dispatch_api_get(self.contract, "/api/search/recommendations", {})
+ with mock.patch.object(web_search, "q_items", side_effect=AssertionError("推荐池被重算了")):
+ again = rm_web.dispatch_api_get(self.contract, "/api/search/recommendations", {})
+ self.assertEqual({row["id"] for row in again["items"]["items"]},
+ {row["id"] for row in first["items"]["items"]})
+ self.assertEqual(sorted(again["words"]), sorted(first["words"]))
+
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = str(fresh_ledger(self.tmp.name))
@@ -293,12 +302,12 @@ class WebDataTests(unittest.TestCase):
calls = 0
- def counted_extra(name, path, **kwargs):
+ def counted_extra(name, path, roots):
nonlocal calls
calls += 1
- return catalog_rules.is_local_extra(name, path, **kwargs)
+ return catalog_rules.is_local_extra(name, path, library_roots=roots)
- with mock.patch("peach.repository.is_local_extra", counted_extra):
+ with mock.patch("peach.repository.sql_is_local_extra", counted_extra):
actual = rm_web.q_facets(self.contract, filters=filters)
# 工作量断言不依赖机器执行速度。
self.assertLess(calls, asset_count * 40, f"asset predicate calls: {calls}")
@@ -4067,14 +4076,14 @@ class TopsRotationTests(unittest.TestCase):
with self.subTest(**scope):
calls = 0
- def counted_extra(name, path, **kwargs):
+ def counted_extra(name, path, roots):
nonlocal calls
calls += 1
- return catalog_rules.is_local_extra(name, path, **kwargs)
+ return catalog_rules.is_local_extra(name, path, library_roots=roots)
# 新契约使用冷缓存和新连接,确保执行的是修改统计后的查询。
contract = rm_web.WebContract(Path(self.db_path), **roots)
- with mock.patch('peach.repository.is_local_extra', counted_extra):
+ with mock.patch('peach.repository.sql_is_local_extra', counted_extra):
actual = rm_web.q_tops(contract, 8, seed='111', **scope)
self.assertTrue(wanted['performers'])
self.assertGreater(calls, 0)