Peach: Local-first video metadata scraper and media manager.
perf(web): 后台任务写账本后页面先拿上一份聚合,重算在后台进行
longmeidao1460688e5083
diff --git a/docs/adr/0146-background-writes-serve-the-previous-aggregate.md b/docs/adr/0146-background-writes-serve-the-previous-aggregate.mdnew file mode 100644index 000000000..359991ba0--- /dev/null+++ b/docs/adr/0146-background-writes-serve-the-previous-aggregate.md@@ -0,0 +1,33 @@+# ADR-0146:后台任务写账本后,页面先拿上一份聚合++- 状态:已采纳+- 日期:2026-11-05+- 关系:修改 `WebContract.cached_until_changed` 带 `tables` 那一支的失效口径;结果版本见 `web_result_versions.py`++## 背景++目录分组快照、筛选项、榜单、索引、统计、形状名单都按依赖表的版本缓存(`cached_until_changed(..., tables=...)`)。版本一变,下一位读者就当场重算,算完才返回。++后台任务成批写账本时,这个口径让页面一直在等。2026-11-05 生产实测:补头像、补别名、采集这类后续任务每几秒提交一次,统计页第一次打开要等 3 秒,重启后定时任务集中补跑那几分钟,作品列表等了 13 秒。重算本身已经压过一轮(解析结果按输入记住),剩下的是 SQL 与聚合,压不到零。++这些写入不是用户刚做的操作。页面上晚几秒看到一条新补的头像或别名,用户察觉不到;但等 3 秒才看到作品列表,他会察觉。++## 决策++**一、区分前台与后台提交。** `repository.in_background` 包住的线程入口算后台写入者:后台任务(`BackgroundJob`)、后续任务调度(`FollowupRunner`)、云下载循环、推送发现。它们提交时,`cache_bust` 记下 `background_generation`;其余线程(请求处理、测试、显式 `cache_bust`)的提交记 `foreground_generation`。没包的线程一律按前台算,保持原有的当场重算。++**二、只有后台提交之后才先回旧值。** 带 `tables` 的键版本变了时,若手上那份出发之后没有前台提交,且本进程确有一次后台提交,就先回那一份,交给一条后台线程重算。同一个键在途时不重复排。前台提交之后照旧当场重算,写的人立刻看到自己写的。进程外的写(命令行、账本同步)不经提交通知,也当场重算。不带 `tables` 的键每次提交都被清掉,没有旧值可回,不在此列。++**三、重算写回后通知页面。** 重算写回时 `cache_landed` 加一,并拼进每个账本结果版本键(`catalog`、`follow`、`discoveries`、`review`)。页面的结果轮询看到版本变了,再取一次,拿到新值。没有这一步,最后一笔后台写入之后页面会停在旧值,直到下一次写入。++## 理由++- 判据落在「谁提交的」而不是「谁在读」:同一页面既要在后台任务刷账本时不等,也要在用户自己批准、改名后立刻看到结果,只有提交方知道这一笔属于哪种。+- 标记后台写入者、默认前台:漏标一个后台线程只是少一处加速,不会让用户的操作结果晚到;反过来标记前台,测试与请求内写完即读都会拿到旧值。+- 一条重算线程:重算多是几秒的 CPU,和请求抢同一个进程;排成一队比并发重算对页面更友好。++## 后果++- 后台任务刚写完的几秒内,页面先显示上一份,重算写回后自动换成新的。+- 每次写回都会让页面上各组读数各刷新一次;它们多半命中缓存,代价是几次轻量请求。+- 新的后台写入线程要用 `in_background` 包入口才享受这一口径;不包也正确,只是照旧当场重算。diff --git a/src/peach/downloads.py b/src/peach/downloads.pyindex 68d0e3a93..5182931bd 100644--- a/src/peach/downloads.py+++ b/src/peach/downloads.py@@ -37,7 +37,7 @@ from urllib.parse import parse_qs, unquote, urlsplitfrom .follow_secrets import CredentialStorefrom .fsutil import atomic_write_textfrom .push_discovery import CloudPathError, CloudPrefix, map_cloud_path, normalise_prefix-from .repository import LedgerDatabase+from .repository import LedgerDatabase, in_backgroundLOGGER = logging.getLogger(__name__)@@ -911,7 +911,7 @@ class DownloadService:if not self.available or self._thread is not None:returnself._stop.clear()- self._thread = threading.Thread(target=self._loop, name="PeachDownloads", daemon=True)+ self._thread = threading.Thread(target=in_background(self._loop), name="PeachDownloads", daemon=True)self._thread.start()if self.after_ingest is not None:try:diff --git a/src/peach/followups.py b/src/peach/followups.pyindex 9e1ea3ed1..87c273d45 100644--- a/src/peach/followups.py+++ b/src/peach/followups.py@@ -25,6 +25,7 @@ from dataclasses import dataclassfrom pathlib import Pathfrom typing import Callable+from .repository import in_backgroundfrom .task_runs import TaskRunHandle, TaskRunStoreLOGGER = logging.getLogger(__name__)@@ -178,7 +179,7 @@ class FollowupRunner:if thread is not None and thread.is_alive():continuethread = threading.Thread(- target=self._work, args=(keys,), daemon=True,+ target=in_background(self._work), args=(keys,), daemon=True,name=f"PeachFollowup-{lane}")self._threads[lane] = threadthread.start()diff --git a/src/peach/jobs.py b/src/peach/jobs.pyindex f610e40fe..170a1b717 100644--- a/src/peach/jobs.py+++ b/src/peach/jobs.py@@ -13,6 +13,7 @@ from dataclasses import dataclassfrom pathlib import Pathfrom .media_sources import SOURCE_COSTS, cost_for, cost_sql+from .repository import in_backgroundDEFAULT_METERED_LOCATIONS = frozenset(location for location, cost in SOURCE_COSTS.items()if cost == 'metered') | {'online'}@@ -501,7 +502,7 @@ class BackgroundJob:"started_at": time.time(), "error": "", **(initial or {})}state.update(self._resuming_fields(job_id, initial))self.state = state- thread = threading.Thread(target=self._run, args=(fn, job_id),+ thread = threading.Thread(target=in_background(self._run), args=(fn, job_id),daemon=True, name=self.name)self.thread = threadsnapshot = copy.deepcopy(state)diff --git a/src/peach/push_discovery.py b/src/peach/push_discovery.pyindex e59ce19b2..cf4d629a3 100644--- a/src/peach/push_discovery.py+++ b/src/peach/push_discovery.py@@ -43,6 +43,7 @@ from pathlib import Path, PureWindowsPathfrom typing import Callablefrom .fsutil import atomic_write_text+from .repository import in_backgroundLOGGER = logging.getLogger(__name__)@@ -402,7 +403,7 @@ class DiscoveryQueue:returnself._stopping = Falseself._thread = threading.Thread(- target=self._loop, name="PeachPushDiscovery", daemon=True)+ target=in_background(self._loop), name="PeachPushDiscovery", daemon=True)self._thread.start()def stop(self) -> None:diff --git a/src/peach/repository.py b/src/peach/repository.pyindex 43800c993..354770a12 100644--- a/src/peach/repository.py+++ b/src/peach/repository.py@@ -10,6 +10,24 @@ from .catalog_rules import is_jav_code, normalise_code_key, sql_is_local_extrafrom .entities import normalize_entity_namefrom .regions import infer_region+#: 当前线程是不是后台写入者,见 `in_background`。+_WRITER = threading.local()+++def in_background(target):+ """把线程入口包成后台写入者:它提交的写入只让页面缓存「过期」,读者可以先拿旧值(ADR-0146)。++ 没包的线程照旧按前台算:测试、请求处理与显式失效写完就要读到新值。+ """+ def run(*args, **kwargs):+ _WRITER.background = True+ return target(*args, **kwargs)+ return run+++def writing_in_background() -> bool:+ return getattr(_WRITER, "background", False)+class LedgerDatabase:"""Shared SQLite connection and transaction boundary for one Peach app."""diff --git a/src/peach/web_result_versions.py b/src/peach/web_result_versions.pyindex 475e7dcc1..ec08c65d4 100644--- a/src/peach/web_result_versions.py+++ b/src/peach/web_result_versions.py@@ -20,8 +20,13 @@ RESULT_TABLES = {def result_versions(contract):- """复用账本触发器和文件版本,跨线程、CLI 写入和进程重启均可比较。"""- result = {key: str(contract.ledger_revision(tables)) for key, tables in RESULT_TABLES.items()}+ """复用账本触发器和文件版本,跨线程、CLI 写入和进程重启均可比较。++ 后台任务写账本后,按表缓存的那几页先回旧值、另起重算(ADR-0146);重算写回时版本已经+ 不会再变,所以每个账本键都带上写回次数,页面看到它变了再取一次新值。+ """+ landed = f":{contract.cache_landed}"+ result = {key: str(contract.ledger_revision(tables)) + landed for key, tables in RESULT_TABLES.items()}result['review'] += ':' + str(candidate_root_version(contract.candidate_root))result['artwork'] = ':'.join(str(path_version(root)) for root in (contract.cover_root, contract.avatar_root, contract.logo_root, contract.poster_root,diff --git a/src/peach/web_state.py b/src/peach/web_state.pyindex b4de8a048..2e2c202af 100644--- a/src/peach/web_state.py+++ b/src/peach/web_state.py@@ -13,6 +13,7 @@from __future__ import annotationsimport json+import loggingimport mathimport osimport sqlite3@@ -44,9 +45,11 @@ from .media import normalized_path# `previews` 是取图那一侧,不是 web 域处理器:依赖方向仍然只有一个走法。落盘名的# 规则必须和 `/logo` 用的是同一个函数,否则可用性判定迟早和取图对不上。from .previews import ENTITY_IMAGE_KINDS, LOGO_VARIANTS, entity_image_key, logo_key-from .repository import LedgerDatabase+from .repository import LedgerDatabase, writing_in_backgroundfrom .task_runs import TaskRunStore+LOGGER = logging.getLogger(__name__)+class AvatarRootIndex(NamedTuple):"""`avatar_root` 一次扫描的产物。两样东西同处一个目录,分开扫就是白扫两遍。@@ -179,6 +182,16 @@ class WebContract:self.cache_generation = 0self.revision_cache_keys: set[str] = set()self.aggregate_locks: dict[str, threading.RLock] = {}+ #: 每个缓存键那一份结果出发时的 `cache_generation`;判它有没有包含最近一次前台写入。+ self.cache_started: dict[str, int] = {}+ #: 最近一次前台提交(不是 `repository.in_background` 包过的线程)之后的 `cache_generation`。+ self.foreground_generation = 0+ #: 最近一次后台提交之后的 `cache_generation`。进程外的写不经提交通知,两个数都不动。+ self.background_generation = 0+ #: 后台重算写回新值的次数,随结果版本下发,页面据此再取一次(ADR-0146)。+ self.cache_landed = 0+ self.stale_refreshing: set[str] = set()+ self._stale_refresher = Noneself.follow_check_lock = threading.Lock()#: 任务中心。每个后台任务的开始、进度与结束都写这一张表,`/tasks` 只读它。#: 只读端由 `api` 关掉写入:那边的账本是复制来的,写一行就是一处合不回去的分叉。@@ -277,9 +290,11 @@ class WebContract:# 时间戳沿用进入时的 now:算得比 TTL 还久的结果直接算过期,# 宁可下次重算,也不要把一份已经旧了的数据当新的用。self.cache[key] = (now, version, value)+ self.cache_started[key] = generationself.cache.move_to_end(key)while len(self.cache) > CACHE_KEYS:evicted, _ = self.cache.popitem(last=False)+ self.cache_started.pop(evicted, None)# 键空间不封闭的聚合(按作者、按筛选口径)的锁和登记跟着结果一起走,# 否则这两份只增不减。正被别的线程拿着的锁摘掉也无妨,持有者手里还有它;# 同键再来时 `cached_until_changed` 会重新登记。@@ -344,11 +359,60 @@ class WebContract:退回 `CACHE_TTL`。`tables` 声明聚合依赖的表;这类缓存经过提交通知后仍按声明的版本核验,同键的并发请求共用一次计算。++ 带 `tables` 的键在版本变了、手上那份之后只有本进程的后台提交时,先回那一份,另起+ 后台重算(ADR-0146):后台任务写的东西晚几秒无妨;前台写入与进程外的写之后照旧+ 当场重算。不带 `tables` 的键每次提交都被清掉,没有旧值可回。"""if not tables:revision = self.ledger_revision()return self.cached(key, fn, ttl=CACHE_TTL if revision is None else math.inf,version=(revision, *inputs))+ revision = self.ledger_revision(tables)+ if revision is not None:+ version = (tuple(tables), revision, *inputs)+ schedule = False+ with self.cache_lock:+ hit = self.cache.get(key)+ started = self.cache_started.get(key, -1)+ # 手上那份之后只有后台提交:没有前台写入,也有一次本进程的后台提交说明版本为什么变了。+ # 进程外的写(命令行、账本同步)不经提交通知,照旧当场重算。+ stale = (hit is not None and hit[1] != version and started >= self.foreground_generation+ and self.background_generation > started)+ if stale:+ self.cache.move_to_end(key)+ schedule = key not in self.stale_refreshing+ self.stale_refreshing.add(key)+ if stale:+ if schedule:+ self._refresh_later(key, fn, inputs, tables)+ return hit[2]+ return self._cached_by_tables(key, fn, inputs, tables)++ def _refresh_later(self, key, fn, inputs, tables) -> None:+ """一条后台线程依次重算过期的键,同一个键在途时不重复排。"""+ from concurrent.futures import ThreadPoolExecutor++ def work():+ try:+ self._cached_by_tables(key, fn, inputs, tables)+ with self.cache_lock:+ self.cache_landed += 1+ except Exception: # noqa: BLE001 - 后台重算失败时下一次读照旧当场重算,原因记日志+ LOGGER.exception("后台重算缓存 %s 失败", key)+ with self.cache_lock:+ self.cache.pop(key, None)+ finally:+ with self.cache_lock:+ self.stale_refreshing.discard(key)++ with self.cache_lock:+ if self._stale_refresher is None:+ self._stale_refresher = ThreadPoolExecutor(max_workers=1, thread_name_prefix="PeachCacheRefresh")+ refresher = self._stale_refresher+ refresher.submit(work)++ def _cached_by_tables(self, key: str, fn, inputs, tables):# 指定依赖表的聚合共用一次计算;提交后仍按这些表的代次核验。with self.cache_lock:lock = self.aggregate_locks.setdefault(key, threading.RLock())@@ -389,6 +453,8 @@ class WebContract:if hasattr(self, "jav_discovery_metadata"):self.jav_discovery_metadata.stop()self.followups.stop()+ if self._stale_refresher is not None:+ self._stale_refresher.shutdown(wait=False, cancel_futures=True)def cache_bust(self):with self.cache_lock:@@ -397,6 +463,11 @@ class WebContract:self.keyed_cache.clear()# 在途计算靠这个数认出「我出发之后缓存失效过」,从而放弃写回。self.cache_generation += 1+ # 前台写入之后,出发更早的那份缓存不许再先回旧值:写的人要立刻看到自己写的。+ if writing_in_background():+ self.background_generation = self.cache_generation+ else:+ self.foreground_generation = self.cache_generationdef read_connection(self):return self.database.read_connection()diff --git a/tests/test_ledger_revision.py b/tests/test_ledger_revision.pyindex ef6ea2f65..5f7f17f40 100644--- a/tests/test_ledger_revision.py+++ b/tests/test_ledger_revision.py@@ -8,6 +8,8 @@ import jsonimport osimport sqlite3import tempfile+import threading+import timeimport unittestfrom contextlib import closingfrom datetime import datetime, timezone@@ -18,7 +20,7 @@ from peach import taste_history, taste_preferences, web_follow, web_state, web_sfrom peach.follow_sources import FollowCandidate, SourceFetchfrom peach.follow_store import FollowStorefrom peach.previews import entity_image_key, logo_key-from peach.repository import LedgerDatabase+from peach.repository import LedgerDatabase, in_backgroundfrom peach.web_catalog import FACETS_TABLES, GROUP_TABLES, TOPS_TABLESfrom peach.web_contract import dispatch_api_getfrom peach.web_entity import INDEX_TABLES@@ -148,6 +150,44 @@ class LedgerRevisionTests(unittest.TestCase):3 * web_state.CACHE_KEYS + 1)self.assertIn("author:0", self.contract.revision_cache_keys)+ def write_asset(self, asset_id: int, *, background: bool) -> None:+ def write():+ with self.contract.write_transaction() as connection:+ connection.execute(+ "INSERT INTO asset(id,location,path,name,medium,size) VALUES(?,?,?,?,?,?)",+ (asset_id, "local", f"R:\\media\\{asset_id}.mp4", f"{asset_id}.mp4", "video", 1))+ thread = threading.Thread(target=in_background(write) if background else write)+ thread.start()+ thread.join(5)++ def wait_refreshed(self, key: str) -> None:+ deadline = time.monotonic() + 5+ while key in self.contract.stale_refreshing and time.monotonic() < deadline:+ time.sleep(0.01)+ self.assertNotIn(key, self.contract.stale_refreshing)++ def test_background_commits_serve_the_previous_result_until_the_refresh_lands(self):+ """后台任务写账本后先回上一份、另起重算;写回后结果版本跟着变,页面再取拿到新值。"""+ self.assertEqual(self.contract.cached_until_changed("catalog", self.compute, tables=("asset",)), 1)+ before = dispatch_api_get(self.contract, "/api/result-updates", {})["result_versions"]["catalog"]+ self.write_asset(1, background=True)+ self.assertEqual(self.contract.cached_until_changed("catalog", self.compute, tables=("asset",)), 1)+ self.wait_refreshed("catalog")+ self.assertEqual(self.contract.cached_until_changed("catalog", self.compute, tables=("asset",)), 2)+ self.assertEqual(self.calls, 2)+ after = dispatch_api_get(self.contract, "/api/result-updates", {})["result_versions"]["catalog"]+ self.assertNotEqual(after, before)++ def test_foreground_and_out_of_process_writes_still_recompute_on_the_spot(self):+ """写的人要立刻看到自己写的:前台提交之后,以及命令行、账本同步这类进程外的写之后,都当场重算。"""+ self.assertEqual(self.contract.cached_until_changed("catalog", self.compute, tables=("asset",)), 1)+ self.write_asset(1, background=True)+ self.write_asset(2, background=False)+ self.assertEqual(self.contract.cached_until_changed("catalog", self.compute, tables=("asset",)), 2)+ insert_asset(self.db_path, 3)+ self.assertEqual(self.contract.cached_until_changed("catalog", self.compute, tables=("asset",)), 3)+ self.assertEqual(self.contract.stale_refreshing, set())+def test_a_commit_during_a_versioned_computation_keeps_the_result(self):def derive():self.calls += 1