Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

perf(web): 后台任务写账本后页面先拿上一份聚合,重算在后台进行

longmeidao1460688e5083
diff --git a/docs/adr/0146-background-writes-serve-the-previous-aggregate.md b/docs/adr/0146-background-writes-serve-the-previous-aggregate.md
new file mode 100644
index 000000000..359991ba0
--- /dev/null
+++ b/docs/adr/0146-background-writes-serve-the-previous-aggregate.md
@@ -0,0 +1,33 @@
+# ADR-0146:后台任务写账本后,页面先拿上一份聚合
+
+- 状态:已采纳
+- 日期:2026-11-05
+- 关系:修改 `WebContract.cached_until_changed` 带 `tables` 那一支的失效口径;结果版本见 `web_result_versions.py`
+
+## 背景
+
+目录分组快照、筛选项、榜单、索引、统计、形状名单都按依赖表的版本缓存(`cached_until_changed(..., tables=...)`)。版本一变,下一位读者就当场重算,算完才返回。
+
+后台任务成批写账本时,这个口径让页面一直在等。2026-11-05 生产实测:补头像、补别名、采集这类后续任务每几秒提交一次,统计页第一次打开要等 3 秒,重启后定时任务集中补跑那几分钟,作品列表等了 13 秒。重算本身已经压过一轮(解析结果按输入记住),剩下的是 SQL 与聚合,压不到零。
+
+这些写入不是用户刚做的操作。页面上晚几秒看到一条新补的头像或别名,用户察觉不到;但等 3 秒才看到作品列表,他会察觉。
+
+## 决策
+
+**一、区分前台与后台提交。** `repository.in_background` 包住的线程入口算后台写入者:后台任务(`BackgroundJob`)、后续任务调度(`FollowupRunner`)、云下载循环、推送发现。它们提交时,`cache_bust` 记下 `background_generation`;其余线程(请求处理、测试、显式 `cache_bust`)的提交记 `foreground_generation`。没包的线程一律按前台算,保持原有的当场重算。
+
+**二、只有后台提交之后才先回旧值。** 带 `tables` 的键版本变了时,若手上那份出发之后没有前台提交,且本进程确有一次后台提交,就先回那一份,交给一条后台线程重算。同一个键在途时不重复排。前台提交之后照旧当场重算,写的人立刻看到自己写的。进程外的写(命令行、账本同步)不经提交通知,也当场重算。不带 `tables` 的键每次提交都被清掉,没有旧值可回,不在此列。
+
+**三、重算写回后通知页面。** 重算写回时 `cache_landed` 加一,并拼进每个账本结果版本键(`catalog`、`follow`、`discoveries`、`review`)。页面的结果轮询看到版本变了,再取一次,拿到新值。没有这一步,最后一笔后台写入之后页面会停在旧值,直到下一次写入。
+
+## 理由
+
+- 判据落在「谁提交的」而不是「谁在读」:同一页面既要在后台任务刷账本时不等,也要在用户自己批准、改名后立刻看到结果,只有提交方知道这一笔属于哪种。
+- 标记后台写入者、默认前台:漏标一个后台线程只是少一处加速,不会让用户的操作结果晚到;反过来标记前台,测试与请求内写完即读都会拿到旧值。
+- 一条重算线程:重算多是几秒的 CPU,和请求抢同一个进程;排成一队比并发重算对页面更友好。
+
+## 后果
+
+- 后台任务刚写完的几秒内,页面先显示上一份,重算写回后自动换成新的。
+- 每次写回都会让页面上各组读数各刷新一次;它们多半命中缓存,代价是几次轻量请求。
+- 新的后台写入线程要用 `in_background` 包入口才享受这一口径;不包也正确,只是照旧当场重算。
diff --git a/src/peach/downloads.py b/src/peach/downloads.py
index 68d0e3a93..5182931bd 100644
--- a/src/peach/downloads.py
+++ b/src/peach/downloads.py
@@ -37,7 +37,7 @@ from urllib.parse import parse_qs, unquote, urlsplit
from .follow_secrets import CredentialStore
from .fsutil import atomic_write_text
from .push_discovery import CloudPathError, CloudPrefix, map_cloud_path, normalise_prefix
-from .repository import LedgerDatabase
+from .repository import LedgerDatabase, in_background
LOGGER = logging.getLogger(__name__)
@@ -911,7 +911,7 @@ class DownloadService:
if not self.available or self._thread is not None:
return
self._stop.clear()
- self._thread = threading.Thread(target=self._loop, name="PeachDownloads", daemon=True)
+ self._thread = threading.Thread(target=in_background(self._loop), name="PeachDownloads", daemon=True)
self._thread.start()
if self.after_ingest is not None:
try:
diff --git a/src/peach/followups.py b/src/peach/followups.py
index 9e1ea3ed1..87c273d45 100644
--- a/src/peach/followups.py
+++ b/src/peach/followups.py
@@ -25,6 +25,7 @@ from dataclasses import dataclass
from pathlib import Path
from typing import Callable
+from .repository import in_background
from .task_runs import TaskRunHandle, TaskRunStore
LOGGER = logging.getLogger(__name__)
@@ -178,7 +179,7 @@ class FollowupRunner:
if thread is not None and thread.is_alive():
continue
thread = threading.Thread(
- target=self._work, args=(keys,), daemon=True,
+ target=in_background(self._work), args=(keys,), daemon=True,
name=f"PeachFollowup-{lane}")
self._threads[lane] = thread
thread.start()
diff --git a/src/peach/jobs.py b/src/peach/jobs.py
index f610e40fe..170a1b717 100644
--- a/src/peach/jobs.py
+++ b/src/peach/jobs.py
@@ -13,6 +13,7 @@ from dataclasses import dataclass
from pathlib import Path
from .media_sources import SOURCE_COSTS, cost_for, cost_sql
+from .repository import in_background
DEFAULT_METERED_LOCATIONS = frozenset(location for location, cost in SOURCE_COSTS.items()
if cost == 'metered') | {'online'}
@@ -501,7 +502,7 @@ class BackgroundJob:
"started_at": time.time(), "error": "", **(initial or {})}
state.update(self._resuming_fields(job_id, initial))
self.state = state
- thread = threading.Thread(target=self._run, args=(fn, job_id),
+ thread = threading.Thread(target=in_background(self._run), args=(fn, job_id),
daemon=True, name=self.name)
self.thread = thread
snapshot = copy.deepcopy(state)
diff --git a/src/peach/push_discovery.py b/src/peach/push_discovery.py
index e59ce19b2..cf4d629a3 100644
--- a/src/peach/push_discovery.py
+++ b/src/peach/push_discovery.py
@@ -43,6 +43,7 @@ from pathlib import Path, PureWindowsPath
from typing import Callable
from .fsutil import atomic_write_text
+from .repository import in_background
LOGGER = logging.getLogger(__name__)
@@ -402,7 +403,7 @@ class DiscoveryQueue:
return
self._stopping = False
self._thread = threading.Thread(
- target=self._loop, name="PeachPushDiscovery", daemon=True)
+ target=in_background(self._loop), name="PeachPushDiscovery", daemon=True)
self._thread.start()
def stop(self) -> None:
diff --git a/src/peach/repository.py b/src/peach/repository.py
index 43800c993..354770a12 100644
--- a/src/peach/repository.py
+++ b/src/peach/repository.py
@@ -10,6 +10,24 @@ from .catalog_rules import is_jav_code, normalise_code_key, sql_is_local_extra
from .entities import normalize_entity_name
from .regions import infer_region
+#: 当前线程是不是后台写入者,见 `in_background`。
+_WRITER = threading.local()
+
+
+def in_background(target):
+ """把线程入口包成后台写入者:它提交的写入只让页面缓存「过期」,读者可以先拿旧值(ADR-0146)。
+
+ 没包的线程照旧按前台算:测试、请求处理与显式失效写完就要读到新值。
+ """
+ def run(*args, **kwargs):
+ _WRITER.background = True
+ return target(*args, **kwargs)
+ return run
+
+
+def writing_in_background() -> bool:
+ return getattr(_WRITER, "background", False)
+
class LedgerDatabase:
"""Shared SQLite connection and transaction boundary for one Peach app."""
diff --git a/src/peach/web_result_versions.py b/src/peach/web_result_versions.py
index 475e7dcc1..ec08c65d4 100644
--- a/src/peach/web_result_versions.py
+++ b/src/peach/web_result_versions.py
@@ -20,8 +20,13 @@ RESULT_TABLES = {
def result_versions(contract):
- """复用账本触发器和文件版本,跨线程、CLI 写入和进程重启均可比较。"""
- result = {key: str(contract.ledger_revision(tables)) for key, tables in RESULT_TABLES.items()}
+ """复用账本触发器和文件版本,跨线程、CLI 写入和进程重启均可比较。
+
+ 后台任务写账本后,按表缓存的那几页先回旧值、另起重算(ADR-0146);重算写回时版本已经
+ 不会再变,所以每个账本键都带上写回次数,页面看到它变了再取一次新值。
+ """
+ landed = f":{contract.cache_landed}"
+ result = {key: str(contract.ledger_revision(tables)) + landed for key, tables in RESULT_TABLES.items()}
result['review'] += ':' + str(candidate_root_version(contract.candidate_root))
result['artwork'] = ':'.join(str(path_version(root)) for root in (
contract.cover_root, contract.avatar_root, contract.logo_root, contract.poster_root,
diff --git a/src/peach/web_state.py b/src/peach/web_state.py
index b4de8a048..2e2c202af 100644
--- a/src/peach/web_state.py
+++ b/src/peach/web_state.py
@@ -13,6 +13,7 @@
from __future__ import annotations
import json
+import logging
import math
import os
import sqlite3
@@ -44,9 +45,11 @@ from .media import normalized_path
# `previews` 是取图那一侧,不是 web 域处理器:依赖方向仍然只有一个走法。落盘名的
# 规则必须和 `/logo` 用的是同一个函数,否则可用性判定迟早和取图对不上。
from .previews import ENTITY_IMAGE_KINDS, LOGO_VARIANTS, entity_image_key, logo_key
-from .repository import LedgerDatabase
+from .repository import LedgerDatabase, writing_in_background
from .task_runs import TaskRunStore
+LOGGER = logging.getLogger(__name__)
+
class AvatarRootIndex(NamedTuple):
"""`avatar_root` 一次扫描的产物。两样东西同处一个目录,分开扫就是白扫两遍。
@@ -179,6 +182,16 @@ class WebContract:
self.cache_generation = 0
self.revision_cache_keys: set[str] = set()
self.aggregate_locks: dict[str, threading.RLock] = {}
+ #: 每个缓存键那一份结果出发时的 `cache_generation`;判它有没有包含最近一次前台写入。
+ self.cache_started: dict[str, int] = {}
+ #: 最近一次前台提交(不是 `repository.in_background` 包过的线程)之后的 `cache_generation`。
+ self.foreground_generation = 0
+ #: 最近一次后台提交之后的 `cache_generation`。进程外的写不经提交通知,两个数都不动。
+ self.background_generation = 0
+ #: 后台重算写回新值的次数,随结果版本下发,页面据此再取一次(ADR-0146)。
+ self.cache_landed = 0
+ self.stale_refreshing: set[str] = set()
+ self._stale_refresher = None
self.follow_check_lock = threading.Lock()
#: 任务中心。每个后台任务的开始、进度与结束都写这一张表,`/tasks` 只读它。
#: 只读端由 `api` 关掉写入:那边的账本是复制来的,写一行就是一处合不回去的分叉。
@@ -277,9 +290,11 @@ class WebContract:
# 时间戳沿用进入时的 now:算得比 TTL 还久的结果直接算过期,
# 宁可下次重算,也不要把一份已经旧了的数据当新的用。
self.cache[key] = (now, version, value)
+ self.cache_started[key] = generation
self.cache.move_to_end(key)
while len(self.cache) > CACHE_KEYS:
evicted, _ = self.cache.popitem(last=False)
+ self.cache_started.pop(evicted, None)
# 键空间不封闭的聚合(按作者、按筛选口径)的锁和登记跟着结果一起走,
# 否则这两份只增不减。正被别的线程拿着的锁摘掉也无妨,持有者手里还有它;
# 同键再来时 `cached_until_changed` 会重新登记。
@@ -344,11 +359,60 @@ class WebContract:
退回 `CACHE_TTL`。
`tables` 声明聚合依赖的表;这类缓存经过提交通知后仍按声明的版本核验,
同键的并发请求共用一次计算。
+
+ 带 `tables` 的键在版本变了、手上那份之后只有本进程的后台提交时,先回那一份,另起
+ 后台重算(ADR-0146):后台任务写的东西晚几秒无妨;前台写入与进程外的写之后照旧
+ 当场重算。不带 `tables` 的键每次提交都被清掉,没有旧值可回。
"""
if not tables:
revision = self.ledger_revision()
return self.cached(key, fn, ttl=CACHE_TTL if revision is None else math.inf,
version=(revision, *inputs))
+ revision = self.ledger_revision(tables)
+ if revision is not None:
+ version = (tuple(tables), revision, *inputs)
+ schedule = False
+ with self.cache_lock:
+ hit = self.cache.get(key)
+ started = self.cache_started.get(key, -1)
+ # 手上那份之后只有后台提交:没有前台写入,也有一次本进程的后台提交说明版本为什么变了。
+ # 进程外的写(命令行、账本同步)不经提交通知,照旧当场重算。
+ stale = (hit is not None and hit[1] != version and started >= self.foreground_generation
+ and self.background_generation > started)
+ if stale:
+ self.cache.move_to_end(key)
+ schedule = key not in self.stale_refreshing
+ self.stale_refreshing.add(key)
+ if stale:
+ if schedule:
+ self._refresh_later(key, fn, inputs, tables)
+ return hit[2]
+ return self._cached_by_tables(key, fn, inputs, tables)
+
+ def _refresh_later(self, key, fn, inputs, tables) -> None:
+ """一条后台线程依次重算过期的键,同一个键在途时不重复排。"""
+ from concurrent.futures import ThreadPoolExecutor
+
+ def work():
+ try:
+ self._cached_by_tables(key, fn, inputs, tables)
+ with self.cache_lock:
+ self.cache_landed += 1
+ except Exception: # noqa: BLE001 - 后台重算失败时下一次读照旧当场重算,原因记日志
+ LOGGER.exception("后台重算缓存 %s 失败", key)
+ with self.cache_lock:
+ self.cache.pop(key, None)
+ finally:
+ with self.cache_lock:
+ self.stale_refreshing.discard(key)
+
+ with self.cache_lock:
+ if self._stale_refresher is None:
+ self._stale_refresher = ThreadPoolExecutor(max_workers=1, thread_name_prefix="PeachCacheRefresh")
+ refresher = self._stale_refresher
+ refresher.submit(work)
+
+ def _cached_by_tables(self, key: str, fn, inputs, tables):
# 指定依赖表的聚合共用一次计算;提交后仍按这些表的代次核验。
with self.cache_lock:
lock = self.aggregate_locks.setdefault(key, threading.RLock())
@@ -389,6 +453,8 @@ class WebContract:
if hasattr(self, "jav_discovery_metadata"):
self.jav_discovery_metadata.stop()
self.followups.stop()
+ if self._stale_refresher is not None:
+ self._stale_refresher.shutdown(wait=False, cancel_futures=True)
def cache_bust(self):
with self.cache_lock:
@@ -397,6 +463,11 @@ class WebContract:
self.keyed_cache.clear()
# 在途计算靠这个数认出「我出发之后缓存失效过」,从而放弃写回。
self.cache_generation += 1
+ # 前台写入之后,出发更早的那份缓存不许再先回旧值:写的人要立刻看到自己写的。
+ if writing_in_background():
+ self.background_generation = self.cache_generation
+ else:
+ self.foreground_generation = self.cache_generation
def read_connection(self):
return self.database.read_connection()
diff --git a/tests/test_ledger_revision.py b/tests/test_ledger_revision.py
index ef6ea2f65..5f7f17f40 100644
--- a/tests/test_ledger_revision.py
+++ b/tests/test_ledger_revision.py
@@ -8,6 +8,8 @@ import json
import os
import sqlite3
import tempfile
+import threading
+import time
import unittest
from contextlib import closing
from datetime import datetime, timezone
@@ -18,7 +20,7 @@ from peach import taste_history, taste_preferences, web_follow, web_state, web_s
from peach.follow_sources import FollowCandidate, SourceFetch
from peach.follow_store import FollowStore
from peach.previews import entity_image_key, logo_key
-from peach.repository import LedgerDatabase
+from peach.repository import LedgerDatabase, in_background
from peach.web_catalog import FACETS_TABLES, GROUP_TABLES, TOPS_TABLES
from peach.web_contract import dispatch_api_get
from peach.web_entity import INDEX_TABLES
@@ -148,6 +150,44 @@ class LedgerRevisionTests(unittest.TestCase):
3 * web_state.CACHE_KEYS + 1)
self.assertIn("author:0", self.contract.revision_cache_keys)
+ def write_asset(self, asset_id: int, *, background: bool) -> None:
+ def write():
+ with self.contract.write_transaction() as connection:
+ connection.execute(
+ "INSERT INTO asset(id,location,path,name,medium,size) VALUES(?,?,?,?,?,?)",
+ (asset_id, "local", f"R:\\media\\{asset_id}.mp4", f"{asset_id}.mp4", "video", 1))
+ thread = threading.Thread(target=in_background(write) if background else write)
+ thread.start()
+ thread.join(5)
+
+ def wait_refreshed(self, key: str) -> None:
+ deadline = time.monotonic() + 5
+ while key in self.contract.stale_refreshing and time.monotonic() < deadline:
+ time.sleep(0.01)
+ self.assertNotIn(key, self.contract.stale_refreshing)
+
+ def test_background_commits_serve_the_previous_result_until_the_refresh_lands(self):
+ """后台任务写账本后先回上一份、另起重算;写回后结果版本跟着变,页面再取拿到新值。"""
+ self.assertEqual(self.contract.cached_until_changed("catalog", self.compute, tables=("asset",)), 1)
+ before = dispatch_api_get(self.contract, "/api/result-updates", {})["result_versions"]["catalog"]
+ self.write_asset(1, background=True)
+ self.assertEqual(self.contract.cached_until_changed("catalog", self.compute, tables=("asset",)), 1)
+ self.wait_refreshed("catalog")
+ self.assertEqual(self.contract.cached_until_changed("catalog", self.compute, tables=("asset",)), 2)
+ self.assertEqual(self.calls, 2)
+ after = dispatch_api_get(self.contract, "/api/result-updates", {})["result_versions"]["catalog"]
+ self.assertNotEqual(after, before)
+
+ def test_foreground_and_out_of_process_writes_still_recompute_on_the_spot(self):
+ """写的人要立刻看到自己写的:前台提交之后,以及命令行、账本同步这类进程外的写之后,都当场重算。"""
+ self.assertEqual(self.contract.cached_until_changed("catalog", self.compute, tables=("asset",)), 1)
+ self.write_asset(1, background=True)
+ self.write_asset(2, background=False)
+ self.assertEqual(self.contract.cached_until_changed("catalog", self.compute, tables=("asset",)), 2)
+ insert_asset(self.db_path, 3)
+ self.assertEqual(self.contract.cached_until_changed("catalog", self.compute, tables=("asset",)), 3)
+ self.assertEqual(self.contract.stale_refreshing, set())
+
def test_a_commit_during_a_versioned_computation_keeps_the_result(self):
def derive():
self.calls += 1