Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

perf(follow): 关注分组按账本版本缓存,单条详情走轻量路径

longmeidaoaf5957e1c59e
diff --git a/src/peach/web_follow.py b/src/peach/web_follow.py
index ca2fdd841..2ab44ed56 100644
--- a/src/peach/web_follow.py
+++ b/src/peach/web_follow.py
@@ -1081,7 +1081,6 @@ def _item_payload(item, credential_providers: frozenset[str] = frozenset(), tag_
"tags": policy.project(tags),
"detail_tags": policy.project(detail_tags),
"tag_types": tag_types,
- "source_tags": detail_tags,
}
@@ -1715,21 +1714,39 @@ def q_follow_suggest(contract, q: str, limit: int = MAX_SUGGESTIONS,
]}
-def _page_groups(store, everything, counted, by_author, *, item_id, statuses, order, window):
- """这一次要回的作品组,以及后面还有没有。"""
- if item_id is not None:
- # 直达详情取这一条所在的整组。组是读时拼的(剥作者名、相似标题),只按库里
- # 同键去捞会漏掉别的站上那几份,详情就比列表里的卡片少。
- ranked = tuple(group for group in store.group(everything, by_author)
- if item_id in {member.id for member in
- (group.primary, *group.variants, *group.duplicates)})
- return ranked, False # 直达详情只取这一组,没有下一页
+#: 关注条目、分组与筛选项读到的账本表:条目与来源、作者别名、来源绑定的实体名,以及
+#: 在线标签归类所在的 profile 设置。播放心跳只写 `follow_playback`,不在其中:看片时
+#: 每十秒一次的上报不让这几份整库结果重算。
+FOLLOW_GROUP_TABLES = ("follow_item", "follow_source", "follow_author_alias", "entity", "profile")
+
+
+def _item_groups(contract, store, everything, by_author, source_id) -> dict:
+ """条目 id → 它所在的整组。直达详情查这一份,整库只分一次组,按账本版本缓存。
+
+ 组是读时拼的(剥作者名、相似标题),只按库里同键去捞会漏掉别的站上那几份,详情
+ 就比列表里的卡片少,所以还是整库分组。
+ """
+ def build():
+ return {member.id: group for group in store.group(everything, by_author)
+ for member in (group.primary, *group.variants, *group.duplicates)}
+ return contract.cached_until_changed(
+ f"follow-item-groups:{source_id}", build, tables=FOLLOW_GROUP_TABLES)
+
+
+def _page_groups(contract, store, everything, counted, by_author, *, statuses, order, window, cache_key):
+ """这一次要回的作品组,以及后面还有没有。
+
+ `cache_key` 不为空时整批排好的组按账本版本缓存,翻页、回到列表都不再重新分组;
+ 带作者、来源、标签等筛选或随机排序的请求传空,现算。
+ """
# 分页在筛选与分组之后:SQL 先分页、前端再筛的话,选个冷门作者会看到一页里
# 只剩两三条,得反复点「加载更多」才凑出一屏;按条目切页再分组的话,同一作品
# 的几份上传落在相邻两页,页面上就是两张卡。所以整批分组,`limit` 数的是组。
+ def rank():
+ page = tuple(item for item in counted if not statuses or item.status in statuses)
+ return _sorted_groups(store.group(page, by_author, everything), *order)
+ ranked = contract.cached_until_changed(cache_key, rank, tables=FOLLOW_GROUP_TABLES) if cache_key else rank()
offset, limit = window
- page = tuple(item for item in counted if not statuses or item.status in statuses)
- ranked = _sorted_groups(store.group(page, by_author, everything), *order)
return ranked[offset:offset + limit], len(ranked) > offset + limit
@@ -1742,6 +1759,10 @@ def _duration_arg(raw) -> float:
return value if value > 0 else 0.0
+#: `_follow_matcher` 读的筛选参数。带了任一个,列表就是全库的一个子集。
+FOLLOW_FILTER_ARGS = ("author", "provider", "tag", "work", "dur_min", "dur_max")
+
+
def _follow_matcher(args, by_source, alias_map, tag_policy=None):
"""关注页筛选条件落到一条条目上:给一条,回它在不在这次筛选里。
@@ -1833,26 +1854,44 @@ def q_follow(contract, args) -> dict:
everything = contract.cached_until_changed(
f"follow-items:{source_id}",
lambda: tuple(item for item in store.items(source_id=source_id, limit=_ALL_ITEMS)
- if item.source_id in enabled_source_ids and not _excluded_item(item)))
- counted = _sorted_items(
- tuple(item for item in everything if _matches(item)), sort, direction, seed)
+ if item.source_id in enabled_source_ids and not _excluded_item(item)),
+ tables=FOLLOW_GROUP_TABLES)
by_author = group_authors(source_rows, alias_map)
- ranked, has_more = ((), False) if summary else _page_groups(
- store, everything, counted, by_author, item_id=item_id, statuses=statuses,
- order=(sort, direction, seed), window=(offset, limit))
# 翻卡与封面计数要知道组里哪几张是同一个画面,判据与缓存见 `follow_faces`。
faces = getattr(contract, "follow_faces", None)
- groups = [annotate_group(_group_payload(group, credential_providers, tag_policy), faces,
- _content_hashes(group))
- for group in ranked]
+
+ def payload(group):
+ return annotate_group(_group_payload(group, credential_providers, tag_policy), faces,
+ _content_hashes(group))
+
+ if item_id is not None:
+ # 直达详情(`fetchFollowItem`)只读这一组和署名要的来源、别名;筛选项、计数与
+ # 「猜你喜欢」都不算,也不随响应下发。
+ group = _item_groups(contract, store, everything, by_author, source_id).get(item_id)
+ return {"ok": True, "sources": sources, "author_aliases": author_aliases,
+ "groups": [payload(group)] if group is not None else [], "has_more": False}
+ counted = _sorted_items(
+ tuple(item for item in everything if _matches(item)), sort, direction, seed)
+ filtered = any(args.get(key) for key in FOLLOW_FILTER_ARGS)
+ ranked, has_more = ((), False) if summary else _page_groups(
+ contract, store, everything, counted, by_author, statuses=statuses,
+ order=(sort, direction, seed), window=(offset, limit),
+ cache_key=None if filtered or sort == "rand"
+ else f"follow-ranked:{source_id}:{','.join(statuses)}:{sort}:{direction}")
+ groups = [payload(group) for group in ranked]
+ # 筛选项按全库算,只随首页下发:页面把几页并成一份时缺的键沿用首页那份
+ # (`mergedPage`),每翻一页再带一遍就是几十 KB 的重复。
+ facet_args = {} if offset else {"facets": {}, "author_facets": {}}
# 题材圆标的有无与取景读题材头像目录,目录版本进键。
icon_root = work_icon_root(contract)
- facets = {} if summary else contract.cached_until_changed(
- f"follow-facets:{source_id}",
- lambda: _follow_facets(store, everything, by_source, alias_map, icon_root, tag_policy),
- path_version(icon_root))
- author_facets = _author_facets(contract, store, everything, by_source, alias_map, args,
- (source_id, summary), tag_policy)
+ if facet_args and not summary:
+ facet_args["facets"] = contract.cached_until_changed(
+ f"follow-facets:{source_id}",
+ lambda: _follow_facets(store, everything, by_source, alias_map, icon_root, tag_policy),
+ path_version(icon_root), tables=FOLLOW_GROUP_TABLES)
+ if facet_args:
+ facet_args["author_facets"] = _author_facets(
+ contract, store, everything, by_source, alias_map, args, (source_id, summary), tag_policy)
# counts 与列表同源,两边都从 `counted` 出发:筛选怎么变,数字就怎么变,
# 扣减逻辑也只写一份。写成一句全库 SQL 再逐条减掉被隐藏的 rule34video 和
# 无资源的 f95zone 的话,同一套排除规则要维护两份,而且它不看作者、来源和
@@ -1874,8 +1913,7 @@ def q_follow(contract, args) -> dict:
"sort": sort,
"dir": direction,
"seed": seed,
- "facets": facets,
- "author_facets": author_facets,
+ **facet_args,
# counts 是全库口径,groups 只是这一页——两个数并排显示过,看起来像自相矛盾。
"offset": offset,
"limit": limit,
diff --git a/tests/test_follow_web.py b/tests/test_follow_web.py
index 2c1a2cac9..867c64dd2 100644
--- a/tests/test_follow_web.py
+++ b/tests/test_follow_web.py
@@ -536,6 +536,46 @@ class FollowContractTests(unittest.TestCase):
payload = self._get(item=str(item_id), limit="1")
self.assertEqual(payload["groups"][0]["primary"]["id"], item_id)
self.assertEqual(len(payload["groups"][0]["variants"]), 1)
+ # 详情只读这一组和署名要的来源、别名;整库筛选项不随单条下发。
+ self.assertNotIn("facets", payload)
+ self.assertEqual(payload["sources"][0]["label"], "LazyProcrastinator")
+ self.assertEqual(self._get(item=str(item_id + 999))["groups"], [])
+
+ def test_feed_groups_once_per_ledger_version_and_ignores_playback_heartbeats(self):
+ """整库分组按账本版本缓存:翻页、直达详情与播放心跳都不再分组,条目状态变了才重分。"""
+ self._seed(candidates=(
+ FollowCandidate(provider="rule34video", external_id="1", title="First Work",
+ url="https://rule34video.com/video/1/x/", published_at="2026-08-18T00:00:00Z"),
+ FollowCandidate(provider="rule34video", external_id="2", title="Second Work",
+ url="https://rule34video.com/video/2/x/", published_at="2026-08-17T00:00:00Z"),
+ ))
+ grouped = []
+ original = FollowStore.group
+
+ def counted(*args, **kwargs):
+ grouped.append(1)
+ return original(*args, **kwargs)
+
+ with mock.patch.object(FollowStore, "group", side_effect=counted):
+ first = self._get(limit="1")
+ item = first["groups"][0]["primary"]["id"]
+ self.assertIn("facets", first)
+ second = self._get(limit="1", offset="1")
+ # 翻页只接组,筛选项沿用首页那份。
+ self.assertNotIn("facets", second)
+ self.assertNotEqual(second["groups"][0]["primary"]["id"], item)
+ self.assertEqual(self._get(item=str(item))["groups"][0]["primary"]["id"], item)
+ settled = len(grouped)
+ self._get(limit="1")
+ self._get(item=str(item))
+ self._post("/api/follow/activity", {"item": item, "position": 5, "duration": 20, "delta": 5})
+ self._get(limit="1")
+ self._get(item=str(item))
+ self.assertEqual(len(grouped), settled)
+ self._post("/api/follow/status", {"item": item, "to": "ignored"})
+ relisted = self._get(limit="1")
+ self.assertGreater(len(grouped), settled)
+ self.assertEqual(relisted["counts"]["ignored"], 1)
def test_feed_never_exposes_the_raw_media_url(self):
# 界面只需要知道有没有媒体;直链是来源层的事,不进公共 JSON。
diff --git a/tests/test_tag_management.py b/tests/test_tag_management.py
index ab557af00..d56e2238a 100644
--- a/tests/test_tag_management.py
+++ b/tests/test_tag_management.py
@@ -242,9 +242,12 @@ class OnlineTagManagementTests(unittest.TestCase):
feed = web_follow.q_follow(self.contract, {})
self.assertNotIn('outdoors', dict(feed['facets']['tags']))
self.assertTrue(all('outdoors' not in group['primary']['detail_tags'] for group in feed['groups']))
- self.assertTrue(any('outdoors' in group['primary']['source_tags'] for group in feed['groups']))
+ self.assertTrue(any('outdoors' in metadata for (metadata,) in self.c.execute('SELECT metadata_json FROM follow_item')))
self.execute('restore', ['outdoors'])
self.assertIn('outdoors', [row['k'] for row in web_follow.q_follow_tags(self.contract, {'types': 'all'})['items']])
+ restored = web_follow.q_follow(self.contract, {})
+ self.assertIn('outdoors', dict(restored['facets']['tags']))
+ self.assertTrue(any('outdoors' in group['primary']['detail_tags'] for group in restored['groups']))
def test_online_conflicts_types_and_stale_plans_are_rejected(self):
self.seed()