Peach: Local-first video metadata scraper and media manager.
feat(tags): 管理词表支持筛选分页与稳定总数
longmeidaoa988ce4a94d1
diff --git a/docs/TAG_MANAGEMENT.md b/docs/TAG_MANAGEMENT.mdindex a82970417..a419dca66 100644--- a/docs/TAG_MANAGEMENT.md+++ b/docs/TAG_MANAGEMENT.md@@ -48,6 +48,21 @@ kind: howto`GET /api/tag-management` 返回标签 ID、名称、分类、别名、资源数量、隐藏状态和重复组。`POST /api/tag-management/preview` 只读预览;`POST /api/tag-management` 收到预览的 `token` 才执行写入。数据变化后计划失效,须重新预览。接口沿用登录验证和 ledger 写入角色门槛。+`GET /api/tag-management` 不带 `limit` 时返回完整的 `items` 和 `duplicate_groups`;带 `limit` 时启用分页,支持以下参数:++| 参数 | 含义 |+| --- | --- |+| `scope` | `local` 或 `online`,默认 `local` |+| `limit` / `offset` | 每页条数和起始位置;条数限制为 1 至 600,格式无效时使用 20;起始位置至少为 0,格式无效时使用 0 |+| `q` | 搜索名称和别名,统一字符宽度、空白和大小写 |+| `category` | 分类键;空值或 `all` 表示全部分类 |+| `state` | `active`、`hidden` 或 `blocked`,默认 `active`;屏蔽状态优先于隐藏状态 |+| `view` | `cloud` 按资源数量倒序,`alphabet` 按归一化名称排序;默认 `cloud`,同值时按名称和标签身份稳定排序 |++分页响应包含当前页 `items`、筛选后的 `total`、`has_more`、实际 `limit` 和 `offset`。`categories` 是当前搜索和状态下的分类计数,不随选中的分类缩小;`duplicate_groups` 保留本地完整重复组,在线为空列表。搜索、状态、分类和排序均先于分页,超出末页时 `items` 为空且保留 `total`。完整词表复用查询缓存,按本地与在线分别保存;账本版本变化或写入触发缓存失效后重新读取。++浏览接口 `GET /api/index?kind=tags` 与 `GET /api/follow/tags?types=all` 返回 `total`,分别表示搜索及分类筛选后、搜索及来源类型筛选后的完整标签数。+写入会检查外键变化并拒绝本次操作新增的关联违规;已有的历史引用记录保持独立。在线词表的读取参数和操作请求使用 `scope=online`;在线标签 ID 是稳定的来源词键,本地标签 ID 是实体 ID。diff --git a/src/peach/web_entity.py b/src/peach/web_entity.pyindex 74ab5aa88..585005e0b 100644--- a/src/peach/web_entity.py+++ b/src/peach/web_entity.py@@ -664,7 +664,11 @@ def q_index(contract: WebContract, kind, q="", limit=600, offset=0, category="")repr(("index", kind, limit, offset, category)), read, tables=INDEX_TABLES)# 下面还要往每行补目录索引的标志,不能写进缓存里那一份。rows = [dict(row) for row in rows]- return _decorate_index(contract, kind, rows, has_more, category_counts)+ result = _decorate_index(contract, kind, rows, has_more, category_counts)+ if kind == 'tags':+ result['total'] = (category_counts.get(category, 0) if category and category != 'all'+ else sum(category_counts.values()))+ return resultdef _index_rows(contract: WebContract, kind, q, limit, offset, category):diff --git a/src/peach/web_follow.py b/src/peach/web_follow.pyindex 6f30acd72..a40d07999 100644--- a/src/peach/web_follow.py+++ b/src/peach/web_follow.py@@ -1467,6 +1467,7 @@ def q_follow_tags(contract, args) -> dict:and (not query or any(query in word.casefold() for word in [row['k'], *row.get('aliases', [])]))]return {"kind": "tags", "scope": "online","items": rows[offset:offset + limit],+ "total": len(rows),"has_more": offset + limit < len(rows),"categories": categories}diff --git a/src/peach/web_tags.py b/src/peach/web_tags.pyindex cac388a40..cc549dcdb 100644--- a/src/peach/web_tags.py+++ b/src/peach/web_tags.py@@ -1,13 +1,15 @@"""标签管理的读取、预览与事务写入端点。"""from __future__ import annotations+from copy import deepcopyfrom datetime import datetime, timezonefrom uuid import uuid4+from .catalog_rules import clean_tag_namefrom .migrations import sqlite_backupfrom .tag_management import apply_plan, duplicate_groups, preview, tag_rowsfrom . import online_tag_management-from .web_follow import online_tag_inventory+from .web_follow import FOLLOW_GROUP_TABLES, online_tag_inventorydef _online(body):@@ -18,12 +20,62 @@ def _online(body):def q_tag_management(contract, args):- with contract.read_connection() as connection:- if _online(args):- return {'items': [{key: value for key, value in row.items() if not key.startswith('_')}- for row in online_tag_inventory(connection)], 'duplicate_groups': []}- rows = tag_rows(connection)- return {'items': rows, 'duplicate_groups': duplicate_groups(rows)}+ online = _online(args)+ def read():+ with contract.read_connection() as connection:+ if online:+ return {'items': [{key: value for key, value in row.items() if not key.startswith('_')}+ for row in online_tag_inventory(connection)], 'duplicate_groups': []}+ rows = tag_rows(connection)+ return {'items': rows, 'duplicate_groups': duplicate_groups(rows)}+ tables = FOLLOW_GROUP_TABLES if online else ('entity', 'entity_alias', 'asset_entity')+ inventory = contract.cached(f'tag-management:{"online" if online else "local"}', read,+ version=contract.ledger_revision(tables))+ if 'limit' not in args:+ return deepcopy(inventory)+ return _tag_page(inventory, args)+++def _name_key(value):+ return online_tag_management.tag_key(clean_tag_name(value))+++def _tag_page(inventory, args):+ state = str(args.get('state') or 'active').strip().casefold()+ view = str(args.get('view') or 'cloud').strip().casefold()+ if state not in ('active', 'hidden', 'blocked'):+ raise ValueError('未知的标签状态')+ if view not in ('cloud', 'alphabet'):+ raise ValueError('未知的标签视图')+ try:+ limit = min(max(int(args.get('limit') or 20), 1), 600)+ except (TypeError, ValueError):+ limit = 20+ try:+ offset = max(int(args.get('offset') or 0), 0)+ except (TypeError, ValueError):+ offset = 0+ query = _name_key(args.get('q') or '')+ rows = [row for row in inventory['items']+ if _tag_state(row) == state and (not query or any(+ query in _name_key(name) for name in [row['k'], *row.get('aliases', [])]))]+ categories: dict[str, int] = {}+ for row in rows:+ categories[row['cat']] = categories.get(row['cat'], 0) + 1+ category = str(args.get('category') or '')+ if category and category != 'all':+ rows = [row for row in rows if row['cat'] == category]+ def order(row):+ name = (_name_key(row['k']), row['k'], str(row.get('entity_id', row.get('id'))))+ return (-row['n'], *name) if view == 'cloud' else name+ rows.sort(key=order)+ return {'items': deepcopy(rows[offset:offset + limit]), 'total': len(rows),+ 'has_more': offset + limit < len(rows), 'limit': limit, 'offset': offset,+ 'categories': categories, 'duplicate_groups': deepcopy(inventory['duplicate_groups'])}+++def _tag_state(row):+ return 'blocked' if row.get('blocked') else 'hidden' if row.get('hidden') else 'active'def w_tag_preview(contract, body):diff --git a/tests/test_tag_management.py b/tests/test_tag_management.pyindex d9a0355c5..37f41d308 100644--- a/tests/test_tag_management.py+++ b/tests/test_tag_management.py@@ -7,7 +7,7 @@ import unittestfrom contextlib import closingfrom pathlib import Pathfrom types import SimpleNamespace-from unittest.mock import Mock+from unittest.mock import Mock, patchfrom support.ledger import fresh_ledgerfrom peach.catalog_rules import current_tags@@ -15,7 +15,8 @@ from peach.entities import upsert_asset_entityfrom peach.repository import LedgerDatabasefrom peach.tag_management import apply_plan, duplicate_groups, preview, tag_rowsfrom peach.web_catalog import con_tags, tag_not_hidden-from peach.web_tags import w_tag_management+from peach.web_tags import q_tag_management, w_tag_management+from peach.web_entity import q_indexfrom peach import online_tag_management, web_follow, web_library_shortcutsfrom peach.follow_sources import FollowCandidate, SourceFetchfrom peach.follow_store import FollowStore@@ -47,6 +48,87 @@ class TagManagementTests(unittest.TestCase):plan = preview(self.c, body)return apply_plan(self.c, dict(body, token=plan['token']))+ def management_contract(self):+ root = Path(self.tmp.name).resolve()+ contract = WebContract(self.db, follow_sources_root=root / 'sources', follow_secrets_root=root / 'secrets',+ follow_shared_root=root / 'shared', candidate_root=root / 'generated', follow_state_root=root / 'state')+ self.addCleanup(contract.stop_background_jobs)+ contract.followups.stop()+ return contract++ def test_management_pages_filter_aliases_width_categories_and_state_before_slicing(self):+ original = self.tag('Original', (1, 2))+ self.execute('rename', [original], name='Alpha')+ self.tag('Beta')+ self.tag('gamma', (1, 2, 3))+ self.tag('眼镜')+ hidden = self.tag('Dormant')+ self.execute('hide', [hidden])+ self.c.commit()+ contract = self.management_contract()+ first = q_tag_management(contract, dict(limit='2'))+ second = q_tag_management(contract, dict(limit='2', offset='2'))+ self.assertEqual([row['k'] for row in first['items']], ['gamma', 'Alpha'])+ self.assertEqual([row['k'] for row in second['items']], ['Beta', '眼镜'])+ self.assertEqual((first['total'], first['has_more'], first['limit'], first['offset']), (4, True, 2, 0))+ self.assertEqual((second['total'], second['has_more']), (4, False))+ alphabet = q_tag_management(contract, dict(limit='2', view='alphabet'))+ self.assertEqual([row['k'] for row in alphabet['items']], ['Alpha', 'Beta'])+ found = q_tag_management(contract, dict(limit='20', q='ORIGINAL'))+ self.assertEqual([row['k'] for row in found['items']], ['Alpha'])+ self.assertEqual((found['total'], found['categories']), (1, {'general': 1}))+ category = q_tag_management(contract, dict(limit='1', category='appearance'))+ self.assertEqual([row['k'] for row in category['items']], ['眼镜'])+ self.assertEqual((category['total'], category['categories']), (1, {'general': 3, 'appearance': 1}))+ hidden_page = q_tag_management(contract, dict(limit='20', state='hidden'))+ self.assertEqual([row['k'] for row in hidden_page['items']], ['Dormant'])+ self.assertEqual(q_tag_management(contract, dict(limit='20', state='blocked'))['total'], 0)+ past_end = q_tag_management(contract, dict(limit='2', offset='99'))+ self.assertEqual((past_end['items'], past_end['total'], past_end['has_more']), ([], 4, False))+ complete = q_tag_management(contract, dict(q='不存在', state='active'))+ self.assertEqual(set(complete), {'items', 'duplicate_groups'})+ self.assertEqual(len(complete['items']), 5)++ def test_management_page_limits_duplicate_groups_and_cache_invalidation(self):+ self.tag('眼镜')+ self.c.execute("INSERT INTO entity(kind,canonical_name,normalized_name,created_at,updated_at) VALUES('tag','眼镜娘','眼镜娘','now','now')")+ self.c.commit()+ contract = self.management_contract()+ with patch('peach.web_tags.tag_rows', wraps=tag_rows) as inventory:+ complete = q_tag_management(contract, {})+ self.assertEqual(len(complete['duplicate_groups']), 1)+ for raw, expected in [('invalid', 20), ('0', 1), ('-5', 1), ('99999', 600)]:+ page = q_tag_management(contract, dict(limit=raw, offset='invalid', q='不存在'))+ self.assertEqual((page['limit'], page['offset'], page['total']), (expected, 0, 0))+ self.assertEqual(page['duplicate_groups'], complete['duplicate_groups'])+ self.assertEqual(inventory.call_count, 1)+ complete['items'][0]['k'] = '客户端临时值'+ self.assertNotIn('客户端临时值', [row['k'] for row in q_tag_management(contract, {})['items']])+ contract.cache_bust()+ q_tag_management(contract, dict(limit='1'))+ self.assertEqual(inventory.call_count, 2)+ self.tag('新标签')+ self.c.commit()+ self.assertEqual(q_tag_management(contract, dict(limit='20'))['total'], 3)+ self.assertEqual(inventory.call_count, 3)+ for args in [dict(limit='20', state='unknown'), dict(limit='20', view='unknown')]:+ with self.subTest(args=args), self.assertRaisesRegex(ValueError, '未知的标签'):+ q_tag_management(contract, args)++ def test_local_browse_tag_total_counts_filtered_rows_on_every_page(self):+ self.tag('Alpha')+ self.tag('Beta')+ self.tag('眼镜')+ self.c.commit()+ contract = self.management_contract()+ first = q_index(contract, 'tags', limit=1)+ self.assertEqual((len(first['items']), first['total'], first['has_more']), (1, 3, True))+ category = q_index(contract, 'tags', limit=1, category='general', offset=20)+ self.assertEqual((category['items'], category['total'], category['has_more']), ([], 2, False))+ searched = q_index(contract, 'tags', q='Alpha', limit=1)+ self.assertEqual((len(searched['items']), searched['total']), (1, 1))+ self.assertNotIn('total', q_index(contract, 'performers', limit=1))+def test_rename_preserves_unrelated_historical_reference_violations(self):entity_id = self.tag('眼镜')self.c.commit()@@ -215,6 +297,47 @@ class OnlineTagManagementTests(unittest.TestCase):command['token'] = online_tag_management.preview(connection, command, web_follow.online_tag_inventory(connection))['token']return w_tag_management(self.contract, command)+ def test_online_management_pages_share_inventory_and_partition_blocked_hidden_active(self):+ self.seed()+ self.execute('rename', ['outdoors'], name='户外')+ self.execute('hide', ['outdoors'])+ self.execute('block', ['outdoors'])+ args = dict(scope='online', limit='1')+ with patch('peach.web_tags.online_tag_inventory', wraps=web_follow.online_tag_inventory) as inventory:+ first = q_tag_management(self.contract, args)+ second = q_tag_management(self.contract, dict(args, offset='1'))+ self.assertEqual([row['k'] for row in first['items']], ['landscape'])+ self.assertEqual([row['k'] for row in second['items']], ['preview_artist'])+ self.assertEqual((first['total'], first['has_more'], second['has_more']), (2, True, False))+ self.assertEqual(first['categories'], {'general': 1, 'artist': 1})+ artist = q_tag_management(self.contract, dict(args, category='artist'))+ self.assertEqual((artist['total'], artist['categories']), (1, first['categories']))+ self.assertEqual([row['k'] for row in artist['items']], ['preview_artist'])+ hidden = q_tag_management(self.contract, dict(args, state='hidden'))+ self.assertEqual((hidden['items'], hidden['total']), ([], 0))+ blocked = q_tag_management(self.contract, dict(args, state='blocked', q='OUTDOORS'))+ self.assertEqual([row['k'] for row in blocked['items']], ['户外'])+ self.assertEqual((blocked['total'], blocked['categories'], blocked['duplicate_groups']), (1, {'general': 1}, []))+ complete = q_tag_management(self.contract, dict(scope='online'))+ self.assertEqual(set(complete), {'items', 'duplicate_groups'})+ self.assertEqual(len(complete['items']), 3)+ self.assertTrue(all(not key.startswith('_') for row in complete['items'] for key in row))+ self.assertEqual(inventory.call_count, 1)+ self.contract.cache_bust()+ q_tag_management(self.contract, args)+ self.assertEqual(inventory.call_count, 2)+ self.execute('unblock', ['outdoors'])+ self.assertEqual([row['k'] for row in q_tag_management(self.contract, dict(args, state='hidden'))['items']], ['户外'])++ def test_online_browse_tag_total_follows_search_and_type_before_paging(self):+ self.seed()+ first = web_follow.q_follow_tags(self.contract, dict(types='all', limit='1'))+ self.assertEqual((len(first['items']), first['total'], first['has_more']), (1, 3, True))+ artist = web_follow.q_follow_tags(self.contract, dict(types='all', type='artist', limit='1', offset='10'))+ self.assertEqual((artist['items'], artist['total'], artist['has_more']), ([], 1, False))+ found = web_follow.q_follow_tags(self.contract, dict(types='all', q='outdoors', limit='1'))+ self.assertEqual((len(found['items']), found['total']), (1, 1))+def test_rename_merge_and_source_updates_share_one_online_identity(self):source = self.seed()originals = list(map(tuple, self.c.execute('SELECT id,metadata_json FROM follow_item ORDER BY id')))