Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(catalog): 限定筛选聚合工作量并保留修订缓存

longmeidao2302cdf64478
diff --git a/src/peach/web_catalog.py b/src/peach/web_catalog.py
index c0d5e85b5..a3c9af969 100644
--- a/src/peach/web_catalog.py
+++ b/src/peach/web_catalog.py
@@ -1422,18 +1422,19 @@ def q_facets(
f"SELECT {EFFECTIVE_REGION_SQL} AS k,count(*) AS n FROM asset a "
"WHERE a.medium='video' " + scope + "GROUP BY k ORDER BY n DESC",
scope_params))
+ # 先独立筛选资产,避免错误统计让每对实体与视频都执行资产谓词。
+ facet_assets = "SELECT a.id FROM asset a WHERE a.medium='video' " + scope
out["creators"] = [dict(r) for r in c.execute(
"SELECT e.canonical_name AS k,count(DISTINCT ae.asset_id) AS n "
"FROM asset_entity ae JOIN entity e ON e.id=ae.entity_id "
- "JOIN asset a ON a.id=ae.asset_id WHERE a.medium='video' AND e.kind='creator' " + scope +
-
+ "WHERE e.kind='creator' AND ae.asset_id IN (" + facet_assets + ") "
"GROUP BY e.id,e.canonical_name ORDER BY n DESC LIMIT 60", scope_params)]
# 标签要分层 —— 一锅端会让「演员:一个ren」和「1080P」「足交」混在一起。
# 三类分开:技术规格(画质/时长/画幅,筛选价值低)、内容维度(真正有用的)、演员(另立一栏)。
rows = [dict(r) for r in c.execute(
"SELECT e.canonical_name AS k, count(DISTINCT ae.asset_id) AS n "
"FROM asset_entity ae JOIN entity e ON e.id=ae.entity_id "
- "JOIN asset a ON a.id=ae.asset_id WHERE a.medium='video' AND e.kind='tag' " + scope +
+ "WHERE e.kind='tag' AND ae.asset_id IN (" + facet_assets + ") "
"AND " + tag_not_hidden('ae.asset_id', 'e.normalized_name') + ' ' +
"AND " + tag_is_not_a_performer_name("e.normalized_name") + " "
"GROUP BY e.id,e.canonical_name ORDER BY n DESC LIMIT 400", scope_params)]
@@ -1449,8 +1450,7 @@ def q_facets(
out["tagperformers"] = [dict(r) for r in c.execute(
"SELECT e.canonical_name AS k,count(DISTINCT ae.asset_id) AS n "
"FROM asset_entity ae JOIN entity e ON e.id=ae.entity_id "
- "JOIN asset a ON a.id=ae.asset_id WHERE a.medium='video' AND e.kind='performer' " + scope +
-
+ "WHERE e.kind='performer' AND ae.asset_id IN (" + facet_assets + ") "
"GROUP BY e.id,e.canonical_name ORDER BY n DESC LIMIT 20", scope_params)]
st = c.execute(
"SELECT count(*) total, COALESCE(sum(size),0) bytes, "
diff --git a/src/peach/web_router.py b/src/peach/web_router.py
index 17d7a8f42..624d659a7 100644
--- a/src/peach/web_router.py
+++ b/src/peach/web_router.py
@@ -203,6 +203,8 @@ def _get_facets(contract, args):
'len', 'dur_min', 'dur_max', 'orient', 'region', 'exclude_vertical', 'q', 'thumb',
'library', 'folder', 'folder_recursive', 'year',
) if args.get(key)}
+ if not filters.get("tag"):
+ filters.pop("tag_match", None)
scope_key += repr(sorted(filters.items()))
compute = lambda: q_facets( # noqa: E731
contract,
diff --git a/tests/test_rm_web.py b/tests/test_rm_web.py
index a01850619..0d7b84f6e 100644
--- a/tests/test_rm_web.py
+++ b/tests/test_rm_web.py
@@ -215,6 +215,104 @@ class WebDataTests(unittest.TestCase):
con.close()
return row
+ def test_facets_bound_asset_predicate_work_with_inaccurate_statistics(self):
+ asset_count = 400
+ with self.contract.write_transaction() as connection:
+ connection.executemany(
+ "INSERT INTO asset(id,location,path,name,medium,size) VALUES(?,'local',?,?,'video',1)",
+ [(1000 + i, f"R:/Media/demo-{i}.mp4", f"demo-{i}.mp4")
+ for i in range(asset_count)])
+ connection.executemany(
+ "INSERT INTO entity(id,kind,canonical_name,normalized_name,created_at,updated_at) "
+ "VALUES(?,?,?,?, '2026-01-01','2026-01-01')",
+ [(1000 + kind_index * 120 + i, kind, f"Demo {kind} {i}", f"demo {kind} {i}")
+ for kind_index, kind in enumerate(("performer", "creator", "tag"))
+ for i in range(120)])
+ connection.executemany(
+ "INSERT INTO asset_entity(asset_id,entity_id,role,source,confidence) VALUES(?,?,?,'test',1)",
+ [(1000 + i, 1000 + kind_index * 120 + i % 20, kind)
+ for kind_index, kind in enumerate(("performer", "creator", "tag"))
+ for i in range(asset_count)])
+ # 同一关联有多个来源时,每个资产仍只计一次。
+ connection.execute(
+ "INSERT INTO asset_entity(asset_id,entity_id,role,source,confidence) "
+ "VALUES(1000,1000,'performer','second-source',1)")
+ connection.execute("ANALYZE")
+
+ filters = {"loc": "local,115", "tag_match": "all", "thumb": "0"}
+ expected = rm_web.q_facets(self.contract, filters=filters)
+ with self.contract.write_transaction() as connection:
+ # 某些 SQLite 构建的采样表会覆盖 sqlite_stat1 估计。
+ if connection.execute(
+ "SELECT 1 FROM sqlite_master WHERE name='sqlite_stat4'").fetchone():
+ connection.execute("DELETE FROM sqlite_stat4")
+ connection.executemany("UPDATE sqlite_stat1 SET stat=? WHERE idx=?", [
+ ("2957 1439", "idx_asset_location"),
+ ("2957 1439 1", "sqlite_autoindex_asset_1"),
+ ("0 0", "idx_asset_region"),
+ ("0 0 0", "idx_asset_disposal_id"),
+ ("2957 1", "idx_asset_name"),
+ ("2957 1", "idx_asset_size"),
+ ("2957 1439 1439", "idx_asset_hash"),
+ ("12861 131 131", "idx_asset_entity_entity_role"),
+ ("12861 10 1 1 1", "sqlite_autoindex_asset_entity_1"),
+ ("1847 360 1", "sqlite_autoindex_entity_1"),
+ ])
+
+ calls = 0
+
+ def counted_extra(name, path, **kwargs):
+ nonlocal calls
+ calls += 1
+ return catalog_rules.is_local_extra(name, path, **kwargs)
+
+ with mock.patch("peach.repository.is_local_extra", counted_extra):
+ actual = rm_web.q_facets(self.contract, filters=filters)
+ # 工作量断言不依赖机器执行速度。
+ self.assertLess(calls, asset_count * 40, f"asset predicate calls: {calls}")
+ for key in expected:
+ with self.subTest(field=key):
+ if isinstance(expected[key], list):
+ self.assertEqual(sorted(actual[key], key=repr), sorted(expected[key], key=repr))
+ else:
+ self.assertEqual(actual[key], expected[key])
+ self.assertEqual(next(row["n"] for row in actual["tagperformers"]
+ if row["k"] == "Demo performer 0"), 20)
+
+ def test_facets_ignore_unused_tag_match_and_keep_revision_invalidation(self):
+ args = {"loc": "local,115", "thumb": "0", "tag_match": "all"}
+ with mock.patch("peach.web_state.time.monotonic", return_value=100):
+ first = rm_web.dispatch_api_get(self.contract, "/api/facets", args)
+ with mock.patch("peach.web_state.time.monotonic", return_value=500):
+ for mode in ("all", "any", ""):
+ with self.subTest(tag_match=mode):
+ self.assertIs(rm_web.dispatch_api_get(
+ self.contract, "/api/facets", {**args, "tag_match": mode}), first)
+ # 独立连接提交写入,验证修订失效而不是显式 cache_bust。
+ with closing(sqlite3.connect(self.db_path)) as connection, connection:
+ connection.execute(
+ "INSERT INTO asset_entity(asset_id,entity_id,role,source,confidence) "
+ "VALUES(2,11,'performer','test',1)")
+ refreshed = rm_web.dispatch_api_get(self.contract, "/api/facets", args)
+ self.assertIsNot(refreshed, first)
+ self.assertEqual(refreshed["tagperformers"], [{"k": "Canonical Alice", "n": 2}])
+
+ def test_facets_preserve_tag_match_modes_when_tags_are_selected(self):
+ with self.contract.write_transaction() as connection:
+ connection.executemany(
+ "INSERT INTO entity(id,kind,canonical_name,normalized_name,created_at,updated_at) "
+ "VALUES(?,'tag',?,?,'2026-01-01','2026-01-01')",
+ [(201, "Demo A", "demo a"), (202, "Demo B", "demo b")])
+ connection.executemany(
+ "INSERT INTO asset_entity(asset_id,entity_id,role,source,confidence) "
+ "VALUES(?,?,'tag','test',1)", [(1, 201), (1, 202), (2, 201)])
+ args = {"tag": "Demo A,Demo B", "tag_match": "all"}
+ both = rm_web.dispatch_api_get(self.contract, "/api/facets", args)
+ either = rm_web.dispatch_api_get(self.contract, "/api/facets", {**args, "tag_match": "any"})
+ self.assertEqual(both["stats"]["total"], 1)
+ self.assertEqual(either["stats"]["total"], 2)
+ self.assertIs(rm_web.dispatch_api_get(self.contract, "/api/facets", args), both)
+
def test_home_aggregates_recompute_only_when_the_tables_they_read_change(self):
"""顶栏候选池、统计、名册、筛选项各按自己读的表缓存:换种子、搜一次不重算,作品变了就重算。"""
# 存储卷一节读真实挂载的根目录,与这里要验的缓存无关。