Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

feat(scripts): 补写 rule34video 存量条目的角色分类

longmeidaoc3d0b72942cc
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 02d99384d..245838c00 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,7 @@ Removed / Fixed / Security。
### 修复
+- **关注**:rule34video 作品里站方单列在 Characters 一组的角色(如 `harry potter (harry potter)`)记成角色,归到题材一排,不再和 `sex`、`doggystyle position` 混在内容标签里;站方没单列成角色的标签照旧算内容标签。已入库的条目用 `scripts/backfill_rule34video_characters.py` 重读详情页补上。
- **关注**:批量删除作品、清空回收站后,关注新作和想要清单里已经取到的封面不再被当成无用缓存一起删掉;没有番号的作品封面同样保留。升级后,被删掉的封面由补封面任务陆续重新取回。
- **界面**:作品详情缺封面时改为空状态卡,可以点「立即重试」当场重新获取官方封面,取不到时写明原因;浅色主题下缺图的缩略图格不再白底白字。
- **馆藏**:`MCDV-24`(无码)与 `MCDV-024`(有码)不再被当成同一部,资料和封面各取各的;升级后,扫描时按旧规则补零写成 `MCDV-024` 的无码片自动改回 `MCDV-24` 并重新取资料。
diff --git a/scripts/backfill_rule34video_characters.py b/scripts/backfill_rule34video_characters.py
new file mode 100644
index 000000000..8d80d6784
--- /dev/null
+++ b/scripts/backfill_rule34video_characters.py
@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+r"""把已入库的 rule34video 追更条目里站方单列的角色改记成 `character`。
+
+详情页的角色和普通标签都是 `a.tag_item`,只有外层 `.vp-group--characters` 分组不同。
+旧解析把它们全记成 general,角色名因此混在关注卡片的内容标签里。有 `tag_types` 的行
+不再进详情页探测(`follow_store._ENRICHED_PREDICATES["tag_types"]`),常规检查改不回来,
+所以要这一趟。判据不重写,直接复用 `Rule34VideoConnector._detail`。
+
+只改一件事:站方列进 Characters 组、库里已经记过类型的名字改成 `character`。别的类型、
+标签列表、封面与日期一格不动;库里还没有 `tag_types` 的行留给常规检查补。
+
+默认 dry-run,只写复核 CSV。`--apply` 必须同时给 `--backup`。`--resume` 跳过同一份
+CSV 里已经写入或确认无需改动的条目。
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sqlite3
+import sys
+import time
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+SRC_DIR = PROJECT_ROOT / "src"
+if str(SRC_DIR) not in sys.path:
+ sys.path.insert(0, str(SRC_DIR))
+
+import httpx
+
+from peach.config import GENERATED_DIR
+from peach.follow_sources import (
+ FollowSourceError, FollowSourceRateLimited, Rule34VideoConnector, build_connector,
+)
+from peach.review_csv import read_rows, write_rows
+from peach.scripting import (
+ BACKUP_REQUIRED, add_ledger_write_args, open_for_write, open_readonly,
+)
+
+FIELDS = ("item_id", "external_id", "result", "characters", "written", "note")
+#: 详情页的最小请求间隔。站方没有公布限流口径,按常规检查的节奏留余量。
+DEFAULT_DELAY = 1.0
+#: 被 429 挡回时静默多久;连续这么多次就停下,留给下一趟 `--resume`。
+COOLDOWN_SECONDS = 120.0
+RATE_LIMIT_STOP = 3
+FLUSH_EVERY = 50
+#: 续跑跳过的结果:写过的,以及问过确认没有要改的。
+SETTLED = frozenset({"无需改动"})
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ description="把 rule34video 追更条目里站方单列的角色改记成 character")
+ add_ledger_write_args(parser)
+ parser.add_argument("--out", type=Path,
+ default=GENERATED_DIR / "rule34video-character-backfill.csv")
+ parser.add_argument("--limit", type=int, default=0, help="只处理前 N 条,0 为全部")
+ parser.add_argument("--delay", type=float, default=DEFAULT_DELAY)
+ parser.add_argument("--resume", action="store_true",
+ help="跳过 --out 里已经写入或无需改动的条目,接着上一趟跑")
+ return parser
+
+
+def settled_ids(path: Path) -> set[int]:
+ return {int(row["item_id"]) for row in read_rows(path, missing_ok=True)
+ if (row.get("written") == "是" or row.get("result") in SETTLED)
+ and str(row.get("item_id") or "").isdigit()}
+
+
+def pending_rows(connection: sqlite3.Connection, limit: int,
+ done: set[int] = frozenset()) -> list[dict]:
+ """待问条目:已经记过 `tag_types` 的 rule34video 条目,扣掉上一趟问定的。
+
+ 不按名字带不带括号筛:那是词形猜测,`melony` 这类不带括号的角色会漏。
+ """
+ rows = []
+ for row in connection.execute(
+ "SELECT i.id, i.external_id, i.url FROM follow_item i"
+ " JOIN follow_source s ON s.id=i.source_id"
+ " WHERE s.provider='rule34video'"
+ " AND json_extract(i.metadata_json,'$.tag_types') IS NOT NULL ORDER BY i.id"):
+ if row["id"] in done:
+ continue
+ rows.append({"id": row["id"], "external_id": str(row["external_id"] or ""),
+ "url": str(row["url"] or "")})
+ if limit and len(rows) >= limit:
+ break
+ return rows
+
+
+def characters_of(detail: dict) -> list[str]:
+ types = (detail.get("extra") or {}).get("tag_types") or {}
+ return [name for name, kind in types.items() if kind == "character"]
+
+
+def retype(metadata: dict, characters: list[str]) -> dict | None:
+ """库里这份 metadata 改完的样子;没有要改的就是 None。只改已经记过类型的名字。"""
+ types = metadata.get("tag_types")
+ if not isinstance(types, dict):
+ return None
+ changed = {name: "character" for name in characters
+ if name in types and types[name] != "character"}
+ if not changed:
+ return None
+ return {**metadata, "tag_types": {**types, **changed}}
+
+
+def _apply(writer: sqlite3.Connection, item_id: int, characters: list[str]) -> bool:
+ """在写事务里重读再改:服务的关注检查同时在写这一列,不拿 dry-run 时读到的旧值覆盖。"""
+ writer.execute("BEGIN IMMEDIATE")
+ row = writer.execute("SELECT metadata_json FROM follow_item WHERE id=?",
+ (item_id,)).fetchone()
+ try:
+ metadata = json.loads(row["metadata_json"] or "{}") if row else {}
+ except ValueError:
+ metadata = {}
+ updated = retype(metadata, characters) if isinstance(metadata, dict) else None
+ if updated is not None:
+ writer.execute("UPDATE follow_item SET metadata_json=? WHERE id=?",
+ (json.dumps(updated, ensure_ascii=False), item_id))
+ writer.commit()
+ return updated is not None
+
+
+def inspect_row(connector, reader: sqlite3.Connection, row: dict) -> tuple[dict, list[str]]:
+ """问一条详情页。返回 CSV 行和要改记成角色的名字;取不到就是「未取得」,不写。"""
+ entry = {"item_id": row["id"], "external_id": row["external_id"], "result": "",
+ "characters": "", "written": "否", "note": ""}
+ if not Rule34VideoConnector._VIDEO_RE.match(row["url"]):
+ entry.update(result="跳过", note="url 不是详情页")
+ return entry, []
+ try:
+ response = connector._get(row["url"], headers={"Accept": "text/html"})
+ connector._raise_if_rate_limited(response)
+ if response.status != 200:
+ entry.update(result="未取得", note=f"HTTP {response.status}")
+ return entry, []
+ detail = Rule34VideoConnector._detail(response.body)
+ except FollowSourceRateLimited:
+ raise
+ except (FollowSourceError, OSError, httpx.HTTPError, ValueError) as error:
+ entry.update(result="未取得", note=f"{type(error).__name__}: {error}"[:120])
+ return entry, []
+ if not detail:
+ entry.update(result="未取得", note="详情页没有解析出细节")
+ return entry, []
+ characters = characters_of(detail)
+ current = reader.execute("SELECT metadata_json FROM follow_item WHERE id=?",
+ (row["id"],)).fetchone()
+ try:
+ metadata = json.loads(current["metadata_json"] or "{}") if current else {}
+ except ValueError:
+ metadata = {}
+ updated = retype(metadata, characters) if isinstance(metadata, dict) else None
+ if updated is None:
+ entry["result"] = "无需改动"
+ return entry, []
+ names = [name for name in characters
+ if updated["tag_types"].get(name) != metadata["tag_types"].get(name)]
+ entry.update(result="取得", characters=" | ".join(names))
+ return entry, names
+
+
+def run(args: argparse.Namespace) -> int:
+ if args.apply and not args.backup:
+ print(BACKUP_REQUIRED, file=sys.stderr)
+ return 2
+ done = settled_ids(args.out) if args.resume else set()
+ previous = [row for row in read_rows(args.out, missing_ok=True)
+ if int(row.get("item_id") or 0) in done] if args.resume else []
+ reader = open_readonly(args.db)
+ rows = pending_rows(reader, args.limit, done)
+ print(f"rule34video 已记类型的条目:上一趟问定 {len(done)} 条,本趟待问 {len(rows)} 条",
+ flush=True)
+ if not rows:
+ reader.close()
+ return 0
+ writer: sqlite3.Connection | None = None
+ before = 0
+ if args.apply:
+ writer = open_for_write(args)
+ writer.isolation_level = None
+ writer.execute("PRAGMA busy_timeout=60000")
+ print(f"备份:{args.backup}", flush=True)
+ before = writer.execute("SELECT COUNT(*) FROM follow_item").fetchone()[0]
+
+ connector = build_connector("rule34video")
+ log: list[dict] = []
+ counts = {"取得": 0, "无需改动": 0, "未取得": 0, "跳过": 0, "names": 0}
+ limited = 0
+ next_request = 0.0
+
+ def flush() -> None:
+ args.out.parent.mkdir(parents=True, exist_ok=True)
+ write_rows(args.out, FIELDS, previous + log, atomic=True)
+
+ stopped = False
+ for index, row in enumerate(rows, 1):
+ time.sleep(max(0.0, next_request - time.monotonic()))
+ next_request = time.monotonic() + args.delay
+ try:
+ entry, names = inspect_row(connector, reader, row)
+ limited = 0
+ except FollowSourceRateLimited as error:
+ limited += 1
+ if limited >= RATE_LIMIT_STOP:
+ print(f"连续 {limited} 次被限流,停在第 {index} 条;用 --resume 接着跑",
+ flush=True)
+ stopped = True
+ break
+ wait = max(COOLDOWN_SECONDS, float(error.retry_after or 0))
+ print(f"被限流,静默 {wait:.0f} 秒", flush=True)
+ time.sleep(wait)
+ continue
+ counts[entry["result"]] += 1
+ if names:
+ counts["names"] += len(names)
+ if writer is not None and _apply(writer, row["id"], names):
+ entry["written"] = "是"
+ log.append(entry)
+ if index % FLUSH_EVERY == 0:
+ flush()
+ print(f"{index}/{len(rows)}:改记角色 {counts['取得']} 条({counts['names']} 个名字),"
+ f"无需改动 {counts['无需改动']},未取得 {counts['未取得']}", flush=True)
+ flush()
+ reader.close()
+ summary = (f"改记角色 {counts['取得']} 条({counts['names']} 个名字),无需改动 "
+ f"{counts['无需改动']},未取得 {counts['未取得']},跳过 {counts['跳过']} → {args.out}")
+ if writer is None:
+ print(f"dry-run:{summary};加 --apply --backup 才真正写入", flush=True)
+ return 1 if stopped else 0
+ after = writer.execute("SELECT COUNT(*) FROM follow_item").fetchone()[0]
+ writer.close()
+ # 条数必须不变:这一趟只改已有条目的一列。
+ print(f"已写入:{summary};follow_item {before} → {after}", flush=True)
+ return 0 if before == after and not stopped else 1
+
+
+def main(argv: list[str] | None = None) -> int:
+ return run(build_parser().parse_args(argv))
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index 0d9d9036d..e52e5ad78 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -1713,6 +1713,68 @@ class OperationalScriptTests(unittest.TestCase):
" ORDER BY id").fetchall(), untouched)
self.assertEqual(backfill.written_ids(out), {2})
+ def test_rule34video_character_backfill_only_retypes_the_site_character_group(self):
+ """存量行只把站方 Characters 组里的名字改记 character,别的类型与列一格不动。"""
+ backfill = load_script("backfill_rule34video_characters")
+ self.assertIs(backfill.open_for_write, scripting.open_for_write)
+ root = Path(tempfile.mkdtemp()).resolve()
+ self.addCleanup(shutil.rmtree, root, True)
+ database = root / "ledger.db"
+ seeding = sqlite3.connect(database)
+ seeding.executescript(
+ "CREATE TABLE follow_source(id INTEGER PRIMARY KEY, provider TEXT);"
+ "CREATE TABLE follow_item(id INTEGER PRIMARY KEY, source_id INTEGER,"
+ " external_id TEXT, url TEXT, metadata_json TEXT);"
+ "INSERT INTO follow_source VALUES(1,'rule34video');")
+ typed = {"tags": ["harry potter (harry potter)", "d.mon (overwatch)", "sex"],
+ "tag_types": {"harry potter (harry potter)": "general",
+ "d.mon (overwatch)": "general", "sex": "general",
+ "3D": "metadata"}}
+ seeding.executemany("INSERT INTO follow_item VALUES(?,1,?,?,?)", [
+ (1, "4708183", "https://rule34video.com/video/4708183/x/",
+ json.dumps(typed)),
+ (2, "4708184", "https://rule34video.com/video/4708184/y/",
+ json.dumps({"tags": []}))])
+ seeding.commit()
+ seeding.close()
+ detail = (
+ b'<html><body><div class="vp-group vp-group--characters">'
+ b'<a class="tag_item" href="https://rule34video.com/tags/h/">'
+ b'harry potter (harry potter)</a></div><div class="vp-group">'
+ b'<a class="tag_item" href="https://rule34video.com/tags/d/">d.mon (overwatch)</a>'
+ b'<a class="tag_item" href="https://rule34video.com/tags/s/">sex</a>'
+ b'</div></body></html>')
+ asked = []
+
+ class Connector:
+ def _get(self, url, headers=None):
+ asked.append(url)
+ return HttpResponse(200, {}, detail)
+
+ def _raise_if_rate_limited(self, response):
+ return None
+
+ out = root / "characters.csv"
+ common = ["--db", str(database), "--out", str(out), "--delay", "0"]
+ with mock.patch.object(backfill, "build_connector", return_value=Connector()), \
+ redirect_stdout(io.StringIO()):
+ self.assertEqual(backfill.run(backfill.build_parser().parse_args(common)), 0)
+ with closing(sqlite3.connect(database)) as peek:
+ self.assertEqual(json.loads(peek.execute(
+ "SELECT metadata_json FROM follow_item WHERE id=1").fetchone()[0]), typed)
+ self.assertEqual(backfill.run(backfill.build_parser().parse_args(
+ common + ["--apply", "--backup", str(root / "b.db")])), 0)
+ # 没有 `tag_types` 的行留给常规检查,不问详情页。
+ self.assertEqual(asked, ["https://rule34video.com/video/4708183/x/"] * 2)
+ after = sqlite3.connect(database)
+ self.addCleanup(after.close)
+ rows = dict(after.execute("SELECT id, metadata_json FROM follow_item"))
+ self.assertEqual(json.loads(rows[1]), {
+ **typed, "tag_types": {**typed["tag_types"],
+ "harry potter (harry potter)": "character"}})
+ self.assertEqual(json.loads(rows[2]), {"tags": []})
+ self.assertEqual(backfill.settled_ids(out), {1})
+
def test_follow_image_dims_backfill_reads_archives_first_and_probes_headers_for_the_rest(self):
"""图片墙的比例占位要每张图都有宽高;存量行里只有 fanbox 记过。
@@ -4651,6 +4713,7 @@ class ScriptingConventionTests(unittest.TestCase):
#: 会真写 ledger、已经收口到本模块的脚本。
LEDGER_WRITERS = (
"backfill_rule34_details",
+ "backfill_rule34video_characters",
"clean_names",
"install_entity_links",
"localize_performer_names",