Peach: Local-first video metadata scraper and media manager.
fix(follow): rule34.xxx 标签分类改走 dapi 的 tag_info
longmeidao8e7bebd99d25
diff --git a/scripts/backfill_rule34_details.py b/scripts/backfill_rule34_details.pyindex 5e13bda91..568d71ded 100644--- a/scripts/backfill_rule34_details.py+++ b/scripts/backfill_rule34_details.py@@ -1,13 +1,15 @@#!/usr/bin/env python3# -*- coding: utf-8 -*--r"""给已入库的 rule34.xxx 追更条目补上只有帖子页和原文件才有的细节。+r"""给已入库的 rule34.xxx 追更条目补上列表请求之外的细节。-posts DAPI 只回扁平标签和最后修改时间,三样东西要另外问:+posts DAPI 的默认 JSON 只回扁平标签和最后修改时间,三样东西要另外问:-- 标签类型(general/artist/character/copyright/metadata)在帖子页 `#tag-sidebar`。-- 上传时间在帖子页 `#stats` 的 Posted。DAPI 的 `change` 是最后一次修改:17361475+- 标签类型(general/artist/character/copyright/metadata):JSON 带+ `fields=tag_info`。帖子页 `#tag-sidebar` 自 2026-10-20 起被 Cloudflare 验证挡住,+ 不再是来源。+- 上传时间:XML 的 `created_at`。DAPI 的 `change` 是最后一次修改:17361475五月上传、九月被改过标签,入库时记成了九月,卡片就排到了新内容里。-- 视频时长只在原文件头的 `mvhd` 里,接口和帖子页都不给。+- 视频时长只在原文件头的 `mvhd` 里,接口都不给。常规「检查更新」只看第一页,而且已经补齐过类型的条目不再进第二阶段,存量行靠它永远改不过来,所以要这一趟。判据不重写,直接复用连接器的 `_detail` 与@@ -16,8 +18,11 @@ posts DAPI 只回扁平标签和最后修改时间,三样东西要另外问:默认 dry-run,只写复核 CSV。`--apply` 必须同时给 `--backup`,与本仓库其它真实写入脚本一致。`--resume` 跳过同一份 CSV 里已经写入的条目,中断后用原命令加它接着跑。-`--duration-only` 只补 mp4 缺的时长:只挑 `duration` 为空的 mp4 条目,不问帖子页,-只读原文件头,也只写 `duration`。时长在视频 CDN 上,不受帖子页每秒一次的限流牵连,+`--missing-types` 只挑账本里还没有标签类型的条目。帖子页被挡住的那段时间入库的行+正是这一批,它们的上传时间也还是 `change` 占位,一趟一起改正。++`--duration-only` 只补 mp4 缺的时长:只挑 `duration` 为空的 mp4 条目,不问 dapi,+只读原文件头,也只写 `duration`。时长在视频 CDN 上,不受 dapi 每秒一次的限流牵连,授权只覆盖时长时用它,别的列一格不动。"""from __future__ import annotations@@ -44,12 +49,11 @@ from peach.scripting import (FIELDS = ("item_id", "external_id", "result", "published_before", "published_after","duration", "tag_types_added", "written", "note")-#: 帖子页的最小请求间隔。rule34.xxx 自己公布的口径是每 60 秒 60 次,也就是-#: 每次至少 1 秒;留一点余量。首轮按 0.35 秒跑过一次:400 条里 372 条「帖子页-#: 没有解析出类型」——不是这些帖子没有类型,是被限流挡了,`_detail`-#: 把非 200 吞成了空字典。速度换来的全是白跑的请求。视频文件头在另一个主机上,-#: 在这段间隔里发,不另占时间。-DEFAULT_DELAY = 1.1+#: 每条之间的最小间隔。rule34.xxx 自己公布的口径是每 60 秒 60 次,`_detail` 每条+#: 问两次 dapi(JSON 的分类、XML 的上传时间),所以每条至少 2 秒;留一点余量。+#: 首轮按 0.35 秒跑过一次:400 条里 372 条被限流挡了,被读成「没有类型」——速度+#: 换来的全是白跑的请求。视频文件头在另一个主机上,在这段间隔里发,不另占时间。+DEFAULT_DELAY = 2.2#: 连续这么多条取不到就认定是被限流,而不是这一段恰好都取不到。MISS_STREAK_LIMIT = 8#: 限流后的静默时长,按站方公布的窗口取满一格。@@ -69,7 +73,9 @@ def build_parser() -> argparse.ArgumentParser:parser.add_argument("--resume", action="store_true",help="跳过 --out 里已经写入的条目,接着上一趟跑")parser.add_argument("--duration-only", action="store_true",- help="只补 mp4 缺的时长:不问帖子页,只读原文件头,只写 duration")+ help="只补 mp4 缺的时长:不问 dapi,只读原文件头,只写 duration")+ parser.add_argument("--missing-types", action="store_true",+ help="只挑还没有标签类型的条目")return parser@@ -80,12 +86,14 @@ def written_ids(path: Path) -> set[int]:def pending_rows(connection: sqlite3.Connection, limit: int,- done: set[int] = frozenset(), *, duration_only: bool = False) -> list[dict]:+ done: set[int] = frozenset(), *, duration_only: bool = False,+ missing_types: bool = False) -> list[dict]:"""待补条目:rule34xxx 来源的全部条目,扣掉上一趟已经写入的。- 上传时间错没错,不问帖子页就看不出来——被改过的帖子 `change` 晚于上传,没改过的+ 上传时间错没错,不问 dapi 就看不出来——被改过的帖子 `change` 晚于上传,没改过的两者逐秒相同,账本里长得一样。所以每条都要问一遍,续跑靠 CSV 而不是账本里的值。- 只补时长时缺不缺看账本就知道,只挑 `duration` 为空的 mp4。+ 只补时长时缺不缺看账本就知道,只挑 `duration` 为空的 mp4;只补类型时同理,只挑+ metadata 里没有 `tag_types` 的。"""rows = []query = (@@ -102,6 +110,9 @@ def pending_rows(connection: sqlite3.Connection, limit: int,metadata = json.loads(row["metadata_json"] or "{}")except ValueError:metadata = {}+ # 空表与缺键同样算缺:`plan_update` 也只在没有非空类型时才补。+ if missing_types and isinstance(metadata, dict) and metadata.get("tag_types"):+ continuerows.append({"id": row["id"], "external_id": str(row["external_id"] or ""),"media_url": row["media_url"], "published_at": row["published_at"],"duration": row["duration"],@@ -114,7 +125,7 @@ def pending_rows(connection: sqlite3.Connection, limit: int,def plan_update(row: dict, detail: dict, duration: float | None) -> dict | None:"""这一条要改哪几列。没有可写的就是 None。- 只补不抹:帖子页没给上传时间就不动 `published_at`;已有时长不覆盖;已有非空+ 只补不抹:dapi 没给上传时间就不动 `published_at`;已有时长不覆盖;已有非空类型不覆盖。取不到的留待下一趟,不写空值冒充已补。"""update: dict = {}@@ -131,10 +142,10 @@ def plan_update(row: dict, detail: dict, duration: float | None) -> dict | None:def inspect_row(connector, row: dict) -> tuple[dict, dict | None]:- """问一条:帖子页,缺时长的视频再问一次文件头。返回 CSV 行和要写的列。+ """问一条:dapi 的分类与上传时间,缺时长的视频再问一次文件头。返回 CSV 行和要写的列。- 帖子页没解析出来时 `result` 是「未取得」:限流时整页都是空的,不能当成「这条- 没有」写进去,留给下一趟再问。+ 分类没取到时 `result` 是「未取得」:被限流挡回来与帖子已删都是这样,不能当成+ 「这条没有」写进去,留给下一趟再问。"""post_id = row["external_id"]entry = {"item_id": row["id"], "external_id": post_id, "result": "",@@ -149,7 +160,7 @@ def inspect_row(connector, row: dict) -> tuple[dict, dict | None]:except (FollowSourceError, OSError) as error:detail, entry["note"] = {}, f"{type(error).__name__}: {error}"[:120]if not detail.get("tag_types"):- entry.update(result="未取得", note=entry["note"] or "帖子页没有解析出细节")+ entry.update(result="未取得", note=entry["note"] or "dapi 没有给出标签分类")return entry, Noneduration = (connector._video_seconds(row["media_url"])if row["duration"] is None else None)@@ -159,7 +170,7 @@ def inspect_row(connector, row: dict) -> tuple[dict, dict | None]:duration=update.get("duration", entry["duration"]),tag_types_added="是" if "metadata_json" in update else "")if not detail.get("published_at"):- entry["note"] = "帖子页没有 Posted"+ entry["note"] = "dapi 没有给出 created_at"elif _is_mp4(row["media_url"]) and row["duration"] is None and not duration:entry["note"] = "时长未取得"return entry, update or None@@ -208,7 +219,8 @@ def run(args: argparse.Namespace) -> int:previous = [row for row in read_rows(args.out, missing_ok=True)if row.get("written") == "是"] if args.resume else []with open_readonly(args.db) as reader:- rows = pending_rows(reader, args.limit, done, duration_only=args.duration_only)+ rows = pending_rows(reader, args.limit, done, duration_only=args.duration_only,+ missing_types=args.missing_types)total = reader.execute("SELECT COUNT(*) FROM follow_item i JOIN follow_source s ON s.id=i.source_id"" WHERE s.provider='rule34xxx'").fetchone()[0]@@ -245,7 +257,7 @@ def run(args: argparse.Namespace) -> int:write_rows(args.out, FIELDS, previous + log, atomic=True)for index, row in enumerate(rows, 1):- # 间隔按帖子页请求的起点算:文件头在另一个主机上,问它的时间落在这段间隔里。+ # 间隔按 dapi 请求的起点算:文件头在另一个主机上,问它的时间落在这段间隔里。time.sleep(max(0.0, next_detail - time.monotonic()))next_detail = time.monotonic() + args.delayentry, update = (inspect_duration if args.duration_only else inspect_row)(connector, row)diff --git a/src/peach/follow_sources.py b/src/peach/follow_sources.pyindex 198d93644..02ba1239a 100644--- a/src/peach/follow_sources.py+++ b/src/peach/follow_sources.py@@ -1558,11 +1558,12 @@ class Rule34VideoConnector(_BaseConnector):class Rule34XxxConnector(_BaseConnector):"""rule34.xxx 的官方 dapi。- 网页版已挂 Cloudflare Turnstile,Peach 不绕验证码,因此这里只走官方 API,- 并且必须带账号自己的 `user_id` + `api_key`(在- `https://rule34.xxx/index.php?page=account&s=options` 生成)。凭据只进查询参数- 以外的位置不可行——dapi 只接受查询参数——所以请求 URL 不进日志也不进 ledger,- 只有 `request_url` 的脱敏形式会被记录。+ 网页版挂了 Cloudflare 验证:2026-11-03 实测帖子页回 HTTP 403、`cf-mitigated:+ challenge`、正文是「Rule34.xxx CAPTCHA」页,10-20 起读 `#tag-sidebar` 的路径整批+ 落空。Peach 不绕验证码,因此这里只走官方 API,并且必须带账号自己的 `user_id` ++ `api_key`(在 `https://rule34.xxx/index.php?page=account&s=options` 生成)。凭据+ 只进查询参数以外的位置不可行——dapi 只接受查询参数——所以请求 URL 不进日志也不进+ ledger,只有 `request_url` 的脱敏形式会被记录。`ref` 是标签,例如 `lazyprocrastinator`。"""@@ -1570,8 +1571,9 @@ class Rule34XxxConnector(_BaseConnector):provider = "rule34xxx"#: 一页 24 条时整页都能补上;额度存在是为了页面变长时请求数不跟着长。DEFAULT_ENRICH_BUDGET = 24- #: 第二阶段补两样:帖子页的标签分类(metadata 的 `tag_types`)和原文件头的时长。- #: mp4 缺时长的行下次检查还会进第二阶段,判据在 `follow_store._ENRICHED_PREDICATES`。+ #: 标签分类(metadata 的 `tag_types`)随列表请求的 `fields=tag_info` 一起到;第二+ #: 阶段补上传时间和原文件头的时长。mp4 缺时长的行下次检查还会进第二阶段,判据在+ #: `follow_store._ENRICHED_PREDICATES`。ENRICHED_MARK = "tag_types_duration"_TAG_RE = re.compile(r"^[^\s&?#]{1,100}$")#: 历史行存的是 250px 的 preview。官方 dapi 的 `sample_url` 与它用同一@@ -1619,29 +1621,37 @@ class Rule34XxxConnector(_BaseConnector):return str(item.media_url or item.thumb_url or "") or Nonereturn str(item.thumb_url or "") or None+ def _dapi_url(self, **parameters: str) -> str:+ """带凭据的 dapi 地址。查询串里有 api_key,只能拿去发请求,不能落进日志或证据。"""+ if self.credential is None:+ raise CredentialError(+ "rule34xxx 需要 user_id 与 api_key;请把它们写进 "+ "peach-data/secrets/follow/rule34xxx.json")+ user_id, api_key = self.credential.require("user_id", "api_key")+ query = urllib.parse.urlencode({"page": "dapi", **parameters,+ "user_id": user_id, "api_key": api_key})+ return f"https://api.rule34.xxx/index.php?{query}"+def _dapi(self, tags: str, *, limit: int, page: int = 0,etag: str | None = None, last_modified: str | None = None,+ tag_info: bool = False,) -> tuple[dict[str, object], HttpResponse | None]:- """问一次 dapi。凭据、地址形状和脱敏都只有这一处。+ """问一次 dapi 帖子列表。地址形状和脱敏都只有这一处。`tags` 原样交给站点:dapi 的标签之间用空格分隔,`sort:score` 这类元标签也走同一个参数,所以这里不解析它,只负责把它编码进查询串。++ `tag_info` 让每条帖子随带 `tag_info`:每个标签的分类和帖子数,一页一次请求+ 就拿全,不必逐条去问帖子页或 tag 接口。2026-11-03 实测 24 条约 99 KB。"""- if self.credential is None:- raise CredentialError(- "rule34xxx 需要 user_id 与 api_key;请把它们写进 "- "peach-data/secrets/follow/rule34xxx.json")- user_id, api_key = self.credential.require("user_id", "api_key")# `pid` 是 0 起的页号,页大小就是 `limit`。- parameters = {- "page": "dapi", "s": "post", "q": "index", "json": "1",- "limit": str(min(limit, 1000)), "tags": tags,- "user_id": user_id, "api_key": api_key,- }+ parameters = {"s": "post", "q": "index", "json": "1",+ "limit": str(min(limit, 1000)), "tags": tags}if page:parameters["pid"] = str(page)- query = urllib.parse.urlencode(parameters)- url = f"https://api.rule34.xxx/index.php?{query}"+ if tag_info:+ parameters["fields"] = "tag_info"+ url = self._dapi_url(**parameters)safe_url = (f"https://api.rule34.xxx/index.php?page=dapi&s=post&q=index"f"&tags={tags}" + (f"&pid={page}" if page else ""))# request_url 传脱敏版:真实 url 的查询串里带 api_key,绝不能落进证据。@@ -1650,6 +1660,108 @@ class Rule34XxxConnector(_BaseConnector):headers={"Accept": "application/json"},request_url=safe_url)+ #: dapi 被限流时的退避节奏,与外网退避规则同一套。站方口径是每 60 秒 60 次。+ _RATE_LIMIT_RETRY_DELAYS = (1.5, 4.0, 9.0)++ def _dapi_patiently(self, url: str, accept: str) -> HttpResponse | None:+ """补细节用的 dapi 请求:被限流就退避重试,仍取不到返回 None。++ 列表那次请求已经成功,这里取的是锦上添花的细节,所以失败不抛:一抛整次检查+ 就丢了已经到手的列表。但也不能把被挡回来读成「没有」——返回 None 让调用方+ 保持 partial,下一轮再问。限流有两种长相:HTTP 429,或者 HTTP 200 加一句+ `You currently have a limit of 60 requests every 60 second(s)`。+ """+ for delay in (0.0, *self._RATE_LIMIT_RETRY_DELAYS):+ if delay:+ self.sleeper(delay)+ try:+ response = self._get(url, headers={"Accept": accept})+ except FollowSourceError:+ return None+ if response.status == 429 or (response.status == 200+ and self._upstream_reason(response)):+ continue+ return response if response.status == 200 else None+ return None++ #: XML 帖子的 `created_at`:`Fri May 01 18:32:21 +0200 2026`。+ _CREATED_AT_RE = re.compile(+ r"^\w{3} (\w{3}) (\d{1,2}) (\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2}) (\d{4})$")+ _MONTHS = {name: number for number, name in enumerate(+ ("jan", "feb", "mar", "apr", "may", "jun",+ "jul", "aug", "sep", "oct", "nov", "dec"), 1)}+ _XML_POST_RE = re.compile(r"<post\b[^>]*>", re.IGNORECASE)+ _XML_ATTR_RE = re.compile(r'\b(id|created_at)="([^"]*)"')++ @classmethod+ def _created_at(cls, text: str) -> str | None:+ """XML 的上传时间转成 UTC。不靠 `%a %b`:那两个格式符跟着系统区域设置走。"""+ matched = cls._CREATED_AT_RE.match(html.unescape(str(text or "")).strip())+ month = cls._MONTHS.get(matched.group(1).casefold()) if matched else None+ if not matched or not month:+ return None+ day, hour, minute, second = (int(matched.group(index)) for index in (2, 3, 4, 5))+ offset = timedelta(hours=int(matched.group(7)), minutes=int(matched.group(8)))+ try:+ local = datetime(int(matched.group(9)), month, day, hour, minute, second,+ tzinfo=timezone(-offset if matched.group(6) == "-" else offset))+ except ValueError:+ return None+ return _iso_utc(local)++ @classmethod+ def _upload_times_from_xml(cls, body: bytes) -> dict[str, str]:+ result: dict[str, str] = {}+ for tag in cls._XML_POST_RE.findall(body.decode("utf-8", errors="replace")):+ attributes = dict(cls._XML_ATTR_RE.findall(tag))+ uploaded = cls._created_at(attributes.get("created_at", ""))+ if attributes.get("id") and uploaded:+ result[attributes["id"]] = uploaded+ return result++ def _upload_times(self, *, page: int = 0, tags: str = "",+ post_id: str = "") -> dict[str, str]:+ """帖子号到上传时间。只有 XML 形态给 `created_at`。++ JSON 形态没有这一栏,`fields=created_at` 实测也不加;反过来 XML 不认+ `fields=tag_info`。所以分类和上传时间各要一次请求:列表一次 JSON、一次 XML,+ 按帖子号对回去。两次之间有新帖插进来时页会错开一格,对不上的那条不写上传时间,+ 候选保持 partial。++ `created_at` 与帖子页 `#stats` 的 Posted 是同一个时刻:17361475 的 XML 是+ `Fri May 01 18:32:21 +0200 2026`,账本里帖子页取到的是 `2026-05-01T16:32:21Z`。+ """+ parameters = {"s": "post", "q": "index"}+ if post_id:+ parameters["id"] = post_id+ else:+ parameters.update(limit=str(min(self.max_items, 1000)), tags=tags)+ if page:+ parameters["pid"] = str(page)+ response = self._dapi_patiently(self._dapi_url(**parameters), "application/xml")+ return self._upload_times_from_xml(response.body) if response is not None else {}++ #: `tag_info` 里普通标签叫 `tag`;tag 接口与帖子页 `#tag-sidebar` 叫 general。+ #: 记成后者,读取层与前端 `r34-*` 色板都按这套词认分类。+ _TAG_INFO_TYPES = {"tag": "general", "general": "general", "artist": "artist",+ "copyright": "copyright", "character": "character",+ "metadata": "metadata"}++ @classmethod+ def _tag_info_types(cls, info) -> dict[str, str]:+ """`fields=tag_info` 给的分类。名字与 `tags` 一样是 HTML 转义形态,要反转义。"""+ if not isinstance(info, list):+ return {}+ result: dict[str, str] = {}+ for row in info:+ if not isinstance(row, dict):+ continue+ name = html.unescape(str(row.get("tag") or "")).strip()+ kind = cls._TAG_INFO_TYPES.get(str(row.get("type") or "").strip().casefold(), "")+ if name and kind:+ result[name] = kind+ return result+def _dapi_posts(self, response: HttpResponse) -> list[dict]:"""一次 dapi 响应里的帖子。"""body = response.body.decode("utf-8", errors="replace").strip()@@ -1687,32 +1799,34 @@ class Rule34XxxConnector(_BaseConnector):if not self._TAG_RE.match(tag):raise FollowSourceError(f"rule34xxx 的 ref 必须是单个标签,收到:{ref!r}")common, response = self._dapi(tag, limit=self.max_items, page=page,- etag=etag, last_modified=last_modified)+ etag=etag, last_modified=last_modified,+ tag_info=True)if response is None:return SourceFetch(not_modified=True, **common)listed = [self._candidate(post, tag)for post in self._dapi_posts(response)[: self.max_items]]- # 第二阶段:标签分类只有帖子详情页给,列表的 dapi 只给一串扁平标签。+ # 第二阶段要的上传时间整页一次问齐;整页都补过就一次也不问。+ wanted = self.enrich_budget and any(+ str(candidate.external_id) not in self.enrich_skip for candidate in listed)+ self._uploaded = self._upload_times(tags=tag, page=page) if wanted else {}candidates, probed = self.enrich(listed)if page and not candidates:raise FollowHistoryEnd("没有更多历史内容")return SourceFetch(candidates=candidates, probed=probed,raw_body=response.body, **common)+ #: 这一页的帖子号到上传时间,`fetch` 每次重新填;类上的空表只是还没抓过时的默认。+ _uploaded: Mapping[str, str] = {}+def _enrich_one(self, candidate: FollowCandidate) -> FollowCandidate:- detail = self._detail(str(candidate.external_id))- # 文件头在另一个主机上,帖子页被限流挡回来不影响它,所以不论分类取没取到都问。+ uploaded = self._uploaded.get(str(candidate.external_id))+ # 文件头在另一个主机上,dapi 被限流不牵连它,所以不论上传时间取没取到都问。duration = self._video_seconds(candidate.media_url)- tag_types = detail.get("tag_types")- if not tag_types:- # 详情页这次没给出分类。保持 partial:上一轮取到的分类还在库里,- # 覆盖成空会让「这条没有分类」和「这次没问到」在数据里长得一样。- # partial 行落库时时长只补空,所以读到的时长照样带上。+ if not uploaded or not candidate.extra.get("tag_types"):+ # 这次缺了一样就保持 partial:落库时不动 `published_at`,上一轮取到的上传+ # 时间不会被 `change` 冲掉;分类经 `json_patch` 照样并进去,时长只补空。return replace(candidate, duration=duration) if duration else candidate- return replace(candidate, partial=False,- published_at=detail.get("published_at") or candidate.published_at,- duration=duration,- extra={**candidate.extra, "tag_types": tag_types})+ return replace(candidate, partial=False, published_at=uploaded, duration=duration)#: 读时长只取文件开头这么多字节。站方转码出的 mp4 都是 faststart,`mvhd` 在开头#: 几百字节里;实测随机 12 条全部读出,64 KiB 留足 `ftyp` 之后夹杂别的盒的余量。@@ -1773,8 +1887,8 @@ class Rule34XxxConnector(_BaseConnector):rows.append((tag, int(matched.group(1).replace(",", "")) if matched else 0))return tuple(rows)- #: dapi 的 tag 接口用数字表示分类。名字取站方自己在详情页 `#tag-sidebar` 上用的- #: 那套(`_detail` 读的就是它),两条路认出来的类型才是同一个词。+ #: dapi 的 tag 接口用数字表示分类。名字取站方帖子页 `#tag-sidebar` 上用的那套,+ #: 与 `_TAG_INFO_TYPES` 归一后的词相同,几条路认出来的类型才是同一个词。_TAG_TYPE_NAMES = {0: "general", 1: "artist", 3: "copyright",4: "character", 5: "metadata"}#: dapi 的 tag 接口只回 XML,`json=1` 实测被忽略。@@ -1795,13 +1909,8 @@ class Rule34XxxConnector(_BaseConnector):tag = (name or "").strip()if not tag or self.credential is None:return ""- user_id, api_key = self.credential.require("user_id", "api_key")- query = urllib.parse.urlencode({- "page": "dapi", "s": "tag", "q": "index", "name": tag,- "user_id": user_id, "api_key": api_key,- })# 查询串里带 api_key,和 `fetch` 一样绝不落进日志或证据。- response = self._get(f"https://api.rule34.xxx/index.php?{query}",+ response = self._get(self._dapi_url(s="tag", q="index", name=tag),headers={"Accept": "application/xml"})self._check_status(response)body = response.body.decode("utf-8", errors="replace")@@ -1817,60 +1926,32 @@ class Rule34XxxConnector(_BaseConnector):"video", "sound", "animated", "mp4", "webm", "3d", "hd", "60fps","tagme", "highres", "absurdres",})- #: 详情页取不到时的退避节奏,与外网退避规则同一套。- _DETAIL_RETRY_DELAYS = (1.5, 4.0, 9.0)- #: 详情页 `#stats` 里的上传时间:`Posted: 2026-09-11 00:48:20 by mhzw666`。- _POSTED_RE = re.compile(r"Posted:\s*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")-def _detail(self, post_id: str) -> dict[str, object]:- """帖子详情页上才有的两样:标签分类和上传时间。取不到就是空字典。+ """单条帖子的标签分类和上传时间,供存量行补齐。取不到的那一样不出现在结果里。- dapi 只给一串扁平标签,分类要读详情页 `#tag-sidebar`,取不到就不给,不按词形- 猜。dapi 也不给上传时间:它的 `change` 是最后一次修改,实测 17361475 在- 2026-05-01 上传、09-14 被改过标签,`change` 就是 09-14。详情页 `#stats` 的- Posted 才是上传时间;没被改过的帖子两者逐秒相同,所以它和 `change` 一样是 UTC。+ 列表请求一页问齐这两样(`fetch`),这里按帖子号各问一次:JSON 带+ `fields=tag_info` 给分类,XML 给 `created_at`。dapi 列表的 `change` 是最后一次+ 修改:实测 17361475 在 2026-05-01 上传、09-14 被改过标签,`change` 就是 09-14。+ 被限流挡回来时退避重试,仍取不到就缺那一样,不按词形猜,也不写空冒充已补。"""- url = ("https://rule34.xxx/index.php?page=post&s=view&id="- f"{urllib.parse.quote(post_id)}")- # 站方公布的口径是每 60 秒 60 次,而列表页一页 24 条、每条都要单独打一次- # 详情页,被挡回来是常态。一次挡回来就返回 {},得到的是「这条没有类型」——- # 和「这条确实没有类型」在日志里长得一模一样。实测:回填首轮 400 条里- # 372 条判成「没有类型」,事后逐条直取,全部 200 且 `#tag-sidebar` 正常。- response = None- for delay in (0.0, *self._DETAIL_RETRY_DELAYS):- if delay:- self.sleeper(delay)+ if not str(post_id).isdigit():+ return {}+ result: dict[str, object] = {}+ response = self._dapi_patiently(+ self._dapi_url(s="post", q="index", json="1", id=str(post_id),+ fields="tag_info"), "application/json")+ if response is not None:try:- response = self._get(url, headers={"Accept": "text/html"})+ posts = self._dapi_posts(response)except FollowSourceError:- response = None- break- if response.status not in {425, 429}:- break- if response is None or response.status != 200:- return {}- soup = BeautifulSoup(response.body, "html.parser")- allowed = {"general", "artist", "copyright", "character", "metadata"}- result: dict[str, str] = {}- for row in soup.select("#tag-sidebar li[class*='tag-type-']"):- tag_type = next((- value.removeprefix("tag-type-") for value in row.get("class", [])- if value.startswith("tag-type-")- ), "")- if tag_type not in allowed:- continue- link = row.select_one("a[href*='page=post'][href*='tags=']")- if link is None:- continue- query = urllib.parse.parse_qs(- urllib.parse.urlsplit(str(link.get("href") or "")).query)- name = html.unescape(str((query.get("tags") or [""])[0])).strip()- if name:- result[name] = tag_type- stats = soup.select_one("#stats")- posted = self._POSTED_RE.search(stats.get_text(" ") if stats is not None else "")- return {"tag_types": result,- "published_at": _iso_from_text(posted.group(1)) if posted else None}+ posts = []+ tag_types = self._tag_info_types(posts[0].get("tag_info")) if posts else {}+ if tag_types:+ result["tag_types"] = tag_types+ uploaded = self._upload_times(post_id=str(post_id)).get(str(post_id))+ if uploaded:+ result["published_at"] = uploaded+ return resultdef _candidate(self, post: dict, tag: str) -> FollowCandidate:post_id = str(post.get("id") or "")@@ -1878,6 +1959,7 @@ class Rule34XxxConnector(_BaseConnector):# 就进 metadata,读取层再转一次就成了双重转义,用户看到的就是 `'`# 字面量,同一个标签还会和反转义后的写法分裂成两个身份。tags = html.unescape(str(post.get("tags") or ""))+ tag_types = self._tag_info_types(post.get("tag_info"))image = str(post.get("image") or "")stem = re.sub(r"\.[a-z0-9]{2,5}$", "", image, flags=re.IGNORECASE)# booru 帖子没有标题。实测 rule34.xxx 的 `image` 全是 32 位十六进制哈希,@@ -1912,18 +1994,20 @@ class Rule34XxxConnector(_BaseConnector):# 对图片则是站点选定的样图。gallery-dl 的 booru 抽取器也把这三层# 作为可配置回退链,不能把最小 preview 固定成 Peach 封面。thumb_url=str(post.get("sample_url") or post.get("preview_url") or "") or None,- # `change` 是最后修改时间,只在详情页取不到时占位;上传时间由 `_detail` 给。+ # `change` 是最后修改时间,只在 XML 的 `created_at` 取不到时占位。published_at=_iso_from_epoch(post.get("change")),group_hint=hint,title_is_name=title_is_name,- # 列表视图:标签分类要等第二阶段的详情页。+ # 列表视图:上传时间要等第二阶段。分类已经随 `tag_info` 到了,partial 行+ # 落库时 metadata 走 `json_patch`,分类照样写进去。partial=True,extra={"tag": tag, "tags": tags, "score": post.get("score"),"source": source, "title_from": title_from,"preview_url": post.get("preview_url"),# dapi 随帖给出原文件的宽高;图片墙靠它在图落地前占好比例。# sample 与 preview 都是等比缩放,比例与原文件一致。- **_dims_extra(post.get("width"), post.get("height"))},+ **_dims_extra(post.get("width"), post.get("height")),+ **({"tag_types": tag_types} if tag_types else {})},)@classmethoddiff --git a/src/peach/follow_store.py b/src/peach/follow_store.pyindex 60b6b502d..07f17aa07 100644--- a/src/peach/follow_store.py+++ b/src/peach/follow_store.py@@ -179,7 +179,7 @@ def _carry_image_dims(metadata: dict, previous_json: str | None) -> dict:#: published_at、group_hint 一概不动,metadata 用 `json_patch` 并进去而#: 不是整块替换——否则跳过第二阶段就等于把上一轮取到的细节抹掉,而抹掉之后#: 「详情本来就没有」和「这次没去问」在数据里长得一模一样。-#: 时长只补空:rule34xxx 的时长读自原文件头,帖子页被限流挡回来时它照样取得到,+#: 时长只补空:rule34xxx 的时长读自原文件头,上传时间被限流挡回来时它照样取得到,#: 这时候选仍是 partial,已有的时长不覆盖,空着的补上。_PARTIAL_UPDATE = (" title=excluded.title, url=excluded.url, version=excluded.version,"@@ -200,8 +200,8 @@ _ENRICHED_PREDICATES = {# kemono 系:详情页只用来判「这一帖交付了资源没有」,没有要补的列。判过保留的帖子# 才会落库,所以行在 ledger 里本身就是判定结果,库里有的帖子不必再探。"kept": "1",- # rule34xxx:分类只有帖子页给,时长只有原文件头给,两样都有才算补齐。只看分类的话,- # 时长功能上线前补过分类的行、文件头那次没读到的行,常规检查永远不会再问。+ # rule34xxx:分类随 dapi 的 `tag_info` 到,时长只有原文件头给,两样都有才算补齐。+ # 只看分类的话,时长功能上线前补过分类的行、文件头那次没读到的行,常规检查永远不会再问。"tag_types_duration": ("json_extract(metadata_json,'$.tag_types') IS NOT NULL"" AND (duration IS NOT NULL OR (lower(COALESCE(media_url,'')) NOT LIKE '%.mp4'"diff --git a/tests/test_follow_sources.py b/tests/test_follow_sources.pyindex d15d0a5d6..26e52c3cb 100644--- a/tests/test_follow_sources.py+++ b/tests/test_follow_sources.py@@ -82,13 +82,17 @@ RULE34VIDEO_DETAIL_HTML = b"""<html><body><script>video_url: 'https://rule34video.com/get_file/51/aa/4542721.mp4/?v-acctoken=test'</script></body></html>"""-RULE34XXX_DETAIL_HTML = b"""<html><body><ul id="tag-sidebar">-<li class="tag-type-metadata tag"><a href="index.php?page=wiki&s=list&search=3d">?</a><a href="index.php?page=post&s=list&tags=3d">3d</a></li>-<li class="tag-type-metadata tag"><a href="index.php?page=wiki&s=list&search=blender">?</a><a href="index.php?page=post&s=list&tags=blender">blender</a></li>-<li class="tag-type-metadata tag"><a href="index.php?page=wiki&s=list&search=3d_model">?</a><a href="index.php?page=post&s=list&tags=3d_model">3d model</a></li>-<li class="tag-type-artist tag"><a href="index.php?page=post&s=list&tags=lazyprocrastinator">lazyprocrastinator</a></li>-<li class="tag-type-general tag"><a href="index.php?page=post&s=list&tags=reverse_cowgirl_position">reverse cowgirl position</a></li>-</ul></body></html>"""+# XML 形态取自 2026-11-03 对 api.rule34.xxx 的实测:只有它带 `created_at`,带时区偏移。+RULE34XXX_XML_FIRST_TWO = (+ b'<?xml version="1.0" encoding="UTF-8"?><posts count="2" offset="0">'+ b'<post height="720" score="72" id="18534395" change="1787445373"'+ b' created_at="Fri May 01 18:32:21 +0200 2026" status="active"/>'+ b'<post height="720" score="70" id="18534396" change="1787445300"'+ b' created_at="Sun Aug 23 00:30:00 -0500 2026" status="active"/>'+ b"</posts>")+RULE34XXX_XML = RULE34XXX_XML_FIRST_TWO.replace(b"</posts>", (+ b'<post id="18534397" created_at="Sun Aug 23 00:20:00 +0000 2026"/>'+ b'<post id="18534398" created_at="Sun Aug 23 00:10:00 +0000 2026"/></posts>'))# 搜索表单与结果页的结构取自 2026-09-01 对 f95zone.to 的实测:关键词 `Ria_neearts`# 在 latest_data.php 的五个分类里全为空,站内搜索命中@@ -523,9 +527,19 @@ class OfficialConnectorTests(unittest.TestCase):# 形状取自 2026-08-25 对 api.rule34.xxx 的实测响应:顶层是裸列表,`image` 是 32 位# 十六进制哈希(15/15),`parent_id` 全是 0(15/15),`source` 13/15 有值。+# `tag_info` 是 `fields=tag_info` 加上的(2026-11-03 实测):普通标签的类型写作 `tag`,+# 名字和 `tags` 一样是 HTML 转义形态。RULE34XXX_JSON = json.dumps([{"id": 18534395, "image": "3df1cbc67e072d6144588d6c80e490ea.mp4", "parent_id": 0,- "tags": "lazyprocrastinator fiona blush video sound", "change": 1787445373,+ "tags": "lazyprocrastinator fiona blush video sound miqo'te", "change": 1787445373,+ "tag_info": [+ {"count": 120, "type": "artist", "tag": "lazyprocrastinator"},+ {"count": 900, "type": "character", "tag": "fiona"},+ {"count": 50000, "type": "tag", "tag": "blush"},+ {"count": 80000, "type": "metadata", "tag": "video"},+ {"count": 70000, "type": "metadata", "tag": "sound"},+ {"count": 20796, "type": "character", "tag": "miqo'te"},+ {"count": 3, "type": "deprecated", "tag": "old_name"}],"file_url": "https://api-cdn-mp4.rule34.xxx/images/1232/3df1.mp4","sample_url": "https://api-cdn.rule34.xxx/images/1232/3df1.jpg","preview_url": "https://api-cdn.rule34.xxx/thumbnails/1232/t_3df1.jpg",@@ -533,13 +547,16 @@ RULE34XXX_JSON = json.dumps(["score": 72, "source": "https://lazyprocrast.fanbox.cc/posts/12304831"},{"id": 18534396, "image": "3bda1572c365b223d8b287c538a38956.mp4", "parent_id": 0,"tags": "lazyprocrastinator fiona video", "change": 1787445300,+ "tag_info": [{"count": 120, "type": "artist", "tag": "lazyprocrastinator"}],"file_url": "https://api-cdn-mp4.rule34.xxx/images/1232/3bda.mp4","score": 70, "source": "https://www.fanbox.cc/@lazyprocrast/posts/12304831"},{"id": 18534397, "image": "93eaeadff5effabe078c738aab85b3bc.mp4", "parent_id": 0,"tags": "lazyprocrastinator sayuri video", "change": 1787445200,+ "tag_info": [{"count": 120, "type": "artist", "tag": "lazyprocrastinator"}],"file_url": "https://api-cdn-mp4.rule34.xxx/images/1232/93ea.mp4", "score": 12},{"id": 18534398, "image": "fiona_paizuri_nude.mp4", "parent_id": 18534397,"tags": "lazyprocrastinator fiona nude", "change": 1787445100,+ "tag_info": [{"count": 120, "type": "artist", "tag": "lazyprocrastinator"}],"file_url": "https://api-cdn-mp4.rule34.xxx/images/1232/fp.mp4", "score": 9},]).encode()@@ -1492,134 +1509,133 @@ class Rule34XxxConnectorTests(unittest.TestCase):self.assertNotIn("width", by_id["18534396"].extra)self.assertNotIn("height", by_id["18534396"].extra)- def test_post_page_taxonomy_is_recorded_without_guessing_from_tag_words(self):- seen = []+ @staticmethod+ def _dapi_transport(*, xml=RULE34XXX_XML, video=None, seen=None):+ """JSON 请求回列表,XML 请求按 `xml` 回(可调用时逐次取),文件头按 `video` 回。"""def transport(request, timeout, max_bytes):- seen.append(request)- body = RULE34XXX_JSON if request.url.startswith("https://api.rule34.xxx/") \- else RULE34XXX_DETAIL_HTML- return HttpResponse(200, {}, body)+ if seen is not None:+ seen.append((request, max_bytes))+ if request.url.startswith("https://api.rule34.xxx/"):+ query = urllib.parse.parse_qs(urllib.parse.urlsplit(request.url).query)+ if query.get("json") == ["1"]:+ return HttpResponse(200, {}, RULE34XXX_JSON)+ return xml() if callable(xml) else HttpResponse(200, {}, xml)+ if video is not None and request.url.startswith("https://api-cdn-mp4.rule34.xxx/"):+ return HttpResponse(206, {}, video)+ return HttpResponse(404, {}, b"")+ return transport- result = Rule34XxxConnector(- transport=transport, max_items=1,+ def _fetch(self, transport, *, max_items=1, sleeper=lambda _: None, **options):+ return Rule34XxxConnector(+ transport=transport, max_items=max_items, sleeper=sleeper,credential=Credential("rule34xxx", {"user_id": "42", "api_key": "sekret"}),- ).fetch("lazyprocrastinator")- types = result.candidates[0].extra["tag_types"]- self.assertEqual(types["3d"], "metadata")- self.assertEqual(types["blender"], "metadata")- self.assertEqual(types["3d_model"], "metadata")- self.assertEqual(types["lazyprocrastinator"], "artist")- self.assertEqual(types["reverse_cowgirl_position"], "general")- self.assertNotIn("api_key", seen[1].url)- # 详情页没有 `#stats` 时,dapi 的 `change` 留作占位。- self.assertEqual(result.candidates[0].published_at, "2026-08-23T00:36:13Z")-- def test_upload_time_comes_from_the_post_page_not_the_last_edit(self):- """dapi 的 `change` 是最后修改时间:17361475 五月上传、九月改过标签就显示成九月。"""- detail = RULE34XXX_DETAIL_HTML.replace(b"</ul>", (- b'</ul><div id="stats"><ul><li>Id: 18534395</li>'- b"<li>Posted: 2026-05-01 16:32:21<br> by <a>rekin3d</a></li></ul></div>"))+ **options).fetch("lazyprocrastinator")- def transport(request, timeout, max_bytes):- body = RULE34XXX_JSON if request.url.startswith("https://api.rule34.xxx/") \- else detail- return HttpResponse(200, {}, body)+ def test_list_request_brings_the_site_taxonomy_without_guessing_from_tag_words(self):+ """帖子页被 Cloudflare 验证挡住后,分类随列表请求的 `fields=tag_info` 一起到。- result = Rule34XxxConnector(- transport=transport, max_items=1,- credential=Credential("rule34xxx", {"user_id": "42", "api_key": "sekret"}),- ).fetch("lazyprocrastinator")- self.assertEqual(result.candidates[0].published_at, "2026-05-01T16:32:21Z")- self.assertFalse(result.candidates[0].partial)+ 一页一次请求就拿全,不逐条打帖子页;普通标签在 `tag_info` 里叫 `tag`,+ 记成读取层与色板认的 general。站方没列进五类的(`deprecated`)不记。+ """+ seen = []+ result = self._fetch(self._dapi_transport(seen=seen))+ self.assertEqual(result.candidates[0].extra["tag_types"], {+ "lazyprocrastinator": "artist", "fiona": "character", "blush": "general",+ "video": "metadata", "sound": "metadata", "miqo'te": "character"})+ query = urllib.parse.parse_qs(urllib.parse.urlsplit(seen[0][0].url).query)+ self.assertEqual(query["fields"], ["tag_info"])+ self.assertTrue(all(request.url.startswith("https://api") for request, _ in seen),+ "不再打 rule34.xxx 的帖子页")+ self.assertNotIn("api_key", result.request_url)++ def test_upload_time_comes_from_the_xml_created_at_not_the_last_edit(self):+ """dapi 的 `change` 是最后修改时间:17361475 五月上传、九月改过标签就显示成九月。++ 只有 XML 给 `created_at`,带时区偏移;按帖子号对回,对不上的那条留 partial。+ """+ candidates = self._fetch(self._dapi_transport(xml=RULE34XXX_XML_FIRST_TWO),+ max_items=3).candidates+ self.assertEqual(candidates[0].published_at, "2026-05-01T16:32:21Z")+ self.assertEqual(candidates[1].published_at, "2026-08-23T05:30:00Z")+ self.assertFalse(candidates[0].partial)+ # 两次请求之间插进新帖、页错开一格时,第三条不在 XML 里:`change` 占位,+ # 保持 partial 等下一轮,分类照样带上。+ self.assertTrue(candidates[2].partial)+ self.assertEqual(candidates[2].extra["tag_types"], {"lazyprocrastinator": "artist"})+ self.assertEqual(candidates[2].published_at, "2026-08-23T00:33:20Z")def test_video_duration_comes_from_the_original_file_head(self):- """接口和帖子页都不给时长,原文件头的 `mvhd` 有;只取头上一段,不拉整片。"""+ """接口都不给时长,原文件头的 `mvhd` 有;只取头上一段,不拉整片。"""video = minimal_mp4(timescale=1000, sample_delta=40, samples=250, keyframe_every=25)seen = []-- def transport(request, timeout, max_bytes):- seen.append((request, max_bytes))- if request.url.startswith("https://api.rule34.xxx/"):- return HttpResponse(200, {}, RULE34XXX_JSON)- if request.url.startswith("https://api-cdn-mp4.rule34.xxx/"):- return HttpResponse(206, {}, video)- return HttpResponse(200, {}, RULE34XXX_DETAIL_HTML)-- result = Rule34XxxConnector(- transport=transport, max_items=1,- credential=Credential("rule34xxx", {"user_id": "42", "api_key": "sekret"}),- ).fetch("lazyprocrastinator")+ result = self._fetch(self._dapi_transport(video=video, seen=seen))self.assertEqual(result.candidates[0].duration, 10.0)head, budget = next((request, size) for request, size in seenif request.url.endswith(".mp4"))self.assertEqual(head.headers["Range"], f"bytes=0-{budget - 1}")self.assertLessEqual(budget, 65536)- def test_a_throttled_post_page_still_brings_the_file_head_duration(self):- """文件头在另一个主机上,帖子页被挡回来不牵连它:时长照样带上,候选仍是 partial。"""- video = minimal_mp4(timescale=1000, sample_delta=40, samples=250, keyframe_every=25)-- def transport(request, timeout, max_bytes):- if request.url.startswith("https://api.rule34.xxx/"):- return HttpResponse(200, {}, RULE34XXX_JSON)- if request.url.startswith("https://api-cdn-mp4.rule34.xxx/"):- return HttpResponse(206, {}, video)- return HttpResponse(503, {}, b"slow down")+ #: 站方限流页的原句:HTTP 200,正文是一句话而不是 XML。+ RATE_LIMITED = b"You currently have a limit of 60 requests every 60 second(s)"- result = Rule34XxxConnector(- transport=transport, max_items=1, sleeper=lambda _: None,- credential=Credential("rule34xxx", {"user_id": "42", "api_key": "sekret"}),- ).fetch("lazyprocrastinator")- self.assertEqual(result.candidates[0].duration, 10.0)- self.assertTrue(result.candidates[0].partial)- self.assertNotIn("tag_types", result.candidates[0].extra)+ def test_a_rate_limited_upload_time_is_retried_not_read_as_missing(self):+ """站方口径是每 60 秒 60 次。挡回来一次就放弃的话,上传时间永远停在 `change`。"""+ replies = [HttpResponse(429, {}, b""), HttpResponse(200, {}, self.RATE_LIMITED),+ HttpResponse(200, {}, RULE34XXX_XML)]+ waits = []+ result = self._fetch(self._dapi_transport(xml=lambda: replies.pop(0)),+ sleeper=waits.append)+ # 退避节奏本身也是行为:注入假 sleeper 才能断言它,而且测试不会真的等。+ self.assertEqual(waits, [1.5, 4.0])+ self.assertEqual(result.candidates[0].published_at, "2026-05-01T16:32:21Z")+ self.assertFalse(result.candidates[0].partial)- def test_a_throttled_detail_page_is_retried_not_read_as_no_types(self):- """站方公布的是每 60 秒 60 次,而列表页一页 24 条、每条都要单独打一次详情页。+ def test_an_upload_time_that_stays_throttled_keeps_types_duration_and_partial(self):+ """重试有界,仍被挡就留 partial:落库不动已有的上传时间,下一轮再问。- 一次挡回来就返回 {},落进库里就是「这条没有类型」——和「这条确实没有- 类型」长得一模一样。实测:回填首轮 400 条里 372 条判成「没有类型」,- 事后逐条直取,全部 200 且 `#tag-sidebar` 正常。+ 列表那次已经成功,分类与它同一次请求到手,照样带上;文件头在另一个主机上,+ 也不受牵连。一抛错就连已经到手的列表一起丢了。"""- attempts = []+ video = minimal_mp4(timescale=1000, sample_delta=40, samples=250, keyframe_every=25)+ waits = []+ result = self._fetch(self._dapi_transport(xml=self.RATE_LIMITED, video=video),+ sleeper=waits.append)+ first = result.candidates[0]+ self.assertEqual(waits, [1.5, 4.0, 9.0], "退避有界且不会叠乘")+ self.assertTrue(first.partial,+ "上传时间没取到就得留着 partial,否则落库会拿 `change` 冲掉上一轮的值")+ self.assertEqual(first.extra["tag_types"]["lazyprocrastinator"], "artist")+ self.assertEqual(first.duration, 10.0)- def transport(request, timeout, max_bytes):- if request.url.startswith("https://api.rule34.xxx/"):- return HttpResponse(200, {}, RULE34XXX_JSON)- if "page=post" not in request.url:- return HttpResponse(404, {}, b"")- attempts.append(request.url)- if len(attempts) < 3:- return HttpResponse(503, {}, b"slow down")- return HttpResponse(200, {}, RULE34XXX_DETAIL_HTML)+ def test_a_page_already_enriched_costs_no_upload_time_request(self):+ seen = []+ self._fetch(self._dapi_transport(seen=seen), max_items=4,+ enrich_skip=("18534395", "18534396", "18534397", "18534398"))+ self.assertEqual(len(seen), 1, "整页都补过,只有列表那一次请求")- waits = []- result = Rule34XxxConnector(- transport=transport, max_items=1, sleeper=waits.append,+ def test_one_post_detail_for_the_backfill_asks_dapi_by_id(self):+ """存量行补齐逐条问:JSON 按帖子号带 `tag_info`,XML 按帖子号给 `created_at`。"""+ seen = []+ detail = Rule34XxxConnector(+ transport=self._dapi_transport(seen=seen), sleeper=lambda _: None,credential=Credential("rule34xxx", {"user_id": "42", "api_key": "sekret"}),- ).fetch("lazyprocrastinator")- self.assertEqual(len(attempts), 3, "被挡回来要重试,不能一次就当没有类型")- # 退避节奏本身也是行为:注入假 sleeper 才能断言它,而且测试不会真的等 5.5 秒。- self.assertEqual(waits, [1.0, 2.0])- self.assertEqual(result.candidates[0].extra["tag_types"]["lazyprocrastinator"],- "artist")-- def test_a_detail_page_that_stays_unavailable_records_no_types(self):- """重试完仍取不到就留空,让下一轮再问;不写空字典冒充已补。"""+ )._detail("18534395")+ self.assertEqual(detail["published_at"], "2026-05-01T16:32:21Z")+ self.assertEqual(detail["tag_types"]["miqo'te"], "character")+ for request, _ in seen:+ query = urllib.parse.parse_qs(urllib.parse.urlsplit(request.url).query)+ self.assertEqual(query["id"], ["18534395"])++ def test_one_post_detail_that_stays_throttled_has_no_types(self):+ """被挡回来不能读成「这条没有分类」:结果里不出现 `tag_types`,脚本记未取得。"""def transport(request, timeout, max_bytes):- if request.url.startswith("https://api.rule34.xxx/"):- return HttpResponse(200, {}, RULE34XXX_JSON)- return HttpResponse(503, {}, b"slow down")+ return HttpResponse(200, {}, self.RATE_LIMITED)- waits = []- result = Rule34XxxConnector(- transport=transport, max_items=1, sleeper=waits.append,+ detail = Rule34XxxConnector(+ transport=transport, sleeper=lambda _: None,credential=Credential("rule34xxx", {"user_id": "42", "api_key": "sekret"}),- ).fetch("lazyprocrastinator")- self.assertNotIn("tag_types", result.candidates[0].extra)- self.assertTrue(result.candidates[0].partial,- "详情没取到就得留着 partial,否则落库会把上一轮的分类抹掉")- self.assertEqual(waits, [1.0, 2.0, 4.0, 8.0], "退避有界且不会叠乘")+ )._detail("18534395")+ self.assertEqual(detail, {})def test_autocomplete_reports_the_real_tag_spelling_without_credentials(self):# 站上写作 `ria-neearts`,手边的手柄是 `Ria_neearts`;补全是唯一能把两者@@ -1661,6 +1677,7 @@ class Rule34XxxConnectorTests(unittest.TestCase):self.assertEqual(query["tags"], ["stellar_blade 3d sort:score"])self.assertEqual(query["limit"], ["3"])self.assertNotIn("pid", query, "一次性的搜索只要第一页")+ self.assertNotIn("fields", query, "不落库的搜索用不上分类,不多拉四倍正文")self.assertEqual(len(candidates), 3)self.assertEqual(candidates[0].thumb_url,"https://api-cdn.rule34.xxx/images/1232/3df1.jpg",@@ -2330,12 +2347,16 @@ class EnrichPhaseTests(unittest.TestCase):补齐过的条目不再问——否则一个作者的来源每次检查都要按条目数付请求。"""+ #: rule34xxx 第二阶段逐条付费的请求是原文件头;上传时间整页一次,不随条目数长。+ HEAD = "https://api-cdn-mp4.rule34.xxx/"+def _rule34xxx(self, seen, **kwargs):def transport(request, timeout, max_bytes):seen.append(request.url)if request.url.startswith("https://api.rule34.xxx/index.php"):- return HttpResponse(200, {}, RULE34XXX_JSON)- return HttpResponse(200, {}, RULE34XXX_DETAIL_HTML)+ return HttpResponse(200, {}, RULE34XXX_JSON if "json=1" in request.url+ else RULE34XXX_XML)+ return HttpResponse(404, {}, b"")return Rule34XxxConnector(transport=transport, max_items=4,@@ -2347,7 +2368,7 @@ class EnrichPhaseTests(unittest.TestCase):result = self._rule34xxx(seen, enrich_budget=2).fetch("lazyprocrastinator")self.assertEqual(len(result.candidates), 4)self.assertEqual(result.probed, 2)- details = [url for url in seen if "s=view" in url]+ details = [url for url in seen if url.startswith(self.HEAD)]self.assertEqual(len(details), 2, "额度是请求数的上限,不是建议值")self.assertEqual([item.partial for item in result.candidates],[False, False, True, True],@@ -2356,7 +2377,7 @@ class EnrichPhaseTests(unittest.TestCase):def test_a_zero_budget_means_the_list_phase_alone(self):seen = []result = self._rule34xxx(seen, enrich_budget=0).fetch("lazyprocrastinator")- self.assertEqual(seen, [url for url in seen if "s=view" not in url])+ self.assertEqual(len(seen), 1, "额度为零连整页的上传时间都不问")self.assertEqual(result.probed, 0)self.assertTrue(all(item.partial for item in result.candidates))@@ -2367,7 +2388,7 @@ class EnrichPhaseTests(unittest.TestCase):def publish(candidate):ready.append(candidate)enriched = sum(not item.partial for item in ready)- self.assertEqual(sum('s=view' in url for url in seen), enriched)+ self.assertEqual(sum(url.startswith(self.HEAD) for url in seen), enriched)connector.candidate_ready = publishresult = connector.fetch('lazyprocrastinator')@@ -2381,8 +2402,9 @@ class EnrichPhaseTests(unittest.TestCase):result = self._rule34xxx(seen, enrich_budget=2, enrich_skip={"18534395", "18534396"},).fetch("lazyprocrastinator")- details = sorted(url.rsplit("id=", 1)[1] for url in seen if "s=view" in url)- self.assertEqual(details, ["18534397", "18534398"])+ details = sorted(url for url in seen if url.startswith(self.HEAD))+ self.assertEqual(details, [self.HEAD + "images/1232/93ea.mp4",+ self.HEAD + "images/1232/fp.mp4"])self.assertEqual([item.partial for item in result.candidates],[True, True, False, False])@@ -2412,8 +2434,8 @@ class EnrichPhaseTests(unittest.TestCase):def test_rule34xxx_is_not_marked_by_a_time_the_list_already_gave(self):"""rule34xxx 的上传时间来自列表的 `change`,第一次落库就有值。- 拿 `published_at` 当它的补齐判据,等于宣布「所有条目都已补齐」,上一轮被- 限流挡掉的分类就永远补不回来了。+ 拿 `published_at` 当它的补齐判据,等于宣布「所有条目都已补齐」,没读到时长的+ mp4 就永远补不回来了。分类随列表一起到,不走第二阶段也照样带上。"""self.assertNotEqual(Rule34XxxConnector.ENRICHED_MARK, "published_at")listed = Rule34XxxConnector(@@ -2421,7 +2443,7 @@ class EnrichPhaseTests(unittest.TestCase):credential=Credential("rule34xxx", {"user_id": "42", "api_key": "sekret"}),).fetch("lazyprocrastinator").candidatesself.assertTrue(all(item.published_at for item in listed))- self.assertTrue(all("tag_types" not in item.extra for item in listed))+ self.assertTrue(all(item.partial and item.extra["tag_types"] for item in listed))class MediaContentHashTests(unittest.TestCase):diff --git a/tests/test_scripts.py b/tests/test_scripts.pyindex e52e5ad78..8b40736b7 100644--- a/tests/test_scripts.py+++ b/tests/test_scripts.py@@ -1618,7 +1618,7 @@ class OperationalScriptTests(unittest.TestCase):self.assertEqual(self.scrape_codes.SETTLED_ERROR_KINDS, frozenset({"not_found"}))def test_rule34_detail_backfill_reuses_the_connector_and_is_resumable(self):- """标签类型、上传时间在帖子页上,视频时长在原文件头里,存量行靠常规检查补不到。+ """标签类型、上传时间要另问 dapi,视频时长在原文件头里,存量行靠常规检查补不到。抓取判据不重写,直接复用连接器的 `_detail` 与 `_video_seconds`;备份先做、分批提交,续跑只跳过上一趟真正写入的条目。@@ -1646,6 +1646,9 @@ class OperationalScriptTests(unittest.TestCase):self.assertEqual([row["external_id"] for row in pending], ["18622796", "18622794"])self.assertEqual([row["external_id"] for row in backfill.pending_rows(connection, 0, {1})],["18622794"])+ # 只补类型时只挑还没有非空类型的条目,空表与缺键一样算缺。+ self.assertEqual([row["external_id"] for row in backfill.pending_rows(+ connection, 0, missing_types=True)], ["18622794"])# 只补不抹:已有的类型与时长不覆盖,帖子页没给上传时间就不动它。typed, untyped = pending@@ -1673,7 +1676,7 @@ class OperationalScriptTests(unittest.TestCase):self.assertEqual(backfill.written_ids(out), {2})def test_rule34_duration_only_backfill_touches_nothing_but_empty_durations(self):- """授权只覆盖时长时:只挑缺时长的 mp4,不问帖子页,只写 `duration`。"""+ """授权只覆盖时长时:只挑缺时长的 mp4,不问 dapi,只写 `duration`。"""backfill = load_script("backfill_rule34_details")database = self.tmp_ledger()seeding = sqlite3.connect(database)@@ -1690,7 +1693,7 @@ class OperationalScriptTests(unittest.TestCase):class Connector:def _detail(self, post_id):- raise AssertionError("只补时长不该打帖子页")+ raise AssertionError("只补时长不该问 dapi")def _video_seconds(self, url):heads.append(url)