Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(follow): rule34.xxx 标签分类改走 dapi 的 tag_info

longmeidao8e7bebd99d25
diff --git a/scripts/backfill_rule34_details.py b/scripts/backfill_rule34_details.py
index 5e13bda91..568d71ded 100644
--- a/scripts/backfill_rule34_details.py
+++ b/scripts/backfill_rule34_details.py
@@ -1,13 +1,15 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
-r"""给已入库的 rule34.xxx 追更条目补上只有帖子页和原文件才有的细节。
+r"""给已入库的 rule34.xxx 追更条目补上列表请求之外的细节。
-posts DAPI 只回扁平标签和最后修改时间,三样东西要另外问:
+posts DAPI 的默认 JSON 只回扁平标签和最后修改时间,三样东西要另外问:
-- 标签类型(general/artist/character/copyright/metadata)在帖子页 `#tag-sidebar`。
-- 上传时间在帖子页 `#stats` 的 Posted。DAPI 的 `change` 是最后一次修改:17361475
+- 标签类型(general/artist/character/copyright/metadata):JSON 带
+ `fields=tag_info`。帖子页 `#tag-sidebar` 自 2026-10-20 起被 Cloudflare 验证挡住,
+ 不再是来源。
+- 上传时间:XML 的 `created_at`。DAPI 的 `change` 是最后一次修改:17361475
五月上传、九月被改过标签,入库时记成了九月,卡片就排到了新内容里。
-- 视频时长只在原文件头的 `mvhd` 里,接口和帖子页都不给。
+- 视频时长只在原文件头的 `mvhd` 里,接口都不给。
常规「检查更新」只看第一页,而且已经补齐过类型的条目不再进第二阶段,存量行靠
它永远改不过来,所以要这一趟。判据不重写,直接复用连接器的 `_detail` 与
@@ -16,8 +18,11 @@ posts DAPI 只回扁平标签和最后修改时间,三样东西要另外问:
默认 dry-run,只写复核 CSV。`--apply` 必须同时给 `--backup`,与本仓库其它真实写入
脚本一致。`--resume` 跳过同一份 CSV 里已经写入的条目,中断后用原命令加它接着跑。
-`--duration-only` 只补 mp4 缺的时长:只挑 `duration` 为空的 mp4 条目,不问帖子页,
-只读原文件头,也只写 `duration`。时长在视频 CDN 上,不受帖子页每秒一次的限流牵连,
+`--missing-types` 只挑账本里还没有标签类型的条目。帖子页被挡住的那段时间入库的行
+正是这一批,它们的上传时间也还是 `change` 占位,一趟一起改正。
+
+`--duration-only` 只补 mp4 缺的时长:只挑 `duration` 为空的 mp4 条目,不问 dapi,
+只读原文件头,也只写 `duration`。时长在视频 CDN 上,不受 dapi 每秒一次的限流牵连,
授权只覆盖时长时用它,别的列一格不动。
"""
from __future__ import annotations
@@ -44,12 +49,11 @@ from peach.scripting import (
FIELDS = ("item_id", "external_id", "result", "published_before", "published_after",
"duration", "tag_types_added", "written", "note")
-#: 帖子页的最小请求间隔。rule34.xxx 自己公布的口径是每 60 秒 60 次,也就是
-#: 每次至少 1 秒;留一点余量。首轮按 0.35 秒跑过一次:400 条里 372 条「帖子页
-#: 没有解析出类型」——不是这些帖子没有类型,是被限流挡了,`_detail`
-#: 把非 200 吞成了空字典。速度换来的全是白跑的请求。视频文件头在另一个主机上,
-#: 在这段间隔里发,不另占时间。
-DEFAULT_DELAY = 1.1
+#: 每条之间的最小间隔。rule34.xxx 自己公布的口径是每 60 秒 60 次,`_detail` 每条
+#: 问两次 dapi(JSON 的分类、XML 的上传时间),所以每条至少 2 秒;留一点余量。
+#: 首轮按 0.35 秒跑过一次:400 条里 372 条被限流挡了,被读成「没有类型」——速度
+#: 换来的全是白跑的请求。视频文件头在另一个主机上,在这段间隔里发,不另占时间。
+DEFAULT_DELAY = 2.2
#: 连续这么多条取不到就认定是被限流,而不是这一段恰好都取不到。
MISS_STREAK_LIMIT = 8
#: 限流后的静默时长,按站方公布的窗口取满一格。
@@ -69,7 +73,9 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--resume", action="store_true",
help="跳过 --out 里已经写入的条目,接着上一趟跑")
parser.add_argument("--duration-only", action="store_true",
- help="只补 mp4 缺的时长:不问帖子页,只读原文件头,只写 duration")
+ help="只补 mp4 缺的时长:不问 dapi,只读原文件头,只写 duration")
+ parser.add_argument("--missing-types", action="store_true",
+ help="只挑还没有标签类型的条目")
return parser
@@ -80,12 +86,14 @@ def written_ids(path: Path) -> set[int]:
def pending_rows(connection: sqlite3.Connection, limit: int,
- done: set[int] = frozenset(), *, duration_only: bool = False) -> list[dict]:
+ done: set[int] = frozenset(), *, duration_only: bool = False,
+ missing_types: bool = False) -> list[dict]:
"""待补条目:rule34xxx 来源的全部条目,扣掉上一趟已经写入的。
- 上传时间错没错,不问帖子页就看不出来——被改过的帖子 `change` 晚于上传,没改过的
+ 上传时间错没错,不问 dapi 就看不出来——被改过的帖子 `change` 晚于上传,没改过的
两者逐秒相同,账本里长得一样。所以每条都要问一遍,续跑靠 CSV 而不是账本里的值。
- 只补时长时缺不缺看账本就知道,只挑 `duration` 为空的 mp4。
+ 只补时长时缺不缺看账本就知道,只挑 `duration` 为空的 mp4;只补类型时同理,只挑
+ metadata 里没有 `tag_types` 的。
"""
rows = []
query = (
@@ -102,6 +110,9 @@ def pending_rows(connection: sqlite3.Connection, limit: int,
metadata = json.loads(row["metadata_json"] or "{}")
except ValueError:
metadata = {}
+ # 空表与缺键同样算缺:`plan_update` 也只在没有非空类型时才补。
+ if missing_types and isinstance(metadata, dict) and metadata.get("tag_types"):
+ continue
rows.append({"id": row["id"], "external_id": str(row["external_id"] or ""),
"media_url": row["media_url"], "published_at": row["published_at"],
"duration": row["duration"],
@@ -114,7 +125,7 @@ def pending_rows(connection: sqlite3.Connection, limit: int,
def plan_update(row: dict, detail: dict, duration: float | None) -> dict | None:
"""这一条要改哪几列。没有可写的就是 None。
- 只补不抹:帖子页没给上传时间就不动 `published_at`;已有时长不覆盖;已有非空
+ 只补不抹:dapi 没给上传时间就不动 `published_at`;已有时长不覆盖;已有非空
类型不覆盖。取不到的留待下一趟,不写空值冒充已补。
"""
update: dict = {}
@@ -131,10 +142,10 @@ def plan_update(row: dict, detail: dict, duration: float | None) -> dict | None:
def inspect_row(connector, row: dict) -> tuple[dict, dict | None]:
- """问一条:帖子页,缺时长的视频再问一次文件头。返回 CSV 行和要写的列。
+ """问一条:dapi 的分类与上传时间,缺时长的视频再问一次文件头。返回 CSV 行和要写的列。
- 帖子页没解析出来时 `result` 是「未取得」:限流时整页都是空的,不能当成「这条
- 没有」写进去,留给下一趟再问。
+ 分类没取到时 `result` 是「未取得」:被限流挡回来与帖子已删都是这样,不能当成
+ 「这条没有」写进去,留给下一趟再问。
"""
post_id = row["external_id"]
entry = {"item_id": row["id"], "external_id": post_id, "result": "",
@@ -149,7 +160,7 @@ def inspect_row(connector, row: dict) -> tuple[dict, dict | None]:
except (FollowSourceError, OSError) as error:
detail, entry["note"] = {}, f"{type(error).__name__}: {error}"[:120]
if not detail.get("tag_types"):
- entry.update(result="未取得", note=entry["note"] or "帖子页没有解析出细节")
+ entry.update(result="未取得", note=entry["note"] or "dapi 没有给出标签分类")
return entry, None
duration = (connector._video_seconds(row["media_url"])
if row["duration"] is None else None)
@@ -159,7 +170,7 @@ def inspect_row(connector, row: dict) -> tuple[dict, dict | None]:
duration=update.get("duration", entry["duration"]),
tag_types_added="是" if "metadata_json" in update else "")
if not detail.get("published_at"):
- entry["note"] = "帖子页没有 Posted"
+ entry["note"] = "dapi 没有给出 created_at"
elif _is_mp4(row["media_url"]) and row["duration"] is None and not duration:
entry["note"] = "时长未取得"
return entry, update or None
@@ -208,7 +219,8 @@ def run(args: argparse.Namespace) -> int:
previous = [row for row in read_rows(args.out, missing_ok=True)
if row.get("written") == "是"] if args.resume else []
with open_readonly(args.db) as reader:
- rows = pending_rows(reader, args.limit, done, duration_only=args.duration_only)
+ rows = pending_rows(reader, args.limit, done, duration_only=args.duration_only,
+ missing_types=args.missing_types)
total = reader.execute(
"SELECT COUNT(*) FROM follow_item i JOIN follow_source s ON s.id=i.source_id"
" WHERE s.provider='rule34xxx'").fetchone()[0]
@@ -245,7 +257,7 @@ def run(args: argparse.Namespace) -> int:
write_rows(args.out, FIELDS, previous + log, atomic=True)
for index, row in enumerate(rows, 1):
- # 间隔按帖子页请求的起点算:文件头在另一个主机上,问它的时间落在这段间隔里。
+ # 间隔按 dapi 请求的起点算:文件头在另一个主机上,问它的时间落在这段间隔里。
time.sleep(max(0.0, next_detail - time.monotonic()))
next_detail = time.monotonic() + args.delay
entry, update = (inspect_duration if args.duration_only else inspect_row)(connector, row)
diff --git a/src/peach/follow_sources.py b/src/peach/follow_sources.py
index 198d93644..02ba1239a 100644
--- a/src/peach/follow_sources.py
+++ b/src/peach/follow_sources.py
@@ -1558,11 +1558,12 @@ class Rule34VideoConnector(_BaseConnector):
class Rule34XxxConnector(_BaseConnector):
"""rule34.xxx 的官方 dapi。
- 网页版已挂 Cloudflare Turnstile,Peach 不绕验证码,因此这里只走官方 API,
- 并且必须带账号自己的 `user_id` + `api_key`(在
- `https://rule34.xxx/index.php?page=account&s=options` 生成)。凭据只进查询参数
- 以外的位置不可行——dapi 只接受查询参数——所以请求 URL 不进日志也不进 ledger,
- 只有 `request_url` 的脱敏形式会被记录。
+ 网页版挂了 Cloudflare 验证:2026-11-03 实测帖子页回 HTTP 403、`cf-mitigated:
+ challenge`、正文是「Rule34.xxx CAPTCHA」页,10-20 起读 `#tag-sidebar` 的路径整批
+ 落空。Peach 不绕验证码,因此这里只走官方 API,并且必须带账号自己的 `user_id` +
+ `api_key`(在 `https://rule34.xxx/index.php?page=account&s=options` 生成)。凭据
+ 只进查询参数以外的位置不可行——dapi 只接受查询参数——所以请求 URL 不进日志也不进
+ ledger,只有 `request_url` 的脱敏形式会被记录。
`ref` 是标签,例如 `lazyprocrastinator`。
"""
@@ -1570,8 +1571,9 @@ class Rule34XxxConnector(_BaseConnector):
provider = "rule34xxx"
#: 一页 24 条时整页都能补上;额度存在是为了页面变长时请求数不跟着长。
DEFAULT_ENRICH_BUDGET = 24
- #: 第二阶段补两样:帖子页的标签分类(metadata 的 `tag_types`)和原文件头的时长。
- #: mp4 缺时长的行下次检查还会进第二阶段,判据在 `follow_store._ENRICHED_PREDICATES`。
+ #: 标签分类(metadata 的 `tag_types`)随列表请求的 `fields=tag_info` 一起到;第二
+ #: 阶段补上传时间和原文件头的时长。mp4 缺时长的行下次检查还会进第二阶段,判据在
+ #: `follow_store._ENRICHED_PREDICATES`。
ENRICHED_MARK = "tag_types_duration"
_TAG_RE = re.compile(r"^[^\s&?#]{1,100}$")
#: 历史行存的是 250px 的 preview。官方 dapi 的 `sample_url` 与它用同一
@@ -1619,29 +1621,37 @@ class Rule34XxxConnector(_BaseConnector):
return str(item.media_url or item.thumb_url or "") or None
return str(item.thumb_url or "") or None
+ def _dapi_url(self, **parameters: str) -> str:
+ """带凭据的 dapi 地址。查询串里有 api_key,只能拿去发请求,不能落进日志或证据。"""
+ if self.credential is None:
+ raise CredentialError(
+ "rule34xxx 需要 user_id 与 api_key;请把它们写进 "
+ "peach-data/secrets/follow/rule34xxx.json")
+ user_id, api_key = self.credential.require("user_id", "api_key")
+ query = urllib.parse.urlencode({"page": "dapi", **parameters,
+ "user_id": user_id, "api_key": api_key})
+ return f"https://api.rule34.xxx/index.php?{query}"
+
def _dapi(self, tags: str, *, limit: int, page: int = 0,
etag: str | None = None, last_modified: str | None = None,
+ tag_info: bool = False,
) -> tuple[dict[str, object], HttpResponse | None]:
- """问一次 dapi。凭据、地址形状和脱敏都只有这一处。
+ """问一次 dapi 帖子列表。地址形状和脱敏都只有这一处。
`tags` 原样交给站点:dapi 的标签之间用空格分隔,`sort:score` 这类元标签也
走同一个参数,所以这里不解析它,只负责把它编码进查询串。
+
+ `tag_info` 让每条帖子随带 `tag_info`:每个标签的分类和帖子数,一页一次请求
+ 就拿全,不必逐条去问帖子页或 tag 接口。2026-11-03 实测 24 条约 99 KB。
"""
- if self.credential is None:
- raise CredentialError(
- "rule34xxx 需要 user_id 与 api_key;请把它们写进 "
- "peach-data/secrets/follow/rule34xxx.json")
- user_id, api_key = self.credential.require("user_id", "api_key")
# `pid` 是 0 起的页号,页大小就是 `limit`。
- parameters = {
- "page": "dapi", "s": "post", "q": "index", "json": "1",
- "limit": str(min(limit, 1000)), "tags": tags,
- "user_id": user_id, "api_key": api_key,
- }
+ parameters = {"s": "post", "q": "index", "json": "1",
+ "limit": str(min(limit, 1000)), "tags": tags}
if page:
parameters["pid"] = str(page)
- query = urllib.parse.urlencode(parameters)
- url = f"https://api.rule34.xxx/index.php?{query}"
+ if tag_info:
+ parameters["fields"] = "tag_info"
+ url = self._dapi_url(**parameters)
safe_url = (f"https://api.rule34.xxx/index.php?page=dapi&s=post&q=index"
f"&tags={tags}" + (f"&pid={page}" if page else ""))
# request_url 传脱敏版:真实 url 的查询串里带 api_key,绝不能落进证据。
@@ -1650,6 +1660,108 @@ class Rule34XxxConnector(_BaseConnector):
headers={"Accept": "application/json"},
request_url=safe_url)
+ #: dapi 被限流时的退避节奏,与外网退避规则同一套。站方口径是每 60 秒 60 次。
+ _RATE_LIMIT_RETRY_DELAYS = (1.5, 4.0, 9.0)
+
+ def _dapi_patiently(self, url: str, accept: str) -> HttpResponse | None:
+ """补细节用的 dapi 请求:被限流就退避重试,仍取不到返回 None。
+
+ 列表那次请求已经成功,这里取的是锦上添花的细节,所以失败不抛:一抛整次检查
+ 就丢了已经到手的列表。但也不能把被挡回来读成「没有」——返回 None 让调用方
+ 保持 partial,下一轮再问。限流有两种长相:HTTP 429,或者 HTTP 200 加一句
+ `You currently have a limit of 60 requests every 60 second(s)`。
+ """
+ for delay in (0.0, *self._RATE_LIMIT_RETRY_DELAYS):
+ if delay:
+ self.sleeper(delay)
+ try:
+ response = self._get(url, headers={"Accept": accept})
+ except FollowSourceError:
+ return None
+ if response.status == 429 or (response.status == 200
+ and self._upstream_reason(response)):
+ continue
+ return response if response.status == 200 else None
+ return None
+
+ #: XML 帖子的 `created_at`:`Fri May 01 18:32:21 +0200 2026`。
+ _CREATED_AT_RE = re.compile(
+ r"^\w{3} (\w{3}) (\d{1,2}) (\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2}) (\d{4})$")
+ _MONTHS = {name: number for number, name in enumerate(
+ ("jan", "feb", "mar", "apr", "may", "jun",
+ "jul", "aug", "sep", "oct", "nov", "dec"), 1)}
+ _XML_POST_RE = re.compile(r"<post\b[^>]*>", re.IGNORECASE)
+ _XML_ATTR_RE = re.compile(r'\b(id|created_at)="([^"]*)"')
+
+ @classmethod
+ def _created_at(cls, text: str) -> str | None:
+ """XML 的上传时间转成 UTC。不靠 `%a %b`:那两个格式符跟着系统区域设置走。"""
+ matched = cls._CREATED_AT_RE.match(html.unescape(str(text or "")).strip())
+ month = cls._MONTHS.get(matched.group(1).casefold()) if matched else None
+ if not matched or not month:
+ return None
+ day, hour, minute, second = (int(matched.group(index)) for index in (2, 3, 4, 5))
+ offset = timedelta(hours=int(matched.group(7)), minutes=int(matched.group(8)))
+ try:
+ local = datetime(int(matched.group(9)), month, day, hour, minute, second,
+ tzinfo=timezone(-offset if matched.group(6) == "-" else offset))
+ except ValueError:
+ return None
+ return _iso_utc(local)
+
+ @classmethod
+ def _upload_times_from_xml(cls, body: bytes) -> dict[str, str]:
+ result: dict[str, str] = {}
+ for tag in cls._XML_POST_RE.findall(body.decode("utf-8", errors="replace")):
+ attributes = dict(cls._XML_ATTR_RE.findall(tag))
+ uploaded = cls._created_at(attributes.get("created_at", ""))
+ if attributes.get("id") and uploaded:
+ result[attributes["id"]] = uploaded
+ return result
+
+ def _upload_times(self, *, page: int = 0, tags: str = "",
+ post_id: str = "") -> dict[str, str]:
+ """帖子号到上传时间。只有 XML 形态给 `created_at`。
+
+ JSON 形态没有这一栏,`fields=created_at` 实测也不加;反过来 XML 不认
+ `fields=tag_info`。所以分类和上传时间各要一次请求:列表一次 JSON、一次 XML,
+ 按帖子号对回去。两次之间有新帖插进来时页会错开一格,对不上的那条不写上传时间,
+ 候选保持 partial。
+
+ `created_at` 与帖子页 `#stats` 的 Posted 是同一个时刻:17361475 的 XML 是
+ `Fri May 01 18:32:21 +0200 2026`,账本里帖子页取到的是 `2026-05-01T16:32:21Z`。
+ """
+ parameters = {"s": "post", "q": "index"}
+ if post_id:
+ parameters["id"] = post_id
+ else:
+ parameters.update(limit=str(min(self.max_items, 1000)), tags=tags)
+ if page:
+ parameters["pid"] = str(page)
+ response = self._dapi_patiently(self._dapi_url(**parameters), "application/xml")
+ return self._upload_times_from_xml(response.body) if response is not None else {}
+
+ #: `tag_info` 里普通标签叫 `tag`;tag 接口与帖子页 `#tag-sidebar` 叫 general。
+ #: 记成后者,读取层与前端 `r34-*` 色板都按这套词认分类。
+ _TAG_INFO_TYPES = {"tag": "general", "general": "general", "artist": "artist",
+ "copyright": "copyright", "character": "character",
+ "metadata": "metadata"}
+
+ @classmethod
+ def _tag_info_types(cls, info) -> dict[str, str]:
+ """`fields=tag_info` 给的分类。名字与 `tags` 一样是 HTML 转义形态,要反转义。"""
+ if not isinstance(info, list):
+ return {}
+ result: dict[str, str] = {}
+ for row in info:
+ if not isinstance(row, dict):
+ continue
+ name = html.unescape(str(row.get("tag") or "")).strip()
+ kind = cls._TAG_INFO_TYPES.get(str(row.get("type") or "").strip().casefold(), "")
+ if name and kind:
+ result[name] = kind
+ return result
+
def _dapi_posts(self, response: HttpResponse) -> list[dict]:
"""一次 dapi 响应里的帖子。"""
body = response.body.decode("utf-8", errors="replace").strip()
@@ -1687,32 +1799,34 @@ class Rule34XxxConnector(_BaseConnector):
if not self._TAG_RE.match(tag):
raise FollowSourceError(f"rule34xxx 的 ref 必须是单个标签,收到:{ref!r}")
common, response = self._dapi(tag, limit=self.max_items, page=page,
- etag=etag, last_modified=last_modified)
+ etag=etag, last_modified=last_modified,
+ tag_info=True)
if response is None:
return SourceFetch(not_modified=True, **common)
listed = [self._candidate(post, tag)
for post in self._dapi_posts(response)[: self.max_items]]
- # 第二阶段:标签分类只有帖子详情页给,列表的 dapi 只给一串扁平标签。
+ # 第二阶段要的上传时间整页一次问齐;整页都补过就一次也不问。
+ wanted = self.enrich_budget and any(
+ str(candidate.external_id) not in self.enrich_skip for candidate in listed)
+ self._uploaded = self._upload_times(tags=tag, page=page) if wanted else {}
candidates, probed = self.enrich(listed)
if page and not candidates:
raise FollowHistoryEnd("没有更多历史内容")
return SourceFetch(candidates=candidates, probed=probed,
raw_body=response.body, **common)
+ #: 这一页的帖子号到上传时间,`fetch` 每次重新填;类上的空表只是还没抓过时的默认。
+ _uploaded: Mapping[str, str] = {}
+
def _enrich_one(self, candidate: FollowCandidate) -> FollowCandidate:
- detail = self._detail(str(candidate.external_id))
- # 文件头在另一个主机上,帖子页被限流挡回来不影响它,所以不论分类取没取到都问。
+ uploaded = self._uploaded.get(str(candidate.external_id))
+ # 文件头在另一个主机上,dapi 被限流不牵连它,所以不论上传时间取没取到都问。
duration = self._video_seconds(candidate.media_url)
- tag_types = detail.get("tag_types")
- if not tag_types:
- # 详情页这次没给出分类。保持 partial:上一轮取到的分类还在库里,
- # 覆盖成空会让「这条没有分类」和「这次没问到」在数据里长得一样。
- # partial 行落库时时长只补空,所以读到的时长照样带上。
+ if not uploaded or not candidate.extra.get("tag_types"):
+ # 这次缺了一样就保持 partial:落库时不动 `published_at`,上一轮取到的上传
+ # 时间不会被 `change` 冲掉;分类经 `json_patch` 照样并进去,时长只补空。
return replace(candidate, duration=duration) if duration else candidate
- return replace(candidate, partial=False,
- published_at=detail.get("published_at") or candidate.published_at,
- duration=duration,
- extra={**candidate.extra, "tag_types": tag_types})
+ return replace(candidate, partial=False, published_at=uploaded, duration=duration)
#: 读时长只取文件开头这么多字节。站方转码出的 mp4 都是 faststart,`mvhd` 在开头
#: 几百字节里;实测随机 12 条全部读出,64 KiB 留足 `ftyp` 之后夹杂别的盒的余量。
@@ -1773,8 +1887,8 @@ class Rule34XxxConnector(_BaseConnector):
rows.append((tag, int(matched.group(1).replace(",", "")) if matched else 0))
return tuple(rows)
- #: dapi 的 tag 接口用数字表示分类。名字取站方自己在详情页 `#tag-sidebar` 上用的
- #: 那套(`_detail` 读的就是它),两条路认出来的类型才是同一个词。
+ #: dapi 的 tag 接口用数字表示分类。名字取站方帖子页 `#tag-sidebar` 上用的那套,
+ #: 与 `_TAG_INFO_TYPES` 归一后的词相同,几条路认出来的类型才是同一个词。
_TAG_TYPE_NAMES = {0: "general", 1: "artist", 3: "copyright",
4: "character", 5: "metadata"}
#: dapi 的 tag 接口只回 XML,`json=1` 实测被忽略。
@@ -1795,13 +1909,8 @@ class Rule34XxxConnector(_BaseConnector):
tag = (name or "").strip()
if not tag or self.credential is None:
return ""
- user_id, api_key = self.credential.require("user_id", "api_key")
- query = urllib.parse.urlencode({
- "page": "dapi", "s": "tag", "q": "index", "name": tag,
- "user_id": user_id, "api_key": api_key,
- })
# 查询串里带 api_key,和 `fetch` 一样绝不落进日志或证据。
- response = self._get(f"https://api.rule34.xxx/index.php?{query}",
+ response = self._get(self._dapi_url(s="tag", q="index", name=tag),
headers={"Accept": "application/xml"})
self._check_status(response)
body = response.body.decode("utf-8", errors="replace")
@@ -1817,60 +1926,32 @@ class Rule34XxxConnector(_BaseConnector):
"video", "sound", "animated", "mp4", "webm", "3d", "hd", "60fps",
"tagme", "highres", "absurdres",
})
- #: 详情页取不到时的退避节奏,与外网退避规则同一套。
- _DETAIL_RETRY_DELAYS = (1.5, 4.0, 9.0)
- #: 详情页 `#stats` 里的上传时间:`Posted: 2026-09-11 00:48:20 by mhzw666`。
- _POSTED_RE = re.compile(r"Posted:\s*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})")
-
def _detail(self, post_id: str) -> dict[str, object]:
- """帖子详情页上才有的两样:标签分类和上传时间。取不到就是空字典。
+ """单条帖子的标签分类和上传时间,供存量行补齐。取不到的那一样不出现在结果里。
- dapi 只给一串扁平标签,分类要读详情页 `#tag-sidebar`,取不到就不给,不按词形
- 猜。dapi 也不给上传时间:它的 `change` 是最后一次修改,实测 17361475 在
- 2026-05-01 上传、09-14 被改过标签,`change` 就是 09-14。详情页 `#stats` 的
- Posted 才是上传时间;没被改过的帖子两者逐秒相同,所以它和 `change` 一样是 UTC。
+ 列表请求一页问齐这两样(`fetch`),这里按帖子号各问一次:JSON 带
+ `fields=tag_info` 给分类,XML 给 `created_at`。dapi 列表的 `change` 是最后一次
+ 修改:实测 17361475 在 2026-05-01 上传、09-14 被改过标签,`change` 就是 09-14。
+ 被限流挡回来时退避重试,仍取不到就缺那一样,不按词形猜,也不写空冒充已补。
"""
- url = ("https://rule34.xxx/index.php?page=post&s=view&id="
- f"{urllib.parse.quote(post_id)}")
- # 站方公布的口径是每 60 秒 60 次,而列表页一页 24 条、每条都要单独打一次
- # 详情页,被挡回来是常态。一次挡回来就返回 {},得到的是「这条没有类型」——
- # 和「这条确实没有类型」在日志里长得一模一样。实测:回填首轮 400 条里
- # 372 条判成「没有类型」,事后逐条直取,全部 200 且 `#tag-sidebar` 正常。
- response = None
- for delay in (0.0, *self._DETAIL_RETRY_DELAYS):
- if delay:
- self.sleeper(delay)
+ if not str(post_id).isdigit():
+ return {}
+ result: dict[str, object] = {}
+ response = self._dapi_patiently(
+ self._dapi_url(s="post", q="index", json="1", id=str(post_id),
+ fields="tag_info"), "application/json")
+ if response is not None:
try:
- response = self._get(url, headers={"Accept": "text/html"})
+ posts = self._dapi_posts(response)
except FollowSourceError:
- response = None
- break
- if response.status not in {425, 429}:
- break
- if response is None or response.status != 200:
- return {}
- soup = BeautifulSoup(response.body, "html.parser")
- allowed = {"general", "artist", "copyright", "character", "metadata"}
- result: dict[str, str] = {}
- for row in soup.select("#tag-sidebar li[class*='tag-type-']"):
- tag_type = next((
- value.removeprefix("tag-type-") for value in row.get("class", [])
- if value.startswith("tag-type-")
- ), "")
- if tag_type not in allowed:
- continue
- link = row.select_one("a[href*='page=post'][href*='tags=']")
- if link is None:
- continue
- query = urllib.parse.parse_qs(
- urllib.parse.urlsplit(str(link.get("href") or "")).query)
- name = html.unescape(str((query.get("tags") or [""])[0])).strip()
- if name:
- result[name] = tag_type
- stats = soup.select_one("#stats")
- posted = self._POSTED_RE.search(stats.get_text(" ") if stats is not None else "")
- return {"tag_types": result,
- "published_at": _iso_from_text(posted.group(1)) if posted else None}
+ posts = []
+ tag_types = self._tag_info_types(posts[0].get("tag_info")) if posts else {}
+ if tag_types:
+ result["tag_types"] = tag_types
+ uploaded = self._upload_times(post_id=str(post_id)).get(str(post_id))
+ if uploaded:
+ result["published_at"] = uploaded
+ return result
def _candidate(self, post: dict, tag: str) -> FollowCandidate:
post_id = str(post.get("id") or "")
@@ -1878,6 +1959,7 @@ class Rule34XxxConnector(_BaseConnector):
# 就进 metadata,读取层再转一次就成了双重转义,用户看到的就是 `&#039;`
# 字面量,同一个标签还会和反转义后的写法分裂成两个身份。
tags = html.unescape(str(post.get("tags") or ""))
+ tag_types = self._tag_info_types(post.get("tag_info"))
image = str(post.get("image") or "")
stem = re.sub(r"\.[a-z0-9]{2,5}$", "", image, flags=re.IGNORECASE)
# booru 帖子没有标题。实测 rule34.xxx 的 `image` 全是 32 位十六进制哈希,
@@ -1912,18 +1994,20 @@ class Rule34XxxConnector(_BaseConnector):
# 对图片则是站点选定的样图。gallery-dl 的 booru 抽取器也把这三层
# 作为可配置回退链,不能把最小 preview 固定成 Peach 封面。
thumb_url=str(post.get("sample_url") or post.get("preview_url") or "") or None,
- # `change` 是最后修改时间,只在详情页取不到时占位;上传时间由 `_detail` 给。
+ # `change` 是最后修改时间,只在 XML 的 `created_at` 取不到时占位。
published_at=_iso_from_epoch(post.get("change")),
group_hint=hint,
title_is_name=title_is_name,
- # 列表视图:标签分类要等第二阶段的详情页。
+ # 列表视图:上传时间要等第二阶段。分类已经随 `tag_info` 到了,partial 行
+ # 落库时 metadata 走 `json_patch`,分类照样写进去。
partial=True,
extra={"tag": tag, "tags": tags, "score": post.get("score"),
"source": source, "title_from": title_from,
"preview_url": post.get("preview_url"),
# dapi 随帖给出原文件的宽高;图片墙靠它在图落地前占好比例。
# sample 与 preview 都是等比缩放,比例与原文件一致。
- **_dims_extra(post.get("width"), post.get("height"))},
+ **_dims_extra(post.get("width"), post.get("height")),
+ **({"tag_types": tag_types} if tag_types else {})},
)
@classmethod
diff --git a/src/peach/follow_store.py b/src/peach/follow_store.py
index 60b6b502d..07f17aa07 100644
--- a/src/peach/follow_store.py
+++ b/src/peach/follow_store.py
@@ -179,7 +179,7 @@ def _carry_image_dims(metadata: dict, previous_json: str | None) -> dict:
#: published_at、group_hint 一概不动,metadata 用 `json_patch` 并进去而
#: 不是整块替换——否则跳过第二阶段就等于把上一轮取到的细节抹掉,而抹掉之后
#: 「详情本来就没有」和「这次没去问」在数据里长得一模一样。
-#: 时长只补空:rule34xxx 的时长读自原文件头,帖子页被限流挡回来时它照样取得到,
+#: 时长只补空:rule34xxx 的时长读自原文件头,上传时间被限流挡回来时它照样取得到,
#: 这时候选仍是 partial,已有的时长不覆盖,空着的补上。
_PARTIAL_UPDATE = (
" title=excluded.title, url=excluded.url, version=excluded.version,"
@@ -200,8 +200,8 @@ _ENRICHED_PREDICATES = {
# kemono 系:详情页只用来判「这一帖交付了资源没有」,没有要补的列。判过保留的帖子
# 才会落库,所以行在 ledger 里本身就是判定结果,库里有的帖子不必再探。
"kept": "1",
- # rule34xxx:分类只有帖子页给,时长只有原文件头给,两样都有才算补齐。只看分类的话,
- # 时长功能上线前补过分类的行、文件头那次没读到的行,常规检查永远不会再问。
+ # rule34xxx:分类随 dapi 的 `tag_info` 到,时长只有原文件头给,两样都有才算补齐。
+ # 只看分类的话,时长功能上线前补过分类的行、文件头那次没读到的行,常规检查永远不会再问。
"tag_types_duration": (
"json_extract(metadata_json,'$.tag_types') IS NOT NULL"
" AND (duration IS NOT NULL OR (lower(COALESCE(media_url,'')) NOT LIKE '%.mp4'"
diff --git a/tests/test_follow_sources.py b/tests/test_follow_sources.py
index d15d0a5d6..26e52c3cb 100644
--- a/tests/test_follow_sources.py
+++ b/tests/test_follow_sources.py
@@ -82,13 +82,17 @@ RULE34VIDEO_DETAIL_HTML = b"""<html><body>
<script>video_url: 'https://rule34video.com/get_file/51/aa/4542721.mp4/?v-acctoken=test'</script>
</body></html>"""
-RULE34XXX_DETAIL_HTML = b"""<html><body><ul id="tag-sidebar">
-<li class="tag-type-metadata tag"><a href="index.php?page=wiki&amp;s=list&amp;search=3d">?</a><a href="index.php?page=post&amp;s=list&amp;tags=3d">3d</a></li>
-<li class="tag-type-metadata tag"><a href="index.php?page=wiki&amp;s=list&amp;search=blender">?</a><a href="index.php?page=post&amp;s=list&amp;tags=blender">blender</a></li>
-<li class="tag-type-metadata tag"><a href="index.php?page=wiki&amp;s=list&amp;search=3d_model">?</a><a href="index.php?page=post&amp;s=list&amp;tags=3d_model">3d model</a></li>
-<li class="tag-type-artist tag"><a href="index.php?page=post&amp;s=list&amp;tags=lazyprocrastinator">lazyprocrastinator</a></li>
-<li class="tag-type-general tag"><a href="index.php?page=post&amp;s=list&amp;tags=reverse_cowgirl_position">reverse cowgirl position</a></li>
-</ul></body></html>"""
+# XML 形态取自 2026-11-03 对 api.rule34.xxx 的实测:只有它带 `created_at`,带时区偏移。
+RULE34XXX_XML_FIRST_TWO = (
+ b'<?xml version="1.0" encoding="UTF-8"?><posts count="2" offset="0">'
+ b'<post height="720" score="72" id="18534395" change="1787445373"'
+ b' created_at="Fri May 01 18:32:21 +0200 2026" status="active"/>'
+ b'<post height="720" score="70" id="18534396" change="1787445300"'
+ b' created_at="Sun Aug 23 00:30:00 -0500 2026" status="active"/>'
+ b"</posts>")
+RULE34XXX_XML = RULE34XXX_XML_FIRST_TWO.replace(b"</posts>", (
+ b'<post id="18534397" created_at="Sun Aug 23 00:20:00 +0000 2026"/>'
+ b'<post id="18534398" created_at="Sun Aug 23 00:10:00 +0000 2026"/></posts>'))
# 搜索表单与结果页的结构取自 2026-09-01 对 f95zone.to 的实测:关键词 `Ria_neearts`
# 在 latest_data.php 的五个分类里全为空,站内搜索命中
@@ -523,9 +527,19 @@ class OfficialConnectorTests(unittest.TestCase):
# 形状取自 2026-08-25 对 api.rule34.xxx 的实测响应:顶层是裸列表,`image` 是 32 位
# 十六进制哈希(15/15),`parent_id` 全是 0(15/15),`source` 13/15 有值。
+# `tag_info` 是 `fields=tag_info` 加上的(2026-11-03 实测):普通标签的类型写作 `tag`,
+# 名字和 `tags` 一样是 HTML 转义形态。
RULE34XXX_JSON = json.dumps([
{"id": 18534395, "image": "3df1cbc67e072d6144588d6c80e490ea.mp4", "parent_id": 0,
- "tags": "lazyprocrastinator fiona blush video sound", "change": 1787445373,
+ "tags": "lazyprocrastinator fiona blush video sound miqo&#039;te", "change": 1787445373,
+ "tag_info": [
+ {"count": 120, "type": "artist", "tag": "lazyprocrastinator"},
+ {"count": 900, "type": "character", "tag": "fiona"},
+ {"count": 50000, "type": "tag", "tag": "blush"},
+ {"count": 80000, "type": "metadata", "tag": "video"},
+ {"count": 70000, "type": "metadata", "tag": "sound"},
+ {"count": 20796, "type": "character", "tag": "miqo&#039;te"},
+ {"count": 3, "type": "deprecated", "tag": "old_name"}],
"file_url": "https://api-cdn-mp4.rule34.xxx/images/1232/3df1.mp4",
"sample_url": "https://api-cdn.rule34.xxx/images/1232/3df1.jpg",
"preview_url": "https://api-cdn.rule34.xxx/thumbnails/1232/t_3df1.jpg",
@@ -533,13 +547,16 @@ RULE34XXX_JSON = json.dumps([
"score": 72, "source": "https://lazyprocrast.fanbox.cc/posts/12304831"},
{"id": 18534396, "image": "3bda1572c365b223d8b287c538a38956.mp4", "parent_id": 0,
"tags": "lazyprocrastinator fiona video", "change": 1787445300,
+ "tag_info": [{"count": 120, "type": "artist", "tag": "lazyprocrastinator"}],
"file_url": "https://api-cdn-mp4.rule34.xxx/images/1232/3bda.mp4",
"score": 70, "source": "https://www.fanbox.cc/@lazyprocrast/posts/12304831"},
{"id": 18534397, "image": "93eaeadff5effabe078c738aab85b3bc.mp4", "parent_id": 0,
"tags": "lazyprocrastinator sayuri video", "change": 1787445200,
+ "tag_info": [{"count": 120, "type": "artist", "tag": "lazyprocrastinator"}],
"file_url": "https://api-cdn-mp4.rule34.xxx/images/1232/93ea.mp4", "score": 12},
{"id": 18534398, "image": "fiona_paizuri_nude.mp4", "parent_id": 18534397,
"tags": "lazyprocrastinator fiona nude", "change": 1787445100,
+ "tag_info": [{"count": 120, "type": "artist", "tag": "lazyprocrastinator"}],
"file_url": "https://api-cdn-mp4.rule34.xxx/images/1232/fp.mp4", "score": 9},
]).encode()
@@ -1492,134 +1509,133 @@ class Rule34XxxConnectorTests(unittest.TestCase):
self.assertNotIn("width", by_id["18534396"].extra)
self.assertNotIn("height", by_id["18534396"].extra)
- def test_post_page_taxonomy_is_recorded_without_guessing_from_tag_words(self):
- seen = []
+ @staticmethod
+ def _dapi_transport(*, xml=RULE34XXX_XML, video=None, seen=None):
+ """JSON 请求回列表,XML 请求按 `xml` 回(可调用时逐次取),文件头按 `video` 回。"""
def transport(request, timeout, max_bytes):
- seen.append(request)
- body = RULE34XXX_JSON if request.url.startswith("https://api.rule34.xxx/") \
- else RULE34XXX_DETAIL_HTML
- return HttpResponse(200, {}, body)
+ if seen is not None:
+ seen.append((request, max_bytes))
+ if request.url.startswith("https://api.rule34.xxx/"):
+ query = urllib.parse.parse_qs(urllib.parse.urlsplit(request.url).query)
+ if query.get("json") == ["1"]:
+ return HttpResponse(200, {}, RULE34XXX_JSON)
+ return xml() if callable(xml) else HttpResponse(200, {}, xml)
+ if video is not None and request.url.startswith("https://api-cdn-mp4.rule34.xxx/"):
+ return HttpResponse(206, {}, video)
+ return HttpResponse(404, {}, b"")
+ return transport
- result = Rule34XxxConnector(
- transport=transport, max_items=1,
+ def _fetch(self, transport, *, max_items=1, sleeper=lambda _: None, **options):
+ return Rule34XxxConnector(
+ transport=transport, max_items=max_items, sleeper=sleeper,
credential=Credential("rule34xxx", {"user_id": "42", "api_key": "sekret"}),
- ).fetch("lazyprocrastinator")
- types = result.candidates[0].extra["tag_types"]
- self.assertEqual(types["3d"], "metadata")
- self.assertEqual(types["blender"], "metadata")
- self.assertEqual(types["3d_model"], "metadata")
- self.assertEqual(types["lazyprocrastinator"], "artist")
- self.assertEqual(types["reverse_cowgirl_position"], "general")
- self.assertNotIn("api_key", seen[1].url)
- # 详情页没有 `#stats` 时,dapi 的 `change` 留作占位。
- self.assertEqual(result.candidates[0].published_at, "2026-08-23T00:36:13Z")
-
- def test_upload_time_comes_from_the_post_page_not_the_last_edit(self):
- """dapi 的 `change` 是最后修改时间:17361475 五月上传、九月改过标签就显示成九月。"""
- detail = RULE34XXX_DETAIL_HTML.replace(b"</ul>", (
- b'</ul><div id="stats"><ul><li>Id: 18534395</li>'
- b"<li>Posted: 2026-05-01 16:32:21<br> by <a>rekin3d</a></li></ul></div>"))
+ **options).fetch("lazyprocrastinator")
- def transport(request, timeout, max_bytes):
- body = RULE34XXX_JSON if request.url.startswith("https://api.rule34.xxx/") \
- else detail
- return HttpResponse(200, {}, body)
+ def test_list_request_brings_the_site_taxonomy_without_guessing_from_tag_words(self):
+ """帖子页被 Cloudflare 验证挡住后,分类随列表请求的 `fields=tag_info` 一起到。
- result = Rule34XxxConnector(
- transport=transport, max_items=1,
- credential=Credential("rule34xxx", {"user_id": "42", "api_key": "sekret"}),
- ).fetch("lazyprocrastinator")
- self.assertEqual(result.candidates[0].published_at, "2026-05-01T16:32:21Z")
- self.assertFalse(result.candidates[0].partial)
+ 一页一次请求就拿全,不逐条打帖子页;普通标签在 `tag_info` 里叫 `tag`,
+ 记成读取层与色板认的 general。站方没列进五类的(`deprecated`)不记。
+ """
+ seen = []
+ result = self._fetch(self._dapi_transport(seen=seen))
+ self.assertEqual(result.candidates[0].extra["tag_types"], {
+ "lazyprocrastinator": "artist", "fiona": "character", "blush": "general",
+ "video": "metadata", "sound": "metadata", "miqo'te": "character"})
+ query = urllib.parse.parse_qs(urllib.parse.urlsplit(seen[0][0].url).query)
+ self.assertEqual(query["fields"], ["tag_info"])
+ self.assertTrue(all(request.url.startswith("https://api") for request, _ in seen),
+ "不再打 rule34.xxx 的帖子页")
+ self.assertNotIn("api_key", result.request_url)
+
+ def test_upload_time_comes_from_the_xml_created_at_not_the_last_edit(self):
+ """dapi 的 `change` 是最后修改时间:17361475 五月上传、九月改过标签就显示成九月。
+
+ 只有 XML 给 `created_at`,带时区偏移;按帖子号对回,对不上的那条留 partial。
+ """
+ candidates = self._fetch(self._dapi_transport(xml=RULE34XXX_XML_FIRST_TWO),
+ max_items=3).candidates
+ self.assertEqual(candidates[0].published_at, "2026-05-01T16:32:21Z")
+ self.assertEqual(candidates[1].published_at, "2026-08-23T05:30:00Z")
+ self.assertFalse(candidates[0].partial)
+ # 两次请求之间插进新帖、页错开一格时,第三条不在 XML 里:`change` 占位,
+ # 保持 partial 等下一轮,分类照样带上。
+ self.assertTrue(candidates[2].partial)
+ self.assertEqual(candidates[2].extra["tag_types"], {"lazyprocrastinator": "artist"})
+ self.assertEqual(candidates[2].published_at, "2026-08-23T00:33:20Z")
def test_video_duration_comes_from_the_original_file_head(self):
- """接口和帖子页都不给时长,原文件头的 `mvhd` 有;只取头上一段,不拉整片。"""
+ """接口都不给时长,原文件头的 `mvhd` 有;只取头上一段,不拉整片。"""
video = minimal_mp4(timescale=1000, sample_delta=40, samples=250, keyframe_every=25)
seen = []
-
- def transport(request, timeout, max_bytes):
- seen.append((request, max_bytes))
- if request.url.startswith("https://api.rule34.xxx/"):
- return HttpResponse(200, {}, RULE34XXX_JSON)
- if request.url.startswith("https://api-cdn-mp4.rule34.xxx/"):
- return HttpResponse(206, {}, video)
- return HttpResponse(200, {}, RULE34XXX_DETAIL_HTML)
-
- result = Rule34XxxConnector(
- transport=transport, max_items=1,
- credential=Credential("rule34xxx", {"user_id": "42", "api_key": "sekret"}),
- ).fetch("lazyprocrastinator")
+ result = self._fetch(self._dapi_transport(video=video, seen=seen))
self.assertEqual(result.candidates[0].duration, 10.0)
head, budget = next((request, size) for request, size in seen
if request.url.endswith(".mp4"))
self.assertEqual(head.headers["Range"], f"bytes=0-{budget - 1}")
self.assertLessEqual(budget, 65536)
- def test_a_throttled_post_page_still_brings_the_file_head_duration(self):
- """文件头在另一个主机上,帖子页被挡回来不牵连它:时长照样带上,候选仍是 partial。"""
- video = minimal_mp4(timescale=1000, sample_delta=40, samples=250, keyframe_every=25)
-
- def transport(request, timeout, max_bytes):
- if request.url.startswith("https://api.rule34.xxx/"):
- return HttpResponse(200, {}, RULE34XXX_JSON)
- if request.url.startswith("https://api-cdn-mp4.rule34.xxx/"):
- return HttpResponse(206, {}, video)
- return HttpResponse(503, {}, b"slow down")
+ #: 站方限流页的原句:HTTP 200,正文是一句话而不是 XML。
+ RATE_LIMITED = b"You currently have a limit of 60 requests every 60 second(s)"
- result = Rule34XxxConnector(
- transport=transport, max_items=1, sleeper=lambda _: None,
- credential=Credential("rule34xxx", {"user_id": "42", "api_key": "sekret"}),
- ).fetch("lazyprocrastinator")
- self.assertEqual(result.candidates[0].duration, 10.0)
- self.assertTrue(result.candidates[0].partial)
- self.assertNotIn("tag_types", result.candidates[0].extra)
+ def test_a_rate_limited_upload_time_is_retried_not_read_as_missing(self):
+ """站方口径是每 60 秒 60 次。挡回来一次就放弃的话,上传时间永远停在 `change`。"""
+ replies = [HttpResponse(429, {}, b""), HttpResponse(200, {}, self.RATE_LIMITED),
+ HttpResponse(200, {}, RULE34XXX_XML)]
+ waits = []
+ result = self._fetch(self._dapi_transport(xml=lambda: replies.pop(0)),
+ sleeper=waits.append)
+ # 退避节奏本身也是行为:注入假 sleeper 才能断言它,而且测试不会真的等。
+ self.assertEqual(waits, [1.5, 4.0])
+ self.assertEqual(result.candidates[0].published_at, "2026-05-01T16:32:21Z")
+ self.assertFalse(result.candidates[0].partial)
- def test_a_throttled_detail_page_is_retried_not_read_as_no_types(self):
- """站方公布的是每 60 秒 60 次,而列表页一页 24 条、每条都要单独打一次详情页。
+ def test_an_upload_time_that_stays_throttled_keeps_types_duration_and_partial(self):
+ """重试有界,仍被挡就留 partial:落库不动已有的上传时间,下一轮再问。
- 一次挡回来就返回 {},落进库里就是「这条没有类型」——和「这条确实没有
- 类型」长得一模一样。实测:回填首轮 400 条里 372 条判成「没有类型」,
- 事后逐条直取,全部 200 且 `#tag-sidebar` 正常。
+ 列表那次已经成功,分类与它同一次请求到手,照样带上;文件头在另一个主机上,
+ 也不受牵连。一抛错就连已经到手的列表一起丢了。
"""
- attempts = []
+ video = minimal_mp4(timescale=1000, sample_delta=40, samples=250, keyframe_every=25)
+ waits = []
+ result = self._fetch(self._dapi_transport(xml=self.RATE_LIMITED, video=video),
+ sleeper=waits.append)
+ first = result.candidates[0]
+ self.assertEqual(waits, [1.5, 4.0, 9.0], "退避有界且不会叠乘")
+ self.assertTrue(first.partial,
+ "上传时间没取到就得留着 partial,否则落库会拿 `change` 冲掉上一轮的值")
+ self.assertEqual(first.extra["tag_types"]["lazyprocrastinator"], "artist")
+ self.assertEqual(first.duration, 10.0)
- def transport(request, timeout, max_bytes):
- if request.url.startswith("https://api.rule34.xxx/"):
- return HttpResponse(200, {}, RULE34XXX_JSON)
- if "page=post" not in request.url:
- return HttpResponse(404, {}, b"")
- attempts.append(request.url)
- if len(attempts) < 3:
- return HttpResponse(503, {}, b"slow down")
- return HttpResponse(200, {}, RULE34XXX_DETAIL_HTML)
+ def test_a_page_already_enriched_costs_no_upload_time_request(self):
+ seen = []
+ self._fetch(self._dapi_transport(seen=seen), max_items=4,
+ enrich_skip=("18534395", "18534396", "18534397", "18534398"))
+ self.assertEqual(len(seen), 1, "整页都补过,只有列表那一次请求")
- waits = []
- result = Rule34XxxConnector(
- transport=transport, max_items=1, sleeper=waits.append,
+ def test_one_post_detail_for_the_backfill_asks_dapi_by_id(self):
+ """存量行补齐逐条问:JSON 按帖子号带 `tag_info`,XML 按帖子号给 `created_at`。"""
+ seen = []
+ detail = Rule34XxxConnector(
+ transport=self._dapi_transport(seen=seen), sleeper=lambda _: None,
credential=Credential("rule34xxx", {"user_id": "42", "api_key": "sekret"}),
- ).fetch("lazyprocrastinator")
- self.assertEqual(len(attempts), 3, "被挡回来要重试,不能一次就当没有类型")
- # 退避节奏本身也是行为:注入假 sleeper 才能断言它,而且测试不会真的等 5.5 秒。
- self.assertEqual(waits, [1.0, 2.0])
- self.assertEqual(result.candidates[0].extra["tag_types"]["lazyprocrastinator"],
- "artist")
-
- def test_a_detail_page_that_stays_unavailable_records_no_types(self):
- """重试完仍取不到就留空,让下一轮再问;不写空字典冒充已补。"""
+ )._detail("18534395")
+ self.assertEqual(detail["published_at"], "2026-05-01T16:32:21Z")
+ self.assertEqual(detail["tag_types"]["miqo'te"], "character")
+ for request, _ in seen:
+ query = urllib.parse.parse_qs(urllib.parse.urlsplit(request.url).query)
+ self.assertEqual(query["id"], ["18534395"])
+
+ def test_one_post_detail_that_stays_throttled_has_no_types(self):
+ """被挡回来不能读成「这条没有分类」:结果里不出现 `tag_types`,脚本记未取得。"""
def transport(request, timeout, max_bytes):
- if request.url.startswith("https://api.rule34.xxx/"):
- return HttpResponse(200, {}, RULE34XXX_JSON)
- return HttpResponse(503, {}, b"slow down")
+ return HttpResponse(200, {}, self.RATE_LIMITED)
- waits = []
- result = Rule34XxxConnector(
- transport=transport, max_items=1, sleeper=waits.append,
+ detail = Rule34XxxConnector(
+ transport=transport, sleeper=lambda _: None,
credential=Credential("rule34xxx", {"user_id": "42", "api_key": "sekret"}),
- ).fetch("lazyprocrastinator")
- self.assertNotIn("tag_types", result.candidates[0].extra)
- self.assertTrue(result.candidates[0].partial,
- "详情没取到就得留着 partial,否则落库会把上一轮的分类抹掉")
- self.assertEqual(waits, [1.0, 2.0, 4.0, 8.0], "退避有界且不会叠乘")
+ )._detail("18534395")
+ self.assertEqual(detail, {})
def test_autocomplete_reports_the_real_tag_spelling_without_credentials(self):
# 站上写作 `ria-neearts`,手边的手柄是 `Ria_neearts`;补全是唯一能把两者
@@ -1661,6 +1677,7 @@ class Rule34XxxConnectorTests(unittest.TestCase):
self.assertEqual(query["tags"], ["stellar_blade 3d sort:score"])
self.assertEqual(query["limit"], ["3"])
self.assertNotIn("pid", query, "一次性的搜索只要第一页")
+ self.assertNotIn("fields", query, "不落库的搜索用不上分类,不多拉四倍正文")
self.assertEqual(len(candidates), 3)
self.assertEqual(candidates[0].thumb_url,
"https://api-cdn.rule34.xxx/images/1232/3df1.jpg",
@@ -2330,12 +2347,16 @@ class EnrichPhaseTests(unittest.TestCase):
补齐过的条目不再问——否则一个作者的来源每次检查都要按条目数付请求。
"""
+ #: rule34xxx 第二阶段逐条付费的请求是原文件头;上传时间整页一次,不随条目数长。
+ HEAD = "https://api-cdn-mp4.rule34.xxx/"
+
def _rule34xxx(self, seen, **kwargs):
def transport(request, timeout, max_bytes):
seen.append(request.url)
if request.url.startswith("https://api.rule34.xxx/index.php"):
- return HttpResponse(200, {}, RULE34XXX_JSON)
- return HttpResponse(200, {}, RULE34XXX_DETAIL_HTML)
+ return HttpResponse(200, {}, RULE34XXX_JSON if "json=1" in request.url
+ else RULE34XXX_XML)
+ return HttpResponse(404, {}, b"")
return Rule34XxxConnector(
transport=transport, max_items=4,
@@ -2347,7 +2368,7 @@ class EnrichPhaseTests(unittest.TestCase):
result = self._rule34xxx(seen, enrich_budget=2).fetch("lazyprocrastinator")
self.assertEqual(len(result.candidates), 4)
self.assertEqual(result.probed, 2)
- details = [url for url in seen if "s=view" in url]
+ details = [url for url in seen if url.startswith(self.HEAD)]
self.assertEqual(len(details), 2, "额度是请求数的上限,不是建议值")
self.assertEqual([item.partial for item in result.candidates],
[False, False, True, True],
@@ -2356,7 +2377,7 @@ class EnrichPhaseTests(unittest.TestCase):
def test_a_zero_budget_means_the_list_phase_alone(self):
seen = []
result = self._rule34xxx(seen, enrich_budget=0).fetch("lazyprocrastinator")
- self.assertEqual(seen, [url for url in seen if "s=view" not in url])
+ self.assertEqual(len(seen), 1, "额度为零连整页的上传时间都不问")
self.assertEqual(result.probed, 0)
self.assertTrue(all(item.partial for item in result.candidates))
@@ -2367,7 +2388,7 @@ class EnrichPhaseTests(unittest.TestCase):
def publish(candidate):
ready.append(candidate)
enriched = sum(not item.partial for item in ready)
- self.assertEqual(sum('s=view' in url for url in seen), enriched)
+ self.assertEqual(sum(url.startswith(self.HEAD) for url in seen), enriched)
connector.candidate_ready = publish
result = connector.fetch('lazyprocrastinator')
@@ -2381,8 +2402,9 @@ class EnrichPhaseTests(unittest.TestCase):
result = self._rule34xxx(
seen, enrich_budget=2, enrich_skip={"18534395", "18534396"},
).fetch("lazyprocrastinator")
- details = sorted(url.rsplit("id=", 1)[1] for url in seen if "s=view" in url)
- self.assertEqual(details, ["18534397", "18534398"])
+ details = sorted(url for url in seen if url.startswith(self.HEAD))
+ self.assertEqual(details, [self.HEAD + "images/1232/93ea.mp4",
+ self.HEAD + "images/1232/fp.mp4"])
self.assertEqual([item.partial for item in result.candidates],
[True, True, False, False])
@@ -2412,8 +2434,8 @@ class EnrichPhaseTests(unittest.TestCase):
def test_rule34xxx_is_not_marked_by_a_time_the_list_already_gave(self):
"""rule34xxx 的上传时间来自列表的 `change`,第一次落库就有值。
- 拿 `published_at` 当它的补齐判据,等于宣布「所有条目都已补齐」,上一轮被
- 限流挡掉的分类就永远补不回来了。
+ 拿 `published_at` 当它的补齐判据,等于宣布「所有条目都已补齐」,没读到时长的
+ mp4 就永远补不回来了。分类随列表一起到,不走第二阶段也照样带上。
"""
self.assertNotEqual(Rule34XxxConnector.ENRICHED_MARK, "published_at")
listed = Rule34XxxConnector(
@@ -2421,7 +2443,7 @@ class EnrichPhaseTests(unittest.TestCase):
credential=Credential("rule34xxx", {"user_id": "42", "api_key": "sekret"}),
).fetch("lazyprocrastinator").candidates
self.assertTrue(all(item.published_at for item in listed))
- self.assertTrue(all("tag_types" not in item.extra for item in listed))
+ self.assertTrue(all(item.partial and item.extra["tag_types"] for item in listed))
class MediaContentHashTests(unittest.TestCase):
diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index e52e5ad78..8b40736b7 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -1618,7 +1618,7 @@ class OperationalScriptTests(unittest.TestCase):
self.assertEqual(self.scrape_codes.SETTLED_ERROR_KINDS, frozenset({"not_found"}))
def test_rule34_detail_backfill_reuses_the_connector_and_is_resumable(self):
- """标签类型、上传时间在帖子页上,视频时长在原文件头里,存量行靠常规检查补不到。
+ """标签类型、上传时间要另问 dapi,视频时长在原文件头里,存量行靠常规检查补不到。
抓取判据不重写,直接复用连接器的 `_detail` 与 `_video_seconds`;备份先做、
分批提交,续跑只跳过上一趟真正写入的条目。
@@ -1646,6 +1646,9 @@ class OperationalScriptTests(unittest.TestCase):
self.assertEqual([row["external_id"] for row in pending], ["18622796", "18622794"])
self.assertEqual([row["external_id"] for row in backfill.pending_rows(connection, 0, {1})],
["18622794"])
+ # 只补类型时只挑还没有非空类型的条目,空表与缺键一样算缺。
+ self.assertEqual([row["external_id"] for row in backfill.pending_rows(
+ connection, 0, missing_types=True)], ["18622794"])
# 只补不抹:已有的类型与时长不覆盖,帖子页没给上传时间就不动它。
typed, untyped = pending
@@ -1673,7 +1676,7 @@ class OperationalScriptTests(unittest.TestCase):
self.assertEqual(backfill.written_ids(out), {2})
def test_rule34_duration_only_backfill_touches_nothing_but_empty_durations(self):
- """授权只覆盖时长时:只挑缺时长的 mp4,不问帖子页,只写 `duration`。"""
+ """授权只覆盖时长时:只挑缺时长的 mp4,不问 dapi,只写 `duration`。"""
backfill = load_script("backfill_rule34_details")
database = self.tmp_ledger()
seeding = sqlite3.connect(database)
@@ -1690,7 +1693,7 @@ class OperationalScriptTests(unittest.TestCase):
class Connector:
def _detail(self, post_id):
- raise AssertionError("只补时长不该打帖子页")
+ raise AssertionError("只补时长不该问 dapi")
def _video_seconds(self, url):
heads.append(url)