Peach: Local-first video metadata scraper and media manager.
refactor(revert): 自动落库撤回的计划与执行挪进 peach.auto_landing_revert
longmeidao050fded901af
diff --git a/scripts/revert_auto_landing.py b/scripts/revert_auto_landing.pyindex 3d3692a67..e435b1a86 100644--- a/scripts/revert_auto_landing.py+++ b/scripts/revert_auto_landing.py@@ -1,45 +1,7 @@-"""把自动落库写下的一批实体结果整批撤回(ADR-0052)。+"""把自动落库写下的一批结果整批撤回(ADR-0052)。-自动落库的每条写入都带归属串和批次号:官网链接记在 `entity_link.metadata_json` 的-`source` 与 `batch`,标识文件记在边车 `.provenance.json` 的同名两项,别名的-`entity_alias.source` 直接就是批次号 `<source>@<任务行 id>`(ADR-0055)。女优资料行的-`performer_profile.source` 同样是批次号,后继登记的站上编号在 `entity_external_ref.metadata_json`-里记 `source` 与 `batch`(ADR-0067),番号样张的 `code_sample_image.source` 也是批次号(ADR-0068),-种子包补的所属事务所 `entity_membership.source` 与 label 的片商 `label_maker.source` 同样(ADR-0073、ADR-0075),-查创作者身份后继写的身份断言 `entity_classification.source` 也是批次号(ADR-0098)。-种子中文规范名按 `entity.metadata_json.name_preference` 的来源和原名快照还原,作品署名同步恢复;后续人工选择保留(ADR-0122)。-判据错了一批,就按它们认出来一起撤掉,不必一条条找。--复核队列的自动否决记在 `review_decision.note` 里(`auto_rejected` 与 `rule`,ADR-0079),-`--source` 给规则名就按它认。决定没有批次号,给了 `--batch` 就一条都不认。撤掉的那几行回到-复核页;下一轮处理任务会按同一条判据再否决一次,所以先改判据再撤。--人批准过的标签上按并集补进来的那几个,`asset_tag.source` 与 `asset_entity.source` 都是批次号-`auto:metadata-tags@<时间>`。撤回删掉这几行标签,并把对应决定 note 里补标签时追加的-`refreshed_candidate_key`、`added_tags` 去掉,那一行重新过期、回到复核页;生词收录后补的-那几个(ADR-0082)连同 `collected_genres` 一起去掉,下一轮按那时的收录结果重补。同样先改判据再撤。--撤回是删除,不是恢复旧值:补厂牌后继只在盘上一张图都没有、账本里一条官网都没有时才写,-补别名后继只写账本里还没有的写法,补女优资料后继只写自动来源的那一行,补样张后继只给还没有-样张的番号写,写下的就是那一格的全部,删掉就回到它写之前的样子。--已消失作品的个人记录接到新文件(ADR-0087)不是删除能撤的:搬运删了旧行,`record_rehome` 存着-旧行快照与搬动的键,批次号 `<source>@<id>`。撤回按快照重建旧行、把搬走的记录改回去,旧行的-id 或路径已被别的行占用时那一批拒绝撤回。登记时自动接回记 `auto:vanished-reattach`,孤儿记录-列表里由人接回记 `user:reattach`。--「想要」清单的入库对账记在 `want_item.acquired_source` 与 `acquired_batch`(批次号-`auto:want-acquired@<登记时刻>`)。撤回把那几条改回待找,查找计数原样留着。--核查差异按确定判据结算的两条(ADR-0139)按规则名撤:`adr-0139-official-replaces-verified`-按 note 的 `previous` 把旧值与旧归属还原,之后被别人改过的资产不动;`adr-0139-verified-tags-union`-删掉那一行补进的标签。两条都删掉决定,`--batch` 给 note 里记的批次号。先改判据再撤。--西方发行方的无番号作品封面(`ASSET-ID-<id>.jpg`)在 `.scraping.json` 边车里记 `source` 与 `batch`-(`auto:western-artwork@<时间>`)。撤回删掉封面与同组边车,作品回到没有封面的样子。--关注作者建档(ADR-0096)建下的创作者在 `entity.metadata_json` 里记 `source` 与 `batch`。撤回先删-这一批的链接与别名,再解绑关注来源;实体还被别的行引用(人挂上的作品、链接)就留着,没有才删。+认哪些写入、怎么撤、哪些不动,全在 `peach.auto_landing_revert`;复核页的「自动写入的资料」+走同一份实现(ADR-0140)。这里只管命令行参数、打印计划和备份。revert_auto_landing.py --source auto:follow-creatorrevert_auto_landing.py --source auto:performer-alias@@ -60,217 +22,15 @@ id 或路径已被别的行占用时那一批拒绝撤回。登记时自动接from __future__ import annotationsimport argparse-import jsonimport sysfrom pathlib import Pathsys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))-from peach import (company_profiles, fc2_sellers, follow_creators, jav_poster_crop, # noqa: E402- record_rehome, sample_images, seed_pack, wants, western_artwork)+from peach import auto_landing_revert # noqa: E402from peach.config import COVER_DIR, GENERATED_DIR # noqa: E402-from peach.metadata_auto_apply import ( # noqa: E402- UNION_TAGS_SOURCE, planned_verified_revert, revert_verified)from peach.scripting import add_ledger_write_args, open_for_write, verify_after_write # noqa: E402-SIDECARS = (".ct", ".provenance.json")-COVER_SIDECARS = (".face.json", jav_poster_crop.SIDECAR_SUFFIX, ".scraping.json")---def matches(record: dict, source: str, batch: str) -> bool:- return record.get("source") == source and (not batch or record.get("batch") == batch)---def planned_links(connection, source: str, batch: str) -> list[dict]:- rows = connection.execute(- "SELECT l.id,l.entity_id,e.canonical_name,l.url,l.metadata_json FROM entity_link l"- " JOIN entity e ON e.id=l.entity_id WHERE l.metadata_json LIKE ?",- (f"%{source}%",)).fetchall()- found = []- for row in rows:- try:- metadata = json.loads(row["metadata_json"] or "{}")- except ValueError:- continue- if matches(metadata, source, batch):- found.append({"id": row["id"], "entity": row["canonical_name"], "url": row["url"],- "batch": metadata.get("batch", "")})- return found---def planned_aliases(connection, source: str, batch: str) -> list[dict]:- """这个来源写下的别名。`source` 列存的是批次号,所以不给批次时按「来源@」前缀认。"""- clause, values = _batch_clause("a.source", source, batch)- return [{"entity_id": row["entity_id"], "entity": row["canonical_name"], "alias": row["alias"],- "normalized": row["normalized_alias"], "source": row["source"]}- for row in connection.execute(- "SELECT a.entity_id,e.canonical_name,a.alias,a.normalized_alias,a.source"- " FROM entity_alias a JOIN entity e ON e.id=a.entity_id WHERE " + clause- + " ORDER BY a.entity_id,a.alias", values)]---def _batch_clause(column: str, source: str, batch: str) -> tuple[str, tuple]:- """`column` 存的是批次号:给了批次就逐字比,不给就按「来源@」前缀认。"""- if batch:- return f"{column}=?", (batch,)- return f"({column}=? OR substr({column},1,?)=?)", (source, len(source) + 1, source + "@")---def planned_profiles(connection, source: str, batch: str) -> list[dict]:- """这个来源写下的女优资料行。"""- clause, values = _batch_clause("p.source", source, batch)- return [{"entity_id": row["entity_id"], "entity": row["canonical_name"], "source": row["source"]}- for row in connection.execute(- "SELECT p.entity_id,e.canonical_name,p.source FROM performer_profile p"- " JOIN entity e ON e.id=p.entity_id WHERE " + clause + " ORDER BY p.entity_id",- values)]---def planned_claims(connection, source: str, batch: str) -> list[dict]:- """这个来源写下的身份断言(`source` 列存批次号)。FC2 卖家那一批连同关系由 `fc2_sellers` 撤。"""- if source == fc2_sellers.SOURCE:- return []- clause, values = _batch_clause("c.source", source, batch)- return [{"entity_id": row["entity_id"], "entity": row["canonical_name"], "facet": row["facet"],- "value": row["value"], "source": row["source"]}- for row in connection.execute(- "SELECT c.entity_id,e.canonical_name,c.facet,c.value,c.source FROM entity_classification c"- " JOIN entity e ON e.id=c.entity_id WHERE " + clause + " ORDER BY c.entity_id,c.facet,c.value",- values)]---def planned_memberships(connection, source: str, batch: str) -> list[dict]:- """这个来源写下的所属事务所(`source` 列存批次号)。"""- clause, values = _batch_clause("m.source", source, batch)- return [{"member_id": row["member_id"], "entity": row["canonical_name"], "agency": row["agency"],- "source": row["source"]}- for row in connection.execute(- "SELECT m.member_id,e.canonical_name,a.canonical_name AS agency,m.source"- " FROM entity_membership m JOIN entity e ON e.id=m.member_id"- " JOIN entity a ON a.id=m.agency_id WHERE " + clause + " ORDER BY m.member_id", values)]---def planned_makers(connection, source: str, batch: str) -> list[dict]:- """这个来源写下的 label 归属片商(`source` 列存批次号)。"""- clause, values = _batch_clause("l.source", source, batch)- return [{"label_id": row["label_id"], "entity": row["canonical_name"], "maker": row["maker"],- "source": row["source"]}- for row in connection.execute(- "SELECT l.label_id,e.canonical_name,m.canonical_name AS maker,l.source"- " FROM label_maker l JOIN entity e ON e.id=l.label_id"- " JOIN entity m ON m.id=l.maker_id WHERE " + clause + " ORDER BY l.label_id", values)]---def planned_refs(connection, source: str, batch: str) -> list[dict]:- """这个来源登记的站上编号(`metadata_json` 里记着 `source` 与 `batch`)。"""- found = []- for row in connection.execute(- "SELECT r.entity_id,e.canonical_name,r.provider,r.external_kind,r.external_id,"- "r.metadata_json FROM entity_external_ref r JOIN entity e ON e.id=r.entity_id"- " WHERE r.metadata_json LIKE ?", (f"%{source}%",)):- try:- metadata = json.loads(row["metadata_json"] or "{}")- except ValueError:- continue- if isinstance(metadata, dict) and matches(metadata, source, batch):- found.append({"entity": row["canonical_name"], "provider": row["provider"],- "kind": row["external_kind"], "id": row["external_id"],- "batch": metadata.get("batch", "")})- return found---def planned_rejections(connection, source: str, batch: str) -> list[dict]:- """这条规则自动写下的复核否决(note 里 `auto_rejected` 为真、`rule` 逐字相同)。-- 用户手工否决的 note 是自由文本或不带 `auto_rejected`,这里认不到,也就不会被撤。- """- if batch:- return []- found = []- for row in connection.execute(- "SELECT category,item_key,note FROM review_decision"- " WHERE status='rejected' AND note LIKE ? ORDER BY category,item_key",- (f"%{source}%",)):- try:- note = json.loads(row["note"] or "{}")- except ValueError:- continue- if (isinstance(note, dict) and note.get("auto_rejected") is True- and note.get("rule") == source):- found.append({"category": row["category"], "item_key": row["item_key"]})- return found---def planned_tags(connection, source: str, batch: str) -> list[dict]:- """人批准过的标签上按并集补进来的那几个(`asset_entity.source` 存批次号)。-- `item_key` 取自实体关系留痕里的 `review_item`:撤掉标签的同时,那一行决定的 note- 要去掉 `refreshed_candidate_key` 与 `added_tags`,否则它不再过期,也就回不到复核页。-- 只认并集补标签这一个归属串:标签的来源列还存着扫描、刮削写下的 `name`、`r18` 等,- 给别的 `--source` 就按它整批删,删掉的是不归自动落库管的标签。- """- if source != UNION_TAGS_SOURCE:- return []- clause, values = _batch_clause("ae.source", source, batch)- found = []- for row in connection.execute(- "SELECT ae.asset_id,ae.entity_id,e.canonical_name,ae.source,ae.metadata_json"- " FROM asset_entity ae JOIN entity e ON e.id=ae.entity_id"- " WHERE ae.role='tag' AND " + clause + " ORDER BY ae.asset_id,e.canonical_name",- values):- try:- metadata = json.loads(row["metadata_json"] or "{}")- except ValueError:- metadata = {}- found.append({"asset_id": row["asset_id"], "entity_id": row["entity_id"],- "tag": row["canonical_name"], "source": row["source"],- "item_key": str(metadata.get("review_item") or "")- if isinstance(metadata, dict) else ""})- return found---def reopen_extended_decisions(connection, item_keys: set[str]) -> int:- """并集补标签的那几行决定去掉补标签时追加的几项,其余原样,返回改了几行。"""- changed = 0- for item_key in sorted(key for key in item_keys if key):- row = connection.execute(- "SELECT note FROM review_decision WHERE category='metadata_fields' AND item_key=?",- (item_key,)).fetchone()- try:- note = json.loads(row["note"] or "{}") if row else None- except ValueError:- note = None- if not isinstance(note, dict) or "added_tags" not in note:- continue- kept = {key: value for key, value in note.items()- if key not in {"added_tags", "refreshed_candidate_key", "collected_genres"}}- connection.execute(- "UPDATE review_decision SET note=? WHERE category='metadata_fields' AND item_key=?",- (json.dumps(kept, ensure_ascii=False, separators=(",", ":")), item_key))- changed += 1- return changed---def planned_files(logo_root: Path, source: str, batch: str) -> list[Path]:- """边车上写着这个来源的标识文件本体。"""- found = []- for sidecar in sorted(logo_root.glob("*.provenance.json")):- try:- record = json.loads(sidecar.read_text(encoding="utf-8"))- except (OSError, ValueError):- continue- if matches(record, source, batch):- found.append(sidecar.with_name(sidecar.name.removesuffix(".provenance.json")))- return found---def print_acquired(planned: list[dict]) -> None:- """想要清单里对账标了已入库的那几条,一条一行。"""- for item in planned:- label = item["code"] or item["title"] or f"#{item['id']}"- print(f" - 想要 {label[:40]:<40} 资产 {item['acquired_asset_id']} {item['acquired_batch']}")-def build_parser() -> argparse.ArgumentParser:parser = argparse.ArgumentParser(description=__doc__)@@ -283,6 +43,13 @@ def build_parser() -> argparse.ArgumentParser:return parser+def print_acquired(planned: list[dict]) -> None:+ """想要清单里对账标了已入库的那几条,一条一行。"""+ for item in planned:+ label = item["code"] or item["title"] or f"#{item['id']}"+ print(f" - 想要 {label[:40]:<40} 资产 {item['acquired_asset_id']} {item['acquired_batch']}")++def print_verified(planned: list[dict]) -> None:"""核查结算的撤回清单:覆盖写还原几条资产、几条已被改过不动,并集删几个标签。"""for entry in planned:@@ -310,136 +77,56 @@ def print_files(files: list[Path], covers: list[Path]) -> None:print(f" - 封面 {path.name}")-def remove_files(files: list[Path], covers: list[Path]) -> int:- """删掉标识文件与作品封面,连同各自的同组边车;返回删掉的文件数。"""- targets = [target for path in files- for target in (path, *(path.with_name(path.name + suffix) for suffix in SIDECARS))]- targets += [target for path in covers- for target in (path, *(path.with_suffix(suffix) for suffix in COVER_SIDECARS))]- removed = 0- for target in targets:- if target.exists():- target.unlink()- removed += 1- return removed+def print_plan(planned: dict) -> None:+ print_acquired(planned["acquired"])+ print_verified(planned["verified"])+ for link in planned["links"]:+ print(f" - 链接 {link['entity'][:20]:<20} {link['url'][:56]} {link['batch']}")+ for alias in planned["aliases"]:+ print(f" - 别名 {alias['entity'][:20]:<20} {alias['alias'][:40]} {alias['source']}")+ for name in planned["names"]:+ print(f" - 中文名 {name['entity']} → {name['previous_name']} {name['source']}")+ print_profiles(planned["profiles"], planned["companies"])+ for creator in planned["creators"]:+ print(f" - 建档 {creator['name'][:20]:<20} {creator['sources']} 个来源 {creator['batch']}")+ for claim in planned["claims"]:+ print(f" - 断言 {claim['entity'][:20]:<20} {claim['facet']}={claim['value']} {claim['source']}")+ for ref in planned["refs"]:+ print(f" - 编号 {ref['entity'][:20]:<20} {ref['provider']} {ref['id']} {ref['batch']}")+ for membership in planned["memberships"]:+ print(f" - 归属 {membership['entity'][:20]:<20} {membership['agency'][:30]} {membership['source']}")+ for maker in planned["makers"]:+ print(f" - 片商 {maker['entity'][:20]:<20} {maker['maker'][:30]} {maker['source']}")+ print_files(planned["files"], planned["covers"])+ for sample in planned["samples"]:+ print(f" - 样张 {sample['code']:<20} {sample['count']} 张 {sample['source']}")+ for rejection in planned["rejections"]:+ print(f" - 否决 {rejection['category']} {rejection['item_key']}")+ for tag in planned["tags"]:+ print(f" - 标签 {tag['asset_id']:<8} {tag['tag'][:30]:<30} {tag['source']}")+ for rehome in planned["rehomes"]:+ print(f" - 接回 {rehome['old_asset_id']:<8} → {rehome['new_asset_id']:<8} "+ f"{rehome['name'][:40]} {rehome['batch']}")+ print(auto_landing_revert.counts(planned))def main(argv: list[str] | None = None) -> int:args = build_parser().parse_args(argv)connection = open_for_write(args)try:- links = planned_links(connection, args.source, args.batch)- aliases = planned_aliases(connection, args.source, args.batch)- names = seed_pack.planned_names(connection, args.source, args.batch)- profiles = planned_profiles(connection, args.source, args.batch)- claims = planned_claims(connection, args.source, args.batch)- companies = company_profiles.planned_revert(connection, args.source, args.batch)- sellers = fc2_sellers.planned_revert(connection, args.source, args.batch)- creators = follow_creators.planned_revert(connection, args.source, args.batch)- refs = planned_refs(connection, args.source, args.batch)- memberships = planned_memberships(connection, args.source, args.batch)- makers = planned_makers(connection, args.source, args.batch)- files = planned_files(args.logo_root, args.source, args.batch)- covers = western_artwork.planned_covers(args.cover_root, args.source, args.batch)- samples = sample_images.planned_revert(connection, args.source, args.batch)- rejections = planned_rejections(connection, args.source, args.batch)- tags = planned_tags(connection, args.source, args.batch)- rehomes = record_rehome.planned_revert(connection, args.source, args.batch)- acquired = wants.planned_revert(connection, args.source, args.batch)- verified = planned_verified_revert(connection, args.source, args.batch)- print_acquired(acquired)- print_verified(verified)- for link in links:- print(f" - 链接 {link['entity'][:20]:<20} {link['url'][:56]} {link['batch']}")- for alias in aliases:- print(f" - 别名 {alias['entity'][:20]:<20} {alias['alias'][:40]} {alias['source']}")- for name in names:- print(f" - 中文名 {name['entity']} → {name['previous_name']} {name['source']}")- print_profiles(profiles, companies)- for creator in creators:- print(f" - 建档 {creator['name'][:20]:<20} {creator['sources']} 个来源 {creator['batch']}")- for claim in claims:- print(f" - 断言 {claim['entity'][:20]:<20} {claim['facet']}={claim['value']} {claim['source']}")- for ref in refs:- print(f" - 编号 {ref['entity'][:20]:<20} {ref['provider']} {ref['id']} {ref['batch']}")- for membership in memberships:- print(f" - 归属 {membership['entity'][:20]:<20} {membership['agency'][:30]} {membership['source']}")- for maker in makers:- print(f" - 片商 {maker['entity'][:20]:<20} {maker['maker'][:30]} {maker['source']}")- print_files(files, covers)- for sample in samples:- print(f" - 样张 {sample['code']:<20} {sample['count']} 张 {sample['source']}")- for rejection in rejections:- print(f" - 否决 {rejection['category']} {rejection['item_key']}")- for tag in tags:- print(f" - 标签 {tag['asset_id']:<8} {tag['tag'][:30]:<30} {tag['source']}")- for rehome in rehomes:- print(f" - 接回 {rehome['old_asset_id']:<8} → {rehome['new_asset_id']:<8} "- f"{rehome['name'][:40]} {rehome['batch']}")- print({"链接": len(links), "别名": len(aliases), "资料": len(profiles), "断言": len(claims),- "公司资料": len(companies), "卖家": len(sellers), "关注建档": len(creators), "编号": len(refs),- "归属": len(memberships), "片商": len(makers), "标识文件": len(files), "封面": len(covers),- "样张": sum(sample["count"] for sample in samples), "否决": len(rejections),- "标签": len(tags), "接回": len(rehomes), "想要入库": len(acquired), "中文名": len(names),- "核查结算": len(verified)})+ planned = auto_landing_revert.plan(connection, args.source, args.batch,+ logo_root=args.logo_root, cover_root=args.cover_root)+ print_plan(planned)if not args.apply:print("dry-run;确认无误后加 --apply --backup <路径>")return 0with connection:- restored_names = seed_pack.revert_names(connection, names)- company_profiles.revert(connection, companies)- connection.executemany("DELETE FROM entity_link WHERE id=?",- [(link["id"],) for link in links])- connection.executemany(- "DELETE FROM entity_alias WHERE entity_id=? AND normalized_alias=? AND source=?",- [(alias["entity_id"], alias["normalized"], alias["source"]) for alias in aliases])- connection.executemany(- "DELETE FROM performer_profile WHERE entity_id=? AND source=?",- [(profile["entity_id"], profile["source"]) for profile in profiles])- connection.executemany(- "DELETE FROM entity_classification WHERE entity_id=? AND facet=? AND value=? AND source=?",- [(claim["entity_id"], claim["facet"], claim["value"], claim["source"]) for claim in claims])- connection.executemany(- "DELETE FROM entity_external_ref WHERE provider=? AND external_kind=?"- " AND external_id=?", [(ref["provider"], ref["kind"], ref["id"]) for ref in refs])- connection.executemany(- "DELETE FROM entity_membership WHERE member_id=? AND source=?",- [(membership["member_id"], membership["source"]) for membership in memberships])- connection.executemany(- "DELETE FROM label_maker WHERE label_id=? AND source=?",- [(maker["label_id"], maker["source"]) for maker in makers])- removed_samples = sample_images.revert(connection, args.source, args.batch)- connection.executemany(- "DELETE FROM review_decision WHERE category=? AND item_key=? AND status='rejected'",- [(rejection["category"], rejection["item_key"]) for rejection in rejections])- connection.executemany(- "DELETE FROM asset_entity WHERE asset_id=? AND entity_id=? AND role='tag'"- " AND source=?", [(tag["asset_id"], tag["entity_id"], tag["source"]) for tag in tags])- # 扁平投影只删计划里列出的那几行:按来源串整表删的话,`--source name` 这类- # 扫描写下的标签来源也会被认成一批。- removed_tag_rows = sum(connection.execute(- "DELETE FROM asset_tag WHERE asset_id=? AND tag=? AND source=?",- (tag["asset_id"], tag["tag"], tag["source"])).rowcount or 0 for tag in tags)- reopened = reopen_extended_decisions(connection, {tag["item_key"] for tag in tags})- restored = record_rehome.revert(connection, args.source, args.batch)- reopened_wants = wants.revert(connection, args.source, args.batch)- removed_seller_relations = fc2_sellers.revert(connection, sellers)- # 链接与别名已在上面按批次删掉,剩下没有别的引用的实体才删得掉。- creator_revert = follow_creators.revert(connection, creators)- verified_revert = revert_verified(connection, verified)+ result = auto_landing_revert.apply(connection, planned)integrity, orphans = verify_after_write(connection)finally:connection.close()- removed = remove_files(files, covers)- print({"删除链接": len(links), "删除别名": len(aliases), "删除资料": len(profiles), "删除断言": len(claims),- "删除编号": len(refs), "删除归属": len(memberships), "删除片商": len(makers),- "删除样张": removed_samples, "删除否决": len(rejections),- "删除标签": len(tags), "删除扁平标签": removed_tag_rows, "重开决定": reopened,- "撤回接回": restored, "想要回到待找": reopened_wants, "卖家关系": removed_seller_relations,- "解绑关注来源": creator_revert["unbound"], "删除建档实体": creator_revert["removed"], "删除文件": removed,- "还原中文名": restored_names, "还原核查覆盖": verified_revert["restored"],- "删除核查并集标签": verified_revert["removed_tags"], "删除核查决定": verified_revert["decisions"],- "integrity_check": integrity, "foreign_key_check": orphans})+ result["删除文件"] = auto_landing_revert.remove_files(planned)+ print({**result, "integrity_check": integrity, "foreign_key_check": orphans})return 0diff --git a/src/peach/auto_landing_revert.py b/src/peach/auto_landing_revert.pynew file mode 100644index 000000000..3c40e2eb4--- /dev/null+++ b/src/peach/auto_landing_revert.py@@ -0,0 +1,357 @@+"""把自动落库写下的一批结果整批撤回(ADR-0052):命令行与复核页共用的同一份判据。++自动落库的每条写入都带归属串和批次号:官网链接记在 `entity_link.metadata_json` 的+`source` 与 `batch`,标识文件记在边车 `.provenance.json` 的同名两项,别名的+`entity_alias.source` 直接就是批次号 `<source>@<任务行 id>`(ADR-0055)。女优资料行的+`performer_profile.source` 同样是批次号,后继登记的站上编号在 `entity_external_ref.metadata_json`+里记 `source` 与 `batch`(ADR-0067),番号样张的 `code_sample_image.source` 也是批次号(ADR-0068),+种子包补的所属事务所 `entity_membership.source` 与 label 的片商 `label_maker.source` 同样(ADR-0073、ADR-0075),+查创作者身份后继写的身份断言 `entity_classification.source` 也是批次号(ADR-0098)。+种子中文规范名按 `entity.metadata_json.name_preference` 的来源和原名快照还原,作品署名同步恢复;后续人工选择保留(ADR-0122)。+判据错了一批,就按它们认出来一起撤掉,不必一条条找。++复核队列的自动否决记在 `review_decision.note` 里(`auto_rejected` 与 `rule`,ADR-0079),+`source` 给规则名就按它认。决定没有批次号,给了 `batch` 就一条都不认。撤掉的那几行回到+复核页;下一轮处理任务会按同一条判据再否决一次,所以先改判据再撤。++人批准过的标签上按并集补进来的那几个,`asset_tag.source` 与 `asset_entity.source` 都是批次号+`auto:metadata-tags@<时间>`。撤回删掉这几行标签,并把对应决定 note 里补标签时追加的+`refreshed_candidate_key`、`added_tags` 去掉,那一行重新过期、回到复核页;生词收录后补的+那几个(ADR-0082)连同 `collected_genres` 一起去掉,下一轮按那时的收录结果重补。同样先改判据再撤。++撤回是删除,不是恢复旧值:补厂牌后继只在盘上一张图都没有、账本里一条官网都没有时才写,+补别名后继只写账本里还没有的写法,补女优资料后继只写自动来源的那一行,补样张后继只给还没有+样张的番号写,写下的就是那一格的全部,删掉就回到它写之前的样子。++已消失作品的个人记录接到新文件(ADR-0087)不是删除能撤的:搬运删了旧行,`record_rehome` 存着+旧行快照与搬动的键,批次号 `<source>@<id>`。撤回按快照重建旧行、把搬走的记录改回去,旧行的+id 或路径已被别的行占用时那一批拒绝撤回。登记时自动接回记 `auto:vanished-reattach`,孤儿记录+列表里由人接回记 `user:reattach`。++「想要」清单的入库对账记在 `want_item.acquired_source` 与 `acquired_batch`(批次号+`auto:want-acquired@<登记时刻>`)。撤回把那几条改回待找,查找计数原样留着。++核查差异按确定判据结算的两条(ADR-0139)按规则名撤:`adr-0139-official-replaces-verified`+按 note 的 `previous` 把旧值与旧归属还原,之后被别人改过的资产不动;`adr-0139-verified-tags-union`+删掉那一行补进的标签。两条都删掉决定,`batch` 给 note 里记的批次号。先改判据再撤。+复核页的「自动写入的资料」只列这两条规则的批次(ADR-0140),撤回走这里同一个 `plan` 与 `apply`。++西方发行方的无番号作品封面(`ASSET-ID-<id>.jpg`)在 `.scraping.json` 边车里记 `source` 与 `batch`+(`auto:western-artwork@<时间>`)。撤回删掉封面与同组边车,作品回到没有封面的样子。++关注作者建档(ADR-0096)建下的创作者在 `entity.metadata_json` 里记 `source` 与 `batch`。撤回先删+这一批的链接与别名,再解绑关注来源;实体还被别的行引用(人挂上的作品、链接)就留着,没有才删。++`plan` 只读,`apply` 只写账本、在调用方的事务里跑,`remove_files` 等事务提交之后再删盘上文件。+"""+from __future__ import annotations++import json+from pathlib import Path++from . import (company_profiles, fc2_sellers, follow_creators, jav_poster_crop, record_rehome,+ sample_images, seed_pack, wants, western_artwork)+from .metadata_auto_apply import UNION_TAGS_SOURCE, planned_verified_revert, revert_verified++SIDECARS = (".ct", ".provenance.json")+COVER_SIDECARS = (".face.json", jav_poster_crop.SIDECAR_SUFFIX, ".scraping.json")+++def matches(record: dict, source: str, batch: str) -> bool:+ return record.get("source") == source and (not batch or record.get("batch") == batch)+++def planned_links(connection, source: str, batch: str) -> list[dict]:+ rows = connection.execute(+ "SELECT l.id,l.entity_id,e.canonical_name,l.url,l.metadata_json FROM entity_link l"+ " JOIN entity e ON e.id=l.entity_id WHERE l.metadata_json LIKE ?",+ (f"%{source}%",)).fetchall()+ found = []+ for row in rows:+ try:+ metadata = json.loads(row["metadata_json"] or "{}")+ except ValueError:+ continue+ if matches(metadata, source, batch):+ found.append({"id": row["id"], "entity": row["canonical_name"], "url": row["url"],+ "batch": metadata.get("batch", "")})+ return found+++def planned_aliases(connection, source: str, batch: str) -> list[dict]:+ """这个来源写下的别名。`source` 列存的是批次号,所以不给批次时按「来源@」前缀认。"""+ clause, values = _batch_clause("a.source", source, batch)+ return [{"entity_id": row["entity_id"], "entity": row["canonical_name"], "alias": row["alias"],+ "normalized": row["normalized_alias"], "source": row["source"]}+ for row in connection.execute(+ "SELECT a.entity_id,e.canonical_name,a.alias,a.normalized_alias,a.source"+ " FROM entity_alias a JOIN entity e ON e.id=a.entity_id WHERE " + clause+ + " ORDER BY a.entity_id,a.alias", values)]+++def _batch_clause(column: str, source: str, batch: str) -> tuple[str, tuple]:+ """`column` 存的是批次号:给了批次就逐字比,不给就按「来源@」前缀认。"""+ if batch:+ return f"{column}=?", (batch,)+ return f"({column}=? OR substr({column},1,?)=?)", (source, len(source) + 1, source + "@")+++def planned_profiles(connection, source: str, batch: str) -> list[dict]:+ """这个来源写下的女优资料行。"""+ clause, values = _batch_clause("p.source", source, batch)+ return [{"entity_id": row["entity_id"], "entity": row["canonical_name"], "source": row["source"]}+ for row in connection.execute(+ "SELECT p.entity_id,e.canonical_name,p.source FROM performer_profile p"+ " JOIN entity e ON e.id=p.entity_id WHERE " + clause + " ORDER BY p.entity_id",+ values)]+++def planned_claims(connection, source: str, batch: str) -> list[dict]:+ """这个来源写下的身份断言(`source` 列存批次号)。FC2 卖家那一批连同关系由 `fc2_sellers` 撤。"""+ if source == fc2_sellers.SOURCE:+ return []+ clause, values = _batch_clause("c.source", source, batch)+ return [{"entity_id": row["entity_id"], "entity": row["canonical_name"], "facet": row["facet"],+ "value": row["value"], "source": row["source"]}+ for row in connection.execute(+ "SELECT c.entity_id,e.canonical_name,c.facet,c.value,c.source FROM entity_classification c"+ " JOIN entity e ON e.id=c.entity_id WHERE " + clause + " ORDER BY c.entity_id,c.facet,c.value",+ values)]+++def planned_memberships(connection, source: str, batch: str) -> list[dict]:+ """这个来源写下的所属事务所(`source` 列存批次号)。"""+ clause, values = _batch_clause("m.source", source, batch)+ return [{"member_id": row["member_id"], "entity": row["canonical_name"], "agency": row["agency"],+ "source": row["source"]}+ for row in connection.execute(+ "SELECT m.member_id,e.canonical_name,a.canonical_name AS agency,m.source"+ " FROM entity_membership m JOIN entity e ON e.id=m.member_id"+ " JOIN entity a ON a.id=m.agency_id WHERE " + clause + " ORDER BY m.member_id", values)]+++def planned_makers(connection, source: str, batch: str) -> list[dict]:+ """这个来源写下的 label 归属片商(`source` 列存批次号)。"""+ clause, values = _batch_clause("l.source", source, batch)+ return [{"label_id": row["label_id"], "entity": row["canonical_name"], "maker": row["maker"],+ "source": row["source"]}+ for row in connection.execute(+ "SELECT l.label_id,e.canonical_name,m.canonical_name AS maker,l.source"+ " FROM label_maker l JOIN entity e ON e.id=l.label_id"+ " JOIN entity m ON m.id=l.maker_id WHERE " + clause + " ORDER BY l.label_id", values)]+++def planned_refs(connection, source: str, batch: str) -> list[dict]:+ """这个来源登记的站上编号(`metadata_json` 里记着 `source` 与 `batch`)。"""+ found = []+ for row in connection.execute(+ "SELECT r.entity_id,e.canonical_name,r.provider,r.external_kind,r.external_id,"+ "r.metadata_json FROM entity_external_ref r JOIN entity e ON e.id=r.entity_id"+ " WHERE r.metadata_json LIKE ?", (f"%{source}%",)):+ try:+ metadata = json.loads(row["metadata_json"] or "{}")+ except ValueError:+ continue+ if isinstance(metadata, dict) and matches(metadata, source, batch):+ found.append({"entity": row["canonical_name"], "provider": row["provider"],+ "kind": row["external_kind"], "id": row["external_id"],+ "batch": metadata.get("batch", "")})+ return found+++def planned_rejections(connection, source: str, batch: str) -> list[dict]:+ """这条规则自动写下的复核否决(note 里 `auto_rejected` 为真、`rule` 逐字相同)。++ 用户手工否决的 note 是自由文本或不带 `auto_rejected`,这里认不到,也就不会被撤。+ """+ if batch:+ return []+ found = []+ for row in connection.execute(+ "SELECT category,item_key,note FROM review_decision"+ " WHERE status='rejected' AND note LIKE ? ORDER BY category,item_key",+ (f"%{source}%",)):+ try:+ note = json.loads(row["note"] or "{}")+ except ValueError:+ continue+ if (isinstance(note, dict) and note.get("auto_rejected") is True+ and note.get("rule") == source):+ found.append({"category": row["category"], "item_key": row["item_key"]})+ return found+++def planned_tags(connection, source: str, batch: str) -> list[dict]:+ """人批准过的标签上按并集补进来的那几个(`asset_entity.source` 存批次号)。++ `item_key` 取自实体关系留痕里的 `review_item`:撤掉标签的同时,那一行决定的 note+ 要去掉 `refreshed_candidate_key` 与 `added_tags`,否则它不再过期,也就回不到复核页。++ 只认并集补标签这一个归属串:标签的来源列还存着扫描、刮削写下的 `name`、`r18` 等,+ 给别的 `source` 就按它整批删,删掉的是不归自动落库管的标签。+ """+ if source != UNION_TAGS_SOURCE:+ return []+ clause, values = _batch_clause("ae.source", source, batch)+ found = []+ for row in connection.execute(+ "SELECT ae.asset_id,ae.entity_id,e.canonical_name,ae.source,ae.metadata_json"+ " FROM asset_entity ae JOIN entity e ON e.id=ae.entity_id"+ " WHERE ae.role='tag' AND " + clause + " ORDER BY ae.asset_id,e.canonical_name",+ values):+ try:+ metadata = json.loads(row["metadata_json"] or "{}")+ except ValueError:+ metadata = {}+ found.append({"asset_id": row["asset_id"], "entity_id": row["entity_id"],+ "tag": row["canonical_name"], "source": row["source"],+ "item_key": str(metadata.get("review_item") or "")+ if isinstance(metadata, dict) else ""})+ return found+++def reopen_extended_decisions(connection, item_keys: set[str]) -> int:+ """并集补标签的那几行决定去掉补标签时追加的几项,其余原样,返回改了几行。"""+ changed = 0+ for item_key in sorted(key for key in item_keys if key):+ row = connection.execute(+ "SELECT note FROM review_decision WHERE category='metadata_fields' AND item_key=?",+ (item_key,)).fetchone()+ try:+ note = json.loads(row["note"] or "{}") if row else None+ except ValueError:+ note = None+ if not isinstance(note, dict) or "added_tags" not in note:+ continue+ kept = {key: value for key, value in note.items()+ if key not in {"added_tags", "refreshed_candidate_key", "collected_genres"}}+ connection.execute(+ "UPDATE review_decision SET note=? WHERE category='metadata_fields' AND item_key=?",+ (json.dumps(kept, ensure_ascii=False, separators=(",", ":")), item_key))+ changed += 1+ return changed+++def planned_files(logo_root: Path, source: str, batch: str) -> list[Path]:+ """边车上写着这个来源的标识文件本体。"""+ found = []+ for sidecar in sorted(logo_root.glob("*.provenance.json")):+ try:+ record = json.loads(sidecar.read_text(encoding="utf-8"))+ except (OSError, ValueError):+ continue+ if matches(record, source, batch):+ found.append(sidecar.with_name(sidecar.name.removesuffix(".provenance.json")))+ return found+++def plan(connection, source: str, batch: str = "", *, logo_root: Path, cover_root: Path) -> dict:+ """这个来源(与批次)要撤的全部条目,只读。`apply` 与 `remove_files` 都按它做。"""+ return {+ "source": source, "batch": batch,+ "links": planned_links(connection, source, batch),+ "aliases": planned_aliases(connection, source, batch),+ "names": seed_pack.planned_names(connection, source, batch),+ "profiles": planned_profiles(connection, source, batch),+ "claims": planned_claims(connection, source, batch),+ "companies": company_profiles.planned_revert(connection, source, batch),+ "sellers": fc2_sellers.planned_revert(connection, source, batch),+ "creators": follow_creators.planned_revert(connection, source, batch),+ "refs": planned_refs(connection, source, batch),+ "memberships": planned_memberships(connection, source, batch),+ "makers": planned_makers(connection, source, batch),+ "files": planned_files(logo_root, source, batch),+ "covers": western_artwork.planned_covers(cover_root, source, batch),+ "samples": sample_images.planned_revert(connection, source, batch),+ "rejections": planned_rejections(connection, source, batch),+ "tags": planned_tags(connection, source, batch),+ "rehomes": record_rehome.planned_revert(connection, source, batch),+ "acquired": wants.planned_revert(connection, source, batch),+ "verified": planned_verified_revert(connection, source, batch),+ }+++def counts(planned: dict) -> dict:+ """计划里每一类各有几条,命令行 dry-run 的那一行摘要。"""+ return {"链接": len(planned["links"]), "别名": len(planned["aliases"]),+ "资料": len(planned["profiles"]), "断言": len(planned["claims"]),+ "公司资料": len(planned["companies"]), "卖家": len(planned["sellers"]),+ "关注建档": len(planned["creators"]), "编号": len(planned["refs"]),+ "归属": len(planned["memberships"]), "片商": len(planned["makers"]),+ "标识文件": len(planned["files"]), "封面": len(planned["covers"]),+ "样张": sum(sample["count"] for sample in planned["samples"]),+ "否决": len(planned["rejections"]), "标签": len(planned["tags"]),+ "接回": len(planned["rehomes"]), "想要入库": len(planned["acquired"]),+ "中文名": len(planned["names"]), "核查结算": len(planned["verified"])}+++def apply(connection, planned: dict) -> dict:+ """按计划改账本,返回各类改了几条。不开不提交事务,由调用方包住;盘上文件不在这里删。"""+ source, batch = planned["source"], planned["batch"]+ links, aliases, profiles, claims = (planned["links"], planned["aliases"], planned["profiles"],+ planned["claims"])+ refs, memberships, makers = planned["refs"], planned["memberships"], planned["makers"]+ rejections, tags = planned["rejections"], planned["tags"]+ restored_names = seed_pack.revert_names(connection, planned["names"])+ company_profiles.revert(connection, planned["companies"])+ connection.executemany("DELETE FROM entity_link WHERE id=?", [(link["id"],) for link in links])+ connection.executemany(+ "DELETE FROM entity_alias WHERE entity_id=? AND normalized_alias=? AND source=?",+ [(alias["entity_id"], alias["normalized"], alias["source"]) for alias in aliases])+ connection.executemany(+ "DELETE FROM performer_profile WHERE entity_id=? AND source=?",+ [(profile["entity_id"], profile["source"]) for profile in profiles])+ connection.executemany(+ "DELETE FROM entity_classification WHERE entity_id=? AND facet=? AND value=? AND source=?",+ [(claim["entity_id"], claim["facet"], claim["value"], claim["source"]) for claim in claims])+ connection.executemany(+ "DELETE FROM entity_external_ref WHERE provider=? AND external_kind=?"+ " AND external_id=?", [(ref["provider"], ref["kind"], ref["id"]) for ref in refs])+ connection.executemany(+ "DELETE FROM entity_membership WHERE member_id=? AND source=?",+ [(membership["member_id"], membership["source"]) for membership in memberships])+ connection.executemany(+ "DELETE FROM label_maker WHERE label_id=? AND source=?",+ [(maker["label_id"], maker["source"]) for maker in makers])+ removed_samples = sample_images.revert(connection, source, batch)+ connection.executemany(+ "DELETE FROM review_decision WHERE category=? AND item_key=? AND status='rejected'",+ [(rejection["category"], rejection["item_key"]) for rejection in rejections])+ connection.executemany(+ "DELETE FROM asset_entity WHERE asset_id=? AND entity_id=? AND role='tag'"+ " AND source=?", [(tag["asset_id"], tag["entity_id"], tag["source"]) for tag in tags])+ # 扁平投影只删计划里列出的那几行:按来源串整表删的话,`--source name` 这类+ # 扫描写下的标签来源也会被认成一批。+ removed_tag_rows = sum(connection.execute(+ "DELETE FROM asset_tag WHERE asset_id=? AND tag=? AND source=?",+ (tag["asset_id"], tag["tag"], tag["source"])).rowcount or 0 for tag in tags)+ reopened = reopen_extended_decisions(connection, {tag["item_key"] for tag in tags})+ restored = record_rehome.revert(connection, source, batch)+ reopened_wants = wants.revert(connection, source, batch)+ removed_seller_relations = fc2_sellers.revert(connection, planned["sellers"])+ # 链接与别名已在上面按批次删掉,剩下没有别的引用的实体才删得掉。+ creator_revert = follow_creators.revert(connection, planned["creators"])+ verified_revert = revert_verified(connection, planned["verified"])+ return {"删除链接": len(links), "删除别名": len(aliases), "删除资料": len(profiles),+ "删除断言": len(claims), "删除编号": len(refs), "删除归属": len(memberships),+ "删除片商": len(makers), "删除样张": removed_samples, "删除否决": len(rejections),+ "删除标签": len(tags), "删除扁平标签": removed_tag_rows, "重开决定": reopened,+ "撤回接回": restored, "想要回到待找": reopened_wants, "卖家关系": removed_seller_relations,+ "解绑关注来源": creator_revert["unbound"], "删除建档实体": creator_revert["removed"],+ "还原中文名": restored_names, "还原核查覆盖": verified_revert["restored"],+ "删除核查并集标签": verified_revert["removed_tags"],+ "删除核查决定": verified_revert["decisions"]}+++def remove_files(planned: dict) -> int:+ """删掉标识文件与作品封面,连同各自的同组边车;返回删掉的文件数。"""+ targets = [target for path in planned["files"]+ for target in (path, *(path.with_name(path.name + suffix) for suffix in SIDECARS))]+ targets += [target for path in planned["covers"]+ for target in (path, *(path.with_suffix(suffix) for suffix in COVER_SIDECARS))]+ removed = 0+ for target in targets:+ if target.exists():+ target.unlink()+ removed += 1+ return removeddiff --git a/tests/test_fc2_sellers.py b/tests/test_fc2_sellers.pyindex a035834b9..5b31c7692 100644--- a/tests/test_fc2_sellers.py+++ b/tests/test_fc2_sellers.py@@ -5,12 +5,11 @@ import tempfileimport unittestfrom pathlib import Path-from peach import fc2_sellers+from peach import auto_landing_revert, fc2_sellersfrom peach.entities import upsert_asset_entityfrom peach.entity_classification import write_claimfrom peach.web_entity import q_entity, q_indexfrom peach.web_state import WebContract-from scripts import revert_auto_landingfrom tests.support.ledger import fresh_ledger@@ -46,9 +45,9 @@ class Fc2SellerTests(unittest.TestCase):def undo(self):with self.connection:- aliases = revert_auto_landing.planned_aliases(self.connection,fc2_sellers.SOURCE,self.batch)- refs = revert_auto_landing.planned_refs(self.connection,fc2_sellers.SOURCE,self.batch)- links = revert_auto_landing.planned_links(self.connection,fc2_sellers.SOURCE,self.batch)+ aliases = auto_landing_revert.planned_aliases(self.connection,fc2_sellers.SOURCE,self.batch)+ refs = auto_landing_revert.planned_refs(self.connection,fc2_sellers.SOURCE,self.batch)+ links = auto_landing_revert.planned_links(self.connection,fc2_sellers.SOURCE,self.batch)sellers = fc2_sellers.planned_revert(self.connection,fc2_sellers.SOURCE,self.batch)for row in aliases:self.connection.execute('DELETE FROM entity_alias WHERE entity_id=? AND source=?',(row['entity_id'],row['source']))