Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(seed): 显式撤销种子断言并绑定版本与内容

longmeidao3908f166e0fe
diff --git a/migrations/0047_seed_pack_history.sql b/migrations/0047_seed_pack_history.sql
new file mode 100644
index 000000000..1dfa9d333
--- /dev/null
+++ b/migrations/0047_seed_pack_history.sql
@@ -0,0 +1,11 @@
+-- 种子版本、内容校验与累计撤销清单;不随任务历史清理。
+CREATE TABLE seed_pack_history(
+ version TEXT PRIMARY KEY,
+ revision INTEGER NOT NULL CHECK(revision >= 0),
+ content_sha256 TEXT NOT NULL,
+ withdrawals_json TEXT NOT NULL DEFAULT '[]',
+ imported_at TEXT NOT NULL
+);
+
+CREATE UNIQUE INDEX idx_seed_pack_history_revision
+ON seed_pack_history(revision) WHERE revision > 0;
diff --git a/scripts/seed_pack.py b/scripts/seed_pack.py
index 0029c4432..293e6b171 100644
--- a/scripts/seed_pack.py
+++ b/scripts/seed_pack.py
@@ -1,13 +1,13 @@
"""实体事实种子包的导出与导入(ADR-0073)。
seed_pack.py export --version 2026-09-25
+ seed_pack.py validate --pack resources/seed/entities.json
seed_pack.py import
seed_pack.py import --apply --backup peach-data/database/ledger.pre-seed-<时间戳>.db
`export` 只读账本,把女优、厂牌与事务所的公开事实写成 `resources/seed/entities.json`;
-同一账本、同一版本号两次导出逐字节一致。内容变了就必须换版本串:`seed-import:<版本>` 是导入
-的幂等键,导过旧内容的机器会把同版本的新包当作已导过而跳过,所以版本串不比现有文件新的导出被
-拒绝、不动文件、退出码 2。`import` 默认 dry-run,只报会补多少;
+格式 2 保留来源置信度、继承累计显式撤销,用内容摘要绑定版本与导入任务。内容变化须推进版本与修订,
+同一版本的不同内容或回退修订被拒绝。`validate` 只校验包,不打开账本。`import` 默认 dry-run,只报会补多少;
`--apply` 必须同时给 `--backup`。撤回:`revert_auto_landing.py --source auto:seed`。
"""
from __future__ import annotations
@@ -33,9 +33,14 @@ def build_parser() -> argparse.ArgumentParser:
export.add_argument("--version", default=date.today().isoformat(),
help="包版本号,也是批次号的后半段;默认当天日期")
export.add_argument("--out", type=Path, default=seed_pack.DEFAULT_PACK, help="落盘路径")
+ export.add_argument("--previous", type=Path, help="继承累计撤销的上一份包;缺省读取现有输出文件")
+ export.add_argument("--revision", type=int, help="格式 2 的单调修订号;缺省由上一包递增")
+ export.add_argument("--withdrawals", type=Path, help="已复核的显式撤销 JSON 数组,原样继承历史撤销")
landing = commands.add_parser("import", help="把种子包补进本机账本")
add_ledger_write_args(landing)
landing.add_argument("--pack", type=Path, default=seed_pack.DEFAULT_PACK, help="种子包路径")
+ validation = commands.add_parser("validate", help="只校验种子包,不打开账本")
+ validation.add_argument("--pack", type=Path, default=seed_pack.DEFAULT_PACK, help="种子包路径")
return parser
@@ -48,9 +53,20 @@ def _version_in(text: str) -> str | None:
def export(args: argparse.Namespace) -> int:
+ previous_path = args.previous or args.out
+ previous = seed_pack.load(previous_path) if previous_path.is_file() else None
+ inherited = list((previous or {}).get("withdrawals", []))
+ withdrawals = json.loads(args.withdrawals.read_text(encoding="utf-8")) if args.withdrawals else []
+ if not isinstance(withdrawals, list):
+ raise ValueError("撤销输入必须是 JSON 数组")
+ same_version = previous is not None and previous["version"] == args.version
+ revision = args.revision if args.revision is not None else (
+ max(1, previous.get("revision", 0)) if same_version else (previous or {}).get("revision", 0) + 1)
connection = open_readonly(args.db)
try:
- pack = seed_pack.export_pack(connection, version=args.version)
+ pack = seed_pack.export_pack(connection, version=args.version, revision=revision,
+ withdrawals=[*inherited, *withdrawals],
+ previous=None if same_version else previous)
finally:
connection.close()
text = seed_pack.dump(pack)
@@ -72,6 +88,14 @@ def export(args: argparse.Namespace) -> int:
return 0
+def validate(args: argparse.Namespace) -> int:
+ pack = seed_pack.load(args.pack)
+ print(json.dumps({"valid": True, "format": pack["format"], "version": pack["version"],
+ "revision": pack.get("revision", 0), "content_sha256": seed_pack.seed_validation.digest(pack),
+ "counts": pack["counts"], "withdrawals": len(pack.get("withdrawals", []))}, ensure_ascii=False))
+ return 0
+
+
def land(args: argparse.Namespace) -> int:
pack = seed_pack.load(args.pack)
connection = open_for_write(args)
@@ -93,7 +117,13 @@ def land(args: argparse.Namespace) -> int:
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
- return export(args) if args.command == "export" else land(args)
+ try:
+ if args.command == "validate":
+ return validate(args)
+ return export(args) if args.command == "export" else land(args)
+ except (ValueError, OSError) as exc:
+ print(json.dumps({"written": False, "error": str(exc)}, ensure_ascii=False))
+ return 2
if __name__ == "__main__":
diff --git a/src/peach/seed_corrections.py b/src/peach/seed_corrections.py
new file mode 100644
index 000000000..d7feeed0a
--- /dev/null
+++ b/src/peach/seed_corrections.py
@@ -0,0 +1,107 @@
+"""显式种子撤销与持久版本历史;不修改其他写入者的断言。"""
+from __future__ import annotations
+
+import json
+import sqlite3
+
+from . import seed_validation as validation
+from .performer_profiles import read_profile
+
+
+def check_history(connection: sqlite3.Connection, pack: dict) -> None:
+ """同版绑定内容,旧格式与较低修订不得穿过已导入的水位。"""
+ revision = pack.get("revision", 0)
+ content_digest = validation.digest(pack)
+ rows = connection.execute("SELECT version,revision,content_sha256,withdrawals_json FROM seed_pack_history").fetchall()
+ incoming = {row["id"]: validation.canonical(row) for row in pack.get("withdrawals", [])}
+ for version, previous_revision, digest, withdrawals in rows:
+ if version == pack["version"] and digest != content_digest:
+ raise ValueError("同版本种子内容与已导入历史不同")
+ if previous_revision > revision or (revision and previous_revision == revision and version != pack["version"]):
+ raise ValueError("种子 revision 不得回退或复用")
+ if not revision and not previous_revision and version > pack["version"]:
+ raise ValueError("旧格式种子版本不得回退")
+ for old in json.loads(withdrawals):
+ if incoming.get(old["id"]) != validation.canonical(old):
+ raise ValueError("种子必须原样继承全部历史撤销")
+
+
+def remember(connection: sqlite3.Connection, pack: dict, stamp: str) -> None:
+ connection.execute("INSERT OR IGNORE INTO seed_pack_history(version,revision,content_sha256,withdrawals_json,imported_at)"
+ " VALUES(?,?,?,?,?)", (pack["version"], pack.get("revision", 0), validation.digest(pack),
+ validation.canonical(pack.get("withdrawals", [])), stamp))
+
+
+def _conflict(report: dict, withdrawal: dict, *, entity: str, detail: str, source: str = "") -> None:
+ report["withdrawal_conflicts"].append({"id": withdrawal["id"], "kind": withdrawal["identity"]["kind"],
+ "entity": entity, "field": withdrawal["field"], "source": source,
+ "reason": withdrawal["reason"], "evidence": withdrawal["evidence"],
+ "detail": detail})
+
+
+def withdraw(connection: sqlite3.Connection, pack: dict, report: dict) -> None:
+ from . import seed_pack
+
+ for withdrawal in pack.get("withdrawals", []):
+ identity = withdrawal["identity"]
+ found = seed_pack.matches(connection, identity)
+ if len(found) != 1:
+ _conflict(report, withdrawal, entity=identity["name"],
+ detail="身份锚点未唯一命中,保留断言" if found else "身份锚点未命中,未取得可撤销实体")
+ continue
+ entity_id = found.pop()
+ field, expected = withdrawal["field"], withdrawal["value"]
+ if field == "alias":
+ rows = connection.execute("SELECT rowid,alias,source FROM entity_alias WHERE entity_id=?", (entity_id,))
+ candidates = [(rowid, source) for rowid, value, source in rows if validation.fact_key(field, value) == validation.fact_key(field, expected)]
+ table = "entity_alias"
+ elif field == "ref":
+ rows = connection.execute("SELECT rowid,provider,external_kind,external_id,metadata_json FROM entity_external_ref WHERE entity_id=?", (entity_id,))
+ candidates = [(rowid, "auto:seed" if seed_pack._metadata_ours(metadata) else _metadata_source(metadata))
+ for rowid, provider, kind, external_id, metadata in rows
+ if (provider, kind, external_id) == validation.ref_key(expected)]
+ table = "entity_external_ref"
+ elif field == "link":
+ rows = connection.execute("SELECT rowid,link_kind,url,metadata_json FROM entity_link WHERE entity_id=?", (entity_id,))
+ candidates = [(rowid, "auto:seed" if seed_pack._metadata_ours(metadata) else _metadata_source(metadata))
+ for rowid, kind, url, metadata in rows
+ if kind == expected["kind"] and seed_pack._normalised(url) == validation.url_key(expected["url"])]
+ table = "entity_link"
+ elif field == "profile":
+ profile = read_profile(connection, entity_id)
+ if profile is None:
+ continue
+ actual = {"fields": {key: profile[key] for key in expected["fields"]},
+ "source_url": profile["source_url"], "fetched_at": profile["fetched_at"]}
+ # 非空列也必须完整匹配;部分资料不能充当整行删除的凭据。
+ from .minnano_av import PROFILE_COLUMNS
+ actual["fields"].update({key: profile[key] for key in PROFILE_COLUMNS if profile.get(key) is not None})
+ actual["fields"]["tags"] = profile["tags"]
+ wanted = {**expected, "fields": {**expected["fields"], "tags": expected["fields"].get("tags", [])}}
+ if validation.fact_key(field, actual) != validation.fact_key(field, wanted):
+ _conflict(report, withdrawal, entity=identity["name"], detail="现有资料与精确撤销值不一致,保留断言", source=profile["source"])
+ continue
+ candidates, table = [(entity_id, profile["source"])], "performer_profile"
+ else:
+ table, own, other, _kind, _count = seed_pack.POINTERS[field]
+ row = connection.execute(f"SELECT p.rowid,e.canonical_name,p.source FROM {table} p JOIN entity e ON e.id=p.{other} WHERE p.{own}=?", (entity_id,)).fetchone()
+ if row is None:
+ continue
+ if validation.fact_key(field, {"name": row[1]}) != validation.fact_key(field, expected):
+ _conflict(report, withdrawal, entity=identity["name"], detail="现有归属与精确撤销值不一致,保留断言", source=row[2])
+ continue
+ candidates = [(row[0], row[2])]
+ for rowid, source in candidates:
+ if not seed_pack._ours(source):
+ _conflict(report, withdrawal, entity=identity["name"], detail="断言归其他写入者所有,保留断言", source=source)
+ continue
+ connection.execute(f"DELETE FROM {table} WHERE rowid=?", (rowid,))
+ report["withdrawn"] += connection.execute("SELECT changes()").fetchone()[0]
+
+
+def _metadata_source(raw: str) -> str:
+ try:
+ parsed = json.loads(raw)
+ except ValueError:
+ return ""
+ return str(parsed.get("source", "")) if isinstance(parsed, dict) else ""
diff --git a/src/peach/seed_followup.py b/src/peach/seed_followup.py
index 7baaa5da8..d0ac2b763 100644
--- a/src/peach/seed_followup.py
+++ b/src/peach/seed_followup.py
@@ -13,31 +13,28 @@ from __future__ import annotations
import sqlite3
from pathlib import Path
-from . import seed_pack
+from . import seed_pack, seed_corrections, seed_validation
from .followups import Followup, FollowupType, register
TASK_KEY = "seed-import"
TASK_LABEL = "导入实体种子"
REVIEW_FILE = "seed-landing.csv"
-FIELDS = ("kind", "entity", "field", "ours", "source", "theirs", "detail", "batch")
+FIELDS = ("kind", "entity", "field", "ours", "source", "theirs", "detail", "batch", "withdrawal_id", "evidence")
#: 活动页摘要里的计数键与它们的中文名,按这个顺序拼结论。
SUMMARY_LABELS = (("aliases", "别名"), ("refs", "编号"), ("links", "链接"), ("profiles", "资料"),
- ("memberships", "归属"), ("makers", "片商"), ("refreshed", "换新"))
+ ("memberships", "归属"), ("makers", "片商"), ("refreshed", "换新"), ("withdrawn", "撤销"))
-def followup_key(version: str) -> str:
- return f"{TASK_KEY}:{version}"
+def followup_key(version: str, content_hash: str = "") -> str:
+ return f"{TASK_KEY}:{version}" + (f":{content_hash}" if content_hash else "")
def pack_version(path: Path | None = None) -> str | None:
- """随仓库走的那份包的版本;没有包或包读不出来就当没有。"""
+ """随仓库走的包版本;缺包时返回 None,包损坏时报告校验错误。"""
path = Path(path) if path is not None else seed_pack.DEFAULT_PACK
if not path.is_file():
return None
- try:
- return str(seed_pack.load(path)["version"])
- except (OSError, ValueError):
- return None
+ return str(seed_pack.load(path)["version"])
def landed(connection: sqlite3.Connection, key: str) -> bool:
@@ -55,10 +52,13 @@ def _new_entities(connection: sqlite3.Connection, since_entity_id: int) -> bool:
def plan(connection: sqlite3.Connection, *, since_entity_id: int, pack_path: Path | None = None) -> list[Followup]:
"""这一轮该不该派:包的版本没成功导过,或这一轮登记了包里那三类的新实体。"""
- version = pack_version(pack_path)
- if version is None:
+ path = Path(pack_path) if pack_path is not None else seed_pack.DEFAULT_PACK
+ if not path.is_file():
return []
- key = followup_key(version)
+ pack = seed_pack.load(path)
+ seed_corrections.check_history(connection, pack)
+ version = pack["version"]
+ key = followup_key(version, seed_validation.digest(pack))
if landed(connection, key) and not _new_entities(connection, since_entity_id):
return []
return [Followup(key=key, task_key=TASK_KEY, label=f"{TASK_LABEL}:{version}")]
@@ -73,6 +73,10 @@ def _rows(report: dict) -> list[dict]:
"source": "", "theirs": item["name"], "batch": report["batch"],
"detail": "包里一条对上本机两位,可能是重复身份;合并不可逆,由人决定"}
for item in report["duplicates"]]
+ rows += [{"kind": item["kind"], "entity": item["entity"], "field": item["field"], "source": item["source"],
+ "detail": item["detail"] + ";" + item["reason"], "withdrawal_id": item["id"],
+ "evidence": item["evidence"], "batch": report["batch"]}
+ for item in report.get("withdrawal_conflicts", [])]
return rows
@@ -90,6 +94,8 @@ def outcome(report: dict) -> str:
parts.append(f"不一致 {len(report['conflicts'])}")
if report["duplicates"]:
parts.append(f"重复身份 {len(report['duplicates'])}")
+ if report.get("withdrawal_conflicts"):
+ parts.append(f"撤销待复核 {len(report['withdrawal_conflicts'])}")
return "、".join(parts) if parts else f"没有可补的(对上 {report['matched']} 位)"
@@ -100,6 +106,8 @@ def run(contract, key: str, handle) -> dict:
会给测试账本里的真人名补上站上编号,后面的后继就真的去联网了。
"""
pack = seed_pack.load(seed_pack.DEFAULT_PACK)
+ if key != followup_key(pack["version"], seed_validation.digest(pack)):
+ raise ValueError("种子内容与排队任务的摘要不一致,请重新派发")
if handle is not None:
handle.progress(label=f"{TASK_LABEL}:{pack['version']}", throttle=0)
with contract.database.write_transaction() as connection:
@@ -108,8 +116,9 @@ def run(contract, key: str, handle) -> dict:
contract.cache_bust()
record_review(contract.candidate_root, report)
return {"outcome": outcome(report), "version": report["version"], "matched": report["matched"],
- "unmatched": report["unmatched"], **{key: report[key] for key, _label in SUMMARY_LABELS},
- "conflicts": len(report["conflicts"]), "duplicates": len(report["duplicates"])}
+ "unmatched": report["unmatched"], **{key: report.get(key, 0) for key, _label in SUMMARY_LABELS},
+ "conflicts": len(report["conflicts"]), "duplicates": len(report["duplicates"]),
+ "withdrawal_conflicts": len(report.get("withdrawal_conflicts", []))}
#: 写账本:别名、编号、链接、资料、归属与片商都进账本,走写账本那一条串行通道(ADR-0040 第三条)。
diff --git a/src/peach/seed_pack.py b/src/peach/seed_pack.py
index 45508dac9..8131fdb05 100644
--- a/src/peach/seed_pack.py
+++ b/src/peach/seed_pack.py
@@ -9,9 +9,9 @@
别名或站上编号对得上)时才补,补的每一格都是空格(ADR-0052 第三条),归属串 `auto:seed`、
批次 `auto:seed@<版本>`,`scripts/revert_auto_landing.py --source auto:seed` 整批撤回。
-会变的事实(所属事务所、label 的片商、资料表)里由种子自己写下的行,新版种子可以整行换掉
-(ADR-0075);人或本机后继写的行不动,包里说的不一样就记进 `conflicts` 等人看。一条对上本机
-两位实体的记进 `duplicates`:那是本机账本里的重复身份信号,合并不可逆,不由种子动手。
+会变的事实(所属事务所、label 的片商、资料表)里由种子自己写下的行,新版种子可以整行换掉。
+格式 2 通过独立身份锚点、累计显式撤销和持久版本历史传递精确纠错;人或本机后继写的行不动。
+一条对上本机两位实体的记进 `duplicates`:合并不可逆,不由种子动手。
"""
from __future__ import annotations
@@ -21,6 +21,7 @@ from datetime import datetime, timezone
from urllib.parse import urlsplit
from . import link_status
+from . import seed_corrections, seed_validation
from .entities import _ref_is_free, normalize_entity_name
from .minnano_av import PROFILE_COLUMNS
from .performer_profiles import read_profile, write_profile
@@ -74,21 +75,26 @@ def _name(connection: sqlite3.Connection, entity_id: int) -> str:
# -- 导出 ----------------------------------------------------------------------
-def _aliases(connection: sqlite3.Connection, entity_id: int, kind: str) -> list[dict]:
+def _aliases(connection: sqlite3.Connection, entity_id: int, kind: str, *, source_confidence: bool = False) -> list[dict]:
"""名下的别名写法,同一写法只留排在前面的那个来源;种子自己写的不再导出。
女优的写法先过 ADR-0055 的判据:罗马字与短单名导入时本来就判「不收」,留在包里只会让
匹配认出别人(`绫乃梓` 的 `Nozomi` 与另一位 `NOZOMI` 同键,两位都对上就整条跳过)。"""
from .performer_alias_followup import rejection
found: dict[str, dict] = {}
- for alias, source in connection.execute(
- "SELECT alias,source FROM entity_alias WHERE entity_id=? ORDER BY normalized_alias,source",
+ for alias, source, confidence in connection.execute(
+ "SELECT alias,source,confidence FROM entity_alias WHERE entity_id=? ORDER BY normalized_alias,source",
(entity_id,)):
if _ours(source) or not str(alias or "").strip():
continue
if kind == "performer" and rejection(str(alias)):
continue
- found.setdefault(normalize_entity_name(str(alias)), {"alias": str(alias), "source": str(source)})
+ if source_confidence and str(source).casefold().split(":", 1)[0] == "stash":
+ continue
+ fact = {"alias": str(alias), "source": str(source)}
+ if source_confidence:
+ fact["confidence"] = float(confidence)
+ found.setdefault(normalize_entity_name(str(alias)), fact)
return list(found.values())
@@ -141,14 +147,15 @@ def _pointer(connection: sqlite3.Connection, entity_id: int, field: str) -> dict
return {"name": str(row[0]), "source": str(row[1])}
-def export_pack(connection: sqlite3.Connection, *, version: str) -> dict:
+def export_pack(connection: sqlite3.Connection, *, version: str, revision: int | None = None,
+ withdrawals: list[dict] | None = None, previous: dict | None = None) -> dict:
"""账本里三类实体的公开事实。只有名字、没有任何可补事实的实体不进包:导入不造实体,
光一个名字对别的账本没有用处。输出按种类与归一名排序,同一账本两次导出逐字节一致。"""
entities, counts = [], dict.fromkeys(COUNT_KEYS, 0)
for entity_id, kind, name in connection.execute(
f"SELECT id,kind,canonical_name FROM entity WHERE kind IN ({','.join('?' * len(KINDS))})"
" ORDER BY kind,normalized_name,id", KINDS):
- item = {"kind": str(kind), "name": str(name), "aliases": _aliases(connection, entity_id, str(kind)),
+ item = {"kind": str(kind), "name": str(name), "aliases": _aliases(connection, entity_id, str(kind), source_confidence=revision is not None),
"refs": _refs(connection, entity_id), "links": _links(connection, entity_id)}
if kind == "performer":
item.update({key: value for key, value in (("profile", _profile(connection, entity_id)),
@@ -167,7 +174,26 @@ def export_pack(connection: sqlite3.Connection, *, version: str) -> dict:
counts["profiles"] += "profile" in item
counts["memberships"] += "agency" in item
counts["makers"] += "maker" in item
- return {"format": FORMAT, "version": version, "source": SOURCE, "counts": counts, "entities": entities}
+ pack = {"format": FORMAT, "version": version, "source": SOURCE, "counts": counts, "entities": entities}
+ if revision is not None:
+ inherited = list((previous or {}).get("withdrawals", []))
+ if previous is not None:
+ seed_validation.validate(previous)
+ if revision <= previous.get("revision", 0) or version <= previous["version"]:
+ raise ValueError("导出需要更新的 version 与 revision")
+ by_id = {item["id"]: item for item in inherited}
+ for item in withdrawals or []:
+ if item["id"] in by_id and item != by_id[item["id"]]:
+ raise ValueError("累计撤销 id 不得改写")
+ by_id[item["id"]] = item
+ for item in entities:
+ item["identity"] = {"kind": item["kind"], "name": item["name"], "refs": item["refs"]}
+ pack.update(format=2, revision=revision, withdrawals=[by_id[key] for key in sorted(by_id)])
+ pack["content_sha256"] = seed_validation.digest(pack)
+ seed_validation.validate(pack)
+ elif withdrawals or previous:
+ raise ValueError("累计撤销需要格式 2 的 revision")
+ return pack
def dump(pack: dict) -> str:
@@ -176,10 +202,16 @@ def dump(pack: dict) -> str:
def load(path) -> dict:
- pack = json.loads(path.read_text(encoding="utf-8"))
- if not isinstance(pack, dict) or pack.get("format") != FORMAT or not pack.get("version"):
- raise ValueError(f"{path} 不是格式 {FORMAT} 的种子包")
- return pack
+ def unique_fields(pairs):
+ result = {}
+ for key, value in pairs:
+ if key in result:
+ raise ValueError(f"种子 JSON 字段重复:{key}")
+ result[key] = value
+ return result
+
+ pack = json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=unique_fields)
+ return seed_validation.validate(pack)
# -- 导入 ----------------------------------------------------------------------
@@ -214,7 +246,7 @@ def match_entity(connection: sqlite3.Connection, item: dict) -> int | None:
return found.pop() if len(found) == 1 else None
-def _land_aliases(connection, entity_id: int, item: dict, version: str, batch: str) -> int:
+def _land_aliases(connection, entity_id: int, item: dict, version: str, batch: str, report: dict) -> int:
"""女优别名走补别名后继的判据(ADR-0055:不收的写法、已有、被占用);厂牌与事务所
只查同类实体里有没有人用着这个写法。"""
names = [item["name"], *(alias["alias"] for alias in item.get("aliases") or [])]
@@ -222,8 +254,10 @@ def _land_aliases(connection, entity_id: int, item: dict, version: str, batch: s
from .performer_alias_followup import WRITE, land
canonical = connection.execute("SELECT canonical_name FROM entity WHERE id=?", (entity_id,)).fetchone()
rows = land(connection, entity_id, str(canonical[0]), "seed", f"{SOURCE}:{version}", names, batch)
+ _preserve_alias_confidence(connection, entity_id, item, batch, report,
+ {row["alias"] for row in rows if row["action"] == WRITE})
return sum(row["action"] == WRITE for row in rows)
- written = 0
+ written, created = 0, set()
for name in dict.fromkeys(names):
key = normalize_entity_name(name)
taken = connection.execute(
@@ -234,10 +268,31 @@ def _land_aliases(connection, entity_id: int, item: dict, version: str, batch: s
connection.execute(
"INSERT OR IGNORE INTO entity_alias(entity_id,alias,normalized_alias,source,confidence)"
" VALUES(?,?,?,?,1.0)", (entity_id, name, key, batch))
- written += connection.execute("SELECT changes()").fetchone()[0]
+ changed = connection.execute("SELECT changes()").fetchone()[0]
+ written += changed
+ if changed:
+ created.add(name)
+ _preserve_alias_confidence(connection, entity_id, item, batch, report, created)
return written
+def _preserve_alias_confidence(connection, entity_id: int, item: dict, batch: str, report: dict,
+ created: set[str]) -> None:
+ """来源置信度与用户复核状态独立,只维护种子自己拥有的行。"""
+ for alias in item.get("aliases", []):
+ if alias.get("confidence") is None:
+ continue
+ rows = connection.execute("SELECT alias,source,confidence FROM entity_alias WHERE entity_id=? AND normalized_alias=?",
+ (entity_id, normalize_entity_name(alias["alias"]))).fetchall()
+ for spelling, source, current in rows:
+ if not _ours(source) or current == alias["confidence"]:
+ continue
+ connection.execute("UPDATE entity_alias SET confidence=?,source=? WHERE entity_id=? AND alias=? AND source=?",
+ (alias["confidence"], batch, entity_id, spelling, source))
+ if spelling not in created:
+ report["refreshed"] += connection.execute("SELECT changes()").fetchone()[0]
+
+
def _land_refs(connection, entity_id: int, item: dict, batch: str, stamp: str) -> int:
"""站上编号只在这个编号没人登记、这位在这家来源下也还没有编号时写:现成的
`ON CONFLICT … DO UPDATE` 会把别人的编号改指到她,这里不用它。"""
@@ -280,16 +335,22 @@ def _land_links(connection, entity_id: int, item: dict, batch: str, stamp: str)
return written
-def _land_profile(connection, entity_id: int, item: dict, batch: str, report: dict) -> None:
- """资料只给没有资料行的写;种子自己写的那一行,包里取回时间更晚才整行换掉。本机后继刮的
- 资料比种子新,也由它自己按期刷新,`write_profile` 对自动来源的整行替换在这里不碰它。"""
+def _land_profile(connection, entity_id: int, item: dict, batch: str, report: dict, *, revision: bool = False) -> None:
+ """资料只补空行与种子自己的行。格式 1 按取回时间换新,格式 2 由包修订与内容决定纠错;
+ 本机后继与人工资料由自己的写入者维护。"""
profile = item.get("profile")
if not profile or item["kind"] != "performer":
return
current = read_profile(connection, entity_id)
- if current is not None and not (_ours(current.get("source"))
- and str(profile.get("fetched_at") or "") > str(current.get("fetched_at") or "")):
- return
+ if current is not None:
+ if not _ours(current.get("source")):
+ return
+ same = (all(current.get(key) == profile["fields"].get(key) for key in PROFILE_COLUMNS)
+ and current.get("tags") == profile["fields"].get("tags", [])
+ and current.get("source_url") == profile.get("source_url")
+ and current.get("fetched_at") == profile.get("fetched_at"))
+ if same or (not revision and str(profile.get("fetched_at") or "") <= str(current.get("fetched_at") or "")):
+ return
fields = dict(profile.get("fields") or {})
fields["raw"] = {}
write_profile(connection, entity_id, fields, source=batch,
@@ -338,28 +399,49 @@ def _land_pointer(connection, entity_id: int, item: dict, field: str, batch: str
"ours": str(row[2]), "source": str(row[1]), "theirs": str(fact["name"])})
-def _land_one(connection, entity_id: int, item: dict, version: str, batch: str, stamp: str, report: dict) -> None:
- report["aliases"] += _land_aliases(connection, entity_id, item, version, batch)
+def _land_one(connection, entity_id: int, item: dict, version: str, batch: str, stamp: str, report: dict, *, revision: bool = False) -> None:
+ report["aliases"] += _land_aliases(connection, entity_id, item, version, batch, report)
report["refs"] += _land_refs(connection, entity_id, item, batch, stamp)
report["links"] += _land_links(connection, entity_id, item, batch, stamp)
- _land_profile(connection, entity_id, item, batch, report)
+ _land_profile(connection, entity_id, item, batch, report, revision=revision)
for field in POINTERS:
_land_pointer(connection, entity_id, item, field, batch, stamp, report)
def written(report: dict) -> int:
"""这次导入动了多少行:补上的加换掉的。"""
- return sum(int(report.get(key, 0)) for key in (*COUNT_KEYS[1:], "refreshed"))
+ return sum(int(report.get(key, 0)) for key in (*COUNT_KEYS[1:], "refreshed", "withdrawn"))
def land(connection: sqlite3.Connection, pack: dict) -> dict:
"""把包里的事实补进这本账,返回逐类计数与两张明细(`conflicts`、`duplicates`)。
调用方负责事务与备份;dry-run 走 `plan`。同一包再跑一遍不会再动任何一行。"""
+ seed_validation.validate(pack)
+ if not connection.in_transaction:
+ connection.execute("BEGIN")
+ connection.execute("SAVEPOINT seed_landing")
+ try:
+ seed_corrections.check_history(connection, pack)
+ report = _land_pack(connection, pack)
+ seed_corrections.remember(connection, pack, _now())
+ connection.execute("RELEASE seed_landing")
+ return report
+ except BaseException:
+ connection.execute("ROLLBACK TO seed_landing")
+ connection.execute("RELEASE seed_landing")
+ raise
+
+
+def _land_pack(connection: sqlite3.Connection, pack: dict) -> dict:
version, batch, stamp = str(pack["version"]), batch_of(str(pack["version"])), _now()
report = {"version": version, "batch": batch, "matched": 0, "unmatched": 0,
**dict.fromkeys(COUNT_KEYS[1:], 0), "refreshed": 0, "conflicts": [], "duplicates": []}
+ revision = pack["format"] == 2
+ if revision:
+ report.update(withdrawn=0, withdrawal_conflicts=[])
+ seed_corrections.withdraw(connection, pack, report)
for item in pack.get("entities") or []:
- found = matches(connection, item)
+ found = matches(connection, item["identity"] if revision else item)
if len(found) > 1:
report["duplicates"].append({"kind": item["kind"], "name": item["name"],
"entities": sorted(_name(connection, entity_id) for entity_id in found)})
@@ -368,7 +450,7 @@ def land(connection: sqlite3.Connection, pack: dict) -> dict:
report["unmatched"] += 1
continue
report["matched"] += 1
- _land_one(connection, found.pop(), item, version, batch, stamp, report)
+ _land_one(connection, found.pop(), item, version, batch, stamp, report, revision=revision)
return report
diff --git a/src/peach/seed_validation.py b/src/peach/seed_validation.py
new file mode 100644
index 000000000..adf72550c
--- /dev/null
+++ b/src/peach/seed_validation.py
@@ -0,0 +1,251 @@
+"""种子包的结构、身份锚点与相互冲突校验。"""
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Annotated, Any, Literal
+from urllib.parse import urlsplit
+
+from pydantic import BaseModel, ConfigDict, Field, StringConstraints, ValidationError
+
+from .entities import normalize_entity_name
+from .minnano_av import PROFILE_COLUMNS
+from .social_links import canonical_url
+
+Text = Annotated[str, StringConstraints(min_length=1)]
+KINDS = Literal["performer", "studio", "agency"]
+FIELDS = Literal["alias", "ref", "link", "profile", "agency", "maker"]
+
+
+class StrictModel(BaseModel):
+ model_config = ConfigDict(extra="forbid", strict=True)
+
+
+class Ref(StrictModel):
+ provider: Text
+ kind: Text
+ id: Text
+
+
+class Alias(StrictModel):
+ alias: Text
+ source: Text
+ confidence: Annotated[float, Field(ge=0, le=1)] = 1.0
+
+
+class Link(StrictModel):
+ kind: Literal["official", "social"]
+ label: str
+ url: Text
+
+
+class Identity(StrictModel):
+ kind: KINDS
+ name: Text
+ refs: list[Ref] = Field(default_factory=list)
+
+
+class Profile(StrictModel):
+ fields: dict[str, Any]
+ source: Text
+ source_url: str
+ fetched_at: str
+
+
+class Pointer(StrictModel):
+ name: Text
+ source: Text
+
+
+class Item(StrictModel):
+ kind: KINDS
+ name: Text
+ aliases: list[Alias]
+ refs: list[Ref]
+ links: list[Link]
+ profile: Profile | None = None
+ agency: Pointer | None = None
+ maker: Pointer | None = None
+ identity: Identity | None = None
+
+
+class Withdrawal(StrictModel):
+ id: Text
+ identity: Identity
+ field: FIELDS
+ value: Any
+ reason: Text
+ evidence: Text
+
+
+class Pack(StrictModel):
+ format: Literal[1, 2]
+ version: Text
+ source: Literal["auto:seed"]
+ counts: dict[str, Annotated[int, Field(ge=0)]]
+ entities: list[Item]
+ revision: Annotated[int, Field(ge=1)] | None = None
+ content_sha256: str | None = None
+ withdrawals: list[Withdrawal] = Field(default_factory=list)
+
+
+def canonical(value: Any) -> str:
+ return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)
+
+
+def digest(pack: dict) -> str:
+ """摘要覆盖版本、修订号、事实与累计撤销,不包含摘要字段自身。"""
+ return hashlib.sha256(canonical({k: v for k, v in pack.items() if k != "content_sha256"}).encode()).hexdigest()
+
+
+def url_key(value: str) -> str:
+ parsed = urlsplit(value)
+ if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:
+ raise ValueError("种子链接必须是无凭据的 HTTP(S) 地址")
+ return canonical_url(value)
+
+
+def ref_key(value: dict) -> tuple[str, str, str]:
+ return value["provider"], value["kind"], value["id"]
+
+
+def identity_keys(identity: dict) -> set[tuple]:
+ return {("name", identity["kind"], normalize_entity_name(identity["name"])),
+ *(("ref", *ref_key(ref)) for ref in identity.get("refs", []))}
+
+
+def fact_key(field: str, value: Any) -> str:
+ if field == "alias":
+ return normalize_entity_name(value)
+ if field == "link":
+ return canonical({"kind": value["kind"], "url": url_key(value["url"])})
+ if field == "profile":
+ return canonical({"fields": value["fields"], "source_url": value["source_url"],
+ "fetched_at": value["fetched_at"]})
+ if field in {"agency", "maker"}:
+ return normalize_entity_name(value["name"])
+ return canonical(value)
+
+
+def _profile(value: dict) -> None:
+ numeric = {"height_cm", "bust_cm", "waist_cm", "hip_cm", "debut_year", "active_until"}
+ for key, item in value["fields"].items():
+ if key == "tags":
+ if not isinstance(item, list) or any(type(tag) is not str for tag in item):
+ raise ValueError("种子资料 tags 必须是字符串数组")
+ elif key not in PROFILE_COLUMNS or (item is not None and type(item) is not (int if key in numeric else str)):
+ raise ValueError(f"种子资料列无效:{key}")
+ if value["source_url"]:
+ url_key(value["source_url"])
+
+
+def _validate_withdrawal(value: dict) -> None:
+ field, old = value["field"], value["value"]
+ if field == "alias":
+ if type(old) is not str or not normalize_entity_name(old):
+ raise ValueError("撤销别名需要精确写法")
+ if normalize_entity_name(old) == normalize_entity_name(value["identity"]["name"]):
+ raise ValueError("被撤销别名不能同时作为身份锚点")
+ elif field == "ref":
+ Ref.model_validate(old)
+ if ref_key(old) in {ref_key(ref) for ref in value["identity"].get("refs", [])}:
+ raise ValueError("被撤销编号不能同时作为身份锚点")
+ elif field == "link":
+ if not isinstance(old, dict) or set(old) != {"kind", "url"} or old["kind"] not in {"official", "social"}:
+ raise ValueError("撤销链接需要用途与精确地址")
+ if type(old["url"]) is not str:
+ raise ValueError("撤销链接地址必须是字符串")
+ url_key(old["url"])
+ elif field == "profile":
+ Profile.model_validate(old)
+ _profile(old)
+ else:
+ Pointer.model_validate(old)
+ kind = value["identity"]["kind"]
+ if not normalize_entity_name(value["identity"]["name"]) or any(ref["provider"] == "stash" for ref in value["identity"].get("refs", [])):
+ raise ValueError("撤销需要有效的独立身份锚点")
+ if field in {"profile", "agency"} and kind != "performer" or field == "maker" and kind != "studio":
+ raise ValueError("撤销字段与实体类型不符")
+
+
+def validate(pack: dict, *, verify_hash: bool = True) -> dict:
+ """只校验,不修补、丢弃或自动批准输入。"""
+ if type(pack) is not dict or type(pack.get("format")) is not int:
+ raise ValueError("种子包必须是带整数 format 的对象")
+ try:
+ parsed = Pack.model_validate(pack)
+ if pack["format"] == 1 and {"revision", "content_sha256", "withdrawals"} & pack.keys():
+ raise ValueError("格式 1 不接受格式 2 字段")
+ if pack["format"] == 2:
+ if parsed.revision is None or type(parsed.content_sha256) is not str or "withdrawals" not in pack:
+ raise ValueError("格式 2 需要 revision、content_sha256 与累计 withdrawals")
+ if verify_hash and pack["content_sha256"] != digest(pack):
+ raise ValueError("种子内容摘要不符")
+ counts = {"entities": len(pack["entities"]), "aliases": 0, "refs": 0, "links": 0,
+ "profiles": 0, "memberships": 0, "makers": 0}
+ for item in pack["entities"]:
+ for field in ("aliases", "refs", "links"):
+ counts[field] += len(item[field])
+ for field, counter in (("profile", "profiles"), ("agency", "memberships"), ("maker", "makers")):
+ counts[counter] += bool(item.get(field))
+ if counts != pack["counts"]:
+ raise ValueError("种子计数与包内容不一致")
+ names, refs = {}, {}
+ for index, item in enumerate(pack["entities"]):
+ kind = item["kind"]
+ identity = item.get("identity")
+ if pack["format"] == 2 and identity is None:
+ raise ValueError("格式 2 的实体必须声明独立身份锚点")
+ if identity and (identity["kind"] != kind or normalize_entity_name(identity["name"]) != normalize_entity_name(item["name"])):
+ raise ValueError("实体与身份锚点不一致")
+ for name in [item["name"], *(alias["alias"] for alias in item["aliases"])]:
+ key = kind, normalize_entity_name(name)
+ if not key[1] or names.setdefault(key, index) != index:
+ raise ValueError(f"种子内身份名称冲突:{name}")
+ for ref in [*item["refs"], *((identity or {}).get("refs", []))]:
+ if ref["provider"] == "stash" or refs.setdefault(ref_key(ref), index) != index:
+ raise ValueError("种子内来源编号身份冲突或使用本机编号")
+ for link in item["links"]:
+ url_key(link["url"])
+ if item.get("profile"):
+ _profile(item["profile"])
+ if item.get("profile") or item.get("agency"):
+ if kind != "performer":
+ raise ValueError("资料与事务所归属只适用于女优")
+ if item.get("maker") and kind != "studio":
+ raise ValueError("片商归属只适用于厂牌")
+ identifiers = set()
+ anchors = [identity_keys(item.get("identity") or {"kind": item["kind"], "name": item["name"], "refs": item["refs"]})
+ for item in pack["entities"]]
+ for withdrawal in pack.get("withdrawals", []):
+ _validate_withdrawal(withdrawal)
+ if withdrawal["id"] in identifiers:
+ raise ValueError("撤销 id 重复")
+ identifiers.add(withdrawal["id"])
+ identity = withdrawal["identity"]
+ if sum(bool(identity_keys(identity) & keys) for keys in anchors) > 1:
+ raise ValueError("撤销身份锚点在包内指向多个实体")
+ for item in pack["entities"]:
+ anchor = item.get("identity", {"kind": item["kind"], "name": item["name"], "refs": item["refs"]})
+ if not identity_keys(identity) & identity_keys(anchor):
+ continue
+ field, old = withdrawal["field"], fact_key(withdrawal["field"], withdrawal["value"])
+ values = {"alias": [item["name"], *(a["alias"] for a in item["aliases"])],
+ "ref": item["refs"], "link": item["links"],
+ "profile": [item["profile"]] if item.get("profile") else [],
+ "agency": [item["agency"]] if item.get("agency") else [],
+ "maker": [item["maker"]] if item.get("maker") else []}[field]
+ if any(fact_key(field, value) == old for value in values):
+ raise ValueError("同一包同时断言与撤销同一事实")
+ studios = {normalize_entity_name(item["name"]): item for item in pack["entities"] if item["kind"] == "studio"}
+ for start in studios:
+ current, visited = start, set()
+ while current in studios and studios[current].get("maker"):
+ if current in visited:
+ raise ValueError("种子片商归属成环")
+ visited.add(current)
+ current = normalize_entity_name(studios[current]["maker"]["name"])
+ canonical(pack)
+ except (ValidationError, TypeError, KeyError, OverflowError) as exc:
+ raise ValueError(f"种子包结构无效:{exc}") from exc
+ return pack
diff --git a/tests/test_seed_corrections.py b/tests/test_seed_corrections.py
new file mode 100644
index 000000000..44322b350
--- /dev/null
+++ b/tests/test_seed_corrections.py
@@ -0,0 +1,282 @@
+"""种子纠错只撤销自己的精确断言,包历史阻止回放。"""
+import copy
+import sqlite3
+from unittest import mock
+
+from peach import seed_pack
+from peach.performer_profiles import read_profile, write_profile
+from test_seed_pack import SeedPackCase, entity, alias, ref, link, membership, label_maker, STAMP
+
+
+class CorrectionTests(SeedPackCase):
+ def setUp(self):
+ super().setUp()
+ with self.source:
+ self.person = entity(self.source, "performer", "天川そら")
+ ref(self.source, self.person, "minnano-av", "123")
+ with self.target:
+ self.recipient = entity(self.target, "performer", "天川そら")
+
+ def pack(self, revision, withdrawals=()):
+ return seed_pack.export_pack(self.source, version=f"2026-10-{revision:02}",
+ revision=revision, withdrawals=list(withdrawals))
+
+ def withdrawal(self, field, value, *, name="天川そら", key="correction-1"):
+ return {"id": key, "identity": {"kind": "performer", "name": name,
+ "refs": [{"provider": "minnano-av", "kind": "performer", "id": "123"}]},
+ "field": field, "value": value, "reason": "来源核对确认指向错误", "evidence": "review:verified.csv"}
+
+ def land(self, pack):
+ with self.target:
+ return seed_pack.land(self.target, pack)
+
+ def test_explicit_alias_ref_and_link_withdrawals_allow_corrected_ref(self):
+ with self.source:
+ alias(self.source, self.person, "错误名字", "review:source")
+ ref(self.source, self.person, "javdb", "wrong")
+ link(self.source, self.person, "social", "https://x.com/wrong")
+ self.land(self.pack(1))
+ withdrawals = [self.withdrawal("alias", "错误名字", key="bad-alias"),
+ self.withdrawal("ref", {"provider": "javdb", "kind": "performer", "id": "wrong"}, key="bad-ref"),
+ self.withdrawal("link", {"kind": "social", "url": "https://x.com/wrong"}, key="bad-link")]
+ with self.source:
+ self.source.execute("DELETE FROM entity_alias")
+ self.source.execute("DELETE FROM entity_external_ref WHERE provider='javdb'")
+ self.source.execute("DELETE FROM entity_link")
+ ref(self.source, self.person, "javdb", "correct")
+ fixed = self.pack(2, withdrawals)
+ report = self.land(fixed)
+ self.assertEqual(report["withdrawn"], 3)
+ self.assertIsNone(self.target.execute("SELECT 1 FROM entity_alias WHERE alias='错误名字'").fetchone())
+ self.assertEqual(self.target.execute("SELECT external_id FROM entity_external_ref WHERE provider='javdb'").fetchone()[0], "correct")
+ self.assertEqual(self.target.execute("SELECT count(*) FROM entity_link").fetchone()[0], 0)
+ self.assertEqual(seed_pack.written(self.land(fixed)), 0)
+
+ def test_withdrawal_preserves_each_nonseed_owner(self):
+ for owner in ("user:manual", "review:verified", "auto:performer-profile@4"):
+ with self.subTest(owner=owner), self.target:
+ self.target.execute("DELETE FROM entity_alias")
+ alias(self.target, self.recipient, "错误名字", owner)
+ report = seed_pack.land(self.target, self.pack(1, [self.withdrawal("alias", "错误名字")]))
+ self.assertEqual(report["withdrawn"], 0)
+ self.assertEqual(self.target.execute("SELECT source FROM entity_alias").fetchone()[0], owner)
+ self.assertEqual(len(report["withdrawal_conflicts"]), 1)
+
+ def test_profile_correction_uses_revision_even_with_same_fetched_at(self):
+ with self.source:
+ write_profile(self.source, self.person, {"height_cm": 164}, source="review:profile", source_url="https://example.com/profile", fetched_at=STAMP)
+ self.land(self.pack(1))
+ with self.source:
+ self.source.execute("UPDATE performer_profile SET height_cm=165")
+ report = self.land(self.pack(2))
+ self.assertEqual(read_profile(self.target, self.recipient)["height_cm"], 165)
+ self.assertEqual(report["refreshed"], 1)
+ self.assertEqual(seed_pack.written(self.land(self.pack(2))), 0)
+
+ def test_cumulative_withdrawal_supports_skipped_revision_and_rejects_replay(self):
+ with self.source:
+ alias(self.source, self.person, "错误名字", "review:source")
+ initial = self.pack(1)
+ self.land(initial)
+ with self.source:
+ self.source.execute("DELETE FROM entity_alias")
+ withdrawal = self.withdrawal("alias", "错误名字")
+ self.land(self.pack(3, [withdrawal]))
+ with self.assertRaises(ValueError):
+ self.land(initial)
+ with self.assertRaises(ValueError):
+ self.land(seed_pack.export_pack(self.source, version="2099-01-01"))
+ self.assertIsNone(self.target.execute("SELECT 1 FROM entity_alias WHERE alias='错误名字'").fetchone())
+ with self.assertRaises(ValueError):
+ self.land(self.pack(4))
+
+ def test_ambiguous_withdrawal_leaves_rows_and_records_review(self):
+ with self.target:
+ other = entity(self.target, "performer", "另一位艺人")
+ ref(self.target, other, "minnano-av", "123")
+ alias(self.target, self.recipient, "错误名字", "auto:seed@old")
+ report = self.land(self.pack(1, [self.withdrawal("alias", "错误名字")]))
+ self.assertEqual(report["withdrawn"], 0)
+ self.assertEqual(len(report["withdrawal_conflicts"]), 1)
+ self.assertEqual(self.target.execute("SELECT count(*) FROM entity_alias").fetchone()[0], 1)
+
+ def test_absence_is_not_a_withdrawal(self):
+ with self.source:
+ alias(self.source, self.person, "已登记别名", "review:source")
+ self.land(self.pack(1))
+ with self.source:
+ self.source.execute("DELETE FROM entity_alias")
+ self.land(self.pack(2))
+ self.assertEqual(self.target.execute("SELECT count(*) FROM entity_alias WHERE alias='已登记别名'").fetchone()[0], 1)
+
+ def test_modified_content_and_invalid_pack_are_rejected_atomically(self):
+ pack = self.pack(1)
+ broken = copy.deepcopy(pack)
+ broken["entities"][0]["aliases"].append({"alias": "无效包的名字", "source": "review:source"})
+ before = list(self.target.iterdump())
+ with self.assertRaises(ValueError):
+ self.land(broken)
+ self.assertEqual(list(self.target.iterdump()), before)
+ self.land(pack)
+ with self.source:
+ alias(self.source, self.person, "同版本新内容", "review:source")
+ with self.assertRaises(ValueError):
+ self.land(self.pack(1))
+
+ def test_schema_rejects_bool_revision_unknown_fields_and_duplicate_ref_identity(self):
+ for mutate in (
+ lambda p: p.update(revision=True),
+ lambda p: p.update(unexpected="value"),
+ lambda p: p["entities"].append({**copy.deepcopy(p["entities"][0]), "name": "另一位艺人"})):
+ with self.subTest(mutate=mutate):
+ pack = self.pack(1)
+ mutate(pack)
+ before = list(self.target.iterdump())
+ with self.assertRaises(ValueError):
+ self.land(pack)
+ self.assertEqual(list(self.target.iterdump()), before)
+
+ def test_same_package_cannot_assert_its_withdrawn_value(self):
+ with self.source:
+ alias(self.source, self.person, "错误名字", "review:source")
+ with self.assertRaises(ValueError):
+ self.pack(1, [self.withdrawal("alias", "错误名字")])
+
+ def test_export_inherits_withdrawals_from_explicit_previous_package(self):
+ previous = self.pack(1, [self.withdrawal("alias", "错误名字")])
+ later = seed_pack.export_pack(self.source, version="2026-10-04", revision=4, previous=previous)
+ self.assertEqual(later["withdrawals"], previous["withdrawals"])
+
+ def test_profile_and_both_pointer_types_are_precisely_withdrawn(self):
+ with self.source:
+ agency = entity(self.source, "agency", "事务所甲")
+ maker = entity(self.source, "studio", "片商甲")
+ label = entity(self.source, "studio", "厂牌甲")
+ membership(self.source, self.person, agency, "review:source")
+ label_maker(self.source, label, maker, "review:source")
+ write_profile(self.source, self.person, {"height_cm": 164}, source="review:source", source_url="https://example.com/profile", fetched_at=STAMP)
+ with self.target:
+ entity(self.target, "agency", "事务所甲")
+ entity(self.target, "studio", "片商甲")
+ entity(self.target, "studio", "厂牌甲")
+ first = self.pack(1)
+ self.land(first)
+ person = next(item for item in first["entities"] if item["kind"] == "performer")
+ studio = next(item for item in first["entities"] if item["name"] == "厂牌甲")
+ withdrawals = [self.withdrawal("profile", person["profile"], key="bad-profile"),
+ self.withdrawal("agency", person["agency"], key="bad-agency"),
+ {"id": "bad-maker", "identity": studio["identity"], "field": "maker", "value": studio["maker"],
+ "reason": "已核对归属", "evidence": "review:verified.csv"}]
+ with self.source:
+ self.source.execute("DELETE FROM performer_profile")
+ self.source.execute("DELETE FROM entity_membership")
+ self.source.execute("DELETE FROM label_maker")
+ self.assertEqual(self.land(self.pack(2, withdrawals))["withdrawn"], 3)
+ self.assertIsNone(read_profile(self.target, self.recipient))
+ self.assertEqual(self.target.execute("SELECT count(*) FROM entity_membership").fetchone()[0], 0)
+ self.assertEqual(self.target.execute("SELECT count(*) FROM label_maker").fetchone()[0], 0)
+
+ def test_profile_withdrawal_requires_all_nonempty_columns(self):
+ with self.target:
+ write_profile(self.target, self.recipient, {"height_cm": 164, "birthplace": "東京都"}, source="auto:seed@old", source_url="https://example.com/profile", fetched_at=STAMP)
+ partial = {"fields": {"height_cm": 164}, "source": "review:source", "source_url": "https://example.com/profile", "fetched_at": STAMP}
+ report = self.land(self.pack(1, [self.withdrawal("profile", partial)]))
+ self.assertEqual(report["withdrawn"], 0)
+ self.assertEqual(len(report["withdrawal_conflicts"]), 1)
+ self.assertEqual(read_profile(self.target, self.recipient)["birthplace"], "東京都")
+
+ def test_runtime_error_rolls_back_withdrawals_facts_and_history(self):
+ with self.source:
+ alias(self.source, self.person, "错误名字", "review:source")
+ self.land(self.pack(1))
+ with self.source:
+ self.source.execute("DELETE FROM entity_alias")
+ fixed = self.pack(2, [self.withdrawal("alias", "错误名字")])
+ before = list(self.target.iterdump())
+ with mock.patch.object(seed_pack.seed_corrections, "remember", side_effect=sqlite3.OperationalError("simulated failure")):
+ with self.assertRaises(sqlite3.OperationalError):
+ seed_pack.land(self.target, fixed)
+ self.assertEqual(list(self.target.iterdump()), before)
+
+ def test_nonseed_profile_is_not_corrected_by_new_revision(self):
+ with self.source:
+ write_profile(self.source, self.person, {"height_cm": 164}, source="review:source", source_url="https://example.com/profile", fetched_at=STAMP)
+ with self.target:
+ write_profile(self.target, self.recipient, {"height_cm": 180}, source="auto:performer-profile@5", source_url="https://example.com/local", fetched_at=STAMP)
+ self.land(self.pack(1))
+ self.assertEqual(read_profile(self.target, self.recipient)["height_cm"], 180)
+
+ def test_schema_failure_after_valid_entity_cannot_land_any_rows(self):
+ pack = seed_pack.export_pack(self.source, version="2026-10-01")
+ pack["entities"].append({"kind": "performer", "name": "另一位", "aliases": False, "refs": [], "links": []})
+ before = list(self.target.iterdump())
+ with self.assertRaises(ValueError):
+ seed_pack.land(self.target, pack)
+ self.assertEqual(list(self.target.iterdump()), before)
+
+ def test_conflicting_reference_identities_are_rejected_without_hash_gate(self):
+ pack = seed_pack.export_pack(self.source, version="2026-10-01")
+ pack["entities"].append({"kind": "performer", "name": "另一位艺人", "aliases": [],
+ "refs": copy.deepcopy(pack["entities"][0]["refs"]), "links": []})
+ before = list(self.target.iterdump())
+ with self.assertRaisesRegex(ValueError, "编号身份冲突"):
+ seed_pack.land(self.target, pack)
+ self.assertEqual(list(self.target.iterdump()), before)
+
+ def test_alias_confidence_rejects_boolean_and_out_of_range_values(self):
+ with self.source:
+ alias(self.source, self.person, "来源别名", "review:source")
+ for confidence in (True, None, -0.1, 1.1):
+ with self.subTest(confidence=confidence):
+ pack = self.pack(1)
+ pack["entities"][0]["aliases"][0]["confidence"] = confidence
+ pack["content_sha256"] = seed_pack.seed_validation.digest(pack)
+ before = list(self.target.iterdump())
+ with self.assertRaises(ValueError):
+ seed_pack.land(self.target, pack)
+ self.assertEqual(list(self.target.iterdump()), before)
+
+ def test_caller_rollback_restores_facts_and_version_history(self):
+ with self.assertRaises(RuntimeError):
+ with self.target:
+ seed_pack.land(self.target, self.pack(1))
+ raise RuntimeError("caller abort")
+ self.assertEqual(self.target.execute("SELECT count(*) FROM entity_external_ref").fetchone()[0], 0)
+ self.assertEqual(self.target.execute("SELECT count(*) FROM seed_pack_history").fetchone()[0], 0)
+
+ def test_source_confidence_round_trip_preserves_manual_values(self):
+ with self.source:
+ alias(self.source, self.person, "来源别名", "r18:performer")
+ self.source.execute("UPDATE entity_alias SET confidence=0.42")
+ pack = self.pack(1)
+ self.assertEqual(pack["entities"][0]["aliases"][0]["confidence"], 0.42)
+ self.land(pack)
+ held = self.target.execute("SELECT confidence,source FROM entity_alias WHERE alias='来源别名'").fetchone()
+ self.assertEqual(tuple(held), (0.42, "auto:seed@2026-10-01"))
+ with self.target:
+ self.target.execute("UPDATE entity_alias SET confidence=0.61,source='user:manual' WHERE alias='来源别名'")
+ with self.source:
+ self.source.execute("UPDATE entity_alias SET confidence=0.3")
+ self.land(self.pack(2))
+ self.assertEqual(tuple(self.target.execute("SELECT confidence,source FROM entity_alias WHERE alias='来源别名'").fetchone()), (0.61, "user:manual"))
+
+ def test_source_confidence_corrects_seed_owned_legacy_confidence(self):
+ with self.source:
+ alias(self.source, self.person, "来源别名", "r18:performer")
+ self.source.execute("UPDATE entity_alias SET confidence=0.42")
+ legacy = seed_pack.export_pack(self.source, version="2026-09-25")
+ self.land(legacy)
+ self.assertEqual(self.target.execute("SELECT confidence FROM entity_alias WHERE alias='来源别名'").fetchone()[0], 1)
+ report = self.land(self.pack(1))
+ self.assertEqual(report["refreshed"], 1)
+ self.assertEqual(self.target.execute("SELECT confidence FROM entity_alias WHERE alias='来源别名'").fetchone()[0], 0.42)
+ self.assertEqual(seed_pack.written(self.land(self.pack(1))), 0)
+
+ def test_stash_aliases_are_not_exported_as_public_v2_facts(self):
+ with self.source:
+ alias(self.source, self.person, "本机导入别名", "stash:performer")
+ self.assertTrue(seed_pack.export_pack(self.source, version="2026-09-25")["entities"][0]["aliases"])
+ pack = self.pack(1)
+ self.assertEqual(pack["entities"][0]["aliases"], [])
+ self.assertEqual(pack["withdrawals"], [])
+
diff --git a/tests/test_seed_followup.py b/tests/test_seed_followup.py
index a18be836f..421662601 100644
--- a/tests/test_seed_followup.py
+++ b/tests/test_seed_followup.py
@@ -5,7 +5,7 @@ from pathlib import Path
from types import SimpleNamespace
from unittest import mock
-from peach import seed_followup, seed_pack
+from peach import seed_followup, seed_pack, seed_validation
from peach.entities import normalize_entity_name
from peach.repository import LedgerDatabase
from peach.review_csv import read_rows
@@ -21,6 +21,7 @@ PACK = {"format": 1, "version": VERSION, "source": "auto:seed", "counts": {}, "e
{"kind": "studio", "name": "PRESTIGE PREMIUM", "aliases": [], "refs": [], "links": [],
"maker": {"name": "Prestige", "source": "review:label-maker-20260923.csv"}},
]}
+KEY = seed_followup.followup_key(VERSION, seed_validation.digest(PACK))
class Case(unittest.TestCase):
@@ -65,6 +66,15 @@ class Case(unittest.TestCase):
class PlanTests(Case):
+ def test_content_hash_changes_the_followup_key(self):
+ first = self.plan(since_entity_id=10 ** 9)[0].key
+ self.finished_run(first)
+ modified = seed_pack.load(self.pack)
+ modified["entities"][0]["links"][0]["url"] = "https://x.com/verified_handle"
+ self.pack.write_text(seed_pack.dump(modified), encoding="utf-8")
+ second = self.plan(since_entity_id=10 ** 9)[0].key
+ self.assertNotEqual(first, second)
+
def test_without_a_pack_nothing_is_planned(self):
with self.database.read_connection() as connection:
self.assertEqual(seed_followup.plan(connection, since_entity_id=0,
@@ -73,10 +83,10 @@ class PlanTests(Case):
def test_a_version_never_landed_is_planned_even_without_new_entities(self):
found = self.plan(since_entity_id=10 ** 9)
self.assertEqual([(item.key, item.task_key, item.label) for item in found],
- [(f"seed-import:{VERSION}", "seed-import", f"导入实体种子:{VERSION}")])
+ [(KEY, "seed-import", f"导入实体种子:{VERSION}")])
def test_a_landed_version_is_planned_again_only_when_new_entities_appear(self):
- self.finished_run(seed_followup.followup_key(VERSION))
+ self.finished_run(KEY)
self.assertEqual(self.plan(since_entity_id=10 ** 9), [], "导过、又没有新实体,不再派")
her = self.add("performer", "天川そら")
self.assertEqual(len(self.plan(since_entity_id=her - 1)), 1, "这一轮登记了新女优,再派一次给她补")
@@ -85,6 +95,17 @@ class PlanTests(Case):
class RunTests(Case):
+ def test_queued_content_change_is_rejected_without_writing(self):
+ self.add("performer", "天川そら")
+ modified = seed_pack.load(self.pack)
+ modified["entities"][0]["links"][0]["url"] = "https://x.com/verified_handle"
+ self.pack.write_text(seed_pack.dump(modified), encoding="utf-8")
+ with self.assertRaises(ValueError):
+ seed_followup.run(self.contract, KEY, None)
+ with self.database.read_connection() as connection:
+ self.assertEqual(connection.execute("SELECT count(*) FROM entity_alias").fetchone()[0], 0)
+ self.assertEqual(connection.execute("SELECT count(*) FROM seed_pack_history").fetchone()[0], 0)
+
def test_run_lands_the_pack_and_writes_the_review_file(self):
her, kmp = self.add("performer", "天川そら"), self.add("studio", "K M Produce")
self.add("agency", "STARTUP")
@@ -93,7 +114,7 @@ class RunTests(Case):
with self.database.write_transaction() as connection:
connection.execute("INSERT INTO label_maker(label_id,maker_id,source,confidence,checked_at)"
" VALUES(?,?,?,1.0,?)", (label, kmp, "review:user", STAMP))
- summary = seed_followup.run(self.contract, seed_followup.followup_key(VERSION), None)
+ summary = seed_followup.run(self.contract, KEY, None)
self.assertEqual((summary["matched"], summary["aliases"], summary["links"], summary["memberships"],
summary["makers"], summary["conflicts"], summary["duplicates"]), (2, 1, 1, 1, 0, 1, 0))
self.assertEqual(summary["outcome"], "别名 1、链接 1、归属 1、不一致 1")
@@ -106,7 +127,7 @@ class RunTests(Case):
rows = read_rows(self.candidates / seed_followup.REVIEW_FILE)
self.assertEqual([(row["entity"], row["field"], row["ours"], row["theirs"]) for row in rows],
[("PRESTIGE PREMIUM", "maker", "K M Produce", "Prestige")])
- again = seed_followup.run(self.contract, seed_followup.followup_key(VERSION), None)
+ again = seed_followup.run(self.contract, KEY, None)
self.assertEqual((again["aliases"], again["links"], again["memberships"], again["refreshed"]), (0, 0, 0, 0),
"同一包再跑一遍没有东西可补")
self.assertEqual(self.busts, 1, "没动账本就不清缓存")
@@ -115,7 +136,7 @@ class RunTests(Case):
def test_two_local_entities_for_one_pack_item_become_a_duplicate_row(self):
self.add("performer", "天川そら")
self.add("performer", "春山心愛")
- summary = seed_followup.run(self.contract, seed_followup.followup_key(VERSION), None)
+ summary = seed_followup.run(self.contract, KEY, None)
self.assertEqual((summary["matched"], summary["duplicates"]), (0, 1))
rows = read_rows(self.candidates / seed_followup.REVIEW_FILE)
self.assertEqual([(row["field"], row["entity"], row["theirs"]) for row in rows],
diff --git a/tests/test_seed_pack.py b/tests/test_seed_pack.py
index d0db3c9b1..4d6210bd3 100644
--- a/tests/test_seed_pack.py
+++ b/tests/test_seed_pack.py
@@ -11,6 +11,7 @@ import sqlite3
import tempfile
import unittest
from pathlib import Path
+from unittest import mock
from peach import seed_pack
from peach.entities import normalize_entity_name
@@ -328,6 +329,19 @@ class ImportTests(SeedPackCase):
class ScriptTests(SeedPackCase):
+ def test_validate_checks_package_without_opening_a_database(self):
+ pack = self.root / "package.json"
+ pack.write_text(seed_pack.dump(self.seeded_source()), encoding="utf-8")
+ script = load_script("seed_pack")
+ with mock.patch.object(script, "open_readonly", side_effect=AssertionError("database must remain closed")):
+ with contextlib.redirect_stdout(io.StringIO()) as output:
+ self.assertEqual(script.main(["validate", "--pack", str(pack)]), 0)
+ self.assertTrue(json.loads(output.getvalue())["valid"])
+ pack.write_text('{"format":true}', encoding="utf-8")
+ with contextlib.redirect_stdout(io.StringIO()) as rejected:
+ self.assertEqual(script.main(["validate", "--pack", str(pack)]), 2)
+ self.assertFalse(json.loads(rejected.getvalue())["written"])
+
def test_export_is_idempotent_and_import_is_dry_run_by_default(self):
self.seeded_source()
self.source.close()
@@ -359,7 +373,7 @@ class ScriptTests(SeedPackCase):
self.assertTrue((self.root / "pre-seed.db").exists())
def test_a_changed_export_that_keeps_the_version_is_refused(self):
- """`seed-import:<版本>` 是导入的幂等键:同版本装新内容,导过旧内容的机器会当作已导过而跳过。"""
+ """版本与内容绑定,导出内容变化须推进版本。"""
self.seeded_source()
script = load_script("seed_pack")
out, db = self.root / "seed" / "entities.json", str(self.root / "source.db")
@@ -388,7 +402,7 @@ class ScriptTests(SeedPackCase):
source.close()
with contextlib.redirect_stdout(io.StringIO()) as older:
self.assertEqual(script.main(["export", "--db", db, "--version", VERSION, "--out", str(out)]), 2)
- self.assertIn(f"不比文件里的 {VERSION}.1 新", json.loads(older.getvalue())["error"])
+ self.assertIn("需要更新的 version 与 revision", json.loads(older.getvalue())["error"])
self.assertEqual(seed_pack.load(out)["version"], f"{VERSION}.1")