Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(catalog): 保留纯汉字现名与艺名原字形

longmeidao553682c85e61
diff --git a/docs/REUSE.md b/docs/REUSE.md
index dc3f3662c..d3396737c 100644
--- a/docs/REUSE.md
+++ b/docs/REUSE.md
@@ -353,6 +353,7 @@ A 盘垃圾隔离使用短名称 `peach-purge-<32 位 UUID>.peach-quarantine`,
- 操作回执复用 Toast(Sonner 的栈,`frontend/src/react/toaster.tsx`;壳里只调 `toast()`/`actionReceipt()`);按钮以 Spinner 和 `aria-busy` 标明忙态。后台任务显示可恢复进度,断线自动重连。
- 实体的统称由用户在资料页自选:菜单只列这条实体名下已有的写法,选中的提为规范名、换下的留成别名,扁平投影跟着改;先过确认弹层并点名两个写法,成功后发可撤销回执;不收自由文本,撞上另一条实体的规范名只报冲突。
- 艺人入口审计复用 `entry_links` 的 JAV 目录闸与日文名选择、只读 SQLite 和 `review_csv`;`audit_performer_entry_names.py` 覆盖全部 performer,分别记录入口模板、名称选择和中文来源缺口,远端页面未请求时保留「未取得」。MISSAV 使用 `/dm3/cn/actresses/` 与当前日文艺名;译名复核后的原日文名按 `performer-localization` 来源优先。`localize_performer_names.py --mapping-source` 标明 CSV 的真实复核来源,`--only` 限定实体并拒绝不完整的同人合并组;别名、兼容标签、来源留痕和备份沿用已有实现。未新增依赖。
+ - MISSAV 的艺名选择排除纯平假名读音,保留同名字形与 `々`;当前汉字艺名读取既有成对译名来源,并要求对应当前规范名及相同姓名前缀。可信假名艺名、不同姓氏或拉丁字母来源字段冲突时不套用原名记录。API 与全库审计共用这一选择函数,派生入口不写 ledger;来源冲突留在复核 CSV,403 验证页记「未取得」。
- FC2 作品计数复用 `ordered_multipart_items`,`web_catalog.video_work_key` 按账本代次缓存已确认的合集键;资料、名册、同台艺人、标签与补全共用。`/api/items.work_total` 供作品读数,`total` 仍用于文件分页;不确定分段不归组。出演名单展开复用 BoardUI `Button`。跨角色人工合并走 `merge_entity`,`user:identity-merge:<kind>` 的别名与墓碑共同证明旧资料页地址,单纯同名不跨角色跳转。
- 馆藏分组缓存声明 `asset`、`asset_entity`、`entity` 依赖;下载进度和复核留痕不使分组重算,同键计算串行共用。作品计数通过 SQLite `create_function` 注册资产到作品的只读字典查找,沿用既有版次与分卷判据。下载后处理复用已绑定资产、整理备份及逐项外键基线比较(`performer_alias_followup`);单部采集只结算本资产候选,进度在 `state/download-processing/`,共享 CSV 仍受全库处理锁保护。资源查询缓存拒收缺字段和来源失败的旧结果。无新增依赖。
- Application 管理路由与资源,域控制器适配原生宿主。文件检查含本地与网盘,来源等分、共用确认弹层;CloudDrive 分档建议首启与配置页共用。
diff --git a/scripts/audit_performer_entry_names.py b/scripts/audit_performer_entry_names.py
index bfb364f5a..9d877a62b 100644
--- a/scripts/audit_performer_entry_names.py
+++ b/scripts/audit_performer_entry_names.py
@@ -29,6 +29,14 @@ HAN = re.compile(r"[\u3400-\u9fff]")
LATIN = re.compile(r"[A-Za-z]")
+def _name_localization(raw: str):
+ try:
+ metadata = json.loads(raw or "{}")
+ except (ValueError, TypeError):
+ return None
+ return metadata.get("name_localization") if isinstance(metadata, dict) else None
+
+
def collect(connection: sqlite3.Connection) -> list[dict]:
"""每位 performer 一行;入口生成正确与远端页面可访问分别留证。"""
connection.row_factory = sqlite3.Row
@@ -44,13 +52,13 @@ def collect(connection: sqlite3.Connection) -> list[dict]:
sources[int(row["entity_id"])].add(str(row["source"]))
assets[int(row["entity_id"])].add(int(row["asset_id"]))
out = []
- for entity in connection.execute("SELECT id,canonical_name FROM entity WHERE kind='performer' ORDER BY id"):
+ for entity in connection.execute("SELECT id,canonical_name,metadata_json FROM entity WHERE kind='performer' ORDER BY id"):
entity_id = int(entity["id"])
name = str(entity["canonical_name"])
chain = aliases[entity_id]
jav = entry_links.is_jav_performer(refs[entity_id])
previous = entry_links.japanese_name("", chain) or name
- selected = entry_links.japanese_name(name, chain)
+ selected = entry_links.japanese_name(name, chain, _name_localization(entity["metadata_json"]))
chinese = sorted({str(row["alias"]) for row in chain
if HAN.search(str(row["alias"])) and not KANA.search(str(row["alias"]))
and not LATIN.search(str(row["alias"]))})
diff --git a/src/peach/entry_links.py b/src/peach/entry_links.py
index 1428c285f..3def4cde1 100644
--- a/src/peach/entry_links.py
+++ b/src/peach/entry_links.py
@@ -17,10 +17,10 @@
**MISSAV 按日文艺名拼。** 入口使用 `/dm3/cn/actresses/`,汉字字形按简中站投影;当前规范名
含假名且不是纯读音时优先使用,否则从别名里挑日文艺名。中文译名与旧艺名不能代替当前日文艺名:
-- 含假名才算日文写法。汉字分不出中日——`佐々木さき` 认得出,纯汉字的艺名认不出,
- 那种只能退回规范名。
-- 纯平假名的那条是读音不是艺名。`r18:performer` 每位都带一条(`しゃくありす`),
- 而站上的页面挂在混着汉字或片假名的艺名下(`釈アリス`),实测纯平假名同样 404。
+- 日文艺名可含假名或纯汉字。汉字现名采用同名字形或已有成对译名的来源证据;
+ 对应的中文名必须仍是当前规范名,不同姓氏与可信假名艺名的冲突保留给复核。
+- 别名里的纯平假名读音不作为入口。`r18:performer` 每位都带一条(`しゃくありす`),
+ 选择艺名时需要区分它与人物的正式写法(`釈アリス`)。可信艺名来源标明的纯平假名正式艺名照用。
- 同样形态的有多条时,优先 `avdb-actor-mapping`、`javdb`、`wiki` 这几类艺名来源。
- 一条都挑不出时才用规范名:拼错的地址和没有地址相比,前者更难发现。
@@ -112,6 +112,7 @@ _HOST_SHAPE = re.compile(r"[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?"
_ORDINALS = "②③④⑤⑥⑦⑧⑨"
#: 艺名类的别名来源。按子串认:来源串后面还挂着采集批次。
_STAGE_NAME_SOURCES = ("avdb-actor-mapping", "javdb", "wiki", "performer-localization")
+_MISSAV_GLYPHS = {**JP_KANJI_TO_SIMPLIFIED, "條": "条"}
def defaults() -> dict:
@@ -233,24 +234,88 @@ def _only_hiragana(written: str) -> bool:
for char in written)
-def japanese_name(canonical_name: str, aliases) -> str:
+def _glyph_key(written: str) -> str:
+ return fold_glyphs(written).translate(str.maketrans(_MISSAV_GLYPHS))
+
+
+def _han_name(written: str) -> bool:
+ return bool(re.fullmatch(r"[\u3400-\u9fff々\s]+", written))
+
+
+def _glyph_equivalents(name: str, aliases) -> list[str]:
+ """与当前艺名仅有汉字字形或叠字符号差异的写法。"""
+ matches = []
+ for row in aliases or ():
+ written = str((row.get("alias") if isinstance(row, dict) else row) or "").strip()
+ if written and _glyph_key(written) == _glyph_key(name):
+ matches.append(written)
+ return matches
+
+
+def _published_kana_alias(aliases) -> bool:
+ """别名中是否有区别于纯平假名读音的艺名。"""
+ for row in aliases or ():
+ written = str((row.get("alias") if isinstance(row, dict) else row) or "")
+ if _has_kana(written) and not _only_hiragana(written):
+ return True
+ return False
+
+
+def _paired_hiragana_name(jp: str, aliases) -> bool:
+ """成对来源中的纯平假名艺名必须同时出现在可信艺名别名中。"""
+ if not _only_hiragana(jp) or not _has_kana(jp):
+ return False
+ return any(isinstance(row, dict) and row.get("alias") == jp
+ and any(mark in str(row.get("source") or "") for mark in _STAGE_NAME_SOURCES)
+ for row in aliases or ())
+
+
+def _localized_han_name(name: str, aliases, localization) -> str:
+ """来源成对记录的当前汉字艺名;不同姓氏或可信假名艺名冲突时不采用。"""
+ if not isinstance(localization, dict) or not _han_name(name):
+ return ""
+ jp = str(localization.get("jp") or "").strip()
+ chinese = str(localization.get("zh_cn") or "").strip()
+ source = str(localization.get("source") or "")
+ trusted = any(mark in source for mark in (*_STAGE_NAME_SOURCES, "kanji-simplification"))
+ if not trusted or _glyph_key(chinese) != _glyph_key(name):
+ return ""
+ if _paired_hiragana_name(jp, aliases):
+ return jp
+ if not _han_name(jp):
+ return ""
+ if _glyph_key(jp) != _glyph_key(name) and _glyph_key(jp)[:2] != _glyph_key(name)[:2]:
+ return ""
+ for row in aliases or ():
+ if not isinstance(row, dict):
+ continue
+ written = str(row.get("alias") or "")
+ alias_source = str(row.get("source") or "")
+ if _has_kana(written) and not _only_hiragana(written) \
+ and any(mark in alias_source for mark in _STAGE_NAME_SOURCES):
+ return ""
+ return jp
+
+
+def japanese_name(canonical_name: str, aliases, localization=None) -> str:
"""这位的日文艺名。挑不出来时退回规范名,判据见模块开头。"""
name = str(canonical_name or "").strip()
- if _has_kana(name) and not _only_hiragana(name):
- equivalents = [name]
- for row in aliases or ():
- written = str((row.get("alias") if isinstance(row, dict) else row) or "").strip()
- if written and fold_glyphs(written) == fold_glyphs(name):
- equivalents.append(written)
- return max(equivalents, key=lambda value: sum(char in JP_KANJI_TO_SIMPLIFIED for char in value))
+ localized = _localized_han_name(name, aliases, localization)
+ current = localized or name
+ equivalents = _glyph_equivalents(current, aliases)
+ if localized or (equivalents and not _published_kana_alias(aliases)) \
+ or (_has_kana(name) and not _only_hiragana(name)):
+ return max([current, *equivalents],
+ key=lambda value: sum(char in _MISSAV_GLYPHS or char == "々" for char in value))
chosen: tuple[tuple[int, int, int], str] | None = None
for index, row in enumerate(aliases or ()):
written = str((row.get("alias") if isinstance(row, dict) else row) or "").strip()
- if not written or not _has_kana(written):
- continue
source = str((row.get("source") if isinstance(row, dict) else "") or "")
+ trusted = any(mark in source for mark in _STAGE_NAME_SOURCES)
+ if not written or not _has_kana(written) or (_only_hiragana(written) and not trusted):
+ continue
rank = (1 if _only_hiragana(written) else 0,
- 0 if any(mark in source for mark in _STAGE_NAME_SOURCES) else 1,
+ 0 if trusted else 1,
index)
if chosen is None or rank < chosen[0]:
chosen = (rank, written)
@@ -264,7 +329,7 @@ def is_jav_performer(refs) -> bool:
def missav_name(japanese: str) -> str:
"""简中入口的汉字字形,假名、叠字符号与其他艺名字符保留。"""
- return japanese.translate(str.maketrans(JP_KANJI_TO_SIMPLIFIED))
+ return japanese.translate(str.maketrans(_MISSAV_GLYPHS))
def _ordinal(index: int) -> str:
@@ -281,11 +346,11 @@ def _ordinal(index: int) -> str:
return f"({index + 1})"
-def build(settings: dict, canonical_name: str, refs, aliases=()) -> list[dict]:
+def build(settings: dict, canonical_name: str, refs, aliases=(), localization=None) -> list[dict]:
"""这条人物实体能直达的地址,按 SITES 的先后排好。缺 id 的站点不出现在结果里。"""
ids = provider_ids(refs)
jav = is_jav_performer(refs)
- written = japanese_name(canonical_name, aliases)
+ written = japanese_name(canonical_name, aliases, localization)
out: list[dict] = []
for site in SITES:
row = settings.get(site.key) or {}
@@ -305,9 +370,9 @@ def build(settings: dict, canonical_name: str, refs, aliases=()) -> list[dict]:
return out
-def entry_links(root: Path, canonical_name: str, refs, aliases=()) -> list[dict]:
+def entry_links(root: Path, canonical_name: str, refs, aliases=(), localization=None) -> list[dict]:
"""读设置并拼出入口。资料页只调这一个。"""
- return build(read(root), canonical_name, refs, aliases)
+ return build(read(root), canonical_name, refs, aliases, localization)
def javdb_actor_pages(canonical_name: str, refs) -> list[str]:
diff --git a/src/peach/web_entity.py b/src/peach/web_entity.py
index 81d202c06..b5db0ef15 100644
--- a/src/peach/web_entity.py
+++ b/src/peach/web_entity.py
@@ -155,8 +155,11 @@ def _performer_entries(contract: WebContract, c, d: dict, alias_rows) -> dict:
前端再拼一遍就会有两份规则。缺 id 的站点不出现在这个列表里。开关只在她有
JavDB 演员页时出现,订阅地址同样由服务端拼,页面只管开和关。
"""
+ metadata = d.get("metadata") or {}
+ localization = metadata.get("name_localization") if isinstance(metadata, dict) else None
out = {"entry_links": entry_links.entry_links(
- contract.entry_links_root, d["canonical_name"], d["external_refs"], alias_rows)}
+ contract.entry_links_root, d["canonical_name"], d["external_refs"], alias_rows,
+ localization)}
if entry_links.javdb_actor_pages(d["canonical_name"], d["external_refs"]):
out["feed"] = {"following": feeds.entity_following(c, d["id"])}
return out
diff --git a/tests/test_entry_links.py b/tests/test_entry_links.py
index b8dd6e748..8f27ae4f9 100644
--- a/tests/test_entry_links.py
+++ b/tests/test_entry_links.py
@@ -107,6 +107,65 @@ class EntryLinkAddressTests(unittest.TestCase):
alias("藍川みれい", "avdb-actor-mapping"), alias("坂本マリア"),
]), "藍川みれい")
+ def test_plain_hiragana_alias_is_not_a_han_stage_name(self):
+ for name, reading in (("三宅美香", "みやけみか"), ("音海里奈", "おとみりな")):
+ with self.subTest(name=name):
+ self.assertEqual(entry_links.japanese_name(name, [alias(reading)]), name)
+
+ def test_a_published_hiragana_stage_name_is_distinct_from_a_profile_reading(self):
+ aliases = [alias("なぎさ", "javdb-actor-page")]
+ self.assertEqual(entry_links.japanese_name("渚向日葵", aliases), "なぎさ")
+ self.assertEqual(entry_links.japanese_name("渚向日葵", aliases,
+ {"source": "javdb-actor-page", "jp": "なぎさ", "zh_cn": "渚向日葵"}),
+ "なぎさ")
+
+ def test_glyph_equivalent_chinese_alias_does_not_hide_a_published_kana_stage_name(self):
+ for name, jp, chinese in (("逢泽美优", "逢沢みゆ", "逢澤美優"),
+ ("优梨舞奈", "優梨まいな", "優梨舞奈"),
+ ("新井爱丽", "新井エリー", "新井愛麗")):
+ with self.subTest(name=name):
+ aliases = [alias(chinese, "review:performer-localization"),
+ alias(jp, "review:performer-localization")]
+ self.assertEqual(entry_links.japanese_name(name, aliases,
+ {"source": "review:performer-localization", "jp": jp, "zh_cn": name}), jp)
+
+ def test_equivalent_han_name_preserves_original_glyphs_and_iteration_mark(self):
+ for name, jp in (("北条麻妃", "北條麻妃"), ("乃乃果花", "乃々果花"),
+ ("梨梨衣", "梨々衣")):
+ with self.subTest(name=name):
+ self.assertEqual(entry_links.japanese_name(name, [alias(jp)]), jp)
+ self.assertEqual(entry_links.missav_name("北條麻妃"), "北条麻妃")
+ self.assertEqual(entry_links.missav_name("乃々果花"), "乃々果花")
+
+ def test_paired_current_han_name_wins_over_an_unreviewed_former_name(self):
+ localized = {"source": "avdb-actor-mapping", "jp": "水谷心音", "zh_cn": "水谷心音"}
+ rows = entry_links.build(entry_links.defaults(), "水谷心音", [ref("javdb", "sample")],
+ [alias("藤崎りお"), alias("みずたにここね")], localized)
+ self.assertEqual(next(row["url"] for row in rows if row["site"] == "missav"),
+ "https://missav.ws/dm3/cn/actresses/"
+ "%E6%B0%B4%E8%B0%B7%E5%BF%83%E9%9F%B3")
+
+ def test_localization_conflicts_do_not_replace_the_current_identity(self):
+ self.assertEqual(entry_links.japanese_name("霞理沙", [alias("かすみりさ")],
+ {"source": "avdb-actor-mapping", "jp": "松冈仁美", "zh_cn": "霞理沙"}),
+ "霞理沙")
+ self.assertEqual(entry_links.japanese_name("泽村玲子", [
+ alias("澤村レイコ", "avdb-actor-mapping")],
+ {"source": "avdb-actor-mapping", "jp": "泽村玲子", "zh_cn": "泽村玲子"}),
+ "澤村レイコ")
+ self.assertEqual(entry_links.japanese_name("鹫尾芽衣", [
+ alias("筧ジュン", "avdb-actor-mapping"), alias("鷲尾めい", "avdb-actor-mapping")],
+ {"source": "avdb-actor-mapping", "jp": "鷲尾めい", "zh_cn": "鹫尾芽衣"}),
+ "筧ジュン")
+
+ def test_paired_han_name_requires_the_same_current_chinese_name(self):
+ self.assertEqual(entry_links.japanese_name("宇野刊菜", [alias("あべかんな")],
+ {"source": "avdb-actor-mapping", "jp": "宇野栞菜", "zh_cn": "宇野刊菜"}),
+ "宇野栞菜")
+ self.assertEqual(entry_links.japanese_name("宇野刊菜", [alias("あべかんな")],
+ {"source": "avdb-actor-mapping", "jp": "宇野栞菜", "zh_cn": "别的写法"}),
+ "宇野刊菜")
+
def test_a_reading_in_plain_hiragana_loses_to_the_written_stage_name(self):
rows = entry_links.build(entry_links.defaults(), "天川空",
[ref("r18dev", "1091104")], [
diff --git a/tests/test_fastapi_api.py b/tests/test_fastapi_api.py
index a8d8fc70f..3a7b1cab8 100644
--- a/tests/test_fastapi_api.py
+++ b/tests/test_fastapi_api.py
@@ -1248,6 +1248,26 @@ class FastApiContractTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(entries[0]["url"], "https://javdb.com/actors/NPD3")
self.assertTrue(entries[1]["url"].startswith("https://missav.ws/dm3/cn/actresses/%"))
+ async def test_entity_entries_use_paired_current_han_stage_name(self):
+ connection = sqlite3.connect(self.db)
+ metadata = {"name_localization": {"source": "avdb-actor-mapping",
+ "jp": "水谷心音", "zh_cn": "水谷心音"}}
+ connection.execute("INSERT INTO entity(id,kind,canonical_name,normalized_name,metadata_json) "
+ "VALUES(3,'performer','水谷心音','水谷心音',?)",
+ (json.dumps(metadata, ensure_ascii=False),))
+ connection.execute("INSERT INTO asset_entity(asset_id,entity_id,role,source,confidence) "
+ "VALUES(1,3,'performer','test',1)")
+ connection.execute("INSERT INTO entity_external_ref(entity_id,provider,external_kind,external_id) "
+ "VALUES(3,'javdb','performer','sample')")
+ connection.execute("INSERT INTO entity_alias(entity_id,alias,normalized_alias,source,confidence) "
+ "VALUES(3,'藤崎りお','藤崎りお','r18:performer',1)")
+ connection.commit()
+ connection.close()
+ response = await self.client.get("/api/entity?t=secret&kind=performer&name=" + quote("水谷心音"))
+ self.assertEqual(response.status_code, 200, response.text)
+ missav = next(row for row in response.json()["entry_links"] if row["site"] == "missav")
+ self.assertEqual(missav["url"], "https://missav.ws/dm3/cn/actresses/" + quote("水谷心音"))
+
async def test_review_queue_is_readable_and_decisions_are_persisted(self):
response = await self.client.get("/api/review?t=secret")
self.assertEqual(response.status_code, 200)