Peach: Local-first video metadata scraper and media manager.
fix(upgrade): 定向重采旧 JavDB 身份与标题证据
longmeidao116159b13135
diff --git a/docs/SOURCING.md b/docs/SOURCING.mdindex 8cd51bd9d..e617a901c 100644--- a/docs/SOURCING.md+++ b/docs/SOURCING.md@@ -657,11 +657,9 @@ Instagram 的独立用户登录会话未取得,自动适配器不进正式依### 确认来源返回的是同一部片-- JavDB 的相同番号可能对应多个来源 ID,不能靠番号、时长或搜索顺序认作同一作品。网页与匿名 API- 都先汇总匹配的来源 ID;只有一个才取详情,多个时报告歧义。详情、图片及资源沿用这次确认的来源 ID,- 不在取资源时重新按番号选片。JavDB 旧的番号缓存、发现边车和原始快照须通过来源专属规则才可复用。-- JavDB 的 `origin-title` / `origin_title` 是原标题,`current-title` 是本地化显示标题。有原题时优先原题,- 显示标题独立留证;缺原题时保留显示标题,不凭文字语言猜测或删改星号片段。+- JavDB 的相同番号可能对应多个来源 ID,不能靠番号、时长或搜索顺序认作同一作品。网页与匿名 API 都先汇总匹配的来源 ID;只有一个才取详情,多个时报告歧义。详情、图片及资源沿用这次确认的来源 ID,不在取资源时重新按番号选片。旧的番号缓存、发现边车和原始快照须通过来源专属规则才可复用。+- JavDB 的 `origin-title` / `origin_title` 是原标题,`current-title` 是本地化显示标题。有原题时优先原题,显示标题独立留证;缺原题时保留显示标题,不凭文字语言猜测或删改星号片段。+- 升级补修通过独立的文字规则回执定向失效旧 JavDB 库内快照,并选择对应影片重新取证;原 JSON、人工字段、既有取消和延期记录保留。旧 JavDB 候选不参与自动采纳,当前候选随来源身份和规则版本进入既有复核流程,不清空其他来源正常证据。- 来源返回的番号必须和查询的番号比对过才算命中。javbus 一侧是拿番号做关键词搜索取首个结果,搜不到就返回近似的别人(`SA-104 → AVSA-104`、`AR-301 → STAR-3016`)。不比对时大部分匹配番号根本不对,整个 MIB 目录(韩国内容)曾因此被写上日本厂牌、系列和标题,还靠这些假证据升级成 JAV。diff --git a/src/peach/metadata_text_repair.py b/src/peach/metadata_text_repair.pyindex c0204e0f7..5e259cf0c 100644--- a/src/peach/metadata_text_repair.py+++ b/src/peach/metadata_text_repair.py@@ -8,6 +8,7 @@ from pathlib import Pathfrom .field_owners import auto_owner, owner_offrom . import artifact_rules, upgrade_rules+from .javdb_identity import current_evidencefrom .metadata import extract_peach_fields, identifies_code, source_text_is_valid, validate_provider_codefrom .metadata_auto_apply import _apply_metadata_candidate, metadata_auto_apply_candidatefrom .metadata_policy import SOURCE_SPECS@@ -17,7 +18,7 @@ FIELDS = {'title': 'catalog_title', 'original_title': 'original_title',def _damaged_snapshots(connection, root: Path) -> tuple[list[dict], list[int]]:- """只失效已知来源的损坏文字快照,并选择同番号资产重新采集。"""+ """失效损坏文字或旧 JavDB 绑定快照,保留原文并选择对应资产重采。"""snapshots, affected = [], []for path in root.glob('*.json'):code, separator, source = path.stem.rpartition('-')@@ -29,7 +30,8 @@ def _damaged_snapshots(connection, root: Path) -> tuple[list[dict], list[int]]:payload = json.loads(path.read_text(encoding='utf-8'))except (OSError, ValueError):continue- if not isinstance(payload, dict) or source_text_is_valid(payload):+ if not isinstance(payload, dict) or (source_text_is_valid(payload)+ and (source != 'javdb' or current_evidence(payload))):continuesnapshots.append({'path': str(path), 'fingerprint': before})affected.extend(row[0] for row in connection.execute(@@ -61,22 +63,26 @@ def _candidates(root: Path, code: str, field: str) -> list[dict]:continueexcept (ValueError, UnicodeError):continue- if not isinstance(payload, dict) or not identifies_code(code, payload):+ if (not isinstance(payload, dict) or not identifies_code(code, payload)+ or source == 'javdb' and not current_evidence(payload)):continuevalue = extract_peach_fields(payload).get(field)if not value:continuekey = hashlib.sha256(json.dumps([source, value['value']], ensure_ascii=False,sort_keys=True).encode()).hexdigest()- candidates.append({**value, 'candidate_key': key, 'source': source,- 'provider_id': payload.get('id'), 'content_id': payload.get('content_id'),- 'source_url': payload.get('source_url'), 'raw_snapshot': str(path),- 'confidence': .75})+ candidate = {**value, 'candidate_key': key, 'source': source,+ 'provider_id': payload.get('id'), 'content_id': payload.get('content_id'),+ 'source_url': payload.get('source_url'), 'raw_snapshot': str(path),+ 'confidence': .75}+ if source == 'javdb':+ candidate.update(javdb_rule=payload['javdb_rule'], source_id=payload.get('source_id'))+ candidates.append(candidate)return candidatesdef collect(connection, snapshots: Path) -> dict:- """只选中有损坏机器字段或来源关系的资产;人物规范名保留给复核。"""+ """选中损坏文字与失效来源快照的资产;人物规范名保留给复核。"""marker = '\ufffd'tests = ' OR '.join(f'instr(COALESCE(a.{column},\'\'),?)>0' for column in FIELDS.values())assets = connection.execute(diff --git a/src/peach/upgrade_rules.py b/src/peach/upgrade_rules.pyindex 1be6e89c8..c42341b6d 100644--- a/src/peach/upgrade_rules.py+++ b/src/peach/upgrade_rules.py@@ -6,5 +6,5 @@ LOCAL_POSTERS = PREVIEWS + ':local'PROCESSING = "processing-v2"PUBLIC_SNAPSHOTS = 2PROVIDER_EVIDENCE = 2-PROVIDER_TEXT = 2+PROVIDER_TEXT = 3OFFICIAL_COVER_CATALOG_MISSES = 1diff --git a/tests/test_upgrade_repairs.py b/tests/test_upgrade_repairs.pyindex 92ccaec30..bda03561c 100644--- a/tests/test_upgrade_repairs.py+++ b/tests/test_upgrade_repairs.py@@ -15,6 +15,80 @@ from support.ledger import fresh_ledgerclass UpgradeRepairTests(unittest.TestCase):+ def test_javdb_upgrade_requeues_legacy_evidence_and_keeps_manual_values(self):+ from peach import artifact_rules, upgrade_rules+ self.asset(1, code='EXM-001', duration=1)+ self.asset(2, code='EXM-002', duration=1)+ with self.database.write_transaction() as connection:+ connection.execute('UPDATE asset SET catalog_title=?,field_owners=? WHERE id=1',+ ('人工题名', json.dumps({'catalog_title':'user:manual'})))+ with patch.object(upgrade_repairs, 'REVISION', f'{upgrade_repairs.CATALOG_REVISION}:text-2'):+ self.service.prepare()+ previous=upgrade_repairs._read(self.service.receipt)+ previous.update(queued={'previous-task':99}, deferred=[42], dispatch_complete=True)+ self.service._save_receipt(previous)+ directory=self.root/'sources/library-metadata'+ directory.mkdir(parents=True,exist_ok=True)+ legacy=directory/'EXM-001-javdb.json'+ current=directory/'EXM-002-javdb.json'+ other=directory/'EXM-001-dmm.json'+ payloads=((legacy,{'id':'EXM-001','title':'来源显示题名'}),+ (current,{'id':'EXM-002','title':'Original title * Limited','javdb_rule':2,+ 'source_id':'CurrentMovie'}),+ (other,{'id':'EXM-001','title':'其他来源题名'}))+ for path,payload in payloads:+ path.write_text(json.dumps(payload,ensure_ascii=False),encoding='utf-8')+ artifact_rules.remember(path,str(upgrade_rules.PROVIDER_EVIDENCE))+ original={path:path.read_bytes() for path,_ in payloads}+ self.service.prepare()+ self.assertFalse(artifact_rules.current(legacy,str(upgrade_rules.PROVIDER_EVIDENCE)))+ self.assertTrue(artifact_rules.current(current,str(upgrade_rules.PROVIDER_EVIDENCE)))+ self.assertTrue(artifact_rules.current(other,str(upgrade_rules.PROVIDER_EVIDENCE)))+ for path,content in original.items():+ self.assertEqual(path.read_bytes(),content)+ receipt=upgrade_repairs._read(self.service.receipt)+ self.assertEqual(receipt['text_media_ids'],[1])+ self.assertEqual(receipt['queued'],{'previous-task':99})+ self.assertEqual(receipt['deferred'],[42])+ self.assertFalse(receipt['dispatch_complete'])+ self.assertEqual(receipt['local']['changed']['text_snapshots_invalidated'],1)+ with self.database.read_connection() as connection:+ self.assertEqual(connection.execute('SELECT catalog_title FROM asset WHERE id=1').fetchone()[0],+ '人工题名')+ self.service.prepare()+ self.assertEqual(len(list(self.root.glob('ledger.pre-upgrade-*.db'))),2)++ def test_legacy_javdb_title_is_excluded_from_machine_text_repair(self):+ from peach import metadata_text_repair+ self.damaged_asset(owner='auto:javdb')+ directory=self.root/'sources/library-metadata'+ directory.mkdir(parents=True,exist_ok=True)+ (directory/'EXM-001-javdb.json').write_text(+ json.dumps({'id':'EXM-001','title':'旧绑定显示题名'}),encoding='utf-8')+ with self.database.read_connection() as connection:+ plan=metadata_text_repair.collect(connection,directory)+ row=next(row for row in plan['rows'] if row['field']=='title')+ self.assertEqual(row['group']['candidates'],[])+ self.assertEqual(row['action'],'review')++ def test_current_javdb_original_is_accepted_for_empty_machine_text(self):+ from peach import metadata_auto_apply, metadata_text_repair+ self.asset(1,code='EXM-001',duration=1)+ with self.database.write_transaction() as connection:+ connection.execute('UPDATE asset SET name=? WHERE id=1',('EXM-001.mp4',))+ directory=self.root/'sources/library-metadata'+ directory.mkdir(parents=True,exist_ok=True)+ (directory/'EXM-001-javdb.json').write_text(json.dumps({+ 'id':'EXM-001','title':'元の題名','origin_title':'元の題名',+ 'javdb_rule':2,'source_id':'AbC123','source_url':'https://javdb.com/v/AbC123'}),encoding='utf-8')+ with self.database.read_connection() as connection:+ candidate=metadata_auto_apply.metadata_auto_apply_candidate(connection,{+ 'asset_id':1,'code':'EXM-001','field':'title','current_value':'',+ 'candidates':metadata_text_repair._candidates(directory,'EXM-001','title')})+ self.assertIsNotNone(candidate)+ self.assertEqual(candidate['value'],'元の題名')+ self.assertEqual(candidate['source_id'],'AbC123')+def test_encoding_upgrade_invalidates_damaged_snapshots_after_a_text_one_receipt(self):from peach import artifact_rules, upgrade_rulesself.asset(1, code='EXM-001', duration=1)@@ -115,7 +189,7 @@ class UpgradeRepairTests(unittest.TestCase):with patch('peach.library_processing._people_allowed',return_value=False), patch('peach.seed_followup.plan',return_value=[]), patch('peach.media_sources.cost_for',return_value='free'):planned=self.service._plan(receipt)- self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-2:1'])+ self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-3:1'])def test_text_only_dispatch_does_not_plan_unrelated_people(self):from peach.settings_file import load_config@@ -126,7 +200,7 @@ class UpgradeRepairTests(unittest.TestCase):'peach.seed_followup.plan',return_value=[]), patch('peach.media_sources.cost_for',return_value='free'), patch('peach.performer_alias_followup.stock',side_effect=AssertionError('不能重放人物补修')):planned=self.service._plan(receipt)- self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-2:1'])+ self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-3:1'])def test_completed_catalog_dispatch_preserves_deferred_work_across_text_revision(self):with patch.object(upgrade_repairs,'REVISION',upgrade_repairs.CATALOG_REVISION):