Peach: Local-first video metadata scraper and media manager.
fix(upgrade): 番号改回短号后撤下旧番号取到的候选与快照并重采
longmeidaofd654e99a593
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6053f4fdb..488c9574b 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -33,6 +33,7 @@ Removed / Fixed / Security。### 修复- **采集**:JavDB 匿名接口的一个主机连不上时自动换下一个,三个都不通才暂停刮削(#315)。+- **复核**:番号改回短号的影片(如 `MCDV-24`)复核页不再摆出按旧番号取到的资料,升级时撤下并按现番号重采,已批准的字段不动(#296)。- **馆藏**:入库、采集和 JAV 新作、发现页认女优与厂牌更快。- **界面**:后台任务写入资料时和重启后,目录、索引与统计页打开不再等重算。- **界面**:任务中心打开从约 1.7 秒缩短到约 0.1 秒。diff --git a/src/peach/upgrade_repairs.py b/src/peach/upgrade_repairs.pyindex d74a58b1d..c1adea568 100644--- a/src/peach/upgrade_repairs.py+++ b/src/peach/upgrade_repairs.py@@ -96,6 +96,73 @@ def stale_filename_codes(connection) -> list[dict]:return rows+#: 撤过旧番号证据的影片按现番号重采时,补全任务键上的规则段。上一轮改回番号时已按+#: `processing-v2` 派过同一部片,键相同就会被当成派过而跳掉。+CODE_EVIDENCE_RULE = 'code-evidence-1'+CANDIDATE_FILE = 'library-metadata-field-candidates.csv'+++def stale_code_evidence(connection, candidates: Path, sources: Path) -> dict:+ """番号改过之后,旧番号下取到的资料候选和来源快照(#296)。++ `MCDV-24` 曾被补零成 `MCDV-024`,按它从 r18.dev 取回的是 2004 年的另一部片;番号改回之后,+ 候选表那一行的 `code` 还写着旧番号,复核页按它核身份,那组资料就一直摆着。++ 判据是候选级的:这一行记的番号与影片现番号不是同一部片(`same_release_code`),且这条+ 候选认得旧番号、不认现番号。同一部片只是写法变了(`HEYZO-707` 改成 `HEYZO-0707`)的整行+ 放过;两个番号都不认的候选身份不明,也放过。用户批准过的行整行不动,那是人的结论。+ 旧番号的快照只在库里已没有任何影片还是那部片时失效:真有 2004 年那部 `MCDV-024` 的,快照是对的。+ """+ from .catalog_rules import same_release_code+ from .metadata_auto_apply import _candidate_identifies_code+ from .review_csv import read_rows+ rows = read_rows(candidates, missing_ok=True)+ codes = {row[0]: row[1] for row in connection.execute(+ "SELECT id,code FROM asset WHERE medium='video' AND disposal IS NULL AND code IS NOT NULL AND code<>''")}+ approved = {row[0] for row in connection.execute(+ "SELECT item_key FROM review_decision WHERE category='metadata_fields' AND status='approved'")}+ kept, withdrawn, held, changed = [], [], [], False+ for row in rows:+ old = str(row.get('code') or '').strip()+ try:+ asset_id = int(row.get('asset_id') or 0)+ except ValueError:+ asset_id = 0+ current = codes.get(asset_id)+ if not old or not current or same_release_code(old, current):+ kept.append(row)+ continue+ if row['item_key'] in approved:+ held.append(row['item_key'])+ kept.append(row)+ continue+ try:+ choices = [choice for choice in json.loads(row.get('candidates_json') or '[]') if isinstance(choice, dict)]+ except ValueError:+ kept.append(row)+ continue+ stale = [choice for choice in choices+ if _candidate_identifies_code(old, choice) and not _candidate_identifies_code(current, choice)]+ remaining = [choice for choice in choices if choice not in stale]+ changed = True+ if stale:+ withdrawn.append({'item_key': row['item_key'], 'asset_id': asset_id, 'code': old, 'target': current,+ 'candidates': stale})+ if remaining:+ # 留下的候选按现番号核身份;`refresh_current_values` 也按这一栏找影片。+ kept.append({**row, 'code': current, 'query': current, 'source_count': len(remaining),+ 'candidates_json': json.dumps(remaining, ensure_ascii=False)})+ snapshots = []+ pairs = {(row['code'], str(choice.get('source') or '')) for row in withdrawn for choice in row['candidates']}+ for old, source in sorted(pairs):+ path = sources / f'{old}-{source}.json'+ if (source and path.is_file() and not path.is_symlink()+ and not any(same_release_code(old, code) for code in codes.values())):+ snapshots.append(str(path))+ return {'rows': kept if changed else None, 'withdrawn': withdrawn, 'snapshots': snapshots,+ 'approved_kept': held, 'asset_ids': sorted({row['asset_id'] for row in withdrawn})}++def _catalog_plan(connection, sources: Path, *, catalog: bool = True) -> dict:text = metadata_text_repair.collect(connection, sources / 'library-metadata')if not catalog:@@ -234,6 +301,7 @@ class UpgradeRepairs:(CATALOG_REVISION, CATALOG_REVISION + ':text-*')).fetchone()if applied is not None:self._restore_receipt(json.loads(applied[0]))+ self._withdraw_stale_code_evidence()returnself._restore_receipt(None)sources = Path(contract.follow_sources_root)@@ -267,12 +335,48 @@ class UpgradeRepairs:+ plan['text']['asset_ids'])),'queued': dict(previous.get('queued', {})),'deferred': list(previous.get('deferred', [])),+ 'code_media_ids': list(previous.get('code_media_ids', [])),'dispatch_complete': bool(previous.get('dispatch_complete')) and not plan['text']['asset_ids'],'text_only': prior is not None,'prior_revision': prior[1] if prior else None}connection.execute('INSERT INTO ledger_upgrade_repair(id,applied_at,result_json) VALUES(?,?,?)',(REVISION, str(time.time()), json.dumps(receipt, ensure_ascii=False)))atomic_write_text(self.receipt, json.dumps(receipt, ensure_ascii=False))+ self._withdraw_stale_code_evidence()++ def _withdraw_stale_code_evidence(self) -> None:+ """撤掉旧番号下取到的候选与快照(`stale_code_evidence`),被撤的影片按现番号重采。++ 每份回执做一次,在 `prepare` 里、处理任务恢复之前做:那时没有采集任务拿着候选表。+ 撤下的候选原样记进回执的 `code_evidence`,快照只加失效标记、正文不动,都能按它整批还原。+ """+ from .library_processing import FIELDS+ from .review_csv import write_rows+ output = Path(self.contract.candidate_root) / CANDIDATE_FILE+ sources = Path(self.contract.follow_sources_root) / 'library-metadata'+ with self.contract.database.write_transaction() as connection:+ stored = connection.execute('SELECT result_json FROM ledger_upgrade_repair WHERE id=?',+ (REVISION,)).fetchone()+ receipt = json.loads(stored[0]) if stored else None+ if receipt is None or 'code_evidence' in receipt:+ return+ plan = stale_code_evidence(connection, output, sources)+ if plan['rows'] is not None:+ write_rows(output, FIELDS, plan['rows'], atomic=True)+ for path in plan['snapshots']:+ artifact_rules.remember(Path(path), f'invalid:{CODE_EVIDENCE_RULE}')+ receipt['code_evidence'] = {'withdrawn': plan['withdrawn'], 'snapshots': plan['snapshots'],+ 'approved_kept': plan['approved_kept']}+ if plan['asset_ids']:+ receipt['code_media_ids'] = sorted(set(receipt.get('code_media_ids', [])) | set(plan['asset_ids']))+ receipt['media_ids'] = sorted(set(receipt.get('media_ids', [])) | set(plan['asset_ids']))+ receipt['dispatch_complete'] = False+ connection.execute('UPDATE ledger_upgrade_repair SET result_json=? WHERE id=?',+ (json.dumps(receipt, ensure_ascii=False), REVISION))+ atomic_write_text(self.receipt, json.dumps(receipt, ensure_ascii=False))+ if plan['withdrawn']:+ LOGGER.info('旧番号下的资料候选:撤下 %s 行,快照失效 %s 份',+ len(plan['withdrawn']), len(plan['snapshots']))def _clear_stale_codes(self) -> None:"""清掉旧版从文件名误认的番号(`stale_filename_codes`),只做一次。"""@@ -420,6 +524,8 @@ class UpgradeRepairs:rule = upgrade_rules.PROCESSINGif row['id'] in receipt.get('text_media_ids', []):rule += f':text-{upgrade_rules.PROVIDER_TEXT}'+ if row['id'] in receipt.get('code_media_ids', []):+ rule += f':{CODE_EVIDENCE_RULE}'key = f'{MEDIA_TASK}:{rule}:{row["id"]}'if key in seen:continuediff --git a/tests/test_upgrade_repairs.py b/tests/test_upgrade_repairs.pyindex e5c5b9c86..4bed2fa6b 100644--- a/tests/test_upgrade_repairs.py+++ b/tests/test_upgrade_repairs.py@@ -383,6 +383,146 @@ class UpgradeRepairTests(unittest.TestCase):self.assertIn(1, receipt['media_ids'])self.assertNotIn(2, receipt['media_ids'])+ def code_evidence_fixture(self, assets):+ """#296 的现场:`MCDV-24` 曾按 `MCDV-024` 取过 r18.dev,标题那一行用户批准过。"""+ from peach import artifact_rules+ from peach.library_processing import FIELDS+ from peach.review_csv import write_rows+ with self.database.write_transaction() as connection:+ for key, name, code in assets:+ connection.execute('INSERT INTO asset(id,location,path,name,medium,code,field_owners,duration) '+ "VALUES(?,'local',?,?,'video',?,?,60)",+ (key, name, name, code, json.dumps({'code': 'scan:filename'})))+ connection.execute('UPDATE asset SET catalog_title=?,field_owners=? WHERE id=1',+ ('クリスタル映像の題名', json.dumps({'code': 'scan:filename',+ 'catalog_title': 'review:r18dev'})))+ connection.execute("INSERT INTO review_decision(category,item_key,status,note,updated_at) "+ "VALUES('metadata_fields','asset:1:title','approved',?,'2026-11-04T00:00:00Z')",+ (json.dumps({'candidate_key': 'r18-title', 'source': 'r18dev'}),))++ def candidate(key, source, provider_id, value, content_id=''):+ return {'candidate_key': key, 'source': source, 'provider_id': provider_id, 'content_id': content_id,+ 'value': value, 'display_value': value, 'source_url': ''}++ def row(asset_id, field, code, *candidates):+ return {'item_key': f'asset:{asset_id}:{field}', 'code': code, 'query': code, 'asset_id': asset_id,+ 'asset_path': f'{asset_id}.mp4', 'field': field, 'field_label': field, 'current_value': '',+ 'candidates_json': json.dumps(list(candidates), ensure_ascii=False),+ 'source_count': len(candidates), 'source_profile': 'library', 'policy_version': 'library-v1',+ 'status': 'candidate'}++ old = ('MCDV-024', '49mcdv024')+ rows = [row(1, 'title', 'MCDV-024', candidate('r18-title', 'r18dev', *old[:1], 'クリスタル映像の題名', old[1])),+ row(1, 'studio', 'MCDV-024', candidate('r18-studio', 'r18dev', old[0], 'クリスタル映像', old[1]),+ candidate('javdb-studio', 'javdb', 'MCDV-24', 'メルシーボークー'),+ candidate('nfo-studio', 'local_nfo', '', 'メルシーボークー')),+ row(1, 'release_date', 'MCDV-024', candidate('r18-date', 'r18dev', old[0], '2004-12-10', old[1])),+ # 同一部片只是写法变了:番号补修把 `HEYZO-707` 改成了 `HEYZO-0707`,候选照样成立。+ row(2, 'title', 'HEYZO-707', candidate('heyzo-title', 'javdb', 'HEYZO-707', 'HEYZO の題名')),+ row(3, 'title', 'ABP-001', candidate('abp-title', 'r18dev', 'ABP-001', 'ABP の題名', 'abp00001'))]+ if any(key == 4 for key, _, _ in assets):+ rows.append(row(4, 'title', 'MCDV-024', candidate('mcdv024-title', 'r18dev', *old[:1], '2004 年那部', old[1])))+ output = self.contract.candidate_root / upgrade_repairs.CANDIDATE_FILE+ output.parent.mkdir(parents=True, exist_ok=True)+ write_rows(output, FIELDS, rows, fill_missing=True)+ directory = self.root / 'sources/library-metadata'+ directory.mkdir(parents=True, exist_ok=True)+ snapshots = {}+ for name in ('MCDV-024-r18dev', 'MCDV-24-javdb', 'HEYZO-707-javdb', 'ABP-001-r18dev'):+ path = directory / f'{name}.json'+ path.write_text(json.dumps({'id': name.rsplit('-', 1)[0]}), encoding='utf-8')+ artifact_rules.remember(path, str(upgrade_rules.PROVIDER_EVIDENCE))+ snapshots[name] = path+ return output, snapshots++ def candidate_rows(self, output):+ from peach.review_csv import read_rows+ return {row['item_key']: row for row in read_rows(output)}++ def test_reverted_short_code_withdraws_old_code_evidence_and_requeues_once(self):+ # 报告者的现场:番号早已改回 `MCDV-24`、那一轮也派过补全,旧番号的候选和快照还在。+ from peach import artifact_rules+ from peach.settings_file import load_config+ output, snapshots = self.code_evidence_fixture(+ ((1, 'mcdv-24.mp4', 'MCDV-24'), (2, 'HEYZO-0707.mp4', 'HEYZO-0707'), (3, 'ABP-001.mp4', 'ABP-001')))+ with self.database.write_transaction() as connection:+ connection.execute('INSERT INTO ledger_upgrade_repair(id,applied_at,result_json) VALUES(?,?,?)',+ (upgrade_repairs.REVISION, '1', json.dumps({+ 'revision': upgrade_repairs.REVISION, 'media_ids': [1], 'deferred': [],+ 'queued': {'upgrade-library:processing-v2:1': 99}, 'dispatch_complete': True})))+ before = self.candidate_rows(output)+ snapshot_bytes = {name: path.read_bytes() for name, path in snapshots.items()}+ self.service.prepare()+ after = self.candidate_rows(output)+ # 批准过的标题行原样留着;未批准的行只摘认旧番号的候选,留下的改按现番号核身份。+ self.assertEqual(after['asset:1:title'], before['asset:1:title'])+ studio = after['asset:1:studio']+ self.assertEqual([choice['candidate_key'] for choice in json.loads(studio['candidates_json'])],+ ['javdb-studio', 'nfo-studio'])+ self.assertEqual((studio['code'], studio['query'], studio['source_count']), ('MCDV-24', 'MCDV-24', '2'))+ self.assertNotIn('asset:1:release_date', after)+ self.assertEqual(after['asset:2:title'], before['asset:2:title'])+ self.assertEqual(after['asset:3:title'], before['asset:3:title'])+ with self.database.read_connection() as connection:+ title, owners = connection.execute('SELECT catalog_title,field_owners FROM asset WHERE id=1').fetchone()+ self.assertEqual((title, json.loads(owners)['catalog_title']), ('クリスタル映像の題名', 'review:r18dev'))+ evidence = str(upgrade_rules.PROVIDER_EVIDENCE)+ self.assertFalse(artifact_rules.current(snapshots['MCDV-024-r18dev'], evidence))+ for name in ('MCDV-24-javdb', 'HEYZO-707-javdb', 'ABP-001-r18dev'):+ self.assertTrue(artifact_rules.current(snapshots[name], evidence), name)+ self.assertEqual({name: path.read_bytes() for name, path in snapshots.items()}, snapshot_bytes)+ receipt = upgrade_repairs._read(self.service.receipt)+ self.assertEqual(receipt['code_media_ids'], [1])+ self.assertEqual(receipt['media_ids'], [1])+ self.assertFalse(receipt['dispatch_complete'])+ self.assertEqual(receipt['code_evidence']['approved_kept'], ['asset:1:title'])+ self.assertEqual(sorted(row['item_key'] for row in receipt['code_evidence']['withdrawn']),+ ['asset:1:release_date', 'asset:1:studio'])+ # 上一轮按 `processing-v2:1` 派过,这次换规则段才会真的重采。+ self.service.config = load_config(environ={'PEACH_DATA_ROOT': str(self.root)})+ with patch('peach.library_processing._people_allowed', return_value=False), patch(+ 'peach.seed_followup.plan', return_value=[]), patch('peach.media_sources.cost_for', return_value='free'):+ planned = self.service._plan(receipt)+ self.assertEqual([row.key for row in planned], ['upgrade-library:processing-v2:code-evidence-1:1'])+ # 一份回执只撤一次,再启动不重做、不重排。+ stored = output.read_bytes()+ self.service.prepare()+ self.assertEqual(output.read_bytes(), stored)+ self.assertEqual(upgrade_repairs._read(self.service.receipt), receipt)++ def test_old_code_snapshot_stays_while_another_video_is_that_release(self):+ from peach import artifact_rules+ output, snapshots = self.code_evidence_fixture(+ ((1, 'mcdv-24.mp4', 'MCDV-24'), (2, 'HEYZO-0707.mp4', 'HEYZO-0707'), (3, 'ABP-001.mp4', 'ABP-001'),+ (4, 'MCDV-024.mp4', 'MCDV-024')))+ before = self.candidate_rows(output)+ self.service.prepare()+ after = self.candidate_rows(output)+ self.assertNotIn('asset:1:release_date', after)+ self.assertEqual(after['asset:4:title'], before['asset:4:title'])+ self.assertTrue(artifact_rules.current(snapshots['MCDV-024-r18dev'], str(upgrade_rules.PROVIDER_EVIDENCE)))+ self.assertNotIn(4, upgrade_repairs._read(self.service.receipt)['code_media_ids'])++ def test_catalog_upgrade_reverting_a_short_code_withdraws_its_old_candidates(self):+ from peach import artifact_rules+ output, snapshots = self.code_evidence_fixture(+ ((1, 'mcdv-24.mp4', 'MCDV-024'), (2, 'HEYZO-0707.mp4', 'HEYZO-0707'), (3, 'ABP-001.mp4', 'ABP-001')))+ with self.database.write_transaction() as connection:+ old = upgrade_repairs.CATALOG_REVISION.replace(upgrade_rules.CATALOG, 'catalog-v2')+ connection.execute('INSERT INTO ledger_upgrade_repair(id,applied_at,result_json) VALUES(?,?,?)',+ (old, '1', json.dumps({'revision': old})))+ self.service.prepare()+ with self.database.read_connection() as connection:+ self.assertEqual(connection.execute('SELECT code FROM asset WHERE id=1').fetchone()[0], 'MCDV-24')+ after = self.candidate_rows(output)+ self.assertNotIn('asset:1:release_date', after)+ self.assertEqual(after['asset:1:studio']['code'], 'MCDV-24')+ self.assertFalse(artifact_rules.current(snapshots['MCDV-024-r18dev'], str(upgrade_rules.PROVIDER_EVIDENCE)))+ receipt = upgrade_repairs._read(self.service.receipt)+ self.assertEqual(receipt['code_media_ids'], [1])+ self.assertNotIn(2, receipt['media_ids'])+ self.assertNotIn(3, receipt['media_ids'])+def test_failure_rolls_back_repairs_and_marker_together(self):self.asset(1)original = upgrade_repairs._catalog_apply