Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(upgrade): 番号改回短号后撤下旧番号取到的候选与快照并重采

longmeidaofd654e99a593
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6053f4fdb..488c9574b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -33,6 +33,7 @@ Removed / Fixed / Security。
### 修复
- **采集**:JavDB 匿名接口的一个主机连不上时自动换下一个,三个都不通才暂停刮削(#315)。
+- **复核**:番号改回短号的影片(如 `MCDV-24`)复核页不再摆出按旧番号取到的资料,升级时撤下并按现番号重采,已批准的字段不动(#296)。
- **馆藏**:入库、采集和 JAV 新作、发现页认女优与厂牌更快。
- **界面**:后台任务写入资料时和重启后,目录、索引与统计页打开不再等重算。
- **界面**:任务中心打开从约 1.7 秒缩短到约 0.1 秒。
diff --git a/src/peach/upgrade_repairs.py b/src/peach/upgrade_repairs.py
index d74a58b1d..c1adea568 100644
--- a/src/peach/upgrade_repairs.py
+++ b/src/peach/upgrade_repairs.py
@@ -96,6 +96,73 @@ def stale_filename_codes(connection) -> list[dict]:
return rows
+#: 撤过旧番号证据的影片按现番号重采时,补全任务键上的规则段。上一轮改回番号时已按
+#: `processing-v2` 派过同一部片,键相同就会被当成派过而跳掉。
+CODE_EVIDENCE_RULE = 'code-evidence-1'
+CANDIDATE_FILE = 'library-metadata-field-candidates.csv'
+
+
+def stale_code_evidence(connection, candidates: Path, sources: Path) -> dict:
+ """番号改过之后,旧番号下取到的资料候选和来源快照(#296)。
+
+ `MCDV-24` 曾被补零成 `MCDV-024`,按它从 r18.dev 取回的是 2004 年的另一部片;番号改回之后,
+ 候选表那一行的 `code` 还写着旧番号,复核页按它核身份,那组资料就一直摆着。
+
+ 判据是候选级的:这一行记的番号与影片现番号不是同一部片(`same_release_code`),且这条
+ 候选认得旧番号、不认现番号。同一部片只是写法变了(`HEYZO-707` 改成 `HEYZO-0707`)的整行
+ 放过;两个番号都不认的候选身份不明,也放过。用户批准过的行整行不动,那是人的结论。
+ 旧番号的快照只在库里已没有任何影片还是那部片时失效:真有 2004 年那部 `MCDV-024` 的,快照是对的。
+ """
+ from .catalog_rules import same_release_code
+ from .metadata_auto_apply import _candidate_identifies_code
+ from .review_csv import read_rows
+ rows = read_rows(candidates, missing_ok=True)
+ codes = {row[0]: row[1] for row in connection.execute(
+ "SELECT id,code FROM asset WHERE medium='video' AND disposal IS NULL AND code IS NOT NULL AND code<>''")}
+ approved = {row[0] for row in connection.execute(
+ "SELECT item_key FROM review_decision WHERE category='metadata_fields' AND status='approved'")}
+ kept, withdrawn, held, changed = [], [], [], False
+ for row in rows:
+ old = str(row.get('code') or '').strip()
+ try:
+ asset_id = int(row.get('asset_id') or 0)
+ except ValueError:
+ asset_id = 0
+ current = codes.get(asset_id)
+ if not old or not current or same_release_code(old, current):
+ kept.append(row)
+ continue
+ if row['item_key'] in approved:
+ held.append(row['item_key'])
+ kept.append(row)
+ continue
+ try:
+ choices = [choice for choice in json.loads(row.get('candidates_json') or '[]') if isinstance(choice, dict)]
+ except ValueError:
+ kept.append(row)
+ continue
+ stale = [choice for choice in choices
+ if _candidate_identifies_code(old, choice) and not _candidate_identifies_code(current, choice)]
+ remaining = [choice for choice in choices if choice not in stale]
+ changed = True
+ if stale:
+ withdrawn.append({'item_key': row['item_key'], 'asset_id': asset_id, 'code': old, 'target': current,
+ 'candidates': stale})
+ if remaining:
+ # 留下的候选按现番号核身份;`refresh_current_values` 也按这一栏找影片。
+ kept.append({**row, 'code': current, 'query': current, 'source_count': len(remaining),
+ 'candidates_json': json.dumps(remaining, ensure_ascii=False)})
+ snapshots = []
+ pairs = {(row['code'], str(choice.get('source') or '')) for row in withdrawn for choice in row['candidates']}
+ for old, source in sorted(pairs):
+ path = sources / f'{old}-{source}.json'
+ if (source and path.is_file() and not path.is_symlink()
+ and not any(same_release_code(old, code) for code in codes.values())):
+ snapshots.append(str(path))
+ return {'rows': kept if changed else None, 'withdrawn': withdrawn, 'snapshots': snapshots,
+ 'approved_kept': held, 'asset_ids': sorted({row['asset_id'] for row in withdrawn})}
+
+
def _catalog_plan(connection, sources: Path, *, catalog: bool = True) -> dict:
text = metadata_text_repair.collect(connection, sources / 'library-metadata')
if not catalog:
@@ -234,6 +301,7 @@ class UpgradeRepairs:
(CATALOG_REVISION, CATALOG_REVISION + ':text-*')).fetchone()
if applied is not None:
self._restore_receipt(json.loads(applied[0]))
+ self._withdraw_stale_code_evidence()
return
self._restore_receipt(None)
sources = Path(contract.follow_sources_root)
@@ -267,12 +335,48 @@ class UpgradeRepairs:
+ plan['text']['asset_ids'])),
'queued': dict(previous.get('queued', {})),
'deferred': list(previous.get('deferred', [])),
+ 'code_media_ids': list(previous.get('code_media_ids', [])),
'dispatch_complete': bool(previous.get('dispatch_complete')) and not plan['text']['asset_ids'],
'text_only': prior is not None,
'prior_revision': prior[1] if prior else None}
connection.execute('INSERT INTO ledger_upgrade_repair(id,applied_at,result_json) VALUES(?,?,?)',
(REVISION, str(time.time()), json.dumps(receipt, ensure_ascii=False)))
atomic_write_text(self.receipt, json.dumps(receipt, ensure_ascii=False))
+ self._withdraw_stale_code_evidence()
+
+ def _withdraw_stale_code_evidence(self) -> None:
+ """撤掉旧番号下取到的候选与快照(`stale_code_evidence`),被撤的影片按现番号重采。
+
+ 每份回执做一次,在 `prepare` 里、处理任务恢复之前做:那时没有采集任务拿着候选表。
+ 撤下的候选原样记进回执的 `code_evidence`,快照只加失效标记、正文不动,都能按它整批还原。
+ """
+ from .library_processing import FIELDS
+ from .review_csv import write_rows
+ output = Path(self.contract.candidate_root) / CANDIDATE_FILE
+ sources = Path(self.contract.follow_sources_root) / 'library-metadata'
+ with self.contract.database.write_transaction() as connection:
+ stored = connection.execute('SELECT result_json FROM ledger_upgrade_repair WHERE id=?',
+ (REVISION,)).fetchone()
+ receipt = json.loads(stored[0]) if stored else None
+ if receipt is None or 'code_evidence' in receipt:
+ return
+ plan = stale_code_evidence(connection, output, sources)
+ if plan['rows'] is not None:
+ write_rows(output, FIELDS, plan['rows'], atomic=True)
+ for path in plan['snapshots']:
+ artifact_rules.remember(Path(path), f'invalid:{CODE_EVIDENCE_RULE}')
+ receipt['code_evidence'] = {'withdrawn': plan['withdrawn'], 'snapshots': plan['snapshots'],
+ 'approved_kept': plan['approved_kept']}
+ if plan['asset_ids']:
+ receipt['code_media_ids'] = sorted(set(receipt.get('code_media_ids', [])) | set(plan['asset_ids']))
+ receipt['media_ids'] = sorted(set(receipt.get('media_ids', [])) | set(plan['asset_ids']))
+ receipt['dispatch_complete'] = False
+ connection.execute('UPDATE ledger_upgrade_repair SET result_json=? WHERE id=?',
+ (json.dumps(receipt, ensure_ascii=False), REVISION))
+ atomic_write_text(self.receipt, json.dumps(receipt, ensure_ascii=False))
+ if plan['withdrawn']:
+ LOGGER.info('旧番号下的资料候选:撤下 %s 行,快照失效 %s 份',
+ len(plan['withdrawn']), len(plan['snapshots']))
def _clear_stale_codes(self) -> None:
"""清掉旧版从文件名误认的番号(`stale_filename_codes`),只做一次。"""
@@ -420,6 +524,8 @@ class UpgradeRepairs:
rule = upgrade_rules.PROCESSING
if row['id'] in receipt.get('text_media_ids', []):
rule += f':text-{upgrade_rules.PROVIDER_TEXT}'
+ if row['id'] in receipt.get('code_media_ids', []):
+ rule += f':{CODE_EVIDENCE_RULE}'
key = f'{MEDIA_TASK}:{rule}:{row["id"]}'
if key in seen:
continue
diff --git a/tests/test_upgrade_repairs.py b/tests/test_upgrade_repairs.py
index e5c5b9c86..4bed2fa6b 100644
--- a/tests/test_upgrade_repairs.py
+++ b/tests/test_upgrade_repairs.py
@@ -383,6 +383,146 @@ class UpgradeRepairTests(unittest.TestCase):
self.assertIn(1, receipt['media_ids'])
self.assertNotIn(2, receipt['media_ids'])
+ def code_evidence_fixture(self, assets):
+ """#296 的现场:`MCDV-24` 曾按 `MCDV-024` 取过 r18.dev,标题那一行用户批准过。"""
+ from peach import artifact_rules
+ from peach.library_processing import FIELDS
+ from peach.review_csv import write_rows
+ with self.database.write_transaction() as connection:
+ for key, name, code in assets:
+ connection.execute('INSERT INTO asset(id,location,path,name,medium,code,field_owners,duration) '
+ "VALUES(?,'local',?,?,'video',?,?,60)",
+ (key, name, name, code, json.dumps({'code': 'scan:filename'})))
+ connection.execute('UPDATE asset SET catalog_title=?,field_owners=? WHERE id=1',
+ ('クリスタル映像の題名', json.dumps({'code': 'scan:filename',
+ 'catalog_title': 'review:r18dev'})))
+ connection.execute("INSERT INTO review_decision(category,item_key,status,note,updated_at) "
+ "VALUES('metadata_fields','asset:1:title','approved',?,'2026-11-04T00:00:00Z')",
+ (json.dumps({'candidate_key': 'r18-title', 'source': 'r18dev'}),))
+
+ def candidate(key, source, provider_id, value, content_id=''):
+ return {'candidate_key': key, 'source': source, 'provider_id': provider_id, 'content_id': content_id,
+ 'value': value, 'display_value': value, 'source_url': ''}
+
+ def row(asset_id, field, code, *candidates):
+ return {'item_key': f'asset:{asset_id}:{field}', 'code': code, 'query': code, 'asset_id': asset_id,
+ 'asset_path': f'{asset_id}.mp4', 'field': field, 'field_label': field, 'current_value': '',
+ 'candidates_json': json.dumps(list(candidates), ensure_ascii=False),
+ 'source_count': len(candidates), 'source_profile': 'library', 'policy_version': 'library-v1',
+ 'status': 'candidate'}
+
+ old = ('MCDV-024', '49mcdv024')
+ rows = [row(1, 'title', 'MCDV-024', candidate('r18-title', 'r18dev', *old[:1], 'クリスタル映像の題名', old[1])),
+ row(1, 'studio', 'MCDV-024', candidate('r18-studio', 'r18dev', old[0], 'クリスタル映像', old[1]),
+ candidate('javdb-studio', 'javdb', 'MCDV-24', 'メルシーボークー'),
+ candidate('nfo-studio', 'local_nfo', '', 'メルシーボークー')),
+ row(1, 'release_date', 'MCDV-024', candidate('r18-date', 'r18dev', old[0], '2004-12-10', old[1])),
+ # 同一部片只是写法变了:番号补修把 `HEYZO-707` 改成了 `HEYZO-0707`,候选照样成立。
+ row(2, 'title', 'HEYZO-707', candidate('heyzo-title', 'javdb', 'HEYZO-707', 'HEYZO の題名')),
+ row(3, 'title', 'ABP-001', candidate('abp-title', 'r18dev', 'ABP-001', 'ABP の題名', 'abp00001'))]
+ if any(key == 4 for key, _, _ in assets):
+ rows.append(row(4, 'title', 'MCDV-024', candidate('mcdv024-title', 'r18dev', *old[:1], '2004 年那部', old[1])))
+ output = self.contract.candidate_root / upgrade_repairs.CANDIDATE_FILE
+ output.parent.mkdir(parents=True, exist_ok=True)
+ write_rows(output, FIELDS, rows, fill_missing=True)
+ directory = self.root / 'sources/library-metadata'
+ directory.mkdir(parents=True, exist_ok=True)
+ snapshots = {}
+ for name in ('MCDV-024-r18dev', 'MCDV-24-javdb', 'HEYZO-707-javdb', 'ABP-001-r18dev'):
+ path = directory / f'{name}.json'
+ path.write_text(json.dumps({'id': name.rsplit('-', 1)[0]}), encoding='utf-8')
+ artifact_rules.remember(path, str(upgrade_rules.PROVIDER_EVIDENCE))
+ snapshots[name] = path
+ return output, snapshots
+
+ def candidate_rows(self, output):
+ from peach.review_csv import read_rows
+ return {row['item_key']: row for row in read_rows(output)}
+
+ def test_reverted_short_code_withdraws_old_code_evidence_and_requeues_once(self):
+ # 报告者的现场:番号早已改回 `MCDV-24`、那一轮也派过补全,旧番号的候选和快照还在。
+ from peach import artifact_rules
+ from peach.settings_file import load_config
+ output, snapshots = self.code_evidence_fixture(
+ ((1, 'mcdv-24.mp4', 'MCDV-24'), (2, 'HEYZO-0707.mp4', 'HEYZO-0707'), (3, 'ABP-001.mp4', 'ABP-001')))
+ with self.database.write_transaction() as connection:
+ connection.execute('INSERT INTO ledger_upgrade_repair(id,applied_at,result_json) VALUES(?,?,?)',
+ (upgrade_repairs.REVISION, '1', json.dumps({
+ 'revision': upgrade_repairs.REVISION, 'media_ids': [1], 'deferred': [],
+ 'queued': {'upgrade-library:processing-v2:1': 99}, 'dispatch_complete': True})))
+ before = self.candidate_rows(output)
+ snapshot_bytes = {name: path.read_bytes() for name, path in snapshots.items()}
+ self.service.prepare()
+ after = self.candidate_rows(output)
+ # 批准过的标题行原样留着;未批准的行只摘认旧番号的候选,留下的改按现番号核身份。
+ self.assertEqual(after['asset:1:title'], before['asset:1:title'])
+ studio = after['asset:1:studio']
+ self.assertEqual([choice['candidate_key'] for choice in json.loads(studio['candidates_json'])],
+ ['javdb-studio', 'nfo-studio'])
+ self.assertEqual((studio['code'], studio['query'], studio['source_count']), ('MCDV-24', 'MCDV-24', '2'))
+ self.assertNotIn('asset:1:release_date', after)
+ self.assertEqual(after['asset:2:title'], before['asset:2:title'])
+ self.assertEqual(after['asset:3:title'], before['asset:3:title'])
+ with self.database.read_connection() as connection:
+ title, owners = connection.execute('SELECT catalog_title,field_owners FROM asset WHERE id=1').fetchone()
+ self.assertEqual((title, json.loads(owners)['catalog_title']), ('クリスタル映像の題名', 'review:r18dev'))
+ evidence = str(upgrade_rules.PROVIDER_EVIDENCE)
+ self.assertFalse(artifact_rules.current(snapshots['MCDV-024-r18dev'], evidence))
+ for name in ('MCDV-24-javdb', 'HEYZO-707-javdb', 'ABP-001-r18dev'):
+ self.assertTrue(artifact_rules.current(snapshots[name], evidence), name)
+ self.assertEqual({name: path.read_bytes() for name, path in snapshots.items()}, snapshot_bytes)
+ receipt = upgrade_repairs._read(self.service.receipt)
+ self.assertEqual(receipt['code_media_ids'], [1])
+ self.assertEqual(receipt['media_ids'], [1])
+ self.assertFalse(receipt['dispatch_complete'])
+ self.assertEqual(receipt['code_evidence']['approved_kept'], ['asset:1:title'])
+ self.assertEqual(sorted(row['item_key'] for row in receipt['code_evidence']['withdrawn']),
+ ['asset:1:release_date', 'asset:1:studio'])
+ # 上一轮按 `processing-v2:1` 派过,这次换规则段才会真的重采。
+ self.service.config = load_config(environ={'PEACH_DATA_ROOT': str(self.root)})
+ with patch('peach.library_processing._people_allowed', return_value=False), patch(
+ 'peach.seed_followup.plan', return_value=[]), patch('peach.media_sources.cost_for', return_value='free'):
+ planned = self.service._plan(receipt)
+ self.assertEqual([row.key for row in planned], ['upgrade-library:processing-v2:code-evidence-1:1'])
+ # 一份回执只撤一次,再启动不重做、不重排。
+ stored = output.read_bytes()
+ self.service.prepare()
+ self.assertEqual(output.read_bytes(), stored)
+ self.assertEqual(upgrade_repairs._read(self.service.receipt), receipt)
+
+ def test_old_code_snapshot_stays_while_another_video_is_that_release(self):
+ from peach import artifact_rules
+ output, snapshots = self.code_evidence_fixture(
+ ((1, 'mcdv-24.mp4', 'MCDV-24'), (2, 'HEYZO-0707.mp4', 'HEYZO-0707'), (3, 'ABP-001.mp4', 'ABP-001'),
+ (4, 'MCDV-024.mp4', 'MCDV-024')))
+ before = self.candidate_rows(output)
+ self.service.prepare()
+ after = self.candidate_rows(output)
+ self.assertNotIn('asset:1:release_date', after)
+ self.assertEqual(after['asset:4:title'], before['asset:4:title'])
+ self.assertTrue(artifact_rules.current(snapshots['MCDV-024-r18dev'], str(upgrade_rules.PROVIDER_EVIDENCE)))
+ self.assertNotIn(4, upgrade_repairs._read(self.service.receipt)['code_media_ids'])
+
+ def test_catalog_upgrade_reverting_a_short_code_withdraws_its_old_candidates(self):
+ from peach import artifact_rules
+ output, snapshots = self.code_evidence_fixture(
+ ((1, 'mcdv-24.mp4', 'MCDV-024'), (2, 'HEYZO-0707.mp4', 'HEYZO-0707'), (3, 'ABP-001.mp4', 'ABP-001')))
+ with self.database.write_transaction() as connection:
+ old = upgrade_repairs.CATALOG_REVISION.replace(upgrade_rules.CATALOG, 'catalog-v2')
+ connection.execute('INSERT INTO ledger_upgrade_repair(id,applied_at,result_json) VALUES(?,?,?)',
+ (old, '1', json.dumps({'revision': old})))
+ self.service.prepare()
+ with self.database.read_connection() as connection:
+ self.assertEqual(connection.execute('SELECT code FROM asset WHERE id=1').fetchone()[0], 'MCDV-24')
+ after = self.candidate_rows(output)
+ self.assertNotIn('asset:1:release_date', after)
+ self.assertEqual(after['asset:1:studio']['code'], 'MCDV-24')
+ self.assertFalse(artifact_rules.current(snapshots['MCDV-024-r18dev'], str(upgrade_rules.PROVIDER_EVIDENCE)))
+ receipt = upgrade_repairs._read(self.service.receipt)
+ self.assertEqual(receipt['code_media_ids'], [1])
+ self.assertNotIn(2, receipt['media_ids'])
+ self.assertNotIn(3, receipt['media_ids'])
+
def test_failure_rolls_back_repairs_and_marker_together(self):
self.asset(1)
original = upgrade_repairs._catalog_apply