Peach: Local-first video metadata scraper and media manager.
fix(upgrade): 独立登记文字补修并保留已有任务状态
longmeidao2fe1781fc34b
diff --git a/docs/REUSE.md b/docs/REUSE.mdindex 13a4e2a6b..9306d618b 100644--- a/docs/REUSE.md+++ b/docs/REUSE.md@@ -912,7 +912,7 @@ Python、npm 与 GitHub Actions 的版本由 `.github/dependabot.yml` 每周检来源文字中的 Unicode 替换字符 `U+FFFD` 表示解码时已有信息丢失。字段提取、既有候选选择和最终写入共用这一判据;含坏值的字段不参与来源优先级结算,原始快照保留。正常字段与正常中日文、重音字母不受影响,不使用启发式转码猜回原文。-存量文字补修复用字段归属、来源身份核对、候选结算与 `UpgradeRepairs` 的备份、事务和回执。仅机器归属的损坏标量字段可按同番号正常来源恢复;人物规范名及人工判断保留复核。待重采的资产使用独立文字规则身份,先前完成的升级任务不吞掉新补修。+存量文字补修复用字段归属、来源身份核对、候选结算与 `UpgradeRepairs` 的备份、事务和回执。仅机器归属的损坏标量字段可按同番号正常来源恢复;人物规范名及人工判断保留复核。已登记完成的目录补修不随文字规则重放;旧任务范围和队列继续保留。待重采的资产使用独立文字规则身份,先前完成的升级任务不吞掉新补修,尚在运行或已由用户取消的任务保持原状态。播放生命周期复用 [PR190](https://github.com/peach-mitao/peach/pull/190) 的 `e41222e312443b5d0fc7be87d2f7eca9c1d61055`:共享分片按消费者计数,最后消费者离开才取消生成,并等待子进程回收;文件响应保留外层取消与运行错误。头像确认的图像和数据库工作共用工作线程,安装按实体加锁。当前 ASGI 响应头中间件、SMB 输入和图像预处理保留 Peach 主线实现;未采用该 PR 的界面、依赖和测试调度改动。diff --git a/src/peach/api.py b/src/peach/api.pyindex c01b8388f..74a0c48ec 100644--- a/src/peach/api.py+++ b/src/peach/api.py@@ -128,7 +128,7 @@ class ResponseHeadersMiddleware:kind = headers.get('content-type','').split(';',1)[0].strip().lower()if kind.startswith(('image/','video/','audio/')):headers.update(MEDIA_SECURITY_HEADERS)- if (scope['method']=='GET' and message['status']==200 and kind=='text/html'+ if (message['status']==200 and kind=='text/html' and scope['method']=='GET'and Request(scope).cookies.get('peach_media_policy')!='1'):headers['Clear-Site-Data']='"cache"'policy=Response()diff --git a/src/peach/upgrade_repairs.py b/src/peach/upgrade_repairs.pyindex 72720b146..21625d030 100644--- a/src/peach/upgrade_repairs.py+++ b/src/peach/upgrade_repairs.py@@ -20,9 +20,9 @@ from .task_runs import MAX_FOLLOWUPSLOGGER = logging.getLogger(__name__)TASK_KEY = 'upgrade-repairs'MEDIA_TASK = 'upgrade-library'-REVISION = ':'.join((upgrade_rules.CATALOG, upgrade_rules.PREVIEWS, upgrade_rules.PROCESSING,- str(upgrade_rules.PUBLIC_SNAPSHOTS), str(upgrade_rules.PROVIDER_EVIDENCE),- str(upgrade_rules.PROVIDER_TEXT)))+CATALOG_REVISION = ':'.join((upgrade_rules.CATALOG, upgrade_rules.PREVIEWS, upgrade_rules.PROCESSING,+ str(upgrade_rules.PUBLIC_SNAPSHOTS), str(upgrade_rules.PROVIDER_EVIDENCE)))+REVISION = f'{CATALOG_REVISION}:text-{upgrade_rules.PROVIDER_TEXT}'def _read(path: Path) -> dict:@@ -54,7 +54,10 @@ def _manual_name(connection, entity_id: int) -> bool:(entity_id,)).fetchone())-def _catalog_plan(connection, sources: Path) -> dict:+def _catalog_plan(connection, sources: Path, *, catalog: bool = True) -> dict:+ text = metadata_text_repair.collect(connection, sources / 'library-metadata')+ if not catalog:+ return {'codes':[], 'fields':[], 'names':[], 'aliases':[], 'text':text, 'catalog':False}from .catalog_rules import compile_code_patterns, release_code_from_filename, same_release_codefrom .profiles import read_settingspatterns = compile_code_patterns(read_settings(connection).get('codePatterns'))@@ -76,10 +79,12 @@ def _catalog_plan(connection, sources: Path) -> dict:if is_protected(row['source']):row.update(verdict='review', note='人工别名保持原样')return {'codes': codes, 'fields': fields, 'names': names, 'aliases': aliases,- 'text': metadata_text_repair.collect(connection, sources / 'library-metadata')}+ 'text': text, 'catalog': True}def _catalog_apply(connection, plan: dict) -> dict:+ if not plan.get('catalog', True):+ return {'codes':0, 'probe_retry':0, **metadata_text_repair.apply(connection, plan['text'])}from datetime import datetime, timezonechanged = {}for row in plan['codes']:@@ -172,16 +177,19 @@ class UpgradeRepairs:with contract.database.read_connection() as connection:applied = connection.execute('SELECT result_json FROM ledger_upgrade_repair WHERE id=?',(REVISION,)).fetchone()+ prior = connection.execute('SELECT result_json FROM ledger_upgrade_repair WHERE id=?',+ (CATALOG_REVISION,)).fetchone()if applied is not None:self._restore_receipt(json.loads(applied[0]))returnself._restore_receipt(None)sources = Path(contract.follow_sources_root)+ previous = json.loads(prior[0]) if prior is not None else {}with contract.database.read_connection() as connection:- plan = _catalog_plan(connection, sources)+ plan = _catalog_plan(connection, sources, catalog=prior is None)before = _counts(connection)failed = [row[0] for row in connection.execute(- "SELECT id FROM asset WHERE medium='video' AND duration<=0 AND disposal IS NULL")]+ "SELECT id FROM asset WHERE medium='video' AND duration<=0 AND disposal IS NULL")] if prior is None else []atomic_write_text(self.root / 'catalog-review.json', json.dumps(plan, ensure_ascii=False, indent=2))backup = contract.db_path.with_name(f'ledger.pre-upgrade-{time.time_ns()}.db')sqlite_backup(contract.db_path, backup)@@ -189,6 +197,7 @@ class UpgradeRepairs:and row['action'] not in ('replace', 'rename') for row in plan['fields'] + plan['names'])review += sum(row['verdict'] == 'review' for row in plan['aliases'])review += sum(row['action'] == 'review' for row in plan['text']['rows'])+ review += previous.get('local', {}).get('review_count', 0)with contract.database.write_transaction() as connection:violations = {tuple(row) for row in connection.execute('PRAGMA foreign_key_check')}changed = _catalog_apply(connection, plan)@@ -201,7 +210,9 @@ class UpgradeRepairs:'review_count': review, 'before': before, 'after': after, 'backup': str(backup)},'text_media_ids': plan['text']['asset_ids'],'media_ids': sorted(set(failed + [row['id'] for row in plan['codes']]- + plan['text']['asset_ids'])), 'queued': {}}+ + previous.get('media_ids', [])+ + plan['text']['asset_ids'])),+ 'queued': dict(previous.get('queued', {})), 'prior_revision': CATALOG_REVISION if prior else None}connection.execute('INSERT INTO ledger_upgrade_repair(id,applied_at,result_json) VALUES(?,?,?)',(REVISION, str(time.time()), json.dumps(receipt, ensure_ascii=False)))atomic_write_text(self.receipt, json.dumps(receipt, ensure_ascii=False))@@ -232,6 +243,12 @@ class UpgradeRepairs:from .settings_file import load_config, DATA_ROOT_ENVconfig = load_config(environ={DATA_ROOT_ENV: str(self.config.data_root)}) if self.config is not None else load_config()seen = set(receipt.get('queued', {}))+ inherited = set()+ for key, run_id in receipt.get('queued', {}).items():+ if key.startswith(MEDIA_TASK + ':'):+ run = self.contract.task_runs.get(run_id)+ if run is not None and run.status in ('pending', 'running', 'cancelled'):+ inherited.add(int(key.rsplit(':', 1)[1]))attempts = Attempts(attempts_root(self.contract.candidate_root))with self.contract.database.read_connection() as connection:maximum = connection.execute('SELECT coalesce(max(id),0) FROM entity').fetchone()[0]@@ -255,6 +272,8 @@ class UpgradeRepairs:selected = set(receipt.get('media_ids', []))deferred = []for row in rows:+ if row['id'] in inherited:+ continueif row['id'] not in selected and not _stale_previews(self.contract, row):continuerule = upgrade_rules.PROCESSINGdiff --git a/tests/test_upgrade_repairs.py b/tests/test_upgrade_repairs.pyindex b7dc93b53..feb250fff 100644--- a/tests/test_upgrade_repairs.py+++ b/tests/test_upgrade_repairs.py@@ -31,10 +31,11 @@ class UpgradeRepairTests(unittest.TestCase):def test_readable_source_repairs_machine_title_after_an_older_upgrade_receipt(self):self.damaged_asset()self.clean_snapshot()- with patch.object(upgrade_repairs, 'REVISION', 'legacy-text-rule'), patch(+ with patch.object(upgrade_repairs, 'REVISION', upgrade_repairs.CATALOG_REVISION), patch('peach.metadata_text_repair.collect', return_value={'rows':[], 'asset_ids':[]}):self.service.prepare()- self.service.prepare()+ with patch('peach.r18_legacy_repair.apply_rows',side_effect=AssertionError('不能重放已完成补修')):+ self.service.prepare()self.service.prepare()with self.database.read_connection() as connection:row=connection.execute('SELECT catalog_title,field_owners FROM asset WHERE id=1').fetchone()@@ -82,6 +83,22 @@ class UpgradeRepairTests(unittest.TestCase):planned=self.service._plan(receipt)self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-1:1'])+ def test_text_rule_preserves_inherited_running_or_cancelled_media_work(self):+ from peach.settings_file import load_config+ self.service.config=load_config(environ={'PEACH_DATA_ROOT':str(self.root)})+ self.damaged_asset()+ parent=self.store.start('old-upgrade',trigger='manual')+ key='upgrade-library:processing-v2:1'+ child=self.store.enqueue_followups(parent.id,[(key,upgrade_repairs.MEDIA_TASK,'处理')])['queued'][0]+ self.store.finish(parent.id,'succeeded')+ receipt={'queued':{key:child},'media_ids':[1],'text_media_ids':[1]}+ for status in ('pending','cancelled'):+ if status=='cancelled':+ self.store.finish(child,'cancelled')+ with self.subTest(status=status), patch('peach.library_processing._people_allowed',return_value=False), patch(+ 'peach.seed_followup.plan',return_value=[]), patch('peach.media_sources.cost_for',return_value='free'):+ self.assertEqual(self.service._plan(receipt),[])+def test_a_snapshot_for_another_work_cannot_repair_machine_text(self):self.damaged_asset()self.clean_snapshot()