Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(upgrade): 独立登记文字补修并保留已有任务状态

longmeidao2fe1781fc34b
diff --git a/docs/REUSE.md b/docs/REUSE.md
index 13a4e2a6b..9306d618b 100644
--- a/docs/REUSE.md
+++ b/docs/REUSE.md
@@ -912,7 +912,7 @@ Python、npm 与 GitHub Actions 的版本由 `.github/dependabot.yml` 每周检
来源文字中的 Unicode 替换字符 `U+FFFD` 表示解码时已有信息丢失。字段提取、既有候选选择和最终写入共用这一判据;含坏值的字段不参与来源优先级结算,原始快照保留。正常字段与正常中日文、重音字母不受影响,不使用启发式转码猜回原文。
-存量文字补修复用字段归属、来源身份核对、候选结算与 `UpgradeRepairs` 的备份、事务和回执。仅机器归属的损坏标量字段可按同番号正常来源恢复;人物规范名及人工判断保留复核。待重采的资产使用独立文字规则身份,先前完成的升级任务不吞掉新补修。
+存量文字补修复用字段归属、来源身份核对、候选结算与 `UpgradeRepairs` 的备份、事务和回执。仅机器归属的损坏标量字段可按同番号正常来源恢复;人物规范名及人工判断保留复核。已登记完成的目录补修不随文字规则重放;旧任务范围和队列继续保留。待重采的资产使用独立文字规则身份,先前完成的升级任务不吞掉新补修,尚在运行或已由用户取消的任务保持原状态。
播放生命周期复用 [PR190](https://github.com/peach-mitao/peach/pull/190) 的 `e41222e312443b5d0fc7be87d2f7eca9c1d61055`:共享分片按消费者计数,最后消费者离开才取消生成,并等待子进程回收;文件响应保留外层取消与运行错误。头像确认的图像和数据库工作共用工作线程,安装按实体加锁。当前 ASGI 响应头中间件、SMB 输入和图像预处理保留 Peach 主线实现;未采用该 PR 的界面、依赖和测试调度改动。
diff --git a/src/peach/api.py b/src/peach/api.py
index c01b8388f..74a0c48ec 100644
--- a/src/peach/api.py
+++ b/src/peach/api.py
@@ -128,7 +128,7 @@ class ResponseHeadersMiddleware:
kind = headers.get('content-type','').split(';',1)[0].strip().lower()
if kind.startswith(('image/','video/','audio/')):
headers.update(MEDIA_SECURITY_HEADERS)
- if (scope['method']=='GET' and message['status']==200 and kind=='text/html'
+ if (message['status']==200 and kind=='text/html' and scope['method']=='GET'
and Request(scope).cookies.get('peach_media_policy')!='1'):
headers['Clear-Site-Data']='"cache"'
policy=Response()
diff --git a/src/peach/upgrade_repairs.py b/src/peach/upgrade_repairs.py
index 72720b146..21625d030 100644
--- a/src/peach/upgrade_repairs.py
+++ b/src/peach/upgrade_repairs.py
@@ -20,9 +20,9 @@ from .task_runs import MAX_FOLLOWUPS
LOGGER = logging.getLogger(__name__)
TASK_KEY = 'upgrade-repairs'
MEDIA_TASK = 'upgrade-library'
-REVISION = ':'.join((upgrade_rules.CATALOG, upgrade_rules.PREVIEWS, upgrade_rules.PROCESSING,
- str(upgrade_rules.PUBLIC_SNAPSHOTS), str(upgrade_rules.PROVIDER_EVIDENCE),
- str(upgrade_rules.PROVIDER_TEXT)))
+CATALOG_REVISION = ':'.join((upgrade_rules.CATALOG, upgrade_rules.PREVIEWS, upgrade_rules.PROCESSING,
+ str(upgrade_rules.PUBLIC_SNAPSHOTS), str(upgrade_rules.PROVIDER_EVIDENCE)))
+REVISION = f'{CATALOG_REVISION}:text-{upgrade_rules.PROVIDER_TEXT}'
def _read(path: Path) -> dict:
@@ -54,7 +54,10 @@ def _manual_name(connection, entity_id: int) -> bool:
(entity_id,)).fetchone())
-def _catalog_plan(connection, sources: Path) -> dict:
+def _catalog_plan(connection, sources: Path, *, catalog: bool = True) -> dict:
+ text = metadata_text_repair.collect(connection, sources / 'library-metadata')
+ if not catalog:
+ return {'codes':[], 'fields':[], 'names':[], 'aliases':[], 'text':text, 'catalog':False}
from .catalog_rules import compile_code_patterns, release_code_from_filename, same_release_code
from .profiles import read_settings
patterns = compile_code_patterns(read_settings(connection).get('codePatterns'))
@@ -76,10 +79,12 @@ def _catalog_plan(connection, sources: Path) -> dict:
if is_protected(row['source']):
row.update(verdict='review', note='人工别名保持原样')
return {'codes': codes, 'fields': fields, 'names': names, 'aliases': aliases,
- 'text': metadata_text_repair.collect(connection, sources / 'library-metadata')}
+ 'text': text, 'catalog': True}
def _catalog_apply(connection, plan: dict) -> dict:
+ if not plan.get('catalog', True):
+ return {'codes':0, 'probe_retry':0, **metadata_text_repair.apply(connection, plan['text'])}
from datetime import datetime, timezone
changed = {}
for row in plan['codes']:
@@ -172,16 +177,19 @@ class UpgradeRepairs:
with contract.database.read_connection() as connection:
applied = connection.execute('SELECT result_json FROM ledger_upgrade_repair WHERE id=?',
(REVISION,)).fetchone()
+ prior = connection.execute('SELECT result_json FROM ledger_upgrade_repair WHERE id=?',
+ (CATALOG_REVISION,)).fetchone()
if applied is not None:
self._restore_receipt(json.loads(applied[0]))
return
self._restore_receipt(None)
sources = Path(contract.follow_sources_root)
+ previous = json.loads(prior[0]) if prior is not None else {}
with contract.database.read_connection() as connection:
- plan = _catalog_plan(connection, sources)
+ plan = _catalog_plan(connection, sources, catalog=prior is None)
before = _counts(connection)
failed = [row[0] for row in connection.execute(
- "SELECT id FROM asset WHERE medium='video' AND duration<=0 AND disposal IS NULL")]
+ "SELECT id FROM asset WHERE medium='video' AND duration<=0 AND disposal IS NULL")] if prior is None else []
atomic_write_text(self.root / 'catalog-review.json', json.dumps(plan, ensure_ascii=False, indent=2))
backup = contract.db_path.with_name(f'ledger.pre-upgrade-{time.time_ns()}.db')
sqlite_backup(contract.db_path, backup)
@@ -189,6 +197,7 @@ class UpgradeRepairs:
and row['action'] not in ('replace', 'rename') for row in plan['fields'] + plan['names'])
review += sum(row['verdict'] == 'review' for row in plan['aliases'])
review += sum(row['action'] == 'review' for row in plan['text']['rows'])
+ review += previous.get('local', {}).get('review_count', 0)
with contract.database.write_transaction() as connection:
violations = {tuple(row) for row in connection.execute('PRAGMA foreign_key_check')}
changed = _catalog_apply(connection, plan)
@@ -201,7 +210,9 @@ class UpgradeRepairs:
'review_count': review, 'before': before, 'after': after, 'backup': str(backup)},
'text_media_ids': plan['text']['asset_ids'],
'media_ids': sorted(set(failed + [row['id'] for row in plan['codes']]
- + plan['text']['asset_ids'])), 'queued': {}}
+ + previous.get('media_ids', [])
+ + plan['text']['asset_ids'])),
+ 'queued': dict(previous.get('queued', {})), 'prior_revision': CATALOG_REVISION if prior else None}
connection.execute('INSERT INTO ledger_upgrade_repair(id,applied_at,result_json) VALUES(?,?,?)',
(REVISION, str(time.time()), json.dumps(receipt, ensure_ascii=False)))
atomic_write_text(self.receipt, json.dumps(receipt, ensure_ascii=False))
@@ -232,6 +243,12 @@ class UpgradeRepairs:
from .settings_file import load_config, DATA_ROOT_ENV
config = load_config(environ={DATA_ROOT_ENV: str(self.config.data_root)}) if self.config is not None else load_config()
seen = set(receipt.get('queued', {}))
+ inherited = set()
+ for key, run_id in receipt.get('queued', {}).items():
+ if key.startswith(MEDIA_TASK + ':'):
+ run = self.contract.task_runs.get(run_id)
+ if run is not None and run.status in ('pending', 'running', 'cancelled'):
+ inherited.add(int(key.rsplit(':', 1)[1]))
attempts = Attempts(attempts_root(self.contract.candidate_root))
with self.contract.database.read_connection() as connection:
maximum = connection.execute('SELECT coalesce(max(id),0) FROM entity').fetchone()[0]
@@ -255,6 +272,8 @@ class UpgradeRepairs:
selected = set(receipt.get('media_ids', []))
deferred = []
for row in rows:
+ if row['id'] in inherited:
+ continue
if row['id'] not in selected and not _stale_previews(self.contract, row):
continue
rule = upgrade_rules.PROCESSING
diff --git a/tests/test_upgrade_repairs.py b/tests/test_upgrade_repairs.py
index b7dc93b53..feb250fff 100644
--- a/tests/test_upgrade_repairs.py
+++ b/tests/test_upgrade_repairs.py
@@ -31,10 +31,11 @@ class UpgradeRepairTests(unittest.TestCase):
def test_readable_source_repairs_machine_title_after_an_older_upgrade_receipt(self):
self.damaged_asset()
self.clean_snapshot()
- with patch.object(upgrade_repairs, 'REVISION', 'legacy-text-rule'), patch(
+ with patch.object(upgrade_repairs, 'REVISION', upgrade_repairs.CATALOG_REVISION), patch(
'peach.metadata_text_repair.collect', return_value={'rows':[], 'asset_ids':[]}):
self.service.prepare()
- self.service.prepare()
+ with patch('peach.r18_legacy_repair.apply_rows',side_effect=AssertionError('不能重放已完成补修')):
+ self.service.prepare()
self.service.prepare()
with self.database.read_connection() as connection:
row=connection.execute('SELECT catalog_title,field_owners FROM asset WHERE id=1').fetchone()
@@ -82,6 +83,22 @@ class UpgradeRepairTests(unittest.TestCase):
planned=self.service._plan(receipt)
self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-1:1'])
+ def test_text_rule_preserves_inherited_running_or_cancelled_media_work(self):
+ from peach.settings_file import load_config
+ self.service.config=load_config(environ={'PEACH_DATA_ROOT':str(self.root)})
+ self.damaged_asset()
+ parent=self.store.start('old-upgrade',trigger='manual')
+ key='upgrade-library:processing-v2:1'
+ child=self.store.enqueue_followups(parent.id,[(key,upgrade_repairs.MEDIA_TASK,'处理')])['queued'][0]
+ self.store.finish(parent.id,'succeeded')
+ receipt={'queued':{key:child},'media_ids':[1],'text_media_ids':[1]}
+ for status in ('pending','cancelled'):
+ if status=='cancelled':
+ self.store.finish(child,'cancelled')
+ with self.subTest(status=status), patch('peach.library_processing._people_allowed',return_value=False), patch(
+ 'peach.seed_followup.plan',return_value=[]), patch('peach.media_sources.cost_for',return_value='free'):
+ self.assertEqual(self.service._plan(receipt),[])
+
def test_a_snapshot_for_another_work_cannot_repair_machine_text(self):
self.damaged_asset()
self.clean_snapshot()