Peach: Local-first video metadata scraper and media manager.
fix(metadata): 拦截来源损坏文字并补修存量机器字段
longmeidao8ec95b150212
diff --git a/src/peach/library_processing.py b/src/peach/library_processing.pyindex 122a7aa7a..673a17e6e 100644--- a/src/peach/library_processing.py+++ b/src/peach/library_processing.py@@ -30,7 +30,7 @@ from .sources.base import SourceFailurefrom .library_nfo import sidecars, SidecarIndexfrom .media_io import directory_files, read_nfo, local_art, root_online, directory_entries, file_statfrom .genre_decisions import load_genre_decisions-from .metadata import extract_catalog_evidence, extract_peach_fields, validate_provider_code+from .metadata import candidate_text_is_valid, extract_catalog_evidence, extract_peach_fields, validate_provider_codefrom . import metadata_routesfrom . import media_identityfrom . import junk@@ -987,7 +987,7 @@ def _fields(payload, genre_decisions=None):values = [value for value in values if value]if values:fields['tags'] = dict(value=values, display_value='、'.join(values), warnings=[])- return fields+ return {field: value for field, value in fields.items() if candidate_text_is_valid(value)}def _require_writer(config, db_path):@@ -1206,7 +1206,8 @@ def _answered_sources(groups, target_key, fields):for field in fields:group = groups.get(f'{target_key}:{field}')if group:- answered.update(entry.get('source') for entry in json.loads(group.get('candidates_json') or '[]'))+ answered.update(entry.get('source') for entry in json.loads(group.get('candidates_json') or '[]')+ if candidate_text_is_valid(entry))return answereddiff --git a/src/peach/metadata.py b/src/peach/metadata.pyindex c826524da..a48908368 100644--- a/src/peach/metadata.py+++ b/src/peach/metadata.py@@ -432,6 +432,23 @@ def _url_list_evidence(raw: object, label: str) -> dict | None:}+def contains_replacement_character(value: object) -> bool:+ """来源解码已丢失的字符不能作为文字证据,保留原始快照供重新取证。"""+ if isinstance(value, str):+ return '\ufffd' in value+ if isinstance(value, dict):+ return any(contains_replacement_character(item) for item in value.values())+ if isinstance(value, (list, tuple)):+ return any(contains_replacement_character(item) for item in value)+ return False+++def candidate_text_is_valid(candidate: dict) -> bool:+ """候选正文与未映射标签均须可解码;来源路径和诊断文字不参与判定。"""+ return not any(contains_replacement_character(candidate.get(key))+ for key in ('value', 'display_value', 'unmapped_genres'))++def extract_catalog_evidence(payload: dict) -> dict[str, dict]:"""Preserve source-only catalog fields without mapping them to ledger truth.@@ -467,7 +484,7 @@ def extract_catalog_evidence(payload: dict) -> dict[str, dict]:screenshots = _url_list_evidence(payload.get("screenshot_urls"), "张截图")if screenshots is not None:out["screenshot_urls"] = screenshots- return out+ return {field: value for field, value in out.items() if candidate_text_is_valid(value)}def extract_peach_fields(payload: dict, genre_decisions=None) -> dict[str, dict]:@@ -520,4 +537,4 @@ def extract_peach_fields(payload: dict, genre_decisions=None) -> dict[str, dict]"unmapped_genres": unmapped,"warnings": [unmapped_genre_warning(unmapped)] if unmapped else [],}- return out+ return {field: value for field, value in out.items() if candidate_text_is_valid(value)}diff --git a/src/peach/metadata_auto_apply.py b/src/peach/metadata_auto_apply.pyindex e24edcff7..fcac081a0 100644--- a/src/peach/metadata_auto_apply.py+++ b/src/peach/metadata_auto_apply.py@@ -45,7 +45,7 @@ from .field_owners import ()from .genre_decisions import load_genre_decisionsfrom .genre_taxonomy import UNMAPPED, genres_in_warning, resolve_genre-from .metadata import identifies_code+from .metadata import candidate_text_is_valid, identifies_codefrom .metadata_alias_resolve import (is_planning_alias as _is_planning_alias,register_planning_alias,@@ -684,7 +684,7 @@ def _row_candidates(row: dict, decided) -> list[dict]:code = str(row.get("code") or "").strip()return [_fc2_seller_as_label(field, code, _fold_genre_decisions(field, candidate, decided))for candidate in _parsed_candidates(row)- if _candidate_identifies_code(code, candidate)]+ if _candidate_identifies_code(code, candidate) and candidate_text_is_valid(candidate)]def _auto_apply_rule(candidate: dict, agreed: int) -> str:@@ -762,6 +762,7 @@ def _evidence_candidates(row: dict, field: str = "") -> list[dict]:if (str(c.get("source") or "").strip() in SOURCE_SPECSor str(c.get("source") or "").strip() == LOCAL_NFO_SOURCE)and not blacklisted(field, str(c.get("source") or "").strip())+ and candidate_text_is_valid(c)and _original_title_candidate(field, c, str(row.get('code') or ''))]@@ -1216,6 +1217,8 @@ def _landing_scope(connection, group: dict, candidate: dict,整套落库(`_apply_metadata_candidate`)与只补新增标签(`_extend_approved_tags`)共用这一份:范围、并发凭据和番号身份不能只在其中一条路上核。"""+ if not candidate_text_is_valid(candidate):+ raise ValueError('来源文字含无法解码的字符,请重新采集资料')code = str(group.get("code") or "").strip()query = str(group.get("query") or code).strip()if group.get('asset_path'):diff --git a/src/peach/metadata_text_repair.py b/src/peach/metadata_text_repair.pynew file mode 100644index 000000000..de7f5e88b--- /dev/null+++ b/src/peach/metadata_text_repair.py@@ -0,0 +1,96 @@+"""按字段归属和同番号来源证据补修已损坏的机器文字。"""+from __future__ import annotations++from datetime import datetime, timezone+import hashlib+import json+from pathlib import Path++from .field_owners import auto_owner, owner_of+from .metadata import extract_peach_fields, identifies_code, validate_provider_code+from .metadata_auto_apply import _apply_metadata_candidate, metadata_auto_apply_candidate+from .metadata_policy import SOURCE_SPECS++FIELDS = {'title': 'catalog_title', 'original_title': 'original_title',+ 'studio': 'studio', 'series': 'series'}+++def _candidates(root: Path, code: str, field: str) -> list[dict]:+ candidates = []+ for source in SOURCE_SPECS:+ path = root / f'{code}-{source}.json'+ try:+ payload = json.loads(path.read_text(encoding='utf-8'))+ except FileNotFoundError:+ continue+ except (ValueError, UnicodeError):+ continue+ if not isinstance(payload, dict) or not identifies_code(code, payload):+ continue+ value = extract_peach_fields(payload).get(field)+ if not value:+ continue+ key = hashlib.sha256(json.dumps([source, value['value']], ensure_ascii=False,+ sort_keys=True).encode()).hexdigest()+ candidates.append({**value, 'candidate_key': key, 'source': source,+ 'provider_id': payload.get('id'), 'content_id': payload.get('content_id'),+ 'source_url': payload.get('source_url'), 'raw_snapshot': str(path),+ 'confidence': .75})+ return candidates+++def collect(connection, snapshots: Path) -> dict:+ """只选中有损坏机器字段或来源关系的资产;人物规范名保留给复核。"""+ marker = '\ufffd'+ tests = ' OR '.join(f'instr(COALESCE(a.{column},\'\'),?)>0' for column in FIELDS.values())+ assets = connection.execute(+ f"SELECT a.* FROM asset a WHERE a.medium='video' AND a.disposal IS NULL AND ({tests} "+ "OR EXISTS(SELECT 1 FROM asset_entity ae JOIN entity e ON e.id=ae.entity_id "+ "WHERE ae.asset_id=a.id AND ae.source LIKE 'javinizer:%' AND instr(e.canonical_name,?)>0))",+ [marker] * (len(FIELDS) + 1)).fetchall()+ rows, affected = [], []+ for asset in assets:+ try:+ code = validate_provider_code(str(asset['code'] or ''))+ except ValueError:+ continue+ selected = False+ for field, column in FIELDS.items():+ current, owner = asset[column] or '', owner_of(asset['field_owners'], column)+ if marker not in current or not owner.startswith('auto:') or owner[5:] not in SOURCE_SPECS:+ continue+ selected = True+ group = {'item_key': f'asset:{asset["id"]}:{field}', 'asset_id': asset['id'],+ 'asset_path': asset['path'], 'field': field, 'code': code,+ 'current_value': current, 'candidates': _candidates(snapshots, code, field)}+ candidate = metadata_auto_apply_candidate(connection, group)+ rows.append({'asset_id': asset['id'], 'field': field, 'column': column,+ 'current': current, 'owner': owner, 'group': group,+ 'candidate': candidate, 'action': 'replace' if candidate else 'review'})+ damaged_roles = connection.execute(+ "SELECT ae.role,e.id FROM asset_entity ae JOIN entity e ON e.id=ae.entity_id "+ "WHERE ae.asset_id=? AND ae.source LIKE 'javinizer:%' AND instr(e.canonical_name,?)>0",+ (asset['id'], marker)).fetchall()+ if selected or damaged_roles:+ affected.append(asset['id'])+ for role, entity_id in damaged_roles:+ rows.append({'asset_id':asset['id'], 'field':role, 'entity_id':entity_id,+ 'action':'review', 'reason':'entity-identity-requires-review'})+ return {'rows':rows, 'asset_ids':affected}+++def apply(connection, plan: dict) -> dict:+ changed = 0+ stamp = datetime.now(timezone.utc).isoformat()+ for row in plan['rows']:+ if row['action'] != 'replace':+ continue+ current = connection.execute(+ f'SELECT {row["column"]},field_owners FROM asset WHERE id=?', (row['asset_id'],)).fetchone()+ if (current is None or current[0] != row['current']+ or owner_of(current[1], row['column']) != row['owner']):+ continue+ candidate = row['candidate']+ changed += bool(_apply_metadata_candidate(connection, row['group'], candidate, stamp,+ auto_owner(candidate['source'])))+ return {'text_fields':changed, 'text_review':sum(r['action']=='review' for r in plan['rows'])}diff --git a/src/peach/upgrade_repairs.py b/src/peach/upgrade_repairs.pyindex 8cbfdaf54..72720b146 100644--- a/src/peach/upgrade_repairs.py+++ b/src/peach/upgrade_repairs.py@@ -10,7 +10,7 @@ from pathlib import Pathfrom apscheduler.schedulers.background import BackgroundSchedulerfrom filelock import FileLock, Timeout-from . import artifact_rules, composite_alias_repair, r18_legacy_repair, upgrade_rules+from . import artifact_rules, composite_alias_repair, metadata_text_repair, r18_legacy_repair, upgrade_rulesfrom .field_owners import SCAN_FILENAME, is_protected, owner_of, write_owned_fieldsfrom .followups import FollowupType, Attempts, attempts_root, registerfrom .fsutil import atomic_write_text@@ -21,7 +21,8 @@ LOGGER = logging.getLogger(__name__)TASK_KEY = 'upgrade-repairs'MEDIA_TASK = 'upgrade-library'REVISION = ':'.join((upgrade_rules.CATALOG, upgrade_rules.PREVIEWS, upgrade_rules.PROCESSING,- str(upgrade_rules.PUBLIC_SNAPSHOTS), str(upgrade_rules.PROVIDER_EVIDENCE)))+ str(upgrade_rules.PUBLIC_SNAPSHOTS), str(upgrade_rules.PROVIDER_EVIDENCE),+ str(upgrade_rules.PROVIDER_TEXT)))def _read(path: Path) -> dict:@@ -74,7 +75,8 @@ def _catalog_plan(connection, sources: Path) -> dict:for row in aliases:if is_protected(row['source']):row.update(verdict='review', note='人工别名保持原样')- return {'codes': codes, 'fields': fields, 'names': names, 'aliases': aliases}+ return {'codes': codes, 'fields': fields, 'names': names, 'aliases': aliases,+ 'text': metadata_text_repair.collect(connection, sources / 'library-metadata')}def _catalog_apply(connection, plan: dict) -> dict:@@ -94,6 +96,7 @@ def _catalog_apply(connection, plan: dict) -> dict:aliases = [row for row in composite_alias_repair.collect(connection)if tuple(row[field] for field in fields) in approved and not is_protected(row['source'])]changed.update(composite_alias_repair.apply_rows(connection, aliases))+ changed.update(metadata_text_repair.apply(connection, plan['text']))failed = connection.execute("SELECT id FROM asset WHERE medium='video' AND duration<=0 AND disposal IS NULL").fetchall()connection.execute("UPDATE asset SET duration=NULL WHERE medium='video' AND duration<=0 AND disposal IS NULL")changed['probe_retry'] = len(failed)@@ -185,6 +188,7 @@ class UpgradeRepairs:review = sum(row['reason'] not in ('already-original', 'already-japanese', 'manual-name')and row['action'] not in ('replace', 'rename') for row in plan['fields'] + plan['names'])review += sum(row['verdict'] == 'review' for row in plan['aliases'])+ review += sum(row['action'] == 'review' for row in plan['text']['rows'])with contract.database.write_transaction() as connection:violations = {tuple(row) for row in connection.execute('PRAGMA foreign_key_check')}changed = _catalog_apply(connection, plan)@@ -195,7 +199,9 @@ class UpgradeRepairs:after = _counts(connection)receipt = {'revision': REVISION, 'local': {'state': 'succeeded', 'changed': changed,'review_count': review, 'before': before, 'after': after, 'backup': str(backup)},- 'media_ids': sorted(set(failed + [row['id'] for row in plan['codes']])), 'queued': {}}+ 'text_media_ids': plan['text']['asset_ids'],+ 'media_ids': sorted(set(failed + [row['id'] for row in plan['codes']]+ + plan['text']['asset_ids'])), 'queued': {}}connection.execute('INSERT INTO ledger_upgrade_repair(id,applied_at,result_json) VALUES(?,?,?)',(REVISION, str(time.time()), json.dumps(receipt, ensure_ascii=False)))atomic_write_text(self.receipt, json.dumps(receipt, ensure_ascii=False))@@ -251,7 +257,10 @@ class UpgradeRepairs:for row in rows:if row['id'] not in selected and not _stale_previews(self.contract, row):continue- key = f'{MEDIA_TASK}:{upgrade_rules.PROCESSING}:{row["id"]}'+ rule = upgrade_rules.PROCESSING+ if row['id'] in receipt.get('text_media_ids', []):+ rule += f':text-{upgrade_rules.PROVIDER_TEXT}'+ key = f'{MEDIA_TASK}:{rule}:{row["id"]}'if key in seen:continueif cost_for(row['location'], row['path'], config) != 'free':diff --git a/src/peach/upgrade_rules.py b/src/peach/upgrade_rules.pyindex f383e6ce6..992b1a872 100644--- a/src/peach/upgrade_rules.py+++ b/src/peach/upgrade_rules.py@@ -6,4 +6,5 @@ LOCAL_POSTERS = PREVIEWS + ':local'PROCESSING = "processing-v2"PUBLIC_SNAPSHOTS = 2PROVIDER_EVIDENCE = 2+PROVIDER_TEXT = 1OFFICIAL_COVER_CATALOG_MISSES = 1diff --git a/tests/test_metadata.py b/tests/test_metadata.pyindex 0723933be..1c25d7e13 100644--- a/tests/test_metadata.py+++ b/tests/test_metadata.py@@ -14,6 +14,25 @@ from peach.metadata import (class MetadataProviderTests(unittest.TestCase):+ def test_damaged_source_fields_leave_healthy_fields_available(self):+ fields = extract_peach_fields({'title':'壊\ufffdれた題名', 'maker':'正しいメーカー',+ 'release_date':'2026-01-02', 'actresses':[{'japanese_name':'名前\ufffd'}],+ 'genres':['未\ufffd分類']})+ self.assertNotIn('title', fields)+ self.assertNotIn('performers', fields)+ self.assertNotIn('tags', fields)+ self.assertEqual(fields['studio']['value'], '正しいメーカー')+ self.assertEqual(fields['release_date']['value'], '2026-01-02')+ self.assertNotIn('title', extract_catalog_evidence({'title':'壊\ufffdれた題名'}))++ def test_valid_non_latin_source_text_and_performer_aliases_remain_available(self):+ fields = extract_peach_fields({'title':'日本語 中文 café',+ 'actresses':[{'japanese_name':'正常な名前','aliases':['別名\ufffd']}],+ 'maker':'正常な会社'})+ self.assertEqual(fields['title']['value'], '日本語 中文 café')+ self.assertNotIn('performers', fields)+ self.assertEqual(fields['studio']['value'], '正常な会社')+def test_all_unmapped_genres_are_reviewable_without_fabricated_tags(self):fields = extract_peach_fields({"genres": ["未知类别甲", "未知类别乙"]})self.assertEqual(fields["tags"]["value"], [])diff --git a/tests/test_upgrade_repairs.py b/tests/test_upgrade_repairs.pyindex bdb2984a8..b7dc93b53 100644--- a/tests/test_upgrade_repairs.py+++ b/tests/test_upgrade_repairs.py@@ -15,6 +15,97 @@ from support.ledger import fresh_ledgerclass UpgradeRepairTests(unittest.TestCase):+ def damaged_asset(self, key=1, *, owner='auto:makers'):+ self.asset(key, code='EXM-001', duration=1)+ with self.database.write_transaction() as connection:+ connection.execute('UPDATE asset SET name=?,catalog_title=?,field_owners=? WHERE id=?',+ ('EXM-001.mp4', '壊\ufffdれた題名', json.dumps({'code':'scan:filename',+ 'catalog_title':owner}), key))++ def clean_snapshot(self):+ snapshots = self.root/'sources/library-metadata'+ snapshots.mkdir(parents=True,exist_ok=True)+ (snapshots/'EXM-001-dmm.json').write_text(json.dumps({'id':'EXM-001','title':'元の題名'}),+ encoding='utf-8')++ def test_readable_source_repairs_machine_title_after_an_older_upgrade_receipt(self):+ self.damaged_asset()+ self.clean_snapshot()+ with patch.object(upgrade_repairs, 'REVISION', 'legacy-text-rule'), patch(+ 'peach.metadata_text_repair.collect', return_value={'rows':[], 'asset_ids':[]}):+ self.service.prepare()+ self.service.prepare()+ self.service.prepare()+ with self.database.read_connection() as connection:+ row=connection.execute('SELECT catalog_title,field_owners FROM asset WHERE id=1').fetchone()+ self.assertEqual(row[0], '元の題名')+ self.assertEqual(json.loads(row[1])['catalog_title'], 'auto:dmm')+ self.assertEqual(connection.execute('SELECT count(*) FROM ledger_upgrade_repair').fetchone()[0], 2)+ self.assertEqual(connection.execute('PRAGMA integrity_check').fetchone()[0], 'ok')+ self.assertEqual(len(list(self.root.glob('ledger.pre-upgrade-*.db'))), 2)+ self.assertEqual(upgrade_repairs._read(self.service.receipt)['text_media_ids'], [1])++ def test_damaged_manual_titles_and_entity_identities_are_not_renamed(self):+ self.damaged_asset(owner='user:manual')+ self.clean_snapshot()+ with self.database.write_transaction() as connection:+ entity=upsert_asset_entity(connection,kind='performer',name='名前\ufffd',asset_id=1,+ role='performer',source='javinizer:makers:performer',confidence=.75,now='now')+ self.service.prepare()+ with self.database.read_connection() as connection:+ self.assertEqual(connection.execute('SELECT catalog_title FROM asset WHERE id=1').fetchone()[0],+ '壊\ufffdれた題名')+ self.assertEqual(connection.execute('SELECT canonical_name FROM entity WHERE id=?', (entity,)).fetchone()[0],+ '名前\ufffd')+ receipt=upgrade_repairs._read(self.service.receipt)+ self.assertGreater(receipt['local']['review_count'],0)+ self.assertEqual(receipt['text_media_ids'],[1])++ def test_missing_alternative_keeps_the_original_and_records_review(self):+ self.damaged_asset()+ self.service.prepare()+ with self.database.read_connection() as connection:+ self.assertEqual(connection.execute('SELECT catalog_title FROM asset WHERE id=1').fetchone()[0],+ '壊\ufffdれた題名')+ receipt=upgrade_repairs._read(self.service.receipt)+ self.assertEqual(receipt['local']['changed']['text_fields'],0)+ self.assertEqual(receipt['local']['changed']['text_review'],1)++ def test_damaged_text_reprocessing_has_an_independent_rule_identity(self):+ from peach.settings_file import load_config+ self.service.config=load_config(environ={'PEACH_DATA_ROOT':str(self.root)})+ self.damaged_asset()+ receipt={'queued':{'upgrade-library:processing-v2:1':99},+ 'media_ids':[1],'text_media_ids':[1]}+ with patch('peach.library_processing._people_allowed',return_value=False), patch(+ 'peach.seed_followup.plan',return_value=[]), patch('peach.media_sources.cost_for',return_value='free'):+ planned=self.service._plan(receipt)+ self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-1:1'])++ def test_a_snapshot_for_another_work_cannot_repair_machine_text(self):+ self.damaged_asset()+ self.clean_snapshot()+ path=self.root/'sources/library-metadata/EXM-001-dmm.json'+ path.write_text(json.dumps({'id':'OTHER-999','title':'別の作品'}),encoding='utf-8')+ self.service.prepare()+ with self.database.read_connection() as connection:+ self.assertEqual(connection.execute('SELECT catalog_title FROM asset WHERE id=1').fetchone()[0],+ '壊\ufffdれた題名')++ def test_manual_change_after_planning_prevents_the_machine_repair(self):+ from peach import metadata_text_repair+ self.damaged_asset()+ self.clean_snapshot()+ with self.database.read_connection() as connection:+ plan=metadata_text_repair.collect(connection,self.root/'sources/library-metadata')+ with self.database.write_transaction() as connection:+ connection.execute('UPDATE asset SET catalog_title=?,field_owners=? WHERE id=1',+ ('人工题名',json.dumps({'catalog_title':'user:manual'})))+ result=metadata_text_repair.apply(connection,plan)+ self.assertEqual(result['text_fields'],0)+ self.assertEqual(connection.execute('SELECT catalog_title FROM asset WHERE id=1').fetchone()[0],+ '人工题名')+def setUp(self):holder = tempfile.TemporaryDirectory()self.addCleanup(holder.cleanup)diff --git a/tests/test_web_review.py b/tests/test_web_review.pyindex 19f21d823..82f131bdf 100644--- a/tests/test_web_review.py+++ b/tests/test_web_review.py@@ -40,6 +40,30 @@ from support.ledger import fresh_ledgerclass ReviewQueueTests(unittest.TestCase):"""复核队列:候选来源、稳定主键、批准的权威值与写入边界。"""+ def test_old_damaged_official_candidate_cannot_win_over_readable_evidence(self):+ self._asset(400, 'EXM-001', 'EXM-001.mp4')+ self.write_metadata_rows([{'item_key':'EXM-001:title', 'code':'EXM-001',+ 'field':'title', 'current':'', 'candidates':[+ {'source':'makers','value':'壊\ufffdれた題名'},+ {'source':'dmm','value':'元の題名'}]}])+ self._auto()+ with sqlite3.connect(self.db_path) as connection:+ self.assertEqual(connection.execute('SELECT catalog_title FROM asset WHERE id=400').fetchone()[0],+ '元の題名')++ def test_direct_approval_rejects_damaged_legacy_candidate_before_writing(self):+ from peach.metadata_auto_apply import _apply_metadata_candidate+ self._asset(400, 'EXM-001', 'EXM-001.mp4')+ group = {'item_key':'asset:400:title','asset_id':400,'asset_path':'/x/EXM-001.mp4',+ 'code':'EXM-001','field':'title'}+ candidate = {'candidate_key':'old','source':'makers','provider_id':'EXM-001',+ 'value':'壊\ufffdれた題名','confidence':.75}+ with sqlite3.connect(self.db_path) as connection:+ connection.row_factory = sqlite3.Row+ with self.assertRaisesRegex(ValueError, '无法解码'):+ _apply_metadata_candidate(connection, group, candidate, 'now', 'review:makers')+ self.assertIsNone(connection.execute('SELECT catalog_title FROM asset WHERE id=400').fetchone()[0])+def setUp(self):self.tmp = tempfile.TemporaryDirectory()root = Path(self.tmp.name)