Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(upgrade): 定向重采旧 JavDB 身份与标题证据

longmeidao116159b13135
diff --git a/docs/SOURCING.md b/docs/SOURCING.md
index 8cd51bd9d..e617a901c 100644
--- a/docs/SOURCING.md
+++ b/docs/SOURCING.md
@@ -657,11 +657,9 @@ Instagram 的独立用户登录会话未取得,自动适配器不进正式依
### 确认来源返回的是同一部片
-- JavDB 的相同番号可能对应多个来源 ID,不能靠番号、时长或搜索顺序认作同一作品。网页与匿名 API
- 都先汇总匹配的来源 ID;只有一个才取详情,多个时报告歧义。详情、图片及资源沿用这次确认的来源 ID,
- 不在取资源时重新按番号选片。JavDB 旧的番号缓存、发现边车和原始快照须通过来源专属规则才可复用。
-- JavDB 的 `origin-title` / `origin_title` 是原标题,`current-title` 是本地化显示标题。有原题时优先原题,
- 显示标题独立留证;缺原题时保留显示标题,不凭文字语言猜测或删改星号片段。
+- JavDB 的相同番号可能对应多个来源 ID,不能靠番号、时长或搜索顺序认作同一作品。网页与匿名 API 都先汇总匹配的来源 ID;只有一个才取详情,多个时报告歧义。详情、图片及资源沿用这次确认的来源 ID,不在取资源时重新按番号选片。旧的番号缓存、发现边车和原始快照须通过来源专属规则才可复用。
+- JavDB 的 `origin-title` / `origin_title` 是原标题,`current-title` 是本地化显示标题。有原题时优先原题,显示标题独立留证;缺原题时保留显示标题,不凭文字语言猜测或删改星号片段。
+- 升级补修通过独立的文字规则回执定向失效旧 JavDB 库内快照,并选择对应影片重新取证;原 JSON、人工字段、既有取消和延期记录保留。旧 JavDB 候选不参与自动采纳,当前候选随来源身份和规则版本进入既有复核流程,不清空其他来源正常证据。
- 来源返回的番号必须和查询的番号比对过才算命中。javbus 一侧是拿番号做关键词搜索取首个结果,搜不到就返回
近似的别人(`SA-104 → AVSA-104`、`AR-301 → STAR-3016`)。不比对时大部分匹配番号根本不对,整个 MIB 目录
(韩国内容)曾因此被写上日本厂牌、系列和标题,还靠这些假证据升级成 JAV。
diff --git a/src/peach/metadata_text_repair.py b/src/peach/metadata_text_repair.py
index c0204e0f7..5e259cf0c 100644
--- a/src/peach/metadata_text_repair.py
+++ b/src/peach/metadata_text_repair.py
@@ -8,6 +8,7 @@ from pathlib import Path
from .field_owners import auto_owner, owner_of
from . import artifact_rules, upgrade_rules
+from .javdb_identity import current_evidence
from .metadata import extract_peach_fields, identifies_code, source_text_is_valid, validate_provider_code
from .metadata_auto_apply import _apply_metadata_candidate, metadata_auto_apply_candidate
from .metadata_policy import SOURCE_SPECS
@@ -17,7 +18,7 @@ FIELDS = {'title': 'catalog_title', 'original_title': 'original_title',
def _damaged_snapshots(connection, root: Path) -> tuple[list[dict], list[int]]:
- """只失效已知来源的损坏文字快照,并选择同番号资产重新采集。"""
+ """失效损坏文字或旧 JavDB 绑定快照,保留原文并选择对应资产重采。"""
snapshots, affected = [], []
for path in root.glob('*.json'):
code, separator, source = path.stem.rpartition('-')
@@ -29,7 +30,8 @@ def _damaged_snapshots(connection, root: Path) -> tuple[list[dict], list[int]]:
payload = json.loads(path.read_text(encoding='utf-8'))
except (OSError, ValueError):
continue
- if not isinstance(payload, dict) or source_text_is_valid(payload):
+ if not isinstance(payload, dict) or (source_text_is_valid(payload)
+ and (source != 'javdb' or current_evidence(payload))):
continue
snapshots.append({'path': str(path), 'fingerprint': before})
affected.extend(row[0] for row in connection.execute(
@@ -61,22 +63,26 @@ def _candidates(root: Path, code: str, field: str) -> list[dict]:
continue
except (ValueError, UnicodeError):
continue
- if not isinstance(payload, dict) or not identifies_code(code, payload):
+ if (not isinstance(payload, dict) or not identifies_code(code, payload)
+ or source == 'javdb' and not current_evidence(payload)):
continue
value = extract_peach_fields(payload).get(field)
if not value:
continue
key = hashlib.sha256(json.dumps([source, value['value']], ensure_ascii=False,
sort_keys=True).encode()).hexdigest()
- candidates.append({**value, 'candidate_key': key, 'source': source,
- 'provider_id': payload.get('id'), 'content_id': payload.get('content_id'),
- 'source_url': payload.get('source_url'), 'raw_snapshot': str(path),
- 'confidence': .75})
+ candidate = {**value, 'candidate_key': key, 'source': source,
+ 'provider_id': payload.get('id'), 'content_id': payload.get('content_id'),
+ 'source_url': payload.get('source_url'), 'raw_snapshot': str(path),
+ 'confidence': .75}
+ if source == 'javdb':
+ candidate.update(javdb_rule=payload['javdb_rule'], source_id=payload.get('source_id'))
+ candidates.append(candidate)
return candidates
def collect(connection, snapshots: Path) -> dict:
- """只选中有损坏机器字段或来源关系的资产;人物规范名保留给复核。"""
+ """选中损坏文字与失效来源快照的资产;人物规范名保留给复核。"""
marker = '\ufffd'
tests = ' OR '.join(f'instr(COALESCE(a.{column},\'\'),?)>0' for column in FIELDS.values())
assets = connection.execute(
diff --git a/src/peach/upgrade_rules.py b/src/peach/upgrade_rules.py
index 1be6e89c8..c42341b6d 100644
--- a/src/peach/upgrade_rules.py
+++ b/src/peach/upgrade_rules.py
@@ -6,5 +6,5 @@ LOCAL_POSTERS = PREVIEWS + ':local'
PROCESSING = "processing-v2"
PUBLIC_SNAPSHOTS = 2
PROVIDER_EVIDENCE = 2
-PROVIDER_TEXT = 2
+PROVIDER_TEXT = 3
OFFICIAL_COVER_CATALOG_MISSES = 1
diff --git a/tests/test_upgrade_repairs.py b/tests/test_upgrade_repairs.py
index 92ccaec30..bda03561c 100644
--- a/tests/test_upgrade_repairs.py
+++ b/tests/test_upgrade_repairs.py
@@ -15,6 +15,80 @@ from support.ledger import fresh_ledger
class UpgradeRepairTests(unittest.TestCase):
+ def test_javdb_upgrade_requeues_legacy_evidence_and_keeps_manual_values(self):
+ from peach import artifact_rules, upgrade_rules
+ self.asset(1, code='EXM-001', duration=1)
+ self.asset(2, code='EXM-002', duration=1)
+ with self.database.write_transaction() as connection:
+ connection.execute('UPDATE asset SET catalog_title=?,field_owners=? WHERE id=1',
+ ('人工题名', json.dumps({'catalog_title':'user:manual'})))
+ with patch.object(upgrade_repairs, 'REVISION', f'{upgrade_repairs.CATALOG_REVISION}:text-2'):
+ self.service.prepare()
+ previous=upgrade_repairs._read(self.service.receipt)
+ previous.update(queued={'previous-task':99}, deferred=[42], dispatch_complete=True)
+ self.service._save_receipt(previous)
+ directory=self.root/'sources/library-metadata'
+ directory.mkdir(parents=True,exist_ok=True)
+ legacy=directory/'EXM-001-javdb.json'
+ current=directory/'EXM-002-javdb.json'
+ other=directory/'EXM-001-dmm.json'
+ payloads=((legacy,{'id':'EXM-001','title':'来源显示题名'}),
+ (current,{'id':'EXM-002','title':'Original title * Limited','javdb_rule':2,
+ 'source_id':'CurrentMovie'}),
+ (other,{'id':'EXM-001','title':'其他来源题名'}))
+ for path,payload in payloads:
+ path.write_text(json.dumps(payload,ensure_ascii=False),encoding='utf-8')
+ artifact_rules.remember(path,str(upgrade_rules.PROVIDER_EVIDENCE))
+ original={path:path.read_bytes() for path,_ in payloads}
+ self.service.prepare()
+ self.assertFalse(artifact_rules.current(legacy,str(upgrade_rules.PROVIDER_EVIDENCE)))
+ self.assertTrue(artifact_rules.current(current,str(upgrade_rules.PROVIDER_EVIDENCE)))
+ self.assertTrue(artifact_rules.current(other,str(upgrade_rules.PROVIDER_EVIDENCE)))
+ for path,content in original.items():
+ self.assertEqual(path.read_bytes(),content)
+ receipt=upgrade_repairs._read(self.service.receipt)
+ self.assertEqual(receipt['text_media_ids'],[1])
+ self.assertEqual(receipt['queued'],{'previous-task':99})
+ self.assertEqual(receipt['deferred'],[42])
+ self.assertFalse(receipt['dispatch_complete'])
+ self.assertEqual(receipt['local']['changed']['text_snapshots_invalidated'],1)
+ with self.database.read_connection() as connection:
+ self.assertEqual(connection.execute('SELECT catalog_title FROM asset WHERE id=1').fetchone()[0],
+ '人工题名')
+ self.service.prepare()
+ self.assertEqual(len(list(self.root.glob('ledger.pre-upgrade-*.db'))),2)
+
+ def test_legacy_javdb_title_is_excluded_from_machine_text_repair(self):
+ from peach import metadata_text_repair
+ self.damaged_asset(owner='auto:javdb')
+ directory=self.root/'sources/library-metadata'
+ directory.mkdir(parents=True,exist_ok=True)
+ (directory/'EXM-001-javdb.json').write_text(
+ json.dumps({'id':'EXM-001','title':'旧绑定显示题名'}),encoding='utf-8')
+ with self.database.read_connection() as connection:
+ plan=metadata_text_repair.collect(connection,directory)
+ row=next(row for row in plan['rows'] if row['field']=='title')
+ self.assertEqual(row['group']['candidates'],[])
+ self.assertEqual(row['action'],'review')
+
+ def test_current_javdb_original_is_accepted_for_empty_machine_text(self):
+ from peach import metadata_auto_apply, metadata_text_repair
+ self.asset(1,code='EXM-001',duration=1)
+ with self.database.write_transaction() as connection:
+ connection.execute('UPDATE asset SET name=? WHERE id=1',('EXM-001.mp4',))
+ directory=self.root/'sources/library-metadata'
+ directory.mkdir(parents=True,exist_ok=True)
+ (directory/'EXM-001-javdb.json').write_text(json.dumps({
+ 'id':'EXM-001','title':'元の題名','origin_title':'元の題名',
+ 'javdb_rule':2,'source_id':'AbC123','source_url':'https://javdb.com/v/AbC123'}),encoding='utf-8')
+ with self.database.read_connection() as connection:
+ candidate=metadata_auto_apply.metadata_auto_apply_candidate(connection,{
+ 'asset_id':1,'code':'EXM-001','field':'title','current_value':'',
+ 'candidates':metadata_text_repair._candidates(directory,'EXM-001','title')})
+ self.assertIsNotNone(candidate)
+ self.assertEqual(candidate['value'],'元の題名')
+ self.assertEqual(candidate['source_id'],'AbC123')
+
def test_encoding_upgrade_invalidates_damaged_snapshots_after_a_text_one_receipt(self):
from peach import artifact_rules, upgrade_rules
self.asset(1, code='EXM-001', duration=1)
@@ -115,7 +189,7 @@ class UpgradeRepairTests(unittest.TestCase):
with patch('peach.library_processing._people_allowed',return_value=False), patch(
'peach.seed_followup.plan',return_value=[]), patch('peach.media_sources.cost_for',return_value='free'):
planned=self.service._plan(receipt)
- self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-2:1'])
+ self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-3:1'])
def test_text_only_dispatch_does_not_plan_unrelated_people(self):
from peach.settings_file import load_config
@@ -126,7 +200,7 @@ class UpgradeRepairTests(unittest.TestCase):
'peach.seed_followup.plan',return_value=[]), patch('peach.media_sources.cost_for',return_value='free'), patch(
'peach.performer_alias_followup.stock',side_effect=AssertionError('不能重放人物补修')):
planned=self.service._plan(receipt)
- self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-2:1'])
+ self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-3:1'])
def test_completed_catalog_dispatch_preserves_deferred_work_across_text_revision(self):
with patch.object(upgrade_repairs,'REVISION',upgrade_repairs.CATALOG_REVISION):