Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(metadata): 固定采集桥 UTF-8 并升级失效损坏快照

longmeidao4271decebc21
diff --git a/.claude/skills/peach-worktree/SKILL.md b/.claude/skills/peach-worktree/SKILL.md
index a391f76eb..f4625898d 100644
--- a/.claude/skills/peach-worktree/SKILL.md
+++ b/.claude/skills/peach-worktree/SKILL.md
@@ -5,7 +5,7 @@ description: Peach 代码任务开工,以及创建、提交、集成或回收
# 工作树与集成交付
-最后复核:2026-10-26
+最后复核:2026-10-27
证据来源:工作树入口、交付门槛、集成与回收事故、用户确认的存量生效判据。
## 开工与职责
@@ -30,6 +30,7 @@ description: Peach 代码任务开工,以及创建、提交、集成或回收
- 替换代码或资源并正常刷新、重启即可生效的修复,跑对应行为回归,不强制增加迁移或新装、升级双轨检查。
- 既有数据、配置、断点、缓存或派生产物需要转换、补修、失效或重算时,复用相应机制,用临时旧样本验证并保留人工资料与个人记录。
+- 刮削解析、匹配或文字处理修复须核对旧快照;旧结果会挡住修复时,在升级补修中按受影响来源和规则定向失效,验证已有回执用户仍能重采,不清空正常缓存。
- 安装器、依赖分发、数据目录或升级协议变化再验证对应制品;源码或单元回归不代替制品验收。
## 多分支一起合入
diff --git a/docs/SOURCING.md b/docs/SOURCING.md
index 73588a76b..7e53999c3 100644
--- a/docs/SOURCING.md
+++ b/docs/SOURCING.md
@@ -222,6 +222,10 @@ Wiki 与视频作品资料无关。
「来源和凭证」页那张卡上重建,钉住的版本只由人改。venv 里读到 amane 的包元数据才算已安装,重建中途失败留下的
半截环境照旧显示未安装。Docker 部署见 [Docker 部署](DOCKER.md#amane-刮削站点)。
+桥的标准输出与错误输出固定为 UTF-8,源码运行和独立制品使用相同协议。主程序严格解码;编码不符时记录 `bridge_stdio_decode`、番号、来源、运行方式、字节位置与 SHA-256,不把替换字符当成正常资料。来源快照中的 `collection_diagnostics` 随诊断包与刮削记录导出,包含编码和采集器版本,不包含代理凭据或响应正文。
+
+升级补修会定向失效包含 `�` 的来源文字快照,保留原文件供诊断,并安排受影响影片重新采集;已有旧补修回执或已完成断点不会挡住新规则。正常缓存和人工资料保留,付费或需下载的来源仍等待显式处理。已入库文字沿用升级补修与人工复核,不通过删字或猜测编码还原。
+
- **社区档 `amane`**:avsox,以及只能由覆盖点名的 fc2club、freejavbt、airav。链上合成一档,一次子进程并发
问完。上游报的 `rate_limited` 按 429 那一档、`cloudflare_*` 与 `ip_banned` 按 403 那一档写进同一份冷却记录;
正在冷却的站不带进子进程。
diff --git a/src/peach/library_processing.py b/src/peach/library_processing.py
index 3e992e02e..a19a46b34 100644
--- a/src/peach/library_processing.py
+++ b/src/peach/library_processing.py
@@ -31,7 +31,8 @@ from .sources.base import SourceFailure
from .library_nfo import sidecars, SidecarIndex
from .media_io import directory_files, read_nfo, local_art, root_online, directory_entries, file_stat
from .genre_decisions import load_genre_decisions
-from .metadata import candidate_text_is_valid, extract_catalog_evidence, extract_peach_fields, validate_provider_code
+from .metadata import (candidate_text_is_valid, source_text_is_valid,
+ extract_catalog_evidence, extract_peach_fields, validate_provider_code)
from . import metadata_routes
from . import media_identity
from . import junk
@@ -1492,7 +1493,10 @@ class _RemoteSession:
continue
if time.time() - path.stat().st_mtime > MISS_TTL_SECONDS:
continue
- entry = (name, json.loads(path.read_text(encoding='utf-8')), path)
+ payload = json.loads(path.read_text(encoding='utf-8'))
+ if not source_text_is_valid(payload):
+ continue
+ entry = (name, payload, path)
self._task_evidence[name, code] = entry
found.append(entry)
except (OSError, ValueError):
diff --git a/src/peach/metadata.py b/src/peach/metadata.py
index a48908368..f95fa0a07 100644
--- a/src/peach/metadata.py
+++ b/src/peach/metadata.py
@@ -449,6 +449,13 @@ def candidate_text_is_valid(candidate: dict) -> bool:
for key in ('value', 'display_value', 'unmapped_genres'))
+def source_text_is_valid(payload: dict) -> bool:
+ """来源文字必须完整;原始旁证、路径和诊断文字不决定缓存有效性。"""
+ return not any(contains_replacement_character(payload.get(key)) for key in (
+ 'title', 'original_title', 'actresses', 'actors', 'performers', 'maker', 'studio',
+ 'label', 'series', 'genres', 'tags', 'director'))
+
+
def extract_catalog_evidence(payload: dict) -> dict[str, dict]:
"""Preserve source-only catalog fields without mapping them to ledger truth.
diff --git a/src/peach/metadata_amane.py b/src/peach/metadata_amane.py
index 825f36062..6e763ed8d 100644
--- a/src/peach/metadata_amane.py
+++ b/src/peach/metadata_amane.py
@@ -17,6 +17,7 @@ from __future__ import annotations
import hashlib
import json
+import logging
import os
import re
import shutil
@@ -497,25 +498,24 @@ class AmaneBridge:
command.extend(["--direct-sites", ",".join(direct_sites)])
try:
completed = self.runner(
- command, capture_output=True, text=True, encoding="utf-8", errors="replace",
+ command, capture_output=True, text=True, encoding="utf-8", errors="strict",
timeout=min(timeout, self.timeout) if timeout else self.timeout,
check=False, shell=False, env=env,
creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))
+ except UnicodeDecodeError as exc:
+ # 不猜编码、不删除字节;保留可在诊断日志中定位的阶段和指纹,不记录正文或代理凭据。
+ message = (f"amane 桥编码错误(bridge_stdio_decode,{number},{','.join(names)},"
+ f"runtime={'executable' if self.script is None else 'python'},expected=utf-8,"
+ f"bytes={len(exc.object)},offset={exc.start}:{exc.end},"
+ f"sha256={hashlib.sha256(exc.object).hexdigest()})")
+ logging.getLogger(__name__).warning('%s', message)
+ raise MetadataProviderError(message, kind="unavailable", detail="bridge_stdio_decode") from exc
except subprocess.TimeoutExpired as exc:
raise MetadataProviderError("amane 桥查询超时", kind="unavailable",
retryable=True, temporary=True) from exc
except OSError as exc:
raise MetadataProviderError(f"无法启动 amane 桥:{exc}") from exc
- line = next((row for row in reversed(completed.stdout.splitlines()) if row.strip()), "")
- try:
- report = json.loads(line)
- except (TypeError, json.JSONDecodeError) as exc:
- detail = (completed.stderr or completed.stdout or "empty output").strip()[-800:]
- raise MetadataProviderError(f"amane 桥返回了非 JSON 输出:{detail}") from exc
- if not isinstance(report, dict) or "sites" not in report:
- message = report.get("error") if isinstance(report, dict) else ""
- raise MetadataProviderError(str(message or f"amane 桥查询失败(exit {completed.returncode})"))
- return report
+ return _bridge_report(completed, runtime='executable' if self.script is None else 'python')
def _serial_query(self, code, sites, *, language, proxy_options, direct_sites, timeout):
"""低占用逐站启动桥查询,保留统一的整档时间预算。"""
@@ -533,6 +533,35 @@ class AmaneBridge:
return {'number': code, 'sites': records}
+def _bridge_report(completed, *, runtime: str) -> dict:
+ """校验桥的 JSON 报告,并给各站快照附加有界的传输诊断。"""
+ line = next((row for row in reversed(completed.stdout.splitlines()) if row.strip()), "")
+ try:
+ report = json.loads(line)
+ except (TypeError, json.JSONDecodeError) as exc:
+ detail = (completed.stderr or completed.stdout or "empty output").strip()[-800:]
+ raise MetadataProviderError(f"amane 桥返回了非 JSON 输出:{detail}") from exc
+ if not isinstance(report, dict) or "sites" not in report:
+ message = report.get("error") if isinstance(report, dict) else ""
+ raise MetadataProviderError(str(message or f"amane 桥查询失败(exit {completed.returncode})"))
+ transport = report.get('transport')
+ transport = transport if isinstance(transport, dict) else {}
+ amane = report.get('amane')
+ amane = amane if isinstance(amane, dict) else {}
+ diagnostics = {
+ 'stage': 'bridge_stdio_decode', 'status': 'ok', 'expected_encoding': 'utf-8',
+ 'runtime': runtime,
+ 'stdout_encoding': str(transport.get('stdout_encoding') or 'unknown')[:40],
+ 'stderr_encoding': str(transport.get('stderr_encoding') or 'unknown')[:40],
+ 'amane_version': str(amane.get('version') or 'unknown')[:64],
+ }
+ records = report['sites'] if isinstance(report['sites'], dict) else {}
+ for record in records.values():
+ if isinstance(record, dict):
+ record['collection_diagnostics'] = dict(diagnostics)
+ return report
+
+
def failure_error(site: str, record: Mapping[str, object]) -> MetadataProviderError:
"""桥记录里一站的失败 → Peach 的错误对象。上游 reason 留在 `detail`,措辞里也带着。"""
return site_failure(site, record).provider_error(SITES.get(site, site))
@@ -654,6 +683,8 @@ def split_report(code: str, report: Mapping[str, object]) -> tuple[list[tuple[st
continue
if record.get("status") == "found" and isinstance(record.get("metadata"), dict):
payload = to_payload(site, code, record["metadata"])
+ if record.get('collection_diagnostics'):
+ payload['collection_diagnostics'] = dict(record['collection_diagnostics'])
if identifies_code(code, payload):
found.append((site, payload))
continue
diff --git a/src/peach/metadata_text_repair.py b/src/peach/metadata_text_repair.py
index de7f5e88b..c0204e0f7 100644
--- a/src/peach/metadata_text_repair.py
+++ b/src/peach/metadata_text_repair.py
@@ -7,7 +7,8 @@ import json
from pathlib import Path
from .field_owners import auto_owner, owner_of
-from .metadata import extract_peach_fields, identifies_code, validate_provider_code
+from . import artifact_rules, upgrade_rules
+from .metadata import extract_peach_fields, identifies_code, source_text_is_valid, validate_provider_code
from .metadata_auto_apply import _apply_metadata_candidate, metadata_auto_apply_candidate
from .metadata_policy import SOURCE_SPECS
@@ -15,6 +16,41 @@ FIELDS = {'title': 'catalog_title', 'original_title': 'original_title',
'studio': 'studio', 'series': 'series'}
+def _damaged_snapshots(connection, root: Path) -> tuple[list[dict], list[int]]:
+ """只失效已知来源的损坏文字快照,并选择同番号资产重新采集。"""
+ snapshots, affected = [], []
+ for path in root.glob('*.json'):
+ code, separator, source = path.stem.rpartition('-')
+ if not separator or source not in SOURCE_SPECS or path.is_symlink():
+ continue
+ try:
+ code = validate_provider_code(code)
+ before = artifact_rules.fingerprint(path)
+ payload = json.loads(path.read_text(encoding='utf-8'))
+ except (OSError, ValueError):
+ continue
+ if not isinstance(payload, dict) or source_text_is_valid(payload):
+ continue
+ snapshots.append({'path': str(path), 'fingerprint': before})
+ affected.extend(row[0] for row in connection.execute(
+ "SELECT id FROM asset WHERE code=? AND medium='video' AND disposal IS NULL", (code,)))
+ return snapshots, affected
+
+
+def _invalidate_snapshots(snapshots: list[dict]) -> int:
+ count = 0
+ for item in snapshots:
+ path = Path(item['path'])
+ try:
+ if artifact_rules.fingerprint(path) != item['fingerprint']:
+ continue
+ artifact_rules.remember(path, f'invalid:provider-text-{upgrade_rules.PROVIDER_TEXT}')
+ count += 1
+ except FileNotFoundError:
+ continue
+ return count
+
+
def _candidates(root: Path, code: str, field: str) -> list[dict]:
candidates = []
for source in SOURCE_SPECS:
@@ -76,7 +112,8 @@ def collect(connection, snapshots: Path) -> dict:
for role, entity_id in damaged_roles:
rows.append({'asset_id':asset['id'], 'field':role, 'entity_id':entity_id,
'action':'review', 'reason':'entity-identity-requires-review'})
- return {'rows':rows, 'asset_ids':affected}
+ invalid_snapshots, snapshot_assets = _damaged_snapshots(connection, snapshots)
+ return {'rows':rows, 'asset_ids':sorted(set(affected + snapshot_assets)), 'snapshots':invalid_snapshots}
def apply(connection, plan: dict) -> dict:
@@ -93,4 +130,5 @@ def apply(connection, plan: dict) -> dict:
candidate = row['candidate']
changed += bool(_apply_metadata_candidate(connection, row['group'], candidate, stamp,
auto_owner(candidate['source'])))
- return {'text_fields':changed, 'text_review':sum(r['action']=='review' for r in plan['rows'])}
+ return {'text_fields':changed, 'text_review':sum(r['action']=='review' for r in plan['rows']),
+ 'text_snapshots_invalidated':_invalidate_snapshots(plan.get('snapshots', []))}
diff --git a/src/peach/scraping_diagnostics.py b/src/peach/scraping_diagnostics.py
index 0ef611d18..b0658c3f9 100644
--- a/src/peach/scraping_diagnostics.py
+++ b/src/peach/scraping_diagnostics.py
@@ -26,7 +26,7 @@ ROW_LIMIT = 30_000
SNAPSHOT_FIELDS = frozenset({
'id', 'content_id', 'title', 'original_title', 'release_date', 'release', 'date',
'actresses', 'actors', 'performers', 'maker', 'studio', 'label', 'series',
- 'genres', 'tags', 'director', 'source', 'source_url', 'catalog_evidence',
+ 'genres', 'tags', 'director', 'source', 'source_url', 'catalog_evidence', 'collection_diagnostics',
})
CANDIDATE_FIELDS = frozenset({
'candidate_key', 'source', 'provider', 'value', 'display_value', 'confidence',
diff --git a/src/peach/upgrade_repairs.py b/src/peach/upgrade_repairs.py
index b1beafaa2..d220fa9ea 100644
--- a/src/peach/upgrade_repairs.py
+++ b/src/peach/upgrade_repairs.py
@@ -177,8 +177,10 @@ class UpgradeRepairs:
with contract.database.read_connection() as connection:
applied = connection.execute('SELECT result_json FROM ledger_upgrade_repair WHERE id=?',
(REVISION,)).fetchone()
- prior = connection.execute('SELECT result_json FROM ledger_upgrade_repair WHERE id=?',
- (CATALOG_REVISION,)).fetchone()
+ prior = connection.execute(
+ 'SELECT result_json,id FROM ledger_upgrade_repair WHERE id=? OR id GLOB ? '
+ 'ORDER BY CAST(applied_at AS REAL) DESC LIMIT 1',
+ (CATALOG_REVISION, CATALOG_REVISION + ':text-*')).fetchone()
if applied is not None:
self._restore_receipt(json.loads(applied[0]))
return
@@ -216,7 +218,7 @@ class UpgradeRepairs:
'deferred': list(previous.get('deferred', [])),
'dispatch_complete': bool(previous.get('dispatch_complete')) and not plan['text']['asset_ids'],
'text_only': prior is not None,
- 'prior_revision': CATALOG_REVISION if prior else None}
+ 'prior_revision': prior[1] if prior else None}
connection.execute('INSERT INTO ledger_upgrade_repair(id,applied_at,result_json) VALUES(?,?,?)',
(REVISION, str(time.time()), json.dumps(receipt, ensure_ascii=False)))
atomic_write_text(self.receipt, json.dumps(receipt, ensure_ascii=False))
@@ -390,6 +392,9 @@ def _media_run(contract, key: str, handle) -> dict:
config = getattr(contract, 'upgrade_config', None) or load_config()
state_file = Path(contract.follow_state_root) / 'upgrade-repairs' / f'{asset_id}.json'
previous = _read(state_file)
+ job_id = key.replace(':', '-')
+ if previous.get('job_id', job_id) != job_id:
+ previous = {}
if previous:
previous.update(issue_count=0, paused_count=0, issue_preview=[], retryable_asset_ids=[])
service = getattr(contract, 'upgrade_repairs', None)
@@ -398,7 +403,7 @@ def _media_run(contract, key: str, handle) -> dict:
try:
result = process_library(config, contract.db_path, contract.candidate_root, contract.cover_root,
database=contract.database, asset_ids=[asset_id], stage='collect',
- recheck_existing=True, job_id=key.replace(':', '-'),
+ recheck_existing=True, job_id=job_id,
state_file=state_file, resume_state=previous or None,
active=active,
report=lambda state: handle.progress(label=state.get('stage', '更新影片资料')))
diff --git a/src/peach/upgrade_rules.py b/src/peach/upgrade_rules.py
index 992b1a872..1be6e89c8 100644
--- a/src/peach/upgrade_rules.py
+++ b/src/peach/upgrade_rules.py
@@ -6,5 +6,5 @@ LOCAL_POSTERS = PREVIEWS + ':local'
PROCESSING = "processing-v2"
PUBLIC_SNAPSHOTS = 2
PROVIDER_EVIDENCE = 2
-PROVIDER_TEXT = 1
+PROVIDER_TEXT = 2
OFFICIAL_COVER_CATALOG_MISSES = 1
diff --git a/tests/test_error_reporting.py b/tests/test_error_reporting.py
index 4998fd5f2..270bc9ee6 100644
--- a/tests/test_error_reporting.py
+++ b/tests/test_error_reporting.py
@@ -122,6 +122,24 @@ class ErrorReportingTests(unittest.TestCase):
with zipfile.ZipFile(io.BytesIO(scraping_diagnostics.bundle_bytes(config, codes='ABC-123'))) as archive:
self.assertIn('snapshots/0.json', archive.namelist())
+ def test_both_exports_preserve_bridge_encoding_provenance(self):
+ config, evidence = self.scraping_config()
+ facts = {'stage': 'bridge_stdio_decode', 'status': 'ok', 'expected_encoding': 'utf-8',
+ 'runtime': 'executable', 'stdout_encoding': 'utf-8', 'stderr_encoding': 'utf-8',
+ 'amane_version': '0.22.1'}
+ (evidence / 'ABC-123-makers.json').write_text(json.dumps({
+ 'id': 'ABC-123', 'title': '完整𠮷野标题', 'collection_diagnostics': facts,
+ 'Cookie': 'private-cookie'}), encoding='utf-8')
+ with mock.patch.object(error_reporting, 'log_directories', return_value=[]):
+ bundles = [(scraping_diagnostics.bundle_bytes(config, codes='ABC-123'), ''),
+ (support_bundle.bundle_bytes(config=config), 'scraping/')]
+ for content, prefix in bundles:
+ with self.subTest(prefix=prefix), zipfile.ZipFile(io.BytesIO(content)) as archive:
+ snapshot = json.loads(archive.read(prefix + 'snapshots/0.json'))
+ self.assertEqual(snapshot['fields']['collection_diagnostics'], facts)
+ self.assertEqual(snapshot['fields']['title'], '完整𠮷野标题')
+ self.assertNotIn('private-cookie', str(snapshot))
+
def test_support_bundle_includes_available_small_records_beyond_manual_code_limit(self):
config, evidence = self.scraping_config()
for number in range(25):
diff --git a/tests/test_metadata_amane.py b/tests/test_metadata_amane.py
index e310c0eff..31d385bab 100644
--- a/tests/test_metadata_amane.py
+++ b/tests/test_metadata_amane.py
@@ -573,6 +573,47 @@ class BridgeQueryTests(unittest.TestCase):
with self.assertRaises(ValueError):
bridge.query("ABW-220", ["javdb"])
+ def test_invalid_utf8_is_a_diagnostic_failure_instead_of_corrupted_metadata(self):
+ raw = report(fc2club=FOUND).encode('gbk')
+ captured = {}
+
+ def runner(command, **kwargs):
+ captured.update(kwargs)
+ return completed(raw.decode(kwargs['encoding'], kwargs['errors']))
+
+ bridge = metadata_amane.AmaneBridge(self.python, script=None, runner=runner)
+ with self.assertLogs('peach.metadata_amane', level='WARNING') as logs, \
+ self.assertRaises(MetadataProviderError) as caught:
+ bridge.query('FC2-PPV-1234567', ['fc2club'],
+ proxy_options={'trust_env': False, 'proxy': 'http://user:secret@proxy:8888'})
+ self.assertEqual(captured['errors'], 'strict')
+ self.assertEqual(caught.exception.detail, 'bridge_stdio_decode')
+ self.assertEqual(caught.exception.kind, 'unavailable')
+ message = str(caught.exception)
+ for value in ('FC2-PPV-1234567', 'fc2club', 'expected=utf-8', 'runtime=executable',
+ f'bytes={len(raw)}', 'offset=', 'sha256='):
+ self.assertIn(value, message)
+ self.assertIn(message, logs.output[0])
+ self.assertNotIn('secret', message)
+ self.assertNotIn('標題', message)
+ self.assertNotIn('\ufffd', message)
+
+ def test_query_preserves_all_unicode_and_exports_collection_provenance(self):
+ title = '標題𠮷野・中文・é・😀'
+ found = {**FOUND, 'metadata': {**FOUND['metadata'], 'title': title}}
+ raw = json.loads(report(fc2club=found))
+ raw['transport'] = {'stdout_encoding': 'utf-8', 'stderr_encoding': 'utf-8', 'frozen': True}
+ bridge = metadata_amane.AmaneBridge(self.python, script=None,
+ runner=lambda *a, **kw: completed(json.dumps(raw, ensure_ascii=False)))
+ result = bridge.query('FC2-PPV-1234567', ['fc2club'], proxy_options={'trust_env': False})
+ payload = metadata_amane.split_report('FC2-PPV-1234567', result)[0][0][1]
+ self.assertEqual(payload['title'], title)
+ self.assertEqual(payload['collection_diagnostics'], {
+ 'stage': 'bridge_stdio_decode', 'status': 'ok', 'expected_encoding': 'utf-8',
+ 'runtime': 'executable', 'stdout_encoding': 'utf-8', 'stderr_encoding': 'utf-8',
+ 'amane_version': '0.16.1',
+ })
+
@unittest.skipUnless(os.name == 'nt', 'Windows 控制台行为')
def test_query_child_has_no_console_window(self):
script = self.tools.resolve() / 'console_probe.py'
diff --git a/tests/test_metadata_amane_bridge.py b/tests/test_metadata_amane_bridge.py
index b92ae6827..ab6aa2314 100644
--- a/tests/test_metadata_amane_bridge.py
+++ b/tests/test_metadata_amane_bridge.py
@@ -73,6 +73,25 @@ def run_main(argv, runtime):
class BridgeContractTests(unittest.TestCase):
+ def test_cli_outputs_utf8_even_when_redirected_streams_start_as_gbk(self):
+ wrapper = (
+ "import runpy, sys; "
+ "sys.stdout.reconfigure(encoding='gbk'); "
+ "sys.stderr.reconfigure(encoding='gbk'); "
+ "sys.argv = sys.argv[1:]; "
+ "runpy.run_path(sys.argv[0], run_name='__main__')"
+ )
+ for arguments in ([], ['--未知参数']):
+ with self.subTest(arguments=arguments):
+ result = subprocess.run([sys.executable, '-I', '-c', wrapper, str(SCRIPT), *arguments],
+ capture_output=True, check=False, timeout=30)
+ self.assertEqual(result.returncode, bridge.EXIT_USAGE)
+ if not arguments:
+ self.assertEqual(json.loads(result.stdout.decode('utf-8')),
+ {'error': '番号与站名都不能为空'})
+ else:
+ self.assertIn('--未知参数', result.stderr.decode('utf-8'))
+
def test_self_check_loads_every_site_without_creating_a_client(self):
runtime = make_runtime({})
factory = mock.Mock(side_effect=AssertionError("离线自检不能创建 HTTP 客户端"))
diff --git a/tests/test_metadata_library.py b/tests/test_metadata_library.py
index 06eca2d89..c2a4fed20 100644
--- a/tests/test_metadata_library.py
+++ b/tests/test_metadata_library.py
@@ -2063,6 +2063,29 @@ class LibraryNfoTests(unittest.TestCase):
[('javdb', ['高跟', '屁眼'])])
self.assertEqual(Path(rows['tags'][0]['raw_snapshot']), directory / 'DASS-468-javdb.json')
+ def test_damaged_snapshot_is_not_reused_even_with_a_current_receipt(self):
+ from peach.library_processing import _MissCache, _RemoteSession, misses_path
+ config = PeachConfig(self.root, self.root / 'config.toml', present=True)
+ path = config.directory('sources') / 'library-metadata' / 'DASS-468-makers.json'
+ path.parent.mkdir(parents=True)
+ for field, value in (('title', '坏\ufffd标题'),
+ ('actresses', [{'japanese_name': '坏\ufffd姓名'}]),
+ ('genres', ['坏\ufffd标签'])):
+ with self.subTest(field=field):
+ path.write_text(json.dumps({'id': 'DASS-468', field: value}), encoding='utf-8')
+ artifact_rules.remember(path, str(upgrade_rules.PROVIDER_EVIDENCE))
+ before = path.read_bytes()
+ session = _RemoteSession(config, None, _MissCache(misses_path(config)), retrying=False)
+ self.assertEqual(session._cached_evidence(['makers'], 'DASS-468'), [])
+ self.assertEqual(path.read_bytes(), before, '原始损坏证据保留供诊断')
+
+ payload = {'id': 'DASS-468', 'title': '完整标题',
+ 'collection_diagnostics': {'detail': '历史诊断\ufffd'}}
+ path.write_text(json.dumps(payload), encoding='utf-8')
+ artifact_rules.remember(path, str(upgrade_rules.PROVIDER_EVIDENCE))
+ session = _RemoteSession(config, None, _MissCache(misses_path(config)), retrying=False)
+ self.assertEqual(session._cached_evidence(['makers'], 'DASS-468'), [('makers', payload, path)])
+
def test_expired_javdb_snapshot_is_requested_once_and_answered_sources_are_skipped(self):
from peach.library_processing import _MissCache, _RemoteSession, misses_path
config = PeachConfig(self.root, self.root / 'config.toml', present=True)
diff --git a/tests/test_upgrade_repairs.py b/tests/test_upgrade_repairs.py
index 8c150cdea..92ccaec30 100644
--- a/tests/test_upgrade_repairs.py
+++ b/tests/test_upgrade_repairs.py
@@ -15,6 +15,40 @@ from support.ledger import fresh_ledger
class UpgradeRepairTests(unittest.TestCase):
+ def test_encoding_upgrade_invalidates_damaged_snapshots_after_a_text_one_receipt(self):
+ from peach import artifact_rules, upgrade_rules
+ self.asset(1, code='EXM-001', duration=1)
+ with self.database.write_transaction() as connection:
+ connection.execute('UPDATE asset SET catalog_title=?,field_owners=? WHERE id=1',
+ ('人工题名', json.dumps({'catalog_title': 'user:manual'})))
+ with patch.object(upgrade_repairs, 'REVISION', f'{upgrade_repairs.CATALOG_REVISION}:text-1'):
+ self.service.prepare()
+ receipt = upgrade_repairs._read(self.service.receipt)
+ receipt.update(queued={'old-work': 99}, deferred=[42], dispatch_complete=True)
+ self.service._save_receipt(receipt)
+ directory = self.root / 'sources/library-metadata'
+ directory.mkdir(parents=True, exist_ok=True)
+ bad, good = directory / 'EXM-001-makers.json', directory / 'EXM-001-dmm.json'
+ for path, title in ((bad, '壊\ufffdれた題名'), (good, '元の題名')):
+ path.write_text(json.dumps({'id': 'EXM-001', 'title': title}), encoding='utf-8')
+ artifact_rules.remember(path, str(upgrade_rules.PROVIDER_EVIDENCE))
+ original = bad.read_bytes()
+ self.service.prepare()
+ self.assertFalse(artifact_rules.current(bad, str(upgrade_rules.PROVIDER_EVIDENCE)))
+ self.assertTrue(artifact_rules.current(good, str(upgrade_rules.PROVIDER_EVIDENCE)))
+ self.assertEqual(bad.read_bytes(), original)
+ receipt = upgrade_repairs._read(self.service.receipt)
+ self.assertEqual(receipt['text_media_ids'], [1])
+ self.assertEqual(receipt['local']['changed']['text_snapshots_invalidated'], 1)
+ self.assertEqual(receipt['queued'], {'old-work': 99})
+ self.assertEqual(receipt['deferred'], [42])
+ self.assertFalse(receipt['dispatch_complete'])
+ with self.database.read_connection() as connection:
+ self.assertEqual(connection.execute('SELECT catalog_title FROM asset WHERE id=1').fetchone()[0],
+ '人工题名')
+ self.service.prepare()
+ self.assertEqual(len(list(self.root.glob('ledger.pre-upgrade-*.db'))), 2)
+
def damaged_asset(self, key=1, *, owner='auto:makers'):
self.asset(key, code='EXM-001', duration=1)
with self.database.write_transaction() as connection:
@@ -81,7 +115,7 @@ class UpgradeRepairTests(unittest.TestCase):
with patch('peach.library_processing._people_allowed',return_value=False), patch(
'peach.seed_followup.plan',return_value=[]), patch('peach.media_sources.cost_for',return_value='free'):
planned=self.service._plan(receipt)
- self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-1:1'])
+ self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-2:1'])
def test_text_only_dispatch_does_not_plan_unrelated_people(self):
from peach.settings_file import load_config
@@ -92,7 +126,7 @@ class UpgradeRepairTests(unittest.TestCase):
'peach.seed_followup.plan',return_value=[]), patch('peach.media_sources.cost_for',return_value='free'), patch(
'peach.performer_alias_followup.stock',side_effect=AssertionError('不能重放人物补修')):
planned=self.service._plan(receipt)
- self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-1:1'])
+ self.assertEqual([row.key for row in planned],['upgrade-library:processing-v2:text-2:1'])
def test_completed_catalog_dispatch_preserves_deferred_work_across_text_revision(self):
with patch.object(upgrade_repairs,'REVISION',upgrade_repairs.CATALOG_REVISION):
@@ -394,6 +428,17 @@ class UpgradeRepairTests(unittest.TestCase):
self.assertEqual(process.call_args.kwargs['asset_ids'], [1])
self.assertTrue(process.call_args.kwargs['recheck_existing'])
+ def test_a_new_text_rule_does_not_resume_completed_metadata_from_the_old_rule(self):
+ from peach.settings_file import load_config
+ self.contract.upgrade_config = load_config(environ={'PEACH_DATA_ROOT': str(self.root)})
+ self.service.root.mkdir(parents=True)
+ previous = {'job_id': 'upgrade-library-processing-v2-text-1-1',
+ 'processing_rule': 'processing-v2', 'completed_metadata': ['asset:1']}
+ (self.service.root / '1.json').write_text(json.dumps(previous), encoding='utf-8')
+ with patch('peach.library_processing.process_library', return_value={'status': 'complete'}) as process:
+ upgrade_repairs._media_run(self.contract, 'upgrade-library:processing-v2:text-2:1', Mock())
+ self.assertIsNone(process.call_args.kwargs['resume_state'])
+
def test_unsigned_sidecar_posters_are_scheduled_without_a_failed_probe_or_snapshot(self):
self.asset(1, duration=20)
posters = self.contract.candidate_root / 'posters'
diff --git a/tools/amane-bridge/bridge.py b/tools/amane-bridge/bridge.py
index 8c336492e..c70040260 100644
--- a/tools/amane-bridge/bridge.py
+++ b/tools/amane-bridge/bridge.py
@@ -193,6 +193,9 @@ async def run(runtime: Runtime, number: str, sites: list[str], *, language: str
"language": language or "",
"sites": dict(zip(sites, records)),
"amane": {"version": runtime.version},
+ "transport": {"stdout_encoding": sys.stdout.encoding,
+ "stderr_encoding": sys.stderr.encoding,
+ "frozen": bool(getattr(sys, 'frozen', False))},
}
@@ -263,4 +266,7 @@ def main(argv: list[str] | None = None, runtime: Runtime | None = None,
if __name__ == "__main__":
+ # 冻结程序不读取 PYTHONIOENCODING;重定向后的默认编码可能仍是 Windows GBK。
+ sys.stdout.reconfigure(encoding="utf-8")
+ sys.stderr.reconfigure(encoding="utf-8")
sys.exit(main())