Peach: Local-first video metadata scraper and media manager.
fix(scraping): 静默启动 amane 并支持复核 CSV 长字段
longmeidaob9b20c7c29f7
diff --git a/docs/REUSE.md b/docs/REUSE.mdindex f94fbe0a5..d2a20cc56 100644--- a/docs/REUSE.md+++ b/docs/REUSE.md@@ -775,6 +775,8 @@ Python、npm 与 GitHub Actions 的版本由 `.github/dependabot.yml` 每周检Windows 媒体子进程复用 Python 标准库 `subprocess.CREATE_NO_WINDOW`:入库探测、抽帧、拼图、预览、关注封面与媒体修复均显式设置,输出捕获、超时和取消按各入口原有契约处理。该标记由 Windows 原生支持,不引入依赖;见 [Python 文档](https://docs.python.org/3/library/subprocess.html#subprocess.CREATE_NO_WINDOW) 与 [Windows 创建标记](https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags)。+amane 查询与桥环境重建也使用 `CREATE_NO_WINDOW`,保留标准输出、错误输出、退出码与超时。复核产物读取复用标准库 `csv.DictReader`、`field_size_limit` 与 `strict=True`,支持单字段 16 Mi 个字符的多行证据;进程全局限制只增不减。解析失败报告文件与行号并保留原文件,截断的引号字段不能作为完整候选继续处理;写入沿用 `review_csv.write_rows(atomic=True)`。依据 [Python CSV 文档](https://docs.python.org/3/library/csv.html),无新增依赖或真实账本写入。+抽帧与拼图通过现有 Pillow 12.3.0(HPND)完整解码输出图像,不按压缩文件大小推断有效性。复用既有 FFmpeg、色彩元数据重试与 `support_bundle`:失败记录包含采样秒数、时长、退出码、超时、工具来源和各次尝试的错误输出,错误原文截断并脱敏。诊断 ZIP 收集处理状态、最近 8 份问题清单及启动、更新、隧道证据,每文件最多读取尾部 96 KiB,媒体名称留在本机。有效的不足 1 KiB JPEG、中文路径、失败重试和诊断导出均有回归;FFmpeg 参数依据[官方文档](https://ffmpeg.org/ffmpeg.html),不引入 ffmpeg-python 等命令包装层,不写真实 ledger。核验范围见 [日志与诊断核验](DIAGNOSTICS.md)。Windows FFmpeg 9.0.2 静态链接 VideoLAN dav1d 1.5.4(BSD-2-Clause,官方源码 1,038,852 字节,Meson + NASM,支持 Windows x64 与 8/10-bit AV1)。版本、HTTPS 来源与 SHA256 固定在 `scripts/ffmpeg-sources.json`,源码和许可证随对应源码归档分发;完整工具的 FFmpeg 二进制增加约 3 MiB,不新增 Python 运行依赖。原生 `av1` 包装器依赖硬件解码,浏览器可播放不能作为 FFmpeg 软件解码证据;CPU 解码直接采用维护中的 dav1d,不为当前仅需解码的链路引入 libaom 编码库。发行验收使用合成 AV1 片段确认 CPU 解码与抽帧。构建参数依据 [FFmpeg libdav1d 文档](https://ffmpeg.org/ffmpeg-codecs.html#libdav1d) 和 [VideoLAN 固定发行源码](https://download.videolan.org/pub/videolan/dav1d/1.5.4/)。diff --git a/src/peach/metadata_amane.py b/src/peach/metadata_amane.pyindex c734a4282..55978d418 100644--- a/src/peach/metadata_amane.py+++ b/src/peach/metadata_amane.py@@ -196,7 +196,8 @@ def rebuild(tools_root: Path, *, runner: Runner = subprocess.run, timeout: int =bridge_home(tools_root).mkdir(parents=True, exist_ok=True)try:completed = runner(command, capture_output=True, text=True, encoding="utf-8",- errors="replace", timeout=timeout, check=False, shell=False, env=env)+ errors="replace", timeout=timeout, check=False, shell=False, env=env,+ creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))except subprocess.TimeoutExpired:return {"ok": False, "error": f"重建 amane 桥超过 {timeout} 秒未完成"}except OSError as exc:@@ -337,7 +338,8 @@ class AmaneBridge:completed = self.runner(command, capture_output=True, text=True, encoding="utf-8", errors="replace",timeout=min(timeout, self.timeout) if timeout else self.timeout,- check=False, shell=False, env=env)+ check=False, shell=False, env=env,+ creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))except subprocess.TimeoutExpired as exc:raise MetadataProviderError("amane 桥查询超时", kind="unavailable",retryable=True, temporary=True) from excdiff --git a/src/peach/review_csv.py b/src/peach/review_csv.pyindex 21a77ea68..dd36e43b4 100644--- a/src/peach/review_csv.py+++ b/src/peach/review_csv.py@@ -26,6 +26,8 @@ from .fsutil import atomic_path#: 复核 CSV 的编码。改它等于改所有复核产物的可读性,不要在调用处覆盖。ENCODING = "utf-8-sig"+#: 候选的 JSON 证据可包含多行原始资料;单字段最多读取 16 Mi 个字符。+MAX_FIELD_CHARS = 16 * 1024 * 1024def read_rows(path: Path | str, *, missing_ok: bool = False) -> list[dict[str, str]]:@@ -41,8 +43,14 @@ def read_rows(path: Path | str, *, missing_ok: bool = False) -> list[dict[str, starget = Path(path)if missing_ok and not target.is_file():return []+ # csv 的限制属于进程全局,只增不减,避免并发读者互相恢复成较小限制。+ csv.field_size_limit(max(csv.field_size_limit(), MAX_FIELD_CHARS))with target.open(encoding=ENCODING, newline="") as handle:- return list(csv.DictReader(handle))+ reader = csv.DictReader(handle, strict=True)+ try:+ return list(reader)+ except csv.Error as exc:+ raise csv.Error(f'复核 CSV 读取失败:{target},第 {reader.reader.line_num} 行:{exc}') from excdef write_rows(diff --git a/tests/test_metadata_amane.py b/tests/test_metadata_amane.pyindex 3041deb53..d4e28a761 100644--- a/tests/test_metadata_amane.py+++ b/tests/test_metadata_amane.py@@ -145,6 +145,8 @@ class RebuildTests(unittest.TestCase):python = metadata_amane.bridge_python(tools)def build(*args, **kwargs):+ self.assertEqual(kwargs['creationflags'],+ getattr(subprocess, 'CREATE_NO_WINDOW', 0))python.parent.mkdir(parents=True, exist_ok=True)python.write_text("", encoding="utf-8")write_amane_metadata(tools, "0.20.0")@@ -295,6 +297,7 @@ class BridgeQueryTests(unittest.TestCase):self.assertEqual(command[command.index("--proxy") + 1], "http://p:1")self.assertEqual(kwargs["timeout"], 42)self.assertFalse(kwargs["shell"])+ self.assertEqual(kwargs['creationflags'], getattr(subprocess, 'CREATE_NO_WINDOW', 0))self.assertEqual(kwargs["env"]["PYTHONIOENCODING"], "utf-8")def test_query_rejects_sites_peach_did_not_open(self):@@ -302,6 +305,17 @@ class BridgeQueryTests(unittest.TestCase):with self.assertRaises(ValueError):bridge.query("ABW-220", ["javdb"])+ @unittest.skipUnless(os.name == 'nt', 'Windows 控制台行为')+ def test_query_child_has_no_console_window(self):+ script = self.tools.resolve() / 'console_probe.py'+ script.write_text(+ 'import ctypes, json\n'+ 'print(json.dumps({"console": ctypes.windll.kernel32.GetConsoleWindow(), "sites": {}}))\n',+ encoding='utf-8')+ bridge = metadata_amane.AmaneBridge(Path(sys.executable), script)+ result = bridge.query('ABW-220', ['makers'], proxy_options={'trust_env': False})+ self.assertEqual(result['console'], 0)+def test_timeouts_and_garbage_output_become_provider_errors(self):def slow(command, **kwargs):raise subprocess.TimeoutExpired(command, kwargs["timeout"])diff --git a/tests/test_review_csv.py b/tests/test_review_csv.pyindex 84c6aa54c..f1a2ad3c4 100644--- a/tests/test_review_csv.py+++ b/tests/test_review_csv.py@@ -3,6 +3,7 @@这里守的是编码契约本身。它此前在 46 个读写点各写一遍,而两条要求都属于「写的时候一切正常、几天后有人用 Excel 打开才发现坏了」那一类,靠人眼复查是拦不住的。"""+import csvimport osimport tempfileimport unittest@@ -18,7 +19,7 @@ class ReviewCsvTests(unittest.TestCase):def setUp(self):self.temp = tempfile.TemporaryDirectory()self.addCleanup(self.temp.cleanup)- self.root = Path(self.temp.name)+ self.root = Path(self.temp.name).resolve()def test_written_file_carries_the_bom_excel_needs(self):"""没有 BOM,一份中文复核表在 Excel 里就是一屏问号。"""@@ -58,6 +59,25 @@ class ReviewCsvTests(unittest.TestCase):read_rows(missing)self.assertEqual(read_rows(missing, missing_ok=True), [])+ def test_large_multiline_evidence_round_trips_without_losing_following_rows(self):+ target = self.root / 'library-metadata-field-candidates.csv'+ rows = [{'code': 'A', 'name': '来源,"证据"\n' * 30000},+ {'code': 'B', 'name': '下一条'}]+ write_rows(target, FIELDS, rows, atomic=True)+ original = target.read_bytes()+ self.assertEqual(read_rows(target), rows)+ self.assertEqual(target.read_bytes(), original)++ def test_incomplete_quoted_evidence_reports_the_file_and_preserves_it(self):+ target = self.root / 'library-metadata-field-candidates.csv'+ original = 'code,name\r\nA,"未写完\r\nB,下一条\r\n'+ target.write_text(original, encoding=ENCODING, newline='')+ with self.assertRaises(csv.Error) as caught:+ read_rows(target)+ self.assertIn(target.name, str(caught.exception))+ self.assertIn(',第 3 行:', str(caught.exception))+ self.assertEqual(target.read_text(encoding=ENCODING, newline=''), original)+def test_an_unexpected_field_is_an_error_by_default(self):"""多余的键几乎总是字段名写错了。