Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(scraping): 静默启动 amane 并支持复核 CSV 长字段

longmeidaob9b20c7c29f7
diff --git a/docs/REUSE.md b/docs/REUSE.md
index f94fbe0a5..d2a20cc56 100644
--- a/docs/REUSE.md
+++ b/docs/REUSE.md
@@ -775,6 +775,8 @@ Python、npm 与 GitHub Actions 的版本由 `.github/dependabot.yml` 每周检
Windows 媒体子进程复用 Python 标准库 `subprocess.CREATE_NO_WINDOW`:入库探测、抽帧、拼图、预览、关注封面与媒体修复均显式设置,输出捕获、超时和取消按各入口原有契约处理。该标记由 Windows 原生支持,不引入依赖;见 [Python 文档](https://docs.python.org/3/library/subprocess.html#subprocess.CREATE_NO_WINDOW) 与 [Windows 创建标记](https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags)。
+amane 查询与桥环境重建也使用 `CREATE_NO_WINDOW`,保留标准输出、错误输出、退出码与超时。复核产物读取复用标准库 `csv.DictReader`、`field_size_limit` 与 `strict=True`,支持单字段 16 Mi 个字符的多行证据;进程全局限制只增不减。解析失败报告文件与行号并保留原文件,截断的引号字段不能作为完整候选继续处理;写入沿用 `review_csv.write_rows(atomic=True)`。依据 [Python CSV 文档](https://docs.python.org/3/library/csv.html),无新增依赖或真实账本写入。
+
抽帧与拼图通过现有 Pillow 12.3.0(HPND)完整解码输出图像,不按压缩文件大小推断有效性。复用既有 FFmpeg、色彩元数据重试与 `support_bundle`:失败记录包含采样秒数、时长、退出码、超时、工具来源和各次尝试的错误输出,错误原文截断并脱敏。诊断 ZIP 收集处理状态、最近 8 份问题清单及启动、更新、隧道证据,每文件最多读取尾部 96 KiB,媒体名称留在本机。有效的不足 1 KiB JPEG、中文路径、失败重试和诊断导出均有回归;FFmpeg 参数依据[官方文档](https://ffmpeg.org/ffmpeg.html),不引入 ffmpeg-python 等命令包装层,不写真实 ledger。核验范围见 [日志与诊断核验](DIAGNOSTICS.md)。
Windows FFmpeg 9.0.2 静态链接 VideoLAN dav1d 1.5.4(BSD-2-Clause,官方源码 1,038,852 字节,Meson + NASM,支持 Windows x64 与 8/10-bit AV1)。版本、HTTPS 来源与 SHA256 固定在 `scripts/ffmpeg-sources.json`,源码和许可证随对应源码归档分发;完整工具的 FFmpeg 二进制增加约 3 MiB,不新增 Python 运行依赖。原生 `av1` 包装器依赖硬件解码,浏览器可播放不能作为 FFmpeg 软件解码证据;CPU 解码直接采用维护中的 dav1d,不为当前仅需解码的链路引入 libaom 编码库。发行验收使用合成 AV1 片段确认 CPU 解码与抽帧。构建参数依据 [FFmpeg libdav1d 文档](https://ffmpeg.org/ffmpeg-codecs.html#libdav1d) 和 [VideoLAN 固定发行源码](https://download.videolan.org/pub/videolan/dav1d/1.5.4/)。
diff --git a/src/peach/metadata_amane.py b/src/peach/metadata_amane.py
index c734a4282..55978d418 100644
--- a/src/peach/metadata_amane.py
+++ b/src/peach/metadata_amane.py
@@ -196,7 +196,8 @@ def rebuild(tools_root: Path, *, runner: Runner = subprocess.run, timeout: int =
bridge_home(tools_root).mkdir(parents=True, exist_ok=True)
try:
completed = runner(command, capture_output=True, text=True, encoding="utf-8",
- errors="replace", timeout=timeout, check=False, shell=False, env=env)
+ errors="replace", timeout=timeout, check=False, shell=False, env=env,
+ creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))
except subprocess.TimeoutExpired:
return {"ok": False, "error": f"重建 amane 桥超过 {timeout} 秒未完成"}
except OSError as exc:
@@ -337,7 +338,8 @@ class AmaneBridge:
completed = self.runner(
command, capture_output=True, text=True, encoding="utf-8", errors="replace",
timeout=min(timeout, self.timeout) if timeout else self.timeout,
- check=False, shell=False, env=env)
+ check=False, shell=False, env=env,
+ creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))
except subprocess.TimeoutExpired as exc:
raise MetadataProviderError("amane 桥查询超时", kind="unavailable",
retryable=True, temporary=True) from exc
diff --git a/src/peach/review_csv.py b/src/peach/review_csv.py
index 21a77ea68..dd36e43b4 100644
--- a/src/peach/review_csv.py
+++ b/src/peach/review_csv.py
@@ -26,6 +26,8 @@ from .fsutil import atomic_path
#: 复核 CSV 的编码。改它等于改所有复核产物的可读性,不要在调用处覆盖。
ENCODING = "utf-8-sig"
+#: 候选的 JSON 证据可包含多行原始资料;单字段最多读取 16 Mi 个字符。
+MAX_FIELD_CHARS = 16 * 1024 * 1024
def read_rows(path: Path | str, *, missing_ok: bool = False) -> list[dict[str, str]]:
@@ -41,8 +43,14 @@ def read_rows(path: Path | str, *, missing_ok: bool = False) -> list[dict[str, s
target = Path(path)
if missing_ok and not target.is_file():
return []
+ # csv 的限制属于进程全局,只增不减,避免并发读者互相恢复成较小限制。
+ csv.field_size_limit(max(csv.field_size_limit(), MAX_FIELD_CHARS))
with target.open(encoding=ENCODING, newline="") as handle:
- return list(csv.DictReader(handle))
+ reader = csv.DictReader(handle, strict=True)
+ try:
+ return list(reader)
+ except csv.Error as exc:
+ raise csv.Error(f'复核 CSV 读取失败:{target},第 {reader.reader.line_num} 行:{exc}') from exc
def write_rows(
diff --git a/tests/test_metadata_amane.py b/tests/test_metadata_amane.py
index 3041deb53..d4e28a761 100644
--- a/tests/test_metadata_amane.py
+++ b/tests/test_metadata_amane.py
@@ -145,6 +145,8 @@ class RebuildTests(unittest.TestCase):
python = metadata_amane.bridge_python(tools)
def build(*args, **kwargs):
+ self.assertEqual(kwargs['creationflags'],
+ getattr(subprocess, 'CREATE_NO_WINDOW', 0))
python.parent.mkdir(parents=True, exist_ok=True)
python.write_text("", encoding="utf-8")
write_amane_metadata(tools, "0.20.0")
@@ -295,6 +297,7 @@ class BridgeQueryTests(unittest.TestCase):
self.assertEqual(command[command.index("--proxy") + 1], "http://p:1")
self.assertEqual(kwargs["timeout"], 42)
self.assertFalse(kwargs["shell"])
+ self.assertEqual(kwargs['creationflags'], getattr(subprocess, 'CREATE_NO_WINDOW', 0))
self.assertEqual(kwargs["env"]["PYTHONIOENCODING"], "utf-8")
def test_query_rejects_sites_peach_did_not_open(self):
@@ -302,6 +305,17 @@ class BridgeQueryTests(unittest.TestCase):
with self.assertRaises(ValueError):
bridge.query("ABW-220", ["javdb"])
+ @unittest.skipUnless(os.name == 'nt', 'Windows 控制台行为')
+ def test_query_child_has_no_console_window(self):
+ script = self.tools.resolve() / 'console_probe.py'
+ script.write_text(
+ 'import ctypes, json\n'
+ 'print(json.dumps({"console": ctypes.windll.kernel32.GetConsoleWindow(), "sites": {}}))\n',
+ encoding='utf-8')
+ bridge = metadata_amane.AmaneBridge(Path(sys.executable), script)
+ result = bridge.query('ABW-220', ['makers'], proxy_options={'trust_env': False})
+ self.assertEqual(result['console'], 0)
+
def test_timeouts_and_garbage_output_become_provider_errors(self):
def slow(command, **kwargs):
raise subprocess.TimeoutExpired(command, kwargs["timeout"])
diff --git a/tests/test_review_csv.py b/tests/test_review_csv.py
index 84c6aa54c..f1a2ad3c4 100644
--- a/tests/test_review_csv.py
+++ b/tests/test_review_csv.py
@@ -3,6 +3,7 @@
这里守的是编码契约本身。它此前在 46 个读写点各写一遍,而两条要求都属于「写的时候
一切正常、几天后有人用 Excel 打开才发现坏了」那一类,靠人眼复查是拦不住的。
"""
+import csv
import os
import tempfile
import unittest
@@ -18,7 +19,7 @@ class ReviewCsvTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
- self.root = Path(self.temp.name)
+ self.root = Path(self.temp.name).resolve()
def test_written_file_carries_the_bom_excel_needs(self):
"""没有 BOM,一份中文复核表在 Excel 里就是一屏问号。"""
@@ -58,6 +59,25 @@ class ReviewCsvTests(unittest.TestCase):
read_rows(missing)
self.assertEqual(read_rows(missing, missing_ok=True), [])
+ def test_large_multiline_evidence_round_trips_without_losing_following_rows(self):
+ target = self.root / 'library-metadata-field-candidates.csv'
+ rows = [{'code': 'A', 'name': '来源,"证据"\n' * 30000},
+ {'code': 'B', 'name': '下一条'}]
+ write_rows(target, FIELDS, rows, atomic=True)
+ original = target.read_bytes()
+ self.assertEqual(read_rows(target), rows)
+ self.assertEqual(target.read_bytes(), original)
+
+ def test_incomplete_quoted_evidence_reports_the_file_and_preserves_it(self):
+ target = self.root / 'library-metadata-field-candidates.csv'
+ original = 'code,name\r\nA,"未写完\r\nB,下一条\r\n'
+ target.write_text(original, encoding=ENCODING, newline='')
+ with self.assertRaises(csv.Error) as caught:
+ read_rows(target)
+ self.assertIn(target.name, str(caught.exception))
+ self.assertIn(',第 3 行:', str(caught.exception))
+ self.assertEqual(target.read_text(encoding=ENCODING, newline=''), original)
+
def test_an_unexpected_field_is_an_error_by_default(self):
"""多余的键几乎总是字段名写错了。