Peach: Local-first video metadata scraper and media manager.
fix(metadata): 修正混合媒体根、STRM评分及采集失败取证
longmeidao66f2f21cd14b
diff --git a/docs/REUSE.md b/docs/REUSE.mdindex 41b872197..3756f43ee 100644--- a/docs/REUSE.md+++ b/docs/REUSE.md@@ -1,5 +1,15 @@# 复用清单+## 首页筛选聚合与缓存++采用 bbingz 的 [PR #205](https://github.com/peach-mitao/peach/pull/205),固定来源提交 `bbf5aac8e9562211c324c2019bff35888266df84`,许可证沿用本项目 AGPL-3.0-or-later。作品筛选集合先在独立子查询中求出,厂牌、标签和演员按既有关系聚合,保留去重计数与标签可见性。没有选中标签时移除无效 `tag_match`,复用既有修订缓存、同键锁与跨连接写入失效机制,不在线改写 SQLite 统计或增加索引迁移。++## 混合媒体根与模型读取++馆藏采集复用 `platform.within_root` 的 SMB 根身份判断,本地路径保留字面归属比较,不逐影片访问挂载目录。STRM 是链接文件,其大小和推广名不作为远端影片的垃圾证据;垃圾队列读取时重新评分,既有人工决定与回收站记录保持原样。++YuNet 与 SFace 沿用既有模型哈希校验,Python 读取模型字节后交给 OpenCV 的 ONNX buffer 入口,避免 Windows 原生文件读取对非 ASCII 路径的限制。没有新增模型副本、依赖或路径重试。隔离验证用同一哈希的模型分别从普通路径与中文路径加载,并用合成数组执行检测和特征计算。+## README Star 图复用 [Star Chart](https://github.com/leereilly/star-chart) 的稳定版 v0.3,固定提交 `e7fa78bdf8f78da6c3feeb3a97b528fd97d62727`,MIT,版权为 Lee Reilly。使用 `neon-glow-stream` 霓虹流图、线性绝对值坐标、浅深双主题与蜜桃粉橙配色;不新增应用运行依赖。数据来自 GitHub 原生公开 Star 历史,不补造缺失记录,也不把历史累计增加数当作完整取消 Star 的净变化账本。diff --git a/docs/SOURCING.md b/docs/SOURCING.mdindex 69427de11..2d76b0432 100644--- a/docs/SOURCING.md+++ b/docs/SOURCING.md@@ -23,6 +23,8 @@## 采集连接与网盘读取+来源失败的问题记录保留阶段、规范番号、来源与脱敏异常链,完整问题日志和刮削诊断导出共用这些证据。界面仍显示简短失败原因;来源未取得不写成正常无结果,也不缓存成已经确认没有封面。模型加载等本地处理故障与来源登录、网络冷却分别定位。HTTP 客户端被拒绝后转用浏览器前,仍检查当前馆藏任务的暂停和停止意图;此控制不接管其他独立任务。+「同步媒体库」与 `scripts/scrape_codes.py` 共用正式来源链。来源设为「直连」时使用直连客户端;设为跟随 Peach 代理时,使用配置页保存的直连、自定义代理或跟随环境设置。在线封面使用同一来源传输,amane 桥把代理及逐站直连设置传给子进程。浏览器取页也遵循显式直连和自定义代理;代理地址带用户名时退回 HTTP 客户端。默认外网 HTTP 请求共用「Peach 代理」,覆盖发现页与作品详情资料、关注媒体、封面与头像、字幕搜索、PikPak API、更新检查与下载、模型下载和远程诊断。独立采集、链接检查、参考资料与发行下载脚本也复用这份设置。来源显式选择「直接连接」、自定义元数据的独立连接策略和注入的传输继续优先。diff --git a/src/peach/junk.py b/src/peach/junk.pyindex b93cbda5d..da5eb3f50 100644--- a/src/peach/junk.py+++ b/src/peach/junk.py@@ -244,9 +244,10 @@ def _junk_rows(connection, *, asset_ids=None, roots=None):def _junk_file_eligible(row) -> bool:- """资料边车和文件自身带番号的分卷不属于垃圾附件。"""+ """资料边车、STRM指针和文件自身带番号的分卷不属于垃圾附件。"""item = PureWindowsPath(row["name"] or row["path"] or "")- if item.suffix.casefold() in MEDIA_SIDECAR_SUFFIXES:+ # STRM 的大小与名字描述的是链接文件,不能作为远端影片的垃圾证据。+ if item.suffix.casefold() in MEDIA_SIDECAR_SUFFIXES or item.suffix.casefold() == '.strm':return Falseidentity = PART_MARK.sub("", PROMO_DOMAIN.sub("", item.stem)).strip("@._- ")return not (row["medium"] == "video" and PART_MARK.search(item.stem)diff --git a/src/peach/library_processing.py b/src/peach/library_processing.pyindex a19a46b34..fde46da7b 100644--- a/src/peach/library_processing.py+++ b/src/peach/library_processing.py@@ -243,6 +243,12 @@ def describe_failure(error):if isinstance(error, (Unavailable, SourceFailure, SourcePaused, PickerError, httpx.TransportError)) and text:return textreturn f'处理出错({type(error).__name__})'+++def _source_failure_details(error, code, action, *, source=None):+ """采集问题保留脱敏异常链和阶段,页面摘要不暴露内部错误正文。"""+ from .error_reporting import exception_facts, scrub+ return scrub({'stage': action, 'code': code, 'source': source, **exception_facts(error)})#: 来源说「没有」不是待办:馆藏里本来就有大量独立资源和创作者作品,任何目录站都收不到#: 它们。逐条记在日志里备查,界面只按这两类各报一个数(`state['notes']`),不进问题清单。MISS_MESSAGES = {'querying_metadata': '外部来源没有这部片的资料,7 天内不再问',@@ -1589,6 +1595,7 @@ class _RemoteSession:wants_tags = 'tags' in missingresults = _MetadataResults(self, code, received)problems, held, entries = [], [], results.entries+ failures = []attempted = set(answered)stages = metadata_routes.stages_for_chain(chain)for source, then in zip(stages, (*stages[1:], '')):@@ -1628,6 +1635,7 @@ class _RemoteSession:problems.append(describe_failure(error)if source in ('community', 'amane', 'amane_official')else f'{SOURCE_LABELS.get(source, source)}:{describe_failure(error)}')+ failures.append(_source_failure_details(error, code, action, source=source))held.append(source_paused(error))continueupdate(stage='保存待确认资料')@@ -1644,7 +1652,8 @@ class _RemoteSession:paused = bool(held) and all(held)issue(row, ('外部资料本趟没轮到:' if paused else '外部资料未取得:') + ';'.join(problems)if problems else MISS_MESSAGES[action],- action=action, retryable=True, paused=paused)+ action=action, retryable=True, paused=paused,+ **({'diagnostics': {'failures': failures}} if failures else {}))return []def _preferred_tags(self, row, code, missing, chain, entries, answered, *, update, issue, deadline):@@ -1668,7 +1677,8 @@ class _RemoteSession:self._record_miss(source, code)else:issue(row, 'JavDB 标签未取得:' + describe_failure(error),- action='querying_metadata', retryable=True, paused=source_paused(error))+ action='querying_metadata', retryable=True, paused=source_paused(error),+ diagnostics=_source_failure_details(error, code, 'querying_metadata', source=source))return []update(stage='保存待确认资料')return [self._evidence(source, code, payload)]@@ -1701,7 +1711,8 @@ class _RemoteSession:_raise_task_failure(error)paused = source_paused(error)issue(row, ('封面本趟没轮到:' if paused else '封面未取得:') + describe_failure(error),- action=action, retryable=True, paused=paused)+ action=action, retryable=True, paused=paused,+ diagnostics=_source_failure_details(error, code, action))return 0@@ -1858,10 +1869,19 @@ def _chosen_asset_paths(db_path, asset_ids):return {source: tuple(paths) for source, paths in chosen.items()}+def _within_processing_root(path, root):+ """按资源身份比较归属,不为逐影片筛选访问挂载目录。"""+ from .platform import within_root+ from .smb_media import SmbResource+ if isinstance(path, SmbResource) or isinstance(root, SmbResource):+ return within_root(path, root)+ return path.is_relative_to(root)++def _online_source_root(source, root, chosen_paths, *, retrying, issue):translated_root = translate_ledger_path(root)relevant = not retrying or any(- path.is_relative_to(translated_root)+ _within_processing_root(path, translated_root)for path in chosen_paths.get(source, ()))if not relevant:return None@@ -2063,7 +2083,7 @@ def _collection_rows(db_path, *, locations, online_roots, asset_ids):if not is_local_extra(row['name'], row['path'],library_roots=library_roots)and row['location'] in locations- and any(translate_ledger_path(row['path']).is_relative_to(root) for root in online_roots)]+ and any(_within_processing_root(translate_ledger_path(row['path']), root) for root in online_roots)]genre_decisions = load_genre_decisions(connection)return rows, genre_decisionsdiff --git a/src/peach/scraping_access.py b/src/peach/scraping_access.pyindex d237afa52..1ecd673ba 100644--- a/src/peach/scraping_access.py+++ b/src/peach/scraping_access.py@@ -529,6 +529,8 @@ class SourceTransport:取到 200 且不是验证页:这个来源固定到浏览器(`fix_to_browser`),本趟之后的请求也走浏览器,返回这一页。本机没有浏览器、浏览器没取到或同样被拦:返回 None,调用方照 HTTP 客户端被拦那样冷却。"""+ from .processing_control import checkpoint+ checkpoint('network')browser = self._browser_for(source)if browser is None:return Nonediff --git a/src/peach/scraping_diagnostics.py b/src/peach/scraping_diagnostics.pyindex b0658c3f9..7fa223e71 100644--- a/src/peach/scraping_diagnostics.py+++ b/src/peach/scraping_diagnostics.py@@ -296,6 +296,8 @@ def _issues(config, asset_ids, export):continuerecord = {key: row[key] for key in ('asset_id', 'message', 'severity', 'failed_action','retryable', 'last_failed_at') if key in row}+ if isinstance(row.get('diagnostics'), dict):+ record['diagnostics'] = error_reporting.scrub(row['diagnostics'])record['job_id'] = path.name[len('library-processing-'):-len('.issues.jsonl')]records.append(record)if len(records) > ROW_LIMIT:diff --git a/tests/test_metadata_library.py b/tests/test_metadata_library.pyindex c2a4fed20..0056a9e5c 100644--- a/tests/test_metadata_library.py+++ b/tests/test_metadata_library.py@@ -89,6 +89,72 @@ def _archive_pages(url):class LibraryNfoTests(unittest.TestCase):+ def test_cover_failure_retains_redacted_exception_in_scraping_export_and_remains_retryable(self):+ from peach.library_processing import _MissCache, _ProcessingIssues, _RemoteSession+ from peach.scraping_diagnostics import _issues++ config = PeachConfig(self.root, self.root / 'config.toml', present=True)+ path = issues_path(config, 'cover-job')+ path.parent.mkdir(parents=True, exist_ok=True)+ provider = stub_provider()+ failure = RuntimeError('model unreadable token=hidden-secret C:\\private\\model.onnx')+ provider.cover.side_effect = [failure, False]+ remote = _RemoteSession(config, lambda: provider, _MissCache(self.root / 'misses.json'), retrying=False)+ state = dict(job_id='cover-job', notes={}, issue_count=0, paused_count=0,+ issue_preview=[], retryable_asset_ids=[], issues_truncated=False)+ issue = _ProcessingIssues(state, path, lambda: None, lambda values: None)+ row = {'id': 1, 'name': 'ABC-199_AI.mp4', 'path': 'R:/Films/ABC-199_AI.mp4'}+ remote._cover(row, 'ABC-199', self.root / 'covers', update=lambda **values: None, issue=issue)+ remote._cover(dict(row, id=2), 'ABC-200', self.root / 'covers', update=lambda **values: None, issue=issue)+ self.assertEqual(provider.cover.call_count, 2)+ self.assertFalse(remote.misses.fresh('cover', 'ABC-199'))+ recorded = json.loads(path.read_text(encoding='utf-8'))+ self.assertEqual(recorded['diagnostics']['code'], 'ABC-199')+ self.assertEqual(recorded['diagnostics']['stage'], 'fetching_cover')+ self.assertEqual(recorded['diagnostics']['exception'][0]['type'], 'RuntimeError')+ self.assertTrue(recorded['diagnostics']['frames'])+ self.assertNotIn('hidden-secret', json.dumps(recorded))+ self.assertNotIn('private', json.dumps(recorded))+ self.assertEqual((state['issue_count'], state['retryable_asset_ids']), (1, [1]))+ exported = {}+ export = SimpleNamespace(add=lambda name, value: exported.update({name: value}),+ unavailable=lambda *args: self.fail(str(args)))+ _issues(config, {1}, export)+ self.assertEqual(exported['issues.json']['records'][0]['diagnostics'], recorded['diagnostics'])+ self.assertEqual(exported['issues.json']['records'][0]['job_id'], 'cover-job')++ def test_collection_keeps_local_and_smb_resources_in_their_online_roots(self):+ from peach.library_processing import _collection_rows, _online_source_root+ from peach.smb_media import SmbResource++ db = fresh_ledger(self.root)+ remote = SmbResource('nas', 'R:/Films', 'smb://files.invalid/films', (), None)+ outside = SmbResource('other', 'R:/Films', 'smb://other.invalid/films', (), None)+ local = self.root / 'local'+ names = ['ABC-101.mp4', 'ABC-102.mp4', 'ABC-103.mp4']+ paths = ['R:/Films/' + names[0], str(local / names[1]), 'R:/Other/' + names[2]]+ resources = [remote / names[0], local / names[1], outside / names[2]]+ translated = dict(zip(paths, resources))+ translated['R:/Films'] = remote+ with closing(sqlite3.connect(db)) as connection, connection:+ connection.executemany('INSERT INTO asset(location,path,name,medium) VALUES(?,?,?,?)',+ [('local', path, name, 'video') for path, name in zip(paths, names)])+ before = db.read_bytes()+ with patch('peach.library_processing.translate_ledger_path', side_effect=translated.__getitem__), \+ patch('peach.library_processing.root_online', return_value=True):+ for roots in ([remote, local], [local, remote]):+ rows, _ = _collection_rows(db, locations={'local': ('R:/Films', str(local))},+ online_roots=roots, asset_ids=None)+ self.assertEqual([row['name'] for row in rows], names[:2])+ rows, _ = _collection_rows(db, locations={'local': ('R:/Films',)},+ online_roots=[remote], asset_ids=[1, 3])+ self.assertEqual([row['name'] for row in rows], names[:1])+ self.assertIsNone(_online_source_root('local', 'R:/Films', {'local': [local / names[1]]},+ retrying=True, issue=Mock()))+ self.assertEqual(_online_source_root('local', 'R:/Films', {'local': [remote / names[0]]},+ retrying=True, issue=Mock()), remote)+ self.assertEqual(db.read_bytes(), before)+def test_collection_selects_main_video_and_preserves_registered_trailer(self):from peach.library_processing import _collection_rowsdb = fresh_ledger(self.root)diff --git a/tests/test_scraping_access.py b/tests/test_scraping_access.pyindex b8cb32d13..28acb0e23 100644--- a/tests/test_scraping_access.py+++ b/tests/test_scraping_access.py@@ -3,6 +3,7 @@ import jsonimport iofrom pathlib import Pathimport tempfile+import threadingimport timefrom types import SimpleNamespaceimport unittest@@ -675,6 +676,67 @@ class BrowserFallbackTests(unittest.TestCase):save(self.root, "minnano-av", {"network": "direct"})self.assertTrue(fixed_to_browser(self.root, "minnano-av"), "改连接方式不丢固定")+ def test_user_pause_between_http_and_browser_waits_before_opening_the_browser(self):+ from peach.processing_control import ProcessingControl++ control = ProcessingControl(self.root / 'control.json')+ paused = threading.Event()+ finished = threading.Event()+ errors = []+ browser = self.browser(lambda request: HttpResponse(200, {}, b'<title>actress</title>'))+ transport = SourceTransport(self.root)+ self.addCleanup(transport.close)++ def http(*args):+ control.request('pause', 'job')+ return HttpResponse(403, {}, b'')++ transport.transports['minnano-av'] = http++ def run():+ try:+ with control.bind('job', publish=lambda **values: paused.set() if values.get('status') == 'paused' else None,+ active=lambda: True, db_path=self.root / 'database/ledger.db', mode='high'):+ transport(self.request, 1, 100)+ except BaseException as error:+ errors.append(error)+ finally:+ finished.set()++ with patch('peach.browser_transport.shared', return_value=browser):+ thread = threading.Thread(target=run)+ thread.start()+ try:+ self.assertTrue(paused.wait(2))+ self.assertEqual(browser.calls, [])+ self.assertFalse(finished.is_set())+ finally:+ control.request('resume', 'job')+ thread.join(2)+ self.assertFalse(thread.is_alive())+ self.assertEqual(errors, [])+ self.assertEqual(browser.calls, [self.SEARCH])++ def test_user_cancel_between_http_and_browser_does_not_open_the_browser(self):+ from peach.jobs import JobCancelled+ from peach.processing_control import ProcessingControl++ control = ProcessingControl(self.root / 'control.json')+ transport = SourceTransport(self.root)+ self.addCleanup(transport.close)++ def http(*args):+ control.request('cancel', 'job')+ return HttpResponse(403, {}, b'')++ transport.transports['minnano-av'] = http+ with patch('peach.browser_transport.shared') as browser, \+ control.bind('job', publish=lambda **values: None, active=lambda: True,+ db_path=self.root / 'database/ledger.db', mode='high'):+ with self.assertRaises(JobCancelled):+ transport(self.request, 1, 100)+ browser.assert_not_called()+def test_a_challenge_page_over_http_also_gets_the_browser_check(self):challenge = HttpResponse(200, {}, b"<html><head><title>Just a moment...</title></head></html>")page = HttpResponse(200, {}, b"<title>actress</title>")diff --git a/tests/test_trash_junk.py b/tests/test_trash_junk.pyindex 3651e568c..628b99bfc 100644--- a/tests/test_trash_junk.py+++ b/tests/test_trash_junk.py@@ -71,6 +71,30 @@ class TrashJunkTests(unittest.TestCase):self.assertIsNone(self.disposal(1))self.assertTrue(self.out.is_file())+ def test_strm_pointers_stay_out_of_junk_and_existing_decisions_are_preserved(self):+ from peach import junk++ for asset_id, name in enumerate(('[email protected]',+ '[email protected]',+ '[email protected]'), 1):+ self.add(asset_id, '115', 'B:\\Films\\' + name, 'video', 1024)+ self.add(4, '115', r'B:\Films\tuu26.com.mp4', 'video', 1024, 60)+ self.add(5, '115', r'B:\Films\previous.strm', 'video', 1024)+ with closing(sqlite3.connect(self.db_path)) as connection, connection:+ connection.execute("UPDATE asset SET disposal='trash',disposal_reason='junk' WHERE id=5")+ connection.execute("INSERT INTO review_decision(category,item_key,status,note,updated_at) "+ "VALUES('junk_file','1','rejected','用户保留','2026-01-01')")+ before = self.db_path.read_bytes()+ queued = self.trash_junk.select_candidates(self.db_path, min_score=40)+ self.assertEqual({row['id'] for row in queued}, {4})+ self.assertEqual(self.db_path.read_bytes(), before)+ result = junk.isolate(self.db_path, asset_ids=[1, 2, 3, 4, 5])+ self.assertEqual(result['ids'], [4])+ self.assertEqual([self.disposal(i) for i in (1, 2, 3, 5)], [None, None, None, 'trash'])+ with closing(sqlite3.connect(self.db_path)) as connection:+ self.assertEqual(connection.execute("SELECT status FROM review_decision "+ "WHERE category='junk_file' AND item_key='1'").fetchone()[0], 'rejected')+def test_apply_trashes_only_candidates_at_or_above_the_bar(self):self.add(1, "115", r"B:\广告\tuu26.com.mp4", "video", 10 * 1024**2, 60)self.add(2, "115", r"B:\番号\正片.mp4", "video", 400 * 1024**2, 900)