Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(metadata): 修正混合媒体根、STRM评分及采集失败取证

longmeidao66f2f21cd14b
diff --git a/docs/REUSE.md b/docs/REUSE.md
index 41b872197..3756f43ee 100644
--- a/docs/REUSE.md
+++ b/docs/REUSE.md
@@ -1,5 +1,15 @@
# 复用清单
+## 首页筛选聚合与缓存
+
+采用 bbingz 的 [PR #205](https://github.com/peach-mitao/peach/pull/205),固定来源提交 `bbf5aac8e9562211c324c2019bff35888266df84`,许可证沿用本项目 AGPL-3.0-or-later。作品筛选集合先在独立子查询中求出,厂牌、标签和演员按既有关系聚合,保留去重计数与标签可见性。没有选中标签时移除无效 `tag_match`,复用既有修订缓存、同键锁与跨连接写入失效机制,不在线改写 SQLite 统计或增加索引迁移。
+
+## 混合媒体根与模型读取
+
+馆藏采集复用 `platform.within_root` 的 SMB 根身份判断,本地路径保留字面归属比较,不逐影片访问挂载目录。STRM 是链接文件,其大小和推广名不作为远端影片的垃圾证据;垃圾队列读取时重新评分,既有人工决定与回收站记录保持原样。
+
+YuNet 与 SFace 沿用既有模型哈希校验,Python 读取模型字节后交给 OpenCV 的 ONNX buffer 入口,避免 Windows 原生文件读取对非 ASCII 路径的限制。没有新增模型副本、依赖或路径重试。隔离验证用同一哈希的模型分别从普通路径与中文路径加载,并用合成数组执行检测和特征计算。
+
## README Star 图
复用 [Star Chart](https://github.com/leereilly/star-chart) 的稳定版 v0.3,固定提交 `e7fa78bdf8f78da6c3feeb3a97b528fd97d62727`,MIT,版权为 Lee Reilly。使用 `neon-glow-stream` 霓虹流图、线性绝对值坐标、浅深双主题与蜜桃粉橙配色;不新增应用运行依赖。数据来自 GitHub 原生公开 Star 历史,不补造缺失记录,也不把历史累计增加数当作完整取消 Star 的净变化账本。
diff --git a/docs/SOURCING.md b/docs/SOURCING.md
index 69427de11..2d76b0432 100644
--- a/docs/SOURCING.md
+++ b/docs/SOURCING.md
@@ -23,6 +23,8 @@
## 采集连接与网盘读取
+来源失败的问题记录保留阶段、规范番号、来源与脱敏异常链,完整问题日志和刮削诊断导出共用这些证据。界面仍显示简短失败原因;来源未取得不写成正常无结果,也不缓存成已经确认没有封面。模型加载等本地处理故障与来源登录、网络冷却分别定位。HTTP 客户端被拒绝后转用浏览器前,仍检查当前馆藏任务的暂停和停止意图;此控制不接管其他独立任务。
+
「同步媒体库」与 `scripts/scrape_codes.py` 共用正式来源链。来源设为「直连」时使用直连客户端;设为跟随 Peach 代理时,使用配置页保存的直连、自定义代理或跟随环境设置。在线封面使用同一来源传输,amane 桥把代理及逐站直连设置传给子进程。浏览器取页也遵循显式直连和自定义代理;代理地址带用户名时退回 HTTP 客户端。
默认外网 HTTP 请求共用「Peach 代理」,覆盖发现页与作品详情资料、关注媒体、封面与头像、字幕搜索、PikPak API、更新检查与下载、模型下载和远程诊断。独立采集、链接检查、参考资料与发行下载脚本也复用这份设置。来源显式选择「直接连接」、自定义元数据的独立连接策略和注入的传输继续优先。
diff --git a/src/peach/junk.py b/src/peach/junk.py
index b93cbda5d..da5eb3f50 100644
--- a/src/peach/junk.py
+++ b/src/peach/junk.py
@@ -244,9 +244,10 @@ def _junk_rows(connection, *, asset_ids=None, roots=None):
def _junk_file_eligible(row) -> bool:
- """资料边车和文件自身带番号的分卷不属于垃圾附件。"""
+ """资料边车、STRM指针和文件自身带番号的分卷不属于垃圾附件。"""
item = PureWindowsPath(row["name"] or row["path"] or "")
- if item.suffix.casefold() in MEDIA_SIDECAR_SUFFIXES:
+ # STRM 的大小与名字描述的是链接文件,不能作为远端影片的垃圾证据。
+ if item.suffix.casefold() in MEDIA_SIDECAR_SUFFIXES or item.suffix.casefold() == '.strm':
return False
identity = PART_MARK.sub("", PROMO_DOMAIN.sub("", item.stem)).strip("@._- ")
return not (row["medium"] == "video" and PART_MARK.search(item.stem)
diff --git a/src/peach/library_processing.py b/src/peach/library_processing.py
index a19a46b34..fde46da7b 100644
--- a/src/peach/library_processing.py
+++ b/src/peach/library_processing.py
@@ -243,6 +243,12 @@ def describe_failure(error):
if isinstance(error, (Unavailable, SourceFailure, SourcePaused, PickerError, httpx.TransportError)) and text:
return text
return f'处理出错({type(error).__name__})'
+
+
+def _source_failure_details(error, code, action, *, source=None):
+ """采集问题保留脱敏异常链和阶段,页面摘要不暴露内部错误正文。"""
+ from .error_reporting import exception_facts, scrub
+ return scrub({'stage': action, 'code': code, 'source': source, **exception_facts(error)})
#: 来源说「没有」不是待办:馆藏里本来就有大量独立资源和创作者作品,任何目录站都收不到
#: 它们。逐条记在日志里备查,界面只按这两类各报一个数(`state['notes']`),不进问题清单。
MISS_MESSAGES = {'querying_metadata': '外部来源没有这部片的资料,7 天内不再问',
@@ -1589,6 +1595,7 @@ class _RemoteSession:
wants_tags = 'tags' in missing
results = _MetadataResults(self, code, received)
problems, held, entries = [], [], results.entries
+ failures = []
attempted = set(answered)
stages = metadata_routes.stages_for_chain(chain)
for source, then in zip(stages, (*stages[1:], '')):
@@ -1628,6 +1635,7 @@ class _RemoteSession:
problems.append(describe_failure(error)
if source in ('community', 'amane', 'amane_official')
else f'{SOURCE_LABELS.get(source, source)}:{describe_failure(error)}')
+ failures.append(_source_failure_details(error, code, action, source=source))
held.append(source_paused(error))
continue
update(stage='保存待确认资料')
@@ -1644,7 +1652,8 @@ class _RemoteSession:
paused = bool(held) and all(held)
issue(row, ('外部资料本趟没轮到:' if paused else '外部资料未取得:') + ';'.join(problems)
if problems else MISS_MESSAGES[action],
- action=action, retryable=True, paused=paused)
+ action=action, retryable=True, paused=paused,
+ **({'diagnostics': {'failures': failures}} if failures else {}))
return []
def _preferred_tags(self, row, code, missing, chain, entries, answered, *, update, issue, deadline):
@@ -1668,7 +1677,8 @@ class _RemoteSession:
self._record_miss(source, code)
else:
issue(row, 'JavDB 标签未取得:' + describe_failure(error),
- action='querying_metadata', retryable=True, paused=source_paused(error))
+ action='querying_metadata', retryable=True, paused=source_paused(error),
+ diagnostics=_source_failure_details(error, code, 'querying_metadata', source=source))
return []
update(stage='保存待确认资料')
return [self._evidence(source, code, payload)]
@@ -1701,7 +1711,8 @@ class _RemoteSession:
_raise_task_failure(error)
paused = source_paused(error)
issue(row, ('封面本趟没轮到:' if paused else '封面未取得:') + describe_failure(error),
- action=action, retryable=True, paused=paused)
+ action=action, retryable=True, paused=paused,
+ diagnostics=_source_failure_details(error, code, action))
return 0
@@ -1858,10 +1869,19 @@ def _chosen_asset_paths(db_path, asset_ids):
return {source: tuple(paths) for source, paths in chosen.items()}
+def _within_processing_root(path, root):
+ """按资源身份比较归属,不为逐影片筛选访问挂载目录。"""
+ from .platform import within_root
+ from .smb_media import SmbResource
+ if isinstance(path, SmbResource) or isinstance(root, SmbResource):
+ return within_root(path, root)
+ return path.is_relative_to(root)
+
+
def _online_source_root(source, root, chosen_paths, *, retrying, issue):
translated_root = translate_ledger_path(root)
relevant = not retrying or any(
- path.is_relative_to(translated_root)
+ _within_processing_root(path, translated_root)
for path in chosen_paths.get(source, ()))
if not relevant:
return None
@@ -2063,7 +2083,7 @@ def _collection_rows(db_path, *, locations, online_roots, asset_ids):
if not is_local_extra(row['name'], row['path'],
library_roots=library_roots)
and row['location'] in locations
- and any(translate_ledger_path(row['path']).is_relative_to(root) for root in online_roots)]
+ and any(_within_processing_root(translate_ledger_path(row['path']), root) for root in online_roots)]
genre_decisions = load_genre_decisions(connection)
return rows, genre_decisions
diff --git a/src/peach/scraping_access.py b/src/peach/scraping_access.py
index d237afa52..1ecd673ba 100644
--- a/src/peach/scraping_access.py
+++ b/src/peach/scraping_access.py
@@ -529,6 +529,8 @@ class SourceTransport:
取到 200 且不是验证页:这个来源固定到浏览器(`fix_to_browser`),本趟之后的请求也走浏览器,返回这一页。
本机没有浏览器、浏览器没取到或同样被拦:返回 None,调用方照 HTTP 客户端被拦那样冷却。
"""
+ from .processing_control import checkpoint
+ checkpoint('network')
browser = self._browser_for(source)
if browser is None:
return None
diff --git a/src/peach/scraping_diagnostics.py b/src/peach/scraping_diagnostics.py
index b0658c3f9..7fa223e71 100644
--- a/src/peach/scraping_diagnostics.py
+++ b/src/peach/scraping_diagnostics.py
@@ -296,6 +296,8 @@ def _issues(config, asset_ids, export):
continue
record = {key: row[key] for key in ('asset_id', 'message', 'severity', 'failed_action',
'retryable', 'last_failed_at') if key in row}
+ if isinstance(row.get('diagnostics'), dict):
+ record['diagnostics'] = error_reporting.scrub(row['diagnostics'])
record['job_id'] = path.name[len('library-processing-'):-len('.issues.jsonl')]
records.append(record)
if len(records) > ROW_LIMIT:
diff --git a/tests/test_metadata_library.py b/tests/test_metadata_library.py
index c2a4fed20..0056a9e5c 100644
--- a/tests/test_metadata_library.py
+++ b/tests/test_metadata_library.py
@@ -89,6 +89,72 @@ def _archive_pages(url):
class LibraryNfoTests(unittest.TestCase):
+ def test_cover_failure_retains_redacted_exception_in_scraping_export_and_remains_retryable(self):
+ from peach.library_processing import _MissCache, _ProcessingIssues, _RemoteSession
+ from peach.scraping_diagnostics import _issues
+
+ config = PeachConfig(self.root, self.root / 'config.toml', present=True)
+ path = issues_path(config, 'cover-job')
+ path.parent.mkdir(parents=True, exist_ok=True)
+ provider = stub_provider()
+ failure = RuntimeError('model unreadable token=hidden-secret C:\\private\\model.onnx')
+ provider.cover.side_effect = [failure, False]
+ remote = _RemoteSession(config, lambda: provider, _MissCache(self.root / 'misses.json'), retrying=False)
+ state = dict(job_id='cover-job', notes={}, issue_count=0, paused_count=0,
+ issue_preview=[], retryable_asset_ids=[], issues_truncated=False)
+ issue = _ProcessingIssues(state, path, lambda: None, lambda values: None)
+ row = {'id': 1, 'name': 'ABC-199_AI.mp4', 'path': 'R:/Films/ABC-199_AI.mp4'}
+ remote._cover(row, 'ABC-199', self.root / 'covers', update=lambda **values: None, issue=issue)
+ remote._cover(dict(row, id=2), 'ABC-200', self.root / 'covers', update=lambda **values: None, issue=issue)
+ self.assertEqual(provider.cover.call_count, 2)
+ self.assertFalse(remote.misses.fresh('cover', 'ABC-199'))
+ recorded = json.loads(path.read_text(encoding='utf-8'))
+ self.assertEqual(recorded['diagnostics']['code'], 'ABC-199')
+ self.assertEqual(recorded['diagnostics']['stage'], 'fetching_cover')
+ self.assertEqual(recorded['diagnostics']['exception'][0]['type'], 'RuntimeError')
+ self.assertTrue(recorded['diagnostics']['frames'])
+ self.assertNotIn('hidden-secret', json.dumps(recorded))
+ self.assertNotIn('private', json.dumps(recorded))
+ self.assertEqual((state['issue_count'], state['retryable_asset_ids']), (1, [1]))
+ exported = {}
+ export = SimpleNamespace(add=lambda name, value: exported.update({name: value}),
+ unavailable=lambda *args: self.fail(str(args)))
+ _issues(config, {1}, export)
+ self.assertEqual(exported['issues.json']['records'][0]['diagnostics'], recorded['diagnostics'])
+ self.assertEqual(exported['issues.json']['records'][0]['job_id'], 'cover-job')
+
+ def test_collection_keeps_local_and_smb_resources_in_their_online_roots(self):
+ from peach.library_processing import _collection_rows, _online_source_root
+ from peach.smb_media import SmbResource
+
+ db = fresh_ledger(self.root)
+ remote = SmbResource('nas', 'R:/Films', 'smb://files.invalid/films', (), None)
+ outside = SmbResource('other', 'R:/Films', 'smb://other.invalid/films', (), None)
+ local = self.root / 'local'
+ names = ['ABC-101.mp4', 'ABC-102.mp4', 'ABC-103.mp4']
+ paths = ['R:/Films/' + names[0], str(local / names[1]), 'R:/Other/' + names[2]]
+ resources = [remote / names[0], local / names[1], outside / names[2]]
+ translated = dict(zip(paths, resources))
+ translated['R:/Films'] = remote
+ with closing(sqlite3.connect(db)) as connection, connection:
+ connection.executemany('INSERT INTO asset(location,path,name,medium) VALUES(?,?,?,?)',
+ [('local', path, name, 'video') for path, name in zip(paths, names)])
+ before = db.read_bytes()
+ with patch('peach.library_processing.translate_ledger_path', side_effect=translated.__getitem__), \
+ patch('peach.library_processing.root_online', return_value=True):
+ for roots in ([remote, local], [local, remote]):
+ rows, _ = _collection_rows(db, locations={'local': ('R:/Films', str(local))},
+ online_roots=roots, asset_ids=None)
+ self.assertEqual([row['name'] for row in rows], names[:2])
+ rows, _ = _collection_rows(db, locations={'local': ('R:/Films',)},
+ online_roots=[remote], asset_ids=[1, 3])
+ self.assertEqual([row['name'] for row in rows], names[:1])
+ self.assertIsNone(_online_source_root('local', 'R:/Films', {'local': [local / names[1]]},
+ retrying=True, issue=Mock()))
+ self.assertEqual(_online_source_root('local', 'R:/Films', {'local': [remote / names[0]]},
+ retrying=True, issue=Mock()), remote)
+ self.assertEqual(db.read_bytes(), before)
+
def test_collection_selects_main_video_and_preserves_registered_trailer(self):
from peach.library_processing import _collection_rows
db = fresh_ledger(self.root)
diff --git a/tests/test_scraping_access.py b/tests/test_scraping_access.py
index b8cb32d13..28acb0e23 100644
--- a/tests/test_scraping_access.py
+++ b/tests/test_scraping_access.py
@@ -3,6 +3,7 @@ import json
import io
from pathlib import Path
import tempfile
+import threading
import time
from types import SimpleNamespace
import unittest
@@ -675,6 +676,67 @@ class BrowserFallbackTests(unittest.TestCase):
save(self.root, "minnano-av", {"network": "direct"})
self.assertTrue(fixed_to_browser(self.root, "minnano-av"), "改连接方式不丢固定")
+ def test_user_pause_between_http_and_browser_waits_before_opening_the_browser(self):
+ from peach.processing_control import ProcessingControl
+
+ control = ProcessingControl(self.root / 'control.json')
+ paused = threading.Event()
+ finished = threading.Event()
+ errors = []
+ browser = self.browser(lambda request: HttpResponse(200, {}, b'<title>actress</title>'))
+ transport = SourceTransport(self.root)
+ self.addCleanup(transport.close)
+
+ def http(*args):
+ control.request('pause', 'job')
+ return HttpResponse(403, {}, b'')
+
+ transport.transports['minnano-av'] = http
+
+ def run():
+ try:
+ with control.bind('job', publish=lambda **values: paused.set() if values.get('status') == 'paused' else None,
+ active=lambda: True, db_path=self.root / 'database/ledger.db', mode='high'):
+ transport(self.request, 1, 100)
+ except BaseException as error:
+ errors.append(error)
+ finally:
+ finished.set()
+
+ with patch('peach.browser_transport.shared', return_value=browser):
+ thread = threading.Thread(target=run)
+ thread.start()
+ try:
+ self.assertTrue(paused.wait(2))
+ self.assertEqual(browser.calls, [])
+ self.assertFalse(finished.is_set())
+ finally:
+ control.request('resume', 'job')
+ thread.join(2)
+ self.assertFalse(thread.is_alive())
+ self.assertEqual(errors, [])
+ self.assertEqual(browser.calls, [self.SEARCH])
+
+ def test_user_cancel_between_http_and_browser_does_not_open_the_browser(self):
+ from peach.jobs import JobCancelled
+ from peach.processing_control import ProcessingControl
+
+ control = ProcessingControl(self.root / 'control.json')
+ transport = SourceTransport(self.root)
+ self.addCleanup(transport.close)
+
+ def http(*args):
+ control.request('cancel', 'job')
+ return HttpResponse(403, {}, b'')
+
+ transport.transports['minnano-av'] = http
+ with patch('peach.browser_transport.shared') as browser, \
+ control.bind('job', publish=lambda **values: None, active=lambda: True,
+ db_path=self.root / 'database/ledger.db', mode='high'):
+ with self.assertRaises(JobCancelled):
+ transport(self.request, 1, 100)
+ browser.assert_not_called()
+
def test_a_challenge_page_over_http_also_gets_the_browser_check(self):
challenge = HttpResponse(200, {}, b"<html><head><title>Just a moment...</title></head></html>")
page = HttpResponse(200, {}, b"<title>actress</title>")
diff --git a/tests/test_trash_junk.py b/tests/test_trash_junk.py
index 3651e568c..628b99bfc 100644
--- a/tests/test_trash_junk.py
+++ b/tests/test_trash_junk.py
@@ -71,6 +71,30 @@ class TrashJunkTests(unittest.TestCase):
self.assertIsNone(self.disposal(1))
self.assertTrue(self.out.is_file())
+ def test_strm_pointers_stay_out_of_junk_and_existing_decisions_are_preserved(self):
+ from peach import junk
+
+ for asset_id, name in enumerate(('[email protected]',
+ self.add(asset_id, '115', 'B:\\Films\\' + name, 'video', 1024)
+ self.add(4, '115', r'B:\Films\tuu26.com.mp4', 'video', 1024, 60)
+ self.add(5, '115', r'B:\Films\previous.strm', 'video', 1024)
+ with closing(sqlite3.connect(self.db_path)) as connection, connection:
+ connection.execute("UPDATE asset SET disposal='trash',disposal_reason='junk' WHERE id=5")
+ connection.execute("INSERT INTO review_decision(category,item_key,status,note,updated_at) "
+ "VALUES('junk_file','1','rejected','用户保留','2026-01-01')")
+ before = self.db_path.read_bytes()
+ queued = self.trash_junk.select_candidates(self.db_path, min_score=40)
+ self.assertEqual({row['id'] for row in queued}, {4})
+ self.assertEqual(self.db_path.read_bytes(), before)
+ result = junk.isolate(self.db_path, asset_ids=[1, 2, 3, 4, 5])
+ self.assertEqual(result['ids'], [4])
+ self.assertEqual([self.disposal(i) for i in (1, 2, 3, 5)], [None, None, None, 'trash'])
+ with closing(sqlite3.connect(self.db_path)) as connection:
+ self.assertEqual(connection.execute("SELECT status FROM review_decision "
+ "WHERE category='junk_file' AND item_key='1'").fetchone()[0], 'rejected')
+
def test_apply_trashes_only_candidates_at_or_above_the_bar(self):
self.add(1, "115", r"B:\广告\tuu26.com.mp4", "video", 10 * 1024**2, 60)
self.add(2, "115", r"B:\番号\正片.mp4", "video", 400 * 1024**2, 900)