Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(media): 修复预览连接与签名清单续播

longmeidaoe5201712d557
diff --git a/docs/MEDIA_CACHE.md b/docs/MEDIA_CACHE.md
index f58aacbc4..2efabb7e7 100644
--- a/docs/MEDIA_CACHE.md
+++ b/docs/MEDIA_CACHE.md
@@ -6,7 +6,7 @@
| --- | --- |
| 关注图片、作品封面与预览图 | 首次实际读取后持久保存;成功内容不自动过期 |
| 关注视频 | 播放时保存完整取得的响应;完整文件可按任意 Range 本地读取,部分响应按原 Range 复用 |
-| JAV 预览 HLS | 清单、嵌套清单、密钥、初始化段与已播放分片按需持久保存 |
+| JAV 预览 HLS | 清单与嵌套清单按来源地址保存,5 分钟后下次读取刷新;浏览器每次读取核对本机清单。密钥、初始化段与已播放分片持久保存 |
| 远程头像 | 90 天后下次读取时刷新;失败保留已有图片,1 小时内不反复重取 |
| favicon、来源图标与外链圆标 | 365 天后下次读取时刷新;失败保留已有图片 |
@@ -14,4 +14,6 @@
浏览器只取得条目 ID 与缓存对象 ID;服务端保留公开 HTTPS 校验、逐跳目标检查和来源凭据隔离。缓存源信息属于本机私有数据,不进入日志和公开产物。用户隐藏图片后,卡面使用对应图片的版本地址。
+公开预览发生 TLS 连接错误时,流式传输复用已安装的 `curl_cffi` 浏览器传输。HTTP 拒绝状态不切换传输;两种传输共用逐跳目标检查。退出播放会关闭上游响应和本次连接池。
+
关注管理在存在已启用的检查失败来源时显示「检查失败项」,按钮提交这些来源的 ID;没有可重试失败来源时隐藏。单条来源检查和「检查全部」继续使用同一任务入口。
diff --git a/docs/REUSE.md b/docs/REUSE.md
index 0d23facdb..cefc4ba00 100644
--- a/docs/REUSE.md
+++ b/docs/REUSE.md
@@ -528,6 +528,8 @@ Python 复用 MIT 许可的 [Ruff](https://docs.astral.sh/ruff/) 0.16.10,版
公开作品资料与评论复用 `jav_cache.PublicSnapshots`、既有 `filelock` 和原子写入;`cache_fresh` 与前端 `publicationFresh` 统一日历月边界、每天补齐及满月后的固定快照。`public_cache_scope` 将快照隔离到调用者数据目录,记录实际来源时间;测试显式传临时目录,默认 `PublicAPI()` 只用内存。媒体字节复用媒体缓存,来源 JSON 不承载本地标签决定、想要状态或观看历史。
+公开预览复用 HTTPX 0.28.1 的流式响应与已安装的 `curl_cffi` 0.16.3(MIT)浏览器传输。`preview_transport` 只适配响应生命周期与 TLS 连接失败,逐跳主机判据仍复用 `open_upstream`;HTTP 拒绝状态不切换传输。已读取固定版本 SDK 的 `Session.request`、`Response.iter_content` 和 `close`,实际公开 MP4 范围请求取得 206,无新增依赖。HLS 根与嵌套清单使用来源地址作为代际,5 分钟后按需刷新,浏览器核对本机清单;分片、密钥与初始化段继续复用持久缓存。MIKR-122 的来源新清单包含可用的当前签名;作品资料及评论的发行日期缓存独立保存。缓存与协议证据沿用 [媒体缓存](MEDIA_CACHE.md) 中现有 `m3u8` 和 Hishel 评估。
+
## 原位改字
`scripts/dev/copy-editor.js` 复用官网本地预览的 `edit-client.js` 交互:文字范围提示、原位编辑、按钮文字就近输入、回车和失焦保存、Esc 取消。
diff --git a/scripts/test_runner.py b/scripts/test_runner.py
index 5ea7713b8..9797205c4 100644
--- a/scripts/test_runner.py
+++ b/scripts/test_runner.py
@@ -65,7 +65,7 @@ SCOPES: dict[str, tuple[str, ...]] = {
"test_web_timeline_thumbnails.py",
"test_interaction.py", "test_media.py", "test_mp4repair.py",
"test_mp4recover.py", "test_web_media_repair.py",
- "test_previews.py",
+ "test_previews.py", "test_preview_transport.py",
"test_providers.py", "test_segments.py", "test_streaming.py",
"test_subtitles.py", "test_transcodes.py"),
"sync": ("test_sync*.py", "test_platform.py", "test_mount.py", "test_tray*.py", "test_log_retention.py",
@@ -163,6 +163,8 @@ SCOPE_TEST_IDS: dict[str, tuple[str, ...]] = {
# 一个文件可以落到多个域:读它的测试登记在哪个域,改它就得跑到哪个域。
# `tests/test_test_planning.py` 按测试源码里真实读到的路径反查这两张表,漏一条本地就红。
AUTO_SCOPE_FILES: dict[str, tuple[str, ...]] = {
+ "src/peach/preview_cache.py": ("follow", "catalog", "media", "web"),
+ "src/peach/preview_transport.py": ("catalog", "media", "web"),
"src/peach/dev_copy.py": ("web",),
"scripts/dev/copy-editor.js": ("web",),
# 路由页面同时被目录、工具与前端三个域的测试读源码。
diff --git a/src/peach/preview_cache.py b/src/peach/preview_cache.py
index 184fda3cd..ed8fdbd49 100644
--- a/src/peach/preview_cache.py
+++ b/src/peach/preview_cache.py
@@ -10,6 +10,7 @@ import m3u8
from .fsutil import atomic_write_text
from .http import public_https_url
+PLAYLIST_REFRESH_SECONDS = 300
class PreviewCache:
def __init__(self, cache):
@@ -46,7 +47,9 @@ class PreviewCache:
if not body.decode("utf-8").removeprefix(chr(0xFEFF)).lstrip().startswith("#EXTM3U"):
raise ValueError("预览清单未取得")
return body
- body = self.cache.bytes(f"preview:{code}:{ref}:playlist", download)
+ generation = hashlib.sha256(url.encode()).hexdigest()
+ body = self.cache.bytes(f"preview:{code}:{ref}:playlist:{generation}", download,
+ ttl=PLAYLIST_REFRESH_SECONDS)
playlist = m3u8.loads(body.decode("utf-8").removeprefix(chr(0xFEFF)), uri=url)
nested = [*playlist.playlists, *playlist.iframe_playlists, *playlist.media]
objects = [*playlist.segments, *nested, *playlist.keys, *playlist.session_keys, *playlist.segment_map]
diff --git a/src/peach/preview_transport.py b/src/peach/preview_transport.py
new file mode 100644
index 000000000..3bb9f70b2
--- /dev/null
+++ b/src/peach/preview_transport.py
@@ -0,0 +1,66 @@
+"""公开预览的流式传输;TLS 连接失败可使用已安装的浏览器传输。"""
+from __future__ import annotations
+
+import httpx
+from curl_cffi import requests
+from curl_cffi.requests.exceptions import CurlError
+
+from . import peach_proxy
+from .follow_stream import open_upstream
+
+
+class _CurlStream(httpx.SyncByteStream):
+ def __init__(self, response):
+ self.response = response
+ self.owner = None
+
+ def __iter__(self):
+ try:
+ yield from self.response.iter_content()
+ except CurlError as error:
+ raise OSError('预览传输中断') from error
+
+ def close(self):
+ try:
+ self.response.close()
+ finally:
+ if self.owner is not None:
+ self.owner.close()
+ self.owner = None
+
+
+class _CurlClient:
+ def __init__(self, root):
+ options = peach_proxy.client_options(root)
+ self.session = requests.Session(impersonate='chrome', proxy=options.get('proxy'),
+ trust_env=options.get('trust_env', True))
+
+ def build_request(self, method, url, *, headers):
+ return httpx.Request(method, url, headers=headers)
+
+ def send(self, request, *, stream, follow_redirects):
+ try:
+ response = self.session.request(request.method, str(request.url), headers=dict(request.headers),
+ stream=stream, allow_redirects=follow_redirects, accept_encoding=None, timeout=20)
+ except CurlError as error:
+ raise OSError('预览连接未取得') from error
+ return httpx.Response(response.status_code, headers=dict(response.headers),
+ stream=_CurlStream(response), request=request)
+
+ def close(self):
+ self.session.close()
+
+
+def open_preview_upstream(client, method, target, *, incoming, secrets_root):
+ """两种传输共用逐跳目标检查;HTTP 拒绝状态不触发传输切换。"""
+ try:
+ return open_upstream(client, method, target, incoming=incoming)
+ except (httpx.ConnectError, httpx.ConnectTimeout):
+ fallback = _CurlClient(secrets_root)
+ try:
+ response = open_upstream(fallback, method, target, incoming=incoming)
+ response.stream.owner = fallback
+ return response
+ except BaseException:
+ fallback.close()
+ raise
diff --git a/src/peach/routes_media.py b/src/peach/routes_media.py
index f1c965e25..70b7de519 100644
--- a/src/peach/routes_media.py
+++ b/src/peach/routes_media.py
@@ -1007,6 +1007,7 @@ def _jav_preview_target(url: str) -> ResolvedFollowMedia:
@router.api_route("/jav-preview/{code}/{ref}", methods=["GET", "HEAD"])
def jav_preview(request: Request, code: str, ref: str, args: dict[str, str] = Depends(require_auth)):
from .web_jav_detail import detail_code
+ from .preview_transport import open_preview_upstream
cache = cache_for(request.app.state.web_contract)
previews = PreviewCache(cache)
try:
@@ -1015,9 +1016,12 @@ def jav_preview(request: Request, code: str, ref: str, args: dict[str, str] = De
url = source["url"]
target = _jav_preview_target(url)
client = request.app.state.http_transport.client
+ def upstream(method, target, incoming):
+ return open_preview_upstream(client, method, target, incoming=incoming,
+ secrets_root=request.app.state.web_contract.follow_secrets_root)
if source["playlist"]:
def fetch(url):
- response = open_upstream(client, "GET", _jav_preview_target(url), incoming={})
+ response = upstream("GET", _jav_preview_target(url), {})
try:
chunks, size = [], 0
for chunk in response.iter_bytes():
@@ -1030,9 +1034,9 @@ def jav_preview(request: Request, code: str, ref: str, args: dict[str, str] = De
response.close()
body = previews.playlist(code, ref, fetch)
return Response(b"" if request.method == "HEAD" else body, media_type="application/vnd.apple.mpegurl",
- headers={"cache-control": "private, max-age=31536000", "content-length": str(len(body))})
+ headers={"cache-control": "private, no-cache", "content-length": str(len(body))})
def media():
- response = open_upstream(client, request.method, target, incoming=request.headers)
+ response = upstream(request.method, target, request.headers)
if source.get('media_type') == 'video/mp4':
kind = response.headers.get('content-type', '').split(';')[0].lower()
if kind not in {'video/mp4', 'application/octet-stream', 'binary/octet-stream'}:
diff --git a/src/peach/web_jav_detail.py b/src/peach/web_jav_detail.py
index 1e2985e2e..a9cfb82b3 100644
--- a/src/peach/web_jav_detail.py
+++ b/src/peach/web_jav_detail.py
@@ -258,7 +258,7 @@ def _preview_fields(contract, code, source, *, refresh=False):
if not url:
return {'preview_video_url': '', 'preview_state': 'missing'}
playlist = url.endswith('.m3u8') or '.m3u8?' in url
- ref = PreviewCache(cache_for(contract)).register(code, url, identity="root", playlist=playlist,
+ ref = PreviewCache(cache_for(contract)).register(code, url, playlist=playlist,
media_type='' if playlist else 'video/mp4')
return {'preview_video_url': f"/jav-preview/{code}/{ref}{'.m3u8' if playlist else ''}", 'preview_state': 'ready'}
diff --git a/tests/test_follow_media_cache.py b/tests/test_follow_media_cache.py
index f516dd4cc..63c0080ee 100644
--- a/tests/test_follow_media_cache.py
+++ b/tests/test_follow_media_cache.py
@@ -162,10 +162,28 @@ class PreviewCacheTests(unittest.TestCase):
with mock.patch.object(cache, 'bytes', wraps=cache.bytes) as saved:
self.assertEqual(previews.playlist('ABP-123', root, fetch).decode(), body)
self.assertEqual(saved.call_count, 1)
- fetch.assert_called_once()
+ self.assertEqual(fetch.call_count, 2)
with self.assertRaises(ValueError):
previews.source("ABP-123", "../private")
with self.assertRaises(ValueError):
previews.register("ABP-123", "https://127.0.0.1/a")
for path in cache.root.rglob("*.source.json"):
self.assertIn("url", json.loads(path.read_text()))
+
+ def test_signed_playlist_refreshes_and_registers_current_segment_addresses(self):
+ with tempfile.TemporaryDirectory() as directory:
+ cache = RemoteMediaCache(Path(directory).resolve())
+ previews = PreviewCache(cache)
+ root = previews.register('EXM-001', 'https://cdn.example/root.m3u8', identity='root')
+ def playlist(signature):
+ return f'#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4,\npart.ts?sign={signature}\n#EXT-X-ENDLIST\n'.encode()
+ fetch = mock.Mock(side_effect=[playlist('first'), playlist('current')])
+ first = previews.playlist('EXM-001', root, fetch)
+ for path in cache.root.rglob('*.data'):
+ os.utime(path, (0, 0))
+ current = previews.playlist('EXM-001', root, fetch)
+ self.assertNotEqual(first, current)
+ segment = next(line for line in current.decode().splitlines() if line.startswith('/jav-preview/'))
+ self.assertTrue(previews.source('EXM-001', segment.rsplit('/', 1)[-1])['url'].endswith('sign=current'))
+ self.assertEqual(previews.playlist('EXM-001', root, fetch), current)
+ self.assertEqual(fetch.call_count, 2)
diff --git a/tests/test_preview_transport.py b/tests/test_preview_transport.py
new file mode 100644
index 000000000..415c249cb
--- /dev/null
+++ b/tests/test_preview_transport.py
@@ -0,0 +1,75 @@
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest import mock
+
+import httpx
+
+from peach.follow_stream import FollowProxyError, ResolvedFollowMedia
+from peach.preview_transport import open_preview_upstream
+
+
+class PreviewTransportTests(unittest.TestCase):
+ def setUp(self):
+ self.native = SimpleNamespace(status_code=206,
+ headers={'content-type': 'video/mp4', 'content-length': '8', 'content-range': 'bytes 0-7/100'},
+ iter_content=mock.Mock(return_value=iter([b'0123', b'4567'])), close=mock.Mock())
+ self.session = mock.Mock()
+ self.session.request.return_value = self.native
+ self.factory = mock.patch('peach.preview_transport.requests.Session', return_value=self.session)
+ self.factory.start()
+ self.network = mock.patch('peach.preview_transport.peach_proxy.client_options', return_value={'trust_env': False})
+ self.network.start()
+ self.addCleanup(self.factory.stop)
+ self.addCleanup(self.network.stop)
+ self.target = ResolvedFollowMedia('https://www.prestige-av.com/api/media/preview.mp4',
+ allowed_hosts=('www.prestige-av.com',), public_hosts=True)
+
+ def open(self, client, method='GET'):
+ return open_preview_upstream(client, method, self.target, incoming={'range': 'bytes=0-'},
+ secrets_root=Path('unused'))
+
+ def test_tls_failure_streams_browser_response_and_closes_its_owner(self):
+ with httpx.Client(transport=httpx.MockTransport(lambda request: self._tls_failure(request))) as client:
+ response = self.open(client)
+ self.session.close.assert_not_called()
+ self.assertEqual(response.status_code, 206)
+ self.assertEqual(response.headers['content-range'], 'bytes 0-7/100')
+ self.assertEqual(b''.join(response.iter_raw()), b'01234567')
+ response.close()
+ self.native.close.assert_called_once()
+ self.session.close.assert_called_once()
+ request = self.session.request.call_args.kwargs
+ self.assertEqual(request['headers']['range'], 'bytes=0-')
+ self.assertTrue(request['stream'])
+ self.assertFalse(request['allow_redirects'])
+ self.assertIsNone(request['accept_encoding'])
+
+ def test_http_refusal_never_switches_transport(self):
+ with httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(403, request=request))) as client:
+ with self.assertRaises(FollowProxyError):
+ self.open(client)
+ self.session.request.assert_not_called()
+
+ def test_browser_redirect_outside_public_boundary_closes_without_second_request(self):
+ self.native.status_code = 302
+ self.native.headers = {'location': 'https://private.example/video.mp4'}
+ with httpx.Client(transport=httpx.MockTransport(self._tls_failure)) as client, \
+ mock.patch('peach.follow_stream.resolves_publicly', return_value=False):
+ with self.assertRaises(FollowProxyError):
+ self.open(client)
+ self.session.request.assert_called_once()
+ self.native.close.assert_called_once()
+ self.session.close.assert_called_once()
+
+ def test_head_and_early_stop_release_the_browser_stream_without_reading_body(self):
+ with httpx.Client(transport=httpx.MockTransport(self._tls_failure)) as client:
+ response = self.open(client, 'HEAD')
+ response.close()
+ self.native.iter_content.assert_not_called()
+ self.native.close.assert_called_once()
+ self.session.close.assert_called_once()
+
+ @staticmethod
+ def _tls_failure(request):
+ raise httpx.ConnectError('TLS connection interrupted', request=request)