Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(media): 回收共享播放任务并隔离活动媒体响应

longmeidaob21bf978b09c
diff --git a/docs/REUSE.md b/docs/REUSE.md
index f0e496c10..13a4e2a6b 100644
--- a/docs/REUSE.md
+++ b/docs/REUSE.md
@@ -908,6 +908,16 @@ CloudDrive2 配置生成复用现有 TOML 模板、mDNS 公布地址和应用服
Python、npm 与 GitHub Actions 的版本由 `.github/dependabot.yml` 每周检查;固定前端文件由
`package-lock.json` 和 `scripts/vendor_web_dependencies.mjs` 重建并核对来源、许可证与 SHA-256。
+## 来源文字质量与生命周期修复
+
+来源文字中的 Unicode 替换字符 `U+FFFD` 表示解码时已有信息丢失。字段提取、既有候选选择和最终写入共用这一判据;含坏值的字段不参与来源优先级结算,原始快照保留。正常字段与正常中日文、重音字母不受影响,不使用启发式转码猜回原文。
+
+存量文字补修复用字段归属、来源身份核对、候选结算与 `UpgradeRepairs` 的备份、事务和回执。仅机器归属的损坏标量字段可按同番号正常来源恢复;人物规范名及人工判断保留复核。待重采的资产使用独立文字规则身份,先前完成的升级任务不吞掉新补修。
+
+播放生命周期复用 [PR190](https://github.com/peach-mitao/peach/pull/190) 的 `e41222e312443b5d0fc7be87d2f7eca9c1d61055`:共享分片按消费者计数,最后消费者离开才取消生成,并等待子进程回收;文件响应保留外层取消与运行错误。头像确认的图像和数据库工作共用工作线程,安装按实体加锁。当前 ASGI 响应头中间件、SMB 输入和图像预处理保留 Peach 主线实现;未采用该 PR 的界面、依赖和测试调度改动。
+
+匿名签名校验复用 [PR150](https://github.com/peach-mitao/peach/pull/150) 的唯一入口、时间戳表达式及字符串约束,适配已存在的动态变量解析接口。媒体响应复用 [PR131](https://github.com/peach-mitao/peach/pull/131) 的被动类型校验、CSP `sandbox` 和 `nosniff`;HLS 清单继续可读,旧活动文档缓存不命中。以上均使用现有标准库、Starlette 和 filelock,无新增依赖。
+
## 静态检查
Windows 媒体子进程复用 Python 标准库 `subprocess.CREATE_NO_WINDOW`:入库探测、抽帧、拼图、预览、关注封面与媒体修复均显式设置,输出捕获、超时和取消按各入口原有契约处理。该标记由 Windows 原生支持,不引入依赖;见 [Python 文档](https://docs.python.org/3/library/subprocess.html#subprocess.CREATE_NO_WINDOW) 与 [Windows 创建标记](https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags)。
diff --git a/docs/ROUTES.md b/docs/ROUTES.md
index cdc533558..8fa032c11 100644
--- a/docs/ROUTES.md
+++ b/docs/ROUTES.md
@@ -133,3 +133,7 @@
`/api/*`、`/stream`、`/follow-stream`、封面、图片及其他媒体端点使用各自的服务契约。实体只读接口 `/api/entity`、`/api/photos` 支持 `kind` 加 `id`,也接受 `kind` 加 `name`。鉴权与本机写入权限由服务端检查。
启用接口文档时,`/docs`、`/docs/oauth2-redirect` 和 `/openapi.json` 是服务文档入口;首次运行表单由未配置状态下的首页展示。
+
+远程媒体代理拒绝 HTML、SVG、XML 和未声明类型的内容;普通媒体和 HLS 清单使用被动响应策略。旧磁盘缓存同样校验类型,缓存字节可复用,浏览器须回源复验。图像响应及其 304 回执携带 CSP `sandbox` 和 `nosniff`。
+
+浏览器首次取得当前媒体策略的 HTML 页面时收到仅清理缓存的 `Clear-Site-Data`,策略 Cookie 避免反复清理,登录和偏好不清除。不支持该响应头的浏览器需手动清理旧站点缓存。
diff --git a/scripts/test_runner.py b/scripts/test_runner.py
index a4c1e5a61..fa839a9f1 100644
--- a/scripts/test_runner.py
+++ b/scripts/test_runner.py
@@ -69,7 +69,7 @@ SCOPES: dict[str, tuple[str, ...]] = {
"test_interaction.py", "test_media.py", "test_mp4repair.py",
"test_mp4recover.py", "test_web_media_repair.py",
"test_previews.py", "test_preview_transport.py",
- "test_providers.py", "test_segments.py", "test_streaming.py",
+ "test_providers.py", "test_segments.py", "test_streaming.py", "test_stream_lifecycle.py",
"test_subtitles.py", "test_transcodes.py", "test_hardware_transcode.py", "test_strm.py",
"test_guangya_sources.py", "test_media_sources.py", "test_smb*.py"),
"sync": ("test_sync*.py", "test_platform.py", "test_mount.py", "test_tray*.py", "test_macos_standalone.py", "test_log_retention.py",
diff --git a/src/peach/api.py b/src/peach/api.py
index c96207af3..c01b8388f 100644
--- a/src/peach/api.py
+++ b/src/peach/api.py
@@ -29,7 +29,7 @@ from contextlib import asynccontextmanager
from urllib.parse import quote
from fastapi import FastAPI, Request
-from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, RedirectResponse
+from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, RedirectResponse, Response
from fastapi.staticfiles import StaticFiles
from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.datastructures import MutableHeaders
@@ -46,6 +46,7 @@ from .follow_covers import FollowCoverService
from .follow_faces import FollowFaceIndex
from .follow_stream import FollowMediaResolver
from .http import HttpxTransport
+from .remote_media_cache import MEDIA_SECURITY_HEADERS
from .health import inspect_database
from .media import (
FilesystemBackend,
@@ -124,6 +125,16 @@ class ResponseHeadersMiddleware:
elif 'cache-control' not in headers:
headers['Cache-Control'] = 'no-store'
headers['X-Content-Type-Options'] = 'nosniff'
+ kind = headers.get('content-type','').split(';',1)[0].strip().lower()
+ if kind.startswith(('image/','video/','audio/')):
+ headers.update(MEDIA_SECURITY_HEADERS)
+ if (scope['method']=='GET' and message['status']==200 and kind=='text/html'
+ and Request(scope).cookies.get('peach_media_policy')!='1'):
+ headers['Clear-Site-Data']='"cache"'
+ policy=Response()
+ policy.set_cookie('peach_media_policy','1',max_age=365*86400,httponly=True,
+ secure=scope['scheme']=='https',samesite='strict')
+ headers.append('set-cookie',policy.headers['set-cookie'])
headers['Referrer-Policy'] = 'same-origin'
headers['X-Robots-Tag'] = routes_pages.ROBOTS_TAG
await send(message)
diff --git a/src/peach/avatar_provider.py b/src/peach/avatar_provider.py
index 4aaa0a93c..6f21ae1ac 100644
--- a/src/peach/avatar_provider.py
+++ b/src/peach/avatar_provider.py
@@ -11,6 +11,7 @@ from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from pathlib import Path
+from filelock import FileLock
from PIL import Image, ImageOps, UnidentifiedImageError
from .avatar_face import FaceProbe, drop_sidecar, write_sidecar
@@ -145,20 +146,22 @@ def install_entity_avatar(avatar_root: Path, kind: str, entity_id: int, body: by
"""
from .previews import entity_image_key
- destination = atomic_write_bytes(
- avatar_root / f"{entity_image_key(kind, entity_id)}.img", body)
- Path(f"{destination}.ct").write_text(content_type, encoding="utf-8")
- record = FaceProbe()(destination) if probe_face and face is None else face
- if record:
- write_sidecar(destination, record)
- else:
- drop_sidecar(destination)
- Path(f"{destination}.provenance.json").write_text(
- json.dumps({**provenance,
- "imported_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
- "purpose": "local performer identity cache"},
- ensure_ascii=False, indent=2),
- encoding="utf-8")
+ destination = avatar_root / f"{entity_image_key(kind, entity_id)}.img"
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ with FileLock(destination.with_suffix('.install.lock'), timeout=30):
+ atomic_write_bytes(destination, body)
+ Path(f"{destination}.ct").write_text(content_type, encoding="utf-8")
+ record = FaceProbe()(destination) if probe_face and face is None else face
+ if record:
+ write_sidecar(destination, record)
+ else:
+ drop_sidecar(destination)
+ Path(f"{destination}.provenance.json").write_text(
+ json.dumps({**provenance,
+ "imported_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+ "purpose": "local performer identity cache"},
+ ensure_ascii=False, indent=2),
+ encoding="utf-8")
return destination
diff --git a/src/peach/remote_media_cache.py b/src/peach/remote_media_cache.py
index 01b7996b6..4eb3d595e 100644
--- a/src/peach/remote_media_cache.py
+++ b/src/peach/remote_media_cache.py
@@ -16,6 +16,19 @@ from .fsutil import atomic_path, atomic_write_text
AVATAR_REFRESH_SECONDS = 90 * 86400
FAVICON_REFRESH_SECONDS = 365 * 86400
MEDIA_HEADERS = ("content-type", "content-length", "content-range", "accept-ranges", "content-encoding")
+MEDIA_SECURITY_HEADERS = {
+ 'content-security-policy': "sandbox; default-src 'none'; frame-ancestors 'none'",
+ 'x-content-type-options': 'nosniff',
+}
+
+
+def passive_media_type(value: str) -> bool:
+ """代理只返回被动媒体;活动文档和未知类型不能在应用域打开。"""
+ kind = value.split(';',1)[0].strip().lower()
+ return (not kind.endswith(('/svg+xml','+xml','/xml'))
+ and (kind.startswith(('image/','video/','audio/'))
+ or kind in ('application/octet-stream','binary/octet-stream',
+ 'application/vnd.apple.mpegurl','application/x-mpegurl')))
class RemoteMediaUnavailable(ValueError):
@@ -44,7 +57,8 @@ class RemoteMediaCache:
try:
metadata = json.loads(slot.with_suffix(".json").read_text(encoding="utf-8"))
path = slot.with_suffix(".bin")
- if path.is_file() and path.stat().st_size == metadata["size"]:
+ if (passive_media_type(metadata['headers'].get('content-type',''))
+ and path.is_file() and path.stat().st_size == metadata["size"]):
return path, metadata
except (OSError, ValueError, KeyError, TypeError):
continue
@@ -55,7 +69,8 @@ class RemoteMediaCache:
if not found:
return None
path, metadata = found
- response_headers = {**metadata["headers"], "cache-control": "private, max-age=31536000", **(headers or {})}
+ response_headers = {**metadata["headers"], "cache-control": "private, no-cache",
+ **(headers or {}), **MEDIA_SECURITY_HEADERS}
if metadata["status"] == 200:
response_headers.pop("content-length", None)
return FileResponse(path, media_type=response_headers.pop("content-type", None), headers=response_headers)
@@ -87,12 +102,13 @@ class RemoteMediaCache:
close()
return hit
upstream = fetch()
+ if not passive_media_type(upstream.headers.get('content-type','')):
+ close()
+ return Response('媒体响应类型不受支持',status_code=502,media_type='text/plain',
+ headers={'cache-control':'no-store',**MEDIA_SECURITY_HEADERS})
forwarded = {name: upstream.headers[name] for name in MEDIA_HEADERS if name in upstream.headers}
- forwarded.update({"cache-control": "private, max-age=31536000", **(headers or {})})
- kind = upstream.headers.get("content-type", "").lower()
- accepted = (upstream.status_code in (200, 206) and
- (kind.startswith(("image/", "video/", "audio/")) or
- kind.split(";")[0] in ("application/octet-stream", "binary/octet-stream")))
+ forwarded.update({'cache-control':'private, no-cache', **(headers or {}), **MEDIA_SECURITY_HEADERS})
+ accepted = upstream.status_code in (200,206)
content_range = re.fullmatch(r"bytes (\d+)-(\d+)/(\d+)", forwarded.get("content-range", ""))
if upstream.status_code == 206 and not content_range:
accepted = False
diff --git a/src/peach/routes_media.py b/src/peach/routes_media.py
index e01199f30..351932c54 100644
--- a/src/peach/routes_media.py
+++ b/src/peach/routes_media.py
@@ -58,8 +58,10 @@ from .routes_auth import require_auth
from .scan import VIDEO
from .segments import SegmentCancelled, SegmentUnavailable, build_hls_playlist
from .streaming import (
+ _cancel_and_wait, _wait_for_disconnect,
BufferedFileResponse, CancellableFileResponse, SplicedMp4Response, run_until_disconnect,
)
+from .remote_media_cache import MEDIA_SECURITY_HEADERS
from .transcodes import TranscodeCancelled, TranscodeNeedsSession, TranscodeUnavailable
router = APIRouter()
@@ -109,9 +111,11 @@ def _image_response(request: Request, path: Path, media_type: str | None = None)
from .smb_response import SmbResponse
return SmbResponse(path, media_type=media_type)
response = FileResponse(path, media_type=media_type, stat_result=path.stat())
+ response.headers.update(MEDIA_SECURITY_HEADERS)
response.headers["Cache-Control"] = "private, no-cache"
if StaticFiles().is_not_modified(response.headers, request.headers):
return Response(status_code=304, headers={
+ **MEDIA_SECURITY_HEADERS,
"ETag": response.headers["etag"],
"Cache-Control": response.headers["cache-control"],
})
@@ -465,11 +469,24 @@ async def _hls_segment(request: Request, id: int, index: int, session: str):
if index < 0 or index >= len(plan):
return JSONResponse({"error": "invalid segment"}, status_code=416)
start, duration = plan[index]
- path = await state.hls_service.generate(
+ generation = asyncio.create_task(state.hls_service.generate(
source, start, duration, asset_id=id, index=index,
session=session, registry=state.stream_sessions,
**({"transcode": True} if conversion else {}),
- )
+ ))
+ disconnect = asyncio.create_task(_wait_for_disconnect(request.receive))
+ try:
+ done, _ = await asyncio.wait({generation, disconnect}, return_when=asyncio.FIRST_COMPLETED)
+ if disconnect in done:
+ disconnect.result()
+ return Response(status_code=204)
+ path = await generation
+ finally:
+ await _cancel_and_wait(generation, disconnect)
+ except asyncio.CancelledError:
+ if state.stream_sessions.is_cancelled(session):
+ return Response(status_code=410, headers={'Cache-Control':'no-store'})
+ raise
except SegmentCancelled:
return Response(status_code=410, headers={"Cache-Control": "no-store"})
except TranscodeCancelled:
@@ -1747,35 +1764,40 @@ async def avatar_pick(request: Request, args: dict[str, str] = Depends(require_a
state = request.app.state.web_contract
providers_root, avatar_root = _picker_roots(state)
payload, body = await _picker_request(request)
- entity_id = _picker_entity(state, payload)
- if entity_id <= 0:
- return JSONResponse({"error": "缺少实体 id"}, status_code=400)
kind = _picker_kind(str(payload.get("kind") or "performer"))
ref = str(payload.get("ref") or "")
version = payload.get("version")
box = _picker_crop(payload.get("crop"))
- try:
+ def confirm():
+ entity_id = _picker_entity(state, payload)
+ if entity_id <= 0:
+ raise avatar_picker.PickerError('缺少实体 id')
+ selected = body
if ref:
with state.read_connection() as connection:
- body, origin = avatar_picker.resolve(
+ selected, origin = avatar_picker.resolve(
ref, connection, providers_root, entity_id,
request.app.state.http_transport, _picker_artwork(request),
version=version if isinstance(version, str) else None,
online=_picker_online(request, connection, entity_id))
- body = avatar_picker.prepare_image(body)
- _remember_library(providers_root, body, origin, avatar_picker.accept_image(body))
- elif body:
- body = await asyncio.to_thread(avatar_picker.prepare_image, body)
+ selected = avatar_picker.prepare_image(selected)
+ _remember_library(providers_root, selected, origin, avatar_picker.accept_image(selected))
+ elif selected:
+ selected = avatar_picker.prepare_image(selected)
origin = {"source": "avatar picker", "provider": "upload",
"external_id": str(payload.get("name") or "")}
else:
- return JSONResponse({"error": "没有可用的图片"}, status_code=400)
+ raise avatar_picker.PickerError('没有可用的图片')
if box not in (None, ""):
- body, cropped = avatar_picker.crop(body, box)
+ selected, cropped = avatar_picker.crop(selected, box)
origin = {**origin, **cropped}
origin = {**origin, "user_selected": True}
result = avatar_picker.install(providers_root, avatar_root, kind,
- entity_id, body, origin)
+ entity_id, selected, origin)
+ return entity_id, result
+
+ try:
+ entity_id, result = await asyncio.to_thread(confirm)
except avatar_picker.PickerError as error:
return JSONResponse({"error": str(error)}, status_code=400)
# 页面按这份索引决定「出 `<img>` 还是首字母垫底」,换完不失效就看不到新图。
diff --git a/src/peach/segments.py b/src/peach/segments.py
index 3f4595612..182201dd2 100644
--- a/src/peach/segments.py
+++ b/src/peach/segments.py
@@ -8,12 +8,13 @@ import os
import subprocess
import uuid
from collections import OrderedDict
+from dataclasses import dataclass, field
from pathlib import Path
from .ffmpeg import FFmpegResolver
from .hardware_transcode import HardwareTranscoder
from .mp4index import keyframe_seconds, segment_plan
-from .streaming import StreamSessionRegistry
+from .streaming import StreamSessionRegistry, _cancel_and_wait, _wait_for_tasks
from .smb_media import input_url
HLS_SEGMENT_SECONDS = 6
@@ -22,6 +23,7 @@ DEFAULT_CACHE_BYTES = 2 << 30
PLAN_CACHE_LIMIT = 256
CODEC_CACHE_LIMIT = 256
PROBE_TIMEOUT_SECONDS = 20
+GENERATION_TIMEOUT_SECONDS = 120
logger = logging.getLogger(__name__)
@@ -65,6 +67,13 @@ def build_hls_playlist(plan: list[tuple[float, float]], segment_url) -> str:
return "\n".join(lines)
+@dataclass
+class _Generation:
+ lock: asyncio.Lock = field(default_factory=asyncio.Lock)
+ users: int = 0
+ task: asyncio.Task[Path] | None = None
+
+
class HlsSegmentService:
"""按关键帧边界 remux 单个时间段,输出可缓存的 MPEG-TS。"""
@@ -91,7 +100,7 @@ class HlsSegmentService:
self._codecs: OrderedDict[tuple, str] = OrderedDict()
# 同一分片常被并发请求:播放器预取、重连、多设备同看都会撞上同一个目标文件。
# 同目标只放一个生成者,后到者等它完事后直接用缓存,不再各起一个 FFmpeg 重读网盘。
- self._generation_locks: dict[Path, asyncio.Lock] = {}
+ self._generation_locks: dict[Path, _Generation] = {}
def _gate(self) -> asyncio.Semaphore:
if self._semaphore is None:
@@ -187,20 +196,45 @@ class HlsSegmentService:
if target.is_file() and target.stat().st_size:
os.utime(target, None) # 命中即续期,淘汰按最后访问时间
return target
- lock = self._generation_locks.setdefault(target, asyncio.Lock())
+ consumer = asyncio.current_task()
+ if consumer is None or not registry.register(session, consumer):
+ raise SegmentCancelled(session)
+ entry = self._generation_locks.setdefault(target, _Generation())
+ entry.users += 1
try:
- async with lock:
- # 等到锁的可能是排在生成者后面的请求,先看缓存再决定要不要自己跑。
- if target.is_file() and target.stat().st_size:
- os.utime(target, None)
- return target
- return await self._generate_uncached(
- source, start, duration, target,
- session=session, registry=registry, transcode=transcode,
- )
+ if entry.task is not None and entry.task.cancelling():
+ await _wait_for_tasks(entry.task)
+ if entry.task is None or entry.task.done():
+ entry.task = asyncio.create_task(self._generate_locked(
+ entry.lock, source, start, duration, target, registry=registry, transcode=transcode))
+ generation = entry.task
+ await asyncio.wait({generation})
+ return generation.result()
finally:
- if not lock.locked():
- self._generation_locks.pop(target, None)
+ registry.unregister(session, consumer)
+ entry.users -= 1
+ if entry.users == 0:
+ try:
+ if entry.task is not None:
+ await _cancel_and_wait(entry.task)
+ finally:
+ if entry.users == 0 and self._generation_locks.get(target) is entry:
+ self._generation_locks.pop(target, None)
+
+ async def _generate_locked(self, lock: asyncio.Lock, source: Path, start: float,
+ duration: float, target: Path, *, registry: StreamSessionRegistry,
+ transcode: bool) -> Path:
+ async with lock:
+ if target.is_file() and target.stat().st_size:
+ os.utime(target, None)
+ return target
+ try:
+ async with asyncio.timeout(GENERATION_TIMEOUT_SECONDS):
+ return await self._generate_uncached(
+ source, start, duration, target, session=f'hls-{uuid.uuid4().hex}',
+ registry=registry, transcode=transcode)
+ except TimeoutError as error:
+ raise SegmentUnavailable('segment generation timed out') from error
async def _generate_uncached(
self, source: Path, start: float, duration: float, target: Path, *,
@@ -301,11 +335,12 @@ class HlsSegmentService:
_kill_process(process)
await process.communicate()
raise SegmentCancelled(session)
+ communication = asyncio.create_task(process.communicate())
try:
- _, stderr = await process.communicate()
+ _, stderr = await asyncio.shield(communication)
except asyncio.CancelledError:
_kill_process(process)
- await process.communicate()
+ await _wait_for_tasks(communication)
raise
finally:
registry.unregister_process(session, process)
diff --git a/src/peach/streaming.py b/src/peach/streaming.py
index 62a6e7713..7c15ba941 100644
--- a/src/peach/streaming.py
+++ b/src/peach/streaming.py
@@ -98,10 +98,30 @@ async def _wait_for_disconnect(receive: Receive) -> None:
"""等到客户端断开。请求体读完后 uvicorn 的 receive() 只会在断开时再返回一次。"""
while True:
message = await receive()
- if message["type"] == "http.disconnect":
+ task = asyncio.current_task()
+ if message["type"] == "http.disconnect" or (task is not None and task.cancelling()):
return
+async def _wait_for_tasks(*tasks) -> None:
+ """清理期间保留外层取消,等子任务结束后再向上交还。"""
+ finished = asyncio.gather(*tasks, return_exceptions=True)
+ cancelled = False
+ while not finished.done():
+ try:
+ await asyncio.shield(finished)
+ except asyncio.CancelledError:
+ cancelled = True
+ if cancelled:
+ raise asyncio.CancelledError
+
+
+async def _cancel_and_wait(*tasks) -> None:
+ for task in tasks:
+ task.cancel()
+ await _wait_for_tasks(*tasks)
+
+
async def run_until_disconnect[T](operation: Awaitable[T], receive: Receive) -> T:
"""等待请求的工作结果;客户端断开时取消工作,并等待其清理完成。"""
work = asyncio.ensure_future(operation)
@@ -113,17 +133,39 @@ async def run_until_disconnect[T](operation: Awaitable[T], receive: Receive) ->
watch.result()
raise ClientDisconnect()
finally:
- work.cancel()
- watch.cancel()
- await asyncio.gather(work, watch, return_exceptions=True)
+ await _cancel_and_wait(work, watch)
-async def _send_until_disconnect(inner, scope: Scope, receive: Receive, send: Send) -> None:
+async def _send_until_disconnect(inner, scope: Scope, receive: Receive, send: Send, *, is_cancelled=None) -> None:
"""跑 `inner` 发响应,客户端一断开就把它掐掉。"""
+ started = False
+
+ async def tracked_send(message):
+ nonlocal started
+ await send(message)
+ if message['type'] == 'http.response.start':
+ started = True
+
+ respond = asyncio.ensure_future(inner(scope, receive, tracked_send))
+ watch = asyncio.ensure_future(_wait_for_disconnect(receive))
+ cancellation = None
try:
- await run_until_disconnect(inner(scope, receive, send), receive)
- except ClientDisconnect:
- return
+ await asyncio.wait({respond, watch}, return_when=asyncio.FIRST_COMPLETED)
+ except asyncio.CancelledError as error:
+ cancellation = error
+ finally:
+ try:
+ await _cancel_and_wait(respond, watch)
+ except asyncio.CancelledError as error:
+ cancellation = cancellation or error
+ if not respond.cancelled():
+ respond.result()
+ if not watch.cancelled():
+ watch.result()
+ if cancellation is not None and (is_cancelled is None or not is_cancelled()):
+ raise cancellation
+ if respond.cancelled() and not started:
+ await Response(status_code=204)(scope, receive, send)
class BufferedFileResponse(FileResponse):
@@ -141,11 +183,14 @@ class BufferedFileResponse(FileResponse):
chunk_size = 1 << 20
+ def _is_cancelled(self) -> bool:
+ return False
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] != "http":
await super().__call__(scope, receive, send)
return
- await _send_until_disconnect(super().__call__, scope, receive, send)
+ await _send_until_disconnect(super().__call__, scope, receive, send, is_cancelled=self._is_cancelled)
class CancellableFileResponse(BufferedFileResponse):
@@ -163,6 +208,9 @@ class CancellableFileResponse(BufferedFileResponse):
self.session = session
self.registry = registry
+ def _is_cancelled(self) -> bool:
+ return self.registry.is_cancelled(self.session)
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
task = asyncio.current_task()
if task is None or not self.registry.register(self.session, task):
@@ -171,7 +219,8 @@ class CancellableFileResponse(BufferedFileResponse):
try:
await super().__call__(scope, receive, send)
except asyncio.CancelledError:
- logging.getLogger(__name__).info("stream session cancelled: %s", self.session)
+ if self._is_cancelled():
+ logging.getLogger(__name__).info("stream session cancelled: %s", self.session)
raise
finally:
self.registry.unregister(self.session, task)
@@ -250,9 +299,11 @@ class SplicedMp4Response(Response):
await Response(status_code=410, headers={"Cache-Control": "no-store"})(scope, receive, send)
return
try:
- await _send_until_disconnect(self._respond, scope, receive, send)
+ await _send_until_disconnect(self._respond, scope, receive, send,
+ is_cancelled=lambda: tracked and self.registry.is_cancelled(self.session))
except asyncio.CancelledError:
- logging.getLogger(__name__).info("stream session cancelled: %s", self.session)
+ if tracked and self.registry.is_cancelled(self.session):
+ logging.getLogger(__name__).info("stream session cancelled: %s", self.session)
raise
finally:
if tracked and task is not None:
diff --git a/tests/test_fastapi_api.py b/tests/test_fastapi_api.py
index 095af3480..2ba331b60 100644
--- a/tests/test_fastapi_api.py
+++ b/tests/test_fastapi_api.py
@@ -26,6 +26,41 @@ from support.mp4 import minimal_mp4
ROOT = Path(__file__).resolve().parents[1]
+class MediaResponseSecurityTests(unittest.IsolatedAsyncioTestCase):
+ async def test_media_policy_clears_only_old_cache_once_and_isolates_images(self):
+ from starlette.applications import Starlette
+ from starlette.responses import HTMLResponse, Response
+ from starlette.routing import Route
+ from peach.api import ResponseHeadersMiddleware
+ from peach.routes_media import _image_response
+ import httpx
+
+ with tempfile.TemporaryDirectory() as directory:
+ path=Path(directory)/'image.svg'
+ path.write_text('<svg xmlns="http://www.w3.org/2000/svg"/>',encoding='utf-8')
+ async def endpoint(request):
+ if request.url.path=='/page':
+ return HTMLResponse('<html>test</html>')
+ if request.url.path=='/image':
+ return _image_response(request,path,'image/svg+xml')
+ return Response(status_code=204)
+ app=ResponseHeadersMiddleware(Starlette(routes=[Route('/{path}',endpoint)]))
+ async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app),base_url='https://test') as client:
+ client.cookies.set('peach_session','keep')
+ first=await client.get('/page')
+ self.assertEqual(first.headers['clear-site-data'],'"cache"')
+ self.assertIn('httponly',first.headers['set-cookie'].lower())
+ second=await client.get('/page')
+ self.assertNotIn('clear-site-data',second.headers)
+ self.assertEqual(client.cookies.get('peach_session'),'keep')
+ image=await client.get('/image')
+ cached=await client.get('/image',headers={'If-None-Match':image.headers['etag']})
+ self.assertEqual(cached.status_code,304)
+ for response in (image,cached):
+ self.assertIn('sandbox',response.headers['content-security-policy'])
+ self.assertEqual(response.headers['x-content-type-options'],'nosniff')
+
+
HAS_DEPS = all(importlib.util.find_spec(name) for name in ("fastapi", "httpx"))
if HAS_DEPS:
import httpx
diff --git a/tests/test_follow_media_cache.py b/tests/test_follow_media_cache.py
index 63c0080ee..8371ea976 100644
--- a/tests/test_follow_media_cache.py
+++ b/tests/test_follow_media_cache.py
@@ -16,6 +16,38 @@ from peach.remote_media_cache import AVATAR_REFRESH_SECONDS, FAVICON_REFRESH_SEC
class RemoteMediaCacheTests(unittest.IsolatedAsyncioTestCase):
+ async def test_active_documents_are_rejected_without_forwarding_or_caching_bytes(self):
+ self.payload=b'<svg><script>active()</script></svg>'
+ for kind in ('image/svg+xml','text/html','application/xhtml+xml','application/xml',''):
+ with self.subTest(kind=kind):
+ self.headers={'content-type':kind,'content-length':str(len(self.payload))}
+ response=await self.client.get('/media')
+ self.assertEqual(response.status_code,502)
+ self.assertNotIn(b'<script>',response.content)
+ self.assertEqual(response.headers['x-content-type-options'],'nosniff')
+ self.assertIn('sandbox',response.headers['content-security-policy'])
+ self.assertIsNone(self.cache.stored('follow:1:media:0'))
+
+ async def test_old_active_document_cache_is_not_served_after_upgrade(self):
+ slot=self.cache._slot('follow:1:media:0')
+ slot.parent.mkdir(parents=True)
+ body=b'<svg><script>active()</script></svg>'
+ slot.with_suffix('.bin').write_bytes(body)
+ slot.with_suffix('.json').write_text(json.dumps({'status':200,'size':len(body),
+ 'headers':{'content-type':'image/svg+xml'}}),encoding='utf-8')
+ response=await self.client.get('/media')
+ self.assertEqual(response.content,self.payload)
+ self.assertEqual(self.calls,1)
+ self.assertEqual(response.headers['cache-control'],'private, no-cache')
+
+ async def test_hls_manifests_remain_available_as_passive_media(self):
+ self.payload=b'#EXTM3U\n'
+ self.headers={'content-type':'application/vnd.apple.mpegurl','content-length':str(len(self.payload))}
+ response=await self.client.get('/media')
+ self.assertEqual(response.status_code,200)
+ self.assertEqual(response.content,self.payload)
+ self.assertIn('sandbox',response.headers['content-security-policy'])
+
async def asyncSetUp(self):
self.temp = tempfile.TemporaryDirectory()
self.root = Path(self.temp.name).resolve()
@@ -102,7 +134,7 @@ class RemoteMediaCacheTests(unittest.IsolatedAsyncioTestCase):
self.headers['content-type'] = 'video/mp4'
response = await self.client.get('/media')
self.assertEqual(response.content, self.payload)
- self.assertIn('max-age=31536000', response.headers['cache-control'])
+ self.assertEqual(response.headers['cache-control'], 'private, no-cache')
await self.client.get('/media')
self.assertEqual(self.calls, 2)
diff --git a/tests/test_stream_lifecycle.py b/tests/test_stream_lifecycle.py
new file mode 100644
index 000000000..ac1b8f963
--- /dev/null
+++ b/tests/test_stream_lifecycle.py
@@ -0,0 +1,593 @@
+"""分片生成与文件响应的取消边界,以及头像确认的线程归属。"""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import io
+import json
+import sqlite3
+import sys
+import tempfile
+import threading
+import unittest
+from contextlib import contextmanager
+from pathlib import Path
+from types import SimpleNamespace
+from unittest import mock
+
+from PIL import Image
+from starlette.responses import FileResponse
+
+from peach import avatar_provider, routes_media
+from peach.api import create_app
+from peach.config import PeachSettings
+from peach.segments import HlsSegmentService, SegmentUnavailable
+from peach.streaming import (
+ BufferedFileResponse, CancellableFileResponse, SplicedMp4Response, StreamSessionRegistry,
+)
+from support.ledger import fresh_ledger
+
+
+class SegmentGenerationLifecycleTests(unittest.IsolatedAsyncioTestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temporary.cleanup)
+ self.root = Path(self.temporary.name).resolve()
+ self.source = self.root / "source.mp4"
+ self.source.write_bytes(b"temporary source")
+ self.registry = StreamSessionRegistry()
+ self.service = HlsSegmentService(
+ SimpleNamespace(ffmpeg=lambda: SimpleNamespace(path=sys.executable)),
+ self.root / "segments", max_concurrent=1)
+
+ def generate(self, session):
+ return self.service.generate(self.source, 0, 6, asset_id=1, index=0,
+ session=session, registry=self.registry)
+
+ async def test_target_has_one_generator_through_failure_and_cancellation(self):
+ for cancelled in (False, True):
+ with self.subTest(cancelled=cancelled):
+ first_started, release_first = asyncio.Event(), asyncio.Event()
+ finish = asyncio.Event()
+ calls = active = peak = 0
+
+ async def generate(source, start, duration, target, *, first_started=first_started,
+ release_first=release_first, finish=finish, **kwargs):
+ nonlocal calls, active, peak
+ calls += 1
+ active += 1
+ peak = max(peak, active)
+ try:
+ if calls == 1:
+ first_started.set()
+ await release_first.wait()
+ raise SegmentUnavailable("controlled failure")
+ await finish.wait()
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_bytes(b"segment")
+ return target
+ finally:
+ active -= 1
+
+ tasks = []
+ with mock.patch.object(self.service, "_generate_uncached", generate):
+ try:
+ tasks.append(asyncio.create_task(self.generate("first")))
+ await asyncio.wait_for(first_started.wait(), 1)
+ tasks.append(asyncio.create_task(self.generate("waiting")))
+ await asyncio.sleep(0)
+ if cancelled:
+ tasks[0].cancel()
+ else:
+ release_first.set()
+ await asyncio.gather(tasks[0], return_exceptions=True)
+ tasks.append(asyncio.create_task(self.generate("new")))
+ tasks.append(asyncio.create_task(self.generate("last")))
+ for _ in range(5):
+ await asyncio.sleep(0)
+ self.assertEqual(peak, 1)
+ self.assertTrue(self.service._generation_locks)
+ finally:
+ release_first.set()
+ finish.set()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ self.service.cached_path(self.source, 1, 0).unlink(missing_ok=True)
+ self.assertEqual(self.service._generation_locks, {})
+
+ async def test_arrivals_share_the_target_while_its_cancelled_generator_is_reaped(self):
+ started, reaping, reaped, finish = (asyncio.Event() for _ in range(4))
+ calls = active = peak = 0
+
+ async def generate(source, start, duration, target, **kwargs):
+ nonlocal calls, active, peak
+ calls += 1
+ active += 1
+ peak = max(peak, active)
+ try:
+ if calls == 1:
+ started.set()
+ try:
+ await asyncio.Event().wait()
+ finally:
+ reaping.set()
+ while not reaped.is_set():
+ try:
+ await reaped.wait()
+ except asyncio.CancelledError:
+ pass
+ await finish.wait()
+ return target
+ finally:
+ active -= 1
+
+ tasks = []
+ with mock.patch.object(self.service, "_generate_uncached", generate):
+ try:
+ tasks.append(asyncio.create_task(self.generate("first")))
+ await asyncio.wait_for(started.wait(), 1)
+ tasks[0].cancel()
+ await asyncio.wait_for(reaping.wait(), 1)
+ tasks.append(asyncio.create_task(self.generate("second")))
+ await asyncio.sleep(0)
+ tasks[1].cancel()
+ for _ in range(8):
+ await asyncio.sleep(0)
+ tasks.append(asyncio.create_task(self.generate("third")))
+ for _ in range(8):
+ await asyncio.sleep(0)
+ self.assertEqual(peak, 1)
+ self.assertTrue(self.service._generation_locks)
+ finally:
+ reaped.set()
+ finish.set()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ self.assertEqual(self.service._generation_locks, {})
+
+ async def test_generation_deadline_reaps_the_child_and_releases_the_slot(self):
+ children = []
+ started = asyncio.Event()
+ deadline = asyncio.Timeout(None)
+ spawn = asyncio.create_subprocess_exec
+
+ async def owned_spawn(*args, **kwargs):
+ child = await spawn(*args, **kwargs)
+ children.append(child)
+ started.set()
+ return child
+
+ attempts = [("owned wait", lambda out: [
+ getattr(sys, '_base_executable', sys.executable), "-c", "import time; time.sleep(60)"])]
+ observed = None
+ generation = None
+ try:
+ with mock.patch.object(self.service, "_attempts", return_value=attempts), \
+ mock.patch("peach.segments.GENERATION_TIMEOUT_SECONDS", 0.2, create=True), \
+ mock.patch("peach.segments.asyncio.create_subprocess_exec", owned_spawn), \
+ mock.patch("peach.segments.asyncio.timeout", return_value=deadline) as timeout:
+ generation = asyncio.create_task(self.generate("deadline"))
+ await asyncio.wait_for(started.wait(), 10)
+ deadline.reschedule(asyncio.get_running_loop().time())
+ try:
+ await asyncio.wait_for(generation, 10)
+ except Exception as error:
+ observed = error
+ timeout.assert_called_once_with(0.2)
+ self.assertIsInstance(observed, SegmentUnavailable)
+ self.assertIn("timed out", str(observed))
+ self.assertTrue(children)
+ self.assertTrue(all(child.returncode is not None for child in children))
+ self.assertFalse(self.registry._processes)
+ self.assertEqual(self.service._gate()._value, 1)
+ self.assertEqual(self.service._generation_locks, {})
+ self.assertEqual(list((self.root / "segments").rglob("*.tmp.ts")), [])
+ finally:
+ if generation is not None:
+ generation.cancel()
+ await asyncio.gather(generation, return_exceptions=True)
+ for child in children:
+ if child.returncode is None:
+ child.kill()
+ await child.wait()
+
+
+class MediaResponseLifecycleTests(unittest.IsolatedAsyncioTestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temporary.cleanup)
+ self.root = Path(self.temporary.name).resolve()
+ self.app = create_app(PeachSettings(
+ configured=True, token="secret", db_path=fresh_ledger(self.root),
+ candidate_root=self.root / "generated", cover_root=self.root / "covers",
+ avatar_root=self.root / "avatars", transcode_root=self.root / "transcodes"))
+ self.addCleanup(self.app.state.hls_plan_executor.shutdown, wait=True, cancel_futures=True)
+ self.source = self.root / "source.mp4"
+ self.source.write_bytes(b"temporary source")
+ self.service = HlsSegmentService(
+ SimpleNamespace(ffmpeg=lambda: SimpleNamespace(path=sys.executable)),
+ self.root / "segments", max_concurrent=1)
+ self.app.state.hls_service = self.service
+ self.children = []
+ self.tasks = []
+ self.spawned = asyncio.Event()
+ self.original_spawn = asyncio.create_subprocess_exec
+ self.addAsyncCleanup(self.stop_owned_work)
+
+ async def stop_owned_work(self):
+ for task in self.tasks:
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(*self.tasks, return_exceptions=True)
+ for child in self.children:
+ if child.returncode is None:
+ child.kill()
+ await child.wait()
+ await asyncio.sleep(0)
+ self.assertEqual([task for task in asyncio.all_tasks()
+ if task is not asyncio.current_task() and not task.done()], [])
+
+ async def owned_spawn(self, *args, **kwargs):
+ child = await self.original_spawn(*args, **kwargs)
+ self.children.append(child)
+ self.spawned.set()
+ return child
+
+ def attempts(self, delay):
+ return [("owned output", lambda out: [getattr(sys, '_base_executable', sys.executable), "-c",
+ "import sys,time; from pathlib import Path; time.sleep(float(sys.argv[1])); "
+ "Path(sys.argv[2]).write_bytes(b'segment')", str(delay), str(out)])]
+
+ def request(self, path, query=b""):
+ messages, sent = asyncio.Queue(), []
+ messages.put_nowait({"type": "http.request", "body": b"", "more_body": False})
+
+ async def receive():
+ message = await messages.get()
+ if isinstance(message, BaseException):
+ raise message
+ return message
+
+ async def send(message):
+ sent.append(message)
+
+ scope = {"type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"},
+ "http_version": "1.1", "method": "GET", "scheme": "http", "root_path": "",
+ "path": path, "raw_path": path.encode(), "query_string": query,
+ "headers": [], "server": ("127.0.0.1", 0), "client": ("127.0.0.1", 0)}
+ task = asyncio.create_task(self.app(scope, receive, send))
+ self.tasks.append(task)
+ return task, messages, sent
+
+ async def finished(self, task):
+ try:
+ await asyncio.wait_for(asyncio.shield(task), 0.3)
+ return True
+ except TimeoutError:
+ return False
+
+ async def test_last_hls_disconnect_reaps_its_generator(self):
+ with mock.patch.object(routes_media, "_hls_plan", return_value=(
+ None, self.source, [(0, 6)], False)), \
+ mock.patch.object(self.service, "_attempts", return_value=self.attempts(60)), \
+ mock.patch("peach.segments.asyncio.create_subprocess_exec", self.owned_spawn):
+ task, messages, sent = self.request("/stream/hls/1/0.ts", b"session=one&t=secret")
+ await asyncio.wait_for(self.spawned.wait(), 2)
+ messages.put_nowait({"type": "http.disconnect"})
+ self.assertTrue(await self.finished(task), "断连请求仍占着生成槽")
+ self.assertIsNotNone(self.children[0].returncode)
+ self.assertFalse(self.app.state.stream_sessions._processes)
+ self.assertEqual(self.app.state.stream_sessions.active_count("one"), 0)
+ self.assertEqual(self.service._gate()._value, 1)
+ self.assertEqual(self.service._generation_locks, {})
+ self.assertNotIn(500, [one["status"] for one in sent if one["type"] == "http.response.start"])
+
+ async def test_one_hls_disconnect_preserves_the_other_clients_generator(self):
+ await self.shared_client_leaves(explicit=False)
+
+ async def test_one_session_cancel_preserves_the_other_clients_generator(self):
+ await self.shared_client_leaves(explicit=True)
+
+ async def shared_client_leaves(self, *, explicit):
+ joined = asyncio.Event()
+ generate = self.service.generate
+
+ async def observed(*args, **kwargs):
+ if kwargs["session"] == "two":
+ joined.set()
+ return await generate(*args, **kwargs)
+
+ with mock.patch.object(routes_media, "_hls_plan", return_value=(
+ None, self.source, [(0, 6)], False)), \
+ mock.patch.object(self.service, "_attempts", return_value=self.attempts(0.6)), \
+ mock.patch.object(self.service, "generate", observed), \
+ mock.patch("peach.segments.asyncio.create_subprocess_exec", self.owned_spawn):
+ first, messages, _ = self.request("/stream/hls/1/0.ts", b"session=one&t=secret")
+ await asyncio.wait_for(self.spawned.wait(), 2)
+ second, _, sent = self.request("/stream/hls/1/0.ts", b"session=two&t=secret")
+ await asyncio.wait_for(joined.wait(), 1)
+ if explicit:
+ self.app.state.stream_sessions.cancel("one")
+ else:
+ messages.put_nowait({"type": "http.disconnect"})
+ self.assertTrue(await self.finished(first), "离开的请求仍在等待共享生成")
+ self.assertIsNone(self.children[0].returncode, "另一个客户端仍在使用该生成者")
+ await asyncio.wait_for(second, 2)
+ self.assertEqual(len(self.children), 1)
+ self.assertEqual([one["status"] for one in sent if one["type"] == "http.response.start"], [200])
+ self.assertEqual(b"".join(one.get("body", b"") for one in sent), b"segment")
+ self.assertFalse(self.app.state.stream_sessions._processes)
+ self.assertEqual(self.service._generation_locks, {})
+
+ async def test_early_file_disconnect_completes_the_application_middleware_stack(self):
+ stopped = asyncio.Event()
+
+ @self.app.get("/owned-stream")
+ async def stream():
+ return BufferedFileResponse(self.source)
+
+ async def slow_headers(response, scope, receive, send):
+ try:
+ await asyncio.Event().wait()
+ finally:
+ stopped.set()
+
+ observed = None
+ with mock.patch.object(FileResponse, "__call__", slow_headers), \
+ mock.patch("peach.api._server_error_headers", return_value={"X-Peach-Error-ID": "test"}) as errors:
+ task, messages, sent = self.request("/owned-stream")
+ messages.put_nowait({"type": "http.disconnect"})
+ try:
+ await asyncio.wait_for(task, 1)
+ except RuntimeError as error:
+ observed = error
+ self.assertIsNone(observed)
+ errors.assert_not_called()
+ self.assertTrue(stopped.is_set())
+ self.assertEqual([one["status"] for one in sent if one["type"] == "http.response.start"], [204])
+
+ async def test_file_session_cancel_before_headers_completes_the_application_stack(self):
+ await self.session_cancel_before_headers(CancellableFileResponse(
+ self.source, session="owned", registry=self.app.state.stream_sessions))
+
+ async def test_file_session_cancel_after_headers_keeps_one_response_start(self):
+ await self.session_cancel_before_headers(CancellableFileResponse(
+ self.source, session="owned", registry=self.app.state.stream_sessions), after_headers=True)
+
+ async def test_spliced_session_cancel_before_headers_completes_the_application_stack(self):
+ await self.session_cancel_before_headers(SplicedMp4Response(
+ self.source, None, session="owned", registry=self.app.state.stream_sessions))
+
+ async def test_spliced_session_cancel_after_headers_keeps_one_response_start(self):
+ await self.session_cancel_before_headers(SplicedMp4Response(
+ self.source, None, session="owned", registry=self.app.state.stream_sessions), after_headers=True)
+
+ async def session_cancel_before_headers(self, response, *, after_headers=False):
+ entered, stopped = asyncio.Event(), asyncio.Event()
+
+ @self.app.get("/owned-session")
+ async def stream():
+ return response
+
+ async def slow_headers(response, scope, receive, send):
+ if after_headers:
+ await send({"type": "http.response.start", "status": 200,
+ "headers": [(b'content-type',b'video/mp4')]})
+ entered.set()
+ try:
+ await asyncio.Event().wait()
+ finally:
+ stopped.set()
+
+ with mock.patch.object(FileResponse, "__call__", slow_headers), \
+ mock.patch.object(SplicedMp4Response, "_respond", slow_headers), \
+ mock.patch("peach.api._server_error_headers", return_value={"X-Peach-Error-ID": "test"}) as errors:
+ task, _, sent = self.request("/owned-session")
+ await asyncio.wait_for(entered.wait(), 1)
+ self.assertEqual(self.app.state.stream_sessions.cancel("owned"), 1)
+ observed = None
+ try:
+ await asyncio.wait_for(task, 1)
+ except BaseException as error:
+ observed = error
+ self.assertIsNone(observed)
+ errors.assert_not_called()
+ self.assertTrue(stopped.is_set())
+ self.assertEqual(self.app.state.stream_sessions.active_count("owned"), 0)
+ self.assertEqual([one["status"] for one in sent if one["type"] == "http.response.start"],
+ [200] if after_headers else [204])
+
+ async def test_external_file_cancellation_propagates_without_a_replacement_response(self):
+ entered, stopped = asyncio.Event(), asyncio.Event()
+ response = CancellableFileResponse(
+ self.source, session="owned", registry=self.app.state.stream_sessions)
+ sent = []
+
+ async def slow_headers(response, scope, receive, send):
+ entered.set()
+ try:
+ await asyncio.Event().wait()
+ finally:
+ stopped.set()
+
+ async def receive():
+ await asyncio.Event().wait()
+
+ async def send(message):
+ sent.append(message)
+
+ with mock.patch.object(FileResponse, "__call__", slow_headers):
+ task = asyncio.create_task(response({"type": "http", "method": "GET"}, receive, send))
+ self.tasks.append(task)
+ await asyncio.wait_for(entered.wait(), 1)
+ task.cancel()
+ with self.assertRaises(asyncio.CancelledError):
+ await task
+ self.assertTrue(stopped.is_set())
+ self.assertEqual(sent, [])
+ self.assertEqual(self.app.state.stream_sessions.active_count("owned"), 0)
+
+ async def test_file_runtime_errors_reach_the_application_error_handler(self):
+ @self.app.get("/owned-stream")
+ async def stream():
+ return BufferedFileResponse(self.source)
+
+ async def broken_headers(response, scope, receive, send):
+ raise RuntimeError("owned response failure")
+
+ with mock.patch.object(FileResponse, "__call__", broken_headers), \
+ mock.patch("peach.api._server_error_headers", return_value={"X-Peach-Error-ID": "test"}) as errors:
+ task, _, sent = self.request("/owned-stream")
+ observed = None
+ try:
+ await asyncio.wait_for(task, 1)
+ except Exception as error:
+ observed = error
+ self.assertIsInstance(observed, RuntimeError)
+ self.assertEqual(str(observed), "owned response failure")
+ errors.assert_called_once()
+ self.assertEqual([one["status"] for one in sent if one["type"] == "http.response.start"], [500])
+
+ async def test_receive_runtime_errors_reach_the_application_error_handler(self):
+ @self.app.get("/owned-stream")
+ async def stream():
+ return BufferedFileResponse(self.source)
+
+ async def waiting_headers(response, scope, receive, send):
+ await asyncio.Event().wait()
+
+ with mock.patch.object(FileResponse, "__call__", waiting_headers), \
+ mock.patch("peach.api._server_error_headers", return_value={"X-Peach-Error-ID": "test"}) as errors:
+ task, messages, sent = self.request("/owned-stream")
+ messages.put_nowait(RuntimeError('owned receive failed'))
+ observed = None
+ try:
+ await asyncio.wait_for(task, 1)
+ except Exception as error:
+ observed = error
+ while isinstance(observed, BaseExceptionGroup) and len(observed.exceptions) == 1:
+ observed = observed.exceptions[0]
+ self.assertIsInstance(observed, RuntimeError)
+ self.assertIn('owned receive failed', str(observed))
+ errors.assert_called_once()
+ self.assertEqual([one["status"] for one in sent if one["type"] == "http.response.start"], [500])
+
+
+class AvatarPickThreadTests(unittest.IsolatedAsyncioTestCase):
+ async def test_concurrent_confirmations_keep_each_installed_file_in_one_image_transaction(self):
+ images = []
+ for color, image_format in (("red", "PNG"), ("blue", "JPEG")):
+ output = io.BytesIO()
+ Image.new("RGB", (8, 8), color).save(output, format=image_format)
+ images.append(output.getvalue())
+ entered, release, second_probe = (threading.Event() for _ in range(3))
+ first_digest = hashlib.sha256(images[0]).hexdigest()
+
+ def probe(destination):
+ digest = hashlib.sha256(destination.read_bytes()).hexdigest()
+ if digest == first_digest:
+ entered.set()
+ if not release.wait(3):
+ raise TimeoutError("controlled avatar transaction")
+ else:
+ second_probe.set()
+ return {"sha256": digest, "px": [8, 8], "face": None}
+
+ with tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp).resolve()
+ state = SimpleNamespace(candidate_root=root / "generated", avatar_root=root / "avatars",
+ cache_bust=mock.Mock())
+ requests = [SimpleNamespace(
+ headers={"content-type": mime}, query_params={"id": "1", "name": f"image-{index}"},
+ body=mock.AsyncMock(return_value=body),
+ app=SimpleNamespace(state=SimpleNamespace(web_contract=state)))
+ for index, (body, mime) in enumerate(zip(images, ("image/png", "image/jpeg")))]
+ tasks = []
+ with mock.patch.object(routes_media, "_picker_entity", return_value=1), \
+ mock.patch.object(avatar_provider, "FaceProbe", return_value=probe):
+ try:
+ tasks.append(asyncio.create_task(routes_media.avatar_pick(requests[0])))
+ self.assertTrue(await asyncio.to_thread(entered.wait, 1))
+ # 其它实体的共享安装入口可以独立完成,不等待这一张图的取景。
+ await asyncio.wait_for(asyncio.to_thread(
+ avatar_provider.install_entity_avatar, state.avatar_root, "performer", 2,
+ images[1], "image/jpeg", {}, probe_face=False), 1)
+ tasks.append(asyncio.create_task(routes_media.avatar_pick(requests[1])))
+ overlapped = await asyncio.to_thread(second_probe.wait, 0.3)
+ finally:
+ release.set()
+ responses = await asyncio.gather(*tasks, return_exceptions=True)
+ self.assertEqual([response.status_code for response in responses], [200, 200])
+ destination = state.avatar_root / "performer-1.img"
+ installed = destination.read_bytes()
+ digest = hashlib.sha256(installed).hexdigest()
+ provenance = json.loads(Path(f"{destination}.provenance.json").read_text(encoding="utf-8"))
+ face = json.loads(destination.with_suffix(".face.json").read_text(encoding="utf-8"))
+ self.assertEqual(installed, images[1])
+ self.assertEqual(provenance["sha256"], digest)
+ self.assertEqual(face["sha256"], digest)
+ self.assertEqual(Path(f"{destination}.ct").read_text(encoding="utf-8"), "image/jpeg")
+ self.assertFalse(overlapped)
+
+ async def test_confirmation_work_and_connection_share_one_worker_thread(self):
+ loop_thread = threading.get_ident()
+ threads, progress = [], threading.Event()
+ progressed = []
+
+ def record():
+ threads.append(threading.get_ident())
+
+ @contextmanager
+ def connection():
+ record()
+ database = sqlite3.connect(":memory:")
+ try:
+ yield database
+ finally:
+ record()
+ database.close()
+
+ def resolve(ref, database, *args, **kwargs):
+ record()
+ database.execute("SELECT 1").fetchone()
+ progressed.append(progress.wait(0.2))
+ return b"source", {"provider": "library"}
+
+ def crop(body, box):
+ record()
+ return b"cropped", {"crop": box}
+
+ def install(*args):
+ record()
+ self.assertEqual(args[-2], b"cropped")
+ self.assertTrue(args[-1]["user_selected"])
+ return {"sha256": "test"}
+
+ async def tick():
+ await asyncio.sleep(0.01)
+ progress.set()
+
+ state = SimpleNamespace(read_connection=connection, cache_bust=mock.Mock())
+ request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(
+ web_contract=state, http_transport=None)))
+ payload = {"id": 1, "ref": "gfriends:test.jpg", "version": "v1",
+ "crop": {"x0": 0, "y0": 0, "x1": 1, "y1": 1}}
+ timer = asyncio.create_task(tick())
+ with tempfile.TemporaryDirectory() as tmp, \
+ mock.patch.object(routes_media, "_picker_roots", return_value=(Path(tmp).resolve(), Path(tmp).resolve())), \
+ mock.patch.object(routes_media, "_picker_request", new=mock.AsyncMock(return_value=(payload, b""))), \
+ mock.patch.object(routes_media, "_picker_entity", side_effect=lambda *_: (record(), 1)[1]), \
+ mock.patch.object(routes_media, "_picker_artwork", return_value=None), \
+ mock.patch.object(routes_media, "_picker_online", return_value=None), \
+ mock.patch.object(routes_media, "_remember_library"), \
+ mock.patch.object(routes_media.avatar_picker, "accept_image"), \
+ mock.patch.object(routes_media.avatar_picker, "prepare_image", side_effect=lambda body:body), \
+ mock.patch.object(routes_media.avatar_picker, "resolve", side_effect=resolve) as resolved, \
+ mock.patch.object(routes_media.avatar_picker, "crop", side_effect=crop), \
+ mock.patch.object(routes_media.avatar_picker, "install", side_effect=install):
+ response = await routes_media.avatar_pick(request)
+ await asyncio.wait_for(timer, 1)
+ self.assertEqual(resolved.call_args.kwargs["version"], "v1")
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(progressed, [True], "图片处理占用了事件循环")
+ self.assertEqual(len(set(threads)), 1)
+ self.assertNotEqual(threads[0], loop_thread)
+ state.cache_bust.assert_called_once()