Peach: Local-first video metadata scraper and media manager.
fix(media): 回收共享播放任务并隔离活动媒体响应
longmeidaob21bf978b09c
diff --git a/docs/REUSE.md b/docs/REUSE.mdindex f0e496c10..13a4e2a6b 100644--- a/docs/REUSE.md+++ b/docs/REUSE.md@@ -908,6 +908,16 @@ CloudDrive2 配置生成复用现有 TOML 模板、mDNS 公布地址和应用服Python、npm 与 GitHub Actions 的版本由 `.github/dependabot.yml` 每周检查;固定前端文件由`package-lock.json` 和 `scripts/vendor_web_dependencies.mjs` 重建并核对来源、许可证与 SHA-256。+## 来源文字质量与生命周期修复++来源文字中的 Unicode 替换字符 `U+FFFD` 表示解码时已有信息丢失。字段提取、既有候选选择和最终写入共用这一判据;含坏值的字段不参与来源优先级结算,原始快照保留。正常字段与正常中日文、重音字母不受影响,不使用启发式转码猜回原文。++存量文字补修复用字段归属、来源身份核对、候选结算与 `UpgradeRepairs` 的备份、事务和回执。仅机器归属的损坏标量字段可按同番号正常来源恢复;人物规范名及人工判断保留复核。待重采的资产使用独立文字规则身份,先前完成的升级任务不吞掉新补修。++播放生命周期复用 [PR190](https://github.com/peach-mitao/peach/pull/190) 的 `e41222e312443b5d0fc7be87d2f7eca9c1d61055`:共享分片按消费者计数,最后消费者离开才取消生成,并等待子进程回收;文件响应保留外层取消与运行错误。头像确认的图像和数据库工作共用工作线程,安装按实体加锁。当前 ASGI 响应头中间件、SMB 输入和图像预处理保留 Peach 主线实现;未采用该 PR 的界面、依赖和测试调度改动。++匿名签名校验复用 [PR150](https://github.com/peach-mitao/peach/pull/150) 的唯一入口、时间戳表达式及字符串约束,适配已存在的动态变量解析接口。媒体响应复用 [PR131](https://github.com/peach-mitao/peach/pull/131) 的被动类型校验、CSP `sandbox` 和 `nosniff`;HLS 清单继续可读,旧活动文档缓存不命中。以上均使用现有标准库、Starlette 和 filelock,无新增依赖。+## 静态检查Windows 媒体子进程复用 Python 标准库 `subprocess.CREATE_NO_WINDOW`:入库探测、抽帧、拼图、预览、关注封面与媒体修复均显式设置,输出捕获、超时和取消按各入口原有契约处理。该标记由 Windows 原生支持,不引入依赖;见 [Python 文档](https://docs.python.org/3/library/subprocess.html#subprocess.CREATE_NO_WINDOW) 与 [Windows 创建标记](https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags)。diff --git a/docs/ROUTES.md b/docs/ROUTES.mdindex cdc533558..8fa032c11 100644--- a/docs/ROUTES.md+++ b/docs/ROUTES.md@@ -133,3 +133,7 @@`/api/*`、`/stream`、`/follow-stream`、封面、图片及其他媒体端点使用各自的服务契约。实体只读接口 `/api/entity`、`/api/photos` 支持 `kind` 加 `id`,也接受 `kind` 加 `name`。鉴权与本机写入权限由服务端检查。启用接口文档时,`/docs`、`/docs/oauth2-redirect` 和 `/openapi.json` 是服务文档入口;首次运行表单由未配置状态下的首页展示。++远程媒体代理拒绝 HTML、SVG、XML 和未声明类型的内容;普通媒体和 HLS 清单使用被动响应策略。旧磁盘缓存同样校验类型,缓存字节可复用,浏览器须回源复验。图像响应及其 304 回执携带 CSP `sandbox` 和 `nosniff`。++浏览器首次取得当前媒体策略的 HTML 页面时收到仅清理缓存的 `Clear-Site-Data`,策略 Cookie 避免反复清理,登录和偏好不清除。不支持该响应头的浏览器需手动清理旧站点缓存。diff --git a/scripts/test_runner.py b/scripts/test_runner.pyindex a4c1e5a61..fa839a9f1 100644--- a/scripts/test_runner.py+++ b/scripts/test_runner.py@@ -69,7 +69,7 @@ SCOPES: dict[str, tuple[str, ...]] = {"test_interaction.py", "test_media.py", "test_mp4repair.py","test_mp4recover.py", "test_web_media_repair.py","test_previews.py", "test_preview_transport.py",- "test_providers.py", "test_segments.py", "test_streaming.py",+ "test_providers.py", "test_segments.py", "test_streaming.py", "test_stream_lifecycle.py","test_subtitles.py", "test_transcodes.py", "test_hardware_transcode.py", "test_strm.py","test_guangya_sources.py", "test_media_sources.py", "test_smb*.py"),"sync": ("test_sync*.py", "test_platform.py", "test_mount.py", "test_tray*.py", "test_macos_standalone.py", "test_log_retention.py",diff --git a/src/peach/api.py b/src/peach/api.pyindex c96207af3..c01b8388f 100644--- a/src/peach/api.py+++ b/src/peach/api.py@@ -29,7 +29,7 @@ from contextlib import asynccontextmanagerfrom urllib.parse import quotefrom fastapi import FastAPI, Request-from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, RedirectResponse+from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, RedirectResponse, Responsefrom fastapi.staticfiles import StaticFilesfrom starlette.exceptions import HTTPException as StarletteHTTPExceptionfrom starlette.datastructures import MutableHeaders@@ -46,6 +46,7 @@ from .follow_covers import FollowCoverServicefrom .follow_faces import FollowFaceIndexfrom .follow_stream import FollowMediaResolverfrom .http import HttpxTransport+from .remote_media_cache import MEDIA_SECURITY_HEADERSfrom .health import inspect_databasefrom .media import (FilesystemBackend,@@ -124,6 +125,16 @@ class ResponseHeadersMiddleware:elif 'cache-control' not in headers:headers['Cache-Control'] = 'no-store'headers['X-Content-Type-Options'] = 'nosniff'+ kind = headers.get('content-type','').split(';',1)[0].strip().lower()+ if kind.startswith(('image/','video/','audio/')):+ headers.update(MEDIA_SECURITY_HEADERS)+ if (scope['method']=='GET' and message['status']==200 and kind=='text/html'+ and Request(scope).cookies.get('peach_media_policy')!='1'):+ headers['Clear-Site-Data']='"cache"'+ policy=Response()+ policy.set_cookie('peach_media_policy','1',max_age=365*86400,httponly=True,+ secure=scope['scheme']=='https',samesite='strict')+ headers.append('set-cookie',policy.headers['set-cookie'])headers['Referrer-Policy'] = 'same-origin'headers['X-Robots-Tag'] = routes_pages.ROBOTS_TAGawait send(message)diff --git a/src/peach/avatar_provider.py b/src/peach/avatar_provider.pyindex 4aaa0a93c..6f21ae1ac 100644--- a/src/peach/avatar_provider.py+++ b/src/peach/avatar_provider.py@@ -11,6 +11,7 @@ from dataclasses import asdict, dataclassfrom datetime import datetime, timezonefrom pathlib import Path+from filelock import FileLockfrom PIL import Image, ImageOps, UnidentifiedImageErrorfrom .avatar_face import FaceProbe, drop_sidecar, write_sidecar@@ -145,20 +146,22 @@ def install_entity_avatar(avatar_root: Path, kind: str, entity_id: int, body: by"""from .previews import entity_image_key- destination = atomic_write_bytes(- avatar_root / f"{entity_image_key(kind, entity_id)}.img", body)- Path(f"{destination}.ct").write_text(content_type, encoding="utf-8")- record = FaceProbe()(destination) if probe_face and face is None else face- if record:- write_sidecar(destination, record)- else:- drop_sidecar(destination)- Path(f"{destination}.provenance.json").write_text(- json.dumps({**provenance,- "imported_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),- "purpose": "local performer identity cache"},- ensure_ascii=False, indent=2),- encoding="utf-8")+ destination = avatar_root / f"{entity_image_key(kind, entity_id)}.img"+ destination.parent.mkdir(parents=True, exist_ok=True)+ with FileLock(destination.with_suffix('.install.lock'), timeout=30):+ atomic_write_bytes(destination, body)+ Path(f"{destination}.ct").write_text(content_type, encoding="utf-8")+ record = FaceProbe()(destination) if probe_face and face is None else face+ if record:+ write_sidecar(destination, record)+ else:+ drop_sidecar(destination)+ Path(f"{destination}.provenance.json").write_text(+ json.dumps({**provenance,+ "imported_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),+ "purpose": "local performer identity cache"},+ ensure_ascii=False, indent=2),+ encoding="utf-8")return destinationdiff --git a/src/peach/remote_media_cache.py b/src/peach/remote_media_cache.pyindex 01b7996b6..4eb3d595e 100644--- a/src/peach/remote_media_cache.py+++ b/src/peach/remote_media_cache.py@@ -16,6 +16,19 @@ from .fsutil import atomic_path, atomic_write_textAVATAR_REFRESH_SECONDS = 90 * 86400FAVICON_REFRESH_SECONDS = 365 * 86400MEDIA_HEADERS = ("content-type", "content-length", "content-range", "accept-ranges", "content-encoding")+MEDIA_SECURITY_HEADERS = {+ 'content-security-policy': "sandbox; default-src 'none'; frame-ancestors 'none'",+ 'x-content-type-options': 'nosniff',+}+++def passive_media_type(value: str) -> bool:+ """代理只返回被动媒体;活动文档和未知类型不能在应用域打开。"""+ kind = value.split(';',1)[0].strip().lower()+ return (not kind.endswith(('/svg+xml','+xml','/xml'))+ and (kind.startswith(('image/','video/','audio/'))+ or kind in ('application/octet-stream','binary/octet-stream',+ 'application/vnd.apple.mpegurl','application/x-mpegurl')))class RemoteMediaUnavailable(ValueError):@@ -44,7 +57,8 @@ class RemoteMediaCache:try:metadata = json.loads(slot.with_suffix(".json").read_text(encoding="utf-8"))path = slot.with_suffix(".bin")- if path.is_file() and path.stat().st_size == metadata["size"]:+ if (passive_media_type(metadata['headers'].get('content-type',''))+ and path.is_file() and path.stat().st_size == metadata["size"]):return path, metadataexcept (OSError, ValueError, KeyError, TypeError):continue@@ -55,7 +69,8 @@ class RemoteMediaCache:if not found:return Nonepath, metadata = found- response_headers = {**metadata["headers"], "cache-control": "private, max-age=31536000", **(headers or {})}+ response_headers = {**metadata["headers"], "cache-control": "private, no-cache",+ **(headers or {}), **MEDIA_SECURITY_HEADERS}if metadata["status"] == 200:response_headers.pop("content-length", None)return FileResponse(path, media_type=response_headers.pop("content-type", None), headers=response_headers)@@ -87,12 +102,13 @@ class RemoteMediaCache:close()return hitupstream = fetch()+ if not passive_media_type(upstream.headers.get('content-type','')):+ close()+ return Response('媒体响应类型不受支持',status_code=502,media_type='text/plain',+ headers={'cache-control':'no-store',**MEDIA_SECURITY_HEADERS})forwarded = {name: upstream.headers[name] for name in MEDIA_HEADERS if name in upstream.headers}- forwarded.update({"cache-control": "private, max-age=31536000", **(headers or {})})- kind = upstream.headers.get("content-type", "").lower()- accepted = (upstream.status_code in (200, 206) and- (kind.startswith(("image/", "video/", "audio/")) or- kind.split(";")[0] in ("application/octet-stream", "binary/octet-stream")))+ forwarded.update({'cache-control':'private, no-cache', **(headers or {}), **MEDIA_SECURITY_HEADERS})+ accepted = upstream.status_code in (200,206)content_range = re.fullmatch(r"bytes (\d+)-(\d+)/(\d+)", forwarded.get("content-range", ""))if upstream.status_code == 206 and not content_range:accepted = Falsediff --git a/src/peach/routes_media.py b/src/peach/routes_media.pyindex e01199f30..351932c54 100644--- a/src/peach/routes_media.py+++ b/src/peach/routes_media.py@@ -58,8 +58,10 @@ from .routes_auth import require_authfrom .scan import VIDEOfrom .segments import SegmentCancelled, SegmentUnavailable, build_hls_playlistfrom .streaming import (+ _cancel_and_wait, _wait_for_disconnect,BufferedFileResponse, CancellableFileResponse, SplicedMp4Response, run_until_disconnect,)+from .remote_media_cache import MEDIA_SECURITY_HEADERSfrom .transcodes import TranscodeCancelled, TranscodeNeedsSession, TranscodeUnavailablerouter = APIRouter()@@ -109,9 +111,11 @@ def _image_response(request: Request, path: Path, media_type: str | None = None)from .smb_response import SmbResponsereturn SmbResponse(path, media_type=media_type)response = FileResponse(path, media_type=media_type, stat_result=path.stat())+ response.headers.update(MEDIA_SECURITY_HEADERS)response.headers["Cache-Control"] = "private, no-cache"if StaticFiles().is_not_modified(response.headers, request.headers):return Response(status_code=304, headers={+ **MEDIA_SECURITY_HEADERS,"ETag": response.headers["etag"],"Cache-Control": response.headers["cache-control"],})@@ -465,11 +469,24 @@ async def _hls_segment(request: Request, id: int, index: int, session: str):if index < 0 or index >= len(plan):return JSONResponse({"error": "invalid segment"}, status_code=416)start, duration = plan[index]- path = await state.hls_service.generate(+ generation = asyncio.create_task(state.hls_service.generate(source, start, duration, asset_id=id, index=index,session=session, registry=state.stream_sessions,**({"transcode": True} if conversion else {}),- )+ ))+ disconnect = asyncio.create_task(_wait_for_disconnect(request.receive))+ try:+ done, _ = await asyncio.wait({generation, disconnect}, return_when=asyncio.FIRST_COMPLETED)+ if disconnect in done:+ disconnect.result()+ return Response(status_code=204)+ path = await generation+ finally:+ await _cancel_and_wait(generation, disconnect)+ except asyncio.CancelledError:+ if state.stream_sessions.is_cancelled(session):+ return Response(status_code=410, headers={'Cache-Control':'no-store'})+ raiseexcept SegmentCancelled:return Response(status_code=410, headers={"Cache-Control": "no-store"})except TranscodeCancelled:@@ -1747,35 +1764,40 @@ async def avatar_pick(request: Request, args: dict[str, str] = Depends(require_astate = request.app.state.web_contractproviders_root, avatar_root = _picker_roots(state)payload, body = await _picker_request(request)- entity_id = _picker_entity(state, payload)- if entity_id <= 0:- return JSONResponse({"error": "缺少实体 id"}, status_code=400)kind = _picker_kind(str(payload.get("kind") or "performer"))ref = str(payload.get("ref") or "")version = payload.get("version")box = _picker_crop(payload.get("crop"))- try:+ def confirm():+ entity_id = _picker_entity(state, payload)+ if entity_id <= 0:+ raise avatar_picker.PickerError('缺少实体 id')+ selected = bodyif ref:with state.read_connection() as connection:- body, origin = avatar_picker.resolve(+ selected, origin = avatar_picker.resolve(ref, connection, providers_root, entity_id,request.app.state.http_transport, _picker_artwork(request),version=version if isinstance(version, str) else None,online=_picker_online(request, connection, entity_id))- body = avatar_picker.prepare_image(body)- _remember_library(providers_root, body, origin, avatar_picker.accept_image(body))- elif body:- body = await asyncio.to_thread(avatar_picker.prepare_image, body)+ selected = avatar_picker.prepare_image(selected)+ _remember_library(providers_root, selected, origin, avatar_picker.accept_image(selected))+ elif selected:+ selected = avatar_picker.prepare_image(selected)origin = {"source": "avatar picker", "provider": "upload","external_id": str(payload.get("name") or "")}else:- return JSONResponse({"error": "没有可用的图片"}, status_code=400)+ raise avatar_picker.PickerError('没有可用的图片')if box not in (None, ""):- body, cropped = avatar_picker.crop(body, box)+ selected, cropped = avatar_picker.crop(selected, box)origin = {**origin, **cropped}origin = {**origin, "user_selected": True}result = avatar_picker.install(providers_root, avatar_root, kind,- entity_id, body, origin)+ entity_id, selected, origin)+ return entity_id, result++ try:+ entity_id, result = await asyncio.to_thread(confirm)except avatar_picker.PickerError as error:return JSONResponse({"error": str(error)}, status_code=400)# 页面按这份索引决定「出 `<img>` 还是首字母垫底」,换完不失效就看不到新图。diff --git a/src/peach/segments.py b/src/peach/segments.pyindex 3f4595612..182201dd2 100644--- a/src/peach/segments.py+++ b/src/peach/segments.py@@ -8,12 +8,13 @@ import osimport subprocessimport uuidfrom collections import OrderedDict+from dataclasses import dataclass, fieldfrom pathlib import Pathfrom .ffmpeg import FFmpegResolverfrom .hardware_transcode import HardwareTranscoderfrom .mp4index import keyframe_seconds, segment_plan-from .streaming import StreamSessionRegistry+from .streaming import StreamSessionRegistry, _cancel_and_wait, _wait_for_tasksfrom .smb_media import input_urlHLS_SEGMENT_SECONDS = 6@@ -22,6 +23,7 @@ DEFAULT_CACHE_BYTES = 2 << 30PLAN_CACHE_LIMIT = 256CODEC_CACHE_LIMIT = 256PROBE_TIMEOUT_SECONDS = 20+GENERATION_TIMEOUT_SECONDS = 120logger = logging.getLogger(__name__)@@ -65,6 +67,13 @@ def build_hls_playlist(plan: list[tuple[float, float]], segment_url) -> str:return "\n".join(lines)+@dataclass+class _Generation:+ lock: asyncio.Lock = field(default_factory=asyncio.Lock)+ users: int = 0+ task: asyncio.Task[Path] | None = None++class HlsSegmentService:"""按关键帧边界 remux 单个时间段,输出可缓存的 MPEG-TS。"""@@ -91,7 +100,7 @@ class HlsSegmentService:self._codecs: OrderedDict[tuple, str] = OrderedDict()# 同一分片常被并发请求:播放器预取、重连、多设备同看都会撞上同一个目标文件。# 同目标只放一个生成者,后到者等它完事后直接用缓存,不再各起一个 FFmpeg 重读网盘。- self._generation_locks: dict[Path, asyncio.Lock] = {}+ self._generation_locks: dict[Path, _Generation] = {}def _gate(self) -> asyncio.Semaphore:if self._semaphore is None:@@ -187,20 +196,45 @@ class HlsSegmentService:if target.is_file() and target.stat().st_size:os.utime(target, None) # 命中即续期,淘汰按最后访问时间return target- lock = self._generation_locks.setdefault(target, asyncio.Lock())+ consumer = asyncio.current_task()+ if consumer is None or not registry.register(session, consumer):+ raise SegmentCancelled(session)+ entry = self._generation_locks.setdefault(target, _Generation())+ entry.users += 1try:- async with lock:- # 等到锁的可能是排在生成者后面的请求,先看缓存再决定要不要自己跑。- if target.is_file() and target.stat().st_size:- os.utime(target, None)- return target- return await self._generate_uncached(- source, start, duration, target,- session=session, registry=registry, transcode=transcode,- )+ if entry.task is not None and entry.task.cancelling():+ await _wait_for_tasks(entry.task)+ if entry.task is None or entry.task.done():+ entry.task = asyncio.create_task(self._generate_locked(+ entry.lock, source, start, duration, target, registry=registry, transcode=transcode))+ generation = entry.task+ await asyncio.wait({generation})+ return generation.result()finally:- if not lock.locked():- self._generation_locks.pop(target, None)+ registry.unregister(session, consumer)+ entry.users -= 1+ if entry.users == 0:+ try:+ if entry.task is not None:+ await _cancel_and_wait(entry.task)+ finally:+ if entry.users == 0 and self._generation_locks.get(target) is entry:+ self._generation_locks.pop(target, None)++ async def _generate_locked(self, lock: asyncio.Lock, source: Path, start: float,+ duration: float, target: Path, *, registry: StreamSessionRegistry,+ transcode: bool) -> Path:+ async with lock:+ if target.is_file() and target.stat().st_size:+ os.utime(target, None)+ return target+ try:+ async with asyncio.timeout(GENERATION_TIMEOUT_SECONDS):+ return await self._generate_uncached(+ source, start, duration, target, session=f'hls-{uuid.uuid4().hex}',+ registry=registry, transcode=transcode)+ except TimeoutError as error:+ raise SegmentUnavailable('segment generation timed out') from errorasync def _generate_uncached(self, source: Path, start: float, duration: float, target: Path, *,@@ -301,11 +335,12 @@ class HlsSegmentService:_kill_process(process)await process.communicate()raise SegmentCancelled(session)+ communication = asyncio.create_task(process.communicate())try:- _, stderr = await process.communicate()+ _, stderr = await asyncio.shield(communication)except asyncio.CancelledError:_kill_process(process)- await process.communicate()+ await _wait_for_tasks(communication)raisefinally:registry.unregister_process(session, process)diff --git a/src/peach/streaming.py b/src/peach/streaming.pyindex 62a6e7713..7c15ba941 100644--- a/src/peach/streaming.py+++ b/src/peach/streaming.py@@ -98,10 +98,30 @@ async def _wait_for_disconnect(receive: Receive) -> None:"""等到客户端断开。请求体读完后 uvicorn 的 receive() 只会在断开时再返回一次。"""while True:message = await receive()- if message["type"] == "http.disconnect":+ task = asyncio.current_task()+ if message["type"] == "http.disconnect" or (task is not None and task.cancelling()):return+async def _wait_for_tasks(*tasks) -> None:+ """清理期间保留外层取消,等子任务结束后再向上交还。"""+ finished = asyncio.gather(*tasks, return_exceptions=True)+ cancelled = False+ while not finished.done():+ try:+ await asyncio.shield(finished)+ except asyncio.CancelledError:+ cancelled = True+ if cancelled:+ raise asyncio.CancelledError+++async def _cancel_and_wait(*tasks) -> None:+ for task in tasks:+ task.cancel()+ await _wait_for_tasks(*tasks)++async def run_until_disconnect[T](operation: Awaitable[T], receive: Receive) -> T:"""等待请求的工作结果;客户端断开时取消工作,并等待其清理完成。"""work = asyncio.ensure_future(operation)@@ -113,17 +133,39 @@ async def run_until_disconnect[T](operation: Awaitable[T], receive: Receive) ->watch.result()raise ClientDisconnect()finally:- work.cancel()- watch.cancel()- await asyncio.gather(work, watch, return_exceptions=True)+ await _cancel_and_wait(work, watch)-async def _send_until_disconnect(inner, scope: Scope, receive: Receive, send: Send) -> None:+async def _send_until_disconnect(inner, scope: Scope, receive: Receive, send: Send, *, is_cancelled=None) -> None:"""跑 `inner` 发响应,客户端一断开就把它掐掉。"""+ started = False++ async def tracked_send(message):+ nonlocal started+ await send(message)+ if message['type'] == 'http.response.start':+ started = True++ respond = asyncio.ensure_future(inner(scope, receive, tracked_send))+ watch = asyncio.ensure_future(_wait_for_disconnect(receive))+ cancellation = Nonetry:- await run_until_disconnect(inner(scope, receive, send), receive)- except ClientDisconnect:- return+ await asyncio.wait({respond, watch}, return_when=asyncio.FIRST_COMPLETED)+ except asyncio.CancelledError as error:+ cancellation = error+ finally:+ try:+ await _cancel_and_wait(respond, watch)+ except asyncio.CancelledError as error:+ cancellation = cancellation or error+ if not respond.cancelled():+ respond.result()+ if not watch.cancelled():+ watch.result()+ if cancellation is not None and (is_cancelled is None or not is_cancelled()):+ raise cancellation+ if respond.cancelled() and not started:+ await Response(status_code=204)(scope, receive, send)class BufferedFileResponse(FileResponse):@@ -141,11 +183,14 @@ class BufferedFileResponse(FileResponse):chunk_size = 1 << 20+ def _is_cancelled(self) -> bool:+ return False+async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:if scope["type"] != "http":await super().__call__(scope, receive, send)return- await _send_until_disconnect(super().__call__, scope, receive, send)+ await _send_until_disconnect(super().__call__, scope, receive, send, is_cancelled=self._is_cancelled)class CancellableFileResponse(BufferedFileResponse):@@ -163,6 +208,9 @@ class CancellableFileResponse(BufferedFileResponse):self.session = sessionself.registry = registry+ def _is_cancelled(self) -> bool:+ return self.registry.is_cancelled(self.session)+async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:task = asyncio.current_task()if task is None or not self.registry.register(self.session, task):@@ -171,7 +219,8 @@ class CancellableFileResponse(BufferedFileResponse):try:await super().__call__(scope, receive, send)except asyncio.CancelledError:- logging.getLogger(__name__).info("stream session cancelled: %s", self.session)+ if self._is_cancelled():+ logging.getLogger(__name__).info("stream session cancelled: %s", self.session)raisefinally:self.registry.unregister(self.session, task)@@ -250,9 +299,11 @@ class SplicedMp4Response(Response):await Response(status_code=410, headers={"Cache-Control": "no-store"})(scope, receive, send)returntry:- await _send_until_disconnect(self._respond, scope, receive, send)+ await _send_until_disconnect(self._respond, scope, receive, send,+ is_cancelled=lambda: tracked and self.registry.is_cancelled(self.session))except asyncio.CancelledError:- logging.getLogger(__name__).info("stream session cancelled: %s", self.session)+ if tracked and self.registry.is_cancelled(self.session):+ logging.getLogger(__name__).info("stream session cancelled: %s", self.session)raisefinally:if tracked and task is not None:diff --git a/tests/test_fastapi_api.py b/tests/test_fastapi_api.pyindex 095af3480..2ba331b60 100644--- a/tests/test_fastapi_api.py+++ b/tests/test_fastapi_api.py@@ -26,6 +26,41 @@ from support.mp4 import minimal_mp4ROOT = Path(__file__).resolve().parents[1]+class MediaResponseSecurityTests(unittest.IsolatedAsyncioTestCase):+ async def test_media_policy_clears_only_old_cache_once_and_isolates_images(self):+ from starlette.applications import Starlette+ from starlette.responses import HTMLResponse, Response+ from starlette.routing import Route+ from peach.api import ResponseHeadersMiddleware+ from peach.routes_media import _image_response+ import httpx++ with tempfile.TemporaryDirectory() as directory:+ path=Path(directory)/'image.svg'+ path.write_text('<svg xmlns="http://www.w3.org/2000/svg"/>',encoding='utf-8')+ async def endpoint(request):+ if request.url.path=='/page':+ return HTMLResponse('<html>test</html>')+ if request.url.path=='/image':+ return _image_response(request,path,'image/svg+xml')+ return Response(status_code=204)+ app=ResponseHeadersMiddleware(Starlette(routes=[Route('/{path}',endpoint)]))+ async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app),base_url='https://test') as client:+ client.cookies.set('peach_session','keep')+ first=await client.get('/page')+ self.assertEqual(first.headers['clear-site-data'],'"cache"')+ self.assertIn('httponly',first.headers['set-cookie'].lower())+ second=await client.get('/page')+ self.assertNotIn('clear-site-data',second.headers)+ self.assertEqual(client.cookies.get('peach_session'),'keep')+ image=await client.get('/image')+ cached=await client.get('/image',headers={'If-None-Match':image.headers['etag']})+ self.assertEqual(cached.status_code,304)+ for response in (image,cached):+ self.assertIn('sandbox',response.headers['content-security-policy'])+ self.assertEqual(response.headers['x-content-type-options'],'nosniff')++HAS_DEPS = all(importlib.util.find_spec(name) for name in ("fastapi", "httpx"))if HAS_DEPS:import httpxdiff --git a/tests/test_follow_media_cache.py b/tests/test_follow_media_cache.pyindex 63c0080ee..8371ea976 100644--- a/tests/test_follow_media_cache.py+++ b/tests/test_follow_media_cache.py@@ -16,6 +16,38 @@ from peach.remote_media_cache import AVATAR_REFRESH_SECONDS, FAVICON_REFRESH_SECclass RemoteMediaCacheTests(unittest.IsolatedAsyncioTestCase):+ async def test_active_documents_are_rejected_without_forwarding_or_caching_bytes(self):+ self.payload=b'<svg><script>active()</script></svg>'+ for kind in ('image/svg+xml','text/html','application/xhtml+xml','application/xml',''):+ with self.subTest(kind=kind):+ self.headers={'content-type':kind,'content-length':str(len(self.payload))}+ response=await self.client.get('/media')+ self.assertEqual(response.status_code,502)+ self.assertNotIn(b'<script>',response.content)+ self.assertEqual(response.headers['x-content-type-options'],'nosniff')+ self.assertIn('sandbox',response.headers['content-security-policy'])+ self.assertIsNone(self.cache.stored('follow:1:media:0'))++ async def test_old_active_document_cache_is_not_served_after_upgrade(self):+ slot=self.cache._slot('follow:1:media:0')+ slot.parent.mkdir(parents=True)+ body=b'<svg><script>active()</script></svg>'+ slot.with_suffix('.bin').write_bytes(body)+ slot.with_suffix('.json').write_text(json.dumps({'status':200,'size':len(body),+ 'headers':{'content-type':'image/svg+xml'}}),encoding='utf-8')+ response=await self.client.get('/media')+ self.assertEqual(response.content,self.payload)+ self.assertEqual(self.calls,1)+ self.assertEqual(response.headers['cache-control'],'private, no-cache')++ async def test_hls_manifests_remain_available_as_passive_media(self):+ self.payload=b'#EXTM3U\n'+ self.headers={'content-type':'application/vnd.apple.mpegurl','content-length':str(len(self.payload))}+ response=await self.client.get('/media')+ self.assertEqual(response.status_code,200)+ self.assertEqual(response.content,self.payload)+ self.assertIn('sandbox',response.headers['content-security-policy'])+async def asyncSetUp(self):self.temp = tempfile.TemporaryDirectory()self.root = Path(self.temp.name).resolve()@@ -102,7 +134,7 @@ class RemoteMediaCacheTests(unittest.IsolatedAsyncioTestCase):self.headers['content-type'] = 'video/mp4'response = await self.client.get('/media')self.assertEqual(response.content, self.payload)- self.assertIn('max-age=31536000', response.headers['cache-control'])+ self.assertEqual(response.headers['cache-control'], 'private, no-cache')await self.client.get('/media')self.assertEqual(self.calls, 2)diff --git a/tests/test_stream_lifecycle.py b/tests/test_stream_lifecycle.pynew file mode 100644index 000000000..ac1b8f963--- /dev/null+++ b/tests/test_stream_lifecycle.py@@ -0,0 +1,593 @@+"""分片生成与文件响应的取消边界,以及头像确认的线程归属。"""+from __future__ import annotations++import asyncio+import hashlib+import io+import json+import sqlite3+import sys+import tempfile+import threading+import unittest+from contextlib import contextmanager+from pathlib import Path+from types import SimpleNamespace+from unittest import mock++from PIL import Image+from starlette.responses import FileResponse++from peach import avatar_provider, routes_media+from peach.api import create_app+from peach.config import PeachSettings+from peach.segments import HlsSegmentService, SegmentUnavailable+from peach.streaming import (+ BufferedFileResponse, CancellableFileResponse, SplicedMp4Response, StreamSessionRegistry,+)+from support.ledger import fresh_ledger+++class SegmentGenerationLifecycleTests(unittest.IsolatedAsyncioTestCase):+ def setUp(self):+ self.temporary = tempfile.TemporaryDirectory()+ self.addCleanup(self.temporary.cleanup)+ self.root = Path(self.temporary.name).resolve()+ self.source = self.root / "source.mp4"+ self.source.write_bytes(b"temporary source")+ self.registry = StreamSessionRegistry()+ self.service = HlsSegmentService(+ SimpleNamespace(ffmpeg=lambda: SimpleNamespace(path=sys.executable)),+ self.root / "segments", max_concurrent=1)++ def generate(self, session):+ return self.service.generate(self.source, 0, 6, asset_id=1, index=0,+ session=session, registry=self.registry)++ async def test_target_has_one_generator_through_failure_and_cancellation(self):+ for cancelled in (False, True):+ with self.subTest(cancelled=cancelled):+ first_started, release_first = asyncio.Event(), asyncio.Event()+ finish = asyncio.Event()+ calls = active = peak = 0++ async def generate(source, start, duration, target, *, first_started=first_started,+ release_first=release_first, finish=finish, **kwargs):+ nonlocal calls, active, peak+ calls += 1+ active += 1+ peak = max(peak, active)+ try:+ if calls == 1:+ first_started.set()+ await release_first.wait()+ raise SegmentUnavailable("controlled failure")+ await finish.wait()+ target.parent.mkdir(parents=True, exist_ok=True)+ target.write_bytes(b"segment")+ return target+ finally:+ active -= 1++ tasks = []+ with mock.patch.object(self.service, "_generate_uncached", generate):+ try:+ tasks.append(asyncio.create_task(self.generate("first")))+ await asyncio.wait_for(first_started.wait(), 1)+ tasks.append(asyncio.create_task(self.generate("waiting")))+ await asyncio.sleep(0)+ if cancelled:+ tasks[0].cancel()+ else:+ release_first.set()+ await asyncio.gather(tasks[0], return_exceptions=True)+ tasks.append(asyncio.create_task(self.generate("new")))+ tasks.append(asyncio.create_task(self.generate("last")))+ for _ in range(5):+ await asyncio.sleep(0)+ self.assertEqual(peak, 1)+ self.assertTrue(self.service._generation_locks)+ finally:+ release_first.set()+ finish.set()+ await asyncio.gather(*tasks, return_exceptions=True)+ self.service.cached_path(self.source, 1, 0).unlink(missing_ok=True)+ self.assertEqual(self.service._generation_locks, {})++ async def test_arrivals_share_the_target_while_its_cancelled_generator_is_reaped(self):+ started, reaping, reaped, finish = (asyncio.Event() for _ in range(4))+ calls = active = peak = 0++ async def generate(source, start, duration, target, **kwargs):+ nonlocal calls, active, peak+ calls += 1+ active += 1+ peak = max(peak, active)+ try:+ if calls == 1:+ started.set()+ try:+ await asyncio.Event().wait()+ finally:+ reaping.set()+ while not reaped.is_set():+ try:+ await reaped.wait()+ except asyncio.CancelledError:+ pass+ await finish.wait()+ return target+ finally:+ active -= 1++ tasks = []+ with mock.patch.object(self.service, "_generate_uncached", generate):+ try:+ tasks.append(asyncio.create_task(self.generate("first")))+ await asyncio.wait_for(started.wait(), 1)+ tasks[0].cancel()+ await asyncio.wait_for(reaping.wait(), 1)+ tasks.append(asyncio.create_task(self.generate("second")))+ await asyncio.sleep(0)+ tasks[1].cancel()+ for _ in range(8):+ await asyncio.sleep(0)+ tasks.append(asyncio.create_task(self.generate("third")))+ for _ in range(8):+ await asyncio.sleep(0)+ self.assertEqual(peak, 1)+ self.assertTrue(self.service._generation_locks)+ finally:+ reaped.set()+ finish.set()+ await asyncio.gather(*tasks, return_exceptions=True)+ self.assertEqual(self.service._generation_locks, {})++ async def test_generation_deadline_reaps_the_child_and_releases_the_slot(self):+ children = []+ started = asyncio.Event()+ deadline = asyncio.Timeout(None)+ spawn = asyncio.create_subprocess_exec++ async def owned_spawn(*args, **kwargs):+ child = await spawn(*args, **kwargs)+ children.append(child)+ started.set()+ return child++ attempts = [("owned wait", lambda out: [+ getattr(sys, '_base_executable', sys.executable), "-c", "import time; time.sleep(60)"])]+ observed = None+ generation = None+ try:+ with mock.patch.object(self.service, "_attempts", return_value=attempts), \+ mock.patch("peach.segments.GENERATION_TIMEOUT_SECONDS", 0.2, create=True), \+ mock.patch("peach.segments.asyncio.create_subprocess_exec", owned_spawn), \+ mock.patch("peach.segments.asyncio.timeout", return_value=deadline) as timeout:+ generation = asyncio.create_task(self.generate("deadline"))+ await asyncio.wait_for(started.wait(), 10)+ deadline.reschedule(asyncio.get_running_loop().time())+ try:+ await asyncio.wait_for(generation, 10)+ except Exception as error:+ observed = error+ timeout.assert_called_once_with(0.2)+ self.assertIsInstance(observed, SegmentUnavailable)+ self.assertIn("timed out", str(observed))+ self.assertTrue(children)+ self.assertTrue(all(child.returncode is not None for child in children))+ self.assertFalse(self.registry._processes)+ self.assertEqual(self.service._gate()._value, 1)+ self.assertEqual(self.service._generation_locks, {})+ self.assertEqual(list((self.root / "segments").rglob("*.tmp.ts")), [])+ finally:+ if generation is not None:+ generation.cancel()+ await asyncio.gather(generation, return_exceptions=True)+ for child in children:+ if child.returncode is None:+ child.kill()+ await child.wait()+++class MediaResponseLifecycleTests(unittest.IsolatedAsyncioTestCase):+ def setUp(self):+ self.temporary = tempfile.TemporaryDirectory()+ self.addCleanup(self.temporary.cleanup)+ self.root = Path(self.temporary.name).resolve()+ self.app = create_app(PeachSettings(+ configured=True, token="secret", db_path=fresh_ledger(self.root),+ candidate_root=self.root / "generated", cover_root=self.root / "covers",+ avatar_root=self.root / "avatars", transcode_root=self.root / "transcodes"))+ self.addCleanup(self.app.state.hls_plan_executor.shutdown, wait=True, cancel_futures=True)+ self.source = self.root / "source.mp4"+ self.source.write_bytes(b"temporary source")+ self.service = HlsSegmentService(+ SimpleNamespace(ffmpeg=lambda: SimpleNamespace(path=sys.executable)),+ self.root / "segments", max_concurrent=1)+ self.app.state.hls_service = self.service+ self.children = []+ self.tasks = []+ self.spawned = asyncio.Event()+ self.original_spawn = asyncio.create_subprocess_exec+ self.addAsyncCleanup(self.stop_owned_work)++ async def stop_owned_work(self):+ for task in self.tasks:+ if not task.done():+ task.cancel()+ await asyncio.gather(*self.tasks, return_exceptions=True)+ for child in self.children:+ if child.returncode is None:+ child.kill()+ await child.wait()+ await asyncio.sleep(0)+ self.assertEqual([task for task in asyncio.all_tasks()+ if task is not asyncio.current_task() and not task.done()], [])++ async def owned_spawn(self, *args, **kwargs):+ child = await self.original_spawn(*args, **kwargs)+ self.children.append(child)+ self.spawned.set()+ return child++ def attempts(self, delay):+ return [("owned output", lambda out: [getattr(sys, '_base_executable', sys.executable), "-c",+ "import sys,time; from pathlib import Path; time.sleep(float(sys.argv[1])); "+ "Path(sys.argv[2]).write_bytes(b'segment')", str(delay), str(out)])]++ def request(self, path, query=b""):+ messages, sent = asyncio.Queue(), []+ messages.put_nowait({"type": "http.request", "body": b"", "more_body": False})++ async def receive():+ message = await messages.get()+ if isinstance(message, BaseException):+ raise message+ return message++ async def send(message):+ sent.append(message)++ scope = {"type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"},+ "http_version": "1.1", "method": "GET", "scheme": "http", "root_path": "",+ "path": path, "raw_path": path.encode(), "query_string": query,+ "headers": [], "server": ("127.0.0.1", 0), "client": ("127.0.0.1", 0)}+ task = asyncio.create_task(self.app(scope, receive, send))+ self.tasks.append(task)+ return task, messages, sent++ async def finished(self, task):+ try:+ await asyncio.wait_for(asyncio.shield(task), 0.3)+ return True+ except TimeoutError:+ return False++ async def test_last_hls_disconnect_reaps_its_generator(self):+ with mock.patch.object(routes_media, "_hls_plan", return_value=(+ None, self.source, [(0, 6)], False)), \+ mock.patch.object(self.service, "_attempts", return_value=self.attempts(60)), \+ mock.patch("peach.segments.asyncio.create_subprocess_exec", self.owned_spawn):+ task, messages, sent = self.request("/stream/hls/1/0.ts", b"session=one&t=secret")+ await asyncio.wait_for(self.spawned.wait(), 2)+ messages.put_nowait({"type": "http.disconnect"})+ self.assertTrue(await self.finished(task), "断连请求仍占着生成槽")+ self.assertIsNotNone(self.children[0].returncode)+ self.assertFalse(self.app.state.stream_sessions._processes)+ self.assertEqual(self.app.state.stream_sessions.active_count("one"), 0)+ self.assertEqual(self.service._gate()._value, 1)+ self.assertEqual(self.service._generation_locks, {})+ self.assertNotIn(500, [one["status"] for one in sent if one["type"] == "http.response.start"])++ async def test_one_hls_disconnect_preserves_the_other_clients_generator(self):+ await self.shared_client_leaves(explicit=False)++ async def test_one_session_cancel_preserves_the_other_clients_generator(self):+ await self.shared_client_leaves(explicit=True)++ async def shared_client_leaves(self, *, explicit):+ joined = asyncio.Event()+ generate = self.service.generate++ async def observed(*args, **kwargs):+ if kwargs["session"] == "two":+ joined.set()+ return await generate(*args, **kwargs)++ with mock.patch.object(routes_media, "_hls_plan", return_value=(+ None, self.source, [(0, 6)], False)), \+ mock.patch.object(self.service, "_attempts", return_value=self.attempts(0.6)), \+ mock.patch.object(self.service, "generate", observed), \+ mock.patch("peach.segments.asyncio.create_subprocess_exec", self.owned_spawn):+ first, messages, _ = self.request("/stream/hls/1/0.ts", b"session=one&t=secret")+ await asyncio.wait_for(self.spawned.wait(), 2)+ second, _, sent = self.request("/stream/hls/1/0.ts", b"session=two&t=secret")+ await asyncio.wait_for(joined.wait(), 1)+ if explicit:+ self.app.state.stream_sessions.cancel("one")+ else:+ messages.put_nowait({"type": "http.disconnect"})+ self.assertTrue(await self.finished(first), "离开的请求仍在等待共享生成")+ self.assertIsNone(self.children[0].returncode, "另一个客户端仍在使用该生成者")+ await asyncio.wait_for(second, 2)+ self.assertEqual(len(self.children), 1)+ self.assertEqual([one["status"] for one in sent if one["type"] == "http.response.start"], [200])+ self.assertEqual(b"".join(one.get("body", b"") for one in sent), b"segment")+ self.assertFalse(self.app.state.stream_sessions._processes)+ self.assertEqual(self.service._generation_locks, {})++ async def test_early_file_disconnect_completes_the_application_middleware_stack(self):+ stopped = asyncio.Event()++ @self.app.get("/owned-stream")+ async def stream():+ return BufferedFileResponse(self.source)++ async def slow_headers(response, scope, receive, send):+ try:+ await asyncio.Event().wait()+ finally:+ stopped.set()++ observed = None+ with mock.patch.object(FileResponse, "__call__", slow_headers), \+ mock.patch("peach.api._server_error_headers", return_value={"X-Peach-Error-ID": "test"}) as errors:+ task, messages, sent = self.request("/owned-stream")+ messages.put_nowait({"type": "http.disconnect"})+ try:+ await asyncio.wait_for(task, 1)+ except RuntimeError as error:+ observed = error+ self.assertIsNone(observed)+ errors.assert_not_called()+ self.assertTrue(stopped.is_set())+ self.assertEqual([one["status"] for one in sent if one["type"] == "http.response.start"], [204])++ async def test_file_session_cancel_before_headers_completes_the_application_stack(self):+ await self.session_cancel_before_headers(CancellableFileResponse(+ self.source, session="owned", registry=self.app.state.stream_sessions))++ async def test_file_session_cancel_after_headers_keeps_one_response_start(self):+ await self.session_cancel_before_headers(CancellableFileResponse(+ self.source, session="owned", registry=self.app.state.stream_sessions), after_headers=True)++ async def test_spliced_session_cancel_before_headers_completes_the_application_stack(self):+ await self.session_cancel_before_headers(SplicedMp4Response(+ self.source, None, session="owned", registry=self.app.state.stream_sessions))++ async def test_spliced_session_cancel_after_headers_keeps_one_response_start(self):+ await self.session_cancel_before_headers(SplicedMp4Response(+ self.source, None, session="owned", registry=self.app.state.stream_sessions), after_headers=True)++ async def session_cancel_before_headers(self, response, *, after_headers=False):+ entered, stopped = asyncio.Event(), asyncio.Event()++ @self.app.get("/owned-session")+ async def stream():+ return response++ async def slow_headers(response, scope, receive, send):+ if after_headers:+ await send({"type": "http.response.start", "status": 200,+ "headers": [(b'content-type',b'video/mp4')]})+ entered.set()+ try:+ await asyncio.Event().wait()+ finally:+ stopped.set()++ with mock.patch.object(FileResponse, "__call__", slow_headers), \+ mock.patch.object(SplicedMp4Response, "_respond", slow_headers), \+ mock.patch("peach.api._server_error_headers", return_value={"X-Peach-Error-ID": "test"}) as errors:+ task, _, sent = self.request("/owned-session")+ await asyncio.wait_for(entered.wait(), 1)+ self.assertEqual(self.app.state.stream_sessions.cancel("owned"), 1)+ observed = None+ try:+ await asyncio.wait_for(task, 1)+ except BaseException as error:+ observed = error+ self.assertIsNone(observed)+ errors.assert_not_called()+ self.assertTrue(stopped.is_set())+ self.assertEqual(self.app.state.stream_sessions.active_count("owned"), 0)+ self.assertEqual([one["status"] for one in sent if one["type"] == "http.response.start"],+ [200] if after_headers else [204])++ async def test_external_file_cancellation_propagates_without_a_replacement_response(self):+ entered, stopped = asyncio.Event(), asyncio.Event()+ response = CancellableFileResponse(+ self.source, session="owned", registry=self.app.state.stream_sessions)+ sent = []++ async def slow_headers(response, scope, receive, send):+ entered.set()+ try:+ await asyncio.Event().wait()+ finally:+ stopped.set()++ async def receive():+ await asyncio.Event().wait()++ async def send(message):+ sent.append(message)++ with mock.patch.object(FileResponse, "__call__", slow_headers):+ task = asyncio.create_task(response({"type": "http", "method": "GET"}, receive, send))+ self.tasks.append(task)+ await asyncio.wait_for(entered.wait(), 1)+ task.cancel()+ with self.assertRaises(asyncio.CancelledError):+ await task+ self.assertTrue(stopped.is_set())+ self.assertEqual(sent, [])+ self.assertEqual(self.app.state.stream_sessions.active_count("owned"), 0)++ async def test_file_runtime_errors_reach_the_application_error_handler(self):+ @self.app.get("/owned-stream")+ async def stream():+ return BufferedFileResponse(self.source)++ async def broken_headers(response, scope, receive, send):+ raise RuntimeError("owned response failure")++ with mock.patch.object(FileResponse, "__call__", broken_headers), \+ mock.patch("peach.api._server_error_headers", return_value={"X-Peach-Error-ID": "test"}) as errors:+ task, _, sent = self.request("/owned-stream")+ observed = None+ try:+ await asyncio.wait_for(task, 1)+ except Exception as error:+ observed = error+ self.assertIsInstance(observed, RuntimeError)+ self.assertEqual(str(observed), "owned response failure")+ errors.assert_called_once()+ self.assertEqual([one["status"] for one in sent if one["type"] == "http.response.start"], [500])++ async def test_receive_runtime_errors_reach_the_application_error_handler(self):+ @self.app.get("/owned-stream")+ async def stream():+ return BufferedFileResponse(self.source)++ async def waiting_headers(response, scope, receive, send):+ await asyncio.Event().wait()++ with mock.patch.object(FileResponse, "__call__", waiting_headers), \+ mock.patch("peach.api._server_error_headers", return_value={"X-Peach-Error-ID": "test"}) as errors:+ task, messages, sent = self.request("/owned-stream")+ messages.put_nowait(RuntimeError('owned receive failed'))+ observed = None+ try:+ await asyncio.wait_for(task, 1)+ except Exception as error:+ observed = error+ while isinstance(observed, BaseExceptionGroup) and len(observed.exceptions) == 1:+ observed = observed.exceptions[0]+ self.assertIsInstance(observed, RuntimeError)+ self.assertIn('owned receive failed', str(observed))+ errors.assert_called_once()+ self.assertEqual([one["status"] for one in sent if one["type"] == "http.response.start"], [500])+++class AvatarPickThreadTests(unittest.IsolatedAsyncioTestCase):+ async def test_concurrent_confirmations_keep_each_installed_file_in_one_image_transaction(self):+ images = []+ for color, image_format in (("red", "PNG"), ("blue", "JPEG")):+ output = io.BytesIO()+ Image.new("RGB", (8, 8), color).save(output, format=image_format)+ images.append(output.getvalue())+ entered, release, second_probe = (threading.Event() for _ in range(3))+ first_digest = hashlib.sha256(images[0]).hexdigest()++ def probe(destination):+ digest = hashlib.sha256(destination.read_bytes()).hexdigest()+ if digest == first_digest:+ entered.set()+ if not release.wait(3):+ raise TimeoutError("controlled avatar transaction")+ else:+ second_probe.set()+ return {"sha256": digest, "px": [8, 8], "face": None}++ with tempfile.TemporaryDirectory() as tmp:+ root = Path(tmp).resolve()+ state = SimpleNamespace(candidate_root=root / "generated", avatar_root=root / "avatars",+ cache_bust=mock.Mock())+ requests = [SimpleNamespace(+ headers={"content-type": mime}, query_params={"id": "1", "name": f"image-{index}"},+ body=mock.AsyncMock(return_value=body),+ app=SimpleNamespace(state=SimpleNamespace(web_contract=state)))+ for index, (body, mime) in enumerate(zip(images, ("image/png", "image/jpeg")))]+ tasks = []+ with mock.patch.object(routes_media, "_picker_entity", return_value=1), \+ mock.patch.object(avatar_provider, "FaceProbe", return_value=probe):+ try:+ tasks.append(asyncio.create_task(routes_media.avatar_pick(requests[0])))+ self.assertTrue(await asyncio.to_thread(entered.wait, 1))+ # 其它实体的共享安装入口可以独立完成,不等待这一张图的取景。+ await asyncio.wait_for(asyncio.to_thread(+ avatar_provider.install_entity_avatar, state.avatar_root, "performer", 2,+ images[1], "image/jpeg", {}, probe_face=False), 1)+ tasks.append(asyncio.create_task(routes_media.avatar_pick(requests[1])))+ overlapped = await asyncio.to_thread(second_probe.wait, 0.3)+ finally:+ release.set()+ responses = await asyncio.gather(*tasks, return_exceptions=True)+ self.assertEqual([response.status_code for response in responses], [200, 200])+ destination = state.avatar_root / "performer-1.img"+ installed = destination.read_bytes()+ digest = hashlib.sha256(installed).hexdigest()+ provenance = json.loads(Path(f"{destination}.provenance.json").read_text(encoding="utf-8"))+ face = json.loads(destination.with_suffix(".face.json").read_text(encoding="utf-8"))+ self.assertEqual(installed, images[1])+ self.assertEqual(provenance["sha256"], digest)+ self.assertEqual(face["sha256"], digest)+ self.assertEqual(Path(f"{destination}.ct").read_text(encoding="utf-8"), "image/jpeg")+ self.assertFalse(overlapped)++ async def test_confirmation_work_and_connection_share_one_worker_thread(self):+ loop_thread = threading.get_ident()+ threads, progress = [], threading.Event()+ progressed = []++ def record():+ threads.append(threading.get_ident())++ @contextmanager+ def connection():+ record()+ database = sqlite3.connect(":memory:")+ try:+ yield database+ finally:+ record()+ database.close()++ def resolve(ref, database, *args, **kwargs):+ record()+ database.execute("SELECT 1").fetchone()+ progressed.append(progress.wait(0.2))+ return b"source", {"provider": "library"}++ def crop(body, box):+ record()+ return b"cropped", {"crop": box}++ def install(*args):+ record()+ self.assertEqual(args[-2], b"cropped")+ self.assertTrue(args[-1]["user_selected"])+ return {"sha256": "test"}++ async def tick():+ await asyncio.sleep(0.01)+ progress.set()++ state = SimpleNamespace(read_connection=connection, cache_bust=mock.Mock())+ request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(+ web_contract=state, http_transport=None)))+ payload = {"id": 1, "ref": "gfriends:test.jpg", "version": "v1",+ "crop": {"x0": 0, "y0": 0, "x1": 1, "y1": 1}}+ timer = asyncio.create_task(tick())+ with tempfile.TemporaryDirectory() as tmp, \+ mock.patch.object(routes_media, "_picker_roots", return_value=(Path(tmp).resolve(), Path(tmp).resolve())), \+ mock.patch.object(routes_media, "_picker_request", new=mock.AsyncMock(return_value=(payload, b""))), \+ mock.patch.object(routes_media, "_picker_entity", side_effect=lambda *_: (record(), 1)[1]), \+ mock.patch.object(routes_media, "_picker_artwork", return_value=None), \+ mock.patch.object(routes_media, "_picker_online", return_value=None), \+ mock.patch.object(routes_media, "_remember_library"), \+ mock.patch.object(routes_media.avatar_picker, "accept_image"), \+ mock.patch.object(routes_media.avatar_picker, "prepare_image", side_effect=lambda body:body), \+ mock.patch.object(routes_media.avatar_picker, "resolve", side_effect=resolve) as resolved, \+ mock.patch.object(routes_media.avatar_picker, "crop", side_effect=crop), \+ mock.patch.object(routes_media.avatar_picker, "install", side_effect=install):+ response = await routes_media.avatar_pick(request)+ await asyncio.wait_for(timer, 1)+ self.assertEqual(resolved.call_args.kwargs["version"], "v1")+ self.assertEqual(response.status_code, 200)+ self.assertEqual(progressed, [True], "图片处理占用了事件循环")+ self.assertEqual(len(set(threads)), 1)+ self.assertNotEqual(threads[0], loop_thread)+ state.cache_bust.assert_called_once()