Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

perf(web): 页面资产的 gzip 压缩体按文件版本只压一次

longmeidao96cf4f00505c
diff --git a/src/peach/routes_pages.py b/src/peach/routes_pages.py
index aa702716f..d7d29b5ad 100644
--- a/src/peach/routes_pages.py
+++ b/src/peach/routes_pages.py
@@ -15,10 +15,13 @@
"""
from __future__ import annotations
+import gzip
import hashlib
import json
import os
import sys
+import threading
+from collections import OrderedDict
from collections.abc import Mapping, Sequence
import re
@@ -170,21 +173,77 @@ def asset_response(request: Request, path: Path, media: str) -> Response:
ETag 取 mtime_ns 加字节数,不读文件内容:这几个文件都由 Git 检出或 `frontend/`
构建产生,改一次就换一次 mtime,不需要为了强校验去算全文哈希。
+
+ 文本类资产在浏览器接受 gzip 时发压缩体,压缩结果按文件版本记住,见 `_gzipped()`。
+ 压缩体与原文是两个表示,ETag 各不相同。
"""
stat = path.stat()
- etag = f'"peach-{stat.st_mtime_ns:x}-{stat.st_size:x}"'
+ compress = (media.startswith(_COMPRESSIBLE_MEDIA) and stat.st_size >= _GZIP_MINIMUM
+ and _accepts_gzip(request.headers.get("accept-encoding", "")))
+ etag = f'"peach-{stat.st_mtime_ns:x}-{stat.st_size:x}{"-gz" if compress else ""}"'
+ # charset 只对文本类型成立。图标是字节流,声明里挂一个字符集,在全站的
+ # `X-Content-Type-Options: nosniff` 之下就是让浏览器照一个自相矛盾的类型解码。
+ media_type = f"{media}; charset=utf-8" if media.startswith("text/") else media
if request.headers.get("if-none-match") == etag:
response: Response = Response(status_code=304)
+ elif compress:
+ body = _gzipped(path, stat)
+ response = Response(b"" if request.method == "HEAD" else body, media_type=media_type,
+ headers={"Content-Encoding": "gzip"})
+ response.headers["Content-Length"] = str(len(body))
else:
- # charset 只对文本类型成立。图标是字节流,声明里挂一个字符集,在全站的
- # `X-Content-Type-Options: nosniff` 之下就是让浏览器照一个自相矛盾的类型解码。
- media_type = f"{media}; charset=utf-8" if media.startswith("text/") else media
response = FileResponse(path, media_type=media_type)
+ if media.startswith(_COMPRESSIBLE_MEDIA):
+ response.headers["Vary"] = "Accept-Encoding"
response.headers["ETag"] = etag
response.headers["Cache-Control"] = "no-cache"
return response
+#: 值得压缩的页面资产类型;图标等位图本身已压缩。
+_COMPRESSIBLE_MEDIA = ("text/", "image/svg+xml")
+#: 小于它的文件压缩后省不下几个字节,与 `GZipMiddleware` 的默认门槛相同。
+_GZIP_MINIMUM = 500
+#: 路径 → (修改时间, 字节数, 压缩体)。主包 3.8 MB,每次冷请求现压要一百多毫秒;同一版本只压一次。
+#: 每个路径只留当前版本,换了版本就替换;路径数按最近使用限界,带哈希的分包每次构建都换名。
+_GZIPPED: OrderedDict[str, tuple[int, int, bytes]] = OrderedDict()
+_GZIPPED_PATHS = 64
+_GZIP_LOCK = threading.Lock()
+
+
+def _accepts_gzip(header: str) -> bool:
+ """`Accept-Encoding` 里有没有 gzip(或 `*`),且没有用 `q=0` 拒绝。"""
+ for item in header.split(","):
+ token, _, params = item.strip().partition(";")
+ if token.strip().lower() not in ("gzip", "*"):
+ continue
+ quality = params.strip().lower()
+ if quality.startswith("q="):
+ try:
+ return float(quality[2:]) > 0
+ except ValueError:
+ return False
+ return True
+ return False
+
+
+def _gzipped(path: Path, stat: os.stat_result) -> bytes:
+ """这一版文件的 gzip 压缩体。压缩在锁外做:并发的首个请求各压一次,结果相同。"""
+ key = str(path)
+ with _GZIP_LOCK:
+ hit = _GZIPPED.get(key)
+ if hit and hit[:2] == (stat.st_mtime_ns, stat.st_size):
+ _GZIPPED.move_to_end(key)
+ return hit[2]
+ body = gzip.compress(path.read_bytes(), compresslevel=9, mtime=0)
+ with _GZIP_LOCK:
+ _GZIPPED[key] = (stat.st_mtime_ns, stat.st_size, body)
+ _GZIPPED.move_to_end(key)
+ while len(_GZIPPED) > _GZIPPED_PATHS:
+ _GZIPPED.popitem(last=False)
+ return body
+
+
#: 拆分后的样式表分区。层叠顺序就是文件名顺序,所以每份都带两位数前缀;
#: 分区名字不接受分隔符。清单由 `tests/test_web_ui.py` 钉住。
CSS_PART_NAME = re.compile(r"\d{2}-[a-z0-9-]+\.css")
diff --git a/tests/test_fastapi_api.py b/tests/test_fastapi_api.py
index c26e49407..eb592aaad 100644
--- a/tests/test_fastapi_api.py
+++ b/tests/test_fastapi_api.py
@@ -581,6 +581,39 @@ class FastApiContractTests(unittest.IsolatedAsyncioTestCase):
denied = await self.client.get(f"/dist/{escape}?t=secret")
self.assertEqual(denied.status_code, 404, f"{escape} 不该被提供")
+ async def test_main_bundle_is_compressed_once_per_file_version(self):
+ import gzip
+ from peach import routes_pages
+ bundle = self.root / "dist" / "peach-app.js"
+ bundle.write_text("export const filler = '" + "peach " * 2000 + "';", encoding="utf-8")
+ path, params = "/dist/peach-app.js", {"t": "secret"}
+ packed, plain = {"Accept-Encoding": "gzip"}, {"Accept-Encoding": "identity"}
+ with patch.object(routes_pages.gzip, "compress", wraps=gzip.compress) as compress:
+ served = await self.client.get(path, params=params, headers=packed)
+ again = await self.client.get(path, params=params, headers=packed)
+ self.assertEqual(compress.call_count, 1)
+ head = await self.client.head(path, params=params, headers=packed)
+ original = await self.client.get(path, params=params, headers=plain)
+ for etag, headers in ((served.headers["etag"], packed), (original.headers["etag"], plain)):
+ cached = await self.client.get(path, params=params, headers={**headers, "If-None-Match": etag})
+ self.assertEqual(cached.status_code, 304)
+ bundle.write_text("export const filler = '" + "pêche " * 2000 + "';", encoding="utf-8")
+ changed = await self.client.get(path, params=params,
+ headers={**packed, "If-None-Match": served.headers["etag"]})
+ self.assertEqual(compress.call_count, 2)
+ self.assertEqual(changed.status_code, 200)
+ self.assertIn("pêche".encode(), changed.content)
+ for response in (served, again):
+ self.assertEqual(response.headers["content-encoding"], "gzip")
+ self.assertEqual(response.headers["vary"], "Accept-Encoding")
+ self.assertEqual(response.content, "export const filler = '{}';".format("peach " * 2000).encode())
+ self.assertLess(int(response.headers["content-length"]), 1000)
+ self.assertEqual((head.content, head.headers["content-length"], head.headers["etag"]),
+ (b"", served.headers["content-length"], served.headers["etag"]))
+ self.assertNotIn("content-encoding", original.headers)
+ self.assertEqual(original.content, served.content)
+ self.assertNotEqual(original.headers["etag"], served.headers["etag"])
+
async def test_page_bundle_is_public_with_only_its_literal_names(self):
for name, media in (("peach-pages.js", "text/javascript"), ("peach-pages.css", "text/css")):
path = f"/dist/{name}"