Peach: Local-first video metadata scraper and media manager.
fix(release): 通过 R2 中转发行构建产物
longmeidao1b7c34d91fea
diff --git a/.gitignore b/.gitignoreindex 23fb28e49..84227fd0d 100644--- a/.gitignore+++ b/.gitignore@@ -9,6 +9,7 @@ __pycache__/htmlcov/build/dist/+/release/*.egg-info/.env.env.*diff --git a/.gitlab-ci.yml b/.gitlab-ci.ymlindex 8b3519729..858f5efc2 100644--- a/.gitlab-ci.yml+++ b/.gitlab-ci.yml@@ -22,6 +22,8 @@ variables:npm_config_fetch_retry_mintimeout: "10000"npm_config_fetch_retry_maxtimeout: "60000"PEACH_DOWNLOAD_BACKEND: r2+ PEACH_R2_BUCKET: peach-releases+ PEACH_R2_PUBLIC_BASE: https://downloads.peach.videoverify-linux:stage: verify@@ -121,49 +123,74 @@ validate-release:windows-media:extends: .release-linuxstage: media+ variables:+ PEACH_SOURCE_CACHE_BASE: "${PEACH_R2_PUBLIC_BASE}/_sources"+ environment:+ name: release+ action: preparescript:- apt-get install -y build-essential gcc-mingw-w64-x86-64-posix g++-mingw-w64-x86-64-posix mingw-w64-x86-64-dev nasm pkg-config clang meson ninja-build curl xz-utils- bash scripts/build_ffmpeg_windows.sh build/ffmpeg-windows- bash scripts/build_untrunc_windows.sh build/untrunc-windows+ - python3 scripts/ci_r2_artifacts.py push windows-mediaartifacts:expire_in: 14 dayspaths:- - build/ffmpeg-windows/ffmpeg/- - build/ffmpeg-windows/ffmpeg-corresponding-source.zip- - build/untrunc-windows/untrunc/- - build/untrunc-windows/untrunc-corresponding-source.zip+ - build/ci-artifacts/windows-media.jsonbuild-windows:stage: buildtags: [peach-windows]rules: !reference [.release-linux, rules]needs: [windows-media]+ environment:+ name: release+ action: prepare+ before_script:+ - python scripts/fetch_release_tools.py --rclone-only+ - if ($LASTEXITCODE -ne 0) { throw 'R2 传输工具校验失败' }+ - python scripts/ci_r2_artifacts.py pull windows-media+ - if ($LASTEXITCODE -ne 0) { throw '媒体构建产物校验失败' }script:- '& ./scripts/ci_windows.ps1 build'- if ($LASTEXITCODE -ne 0) { throw 'Windows 构建失败' }+ - python scripts/ci_r2_artifacts.py push windows-release+ - if ($LASTEXITCODE -ne 0) { throw 'Windows 产物暂存失败' }artifacts:expire_in: 14 days- paths: [release/]+ paths: [build/ci-artifacts/windows-release.json]build-macos:stage: buildtags: [peach-macos]rules: !reference [.release-linux, rules]needs: [validate-release]+ environment:+ name: release+ action: prepare+ before_script:+ - python3 scripts/fetch_release_tools.py --rclone-onlyscript:- bash scripts/ci_macos.sh build arm64- bash scripts/ci_macos.sh build x86_64+ - python3 scripts/ci_r2_artifacts.py push macos-releaseartifacts:expire_in: 14 days- paths: [dist/macos.noindex/]+ paths: [build/ci-artifacts/macos-release.json]smoke-windows:stage: smoketags: [peach-windows]rules: !reference [.release-linux, rules]needs: [build-windows]- variables:- GIT_STRATEGY: empty+ environment:+ name: release+ action: verify+ before_script:+ - python scripts/fetch_release_tools.py --rclone-only+ - if ($LASTEXITCODE -ne 0) { throw 'R2 传输工具校验失败' }+ - python scripts/ci_r2_artifacts.py pull windows-release+ - if ($LASTEXITCODE -ne 0) { throw 'Windows 产物下载校验失败' }script:- '& ./release/ci_windows.ps1 smoke'- if ($LASTEXITCODE -ne 0) { throw 'Windows 独立制品验收失败' }@@ -180,5 +207,8 @@ publish-release:PEACH_R2_BUCKET: peach-releasesPEACH_R2_PUBLIC_BASE: https://downloads.peach.videoscript:+ - python3 scripts/ci_r2_artifacts.py pull windows-release+ - python3 scripts/ci_r2_artifacts.py pull macos-release- cp dist/macos.noindex/arm64/*.dmg dist/macos.noindex/x86_64/*.dmg release/- python3 scripts/publish_gitlab_release.py+ - python3 scripts/ci_r2_artifacts.py cleanupdiff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.mdindex 4bd4babee..c847fa2a1 100644--- a/docs/OPERATIONS.md+++ b/docs/OPERATIONS.md@@ -57,8 +57,14 @@ Windows 原生 Runner 在深目录中检出项目时,需为 Git 启用 `core.lLinux 发布任务使用官方 uv Python 3.14 容器,并在获取制品工具前核对解释器版本;源码归档使用 `tarfile` 的 `data` 过滤器解压。CI 的 `.npm/` 与 `.uv-cache/` 属于生成缓存,不计入源码工作区变化。+媒体工具源码可通过 `PEACH_SOURCE_CACHE_BASE` 指定 HTTPS 缓存入口,按 `<SHA-256>/<归档名>` 定位文件。标签流水线使用发布桶的 `_sources/` 前缀;维护者填充缓存前须核对锁文件的原始 SHA-256,缓存同样计入桶容量预算。下载缓存失败时尝试锁定的上游地址;任一来源的哈希不符立即拒绝,不能放宽清单。未配置缓存的对应源码包直接使用上游。+可选的 `r2` 后端使用独立 Standard 桶与 HTTPS 自定义域名。`PEACH_R2_ACCESS_KEY_ID`、`PEACH_R2_SECRET_ACCESS_KEY`、`PEACH_R2_ENDPOINT` 放入受保护且隐藏的 CI 变量,环境作用域为 `release`;`PEACH_R2_BUCKET` 和 `PEACH_R2_PUBLIC_BASE` 指定发布目标。S3 传输复用 rclone,版本文件禁止覆盖;每份文件必须从公开入口完整下载并核对大小及 SHA-256 后,才更新公开版本索引和 Release 链接。+GitLab 标签流水线通过 R2 暂存传递媒体工具、Windows 发行目录与 macOS 镜像,job artifacts 只保存 JSON 清单。中转需要上述 R2 变量;构建与验收任务使用 `release` 环境的 `prepare` 或 `verify` 动作,仅受保护标签能够取得凭据,普通分支回归不使用这些变量。构建期 rclone 按当前操作系统和 CPU 架构从官方固定版本归档安装并核对 SHA-256。++暂存对象位于 `_ci/<项目 ID>/<流水线 ID>/<提交 SHA>/<产物组>/<作业 ID>/`,纳入同一桶的容量预算。每次重试使用自己的作业 ID,构建时间等元数据变化不会覆盖其他尝试的文件。下载任务先核对清单的版本、提交和流水线身份,再校验每份文件的大小与 SHA-256;产物路径限定在对应构建输出目录,不接收软链接或路径穿越。macOS 仅上传两种架构的 DMG、`build-info.json` 与 `SHA256SUMS.txt`,不传递展开应用。所有构建与独立制品验收成功后,发布任务才写入正式版本目录和公开版本索引;GitLab Release 创建成功后,只回收当前流水线的暂存前缀。失败流水线保留暂存以便排障和重试,不覆盖已有公开版本。+Windows 轻量安装器缺少本地工具包时,先从 R2 下载当前版本的工具;连接失败或内容校验失败时,尝试 GitHub 的同版本文件。两个来源均须匹配安装器内置清单的大小与 SHA-256,不能混用其他版本。发布完成后,用 `scripts/smoke_split_update.py` 的 `--prepare-with-app --online` 在空缓存中运行实际制品,核对联网取得的工具能否重建完整程序。GitLab Release 的下载表指向当前后端,文档链接指向对应版本标签。R2 公网下载流量免费,存储与请求按账户共享额度计费。预算告警只通知,不会停止服务;R2 后端不提供零费用硬封顶。要求绝不产生按量费用的部署应使用不绑定按量计费的下载服务,不启用 R2 自动发布。diff --git a/docs/REUSE.md b/docs/REUSE.mdindex 9296099ca..f1b0b488d 100644--- a/docs/REUSE.md+++ b/docs/REUSE.md@@ -8,6 +8,8 @@ GitLab API、CI 认证和 Release 创建复用官方 [glab 1.124.0](https://gitl`fetch_release_tools.py` 从固定版本官方发行站读取 SHA-256 清单,仅提取指定二进制。Python 标准库负责制品清单、版本与提交判据,不另实现 GitLab 认证、S3 签名或多段上传协议。Windows 与 macOS 的 CI 入口复用既有测试、构建和独立制品验收脚本。下载后端及费用边界见[运维说明](OPERATIONS.md#gitlab-发布与下载后端)。+CI 大产物中转复用相同的 rclone 1.75.2 与官方 [R2 S3 适配](https://developers.cloudflare.com/r2/examples/rclone/),覆盖 Linux amd64、macOS arm64/x86_64 与 Windows amd64;MIT 许可,仅构建期使用。传输、校验和多段上传由 rclone 负责,Peach 只维护流水线身份、输出路径白名单、SHA-256 清单与桶容量预算。macOS 只传 DMG 和构建清单,展开应用保留在构建机器。GitLab job artifacts 只传 JSON 清单;Generic Package 的单文件配额不能保证前置代理接受同等大小的请求。官方 rclone 归档名称与校验清单按实际输入核验,大文件往返使用独立临时仓库验证,不将传输测试当作正式制品验收。无新增 Python 依赖。源码获取复用共享 User-Agent 与标准库 urllib,针对连接中断、超时及暂时性 HTTP 错误最多尝试三次,完成 SHA-256 校验后才登记缓存;对应源码包包含共享请求标识及其许可证,可脱离开发环境执行。+细粒度凭据缺少公开 CI 元数据的读取权限时,发版查询可在官方 API 上进行一次匿名 GET,使用标准库 HTTPS 与项目共用 User-Agent;不扩大凭据权限,也不携带本机认证头。私有资源仍须通过服务端权限检查。npm 安装沿用 Node.js 24 LTS 附带的官方 [`npm ci`](https://docs.npmjs.com/cli/v11/commands/npm-ci/),锁文件与完整性校验由 npm 维护。`npm_ci.py` 复用既有 `npm_audit.py` 的有限重试方式及 Python 3.14 标准库 `subprocess`(PSF 许可,无额外安装体积),只为明确的 `ECONNRESET`、`ETIMEDOUT`、`EAI_AGAIN` 下载错误重试,最多三次;锁文件、完整性与生命周期错误保留失败。Linux、Windows、macOS CI 和 Windows 打包入口共用此策略,不重试测试失败、不变更包版本。真实锁定安装与注入失败回归覆盖成功、重试耗尽和非网络错误,无新增应用依赖、生产入口或账本改动。diff --git a/scripts/ci_r2_artifacts.py b/scripts/ci_r2_artifacts.pynew file mode 100644index 000000000..f881bad42--- /dev/null+++ b/scripts/ci_r2_artifacts.py@@ -0,0 +1,229 @@+"""通过 R2 传递同一流水线的大产物,GitLab 只保存带 SHA-256 的清单。"""+from __future__ import annotations++import argparse+import hashlib+import json+import os+from pathlib import Path, PurePosixPath+import re+import shutil+import subprocess+import sys+import tempfile++if not __package__:+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))+ __package__ = "scripts"+from .release_r2 import rclone_environment, require_capacity++GROUPS = {+ "windows-media": ("build/ffmpeg-windows/ffmpeg", "build/ffmpeg-windows/ffmpeg-corresponding-source.zip",+ "build/untrunc-windows/untrunc", "build/untrunc-windows/untrunc-corresponding-source.zip"),+ "windows-release": ("release",),+ "macos-release": ("dist/macos.noindex/arm64", "dist/macos.noindex/x86_64"),+}+++def identity() -> dict:+ result = {key: os.environ.get("CI_" + variable, "") for key, variable in+ (("project", "PROJECT_ID"), ("pipeline", "PIPELINE_ID"), ("commit", "COMMIT_SHA"), ("tag", "COMMIT_TAG"))}+ if (not all(re.fullmatch(r"[1-9][0-9]*", result[key]) for key in ("project", "pipeline"))+ or not re.fullmatch(r"[0-9a-f]{40}", result["commit"])+ or not re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+", result["tag"])+ or os.environ.get("CI_COMMIT_REF_PROTECTED") != "true"):+ raise ValueError("R2 暂存仅允许受保护版本标签与明确的项目、流水线和提交")+ return result+++def prefix(build: dict, group: str | None = None, producer_job: str | None = None) -> str:+ value = f"_ci/{build['project']}/{build['pipeline']}/{build['commit']}"+ if group is not None:+ if group not in GROUPS or not isinstance(producer_job, str) or not re.fullmatch(r"[1-9][0-9]*", producer_job):+ raise ValueError("产物暂存需要明确的生成作业 ID")+ return value + "/" + group + "/" + producer_job+ return value+++def allowed_path(name: str, group: str) -> bool:+ parts = PurePosixPath(name).parts+ if (not parts or any(part in {".", ".."} or not re.fullmatch(r"[A-Za-z0-9_.-]+", part) for part in parts)+ or PurePosixPath(name).is_absolute() or str(PurePosixPath(name)) != name):+ return False+ if group == "macos-release":+ return (len(parts) == 4 and "/".join(parts[:3]) in GROUPS[group]+ and (parts[-1] in {"build-info.json", "SHA256SUMS.txt"} or parts[-1].endswith(".dmg")))+ if group == "windows-release":+ return len(parts) == 2 and parts[0] == "release" and (+ bool(re.fullmatch(r"Peach-[0-9]+\.[0-9]+\.[0-9]+-windows-x64[-A-Za-z0-9.]*\.(zip|exe)", parts[-1]))+ or parts[-1] in {"ffmpeg-corresponding-source.zip", "untrunc-corresponding-source.zip",+ "RELEASE_NOTES.md", "DOWNLOADS.md", "version.txt", "ci_windows.ps1",+ "smoke_desktop.py", "smoke_update_tasks.py", "smoke_split_update.py"})+ return any(name == base or name.startswith(base + "/") for base in GROUPS[group])+++def local_path(root: Path, name: str, group: str) -> Path:+ if not allowed_path(name, group):+ raise ValueError("产物路径不属于本任务输出目录")+ path = root / name+ if not path.resolve().is_relative_to(root) or any(parent.is_symlink() for parent in (path, *path.parents) if parent != root and parent.is_relative_to(root)):+ raise ValueError("产物路径包含软链接或越出工作目录")+ return path+++def checksum(path: Path) -> str:+ with path.open("rb") as stream:+ return hashlib.file_digest(stream, "sha256").hexdigest()+++def rclone_executable(root: Path) -> str:+ suffix = ".exe" if os.name == "nt" else ""+ path = root / "build/release-tools" / ("rclone" + suffix)+ if not path.is_file() or path.is_symlink():+ raise ValueError("CI 中转仅使用项目内已核对官方 SHA-256 的 rclone")+ return str(path)+++def inventory(root: Path, group: str) -> list[dict]:+ paths = []+ for base in GROUPS[group]:+ source = root / base+ if not source.exists():+ raise ValueError(f"缺少构建产物目录或文件:{base}")+ paths.extend(source.rglob("*") if source.is_dir() else [source])+ items = []+ for path in sorted(paths):+ name = path.relative_to(root).as_posix()+ if group == "macos-release" and not allowed_path(name, group):+ continue+ if path.is_symlink():+ raise ValueError("CI 产物不允许软链接")+ if not path.is_file():+ continue+ local_path(root, name, group)+ items.append({"path": name, "size": path.stat().st_size, "sha256": checksum(path)})+ if not items or len({item["path"].casefold() for item in items}) != len(items):+ raise ValueError("产物清单为空或包含跨平台同名文件")+ if group == "macos-release":+ for arch in ("arm64", "x86_64"):+ if sum(item["path"].startswith(f"dist/macos.noindex/{arch}/") for item in items) != 3:+ raise ValueError("每种 macOS 架构须包含 DMG、构建身份与 SHA-256 清单")+ folder = root / "dist/macos.noindex" / arch+ info = json.loads((folder / "build-info.json").read_text(encoding="utf-8"))+ build = identity()+ image = folder / f"Peach-{build['tag'][1:]}-macos-{arch}.dmg"+ if (info.get("commit") != build["commit"] or info.get("version") != build["tag"][1:]+ or info.get("architecture") != arch or info.get("dirty") is not False+ or not image.is_file()+ or (folder / "SHA256SUMS.txt").read_text(encoding="utf-8").strip() != f"{checksum(image)} {image.name}"):+ raise ValueError("macOS 构建身份或镜像 SHA-256 不符")+ return items+++def validate_manifest(manifest: dict, group: str, build: dict) -> list[dict]:+ if manifest.get("format") != 1 or manifest.get("identity") != build or manifest.get("group") != group:+ raise ValueError("产物清单不属于当前版本、提交或流水线")+ prefix(build, group, manifest.get("producer_job"))+ items = manifest.get("files")+ if not isinstance(items, list) or not items:+ raise ValueError("产物清单缺少文件")+ names = set()+ for item in items:+ name = item.get("path", "")+ if (not allowed_path(name, group) or name.casefold() in names+ or type(item.get("size")) is not int or item["size"] < 0+ or not re.fullmatch(r"[0-9a-f]{64}", item.get("sha256", ""))):+ raise ValueError("产物清单的路径、大小或 SHA-256 无效")+ names.add(name.casefold())+ return items+++def transfer(root: Path, group: str, mode: str) -> dict:+ root = root.resolve(strict=True)+ build = identity()+ environment = rclone_environment()+ executable = rclone_executable(root)+ manifest_path = root / "build/ci-artifacts" / (group + ".json")+ if mode == "push":+ producer_job = os.environ.get("CI_JOB_ID", "")+ else:+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))+ items = validate_manifest(manifest, group, build)+ producer_job = manifest["producer_job"]+ storage_prefix = prefix(build, group, producer_job)+ remote = f"peach:{environment['PEACH_R2_BUCKET']}/{storage_prefix}"+ with tempfile.TemporaryDirectory(prefix="peach-ci-r2-") as directory:+ temporary = Path(directory).resolve()+ configuration = temporary / "rclone.conf"+ configuration.touch()++ def command(*args, read=False):+ args = [executable, *args, "--config", str(configuration)]+ if read:+ return subprocess.check_output(args, env=environment, text=True, encoding="utf-8")+ subprocess.run(args, env=environment, check=True)++ if mode == "push":+ actual = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True, encoding="utf-8").strip()+ dirty = subprocess.check_output(["git", "status", "--porcelain"], cwd=root, text=True, encoding="utf-8").strip()+ if actual != build["commit"] or dirty:+ raise ValueError("暂存需要当前提交的干净构建目录")+ items = inventory(root, group)+ existing = json.loads(command("lsjson", f"peach:{environment['PEACH_R2_BUCKET']}", "--recursive", "--files-only", read=True))+ require_capacity(existing, [{"name": item["path"], "size": item["size"]} for item in items],+ storage_prefix, int(environment.get("PEACH_R2_MAX_BYTES", "8000000000")))+ listing = temporary / "files.txt"+ listing.write_text("".join(item["path"] + "\n" for item in items), encoding="utf-8")+ command("copy", str(root), remote, "--files-from-raw", str(listing), "--checksum", "--immutable")+ manifest = {"format": 1, "identity": build, "group": group, "producer_job": producer_job, "files": items}+ manifest_path.parent.mkdir(parents=True, exist_ok=True)+ manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")+ else:+ for item in items:+ local_path(root, item["path"], group)+ listing = temporary / "files.txt"+ listing.write_text("".join(item["path"] + "\n" for item in items), encoding="utf-8")+ command("copy", remote, str(temporary / "download"), "--files-from-raw", str(listing), "--checksum")+ for item in items:+ source = temporary / "download" / item["path"]+ if not source.is_file() or source.stat().st_size != item["size"] or checksum(source) != item["sha256"]:+ raise ValueError(f"R2 暂存产物的大小或 SHA-256 不符:{item['path']}")+ for item in items:+ target = local_path(root, item["path"], group)+ target.parent.mkdir(parents=True, exist_ok=True)+ shutil.copyfile(temporary / "download" / item["path"], target)+ return {"group": group, "mode": mode, "files": len(items), "bytes": sum(item["size"] for item in items), "commit": build["commit"]}+++def cleanup(root: Path) -> dict:+ """公开发布成功后只回收当前流水线的暂存前缀。"""+ build = identity()+ environment = rclone_environment()+ executable = rclone_executable(root.resolve(strict=True))+ remote = f"peach:{environment['PEACH_R2_BUCKET']}/{prefix(build)}"+ with tempfile.TemporaryDirectory(prefix="peach-ci-r2-cleanup-") as directory:+ configuration = Path(directory) / "rclone.conf"+ configuration.touch()+ index = json.loads(subprocess.check_output(+ [executable, "cat", f"peach:{environment['PEACH_R2_BUCKET']}/releases.json", "--config", str(configuration)],+ env=environment, text=True, encoding="utf-8"))+ if not any(item.get("tag_name") == build["tag"] and item.get("commit_sha") == build["commit"]+ for item in index):+ raise ValueError("公开版本索引尚未登记当前提交,保留暂存产物")+ subprocess.run([executable, "purge", remote, "--config", str(configuration)], env=environment, check=True)+ return {"cleaned_pipeline": build["pipeline"], "commit": build["commit"]}+++def main() -> None:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("mode", choices=("push", "pull", "cleanup"))+ parser.add_argument("group", nargs="?", choices=tuple(GROUPS))+ args = parser.parse_args()+ if args.mode != "cleanup" and not args.group:+ parser.error("上传和下载须指定产物组")+ result = cleanup(Path.cwd()) if args.mode == "cleanup" else transfer(Path.cwd(), args.group, args.mode)+ print(json.dumps(result, ensure_ascii=False))+++if __name__ == "__main__":+ main()diff --git a/scripts/fetch_ffmpeg_sources.py b/scripts/fetch_ffmpeg_sources.pyindex 96c75a291..7287f9ac9 100644--- a/scripts/fetch_ffmpeg_sources.py+++ b/scripts/fetch_ffmpeg_sources.py@@ -3,14 +3,23 @@ from __future__ import annotationsimport argparseimport hashlib+import http.clientimport json+import osfrom pathlib import Pathimport shutil+import sysimport tarfile+import time+import urllib.errorimport urllib.request+from urllib.parse import urlsplitimport zipfileSCRIPT_DIR = Path(__file__).resolve().parent+sys.path.insert(0, str(SCRIPT_DIR.parent / "src"))+from peach.user_agent import USER_AGENT+LOCK_PATH = SCRIPT_DIR / "ffmpeg-sources.json"@@ -19,6 +28,45 @@ def digest(path: Path) -> str:return hashlib.file_digest(stream, "sha256").hexdigest()+def download_source(source: dict, archive: Path) -> None:+ """限次重试源码传输,完整校验后才登记缓存。"""+ temporary = archive.with_suffix(archive.suffix + ".part")+ request = urllib.request.Request(source["url"], headers={"User-Agent": USER_AGENT})+ for attempt in range(3):+ try:+ print(f"下载固定源码:{source['name']},第 {attempt + 1}/3 次", flush=True)+ with urllib.request.urlopen(request, timeout=60) as response, temporary.open("wb") as stream:+ shutil.copyfileobj(response, stream)+ if digest(temporary) != source["sha256"]:+ raise ValueError(f"源码 SHA256 不匹配: {source['name']}")+ temporary.replace(archive)+ return+ except (TimeoutError, ConnectionError, http.client.IncompleteRead, urllib.error.URLError) as error:+ retryable = not isinstance(error, urllib.error.HTTPError) or error.code in {408, 429, 500, 502, 503, 504}+ if not retryable or attempt == 2:+ raise+ time.sleep(2 ** (attempt + 1))+ finally:+ temporary.unlink(missing_ok=True)+++def obtain_source(source: dict, archive: Path) -> None:+ """从可选源码缓存或固定上游获取同一份已锁定字节。"""+ base = os.environ.get("PEACH_SOURCE_CACHE_BASE", "").rstrip("/")+ if base:+ parsed = urlsplit(base)+ if (parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password+ or parsed.query or parsed.fragment):+ raise ValueError("源码缓存须为无凭据的 HTTPS 地址")+ cached = {**source, "url": f"{base}/{source['sha256']}/{source['archive']}"}+ try:+ download_source(cached, archive)+ return+ except (TimeoutError, ConnectionError, http.client.IncompleteRead, urllib.error.URLError):+ print(f"源码缓存未取得,尝试固定上游:{source['name']}", flush=True)+ download_source(source, archive)++def load_sources(path: Path = LOCK_PATH) -> dict:lock = json.loads(path.read_text(encoding="utf-8"))if {source["name"] for source in lock["sources"]} != {"ffmpeg", "x264", "nv-codec-headers", "dav1d"}:@@ -39,12 +87,7 @@ def prepare(output: Path, lock: dict) -> None:for source in lock["sources"]:archive = sources / source["archive"]if not archive.exists():- temporary = archive.with_suffix(archive.suffix + ".part")- with urllib.request.urlopen(source["url"], timeout=60) as response, temporary.open("wb") as stream:- shutil.copyfileobj(response, stream)- if digest(temporary) != source["sha256"]:- raise ValueError(f"源码 SHA256 不匹配: {source['name']}")- temporary.replace(archive)+ obtain_source(source, archive)if digest(archive) != source["sha256"]:raise ValueError(f"源码 SHA256 不匹配: {source['name']}")destination = work / source["name"]@@ -105,6 +148,9 @@ def finish(output: Path, lock: dict) -> dict:package.write(path, f"sources/{path.name}")for name in ("build_ffmpeg_windows.sh", "fetch_ffmpeg_sources.py", "ffmpeg-sources.json", "dav1d-mingw64.ini"):package.write(SCRIPT_DIR / name, f"scripts/{name}")+ for name in ("__init__.py", "user_agent.py"):+ package.write(SCRIPT_DIR.parent / "src/peach" / name, f"src/peach/{name}")+ package.write(SCRIPT_DIR.parent / "LICENSE", "licenses/Peach-LICENSE.txt")for license_file in sorted(bundle.glob("LICENSE*")):package.write(license_file, f"licenses/{license_file.name}")package.writestr("README.md", instructions)diff --git a/scripts/fetch_release_tools.py b/scripts/fetch_release_tools.pyindex 3ec571abc..d7b012493 100644--- a/scripts/fetch_release_tools.py+++ b/scripts/fetch_release_tools.py@@ -1,10 +1,12 @@"""从官方发行站下载固定版本 CLI,校验 SHA-256 后提取指定二进制。"""from __future__ import annotations+import argparseimport hashlibimport iofrom pathlib import Pathimport re+import platformimport sysimport tarfilefrom urllib.request import Request, urlopen@@ -25,9 +27,26 @@ def download(url: str) -> bytes:return response.read()-def install(destination: Path) -> None:+def tool_specs(*, rclone_only: bool = False, system: str | None = None,+ machine: str | None = None) -> tuple:+ system = system or platform.system()+ machine = (machine or platform.machine()).lower()+ arch = {"amd64": "amd64", "x86_64": "amd64", "arm64": "arm64", "aarch64": "arm64"}.get(machine)+ operating = {"Linux": "linux", "Darwin": "osx", "Windows": "windows"}.get(system)+ if not arch or not operating or (not rclone_only and system != "Linux"):+ raise ValueError("此平台仅支持通过 --rclone-only 安装 rclone,且架构须为 amd64 或 arm64")+ suffix = ".exe" if system == "Windows" else ""+ stem = f"rclone-v1.75.2-{operating}-{arch}"+ rclone = ("rclone" + suffix, stem + ".zip", TOOLS[1][2], "SHA256SUMS", stem + "/rclone" + suffix)+ glab = TOOLS[0]+ if arch == "arm64":+ glab = tuple(value.replace("amd64", "arm64") for value in glab)+ return (rclone,) if rclone_only else (glab, rclone)+++def install(destination: Path, *, rclone_only: bool = False) -> None:destination.mkdir(parents=True, exist_ok=True)- for name, archive, base, checksums, member in TOOLS:+ for name, archive, base, checksums, member in tool_specs(rclone_only=rclone_only):document = download(f"{base}/{checksums}").decode("utf-8")expected = re.search(rf"^([0-9a-fA-F]{{64}})\s+\*?{re.escape(archive)}$", document, re.MULTILINE)content = download(f"{base}/{archive}")@@ -49,4 +68,7 @@ def install(destination: Path) -> None:if __name__ == "__main__":- install(Path("build/release-tools").resolve())+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--rclone-only", action="store_true")+ args = parser.parse_args()+ install(Path("build/release-tools").resolve(), rclone_only=args.rclone_only)diff --git a/scripts/fetch_untrunc_sources.py b/scripts/fetch_untrunc_sources.pyindex 94b654a0a..8252bf6c4 100644--- a/scripts/fetch_untrunc_sources.py+++ b/scripts/fetch_untrunc_sources.py@@ -61,6 +61,9 @@ def finish(output: Path, lock: dict) -> dict:for name in ('build_untrunc_windows.sh', 'fetch_untrunc_sources.py','fetch_ffmpeg_sources.py', 'untrunc-sources.json'):package.write(SCRIPT_DIR / name, f'scripts/{name}')+ for name in ('__init__.py', 'user_agent.py'):+ package.write(SCRIPT_DIR.parent / 'src/peach' / name, f'src/peach/{name}')+ package.write(SCRIPT_DIR.parent / 'LICENSE', 'licenses/Peach-LICENSE.txt')package.writestr('README.md', instructions)package.write(output / 'buildconf.txt', 'buildconf.txt')package.write(output / 'build-environment.txt', 'build-environment.txt')diff --git a/tests/test_ffmpeg_build.py b/tests/test_ffmpeg_build.pyindex 85f161f36..20488bcf0 100644--- a/tests/test_ffmpeg_build.py+++ b/tests/test_ffmpeg_build.py@@ -4,10 +4,15 @@ from __future__ import annotationsimport importlib.utilimport ioimport json+import osfrom pathlib import Pathimport tarfileimport tempfile+import subprocess+import sysimport unittest+from unittest import mock+import urllib.errorimport zipfile@@ -19,6 +24,72 @@ SPEC.loader.exec_module(builder)class FFmpegSourceBuildTests(unittest.TestCase):+ def test_source_cache_falls_back_to_pinned_upstream_on_transport_failure(self):+ import hashlib+ data = b"pinned source bytes"+ source = {"name": "ffmpeg", "url": "https://upstream.example/source.tar.xz", "archive": "source.tar.xz",+ "sha256": hashlib.sha256(data).hexdigest()}+ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(os.environ, {"PEACH_SOURCE_CACHE_BASE": "https://cache.example/_sources"}), \+ mock.patch.object(builder.urllib.request, "urlopen", side_effect=[+ urllib.error.HTTPError("https://cache.example", 404, "missing", {}, None), io.BytesIO(data)]) as opened:+ archive = Path(temporary).resolve() / "source.tar.xz"+ builder.obtain_source(source, archive)+ self.assertEqual(archive.read_bytes(), data)+ self.assertEqual([call.args[0].full_url for call in opened.call_args_list],+ [f"https://cache.example/_sources/{source['sha256']}/source.tar.xz", source["url"]])++ def test_source_cache_hash_mismatch_is_fatal(self):+ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(os.environ, {"PEACH_SOURCE_CACHE_BASE": "https://cache.example/_sources"}), \+ mock.patch.object(builder.urllib.request, "urlopen", return_value=io.BytesIO(b"corrupt")) as opened:+ archive = Path(temporary).resolve() / "source.tar.xz"+ with self.assertRaisesRegex(ValueError, "SHA256"):+ builder.obtain_source({"name": "ffmpeg", "url": "https://upstream.example/source.tar.xz",+ "archive": "source.tar.xz", "sha256": "a" * 64}, archive)+ self.assertEqual(opened.call_count, 1)+ self.assertFalse(archive.exists())++ def test_download_retries_transport_failure_and_keeps_only_verified_archive(self):+ data = b"fixed upstream source"+ import hashlib+ source = {"name": "ffmpeg", "url": "https://upstream.example/source.tar.xz",+ "sha256": hashlib.sha256(data).hexdigest()}+ with tempfile.TemporaryDirectory() as temporary:+ archive = Path(temporary).resolve() / "source.tar.xz"+ with mock.patch.object(builder.urllib.request, "urlopen", side_effect=[TimeoutError(), io.BytesIO(data)]) as opened, \+ mock.patch.object(builder.time, "sleep") as sleep:+ builder.download_source(source, archive)+ self.assertEqual(archive.read_bytes(), data)+ self.assertEqual(opened.call_count, 2)+ sleep.assert_called_once_with(2)+ self.assertFalse(archive.with_suffix(".xz.part").exists())++ def test_download_integrity_or_permanent_http_error_is_not_retried(self):+ for failure in (None, urllib.error.HTTPError("https://upstream.example", 403, "Forbidden", {}, None)):+ with self.subTest(failure=failure), tempfile.TemporaryDirectory() as temporary:+ archive = Path(temporary).resolve() / "source.tar.xz"+ with mock.patch.object(builder.urllib.request, "urlopen", side_effect=failure,+ return_value=io.BytesIO(b"corrupt")) as opened, \+ mock.patch.object(builder.time, "sleep") as sleep:+ with self.assertRaises((ValueError, urllib.error.HTTPError)):+ builder.download_source({"name": "ffmpeg", "url": "https://upstream.example/source.tar.xz",+ "sha256": "a" * 64}, archive)+ self.assertEqual(opened.call_count, 1)+ sleep.assert_not_called()+ self.assertFalse(archive.exists())+ self.assertFalse(archive.with_suffix(".xz.part").exists())++ def test_download_stops_after_three_transport_attempts(self):+ with tempfile.TemporaryDirectory() as temporary:+ archive = Path(temporary).resolve() / "source.tar.xz"+ with mock.patch.object(builder.urllib.request, "urlopen", side_effect=TimeoutError()) as opened, \+ mock.patch.object(builder.time, "sleep") as sleep:+ with self.assertRaises(TimeoutError):+ builder.download_source({"name": "ffmpeg", "url": "https://upstream.example/source.tar.xz",+ "sha256": "a" * 64}, archive)+ self.assertEqual(opened.call_count, 3)+ self.assertEqual(sleep.call_args_list, [mock.call(2), mock.call(4)])+ self.assertFalse(archive.exists())+def test_source_lock_pins_all_linked_external_dependencies(self):lock = builder.load_sources()self.assertEqual(lock["version"], "9.0.2")@@ -85,6 +156,12 @@ class FFmpegSourceBuildTests(unittest.TestCase):self.assertTrue({"scripts/build_ffmpeg_windows.sh", "scripts/fetch_ffmpeg_sources.py","scripts/ffmpeg-sources.json", "README.md", "buildconf.txt"} <= names)self.assertFalse(any("manifest.json" in name for name in names))+ independent = output / "independent-source"+ package.extractall(independent)+ help_result = subprocess.run([sys.executable, "-S", str(independent / "scripts/fetch_ffmpeg_sources.py"), "--help"],+ cwd=independent, env={**os.environ, "PYTHONPATH": ""},+ capture_output=True, text=True, encoding="utf-8", check=True)+ self.assertIn("prepare", help_result.stdout)stored = json.loads((output / "ffmpeg/manifest.json").read_text(encoding="utf-8"))self.assertEqual(stored, manifest)diff --git a/tests/test_scripts.py b/tests/test_scripts.pyindex 503332b1a..934622994 100644--- a/tests/test_scripts.py+++ b/tests/test_scripts.py@@ -4695,6 +4695,167 @@ class ScriptingConventionTests(unittest.TestCase):self.assertFalse(scripting.host_under("", ("x.com",)))+class R2ArtifactTransferTests(unittest.TestCase):+ def tools(self, root):+ path = root / "build/release-tools" / ("rclone.exe" if os.name == "nt" else "rclone")+ path.parent.mkdir(parents=True)+ path.write_bytes(b"test double; subprocess calls are mocked")++ def environment(self):+ return {"CI_PROJECT_ID": "123", "CI_PIPELINE_ID": "456", "CI_COMMIT_SHA": "a" * 40,+ "CI_COMMIT_TAG": "v1.2.3", "CI_COMMIT_REF_PROTECTED": "true", "CI_JOB_ID": "789"}++ def rclone_environment(self):+ return {"PEACH_RCLONE": "rclone", "PEACH_R2_BUCKET": "peach-releases"}++ def test_platform_cli_archives_use_native_architecture_and_binary(self):+ module = load_script("fetch_release_tools")+ for system, machine, archive, member in (+ ("Windows", "AMD64", "windows-amd64", "rclone.exe"),+ ("Darwin", "arm64", "osx-arm64", "rclone"),+ ("Darwin", "x86_64", "osx-amd64", "rclone"),+ ("Linux", "x86_64", "linux-amd64", "rclone")):+ with self.subTest(system=system, machine=machine):+ tool, = module.tool_specs(rclone_only=True, system=system, machine=machine)+ self.assertEqual(tool[0], member)+ self.assertEqual(tool[1], f"rclone-v1.75.2-{archive}.zip")+ self.assertEqual(tool[4], f"rclone-v1.75.2-{archive}/{member}")++ def test_round_trip_restores_only_verified_files_from_current_pipeline(self):+ module = load_script("ci_r2_artifacts")+ objects = {}++ def run(command, **_kwargs):+ self.assertEqual(command[1], "copy")+ names = Path(command[command.index("--files-from-raw") + 1]).read_text().splitlines()+ source, destination = command[2:4]+ if destination.startswith("peach:"):+ for name in names:+ objects[destination + "/" + name] = (Path(source) / name).read_bytes()+ else:+ for name in names:+ path = Path(destination) / name+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_bytes(objects[source + "/" + name])++ def read(command, **_kwargs):+ if command[0] == "git":+ return "a" * 40 if "rev-parse" in command else ""+ return json.dumps([{"Path": key.split("peach:peach-releases/", 1)[1], "Size": len(value)}+ for key, value in objects.items()])++ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(os.environ, self.environment(), clear=True), \+ mock.patch.object(module, "rclone_environment", side_effect=self.rclone_environment), \+ mock.patch.object(module.subprocess, "check_output", side_effect=read), \+ mock.patch.object(module.subprocess, "run", side_effect=run):+ root = Path(temporary).resolve()+ self.tools(root)+ (root / "release").mkdir()+ artifact = root / "release/Peach-1.2.3-windows-x64.zip"+ artifact.write_bytes(b"verified executable archive")+ result = module.transfer(root, "windows-release", "push")+ self.assertEqual(result["files"], 1)+ manifest_path = root / "build/ci-artifacts/windows-release.json"+ self.assertLess(manifest_path.stat().st_size, 2000)+ artifact.unlink()+ module.transfer(root, "windows-release", "pull")+ self.assertEqual(artifact.read_bytes(), b"verified executable archive")+ key, = objects+ self.assertTrue(key.startswith("peach:peach-releases/_ci/123/456/" + "a" * 40 + "/"))+ objects[key] = b"corrupted"+ with self.assertRaisesRegex(ValueError, "SHA-256"):+ module.transfer(root, "windows-release", "pull")+ self.assertEqual(artifact.read_bytes(), b"verified executable archive")++ def test_manifest_rejects_other_pipeline_path_traversal_and_duplicates(self):+ module = load_script("ci_r2_artifacts")+ with mock.patch.dict(os.environ, self.environment(), clear=True):+ build = module.identity()+ item = {"path": "release/version.txt", "size": 5, "sha256": "b" * 64}+ manifest = {"format": 1, "identity": build, "group": "windows-release", "producer_job": "789", "files": [item]}+ self.assertEqual(module.validate_manifest(manifest, "windows-release", build), [item])+ with self.assertRaises(ValueError):+ module.validate_manifest(manifest, "windows-release", {**build, "pipeline": "789"})+ with self.assertRaises(ValueError):+ module.validate_manifest({**manifest, "producer_job": "../other"}, "windows-release", build)+ self.assertNotEqual(module.prefix(build, "windows-release", "789"),+ module.prefix(build, "windows-release", "790"))+ for name in ("release/../../secrets/token", "release/Peach.exe:token", "release\\version.txt",+ "/release/version.txt", "release/private-config.toml", "src/peach/__init__.py"):+ with self.subTest(name=name), self.assertRaises(ValueError):+ module.validate_manifest({**manifest, "files": [{**item, "path": name}]}, "windows-release", build)+ with self.assertRaises(ValueError):+ module.validate_manifest({**manifest, "files": [item, {**item, "path": "release/VERSION.txt"}]},+ "windows-release", build)++ def test_macos_inventory_keeps_signed_images_and_checks_build_identity(self):+ module = load_script("ci_r2_artifacts")+ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(os.environ, self.environment(), clear=True):+ root = Path(temporary).resolve()+ for arch in ("arm64", "x86_64"):+ folder = root / "dist/macos.noindex" / arch+ folder.mkdir(parents=True)+ image = folder / f"Peach-1.2.3-macos-{arch}.dmg"+ image.write_bytes(b"signed application image")+ (folder / "Peach.app").mkdir()+ (folder / "Peach.app/app").write_bytes(b"expanded application")+ (folder / "build-info.json").write_text(json.dumps({"version": "1.2.3", "commit": "a" * 40,+ "architecture": arch, "dirty": False}))+ (folder / "SHA256SUMS.txt").write_text(f"{module.checksum(image)} {image.name}\n")+ items = module.inventory(root, "macos-release")+ self.assertEqual(len(items), 6)+ self.assertFalse(any(".app/" in item["path"] for item in items))+ image.write_bytes(b"tampered application")+ with self.assertRaisesRegex(ValueError, "SHA-256"):+ module.inventory(root, "macos-release")++ def test_rclone_path_requires_installed_project_tool(self):+ module = load_script("ci_r2_artifacts")+ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(os.environ, {"PEACH_RCLONE": "arbitrary-cli"}):+ root = Path(temporary).resolve()+ with self.assertRaisesRegex(ValueError, "项目内"):+ module.rclone_executable(root)+ self.tools(root)+ self.assertTrue(module.rclone_executable(root).startswith(str(root)))++ def test_capacity_failure_prevents_upload_and_manifest_creation(self):+ module = load_script("ci_r2_artifacts")+ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(os.environ, self.environment(), clear=True), \+ mock.patch.object(module, "rclone_environment", side_effect=self.rclone_environment), \+ mock.patch.object(module.subprocess, "check_output", side_effect=["a" * 40, "", '[{"Path":"v1.0.0/a.zip","Size":8000000000}]']), \+ mock.patch.object(module.subprocess, "run") as upload:+ root = Path(temporary).resolve()+ self.tools(root)+ (root / "release").mkdir()+ (root / "release/version.txt").write_text("1.2.3")+ with self.assertRaisesRegex(ValueError, "停止上传"):+ module.transfer(root, "windows-release", "push")+ upload.assert_not_called()+ self.assertFalse((root / "build/ci-artifacts/windows-release.json").exists())++ def test_cleanup_requires_published_commit_and_only_purges_own_prefix(self):+ module = load_script("ci_r2_artifacts")+ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(os.environ, self.environment(), clear=True), \+ mock.patch.object(module, "rclone_environment", side_effect=self.rclone_environment), \+ mock.patch.object(module.subprocess, "check_output", return_value="[]") as read, \+ mock.patch.object(module.subprocess, "run") as purge:+ root = Path(temporary).resolve()+ self.tools(root)+ with self.assertRaisesRegex(ValueError, "保留暂存"):+ module.cleanup(root)+ purge.assert_not_called()+ read.return_value = json.dumps([{"tag_name": "v1.2.3", "commit_sha": "a" * 40}])+ module.cleanup(root)+ command = purge.call_args.args[0]+ self.assertEqual(command[1:3], ["purge", "peach:peach-releases/_ci/123/456/" + "a" * 40])++ def test_unprotected_or_incomplete_build_identity_is_rejected(self):+ module = load_script("ci_r2_artifacts")+ for patch in ({"CI_COMMIT_REF_PROTECTED": "false"}, {"CI_PIPELINE_ID": "../other"}, {"CI_COMMIT_TAG": "master"}):+ with mock.patch.dict(os.environ, {**self.environment(), **patch}, clear=True), self.assertRaises(ValueError):+ module.identity()++class R2ReleaseTests(unittest.TestCase):def test_publication_preserves_existing_index_and_verifies_published_metadata(self):module = load_script("release_r2")