Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(release): 通过 R2 中转发行构建产物

longmeidao1b7c34d91fea
diff --git a/.gitignore b/.gitignore
index 23fb28e49..84227fd0d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,7 @@ __pycache__/
htmlcov/
build/
dist/
+/release/
*.egg-info/
.env
.env.*
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8b3519729..858f5efc2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,6 +22,8 @@ variables:
npm_config_fetch_retry_mintimeout: "10000"
npm_config_fetch_retry_maxtimeout: "60000"
PEACH_DOWNLOAD_BACKEND: r2
+ PEACH_R2_BUCKET: peach-releases
+ PEACH_R2_PUBLIC_BASE: https://downloads.peach.video
verify-linux:
stage: verify
@@ -121,49 +123,74 @@ validate-release:
windows-media:
extends: .release-linux
stage: media
+ variables:
+ PEACH_SOURCE_CACHE_BASE: "${PEACH_R2_PUBLIC_BASE}/_sources"
+ environment:
+ name: release
+ action: prepare
script:
- apt-get install -y build-essential gcc-mingw-w64-x86-64-posix g++-mingw-w64-x86-64-posix mingw-w64-x86-64-dev nasm pkg-config clang meson ninja-build curl xz-utils
- bash scripts/build_ffmpeg_windows.sh build/ffmpeg-windows
- bash scripts/build_untrunc_windows.sh build/untrunc-windows
+ - python3 scripts/ci_r2_artifacts.py push windows-media
artifacts:
expire_in: 14 days
paths:
- - build/ffmpeg-windows/ffmpeg/
- - build/ffmpeg-windows/ffmpeg-corresponding-source.zip
- - build/untrunc-windows/untrunc/
- - build/untrunc-windows/untrunc-corresponding-source.zip
+ - build/ci-artifacts/windows-media.json
build-windows:
stage: build
tags: [peach-windows]
rules: !reference [.release-linux, rules]
needs: [windows-media]
+ environment:
+ name: release
+ action: prepare
+ before_script:
+ - python scripts/fetch_release_tools.py --rclone-only
+ - if ($LASTEXITCODE -ne 0) { throw 'R2 传输工具校验失败' }
+ - python scripts/ci_r2_artifacts.py pull windows-media
+ - if ($LASTEXITCODE -ne 0) { throw '媒体构建产物校验失败' }
script:
- '& ./scripts/ci_windows.ps1 build'
- if ($LASTEXITCODE -ne 0) { throw 'Windows 构建失败' }
+ - python scripts/ci_r2_artifacts.py push windows-release
+ - if ($LASTEXITCODE -ne 0) { throw 'Windows 产物暂存失败' }
artifacts:
expire_in: 14 days
- paths: [release/]
+ paths: [build/ci-artifacts/windows-release.json]
build-macos:
stage: build
tags: [peach-macos]
rules: !reference [.release-linux, rules]
needs: [validate-release]
+ environment:
+ name: release
+ action: prepare
+ before_script:
+ - python3 scripts/fetch_release_tools.py --rclone-only
script:
- bash scripts/ci_macos.sh build arm64
- bash scripts/ci_macos.sh build x86_64
+ - python3 scripts/ci_r2_artifacts.py push macos-release
artifacts:
expire_in: 14 days
- paths: [dist/macos.noindex/]
+ paths: [build/ci-artifacts/macos-release.json]
smoke-windows:
stage: smoke
tags: [peach-windows]
rules: !reference [.release-linux, rules]
needs: [build-windows]
- variables:
- GIT_STRATEGY: empty
+ environment:
+ name: release
+ action: verify
+ before_script:
+ - python scripts/fetch_release_tools.py --rclone-only
+ - if ($LASTEXITCODE -ne 0) { throw 'R2 传输工具校验失败' }
+ - python scripts/ci_r2_artifacts.py pull windows-release
+ - if ($LASTEXITCODE -ne 0) { throw 'Windows 产物下载校验失败' }
script:
- '& ./release/ci_windows.ps1 smoke'
- if ($LASTEXITCODE -ne 0) { throw 'Windows 独立制品验收失败' }
@@ -180,5 +207,8 @@ publish-release:
PEACH_R2_BUCKET: peach-releases
PEACH_R2_PUBLIC_BASE: https://downloads.peach.video
script:
+ - python3 scripts/ci_r2_artifacts.py pull windows-release
+ - python3 scripts/ci_r2_artifacts.py pull macos-release
- cp dist/macos.noindex/arm64/*.dmg dist/macos.noindex/x86_64/*.dmg release/
- python3 scripts/publish_gitlab_release.py
+ - python3 scripts/ci_r2_artifacts.py cleanup
diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md
index 4bd4babee..c847fa2a1 100644
--- a/docs/OPERATIONS.md
+++ b/docs/OPERATIONS.md
@@ -57,8 +57,14 @@ Windows 原生 Runner 在深目录中检出项目时,需为 Git 启用 `core.l
Linux 发布任务使用官方 uv Python 3.14 容器,并在获取制品工具前核对解释器版本;源码归档使用 `tarfile` 的 `data` 过滤器解压。CI 的 `.npm/` 与 `.uv-cache/` 属于生成缓存,不计入源码工作区变化。
+媒体工具源码可通过 `PEACH_SOURCE_CACHE_BASE` 指定 HTTPS 缓存入口,按 `<SHA-256>/<归档名>` 定位文件。标签流水线使用发布桶的 `_sources/` 前缀;维护者填充缓存前须核对锁文件的原始 SHA-256,缓存同样计入桶容量预算。下载缓存失败时尝试锁定的上游地址;任一来源的哈希不符立即拒绝,不能放宽清单。未配置缓存的对应源码包直接使用上游。
+
可选的 `r2` 后端使用独立 Standard 桶与 HTTPS 自定义域名。`PEACH_R2_ACCESS_KEY_ID`、`PEACH_R2_SECRET_ACCESS_KEY`、`PEACH_R2_ENDPOINT` 放入受保护且隐藏的 CI 变量,环境作用域为 `release`;`PEACH_R2_BUCKET` 和 `PEACH_R2_PUBLIC_BASE` 指定发布目标。S3 传输复用 rclone,版本文件禁止覆盖;每份文件必须从公开入口完整下载并核对大小及 SHA-256 后,才更新公开版本索引和 Release 链接。
+GitLab 标签流水线通过 R2 暂存传递媒体工具、Windows 发行目录与 macOS 镜像,job artifacts 只保存 JSON 清单。中转需要上述 R2 变量;构建与验收任务使用 `release` 环境的 `prepare` 或 `verify` 动作,仅受保护标签能够取得凭据,普通分支回归不使用这些变量。构建期 rclone 按当前操作系统和 CPU 架构从官方固定版本归档安装并核对 SHA-256。
+
+暂存对象位于 `_ci/<项目 ID>/<流水线 ID>/<提交 SHA>/<产物组>/<作业 ID>/`,纳入同一桶的容量预算。每次重试使用自己的作业 ID,构建时间等元数据变化不会覆盖其他尝试的文件。下载任务先核对清单的版本、提交和流水线身份,再校验每份文件的大小与 SHA-256;产物路径限定在对应构建输出目录,不接收软链接或路径穿越。macOS 仅上传两种架构的 DMG、`build-info.json` 与 `SHA256SUMS.txt`,不传递展开应用。所有构建与独立制品验收成功后,发布任务才写入正式版本目录和公开版本索引;GitLab Release 创建成功后,只回收当前流水线的暂存前缀。失败流水线保留暂存以便排障和重试,不覆盖已有公开版本。
+
Windows 轻量安装器缺少本地工具包时,先从 R2 下载当前版本的工具;连接失败或内容校验失败时,尝试 GitHub 的同版本文件。两个来源均须匹配安装器内置清单的大小与 SHA-256,不能混用其他版本。发布完成后,用 `scripts/smoke_split_update.py` 的 `--prepare-with-app --online` 在空缓存中运行实际制品,核对联网取得的工具能否重建完整程序。GitLab Release 的下载表指向当前后端,文档链接指向对应版本标签。
R2 公网下载流量免费,存储与请求按账户共享额度计费。预算告警只通知,不会停止服务;R2 后端不提供零费用硬封顶。要求绝不产生按量费用的部署应使用不绑定按量计费的下载服务,不启用 R2 自动发布。
diff --git a/docs/REUSE.md b/docs/REUSE.md
index 9296099ca..f1b0b488d 100644
--- a/docs/REUSE.md
+++ b/docs/REUSE.md
@@ -8,6 +8,8 @@ GitLab API、CI 认证和 Release 创建复用官方 [glab 1.124.0](https://gitl
`fetch_release_tools.py` 从固定版本官方发行站读取 SHA-256 清单,仅提取指定二进制。Python 标准库负责制品清单、版本与提交判据,不另实现 GitLab 认证、S3 签名或多段上传协议。Windows 与 macOS 的 CI 入口复用既有测试、构建和独立制品验收脚本。下载后端及费用边界见[运维说明](OPERATIONS.md#gitlab-发布与下载后端)。
+CI 大产物中转复用相同的 rclone 1.75.2 与官方 [R2 S3 适配](https://developers.cloudflare.com/r2/examples/rclone/),覆盖 Linux amd64、macOS arm64/x86_64 与 Windows amd64;MIT 许可,仅构建期使用。传输、校验和多段上传由 rclone 负责,Peach 只维护流水线身份、输出路径白名单、SHA-256 清单与桶容量预算。macOS 只传 DMG 和构建清单,展开应用保留在构建机器。GitLab job artifacts 只传 JSON 清单;Generic Package 的单文件配额不能保证前置代理接受同等大小的请求。官方 rclone 归档名称与校验清单按实际输入核验,大文件往返使用独立临时仓库验证,不将传输测试当作正式制品验收。无新增 Python 依赖。源码获取复用共享 User-Agent 与标准库 urllib,针对连接中断、超时及暂时性 HTTP 错误最多尝试三次,完成 SHA-256 校验后才登记缓存;对应源码包包含共享请求标识及其许可证,可脱离开发环境执行。
+
细粒度凭据缺少公开 CI 元数据的读取权限时,发版查询可在官方 API 上进行一次匿名 GET,使用标准库 HTTPS 与项目共用 User-Agent;不扩大凭据权限,也不携带本机认证头。私有资源仍须通过服务端权限检查。
npm 安装沿用 Node.js 24 LTS 附带的官方 [`npm ci`](https://docs.npmjs.com/cli/v11/commands/npm-ci/),锁文件与完整性校验由 npm 维护。`npm_ci.py` 复用既有 `npm_audit.py` 的有限重试方式及 Python 3.14 标准库 `subprocess`(PSF 许可,无额外安装体积),只为明确的 `ECONNRESET`、`ETIMEDOUT`、`EAI_AGAIN` 下载错误重试,最多三次;锁文件、完整性与生命周期错误保留失败。Linux、Windows、macOS CI 和 Windows 打包入口共用此策略,不重试测试失败、不变更包版本。真实锁定安装与注入失败回归覆盖成功、重试耗尽和非网络错误,无新增应用依赖、生产入口或账本改动。
diff --git a/scripts/ci_r2_artifacts.py b/scripts/ci_r2_artifacts.py
new file mode 100644
index 000000000..f881bad42
--- /dev/null
+++ b/scripts/ci_r2_artifacts.py
@@ -0,0 +1,229 @@
+"""通过 R2 传递同一流水线的大产物,GitLab 只保存带 SHA-256 的清单。"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path, PurePosixPath
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+if not __package__:
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+ __package__ = "scripts"
+from .release_r2 import rclone_environment, require_capacity
+
+GROUPS = {
+ "windows-media": ("build/ffmpeg-windows/ffmpeg", "build/ffmpeg-windows/ffmpeg-corresponding-source.zip",
+ "build/untrunc-windows/untrunc", "build/untrunc-windows/untrunc-corresponding-source.zip"),
+ "windows-release": ("release",),
+ "macos-release": ("dist/macos.noindex/arm64", "dist/macos.noindex/x86_64"),
+}
+
+
+def identity() -> dict:
+ result = {key: os.environ.get("CI_" + variable, "") for key, variable in
+ (("project", "PROJECT_ID"), ("pipeline", "PIPELINE_ID"), ("commit", "COMMIT_SHA"), ("tag", "COMMIT_TAG"))}
+ if (not all(re.fullmatch(r"[1-9][0-9]*", result[key]) for key in ("project", "pipeline"))
+ or not re.fullmatch(r"[0-9a-f]{40}", result["commit"])
+ or not re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+", result["tag"])
+ or os.environ.get("CI_COMMIT_REF_PROTECTED") != "true"):
+ raise ValueError("R2 暂存仅允许受保护版本标签与明确的项目、流水线和提交")
+ return result
+
+
+def prefix(build: dict, group: str | None = None, producer_job: str | None = None) -> str:
+ value = f"_ci/{build['project']}/{build['pipeline']}/{build['commit']}"
+ if group is not None:
+ if group not in GROUPS or not isinstance(producer_job, str) or not re.fullmatch(r"[1-9][0-9]*", producer_job):
+ raise ValueError("产物暂存需要明确的生成作业 ID")
+ return value + "/" + group + "/" + producer_job
+ return value
+
+
+def allowed_path(name: str, group: str) -> bool:
+ parts = PurePosixPath(name).parts
+ if (not parts or any(part in {".", ".."} or not re.fullmatch(r"[A-Za-z0-9_.-]+", part) for part in parts)
+ or PurePosixPath(name).is_absolute() or str(PurePosixPath(name)) != name):
+ return False
+ if group == "macos-release":
+ return (len(parts) == 4 and "/".join(parts[:3]) in GROUPS[group]
+ and (parts[-1] in {"build-info.json", "SHA256SUMS.txt"} or parts[-1].endswith(".dmg")))
+ if group == "windows-release":
+ return len(parts) == 2 and parts[0] == "release" and (
+ bool(re.fullmatch(r"Peach-[0-9]+\.[0-9]+\.[0-9]+-windows-x64[-A-Za-z0-9.]*\.(zip|exe)", parts[-1]))
+ or parts[-1] in {"ffmpeg-corresponding-source.zip", "untrunc-corresponding-source.zip",
+ "RELEASE_NOTES.md", "DOWNLOADS.md", "version.txt", "ci_windows.ps1",
+ "smoke_desktop.py", "smoke_update_tasks.py", "smoke_split_update.py"})
+ return any(name == base or name.startswith(base + "/") for base in GROUPS[group])
+
+
+def local_path(root: Path, name: str, group: str) -> Path:
+ if not allowed_path(name, group):
+ raise ValueError("产物路径不属于本任务输出目录")
+ path = root / name
+ if not path.resolve().is_relative_to(root) or any(parent.is_symlink() for parent in (path, *path.parents) if parent != root and parent.is_relative_to(root)):
+ raise ValueError("产物路径包含软链接或越出工作目录")
+ return path
+
+
+def checksum(path: Path) -> str:
+ with path.open("rb") as stream:
+ return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def rclone_executable(root: Path) -> str:
+ suffix = ".exe" if os.name == "nt" else ""
+ path = root / "build/release-tools" / ("rclone" + suffix)
+ if not path.is_file() or path.is_symlink():
+ raise ValueError("CI 中转仅使用项目内已核对官方 SHA-256 的 rclone")
+ return str(path)
+
+
+def inventory(root: Path, group: str) -> list[dict]:
+ paths = []
+ for base in GROUPS[group]:
+ source = root / base
+ if not source.exists():
+ raise ValueError(f"缺少构建产物目录或文件:{base}")
+ paths.extend(source.rglob("*") if source.is_dir() else [source])
+ items = []
+ for path in sorted(paths):
+ name = path.relative_to(root).as_posix()
+ if group == "macos-release" and not allowed_path(name, group):
+ continue
+ if path.is_symlink():
+ raise ValueError("CI 产物不允许软链接")
+ if not path.is_file():
+ continue
+ local_path(root, name, group)
+ items.append({"path": name, "size": path.stat().st_size, "sha256": checksum(path)})
+ if not items or len({item["path"].casefold() for item in items}) != len(items):
+ raise ValueError("产物清单为空或包含跨平台同名文件")
+ if group == "macos-release":
+ for arch in ("arm64", "x86_64"):
+ if sum(item["path"].startswith(f"dist/macos.noindex/{arch}/") for item in items) != 3:
+ raise ValueError("每种 macOS 架构须包含 DMG、构建身份与 SHA-256 清单")
+ folder = root / "dist/macos.noindex" / arch
+ info = json.loads((folder / "build-info.json").read_text(encoding="utf-8"))
+ build = identity()
+ image = folder / f"Peach-{build['tag'][1:]}-macos-{arch}.dmg"
+ if (info.get("commit") != build["commit"] or info.get("version") != build["tag"][1:]
+ or info.get("architecture") != arch or info.get("dirty") is not False
+ or not image.is_file()
+ or (folder / "SHA256SUMS.txt").read_text(encoding="utf-8").strip() != f"{checksum(image)} {image.name}"):
+ raise ValueError("macOS 构建身份或镜像 SHA-256 不符")
+ return items
+
+
+def validate_manifest(manifest: dict, group: str, build: dict) -> list[dict]:
+ if manifest.get("format") != 1 or manifest.get("identity") != build or manifest.get("group") != group:
+ raise ValueError("产物清单不属于当前版本、提交或流水线")
+ prefix(build, group, manifest.get("producer_job"))
+ items = manifest.get("files")
+ if not isinstance(items, list) or not items:
+ raise ValueError("产物清单缺少文件")
+ names = set()
+ for item in items:
+ name = item.get("path", "")
+ if (not allowed_path(name, group) or name.casefold() in names
+ or type(item.get("size")) is not int or item["size"] < 0
+ or not re.fullmatch(r"[0-9a-f]{64}", item.get("sha256", ""))):
+ raise ValueError("产物清单的路径、大小或 SHA-256 无效")
+ names.add(name.casefold())
+ return items
+
+
+def transfer(root: Path, group: str, mode: str) -> dict:
+ root = root.resolve(strict=True)
+ build = identity()
+ environment = rclone_environment()
+ executable = rclone_executable(root)
+ manifest_path = root / "build/ci-artifacts" / (group + ".json")
+ if mode == "push":
+ producer_job = os.environ.get("CI_JOB_ID", "")
+ else:
+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+ items = validate_manifest(manifest, group, build)
+ producer_job = manifest["producer_job"]
+ storage_prefix = prefix(build, group, producer_job)
+ remote = f"peach:{environment['PEACH_R2_BUCKET']}/{storage_prefix}"
+ with tempfile.TemporaryDirectory(prefix="peach-ci-r2-") as directory:
+ temporary = Path(directory).resolve()
+ configuration = temporary / "rclone.conf"
+ configuration.touch()
+
+ def command(*args, read=False):
+ args = [executable, *args, "--config", str(configuration)]
+ if read:
+ return subprocess.check_output(args, env=environment, text=True, encoding="utf-8")
+ subprocess.run(args, env=environment, check=True)
+
+ if mode == "push":
+ actual = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True, encoding="utf-8").strip()
+ dirty = subprocess.check_output(["git", "status", "--porcelain"], cwd=root, text=True, encoding="utf-8").strip()
+ if actual != build["commit"] or dirty:
+ raise ValueError("暂存需要当前提交的干净构建目录")
+ items = inventory(root, group)
+ existing = json.loads(command("lsjson", f"peach:{environment['PEACH_R2_BUCKET']}", "--recursive", "--files-only", read=True))
+ require_capacity(existing, [{"name": item["path"], "size": item["size"]} for item in items],
+ storage_prefix, int(environment.get("PEACH_R2_MAX_BYTES", "8000000000")))
+ listing = temporary / "files.txt"
+ listing.write_text("".join(item["path"] + "\n" for item in items), encoding="utf-8")
+ command("copy", str(root), remote, "--files-from-raw", str(listing), "--checksum", "--immutable")
+ manifest = {"format": 1, "identity": build, "group": group, "producer_job": producer_job, "files": items}
+ manifest_path.parent.mkdir(parents=True, exist_ok=True)
+ manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ else:
+ for item in items:
+ local_path(root, item["path"], group)
+ listing = temporary / "files.txt"
+ listing.write_text("".join(item["path"] + "\n" for item in items), encoding="utf-8")
+ command("copy", remote, str(temporary / "download"), "--files-from-raw", str(listing), "--checksum")
+ for item in items:
+ source = temporary / "download" / item["path"]
+ if not source.is_file() or source.stat().st_size != item["size"] or checksum(source) != item["sha256"]:
+ raise ValueError(f"R2 暂存产物的大小或 SHA-256 不符:{item['path']}")
+ for item in items:
+ target = local_path(root, item["path"], group)
+ target.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copyfile(temporary / "download" / item["path"], target)
+ return {"group": group, "mode": mode, "files": len(items), "bytes": sum(item["size"] for item in items), "commit": build["commit"]}
+
+
+def cleanup(root: Path) -> dict:
+ """公开发布成功后只回收当前流水线的暂存前缀。"""
+ build = identity()
+ environment = rclone_environment()
+ executable = rclone_executable(root.resolve(strict=True))
+ remote = f"peach:{environment['PEACH_R2_BUCKET']}/{prefix(build)}"
+ with tempfile.TemporaryDirectory(prefix="peach-ci-r2-cleanup-") as directory:
+ configuration = Path(directory) / "rclone.conf"
+ configuration.touch()
+ index = json.loads(subprocess.check_output(
+ [executable, "cat", f"peach:{environment['PEACH_R2_BUCKET']}/releases.json", "--config", str(configuration)],
+ env=environment, text=True, encoding="utf-8"))
+ if not any(item.get("tag_name") == build["tag"] and item.get("commit_sha") == build["commit"]
+ for item in index):
+ raise ValueError("公开版本索引尚未登记当前提交,保留暂存产物")
+ subprocess.run([executable, "purge", remote, "--config", str(configuration)], env=environment, check=True)
+ return {"cleaned_pipeline": build["pipeline"], "commit": build["commit"]}
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("mode", choices=("push", "pull", "cleanup"))
+ parser.add_argument("group", nargs="?", choices=tuple(GROUPS))
+ args = parser.parse_args()
+ if args.mode != "cleanup" and not args.group:
+ parser.error("上传和下载须指定产物组")
+ result = cleanup(Path.cwd()) if args.mode == "cleanup" else transfer(Path.cwd(), args.group, args.mode)
+ print(json.dumps(result, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/fetch_ffmpeg_sources.py b/scripts/fetch_ffmpeg_sources.py
index 96c75a291..7287f9ac9 100644
--- a/scripts/fetch_ffmpeg_sources.py
+++ b/scripts/fetch_ffmpeg_sources.py
@@ -3,14 +3,23 @@ from __future__ import annotations
import argparse
import hashlib
+import http.client
import json
+import os
from pathlib import Path
import shutil
+import sys
import tarfile
+import time
+import urllib.error
import urllib.request
+from urllib.parse import urlsplit
import zipfile
SCRIPT_DIR = Path(__file__).resolve().parent
+sys.path.insert(0, str(SCRIPT_DIR.parent / "src"))
+from peach.user_agent import USER_AGENT
+
LOCK_PATH = SCRIPT_DIR / "ffmpeg-sources.json"
@@ -19,6 +28,45 @@ def digest(path: Path) -> str:
return hashlib.file_digest(stream, "sha256").hexdigest()
+def download_source(source: dict, archive: Path) -> None:
+ """限次重试源码传输,完整校验后才登记缓存。"""
+ temporary = archive.with_suffix(archive.suffix + ".part")
+ request = urllib.request.Request(source["url"], headers={"User-Agent": USER_AGENT})
+ for attempt in range(3):
+ try:
+ print(f"下载固定源码:{source['name']},第 {attempt + 1}/3 次", flush=True)
+ with urllib.request.urlopen(request, timeout=60) as response, temporary.open("wb") as stream:
+ shutil.copyfileobj(response, stream)
+ if digest(temporary) != source["sha256"]:
+ raise ValueError(f"源码 SHA256 不匹配: {source['name']}")
+ temporary.replace(archive)
+ return
+ except (TimeoutError, ConnectionError, http.client.IncompleteRead, urllib.error.URLError) as error:
+ retryable = not isinstance(error, urllib.error.HTTPError) or error.code in {408, 429, 500, 502, 503, 504}
+ if not retryable or attempt == 2:
+ raise
+ time.sleep(2 ** (attempt + 1))
+ finally:
+ temporary.unlink(missing_ok=True)
+
+
+def obtain_source(source: dict, archive: Path) -> None:
+ """从可选源码缓存或固定上游获取同一份已锁定字节。"""
+ base = os.environ.get("PEACH_SOURCE_CACHE_BASE", "").rstrip("/")
+ if base:
+ parsed = urlsplit(base)
+ if (parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password
+ or parsed.query or parsed.fragment):
+ raise ValueError("源码缓存须为无凭据的 HTTPS 地址")
+ cached = {**source, "url": f"{base}/{source['sha256']}/{source['archive']}"}
+ try:
+ download_source(cached, archive)
+ return
+ except (TimeoutError, ConnectionError, http.client.IncompleteRead, urllib.error.URLError):
+ print(f"源码缓存未取得,尝试固定上游:{source['name']}", flush=True)
+ download_source(source, archive)
+
+
def load_sources(path: Path = LOCK_PATH) -> dict:
lock = json.loads(path.read_text(encoding="utf-8"))
if {source["name"] for source in lock["sources"]} != {"ffmpeg", "x264", "nv-codec-headers", "dav1d"}:
@@ -39,12 +87,7 @@ def prepare(output: Path, lock: dict) -> None:
for source in lock["sources"]:
archive = sources / source["archive"]
if not archive.exists():
- temporary = archive.with_suffix(archive.suffix + ".part")
- with urllib.request.urlopen(source["url"], timeout=60) as response, temporary.open("wb") as stream:
- shutil.copyfileobj(response, stream)
- if digest(temporary) != source["sha256"]:
- raise ValueError(f"源码 SHA256 不匹配: {source['name']}")
- temporary.replace(archive)
+ obtain_source(source, archive)
if digest(archive) != source["sha256"]:
raise ValueError(f"源码 SHA256 不匹配: {source['name']}")
destination = work / source["name"]
@@ -105,6 +148,9 @@ def finish(output: Path, lock: dict) -> dict:
package.write(path, f"sources/{path.name}")
for name in ("build_ffmpeg_windows.sh", "fetch_ffmpeg_sources.py", "ffmpeg-sources.json", "dav1d-mingw64.ini"):
package.write(SCRIPT_DIR / name, f"scripts/{name}")
+ for name in ("__init__.py", "user_agent.py"):
+ package.write(SCRIPT_DIR.parent / "src/peach" / name, f"src/peach/{name}")
+ package.write(SCRIPT_DIR.parent / "LICENSE", "licenses/Peach-LICENSE.txt")
for license_file in sorted(bundle.glob("LICENSE*")):
package.write(license_file, f"licenses/{license_file.name}")
package.writestr("README.md", instructions)
diff --git a/scripts/fetch_release_tools.py b/scripts/fetch_release_tools.py
index 3ec571abc..d7b012493 100644
--- a/scripts/fetch_release_tools.py
+++ b/scripts/fetch_release_tools.py
@@ -1,10 +1,12 @@
"""从官方发行站下载固定版本 CLI,校验 SHA-256 后提取指定二进制。"""
from __future__ import annotations
+import argparse
import hashlib
import io
from pathlib import Path
import re
+import platform
import sys
import tarfile
from urllib.request import Request, urlopen
@@ -25,9 +27,26 @@ def download(url: str) -> bytes:
return response.read()
-def install(destination: Path) -> None:
+def tool_specs(*, rclone_only: bool = False, system: str | None = None,
+ machine: str | None = None) -> tuple:
+ system = system or platform.system()
+ machine = (machine or platform.machine()).lower()
+ arch = {"amd64": "amd64", "x86_64": "amd64", "arm64": "arm64", "aarch64": "arm64"}.get(machine)
+ operating = {"Linux": "linux", "Darwin": "osx", "Windows": "windows"}.get(system)
+ if not arch or not operating or (not rclone_only and system != "Linux"):
+ raise ValueError("此平台仅支持通过 --rclone-only 安装 rclone,且架构须为 amd64 或 arm64")
+ suffix = ".exe" if system == "Windows" else ""
+ stem = f"rclone-v1.75.2-{operating}-{arch}"
+ rclone = ("rclone" + suffix, stem + ".zip", TOOLS[1][2], "SHA256SUMS", stem + "/rclone" + suffix)
+ glab = TOOLS[0]
+ if arch == "arm64":
+ glab = tuple(value.replace("amd64", "arm64") for value in glab)
+ return (rclone,) if rclone_only else (glab, rclone)
+
+
+def install(destination: Path, *, rclone_only: bool = False) -> None:
destination.mkdir(parents=True, exist_ok=True)
- for name, archive, base, checksums, member in TOOLS:
+ for name, archive, base, checksums, member in tool_specs(rclone_only=rclone_only):
document = download(f"{base}/{checksums}").decode("utf-8")
expected = re.search(rf"^([0-9a-fA-F]{{64}})\s+\*?{re.escape(archive)}$", document, re.MULTILINE)
content = download(f"{base}/{archive}")
@@ -49,4 +68,7 @@ def install(destination: Path) -> None:
if __name__ == "__main__":
- install(Path("build/release-tools").resolve())
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--rclone-only", action="store_true")
+ args = parser.parse_args()
+ install(Path("build/release-tools").resolve(), rclone_only=args.rclone_only)
diff --git a/scripts/fetch_untrunc_sources.py b/scripts/fetch_untrunc_sources.py
index 94b654a0a..8252bf6c4 100644
--- a/scripts/fetch_untrunc_sources.py
+++ b/scripts/fetch_untrunc_sources.py
@@ -61,6 +61,9 @@ def finish(output: Path, lock: dict) -> dict:
for name in ('build_untrunc_windows.sh', 'fetch_untrunc_sources.py',
'fetch_ffmpeg_sources.py', 'untrunc-sources.json'):
package.write(SCRIPT_DIR / name, f'scripts/{name}')
+ for name in ('__init__.py', 'user_agent.py'):
+ package.write(SCRIPT_DIR.parent / 'src/peach' / name, f'src/peach/{name}')
+ package.write(SCRIPT_DIR.parent / 'LICENSE', 'licenses/Peach-LICENSE.txt')
package.writestr('README.md', instructions)
package.write(output / 'buildconf.txt', 'buildconf.txt')
package.write(output / 'build-environment.txt', 'build-environment.txt')
diff --git a/tests/test_ffmpeg_build.py b/tests/test_ffmpeg_build.py
index 85f161f36..20488bcf0 100644
--- a/tests/test_ffmpeg_build.py
+++ b/tests/test_ffmpeg_build.py
@@ -4,10 +4,15 @@ from __future__ import annotations
import importlib.util
import io
import json
+import os
from pathlib import Path
import tarfile
import tempfile
+import subprocess
+import sys
import unittest
+from unittest import mock
+import urllib.error
import zipfile
@@ -19,6 +24,72 @@ SPEC.loader.exec_module(builder)
class FFmpegSourceBuildTests(unittest.TestCase):
+ def test_source_cache_falls_back_to_pinned_upstream_on_transport_failure(self):
+ import hashlib
+ data = b"pinned source bytes"
+ source = {"name": "ffmpeg", "url": "https://upstream.example/source.tar.xz", "archive": "source.tar.xz",
+ "sha256": hashlib.sha256(data).hexdigest()}
+ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(os.environ, {"PEACH_SOURCE_CACHE_BASE": "https://cache.example/_sources"}), \
+ mock.patch.object(builder.urllib.request, "urlopen", side_effect=[
+ urllib.error.HTTPError("https://cache.example", 404, "missing", {}, None), io.BytesIO(data)]) as opened:
+ archive = Path(temporary).resolve() / "source.tar.xz"
+ builder.obtain_source(source, archive)
+ self.assertEqual(archive.read_bytes(), data)
+ self.assertEqual([call.args[0].full_url for call in opened.call_args_list],
+ [f"https://cache.example/_sources/{source['sha256']}/source.tar.xz", source["url"]])
+
+ def test_source_cache_hash_mismatch_is_fatal(self):
+ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(os.environ, {"PEACH_SOURCE_CACHE_BASE": "https://cache.example/_sources"}), \
+ mock.patch.object(builder.urllib.request, "urlopen", return_value=io.BytesIO(b"corrupt")) as opened:
+ archive = Path(temporary).resolve() / "source.tar.xz"
+ with self.assertRaisesRegex(ValueError, "SHA256"):
+ builder.obtain_source({"name": "ffmpeg", "url": "https://upstream.example/source.tar.xz",
+ "archive": "source.tar.xz", "sha256": "a" * 64}, archive)
+ self.assertEqual(opened.call_count, 1)
+ self.assertFalse(archive.exists())
+
+ def test_download_retries_transport_failure_and_keeps_only_verified_archive(self):
+ data = b"fixed upstream source"
+ import hashlib
+ source = {"name": "ffmpeg", "url": "https://upstream.example/source.tar.xz",
+ "sha256": hashlib.sha256(data).hexdigest()}
+ with tempfile.TemporaryDirectory() as temporary:
+ archive = Path(temporary).resolve() / "source.tar.xz"
+ with mock.patch.object(builder.urllib.request, "urlopen", side_effect=[TimeoutError(), io.BytesIO(data)]) as opened, \
+ mock.patch.object(builder.time, "sleep") as sleep:
+ builder.download_source(source, archive)
+ self.assertEqual(archive.read_bytes(), data)
+ self.assertEqual(opened.call_count, 2)
+ sleep.assert_called_once_with(2)
+ self.assertFalse(archive.with_suffix(".xz.part").exists())
+
+ def test_download_integrity_or_permanent_http_error_is_not_retried(self):
+ for failure in (None, urllib.error.HTTPError("https://upstream.example", 403, "Forbidden", {}, None)):
+ with self.subTest(failure=failure), tempfile.TemporaryDirectory() as temporary:
+ archive = Path(temporary).resolve() / "source.tar.xz"
+ with mock.patch.object(builder.urllib.request, "urlopen", side_effect=failure,
+ return_value=io.BytesIO(b"corrupt")) as opened, \
+ mock.patch.object(builder.time, "sleep") as sleep:
+ with self.assertRaises((ValueError, urllib.error.HTTPError)):
+ builder.download_source({"name": "ffmpeg", "url": "https://upstream.example/source.tar.xz",
+ "sha256": "a" * 64}, archive)
+ self.assertEqual(opened.call_count, 1)
+ sleep.assert_not_called()
+ self.assertFalse(archive.exists())
+ self.assertFalse(archive.with_suffix(".xz.part").exists())
+
+ def test_download_stops_after_three_transport_attempts(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ archive = Path(temporary).resolve() / "source.tar.xz"
+ with mock.patch.object(builder.urllib.request, "urlopen", side_effect=TimeoutError()) as opened, \
+ mock.patch.object(builder.time, "sleep") as sleep:
+ with self.assertRaises(TimeoutError):
+ builder.download_source({"name": "ffmpeg", "url": "https://upstream.example/source.tar.xz",
+ "sha256": "a" * 64}, archive)
+ self.assertEqual(opened.call_count, 3)
+ self.assertEqual(sleep.call_args_list, [mock.call(2), mock.call(4)])
+ self.assertFalse(archive.exists())
+
def test_source_lock_pins_all_linked_external_dependencies(self):
lock = builder.load_sources()
self.assertEqual(lock["version"], "9.0.2")
@@ -85,6 +156,12 @@ class FFmpegSourceBuildTests(unittest.TestCase):
self.assertTrue({"scripts/build_ffmpeg_windows.sh", "scripts/fetch_ffmpeg_sources.py",
"scripts/ffmpeg-sources.json", "README.md", "buildconf.txt"} <= names)
self.assertFalse(any("manifest.json" in name for name in names))
+ independent = output / "independent-source"
+ package.extractall(independent)
+ help_result = subprocess.run([sys.executable, "-S", str(independent / "scripts/fetch_ffmpeg_sources.py"), "--help"],
+ cwd=independent, env={**os.environ, "PYTHONPATH": ""},
+ capture_output=True, text=True, encoding="utf-8", check=True)
+ self.assertIn("prepare", help_result.stdout)
stored = json.loads((output / "ffmpeg/manifest.json").read_text(encoding="utf-8"))
self.assertEqual(stored, manifest)
diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index 503332b1a..934622994 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -4695,6 +4695,167 @@ class ScriptingConventionTests(unittest.TestCase):
self.assertFalse(scripting.host_under("", ("x.com",)))
+class R2ArtifactTransferTests(unittest.TestCase):
+ def tools(self, root):
+ path = root / "build/release-tools" / ("rclone.exe" if os.name == "nt" else "rclone")
+ path.parent.mkdir(parents=True)
+ path.write_bytes(b"test double; subprocess calls are mocked")
+
+ def environment(self):
+ return {"CI_PROJECT_ID": "123", "CI_PIPELINE_ID": "456", "CI_COMMIT_SHA": "a" * 40,
+ "CI_COMMIT_TAG": "v1.2.3", "CI_COMMIT_REF_PROTECTED": "true", "CI_JOB_ID": "789"}
+
+ def rclone_environment(self):
+ return {"PEACH_RCLONE": "rclone", "PEACH_R2_BUCKET": "peach-releases"}
+
+ def test_platform_cli_archives_use_native_architecture_and_binary(self):
+ module = load_script("fetch_release_tools")
+ for system, machine, archive, member in (
+ ("Windows", "AMD64", "windows-amd64", "rclone.exe"),
+ ("Darwin", "arm64", "osx-arm64", "rclone"),
+ ("Darwin", "x86_64", "osx-amd64", "rclone"),
+ ("Linux", "x86_64", "linux-amd64", "rclone")):
+ with self.subTest(system=system, machine=machine):
+ tool, = module.tool_specs(rclone_only=True, system=system, machine=machine)
+ self.assertEqual(tool[0], member)
+ self.assertEqual(tool[1], f"rclone-v1.75.2-{archive}.zip")
+ self.assertEqual(tool[4], f"rclone-v1.75.2-{archive}/{member}")
+
+ def test_round_trip_restores_only_verified_files_from_current_pipeline(self):
+ module = load_script("ci_r2_artifacts")
+ objects = {}
+
+ def run(command, **_kwargs):
+ self.assertEqual(command[1], "copy")
+ names = Path(command[command.index("--files-from-raw") + 1]).read_text().splitlines()
+ source, destination = command[2:4]
+ if destination.startswith("peach:"):
+ for name in names:
+ objects[destination + "/" + name] = (Path(source) / name).read_bytes()
+ else:
+ for name in names:
+ path = Path(destination) / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(objects[source + "/" + name])
+
+ def read(command, **_kwargs):
+ if command[0] == "git":
+ return "a" * 40 if "rev-parse" in command else ""
+ return json.dumps([{"Path": key.split("peach:peach-releases/", 1)[1], "Size": len(value)}
+ for key, value in objects.items()])
+
+ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(os.environ, self.environment(), clear=True), \
+ mock.patch.object(module, "rclone_environment", side_effect=self.rclone_environment), \
+ mock.patch.object(module.subprocess, "check_output", side_effect=read), \
+ mock.patch.object(module.subprocess, "run", side_effect=run):
+ root = Path(temporary).resolve()
+ self.tools(root)
+ (root / "release").mkdir()
+ artifact = root / "release/Peach-1.2.3-windows-x64.zip"
+ artifact.write_bytes(b"verified executable archive")
+ result = module.transfer(root, "windows-release", "push")
+ self.assertEqual(result["files"], 1)
+ manifest_path = root / "build/ci-artifacts/windows-release.json"
+ self.assertLess(manifest_path.stat().st_size, 2000)
+ artifact.unlink()
+ module.transfer(root, "windows-release", "pull")
+ self.assertEqual(artifact.read_bytes(), b"verified executable archive")
+ key, = objects
+ self.assertTrue(key.startswith("peach:peach-releases/_ci/123/456/" + "a" * 40 + "/"))
+ objects[key] = b"corrupted"
+ with self.assertRaisesRegex(ValueError, "SHA-256"):
+ module.transfer(root, "windows-release", "pull")
+ self.assertEqual(artifact.read_bytes(), b"verified executable archive")
+
+ def test_manifest_rejects_other_pipeline_path_traversal_and_duplicates(self):
+ module = load_script("ci_r2_artifacts")
+ with mock.patch.dict(os.environ, self.environment(), clear=True):
+ build = module.identity()
+ item = {"path": "release/version.txt", "size": 5, "sha256": "b" * 64}
+ manifest = {"format": 1, "identity": build, "group": "windows-release", "producer_job": "789", "files": [item]}
+ self.assertEqual(module.validate_manifest(manifest, "windows-release", build), [item])
+ with self.assertRaises(ValueError):
+ module.validate_manifest(manifest, "windows-release", {**build, "pipeline": "789"})
+ with self.assertRaises(ValueError):
+ module.validate_manifest({**manifest, "producer_job": "../other"}, "windows-release", build)
+ self.assertNotEqual(module.prefix(build, "windows-release", "789"),
+ module.prefix(build, "windows-release", "790"))
+ for name in ("release/../../secrets/token", "release/Peach.exe:token", "release\\version.txt",
+ "/release/version.txt", "release/private-config.toml", "src/peach/__init__.py"):
+ with self.subTest(name=name), self.assertRaises(ValueError):
+ module.validate_manifest({**manifest, "files": [{**item, "path": name}]}, "windows-release", build)
+ with self.assertRaises(ValueError):
+ module.validate_manifest({**manifest, "files": [item, {**item, "path": "release/VERSION.txt"}]},
+ "windows-release", build)
+
+ def test_macos_inventory_keeps_signed_images_and_checks_build_identity(self):
+ module = load_script("ci_r2_artifacts")
+ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(os.environ, self.environment(), clear=True):
+ root = Path(temporary).resolve()
+ for arch in ("arm64", "x86_64"):
+ folder = root / "dist/macos.noindex" / arch
+ folder.mkdir(parents=True)
+ image = folder / f"Peach-1.2.3-macos-{arch}.dmg"
+ image.write_bytes(b"signed application image")
+ (folder / "Peach.app").mkdir()
+ (folder / "Peach.app/app").write_bytes(b"expanded application")
+ (folder / "build-info.json").write_text(json.dumps({"version": "1.2.3", "commit": "a" * 40,
+ "architecture": arch, "dirty": False}))
+ (folder / "SHA256SUMS.txt").write_text(f"{module.checksum(image)} {image.name}\n")
+ items = module.inventory(root, "macos-release")
+ self.assertEqual(len(items), 6)
+ self.assertFalse(any(".app/" in item["path"] for item in items))
+ image.write_bytes(b"tampered application")
+ with self.assertRaisesRegex(ValueError, "SHA-256"):
+ module.inventory(root, "macos-release")
+
+ def test_rclone_path_requires_installed_project_tool(self):
+ module = load_script("ci_r2_artifacts")
+ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(os.environ, {"PEACH_RCLONE": "arbitrary-cli"}):
+ root = Path(temporary).resolve()
+ with self.assertRaisesRegex(ValueError, "项目内"):
+ module.rclone_executable(root)
+ self.tools(root)
+ self.assertTrue(module.rclone_executable(root).startswith(str(root)))
+
+ def test_capacity_failure_prevents_upload_and_manifest_creation(self):
+ module = load_script("ci_r2_artifacts")
+ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(os.environ, self.environment(), clear=True), \
+ mock.patch.object(module, "rclone_environment", side_effect=self.rclone_environment), \
+ mock.patch.object(module.subprocess, "check_output", side_effect=["a" * 40, "", '[{"Path":"v1.0.0/a.zip","Size":8000000000}]']), \
+ mock.patch.object(module.subprocess, "run") as upload:
+ root = Path(temporary).resolve()
+ self.tools(root)
+ (root / "release").mkdir()
+ (root / "release/version.txt").write_text("1.2.3")
+ with self.assertRaisesRegex(ValueError, "停止上传"):
+ module.transfer(root, "windows-release", "push")
+ upload.assert_not_called()
+ self.assertFalse((root / "build/ci-artifacts/windows-release.json").exists())
+
+ def test_cleanup_requires_published_commit_and_only_purges_own_prefix(self):
+ module = load_script("ci_r2_artifacts")
+ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(os.environ, self.environment(), clear=True), \
+ mock.patch.object(module, "rclone_environment", side_effect=self.rclone_environment), \
+ mock.patch.object(module.subprocess, "check_output", return_value="[]") as read, \
+ mock.patch.object(module.subprocess, "run") as purge:
+ root = Path(temporary).resolve()
+ self.tools(root)
+ with self.assertRaisesRegex(ValueError, "保留暂存"):
+ module.cleanup(root)
+ purge.assert_not_called()
+ read.return_value = json.dumps([{"tag_name": "v1.2.3", "commit_sha": "a" * 40}])
+ module.cleanup(root)
+ command = purge.call_args.args[0]
+ self.assertEqual(command[1:3], ["purge", "peach:peach-releases/_ci/123/456/" + "a" * 40])
+
+ def test_unprotected_or_incomplete_build_identity_is_rejected(self):
+ module = load_script("ci_r2_artifacts")
+ for patch in ({"CI_COMMIT_REF_PROTECTED": "false"}, {"CI_PIPELINE_ID": "../other"}, {"CI_COMMIT_TAG": "master"}):
+ with mock.patch.dict(os.environ, {**self.environment(), **patch}, clear=True), self.assertRaises(ValueError):
+ module.identity()
+
+
class R2ReleaseTests(unittest.TestCase):
def test_publication_preserves_existing_index_and_verifies_published_metadata(self):
module = load_script("release_r2")