Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

ci(release): 全量先过快速门槛、偶发失败隔离重跑、整机锁,版本提交先演练再发布

longmeidao1531de31782e
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 374ec0900..0cd59aade 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -1,12 +1,13 @@
name: Docker
+# 只验收不发布:镜像的版本号与 edge 由 release.yml 在 GitHub Release 成功后推送,
+# edge 始终指向最新发布的版本(ADR-0121)。
on:
pull_request:
paths: [Dockerfile, .dockerignore, compose.yaml, 'src/**', 'migrations/**', 'web/**', 'resources/**', pyproject.toml, uv.lock, setup.py, MANIFEST.in, scripts/smoke_docker.py, .github/workflows/docker.yml]
push:
branches: [master]
- tags: ['v*.*.*']
- paths: [Dockerfile, .dockerignore, compose.yaml, 'src/**', 'migrations/**', 'web/**', 'resources/**', pyproject.toml, uv.lock, setup.py, MANIFEST.in, scripts/smoke_docker.py, scripts/release_tag.py, .github/workflows/docker.yml]
+ paths: [Dockerfile, .dockerignore, compose.yaml, 'src/**', 'migrations/**', 'web/**', 'resources/**', pyproject.toml, uv.lock, setup.py, MANIFEST.in, scripts/smoke_docker.py, .github/workflows/docker.yml]
workflow_dispatch:
permissions:
@@ -27,11 +28,6 @@ jobs:
- uses: actions/setup-python@v7
with:
python-version: '3.14'
- - name: 核对标签提交的主线 CI
- if: github.ref_type == 'tag'
- env:
- GH_TOKEN: ${{ github.token }}
- run: python3 scripts/release_tag.py --repo "$GITHUB_REPOSITORY" --verify-sha "$GITHUB_SHA"
- uses: docker/setup-buildx-action@v4
- name: 构建候选镜像
uses: docker/build-push-action@v7
@@ -43,27 +39,3 @@ jobs:
cache-to: type=gha,mode=max
- name: 验收临时容器
run: python3 scripts/smoke_docker.py longmeidao/peach:test
- - name: 登录 Docker Hub
- if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v')) && vars.DOCKERHUB_USERNAME != ''
- uses: docker/login-action@v4
- with:
- username: ${{ vars.DOCKERHUB_USERNAME }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
- - name: 计算镜像标签
- id: tags
- env:
- REF_TYPE: ${{ github.ref_type }}
- REF_NAME: ${{ github.ref_name }}
- run: |
- if [ "$REF_TYPE" = tag ]; then
- echo "value=longmeidao/peach:${REF_NAME#v}" >> "$GITHUB_OUTPUT"
- else
- echo 'value=longmeidao/peach:edge' >> "$GITHUB_OUTPUT"
- fi
- - name: 发布已验收镜像
- if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v')) && vars.DOCKERHUB_USERNAME != ''
- env:
- PUBLISH_TAG: ${{ steps.tags.outputs.value }}
- run: |
- docker tag longmeidao/peach:test "$PUBLISH_TAG"
- docker push "$PUBLISH_TAG"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 701f192ae..d8686516a 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,7 +1,10 @@
name: Release
+# 版本提交 `chore(release): 版本 X` 推上 master 时演练整条发布链(只构建、验收,不发布);
+# `release_tag.py --ship` 等这次演练通过才打标签,标签上才发布(ADR-0121)。
on:
push:
+ branches: [master]
tags: ["v*.*.*"]
workflow_dispatch:
@@ -22,6 +25,8 @@ env:
jobs:
validate:
name: 核对发布提交与版本
+ # 其余作业都依赖它:普通的 master 推送在这里就结束,只有版本提交、标签与手动运行往下走。
+ if: "${{ github.ref_type == 'tag' || github.event_name == 'workflow_dispatch' || startsWith(github.event.head_commit.message, 'chore(release): 版本 ') }}"
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
@@ -348,7 +353,7 @@ jobs:
release:
name: GitHub 预发布
if: github.ref_type == 'tag'
- needs: [windows, desktop-smoke, macos-smoke]
+ needs: [windows, desktop-smoke, macos-smoke, docker]
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
@@ -375,6 +380,63 @@ jobs:
sha256sum ./*.zip ./*-setup.exe ./*.dmg > SHA256SUMS.txt
gh release create "$GITHUB_REF_NAME" ./*.zip ./*-setup.exe ./*.dmg SHA256SUMS.txt --verify-tag --prerelease --title "Peach $VERSION · Windows / macOS 测试版" --notes-file RELEASE_NOTES.md
+ # 镜像先构建、验收(演练与标签都跑),发布排在 GitHub Release 成功之后:
+ # 桌面包任何一步没发出去,Docker Hub 的版本号与 edge 都不动(ADR-0121)。
+ docker:
+ name: Linux amd64 镜像构建与验收
+ needs: validate
+ runs-on: ubuntu-latest
+ timeout-minutes: 25
+ steps:
+ - uses: actions/checkout@v7
+ - uses: actions/setup-python@v7
+ with:
+ python-version: '3.14'
+ - uses: docker/setup-buildx-action@v4
+ - name: 构建候选镜像
+ uses: docker/build-push-action@v7
+ with:
+ context: .
+ load: true
+ tags: longmeidao/peach:candidate
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
+ - name: 验收临时容器
+ run: python3 scripts/smoke_docker.py longmeidao/peach:candidate
+
+ docker-publish:
+ name: 发布已验收镜像
+ if: github.ref_type == 'tag' && vars.DOCKERHUB_USERNAME != ''
+ needs: [release, docker]
+ runs-on: ubuntu-latest
+ timeout-minutes: 25
+ steps:
+ - uses: actions/checkout@v7
+ - uses: actions/setup-python@v7
+ with:
+ python-version: '3.14'
+ - uses: docker/setup-buildx-action@v4
+ # 作业之间不共享 Docker 守护进程:按同一提交、同一缓存重建,再验收一次才推送。
+ - name: 构建镜像
+ uses: docker/build-push-action@v7
+ with:
+ context: .
+ load: true
+ tags: longmeidao/peach:candidate
+ cache-from: type=gha
+ - name: 验收临时容器
+ run: python3 scripts/smoke_docker.py longmeidao/peach:candidate
+ - uses: docker/login-action@v4
+ with:
+ username: ${{ vars.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - name: 推送版本号与 edge
+ run: |
+ for tag in "${GITHUB_REF_NAME#v}" edge; do
+ docker tag longmeidao/peach:candidate "longmeidao/peach:$tag"
+ docker push "longmeidao/peach:$tag"
+ done
+
project-board:
name: 同步已发布版本到看板
if: github.ref_type == 'tag'
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 42dce2ca5..86d72aa4b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -71,9 +71,10 @@ verify-linux:
variables:
PEACH_CI_COVERAGE: changed
+# 三平台全量只在版本提交的演练里跑;标签流水线由 validate-release 核对同一提交的演练已通过。
release-verify-linux:
extends: .verify-linux
- rules: !reference [.release-linux, rules]
+ rules: !reference [.release-rehearsal, rules]
variables:
PEACH_CI_COVERAGE: full
@@ -99,7 +100,7 @@ verify-windows:
release-verify-windows:
extends: .verify-windows
- rules: !reference [.release-linux, rules]
+ rules: !reference [.release-rehearsal, rules]
variables:
PEACH_CI_COVERAGE: full
@@ -124,7 +125,7 @@ verify-macos:
release-verify-macos:
extends: .verify-macos
- rules: !reference [.release-linux, rules]
+ rules: !reference [.release-rehearsal, rules]
variables:
PEACH_CI_COVERAGE: full
@@ -151,10 +152,24 @@ wheel-smoke:
expire_in: 14 days
paths: [build/wheelhouse/]
+# 发版分两步(ADR-0121):版本提交 `chore(release): 版本 X` 推上 master 时演练整条发布链,三平台全量、
+# 媒体、构建、冒烟与镜像都跑,只是不发布;`release_tag.py --ship` 等这条流水线转绿才打标签。
+# 标签流水线核对同一提交的演练已通过,重新构建后发布,不再重跑全量;任何一步没过都不发布任何东西。
+.release-rehearsal:
+ rules:
+ - if: '$CI_COMMIT_BRANCH == "master" && $CI_COMMIT_REF_PROTECTED == "true" && $CI_COMMIT_MESSAGE =~ /^chore\(release\): 版本 /'
+ variables:
+ PEACH_RELEASE_REHEARSAL: "1"
+
+.release-publish:
+ rules:
+ - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ && $CI_COMMIT_REF_PROTECTED == "true"'
+
.release-linux:
image: ghcr.io/astral-sh/uv:0.13.3-python3.14-trixie-slim
rules:
- - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ && $CI_COMMIT_REF_PROTECTED == "true"'
+ - !reference [.release-publish, rules]
+ - !reference [.release-rehearsal, rules]
before_script:
- apt-get update
- apt-get install -y --no-install-recommends python3 ca-certificates git
@@ -169,6 +184,8 @@ wheel-smoke:
validate-release:
extends: .release-linux
stage: validate
+ # 核对这个提交的主线流水线(含演练)已经成功:演练里正在跑的就是它自己,所以只在标签上跑。
+ rules: !reference [.release-publish, rules]
script:
- python3 scripts/release_tag.py --gitlab-host "$CI_SERVER_HOST" --repo "$CI_PROJECT_PATH" --verify-sha "$CI_COMMIT_SHA"
- python3 -c 'import os; from pathlib import Path; from scripts.version_bump import read_version; assert os.environ["CI_COMMIT_TAG"] == "v" + read_version(Path.cwd())'
@@ -196,7 +213,7 @@ build-windows:
stage: build
tags: [peach-windows]
rules: !reference [.release-linux, rules]
- needs: [windows-media, {job: release-verify-windows, artifacts: false}]
+ needs: [windows-media, {job: release-verify-windows, artifacts: false, optional: true}]
environment:
name: release
action: prepare
@@ -206,7 +223,10 @@ build-windows:
- python scripts/ci_r2_artifacts.py pull windows-media
- if ($LASTEXITCODE -ne 0) { throw '媒体构建产物校验失败' }
script:
- - '& ./scripts/ci_windows.ps1 build'
+ # 本机 Windows runner 就是开发机:构建、冒烟与镜像作业和本机测试排同一把整机重负载锁
+ # (scripts/machine_lock.py)。runner 的系统 Python 没有 filelock,用 uv 起一个只带它的临时环境,
+ # 不碰作业里会被 uv sync 改写的 .venv。验证作业经测试入口,入口自己拿这把锁。
+ - uv run --no-project --with filelock==4.2.0 --python 3.14 python -X utf8 scripts/machine_lock.py -- pwsh -NoProfile -File ./scripts/ci_windows.ps1 build
- if ($LASTEXITCODE -ne 0) { throw 'Windows 构建失败' }
- python scripts/ci_r2_artifacts.py push windows-release
- if ($LASTEXITCODE -ne 0) { throw 'Windows 产物暂存失败' }
@@ -220,7 +240,7 @@ build-macos:
# 两个架构各自从源码编译 x264、ffmpeg 与 untrunc,不走缓存;windows-media 单架构交叉编译 ffmpeg 与 untrunc 约 17 分钟。
timeout: 2h
rules: !reference [.release-linux, rules]
- needs: [validate-release, {job: release-verify-macos, artifacts: false}]
+ needs: [{job: validate-release, optional: true}, {job: release-verify-macos, artifacts: false, optional: true}]
environment:
name: release
action: prepare
@@ -248,26 +268,53 @@ smoke-windows:
- python scripts/ci_r2_artifacts.py pull windows-release
- if ($LASTEXITCODE -ne 0) { throw 'Windows 产物下载校验失败' }
script:
- - '& ./release/ci_windows.ps1 smoke'
+ - uv run --no-project --with filelock==4.2.0 --python 3.14 python -X utf8 scripts/machine_lock.py -- pwsh -NoProfile -File ./release/ci_windows.ps1 smoke
- if ($LASTEXITCODE -ne 0) { throw 'Windows 独立制品验收失败' }
+# 镜像先构建、冒烟,留在这台 runner 上;推送放在 publish-release 成功之后(publish-docker),
+# 安装包、R2 与 Release 任何一步没发出去,Docker Hub 的版本号与 edge 都不动。
+docker-build:
+ stage: smoke
+ tags: [peach-windows]
+ rules: !reference [.release-linux, rules]
+ needs: [{job: validate-release, optional: true}, {job: release-verify-linux, artifacts: false, optional: true}]
+ environment:
+ name: release
+ action: prepare
+ script:
+ - uv run --no-project --with filelock==4.2.0 --python 3.14 python -X utf8 scripts/machine_lock.py -- pwsh -NoProfile -File ./scripts/ci_windows.ps1 docker-build
+ - if ($LASTEXITCODE -ne 0) { throw 'Docker 镜像构建或验收失败' }
+
publish-docker:
stage: publish
tags: [peach-windows]
- rules: !reference [.release-linux, rules]
- needs: [validate-release, {job: release-verify-linux, artifacts: false}]
+ rules: !reference [.release-publish, rules]
+ needs: [publish-release, docker-build]
resource_group: peach-docker-publication
environment:
name: release
action: prepare
script:
- - '& ./scripts/ci_windows.ps1 docker'
+ - pwsh -NoProfile -File ./scripts/ci_windows.ps1 docker-push
- if ($LASTEXITCODE -ne 0) { throw 'Docker 镜像发布失败' }
+# 演练不发布:暂存在 R2 的本流水线产物删掉,不占容量预算。
+release-rehearsal-cleanup:
+ extends: .release-linux
+ stage: publish
+ rules: !reference [.release-rehearsal, rules]
+ needs: [smoke-windows, build-macos, docker-build]
+ environment:
+ name: release
+ action: prepare
+ script:
+ - python3 scripts/ci_r2_artifacts.py cleanup
+
publish-release:
extends: .release-linux
stage: publish
- needs: [build-windows, smoke-windows, build-macos, {job: release-verify-linux, artifacts: false}]
+ rules: !reference [.release-publish, rules]
+ needs: [validate-release, build-windows, smoke-windows, build-macos, docker-build]
resource_group: peach-release-publication
environment:
name: release
diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md
index 84c0d747f..cfeb7d674 100644
--- a/docs/OPERATIONS.md
+++ b/docs/OPERATIONS.md
@@ -71,7 +71,7 @@ sudo /usr/bin/xattr -rd com.apple.quarantine "/Applications/Peach.app"
### GitLab 发布与下载后端
-GitLab 兼容平台使用 `.gitlab-ci.yml`。唯一版本入口支持 `--gitlab-host` 指定主机,默认入口使用 GitHub。主线发布判据要求同一提交最新的 push 流水线成功,且 Linux、Windows、macOS 和 wheel 验收任务全部成功;缺少原生系统任务时拒绝打标签。主线上的三个平台验收只跑改动域与平台敏感用例,三平台全量在版本标签流水线上运行,Windows、macOS 构建与 Docker、Release 发布都依赖它([ADR-0119](adr/0119-mainline-changed-scope-release-full-suite.md))。CLI 传输复用官方 glab,不在仓库或命令参数里存访问令牌。
+GitLab 兼容平台使用 `.gitlab-ci.yml`。唯一版本入口支持 `--gitlab-host` 指定主机,默认入口使用 GitHub。主线发布判据要求同一提交最新的 push 流水线成功,且 Linux、Windows、macOS 和 wheel 验收任务全部成功;缺少原生系统任务时拒绝打标签。主线上的三个平台验收只跑改动域与平台敏感用例([ADR-0119](adr/0119-mainline-changed-scope-release-full-suite.md))。版本提交 `chore(release): 版本 X` 的主线流水线同时是发布演练:三平台全量、媒体工具、Windows 与 macOS 构建、Windows 冒烟和 Docker 镜像都跑一遍,只是不发布。`--ship` 等演练全部成功才打标签;标签流水线核对同一提交的演练已通过,只重新构建并发布([ADR-0121](adr/0121-release-rehearsal-and-atomic-publication.md))。GitHub 的 `release.yml` 用同一套判据,迁回 GitHub 时不用改流程。CLI 传输复用官方 glab,不在仓库或命令参数里存访问令牌。
平台共享 Runner 的系统由管理员提供。注册页面列出 Windows 和 macOS,表示可以接入这些系统上的 Runner;不能据此判断平台提供相应云主机。临时原生 Runner 只接入受保护主线和版本标签,使用项目专属标签,验收后停止并注销。Mac 的 Intel 构建需要真实 x86_64 Python 和依赖;在 Apple Silicon 上通过 Rosetta 验收时,应明确记录未覆盖 Intel 真机。
@@ -79,19 +79,19 @@ Windows 原生 Runner 在深目录中检出项目时,需为 Git 启用 `core.l
用计划任务常驻 Windows 原生 Runner 时,任务设置的 `Priority` 取 `4`,例如 `New-ScheduledTaskSettingsSet -Priority 4`;已注册的任务可以读出对象,把 `Settings.Priority` 改成 `4`,再用 `Set-ScheduledTask -InputObject` 写回。省略时默认 `7`,`gitlab-runner` 与所有作业子进程以 BelowNormal 的 CPU 优先级、Low 的 I/O 优先级和 2 级内存优先级运行。Runner 与开发会话共用这台机器,别的会话跑全量测试时,作业里的 vitest 工作进程、首次设置后的服务就绪探测和快捷方式 COM 调用拿不到 CPU 与磁盘,因等待超时失败,全量回归也会超过作业时限。设置只对新启动的进程生效:在没有作业运行时重启该任务,再核对 `gitlab-runner` 进程的优先级类为 `Normal`。
-版本标签须受保护。标签流水线重新核对提交与版本,构建固定来源的媒体工具、Windows 完整包与拆分包、macOS 双架构 DMG,并进行独立制品验收。只有全部成功后才创建 GitLab Release。项目流水线使用 `PEACH_DOWNLOAD_BACKEND=both`,将同一份制品分别保存到 R2 与 GitGud Generic Package Registry;不把会过期的 job artifacts 当作永久下载地址。独立部署可明确选用 `r2` 或 `gitgud` 单后端。
+版本标签须受保护。标签流水线重新核对提交与版本,构建固定来源的媒体工具、Windows 完整包与拆分包、macOS 双架构 DMG,并进行独立制品验收。发布全有或全无:`publish-release` 要等全部构建、冒烟与 `docker-build` 通过才上传 R2 并创建 GitLab Release,`publish-docker` 排在它成功之后才推送镜像。任何一步没过,安装包、R2、Release、Docker Hub 一个都不动;发布作业自身失败时,重试同一个作业即可,标签不动。项目流水线使用 `PEACH_DOWNLOAD_BACKEND=both`,将同一份制品分别保存到 R2 与 GitGud Generic Package Registry;不把会过期的 job artifacts 当作永久下载地址。独立部署可明确选用 `r2` 或 `gitgud` 单后端。
Linux 发布任务使用官方 uv Python 3.14 容器,并在获取制品工具前核对解释器版本;源码归档使用 `tarfile` 的 `data` 过滤器解压。CI 的 `.npm/` 与 `.uv-cache/` 属于生成缓存,不计入源码工作区变化。
-Windows 安装覆盖与任务恢复验收允许 GitHub Actions,以及受保护版本标签上的 GitLab `smoke-windows` 作业;主机已有 Peach 安装登记时拒绝执行。所有程序、账本与任务夹具均放在临时目录,安装与卸载完成后核对登记清理。`fetch_cloudflared.ps1` 的相对输出路径按 PowerShell 当前工作位置解析,下载仍须通过固定 SHA-256 校验。
+Windows 安装覆盖与任务恢复验收允许 GitHub Actions,以及受保护版本标签或版本提交演练上的 GitLab `smoke-windows` 作业;主机已有 Peach 安装登记时拒绝执行。所有程序、账本与任务夹具均放在临时目录,安装与卸载完成后核对登记清理。`fetch_cloudflared.ps1` 的相对输出路径按 PowerShell 当前工作位置解析,下载仍须通过固定 SHA-256 校验。
-GitGud 的 Linux 平台 Runner 不提供 Docker 守护进程,Docker 镜像由受保护标签上的 `publish-docker` 作业在 Windows 原生 Runner 构建,宿主须运行 Docker Desktop。作业核对标签与版本,用 `smoke_docker.py` 验收临时容器,再推送版本号与 `edge` 两个标签。`DOCKERHUB_USERNAME` 与 `DOCKERHUB_TOKEN` 放入受保护且隐藏的 CI 变量,环境作用域为 `release`;令牌只授予仓库读写。登录写入作业专用的临时 `DOCKER_CONFIG`,结束时登出并删除,不改动宿主用户的 Docker 登录。
+GitGud 的 Linux 平台 Runner 不提供 Docker 守护进程,Docker 镜像由 `docker-build` 作业在 Windows 原生 Runner 构建,宿主须运行 Docker Desktop。作业核对标签与版本,用 `smoke_docker.py` 验收临时容器,镜像留在本机;演练里验收完就删除。标签流水线上,`publish-docker` 等 `publish-release` 成功后才推送版本号与 `edge` 两个标签,所以 `edge` 只会指向已经完整发布的版本。`DOCKERHUB_USERNAME` 与 `DOCKERHUB_TOKEN` 放入受保护且隐藏的 CI 变量,环境作用域为 `release`;令牌只授予仓库读写。登录写入作业专用的临时 `DOCKER_CONFIG`,结束时登出并删除,不改动宿主用户的 Docker 登录。
媒体工具源码可通过 `PEACH_SOURCE_CACHE_BASE` 指定 HTTPS 缓存入口,按 `<SHA-256>/<归档名>` 定位文件。标签流水线使用发布桶的 `_sources/` 前缀;维护者填充缓存前须核对锁文件的原始 SHA-256,缓存同样计入桶容量预算。下载缓存失败时尝试锁定的上游地址;任一来源的哈希不符立即拒绝,不能放宽清单。未配置缓存的对应源码包直接使用上游。
可选的 `r2` 后端使用独立 Standard 桶与 HTTPS 自定义域名。`PEACH_R2_ACCESS_KEY_ID`、`PEACH_R2_SECRET_ACCESS_KEY`、`PEACH_R2_ENDPOINT` 放入受保护且隐藏的 CI 变量,环境作用域为 `release`;`PEACH_R2_BUCKET` 和 `PEACH_R2_PUBLIC_BASE` 指定发布目标。S3 传输复用 rclone,版本文件禁止覆盖;每份文件必须从公开入口完整下载并核对大小及 SHA-256 后,才更新公开版本索引和 Release 链接。
-GitLab 标签流水线通过 R2 暂存传递媒体工具、Windows 发行目录与 macOS 镜像,job artifacts 只保存 JSON 清单。中转需要上述 R2 变量;构建与验收任务使用 `release` 环境的 `prepare` 或 `verify` 动作,仅受保护标签能够取得凭据,普通分支回归不使用这些变量。构建期 rclone 按当前操作系统和 CPU 架构从官方固定版本归档安装并核对 SHA-256。
+GitLab 标签流水线通过 R2 暂存传递媒体工具、Windows 发行目录与 macOS 镜像,job artifacts 只保存 JSON 清单。中转需要上述 R2 变量;构建与验收任务使用 `release` 环境的 `prepare` 或 `verify` 动作,只有受保护的版本标签与受保护 master 上的版本提交演练能取得凭据,普通分支回归不使用这些变量。演练的暂存产物由 `release-rehearsal-cleanup` 在验收后回收。构建期 rclone 按当前操作系统和 CPU 架构从官方固定版本归档安装并核对 SHA-256。
暂存对象位于 `_ci/<项目 ID>/<流水线 ID>/<提交 SHA>/<产物组>/<作业 ID>/`,纳入同一桶的容量预算。每次重试使用自己的作业 ID,构建时间等元数据变化不会覆盖其他尝试的文件。下载任务先核对清单的版本、提交和流水线身份,再校验每份文件的大小与 SHA-256;产物路径限定在对应构建输出目录,不接收软链接或路径穿越。macOS 仅上传两种架构的 DMG、`build-info.json` 与 `SHA256SUMS.txt`,不传递展开应用。所有构建与独立制品验收成功后,发布任务才写入正式版本目录和公开版本索引;GitLab Release 创建成功后,只回收当前流水线的暂存前缀。失败流水线保留暂存以便排障和重试,不覆盖已有公开版本。
diff --git a/docs/TESTING.md b/docs/TESTING.md
index bdc2690e8..405450b9f 100644
--- a/docs/TESTING.md
+++ b/docs/TESTING.md
@@ -90,7 +90,7 @@ PR 计划按改动选择前端、vendor、wheel 与浏览器任务,文档 PR
Python 分片上传完整测试输出;浏览器分片上传 TAP、服务日志及已脱敏网络日志,保留 14 天。npm 审计遇到 high 或 critical 漏洞直接失败;只有临时网络错误最多请求三次,在前两次失败后分别等待 15 与 30 秒,未知响应不当成通过。
-GitGud 流水线(`.gitlab-ci.yml`)的 `verify-linux`、`verify-windows`、`verify-macos` 在分支与合并请求上跑改动域加本平台的平台敏感用例:基线是最近一条成功的主线流水线里、作为本提交祖先的那个提交,由 `scripts/ci_scope.py` 判定;先用正式入口跑 `auto --base <基线>`,选域与退回全量的规则同本机 `auto`,再用 `-Focus` 跑 `PLATFORM_TESTS` 里本平台的用例。取不到基线(首次推送、强推、接口不可达)、改了 `.gitlab-ci.yml` 本身,或改动落到必须全量的面时只跑一次全量。版本标签上的 `release-verify-*` 三平台各跑一次全量,构建与发布作业依赖它们。口径与平台集合的理由见 [ADR-0119](adr/0119-mainline-changed-scope-release-full-suite.md)。
+GitGud 流水线(`.gitlab-ci.yml`)的 `verify-linux`、`verify-windows`、`verify-macos` 在分支与合并请求上跑改动域加本平台的平台敏感用例:基线是最近一条成功的主线流水线里、作为本提交祖先的那个提交,由 `scripts/ci_scope.py` 判定;先用正式入口跑 `auto --base <基线>`,选域与退回全量的规则同本机 `auto`,再用 `-Focus` 跑 `PLATFORM_TESTS` 里本平台的用例。取不到基线(首次推送、强推、接口不可达)、改了 `.gitlab-ci.yml` 本身,或改动落到必须全量的面时只跑一次全量。版本提交(`chore(release): 版本 X`)的主线流水线是发布演练:`release-verify-*` 三平台各跑一次全量,媒体、构建、冒烟与镜像也都跑,只是不发布;`--ship` 等它全部通过才打标签,标签流水线只构建和发布([ADR-0121](adr/0121-release-rehearsal-and-atomic-publication.md))。口径与平台集合的理由见 [ADR-0119](adr/0119-mainline-changed-scope-release-full-suite.md)。
Release 在媒体工具编译前核对同 SHA 主线 CI,缓存键绑定媒体工具源码清单与构建脚本,缓存内容同时含二进制和对应源码包;制品与独立开箱验收仍是发布前置。Docker 只在镜像相关主线改动、匹配 PR、版本标签或手动请求时运行,验收后推送同一个本地镜像。版本发布与覆盖升级验收不取消进行中的同目标运行;覆盖升级需明确选择目标版本及完整构建提交。
@@ -145,6 +145,12 @@ macOS 按影响域验证:
主检出只按 ADR-0091 只读检查依赖;缺包或检查失败时以 `3` 退出,依赖变更由重启流程执行。找不到 uv、锁文件过期或同步失败也会停止测试。前端依赖与构建仍按上文安装。完成安装和编辑后再启动最终验证;运行中发生改动时,完整批次按下述规则保留未受影响的文件结果。
+会起浏览器或整套 vitest 的运行(全量、选中 `test_web_e2e` 或 `test_frontend_build` 的域、含 `frontend/` 选择器的定向调试)先排整机的重负载锁 `%LOCALAPPDATA%\peach-test-locks\test-machine.lock`(macOS/Linux 在 `~/.local/state/peach-test-locks/`),再拿仓库的全量锁。所有工作树、所有会话与本机 GitGud runner 的作业都排这一把:Windows runner 就是开发机,本机测试与 CI 作业叠在一起跑时,固定时限的用例会因负载偶发超时。runner 的构建、冒烟与镜像作业用 `scripts/machine_lock.py` 包一层拿同一把锁;验证作业经测试入口,入口自己拿。
+
+全量先单独跑一片秒级检查(`test_runner.GATE_TESTS`:仓库策略、编码、文案终态、上下文预算、源码断言棘轮、复杂度、Python lint、外发代理、仓库整洁与依赖清单,合计约 1 分钟),有一项不过就不起其余分片。
+
+全量失败后,入口从逐条结果里取出失败的 Python 用例、e2e 文件与 vitest 文件,不超过 6 个文件时用定向入口逐个单独重跑一次(父进程已持有整机锁,重跑不再排队)。单独重跑通过的,记录照样签为通过,名字写进记录的 `flaky` 字段并打印出来;单独重跑仍失败、失败面更大、或浏览器容器整个没起来的,按真实失败处理。同一用例反复出现在 `flaky` 里,说明它的等待写法或时限需要修。
+
本机全量验证与 `integrate` 共用 `full-suite.lock`,从取得代码和环境快照一直持有到记录写入完成。
锁忙时按到达先后排队:等待者在锁旁的 `full-suite.lock.queue/` 放一张以纳秒时刻开头的票,
前面没有活票时才去抢锁,拿到即撤票;提示里打印持有者与前面排队的个数。票主进程退出后,
@@ -190,7 +196,13 @@ macOS/Linux 的相同入口:
./scripts/test.sh auto --focus frontend/e2e/jav-detail.test.ts
```
-定向调试不签发记录,不能作为 `ready` / `integrate` 的验收证明。浏览器文件选择只在调试环境生效;正式入口拒绝残留的 `PEACH_TEST_FOCUS` / `PEACH_E2E_FILES`,防止局部运行被签成整域。浏览器文件失败或超时时保留它的完整 TAP 与服务日志。
+定向调试一个或几个 vitest 文件(路径从 `frontend/test/` 起),只跑这几个文件:
+
+```powershell
+& .\scripts\test.ps1 -Focus frontend/test/jav-card-title.test.ts
+```
+
+定向调试不签发记录,不能作为 `ready` / `integrate` 的验收证明。浏览器文件选择只在调试环境生效;正式入口拒绝残留的 `PEACH_TEST_FOCUS` / `PEACH_E2E_FILES` / `PEACH_VITEST_FILES`,防止局部运行被签成整域。浏览器文件失败或超时时保留它的完整 TAP 与服务日志。
浏览器沿用 Playwright 的默认进程模型与静音设置,不限制渲染进程数,也不把 GPU 或音频服务强制并入浏览器进程。原生浏览器启动限时 60 秒,页面主体等待限时 15 秒,每批限时 600 秒;进程启动与页面行为分别验收。需要任务状态桩的页面通过 `visit` 的准备回调在首次导航前安装响应,等对应的 React 主体出现后再检查任务状态;进行中的任务不作为主体等待态。页面主体等待失败时保留 URL、关闭状态、已收集的运行期问题与无障碍树,夹具关闭本次浏览器上下文。
diff --git a/docs/adr/0119-mainline-changed-scope-release-full-suite.md b/docs/adr/0119-mainline-changed-scope-release-full-suite.md
index c656ef240..62646bdd6 100644
--- a/docs/adr/0119-mainline-changed-scope-release-full-suite.md
+++ b/docs/adr/0119-mainline-changed-scope-release-full-suite.md
@@ -1,6 +1,6 @@
# ADR-0119:主线推送按改动选测,版本标签跑三平台全量
-- 状态:已采纳
+- 状态:已采纳;第四条由 [ADR-0121](0121-release-rehearsal-and-atomic-publication.md) 修订:三平台全量移到版本提交的发布演练里跑,标签流水线只构建和发布
- 日期:2026-10-24
- 关系:补充 [ADR-0012](0012-version-and-update-boundary.md)「版本号跟发布走」修订里「Test 没绿绝不打标签」一句:主线 Test 绿表示改动域与平台敏感用例通过,三平台全量改由标签流水线把关,其余照旧
diff --git a/docs/adr/0121-release-rehearsal-and-atomic-publication.md b/docs/adr/0121-release-rehearsal-and-atomic-publication.md
new file mode 100644
index 000000000..bf84a0595
--- /dev/null
+++ b/docs/adr/0121-release-rehearsal-and-atomic-publication.md
@@ -0,0 +1,62 @@
+# ADR-0121:版本提交先演练,标签只负责发布且全有或全无
+
+- 状态:已采纳
+- 日期:2026-10-24
+- 关系:修订 [ADR-0119](0119-mainline-changed-scope-release-full-suite.md) 第四条「版本标签流水线:三平台全量是发布门槛」;补充 [ADR-0012](0012-version-and-update-boundary.md) 的发布边界
+
+## 背景
+
+0.47.1 到 0.48.1 有四个版本标签没有发出完整制品,版本号作废了四个。失败点都在只有标签流水线才执行的地方:
+
+- macOS 产物清点没跟上新增的源码包;
+- 打标签之后才做的依赖审计;
+- 标签流水线里三平台全量的一次负载偶发。
+
+主线流水线不跑这些作业,所以主线绿了也说明不了「能发」。另外,`publish-docker` 只等 `validate-release` 与 Linux 全量:0.48.0 的 `build-macos` 失败了,`0.48.0` 镜像却已经推到 Docker Hub,`edge` 也指过去约 5 小时。标签流水线没通过,东西却发出去了一部分。
+
+## 决策
+
+**一、版本提交的主线流水线就是发布演练。** `release_tag.py --ship` 推上 master 的提交主题是 `chore(release): 版本 X`。`.gitlab-ci.yml` 的 `.release-rehearsal` 规则匹配这类提交,给作业设置 `PEACH_RELEASE_REHEARSAL=1`,然后跑完标签上除发布以外的全部作业:三平台全量(`release-verify-*`)、`windows-media`、`build-windows`、`build-macos`、`smoke-windows` 和 `docker-build`。最后由 `release-rehearsal-cleanup` 回收 R2 上的暂存产物。
+
+需要「标签」的脚本,在演练里改用提交里的版本号:
+
+- `ci_r2_artifacts.identity`;
+- `ci_windows.ps1` 的 `Assert-ReleaseRef`;
+- `smoke_update_tasks` 的安装验收主机核对。
+
+**二、演练全部通过才打标签。** `--ship` 等版本提交的流水线转绿。等待中任一作业失败(不限于原来的四个必需作业),就立即报错。打标签前,`GitLabForge.require_rehearsal` 核对 `REHEARSAL_JOBS` 每一项都在、都成功:规则没匹配上导致演练没跑,也不打标签。默认等待上限改为 4 小时,因为 macOS 的两个架构要从源码编译。
+
+**三、标签流水线只构建和发布,不再重跑全量。** `validate-release` 核对同一提交的主线流水线(含演练)已经成功,所以标签上不重跑三平台全量,构建、冒烟后直接发布。
+
+**四、发布全有或全无。** 镜像拆成两步:
+
+- `docker-build` 构建、冒烟,镜像留在这台 runner 上;
+- `publish-docker` 只负责推送,而且排在 `publish-release` 成功之后。
+
+`publish-release` 等 `validate-release`、两个平台的构建、Windows 冒烟和 `docker-build` 全部通过才运行。任何一步没过,安装包、R2、GitGud Release、Docker Hub 的版本号与 `edge` 一个都不动。`publish-release` 与 `publish-docker` 自身失败时,重试同一个作业即可,标签不用动。
+
+## 迁回 GitHub
+
+同一套判据在两个平台各有实现,迁回 GitHub 时不用重新设计:
+
+| 机制 | GitGud(`.gitlab-ci.yml`) | GitHub(`.github/workflows`) |
+| --- | --- | --- |
+| 演练触发 | `.release-rehearsal` 规则匹配版本提交主题,设置 `PEACH_RELEASE_REHEARSAL=1` | `release.yml` 也在 master 推送时触发;`validate` 只放行标签、手动运行与 `chore(release): 版本 ` 开头的提交,其余作业都依赖它 |
+| 打标签前核对 | `release_tag.require_rehearsal` 调 `GitLabForge.require_rehearsal`,核对同一条主线流水线里的演练作业 | `release_tag.require_rehearsal` 等同一提交推送触发的 Release 工作流跑完,结论必须是成功 |
+| 标签只发布 | `validate-release` 核对演练已通过,标签上不重跑全量 | 发布作业 `release`、`docker-publish` 只在标签上运行 |
+| 发布顺序 | `docker-build` 先验收;`publish-docker` 依赖 `publish-release` | `docker` 先验收;`docker-publish` 依赖 `release` |
+| `edge` 的语义 | 只由 `publish-docker` 推送,指向最新发布的版本 | `docker.yml` 只验收不推送;`edge` 只由 `docker-publish` 推送 |
+
+本机测试设施(快速门槛、隔离重跑、整机重负载锁)与平台无关。GitHub 托管 runner 每个作业一台独立机器,整机锁在那里不会产生等待;迁回后如果仍用自托管 Windows runner,用同一个 `scripts/machine_lock.py` 包住构建与冒烟。
+
+## 后果
+
+- 发版多一轮演练。演练与原来的标签全量同口径;标签流水线去掉了全量,两轮合计比「标签全量失败后改版本号重来」快,而且不再作废版本号。
+- 演练要用 `release` 环境的变量。master 是受保护分支,与版本标签同样能取得这些变量;如果项目把环境限制成只允许标签,演练的构建作业会在取凭据时失败,`--ship` 不会打标签,改环境设置后重跑即可。
+- 演练与标签各构建一次,产物不共用。两次构建来自同一个提交,标签流水线的构建失败只可能来自基础设施,重试作业即可。
+
+## 已拒绝方案
+
+- **在标签流水线里重用演练的产物**:可以省一次构建,但 R2 暂存按流水线编号核对身份,跨流水线取产物要另建一套身份校验。等构建时间成为瓶颈时再做。
+- **手动触发的候选流水线**:要多一个人工步骤,还得把候选结果和版本提交对上号。直接在版本提交的流水线里演练,`--ship` 等的本来就是它。
+- **标签失败后删标签、用同一版本号重打**:删远端标签不可逆,而且每次都要单独授权;演练把失败挡在打标签之前。
diff --git a/scripts/ci_r2_artifacts.py b/scripts/ci_r2_artifacts.py
index babb112d5..668076a81 100644
--- a/scripts/ci_r2_artifacts.py
+++ b/scripts/ci_r2_artifacts.py
@@ -25,9 +25,18 @@ GROUPS = {
}
+def rehearsal_tag() -> str:
+ """版本提交演练(ADR-0121)还没有标签:用提交里的版本号代替,流水线规则保证它是版本提交。"""
+ source = (Path(__file__).resolve().parents[1] / "src/peach/__init__.py").read_text(encoding="utf-8")
+ found = re.search(r'^__version__ = "([0-9]+\.[0-9]+\.[0-9]+)"$', source, re.M)
+ return "v" + found.group(1) if found else ""
+
+
def identity() -> dict:
result = {key: os.environ.get("CI_" + variable, "") for key, variable in
(("project", "PROJECT_ID"), ("pipeline", "PIPELINE_ID"), ("commit", "COMMIT_SHA"), ("tag", "COMMIT_TAG"))}
+ if os.environ.get("PEACH_RELEASE_REHEARSAL") == "1" and not result["tag"]:
+ result["tag"] = rehearsal_tag()
if (not all(re.fullmatch(r"[1-9][0-9]*", result[key]) for key in ("project", "pipeline"))
or not re.fullmatch(r"[0-9a-f]{40}", result["commit"])
or not re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+", result["tag"])
@@ -205,20 +214,25 @@ def transfer(root: Path, group: str, mode: str) -> dict:
def cleanup(root: Path) -> dict:
- """公开发布成功后只回收当前流水线的暂存前缀。"""
+ """公开发布成功后只回收当前流水线的暂存前缀。
+
+ 发布演练(ADR-0121)从不发布,暂存只是验收用的草稿,演练跑完直接回收。
+ """
build = identity()
+ rehearsal = os.environ.get("PEACH_RELEASE_REHEARSAL") == "1" and not os.environ.get("CI_COMMIT_TAG")
environment = rclone_environment()
executable = rclone_executable(root.resolve(strict=True))
remote = f"peach:{environment['PEACH_R2_BUCKET']}/{prefix(build)}"
with tempfile.TemporaryDirectory(prefix="peach-ci-r2-cleanup-") as directory:
configuration = Path(directory) / "rclone.conf"
configuration.touch()
- index = json.loads(subprocess.check_output(
- [executable, "cat", f"peach:{environment['PEACH_R2_BUCKET']}/releases.json", "--config", str(configuration)],
- env=environment, text=True, encoding="utf-8"))
- if not any(item.get("tag_name") == build["tag"] and item.get("commit_sha") == build["commit"]
- for item in index):
- raise ValueError("公开版本索引尚未登记当前提交,保留暂存产物")
+ if not rehearsal:
+ index = json.loads(subprocess.check_output(
+ [executable, "cat", f"peach:{environment['PEACH_R2_BUCKET']}/releases.json", "--config", str(configuration)],
+ env=environment, text=True, encoding="utf-8"))
+ if not any(item.get("tag_name") == build["tag"] and item.get("commit_sha") == build["commit"]
+ for item in index):
+ raise ValueError("公开版本索引尚未登记当前提交,保留暂存产物")
subprocess.run([executable, "purge", remote, "--config", str(configuration)], env=environment, check=True)
return {"cleaned_pipeline": build["pipeline"], "commit": build["commit"]}
diff --git a/scripts/ci_windows.ps1 b/scripts/ci_windows.ps1
index 2bb4f3e0c..936009065 100644
--- a/scripts/ci_windows.ps1
+++ b/scripts/ci_windows.ps1
@@ -1,4 +1,4 @@
-param([ValidateSet('verify', 'build', 'smoke', 'docker')][string]$Mode = 'verify',
+param([ValidateSet('verify', 'build', 'smoke', 'docker-build', 'docker-push')][string]$Mode = 'verify',
# verify 默认全量;changed 只跑改动域与平台敏感用例,供主线推送使用(ADR-0119)。
[ValidateSet('full', 'changed')][string]$Coverage = 'full')
$ErrorActionPreference = 'Stop'
@@ -7,6 +7,12 @@ function Assert-Exit([string]$Message) {
if ($LASTEXITCODE -ne 0) { throw $Message }
}
+# 标签流水线核对标签与版本号一致;版本提交的演练(ADR-0121)还没有标签,由流水线规则保证是版本提交。
+function Assert-ReleaseRef([string]$Version) {
+ if ($env:PEACH_RELEASE_REHEARSAL -eq '1') { return }
+ if ($env:CI_COMMIT_TAG -ne "v$Version") { throw 'Tag 与版本不一致' }
+}
+
if ($Mode -eq 'verify' -or $Mode -eq 'build') {
uv sync --locked --all-extras --python 3.14
Assert-Exit '锁定依赖安装失败'
@@ -42,7 +48,7 @@ if ($Mode -eq 'build') {
if (-not $env:PEACH_ISCC -and (Test-Path -LiteralPath $LocalCompiler)) { $env:PEACH_ISCC = $LocalCompiler }
$Version = (& .venv/Scripts/python.exe -c 'from peach import __version__; print(__version__)').Trim()
Assert-Exit '无法读取发布版本'
- if ($env:CI_COMMIT_TAG -ne "v$Version") { throw 'Tag 与版本不一致' }
+ Assert-ReleaseRef $Version
$Changes = git status --porcelain --untracked-files=all
Assert-Exit '无法核对提交状态'
if ($Changes) { throw '构建必须使用干净提交' }
@@ -67,25 +73,42 @@ if ($Mode -eq 'build') {
exit 0
}
-if ($Mode -eq 'docker') {
- # Linux 平台 Runner 不提供 Docker 守护进程;镜像在本机 Docker Desktop 构建、验收后发布。
+if ($Mode -eq 'docker-build' -or $Mode -eq 'docker-push') {
+ # Linux 平台 Runner 不提供 Docker 守护进程;镜像在本机 Docker Desktop 构建、验收,
+ # 留在本机等 publish-release 成功后再推(同一条流水线只有这一台 Windows runner)。
$Match = Select-String -LiteralPath src/peach/__init__.py -Pattern '^__version__ = "([^"]+)"'
if (-not $Match) { throw '无法读取发布版本' }
$Version = $Match.Matches[0].Groups[1].Value
- if ($env:CI_COMMIT_TAG -ne "v$Version") { throw 'Tag 与版本不一致' }
- if (-not $env:DOCKERHUB_USERNAME -or -not $env:DOCKERHUB_TOKEN) { throw '缺少 Docker Hub 发布变量' }
+ Assert-ReleaseRef $Version
$Endpoint = (docker context inspect --format '{{.Endpoints.docker.Host}}')
Assert-Exit '需要运行 Docker Desktop'
$Candidate = "peach-ci:$Version-$env:CI_PIPELINE_ID"
+}
+
+if ($Mode -eq 'docker-build') {
$Revision = (git rev-parse HEAD).Trim()
Assert-Exit '无法读取提交'
docker build --pull --label "org.opencontainers.image.version=$Version" --label "org.opencontainers.image.revision=$Revision" `
--build-arg PEACH_SENTRY_DSN --build-arg PEACH_LIBRARY_SENTRY_DSN --build-arg PEACH_REQUIRE_REPORTING=1 -t $Candidate .
Assert-Exit '镜像构建失败'
- & (Get-Command python.exe).Source -X utf8 scripts/smoke_docker.py $Candidate
- Assert-Exit '镜像验收失败'
- docker run --rm --entrypoint python $Candidate -c 'from peach import library_sharing, remote_reporting, reporting_identity; assert remote_reporting.dsn() and library_sharing.target() and reporting_identity.distribution() == ''docker'''
- Assert-Exit '镜像内 Sentry DSN 或分发标记缺失'
+ try {
+ & (Get-Command python.exe).Source -X utf8 scripts/smoke_docker.py $Candidate
+ Assert-Exit '镜像验收失败'
+ docker run --rm --entrypoint python $Candidate -c 'from peach import library_sharing, remote_reporting, reporting_identity; assert remote_reporting.dsn() and library_sharing.target() and reporting_identity.distribution() == ''docker'''
+ Assert-Exit '镜像内 Sentry DSN 或分发标记缺失'
+ } catch {
+ docker image rm $Candidate | Out-Null
+ throw
+ }
+ # 演练不推送,验收完就删;标签流水线留给 publish-docker 推送后再删。
+ if ($env:PEACH_RELEASE_REHEARSAL -eq '1') { docker image rm $Candidate | Out-Null }
+ exit 0
+}
+
+if ($Mode -eq 'docker-push') {
+ if (-not $env:DOCKERHUB_USERNAME -or -not $env:DOCKERHUB_TOKEN) { throw '缺少 Docker Hub 发布变量' }
+ docker image inspect $Candidate | Out-Null
+ Assert-Exit "本机没有 docker-build 验收过的 $Candidate;重试 docker-build 后再推送"
# 登录写进作业专用配置目录,不改动宿主用户的 Docker 登录。
$SavedConfig, $SavedHost = $env:DOCKER_CONFIG, $env:DOCKER_HOST
$env:DOCKER_HOST = $Endpoint
diff --git a/scripts/machine_lock.py b/scripts/machine_lock.py
new file mode 100644
index 000000000..fe57173d7
--- /dev/null
+++ b/scripts/machine_lock.py
@@ -0,0 +1,41 @@
+"""拿着整机的重负载锁运行一条命令,结束后释放。
+
+本机 CI runner 的构建、冒烟与镜像作业用它和本机测试排同一把锁(`test_evidence.machine_lock_path`);
+测试入口自己会拿这把锁,经测试入口的作业不要再套一层,否则会自己等自己。
+
+用法:python scripts/machine_lock.py [--wait 秒] -- 命令 参数...
+"""
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import subprocess
+import sys
+
+ROOT = Path(__file__).resolve().parents[1]
+if not __package__:
+ sys.path.insert(0, str(ROOT))
+ __package__ = "scripts"
+
+from . import test_evidence # noqa: E402
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("--wait", type=float, default=3600, help="最多等多少秒;CI 作业时限内足够让本机一轮全量跑完")
+ parser.add_argument("command", nargs=argparse.REMAINDER)
+ args = parser.parse_args(argv)
+ command = args.command[1:] if args.command[:1] == ["--"] else args.command
+ if not command:
+ parser.error("缺少要运行的命令")
+ lock = test_evidence.machine_lock_path()
+ try:
+ with test_evidence.held(lock, wait_seconds=args.wait, scope=" ".join(command)[:120], root=str(ROOT)):
+ return subprocess.run(command, check=False).returncode
+ except test_evidence.Timeout:
+ print(f"等了 {args.wait:g} 秒仍拿不到整机重负载锁({test_evidence.describe_holder(lock)})。", flush=True)
+ return 2
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/release_forge.py b/scripts/release_forge.py
index 9ce7faf06..4611c76fb 100644
--- a/scripts/release_forge.py
+++ b/scripts/release_forge.py
@@ -17,6 +17,10 @@ from peach.peach_proxy import read_url
REQUIRED_TEST_JOBS = frozenset({"verify-linux", "verify-windows", "verify-macos", "wheel-smoke"})
+#: 版本提交的主线流水线里演练的标签作业(ADR-0121):全部成功才打标签。规则没匹配上、演练没跑,
+#: 也不能当作通过。
+REHEARSAL_JOBS = frozenset({"release-verify-linux", "release-verify-windows", "release-verify-macos",
+ "windows-media", "build-windows", "build-macos", "smoke-windows", "docker-build"})
TERMINAL_STATUSES = frozenset({"success", "failed", "canceled", "skipped"})
#: 必需任务落在这些状态时,整条流水线等多久都不会转绿;重试会生成同名的新任务。
RED_JOB_STATUSES = frozenset({"failed", "canceled"})
@@ -90,10 +94,10 @@ class GitLabForge:
if status not in TERMINAL_STATUSES:
# 流水线整体是 running 时也可能有任务排着没人接,所以只要没结束就看任务。
jobs = self.jobs(repo, pipeline["id"])
+ # 任何一个不许失败的任务红了,整条就不会转绿:不只看必需的四个,演练里的构建也算。
run["failed_jobs"] = sorted(
- name for name in REQUIRED_TEST_JOBS
- if name in jobs and jobs[name]["status"] in RED_JOB_STATUSES
- and not jobs[name].get("allow_failure"))
+ name for name, job in jobs.items()
+ if job["status"] in RED_JOB_STATUSES and not job.get("allow_failure"))
run["queued_jobs"] = self.queued_jobs(repo, jobs.values())
return [run]
@@ -162,3 +166,12 @@ class GitLabForge:
or selected[name].get("commit", {}).get("id") != sha]
if bad:
raise ValueError("GitLab 发布验证缺少同一提交的必需成功任务:" + "、".join(bad))
+
+ def require_rehearsal(self, repo: str, pipeline: int) -> None:
+ """版本提交的流水线里,演练的标签作业必须都在且成功;缺一个都不打标签。"""
+ selected = self.jobs(repo, pipeline)
+ bad = [name for name in sorted(REHEARSAL_JOBS)
+ if name not in selected or selected[name]["status"] != "success"]
+ if bad:
+ raise ValueError("版本提交的发布演练没有全部通过,不打标签:" + "、".join(bad)
+ + "。检查 .gitlab-ci.yml 的演练规则是否匹配版本提交,修好后再跑一次 --ship --apply")
diff --git a/scripts/release_tag.py b/scripts/release_tag.py
index d810d402d..6d3642176 100644
--- a/scripts/release_tag.py
+++ b/scripts/release_tag.py
@@ -40,8 +40,9 @@ MASTER_WRITER = "peach.masterWriter"
#: 定版只该改这两份:版本号一行,变更日志一节。`--ship` 拒绝夹带别的文件。
RELEASE_FILES = (version_bump.VERSION_FILE, changelog.CHANGELOG)
-#: `--ship` 等 Test 工作流的默认秒数上限,以及两次查询之间的间隔。
-TEST_TIMEOUT = 1800.0
+#: `--ship` 等 Test 工作流的默认秒数上限,以及两次查询之间的间隔。版本提交的流水线里还有发布
+#: 演练(macOS 两个架构从源码编译,作业时限 2 小时),所以按整条演练定。
+TEST_TIMEOUT = 4 * 3600.0
TEST_POLL = 30.0
#: 任务排队多久仍没开始就判定等不到。空闲 runner 几秒内轮询接单,GitLab Runner 默认
#: 3 秒一轮,平台创建流水线也只要几秒;五分钟还没动静就不是正常排队,而是 runner
@@ -191,6 +192,34 @@ def await_test(repo: str, sha: str, *, timeout: float = TEST_TIMEOUT, poll: floa
sleep(poll)
+def require_rehearsal(repo: str, sha: str, checked: dict, *, timeout: float = TEST_TIMEOUT,
+ poll: float = TEST_POLL, sleep=time.sleep, clock=time.monotonic) -> None:
+ """版本提交的发布演练全部通过才打标签(ADR-0121)。
+
+ GitGud 上演练就在版本提交自己的主线流水线里,Test 绿了以后再核对演练作业都在、都成功;
+ GitHub 上演练是同一提交推送触发的 Release 工作流(不带标签时只构建、验收不发布),等它跑完。
+ """
+ if GITLAB:
+ GITLAB.require_rehearsal(repo, checked["id"])
+ return
+ start = clock()
+ while True:
+ runs = api(repo, f"actions/workflows/release.yml/runs?head_sha={sha}&event=push&per_page=100")["workflow_runs"]
+ latest = latest_run(runs, sha)
+ now = clock()
+ if latest is not None and latest.get("status") == "completed":
+ if latest.get("conclusion") != "success":
+ raise ValueError(f"版本提交的发布演练没有通过,不打标签:{latest.get('html_url', '')}")
+ return
+ if latest is None and now - start >= STALL_SECONDS:
+ raise ValueError(f"推送后 {(now - start) / 60:.0f} 分钟仍没有 {sha[:8]} 的 Release 演练记录;"
+ "检查 release.yml 的触发条件,演练跑起来后再跑一次 --ship --apply")
+ if now - start >= timeout:
+ raise ValueError(f"等了 {timeout / 60:.0f} 分钟,{sha[:8]} 的发布演练还没有结果;"
+ "版本提交已在 master 上,演练跑完后再跑一次 --ship --apply")
+ sleep(poll)
+
+
def verify(repo: str, sha: str) -> dict:
if GITLAB:
GITLAB.require_ancestor(repo, sha)
@@ -317,6 +346,7 @@ def ship(repo: str, *, apply: bool, timeout: float = TEST_TIMEOUT) -> dict:
if master_sha(repo) != sha:
command("git", "push", remote_url(repo), "refs/heads/master")
checked = await_test(repo, sha, timeout=timeout)
+ require_rehearsal(repo, sha, checked, timeout=timeout)
require_clean_dependencies()
command("git", "tag", "-a", tag, sha, "-m", f"Peach {tag} Windows 测试版")
command("git", "push", remote_url(repo), f"refs/tags/{tag}")
diff --git a/scripts/smoke_update_tasks.py b/scripts/smoke_update_tasks.py
index 67a9078ac..b5f44b311 100644
--- a/scripts/smoke_update_tasks.py
+++ b/scripts/smoke_update_tasks.py
@@ -22,7 +22,9 @@ def validate_installer_host():
gitlab_release = (os.environ.get('GITLAB_CI') == 'true'
and os.environ.get('CI_JOB_NAME') == 'smoke-windows'
and os.environ.get('CI_COMMIT_REF_PROTECTED') == 'true'
- and re.fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', os.environ.get('CI_COMMIT_TAG', '')))
+ # 版本标签,或受保护 master 上版本提交的发布演练(ADR-0121)。
+ and (re.fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', os.environ.get('CI_COMMIT_TAG', ''))
+ or os.environ.get('PEACH_RELEASE_REHEARSAL') == '1'))
assert os.environ.get('GITHUB_ACTIONS') == 'true' or gitlab_release, (
'安装包验收仅允许 GitHub Actions 或受保护版本标签的 GitLab Windows 验收作业')
import winreg
diff --git a/scripts/test_evidence.py b/scripts/test_evidence.py
index c8baeebac..e7152644b 100644
--- a/scripts/test_evidence.py
+++ b/scripts/test_evidence.py
@@ -266,7 +266,7 @@ def covers(record: dict, scopes: tuple[str, ...]) -> bool:
def write(root: Path, state: str, scopes: tuple[str, ...], *, success: bool,
elapsed: float, slowest: list, count: int, previous: dict | None = None,
context: dict | None = None, baseline: dict | None = None,
- cohort: str | None = None, coverage: dict | None = None) -> None:
+ cohort: str | None = None, coverage: dict | None = None, flaky: list | None = None) -> None:
folder = evidence_dir(root)
folder.mkdir(parents=True, exist_ok=True)
prior = previous if previous is not None else read(root, state)
@@ -287,7 +287,9 @@ def write(root: Path, state: str, scopes: tuple[str, ...], *, success: bool,
"cohort": cohort, "coverage": coverage or {},
"baseline": baseline.get("state") if baseline else None,
"failed_scopes": sorted(failed),
- "elapsed": elapsed, "count": count, "slowest": slowest}
+ "elapsed": elapsed, "count": count, "slowest": slowest,
+ # 全量里失败、单独重跑通过的用例;记录照样算通过,但名字留在这里可查。
+ "flaky": list(flaky or [])}
temporary = folder / f"{state}.{os.getpid()}.tmp"
temporary.write_text(json.dumps(record, ensure_ascii=False, indent=2), encoding="utf-8")
temporary.replace(folder / f"{state}.json")
@@ -298,6 +300,17 @@ def holder_path(lock_path: Path) -> Path:
return lock_path.with_name(lock_path.name + ".holder.json")
+def machine_lock_path() -> Path:
+ """整台机器共用的重负载锁:所有工作树、所有会话和本机 CI runner 的作业都排这一把。
+
+ 放在用户目录而不是仓库里:runner 的作业检出在别处,仓库级的全量锁管不到它;
+ 浏览器、vitest 与构建同时跑会把机器压满,固定时限的用例随之偶发超时。
+ """
+ base = os.environ.get("LOCALAPPDATA") or str(Path.home() / ".local" / "state")
+ # 不放进应用自己的 `%LOCALAPPDATA%\Peach`:那是已装 Peach 的目录,测试设施不往里写。
+ return Path(base) / "peach-test-locks" / "test-machine.lock"
+
+
QUEUE_POLL_SECONDS = 0.1
#: 没有 psutil 判不了票主是否还活着时,票的最长寿命。
QUEUE_TICKET_SECONDS = 7200
diff --git a/scripts/test_focus.py b/scripts/test_focus.py
index 1492da726..234e4fa0f 100644
--- a/scripts/test_focus.py
+++ b/scripts/test_focus.py
@@ -9,10 +9,14 @@ import unittest
from pathlib import Path
from unittest import mock
+VITEST_SUITE = "test_frontend_build.VitestTests.test_the_island_suite_passes"
-def selection(root: Path, selectors: list[str]) -> tuple[unittest.TestSuite, tuple[str, ...]]:
+
+def selection(root: Path, selectors: list[str]) -> tuple[unittest.TestSuite, tuple[str, ...], tuple[str, ...]]:
+ """定向选择器对应的用例;另返回要跑的浏览器文件与 vitest 文件(相对 `frontend/`)。"""
suite = unittest.TestSuite()
browsers = []
+ islands = []
names = []
for selector in dict.fromkeys(selectors):
if selector.startswith("frontend/e2e/") and selector.endswith(".test.ts"):
@@ -20,33 +24,43 @@ def selection(root: Path, selectors: list[str]) -> tuple[unittest.TestSuite, tup
if not path.is_relative_to((root / "frontend/e2e").resolve()) or not path.is_file():
raise ValueError(f"浏览器用例不存在或超出 e2e 目录:{selector}")
browsers.append(path.relative_to(root / "frontend").as_posix())
+ elif selector.startswith("frontend/test/") and selector.endswith((".test.ts", ".test.tsx")):
+ path = (root / selector).resolve()
+ if not path.is_relative_to((root / "frontend/test").resolve()) or not path.is_file():
+ raise ValueError(f"vitest 用例不存在或超出 test 目录:{selector}")
+ islands.append(path.relative_to(root / "frontend").as_posix())
else:
name = selector.removeprefix("tests/").removesuffix(".py")
if not re.fullmatch(r"test_\w+(?:\.\w+)*", name, flags=re.ASCII) \
or not (root / "tests" / (name.split(".")[0] + ".py")).is_file():
- raise ValueError(f"用例必须是 tests/test_*.py、Python 用例名或 frontend/e2e/*.test.ts:{selector}")
+ raise ValueError("用例必须是 tests/test_*.py、Python 用例名、frontend/e2e/*.test.ts "
+ f"或 frontend/test/**/*.test.ts(x):{selector}")
names.append(name)
if "e2e/setup.test.ts" in browsers:
names.append("test_web_e2e.SetupE2ETests")
if any(name != "e2e/setup.test.ts" for name in browsers):
names.append("test_web_e2e.WebE2ESmokeTests")
+ if islands:
+ names.append(VITEST_SUITE)
sys.path[:0] = [str(root), str(root / "tests")]
try:
for name in dict.fromkeys(names):
suite.addTests(unittest.defaultTestLoader.loadTestsFromName(name))
finally:
del sys.path[:2]
- return suite, tuple(browsers)
+ return suite, tuple(browsers), tuple(islands)
def run(runner, args) -> int:
try:
- suite, browsers = selection(runner.ROOT, args.focus)
+ suite, browsers, islands = selection(runner.ROOT, args.focus)
except ValueError as error:
print(error, file=sys.stderr)
return 2
print("定向调试:只运行指定用例,不签发正式记录;提交前仍需 auto。", flush=True)
flags = {"PEACH_TEST_FOCUS": "1", "PEACH_E2E_FILES": json.dumps(browsers)} if browsers else {}
+ if islands:
+ flags["PEACH_VITEST_FILES"] = json.dumps(islands)
with mock.patch.dict(os.environ, flags):
with runner.test_history.running():
result = unittest.TextTestRunner(verbosity=2, resultclass=runner.TimedResult).run(suite)
diff --git a/scripts/test_runner.py b/scripts/test_runner.py
index ccf55fbec..5caed4368 100644
--- a/scripts/test_runner.py
+++ b/scripts/test_runner.py
@@ -970,21 +970,91 @@ def environment_preflight(scopes: tuple[str, ...], timings: Path | None = None,
raise SystemExit(3)
+#: 分片之前单独先跑的秒级检查(合计约 1 分钟):仓库策略、ADR 编号、复杂度与依赖清单。
+#: 这几类问题以前要等 20 分钟的全量才报出来;有一项不过就不再起其余分片。
+GATE_TESTS = frozenset({
+ "test_script_policy", "test_subprocess_encoding", "test_copy_final_state", "test_context_budget",
+ "test_source_assertion_ratchet", "test_complexity_ratchet", "test_python_lint",
+ "test_outbound_proxy_policy", "test_repo_hygiene", "test_dependency_policy",
+})
+#: 失败后单独重跑的文件数上限。失败面更大多半是真回归,重跑只会再白等一轮。
+RERUN_LIMIT = 6
+#: 浏览器与 vitest 的容器用例:它们的失败由各自的逐条结果说明,重跑只重跑那几个文件。
+E2E_CONTAINER = "test_web_e2e."
+VITEST_CONTAINER = "test_frontend_build.VitestTests.test_the_island_suite_passes"
+#: 本次运行里失败、单独重跑通过的用例;`verified_run` 写进测试记录。
+FLAKY: list[str] = []
+
+
+def rerun_selectors(rows: list[dict]) -> list[str] | None:
+ """失败的逐条结果对应的定向选择器;分不出具体文件或失败面过大时返回 None。"""
+ failed = [str(row.get("id", "")) for row in rows if row.get("outcome") in {"fail", "error"}]
+ e2e = sorted({item.split(" > ")[0].removeprefix("e2e:") for item in failed if item.startswith("e2e:")})
+ vitest = sorted({item.split(" > ")[0].removeprefix("vitest:") for item in failed if item.startswith("vitest:")})
+ python = []
+ for item in failed:
+ if item.startswith(("e2e:", "vitest:")):
+ continue
+ name = item.split(" ")[0]
+ if (name.startswith(E2E_CONTAINER) and e2e) or (name == VITEST_CONTAINER and vitest):
+ continue
+ if not re.fullmatch(r"test_\w+(?:\.\w+)+", name) or name.startswith(E2E_CONTAINER) or name == VITEST_CONTAINER:
+ # 类级准备失败、或浏览器容器整个没起来:没有可单独重跑的那一条。
+ return None
+ python.append(name)
+ if not failed or len({name.split(".")[0] for name in python}) + len(e2e) + len(vitest) > RERUN_LIMIT:
+ return None
+ return [*dict.fromkeys(python), *(f"frontend/{name}" for name in e2e), *(f"frontend/{name}" for name in vitest)]
+
+
+def rerun_failures(rows: list[dict], spawn=subprocess.run) -> bool:
+ """把全量里失败的文件单独重跑一次:通过的记为偶发,仍失败或无法隔离的照常算失败。"""
+ selectors = rerun_selectors(rows)
+ if not selectors:
+ return False
+ print(f"单独重跑失败的 {len(selectors)} 项,排除负载造成的偶发:{', '.join(selectors)}", flush=True)
+ command = [sys.executable, str(ROOT / "scripts" / "test_runner.py"), "--machine-lock-held",
+ *(item for selector in selectors for item in ("--focus", selector))]
+ if spawn(command, cwd=str(ROOT), check=False).returncode != 0:
+ print("单独重跑仍失败:按真实失败处理。", flush=True)
+ return False
+ FLAKY[:] = selectors
+ print(f"偶发通过(全量失败、单独重跑通过,已写入测试记录):{', '.join(selectors)}", flush=True)
+ return True
+
+
def run_local_suite(scopes: tuple[str, ...], jobs: int, label: str | None = None) -> tuple[bool, int, list]:
"""按调用方选择的并发运行,串行全量通过分片释放测试缓存。"""
chosen = {path for scope in scopes for path in selected_files(scope)}
+ FLAKY.clear()
if (jobs > 1 or "full" in scopes) and len(chosen) > 1:
- shard_count = min(len(chosen), 4 * jobs)
- print(f"本机分片:{shard_count} 片、同时 {min(jobs, shard_count)} 个子进程,按实测耗时先长后短", flush=True)
- plan = shard_plan((path.name for path in chosen), shard_count, load_file_seconds())
+ names = sorted(path.name for path in chosen)
+ gate = [name for name in names if name.removesuffix(".py") in GATE_TESTS]
+ rest = [name for name in names if name not in gate] if gate and len(gate) < len(names) else names
measured: dict[str, float] = {}
rows: list[dict] = []
- outcome = run_shards(scopes, jobs=jobs, shard_count=shard_count, measured=measured, plan=plan,
- outcomes=rows)
+ count, timings = 0, []
+ if rest is not names:
+ print(f"快速门槛:先跑 {len(gate)} 个秒级检查,有一项不过就不起其余分片", flush=True)
+ passed, count, timings = run_shards(scopes, jobs=1, shard_count=1, measured=measured,
+ plan=[gate], outcomes=rows)
+ if not passed:
+ print("快速门槛未通过,其余测试本轮不跑。", flush=True)
+ save_file_seconds(measured)
+ test_history.append(ROOT, scope=label or " ".join(scopes), rows=rows,
+ files=[name.removesuffix(".py") for name in gate])
+ return False, count, timings
+ shard_count = min(len(rest), 4 * jobs)
+ print(f"本机分片:{shard_count} 片、同时 {min(jobs, shard_count)} 个子进程,按实测耗时先长后短", flush=True)
+ plan = shard_plan(rest, shard_count, load_file_seconds())
+ passed, more, slow = run_shards(scopes, jobs=jobs, shard_count=shard_count, measured=measured, plan=plan,
+ outcomes=rows)
save_file_seconds(measured)
test_history.append(ROOT, scope=label or " ".join(scopes), rows=rows,
files=[path.stem for path in sorted(chosen)])
- return outcome
+ if not passed:
+ passed = rerun_failures(rows)
+ return passed, count + more, [*timings, *slow]
suite = build_suite(*scopes)
with test_history.running():
result = unittest.TextTestRunner(verbosity=2, resultclass=TimedResult).run(suite)
@@ -1038,6 +1108,7 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument("--jobs", type=resolve_jobs, default=1,
help="本机同时跑几个分片子进程;auto 按核数定。默认串行,正式入口传 auto")
parser.add_argument("--list-scopes", action="store_true")
+ parser.add_argument("--machine-lock-held", action="store_true", help=argparse.SUPPRESS)
args = parser.parse_args(argv)
if not 0 <= args.lock_timeout <= 1800:
parser.error("锁等待时间必须在 0~1800 秒之间")
@@ -1050,9 +1121,14 @@ def main(argv: list[str] | None = None) -> int:
if args.shard_count != 1:
parser.error("定向调试不能与分片一起使用")
from . import test_focus
- return test_focus.run(sys.modules[__name__], args)
- if os.environ.get("PEACH_TEST_FOCUS") or os.environ.get("PEACH_E2E_FILES"):
- parser.error("定向调试环境不能签发正式记录;请清除 PEACH_TEST_FOCUS 与 PEACH_E2E_FILES")
+ heavy = any(selector.startswith("frontend/") for selector in args.focus)
+ try:
+ with machine_guard(args, heavy):
+ return test_focus.run(sys.modules[__name__], args)
+ except test_evidence.Timeout as error:
+ return busy(error)
+ if os.environ.get("PEACH_TEST_FOCUS") or os.environ.get("PEACH_E2E_FILES") or os.environ.get("PEACH_VITEST_FILES"):
+ parser.error("定向调试环境不能签发正式记录;请清除 PEACH_TEST_FOCUS、PEACH_E2E_FILES 与 PEACH_VITEST_FILES")
requested: tuple[str, ...] = tuple(dict.fromkeys(args.scopes or ("auto",)))
if "auto" in requested and len(requested) > 1:
parser.error("auto 不能与别的域同时指定")
@@ -1067,26 +1143,48 @@ def main(argv: list[str] | None = None) -> int:
environment_preflight(scopes, args.timings, shard_count=args.shard_count)
files = {path for scope in scopes for path in selected_files(scope)}
print(f"Peach test scope: {' '.join(scopes)} ({len(files)} files)", flush=True)
- if args.shard_count > 1:
- # CI 的每片独立 runner;局部分片绝不签发本机全量证明。
+ if args.shard_count > 1 or args.shard_plan is not None:
+ # CI 的每片独立 runner,或父进程派出的分片(含只有一片的快速门槛);分片绝不签发本机记录。
return run_shard_child(args, requested, scopes, files)
if ci_frontend_delegated():
print("CI 前端工具检查由 web-bundle 验证;本次不签发本机测试记录。", flush=True)
success, count, _ = run_local_suite(scopes, args.jobs, f"{' '.join(requested)}:{' '.join(scopes)}")
return 0 if success and count > 0 else 1
+ heavy = "full" in scopes or any(path.stem in HEAVY_FILES for path in files)
try:
- guard = test_evidence.held(test_evidence.evidence_dir(ROOT) / "full-suite.lock",
- wait_seconds=args.lock_timeout,
- scope=" ".join(scopes), root=str(ROOT)) \
- if "full" in scopes else nullcontext()
- with guard:
- return verified_run(args, requested, scopes, files)
+ # 先拿整机的重负载锁,再拿仓库的全量锁;`integrate` 只拿后者,顺序固定就不会互相等死。
+ with machine_guard(args, heavy, scope=" ".join(scopes)):
+ guard = test_evidence.held(test_evidence.evidence_dir(ROOT) / "full-suite.lock",
+ wait_seconds=args.lock_timeout,
+ scope=" ".join(scopes), root=str(ROOT)) \
+ if "full" in scopes else nullcontext()
+ with guard:
+ return verified_run(args, requested, scopes, files)
except test_evidence.Timeout as error:
- lock = Path(error.lock_file)
- what = "本仓库全量测试" if lock.name == "full-suite.lock" else "相同状态的验证"
- print(f"{what}正在运行({test_evidence.describe_holder(lock)}),请等待该次结果。",
- flush=True)
- return 2
+ return busy(error)
+
+
+#: 会起浏览器或整套 vitest 的测试文件。选中它们的运行与全量一样要排整机的重负载锁。
+HEAVY_FILES = frozenset({"test_web_e2e", "test_frontend_build"})
+
+
+def machine_guard(args, heavy: bool, **note):
+ """重负载运行排整机锁;父进程已持有时(全量里的隔离重跑)不再重复等待。
+
+ 用例里调用运行器(测运行器自己的那些用例)时外层正式运行已持有这把锁,不能再去等它。
+ """
+ if not heavy or args.machine_lock_held or test_history.nested():
+ return nullcontext()
+ return test_evidence.held(test_evidence.machine_lock_path(), wait_seconds=args.lock_timeout,
+ root=str(ROOT), **note)
+
+
+def busy(error) -> int:
+ lock = Path(error.lock_file)
+ what = {"full-suite.lock": "本仓库全量测试", "test-machine.lock": "本机另一项重负载测试或 CI 作业"}.get(
+ lock.name, "相同状态的验证")
+ print(f"{what}正在运行({test_evidence.describe_holder(lock)}),请等待该次结果。", flush=True)
+ return 2
def scoped_baseline(record, scopes, needed):
@@ -1158,6 +1256,7 @@ def verified_run(args, requested, scopes, files) -> int:
print(f"实际运行:{' '.join(scopes) or '无需补测'}({actual} 个测试文件)", flush=True)
folder = test_evidence.evidence_dir(ROOT)
(folder / f"{state}.json").unlink(missing_ok=True)
+ FLAKY.clear()
started = time.monotonic()
passed, count, timings = run_local_suite(scopes, args.jobs, f"{' '.join(requested)}:{' '.join(scopes)}") \
if scopes else (True, 0, [])
@@ -1171,7 +1270,8 @@ def verified_run(args, requested, scopes, files) -> int:
test_evidence.write(ROOT, state, scopes, success=success, previous=previous,
context=context, baseline=baseline,
cohort=cohort, coverage=coverage,
- elapsed=time.monotonic() - started, slowest=slowest, count=count)
+ elapsed=time.monotonic() - started, slowest=slowest, count=count,
+ flaky=FLAKY if success else None)
for seconds, name in slowest[:5]:
print(f"慢测试 {seconds:.3f}s:{name}", flush=True)
if not stable:
diff --git a/tests/test_frontend_build.py b/tests/test_frontend_build.py
index afc3a1b10..9cf2dda1d 100644
--- a/tests/test_frontend_build.py
+++ b/tests/test_frontend_build.py
@@ -20,6 +20,7 @@ from __future__ import annotations
import hashlib
from html.parser import HTMLParser
import json
+import os
from pathlib import Path
import re
import shutil
@@ -410,12 +411,14 @@ class VitestTests(unittest.TestCase):
def test_the_island_suite_passes(self):
npm = self._npm("vitest", "vitest")
+ # 定向入口(含全量失败后的单独重跑)只跑指定的几个文件,路径相对 frontend/。
+ selected = json.loads(os.environ.get("PEACH_VITEST_FILES", "[]"))
# 另挂一个 JSON 报告器,把逐条结果交给测试结果历史;控制台输出与判据不变。
with tempfile.TemporaryDirectory() as folder:
report = Path(folder) / "vitest.json"
completed = subprocess.run(
[npm, "--prefix", str(FRONTEND), "test", "--silent", "--",
- "--reporter=default", "--reporter=json", f"--outputFile.json={report}"],
+ "--reporter=default", "--reporter=json", f"--outputFile.json={report}", *selected],
capture_output=True, text=True, encoding="utf-8", errors="replace",
cwd=str(FRONTEND), check=False)
if report.is_file():
@@ -430,7 +433,7 @@ class VitestTests(unittest.TestCase):
self.assertNotRegex(plain, r"Tests\s+0 passed", "vitest 一个用例都没跑")
counted = re.search(r"Tests\s+(\d+) passed", plain)
assert counted is not None
- self.assertGreaterEqual(int(counted.group(1)), 10, output)
+ self.assertGreaterEqual(int(counted.group(1)), 1 if selected else 10, output)
class BrowserSuiteWorkflowTests(unittest.TestCase):
diff --git a/tests/test_scripts.py b/tests/test_scripts.py
index c91b377b1..6629439ff 100644
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -221,7 +221,8 @@ class UpdateTaskSmokeTests(unittest.TestCase):
def test_protected_gitlab_release_and_actions_require_an_unregistered_host(self):
gitlab = {'GITLAB_CI': 'true', 'CI_JOB_NAME': 'smoke-windows',
'CI_COMMIT_REF_PROTECTED': 'true', 'CI_COMMIT_TAG': 'v1.2.3'}
- for environment in ({'GITHUB_ACTIONS': 'true'}, gitlab):
+ rehearsal = {**gitlab, 'CI_COMMIT_TAG': '', 'PEACH_RELEASE_REHEARSAL': '1'}
+ for environment in ({'GITHUB_ACTIONS': 'true'}, gitlab, rehearsal):
with self.subTest(environment=environment), mock.patch.dict(os.environ, environment, clear=True):
registry = SimpleNamespace(HKEY_CURRENT_USER=object(),
OpenKey=mock.Mock(side_effect=FileNotFoundError))
@@ -4949,10 +4950,22 @@ class R2ArtifactTransferTests(unittest.TestCase):
def test_unprotected_or_incomplete_build_identity_is_rejected(self):
module = load_script("ci_r2_artifacts")
- for patch in ({"CI_COMMIT_REF_PROTECTED": "false"}, {"CI_PIPELINE_ID": "../other"}, {"CI_COMMIT_TAG": "master"}):
+ for patch in ({"CI_COMMIT_REF_PROTECTED": "false"}, {"CI_PIPELINE_ID": "../other"}, {"CI_COMMIT_TAG": "master"},
+ {"CI_COMMIT_TAG": ""}):
with mock.patch.dict(os.environ, {**self.environment(), **patch}, clear=True), self.assertRaises(ValueError):
module.identity()
+ def test_a_release_rehearsal_stages_under_the_version_of_its_commit(self):
+ """版本提交演练(ADR-0121)还没有标签:暂存身份取提交里的版本号,受保护的要求不变。"""
+ module = load_script("ci_r2_artifacts")
+ rehearsal = {**self.environment(), "CI_COMMIT_TAG": "", "PEACH_RELEASE_REHEARSAL": "1"}
+ with mock.patch.dict(os.environ, rehearsal, clear=True):
+ self.assertEqual(module.identity()["tag"], module.rehearsal_tag())
+ self.assertRegex(module.rehearsal_tag(), r"^v\d+\.\d+\.\d+$")
+ with mock.patch.dict(os.environ, {**rehearsal, "CI_COMMIT_REF_PROTECTED": "false"}, clear=True), \
+ self.assertRaises(ValueError):
+ module.identity()
+
class R2ReleaseTests(unittest.TestCase):
def test_publication_preserves_existing_index_and_verifies_published_metadata(self):
@@ -5233,7 +5246,7 @@ class MainlineScopeTests(unittest.TestCase):
from scripts import ci_scope, test_focus
for platform, selectors in ci_scope.PLATFORM_TESTS.items():
with self.subTest(platform=platform):
- suite, browsers = test_focus.selection(ROOT, list(selectors))
+ suite, browsers, _islands = test_focus.selection(ROOT, list(selectors))
self.assertGreater(suite.countTestCases(), 0)
self.assertIn("e2e/setup.test.ts", browsers)
self.assertEqual(len(selectors), len(set(selectors)))
@@ -5393,6 +5406,24 @@ class GitLabReleaseTests(unittest.TestCase):
run, _ = self.running(jobs, {})
self.assertEqual(run["failed_jobs"], ["verify-linux"], "重试过的以最新一次为准")
+ def test_a_failed_rehearsal_build_stops_the_wait_too(self):
+ """版本提交的流水线里有发布演练:构建红了整条也不会转绿,不能只盯四个必需任务。"""
+ jobs = [self.job(1, "verify-linux", "running", started_at="t", runner={"id": 1}),
+ self.job(2, "build-macos", "failed", ("peach-macos",))]
+ run, _ = self.running(jobs, {})
+ self.assertEqual(run["failed_jobs"], ["build-macos"])
+
+ def test_tagging_requires_every_rehearsal_job_to_have_passed(self):
+ from scripts.release_forge import REHEARSAL_JOBS
+ forge = self.forge()
+ passed = [self.job(index, name, "success") for index, name in enumerate(sorted(REHEARSAL_JOBS), 1)]
+ with mock.patch.object(forge, "api", return_value=passed):
+ forge.require_rehearsal("owner/repo", 7)
+ for broken in (passed[1:], [{**passed[0], "status": "failed"}, *passed[1:]]):
+ with self.subTest(jobs=len(broken)), mock.patch.object(forge, "api", return_value=broken), \
+ self.assertRaisesRegex(ValueError, "发布演练没有全部通过"):
+ forge.require_rehearsal("owner/repo", 7)
+
def test_mainline_membership_uses_the_git_commit_graph(self):
forge = self.forge()
forge.command.side_effect = ["", self.SHA, "ancestor"]
@@ -5709,7 +5740,7 @@ class ShipTests(unittest.TestCase):
return mock.patch.object(self.release, "command",
side_effect=lambda *args, **_: answers.get(args, ""))
- def _api(self, *, tags=(), remote="old", runs=(SUCCESS,)):
+ def _api(self, *, tags=(), remote="old", runs=(SUCCESS,), rehearsal=(SUCCESS,)):
def answer(_repo, endpoint):
if endpoint.startswith("git/matching-refs/tags/"):
return list(tags)
@@ -5717,6 +5748,8 @@ class ShipTests(unittest.TestCase):
return {"object": {"sha": remote}}
if endpoint.startswith("actions/workflows/test.yml/runs"):
return {"workflow_runs": list(runs)}
+ if endpoint.startswith("actions/workflows/release.yml/runs"):
+ return {"workflow_runs": list(rehearsal)}
raise AssertionError(f"没预料到的调用:{endpoint}")
return mock.patch.object(self.release, "api", side_effect=answer)
@@ -5734,6 +5767,28 @@ class ShipTests(unittest.TestCase):
if args[:2] in {("git", "add"), ("git", "commit"), ("git", "push"), ("git", "-c")}
or args[:3] == ("git", "tag", "-a")]
+ def test_an_incomplete_release_rehearsal_stops_before_the_tag(self):
+ """GitLab 上版本提交的流水线带发布演练:Test 绿了但演练缺作业或没过,不打标签。"""
+ forge = mock.Mock()
+ forge.require_rehearsal.side_effect = ValueError("版本提交的发布演练没有全部通过,不打标签:build-macos")
+ with self._shell() as command, self._version(), self._document(), \
+ mock.patch.object(self.release, "GITLAB", forge), \
+ mock.patch.object(self.release, "master_sha", return_value="abc"), \
+ mock.patch.object(self.release, "await_test", return_value={**self.SUCCESS, "id": 7}), \
+ self.assertRaisesRegex(ValueError, "发布演练"):
+ self.release.ship("owner/repo", apply=True)
+ forge.require_rehearsal.assert_called_once_with("owner/repo", 7)
+ self.assertFalse([args for args in self._writes(command) if args[:3] == ("git", "tag", "-a")])
+
+ def test_on_github_a_red_release_rehearsal_stops_before_the_tag(self):
+ """迁回 GitHub 时演练是同一提交推送触发的 Release 工作流:它没过同样不打标签。"""
+ red = {**self.SUCCESS, "conclusion": "failure", "html_url": "rehearsal-url"}
+ with self._shell() as command, self._api(remote="abc", rehearsal=(red,)), self._version(), \
+ self._document(), mock.patch.object(self.release, "GITLAB", None), \
+ self.assertRaisesRegex(ValueError, "发布演练没有通过.*rehearsal-url"):
+ self.release.ship("owner/repo", apply=True)
+ self.assertFalse([args for args in self._writes(command) if args[:3] == ("git", "tag", "-a")])
+
def test_a_plan_without_apply_writes_nothing(self):
with self._shell() as command, self._api(), self._version(), self._document():
result = self.release.ship("owner/repo", apply=False)
diff --git a/tests/test_test_evidence.py b/tests/test_test_evidence.py
index fdfc5cc86..ca6dcf057 100644
--- a/tests/test_test_evidence.py
+++ b/tests/test_test_evidence.py
@@ -547,6 +547,7 @@ class VerificationTests(unittest.TestCase):
with redirect_stderr(io.StringIO()), redirect_stdout(io.StringIO()), \
mock.patch.object(runner, "ROOT", self.repo), \
mock.patch.object(runner, "build_suite", side_effect=AssertionError("父进程不该自己跑")), \
+ mock.patch.object(runner, "GATE_TESTS", frozenset()), \
mock.patch.object(runner, "run_shards", return_value=(True, 3, [(0.5, "x"), (0.1, "y")])) as shards:
self.assertEqual(runner.main(["--scope", "checks", "--jobs", "2"]), 0)
shards.assert_called_once()
@@ -567,6 +568,7 @@ class VerificationTests(unittest.TestCase):
with redirect_stderr(io.StringIO()), redirect_stdout(io.StringIO()), \
mock.patch.object(runner, "ROOT", self.repo), \
mock.patch.object(runner, "build_suite", side_effect=AssertionError("父进程不加载全量用例")), \
+ mock.patch.object(runner, "GATE_TESTS", frozenset()), \
mock.patch.object(runner, "run_shards", return_value=(True, 7, [])) as shards:
self.assertEqual(runner.main(["--scope", "full", "--jobs", "1"]), 0)
shards.assert_called_once()
@@ -576,6 +578,28 @@ class VerificationTests(unittest.TestCase):
self.assertTrue(evidence.covers(record, ("full",)))
self.assertEqual(record["count"], 7)
+ def test_a_failure_that_passes_alone_signs_the_record_and_names_the_flaky_file(self):
+ """全量里挂、单独重跑过的:记录照样算通过,偶发的文件名留在记录里;单独也挂就没有记录。"""
+ def shards(*_args, outcomes, **_kwargs):
+ outcomes.append({"id": "e2e:e2e/folder-layout.test.ts > 目录 > 滚到底", "outcome": "fail",
+ "seconds": 2.0, "message": "height 0"})
+ return False, 7, []
+
+ rerun = runner.rerun_failures
+ for code, signed in ((0, True), (1, False)):
+ spawn = mock.Mock(return_value=mock.Mock(returncode=code))
+ with self.subTest(rerun_exit=code), \
+ redirect_stderr(io.StringIO()), redirect_stdout(io.StringIO()), \
+ mock.patch.object(runner, "ROOT", self.repo), \
+ mock.patch.object(runner, "GATE_TESTS", frozenset()), \
+ mock.patch.object(runner, "run_shards", side_effect=shards), \
+ mock.patch.object(runner, "rerun_failures", lambda rows, spawn=spawn: rerun(rows, spawn=spawn)):
+ self.assertEqual(runner.main(["--scope", "full", "--jobs", "1", "--fresh"]), 0 if signed else 1)
+ record = evidence.read(self.repo, evidence.key(self.repo))
+ self.assertEqual(evidence.covers(record, ("full",)), signed)
+ if signed:
+ self.assertEqual(record["flaky"], ["frontend/e2e/folder-layout.test.ts"])
+
def test_full_baseline_requires_only_new_scopes_without_extending_its_age(self):
original = self.certify(self.repo, ("full",))
stamp = evidence.read(self.repo, original)["validated"]["full"]
diff --git a/tests/test_test_planning.py b/tests/test_test_planning.py
index 47c6787c2..09717292a 100644
--- a/tests/test_test_planning.py
+++ b/tests/test_test_planning.py
@@ -592,3 +592,93 @@ class TestPlanningTests(unittest.TestCase):
self.assertIsNone(connection.execute("SELECT name FROM sqlite_master WHERE name='isolation_probe'").fetchone())
finally:
connection.close()
+
+
+class GateAndFlakyRerunTests(unittest.TestCase):
+ """全量先过秒级门槛;全量里失败的少数文件单独重跑,过了记为偶发,并排整机锁。"""
+
+ @staticmethod
+ def fail(test_id):
+ return {'id': test_id, 'outcome': 'fail', 'seconds': 1.0, 'message': 'x'}
+
+ def test_failed_rows_name_the_files_to_rerun_and_refuse_what_cannot_be_isolated(self):
+ rows = [self.fail('e2e:e2e/folder-layout.test.ts > 目录 > 滚到底'),
+ self.fail('test_web_e2e.WebE2ESmokeTests.test_every_route_holds_the_layout_and_runtime_invariants'),
+ self.fail('vitest:test/react/follow-feed.test.tsx > 页面 > 加载更多'),
+ self.fail(runner.VITEST_CONTAINER),
+ self.fail('test_feeds_web.FeedsWebTests.test_second_round'),
+ {'id': 'test_ok.Case.test_pass', 'outcome': 'pass', 'seconds': 0.1}]
+ self.assertEqual(runner.rerun_selectors(rows), [
+ 'test_feeds_web.FeedsWebTests.test_second_round', 'frontend/e2e/folder-layout.test.ts',
+ 'frontend/test/react/follow-feed.test.tsx'])
+ self.assertIsNone(runner.rerun_selectors([self.fail('setUpClass (test_web_e2e.WebE2ESmokeTests)')]),
+ '类级准备失败没有可单独重跑的那一条')
+ self.assertIsNone(runner.rerun_selectors([self.fail(
+ 'test_web_e2e.WebE2ESmokeTests.test_every_route_holds_the_layout_and_runtime_invariants')]),
+ '浏览器容器失败却没有逐条结果:说不清是哪个文件')
+ many = [self.fail(f'e2e:e2e/page-{index}.test.ts > 用例') for index in range(runner.RERUN_LIMIT + 1)]
+ self.assertIsNone(runner.rerun_selectors(many), '失败面大多半是真回归,不再重跑一轮')
+ self.assertIsNone(runner.rerun_selectors([]))
+
+ def test_a_rerun_that_passes_is_recorded_as_flaky_and_one_that_fails_stays_red(self):
+ class Done:
+ def __init__(self, code):
+ self.returncode = code
+
+ rows = [self.fail('e2e:e2e/folder-layout.test.ts > 目录 > 滚到底')]
+ commands = []
+ with contextlib.redirect_stdout(io.StringIO()):
+ self.assertTrue(runner.rerun_failures(rows, spawn=lambda command, **_: commands.append(command) or Done(0)))
+ self.assertEqual(runner.FLAKY, ['frontend/e2e/folder-layout.test.ts'])
+ self.assertIn('--machine-lock-held', commands[0], '父进程已持有整机锁,重跑不能再去等它')
+ self.assertEqual(commands[0][-2:], ['--focus', 'frontend/e2e/folder-layout.test.ts'])
+ runner.FLAKY.clear()
+ with contextlib.redirect_stdout(io.StringIO()):
+ self.assertFalse(runner.rerun_failures(rows, spawn=lambda command, **_: Done(1)))
+ self.assertEqual(runner.FLAKY, [])
+
+ def test_the_gate_runs_first_and_a_red_gate_skips_the_other_shards(self):
+ files = [runner.TESTS / name for name in ('test_script_policy.py', 'test_media.py', 'test_scan.py')]
+ plans = []
+
+ def shards(scopes, *, jobs, shard_count, measured, plan, outcomes):
+ plans.append(plan)
+ return (False, 1, []) if plan == [['test_script_policy.py']] else (True, 2, [])
+
+ with patch.object(runner, 'selected_files', return_value=files), \
+ patch.object(runner, 'run_shards', side_effect=shards), \
+ patch.object(runner, 'save_file_seconds'), patch.object(runner.test_history, 'append'), \
+ contextlib.redirect_stdout(io.StringIO()):
+ self.assertEqual(runner.run_local_suite(('full',), jobs=2), (False, 1, []))
+ self.assertEqual(plans, [[['test_script_policy.py']]], '门槛没过就不起其余分片')
+
+ plans.clear()
+ with patch.object(runner, 'selected_files', return_value=files), \
+ patch.object(runner, 'run_shards', side_effect=lambda *a, **k: plans.append(k['plan']) or (True, 2, [])), \
+ patch.object(runner, 'save_file_seconds'), patch.object(runner.test_history, 'append'), \
+ contextlib.redirect_stdout(io.StringIO()):
+ self.assertTrue(runner.run_local_suite(('full',), jobs=2)[0])
+ self.assertEqual(plans[0], [['test_script_policy.py']])
+ self.assertEqual(sorted(name for shard in plans[1] for name in shard), ['test_media.py', 'test_scan.py'],
+ '门槛里的文件不再进后面的分片')
+
+ def test_heavy_runs_queue_on_the_machine_lock_except_nested_or_already_held(self):
+ args = runner.argparse.Namespace(machine_lock_held=False, lock_timeout=5)
+ with patch.object(runner.test_evidence, 'held') as held, \
+ patch.object(runner.test_history, 'nested', return_value=False):
+ runner.machine_guard(args, True, scope='full')
+ self.assertEqual(held.call_args.args[0], runner.test_evidence.machine_lock_path())
+ for light, nested, already in ((True, False, False), (False, True, False), (False, False, True)):
+ with self.subTest(light=light, nested=nested, already=already), \
+ patch.object(runner.test_evidence, 'held') as held, \
+ patch.object(runner.test_history, 'nested', return_value=nested):
+ runner.machine_guard(runner.argparse.Namespace(machine_lock_held=already, lock_timeout=5), not light)
+ held.assert_not_called()
+
+ def test_focus_accepts_vitest_files_and_hands_them_to_the_island_suite(self):
+ from scripts import test_focus
+ suite, browsers, islands = test_focus.selection(runner.ROOT, ['frontend/test/jav-card-title.test.ts'])
+ self.assertEqual((browsers, islands), ((), ('test/jav-card-title.test.ts',)))
+ self.assertEqual([test.id() for test in suite], [test_focus.VITEST_SUITE])
+ with self.assertRaises(ValueError):
+ test_focus.selection(runner.ROOT, ['frontend/test/../../scripts/test_runner.py'])