Peach: Local-first video metadata scraper and media manager.
ci(gitgud): 主线按改动域与平台敏感用例验证,标签跑三平台全量
longmeidaoabae396877d7
diff --git a/.gitlab-ci.yml b/.gitlab-ci.ymlindex 9dad638a8..aa5572695 100644--- a/.gitlab-ci.yml+++ b/.gitlab-ci.yml@@ -25,11 +25,14 @@ variables:PEACH_R2_BUCKET: peach-releasesPEACH_R2_PUBLIC_BASE: https://downloads.peach.video-verify-linux:+# 回归口径(ADR-0119):分支与合并请求跑相对上一次已验证主线的改动域,加上本平台的平台敏感用例,+# 取不到基线或改动落到必须全量的面时退回全量;版本标签上三个平台都跑全量,构建与发布依赖它们。+# 改动域随时可能退成全量,所以两种作业的时限都按全量定。+.verify-linux:stage: verifyimage: node:24-bookworm- rules:- - if: '$CI_COMMIT_BRANCH || $CI_PIPELINE_SOURCE == "merge_request_event"'+ # 全量回归连同装依赖 13–20 分钟。+ timeout: 45mcache:when: alwayskey:@@ -51,7 +54,7 @@ verify-linux:- test -z "$(git status --porcelain --untracked-files=all -- web/dist)"- uv run --locked python scripts/npm_audit.py- uv run --locked python scripts/npm_audit.py --prefix frontend- - xvfb-run -a ./scripts/test.sh full --fresh+ - xvfb-run -a bash scripts/ci_verify.sh linux "$PEACH_CI_COVERAGE"artifacts:when: alwaysexpire_in: 14 days@@ -59,27 +62,71 @@ verify-linux:- build/agent-verification/browser/*.tap- build/agent-verification/browser/serve.log- build/agent-verification/browser/setup-serve.log+ - build/test-history/-verify-windows:+verify-linux:+ extends: .verify-linux+ rules:+ - if: '$CI_COMMIT_BRANCH || $CI_PIPELINE_SOURCE == "merge_request_event"'+ variables:+ PEACH_CI_COVERAGE: changed++release-verify-linux:+ extends: .verify-linux+ rules: !reference [.release-linux, rules]+ variables:+ PEACH_CI_COVERAGE: full++.verify-windows:stage: verifytags: [peach-windows]# 全量回归连同依赖安装约 20–25 分钟;runner 与本机开发会话共用 CPU,并发全量时会翻倍。timeout: 1h 30m+ script:+ - '& ./scripts/ci_windows.ps1 verify -Coverage $env:PEACH_CI_COVERAGE'+ - if ($LASTEXITCODE -ne 0) { throw 'Windows 回归未通过' }+ artifacts:+ when: always+ expire_in: 14 days+ paths: [build/test-history/]++verify-windows:+ extends: .verify-windowsrules:- if: '$CI_COMMIT_BRANCH == "master"'- script:- - '& ./scripts/ci_windows.ps1 verify'- - if ($LASTEXITCODE -ne 0) { throw 'Windows 全量回归未通过' }+ variables:+ PEACH_CI_COVERAGE: changed-verify-macos:+release-verify-windows:+ extends: .verify-windows+ rules: !reference [.release-linux, rules]+ variables:+ PEACH_CI_COVERAGE: full++.verify-macos:stage: verifytags: [peach-macos]# 作业脚本约 3 分钟,新 runner 首次检出完整历史约 16 分钟;下载卡住时提前结束。timeout: 45m+ script:+ - bash scripts/ci_macos.sh verify arm64 "$PEACH_CI_COVERAGE"+ artifacts:+ when: always+ expire_in: 14 days+ paths: [build/test-history/]++verify-macos:+ extends: .verify-macosrules:- if: '$CI_COMMIT_BRANCH == "master"'- script:- - bash scripts/ci_macos.sh verify arm64+ variables:+ PEACH_CI_COVERAGE: changed++release-verify-macos:+ extends: .verify-macos+ rules: !reference [.release-linux, rules]+ variables:+ PEACH_CI_COVERAGE: fullwheel-smoke:stage: package@@ -149,7 +196,7 @@ build-windows:stage: buildtags: [peach-windows]rules: !reference [.release-linux, rules]- needs: [windows-media]+ needs: [windows-media, {job: release-verify-windows, artifacts: false}]environment:name: releaseaction: prepare@@ -173,7 +220,7 @@ build-macos:# 两个架构各自从源码编译 x264、ffmpeg 与 untrunc,不走缓存;windows-media 单架构交叉编译 ffmpeg 与 untrunc 约 17 分钟。timeout: 2hrules: !reference [.release-linux, rules]- needs: [validate-release]+ needs: [validate-release, {job: release-verify-macos, artifacts: false}]environment:name: releaseaction: prepare@@ -208,7 +255,7 @@ publish-docker:stage: publishtags: [peach-windows]rules: !reference [.release-linux, rules]- needs: [validate-release]+ needs: [validate-release, {job: release-verify-linux, artifacts: false}]resource_group: peach-docker-publicationenvironment:name: release@@ -220,7 +267,7 @@ publish-docker:publish-release:extends: .release-linuxstage: publish- needs: [build-windows, smoke-windows, build-macos]+ needs: [build-windows, smoke-windows, build-macos, {job: release-verify-linux, artifacts: false}]resource_group: peach-release-publicationenvironment:name: releasediff --git a/scripts/ci_macos.sh b/scripts/ci_macos.shindex ec08494fd..160ad3d5d 100644--- a/scripts/ci_macos.sh+++ b/scripts/ci_macos.sh@@ -1,23 +1,26 @@#!/usr/bin/env bash-# macOS CI:原生架构依赖、统一全量回归与独立安装镜像验收。+# macOS CI:原生架构依赖、回归验证与独立安装镜像验收。+# verify 默认全量;第三个参数 changed 只跑改动域与平台敏感用例,供主线推送使用。set -euo pipefailusage() {- printf '%s\n' '用法:bash scripts/ci_macos.sh <verify|build> <arm64|x86_64>'+ printf '%s\n' '用法:bash scripts/ci_macos.sh <verify|build> <arm64|x86_64> [full|changed]'}if [[ "${1:-}" == --help ]]; thenusageexit 0fi-if [[ $# -ne 2 ]]; then+if [[ $# -lt 2 || $# -gt 3 ]]; thenusage >&2exit 2fimode="$1"architecture="$2"+coverage="${3:-full}"case "$mode" in verify|build) ;; *) usage >&2; exit 2 ;; esaccase "$architecture" in arm64|x86_64) ;; *) usage >&2; exit 2 ;; esac+case "$coverage" in full|changed) ;; *) usage >&2; exit 2 ;; esacif [[ "$(uname -s)" != Darwin ]]; thenprintf '%s\n' 'macOS 验证与构建必须在 Mac 上运行。' >&2exit 1@@ -74,7 +77,7 @@ uv pip check --python "$python"if [[ "$mode" == verify ]]; thennpm run build --prefix frontend- ./scripts/test.sh full+ bash scripts/ci_verify.sh macos "$coverage"exit 0fidiff --git a/scripts/ci_scope.py b/scripts/ci_scope.pynew file mode 100644index 000000000..03acf6eb1--- /dev/null+++ b/scripts/ci_scope.py@@ -0,0 +1,114 @@+"""GitGud 主线流水线的选测:相对上一次已验证主线的改动域,加上各平台的平台敏感用例。++输出给 CI 脚本逐行读:第一行是 `full`,或已验证基线的提交;基线可用时,其后每行一个+`-Focus` 选择器。取不到基线、基线不在当前历史上、改动落到必须 full 的面,一律 `full`。+标签流水线不经过这里,直接跑全量(ADR-0119)。+"""+from __future__ import annotations++import argparse+import json+import os+import subprocess+import sys+from collections.abc import Callable, Iterable+from pathlib import Path+from urllib.parse import urlencode++ROOT = Path(__file__).resolve().parents[1]+if not __package__:+ sys.path.insert(0, str(ROOT))+ __package__ = "scripts"+sys.path.insert(0, str(ROOT / "src"))++#: 三个平台都要验的面:路径与数据目录、证书与权限、首次设置拉起服务、调用本机 ffmpeg,+#: 以及在本平台浏览器里走首启、播放链和普通 HTTP 来源。+_SHARED = (+ "test_platform", "test_fsutil", "test_folder_picker", "test_certs", "test_auth", "test_onboarding",+ "test_media", "test_media_probe", "test_frame_capture",+ "frontend/e2e/setup.test.ts", "frontend/e2e/playback-resume.test.ts",+ "frontend/e2e/player-v10.test.ts", "frontend/e2e/http-configuration.test.ts",+)+#: 代码按平台分叉、或调用平台专属程序与系统接口的测试;只是在某平台下拼路径的不算。+PLATFORM_TESTS: dict[str, tuple[str, ...]] = {+ "windows": (*_SHARED,+ "test_tray", "test_tray_detached", "test_tray_diagnostics", "test_tray_lifecycle",+ "test_restart_windows_tray", "test_process_job", "test_windows_update",+ "test_desktop_settings", "test_desktop_installer", "test_standalone_update",+ "test_data_storage", "test_interaction", "test_jobs", "test_scan", "test_media_tool_import",+ "frontend/e2e/external-playback.test.ts"),+ "macos": (*_SHARED,+ "test_tray", "test_mount", "test_netwatch", "test_mdns", "test_macos_standalone", "test_macos_media",+ "frontend/e2e/external-playback.test.ts"),+ "linux": (*_SHARED, "test_mdns"),+}+#: 基线只在最近这么多条成功的主线流水线里找;更早的通过记录覆盖不到之后的改动。+PIPELINE_WINDOW = 20+++def verified_base(pipelines: Iterable[dict], current: str, usable: Callable[[str], bool]) -> str | None:+ """最近一条成功的主线流水线里,不是本提交、且是本提交祖先的那个提交。++ 强推之后旧提交不再是祖先,取不到就返回 None,由调用方退回全量;不拿推送前的提交+ 顶替:上一轮红了的话,那个提交从没被验证过。+ """+ for pipeline in pipelines:+ sha = pipeline.get("sha") or ""+ if pipeline.get("status") == "success" and sha and sha != current and usable(sha):+ return sha+ return None+++def successful_pipelines(environ: dict[str, str]) -> list[dict]:+ """匿名读本项目主线的成功流水线;项目公开,作业里不需要任何凭据。"""+ from peach.peach_proxy import read_url+ from peach.user_agent import USER_AGENT+ query = urlencode({"ref": environ.get("CI_DEFAULT_BRANCH") or "master", "status": "success",+ "order_by": "id", "sort": "desc", "per_page": PIPELINE_WINDOW})+ url = f"{environ['CI_API_V4_URL'].rstrip('/')}/projects/{environ['CI_PROJECT_ID']}/pipelines?{query}"+ payload = json.loads(read_url(url, headers={"Accept": "application/json", "User-Agent": USER_AGENT}, timeout=60))+ if not isinstance(payload, list):+ raise ValueError("流水线列表不是数组")+ return payload+++def ancestor_of_head(root: Path) -> Callable[[str], bool]:+ def usable(sha: str) -> bool:+ command = ["git", "-c", f"safe.directory={root.as_posix()}", "-C", str(root), "merge-base", "--is-ancestor", sha, "HEAD"]+ return subprocess.run(command, capture_output=True, check=False).returncode == 0+ return usable+++def plan(platform: str, root: Path, environ: dict[str, str], *,+ pipelines: Callable[[dict[str, str]], list[dict]] = successful_pipelines) -> tuple[list[str], str]:+ """返回输出行与原因。改动域交给 `test_runner` 的 auto 规则判,这里只预判会不会退成全量。"""+ from . import test_runner as runner+ current = environ.get("CI_COMMIT_SHA", "")+ try:+ base = verified_base(pipelines(environ), current, ancestor_of_head(root))+ except (OSError, KeyError, ValueError) as error:+ return ["full"], f"取不到已验证的主线流水线({error})"+ if base is None:+ return ["full"], "最近的成功主线流水线里没有本提交的祖先"+ try:+ paths = runner.changed_files(root, base)+ scopes, explanation = runner.scopes_for_changes(paths, contents=runner.changed_contents(root, base, paths))+ except (OSError, subprocess.CalledProcessError) as error:+ return ["full"], f"无法比较 {base[:12]}({error})"+ if "full" in scopes:+ return ["full"], explanation+ return [base, *PLATFORM_TESTS[platform]], f"基线 {base[:12]};{explanation}"+++def main(argv: list[str] | None = None) -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--platform", choices=tuple(PLATFORM_TESTS), required=True)+ args = parser.parse_args(argv)+ lines, reason = plan(args.platform, ROOT, dict(os.environ))+ print(f"主线选测:{reason}", file=sys.stderr, flush=True)+ print("\n".join(lines), flush=True)+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/scripts/ci_verify.sh b/scripts/ci_verify.shnew file mode 100644index 000000000..03b13505c--- /dev/null+++ b/scripts/ci_verify.sh@@ -0,0 +1,37 @@+#!/usr/bin/env bash+# GitGud 的 Linux 与 macOS 回归:标签流水线跑全量;主线推送跑改动域,再跑本平台的平台敏感用例。+# 选测由 scripts/ci_scope.py 判,取不到已验证基线时它给出 full(ADR-0119)。+set -euo pipefail++usage() {+ printf '%s\n' '用法:bash scripts/ci_verify.sh <linux|macos> <full|changed>'+}+if [[ $# -ne 2 ]]; then+ usage >&2+ exit 2+fi+platform="$1"+coverage="$2"+case "$platform" in linux|macos) ;; *) usage >&2; exit 2 ;; esac+case "$coverage" in full|changed) ;; *) usage >&2; exit 2 ;; esac++root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"+cd "$root"+plan=full+if [[ "$coverage" == changed ]]; then+ plan="$(.venv/bin/python -X utf8 scripts/ci_scope.py --platform "$platform")"+fi+base="$(printf '%s\n' "$plan" | head -n 1)"+if [[ "$base" == full ]]; then+ ./scripts/test.sh full --fresh+ exit 0+fi+./scripts/test.sh auto --fresh --base "$base"+# bash 3.2 没有 mapfile,逐行读选择器。+focus=()+while IFS= read -r selector; do+ if [[ -n "$selector" ]]; then+ focus+=(--focus "$selector")+ fi+done < <(printf '%s\n' "$plan" | tail -n +2)+./scripts/test.sh auto "${focus[@]}"diff --git a/scripts/ci_windows.ps1 b/scripts/ci_windows.ps1index ab387a946..332ad5d30 100644--- a/scripts/ci_windows.ps1+++ b/scripts/ci_windows.ps1@@ -1,4 +1,6 @@-param([ValidateSet('verify', 'build', 'smoke', 'docker')][string]$Mode = 'verify')+param([ValidateSet('verify', 'build', 'smoke', 'docker')][string]$Mode = 'verify',+ # verify 默认全量;changed 只跑改动域与平台敏感用例,供主线推送使用(ADR-0119)。+ [ValidateSet('full', 'changed')][string]$Coverage = 'full')$ErrorActionPreference = 'Stop'function Assert-Exit([string]$Message) {@@ -18,8 +20,20 @@ if ($Mode -eq 'verify' -or $Mode -eq 'build') {if ($Mode -eq 'verify') {$env:PEACH_E2E_CHROME = Join-Path $env:ProgramFiles 'Google/Chrome/Application/chrome.exe'- & ./scripts/test.ps1 -Scope full -Fresh- Assert-Exit 'Windows 全量回归未通过'+ $Plan = @('full')+ if ($Coverage -eq 'changed') {+ $Plan = @(& .venv/Scripts/python.exe -X utf8 scripts/ci_scope.py --platform windows)+ Assert-Exit '无法判定主线选测范围'+ }+ if ($Plan[0] -eq 'full') {+ & ./scripts/test.ps1 -Scope full -Fresh+ Assert-Exit 'Windows 全量回归未通过'+ exit 0+ }+ & ./scripts/test.ps1 -Scope auto -Base $Plan[0] -Fresh+ Assert-Exit 'Windows 改动域回归未通过'+ & ./scripts/test.ps1 -Focus @($Plan | Select-Object -Skip 1)+ Assert-Exit 'Windows 平台敏感用例未通过'exit 0}diff --git a/tests/test_scripts.py b/tests/test_scripts.pyindex 24584377f..10b8fe947 100644--- a/tests/test_scripts.py+++ b/tests/test_scripts.py@@ -5159,6 +5159,90 @@ class GitGudPackageMirrorTests(unittest.TestCase):(root / 'RELEASE_NOTES.md').read_text(encoding='utf-8'))+class MainlineScopeTests(unittest.TestCase):+ """主线推送只跑改动域与平台敏感用例;基线取不稳就退回全量(ADR-0119)。"""++ @staticmethod+ def git(root, *args):+ command = ["git", "-c", "user.name=Peach", "-c", "[email protected]",+ "-c", "commit.gpgsign=false", "-C", str(root), *args]+ return subprocess.run(command, capture_output=True, text=True, encoding="utf-8", check=True).stdout.strip()++ def repository(self):+ """三个只改文档的提交;返回仓库根和提交列表(旧到新)。"""+ directory = tempfile.TemporaryDirectory()+ self.addCleanup(directory.cleanup)+ root = Path(directory.name).resolve()+ self.git(root, "init", "-q", "-b", "master")+ (root / "docs").mkdir()+ commits = []+ for index in range(3):+ (root / "docs" / "guide.md").write_text(f"第 {index} 版\n", encoding="utf-8")+ self.git(root, "add", "docs/guide.md")+ self.git(root, "commit", "-q", "-m", f"docs: 第 {index} 版")+ commits.append(self.git(root, "rev-parse", "HEAD"))+ return root, commits++ def plan(self, root, pipelines, current):+ from scripts import ci_scope+ source = pipelines if callable(pipelines) else (lambda _environ: pipelines)+ return ci_scope.plan("linux", root, {"CI_COMMIT_SHA": current}, pipelines=source)[0]++ def test_the_base_is_the_newest_verified_ancestor_so_a_red_push_is_retested(self):+ from scripts import ci_scope+ root, (first, second, head) = self.repository()+ pipelines = [{"sha": head, "status": "success"}, {"sha": second, "status": "failed"},+ {"sha": first, "status": "success"}]+ self.assertEqual(self.plan(root, pipelines, head), [first, *ci_scope.PLATFORM_TESTS["linux"]])++ def test_without_a_reachable_verified_base_the_suite_is_full(self):+ root, (first, _second, head) = self.repository()+ self.git(root, "checkout", "-q", "-b", "rewritten", first)+ (root / "docs" / "other.md").write_text("强推前的提交\n", encoding="utf-8")+ self.git(root, "add", "docs/other.md")+ self.git(root, "commit", "-q", "-m", "docs: 被改写的历史")+ rewritten = self.git(root, "rev-parse", "HEAD")+ self.git(root, "checkout", "-q", "master")++ def unavailable(_environ):+ raise OSError("外部请求未取得(ConnectError)")+ for name, pipelines in (("首次推送", []), ("强推后旧提交不是祖先", [{"sha": rewritten, "status": "success"}]),+ ("克隆里没有这个提交", [{"sha": "f" * 40, "status": "success"}]),+ ("只有本提交自己", [{"sha": head, "status": "success"}]),+ ("接口取不到", unavailable)):+ with self.subTest(name):+ self.assertEqual(self.plan(root, pipelines, head), ["full"])++ def test_changes_that_need_the_full_suite_skip_the_platform_subset(self):+ root, (_first, _second, base) = self.repository()+ (root / ".gitlab-ci.yml").write_text("stages: [verify]\n", encoding="utf-8")+ self.git(root, "add", ".gitlab-ci.yml")+ self.git(root, "commit", "-q", "-m", "ci: 新流水线")+ head = self.git(root, "rev-parse", "HEAD")+ self.assertEqual(self.plan(root, [{"sha": base, "status": "success"}], head), ["full"])++ def test_every_platform_subset_resolves_through_the_focus_entry(self):+ from scripts import ci_scope, test_focus+ for platform, selectors in ci_scope.PLATFORM_TESTS.items():+ with self.subTest(platform=platform):+ suite, browsers = test_focus.selection(ROOT, list(selectors))+ self.assertGreater(suite.countTestCases(), 0)+ self.assertIn("e2e/setup.test.ts", browsers)+ self.assertEqual(len(selectors), len(set(selectors)))++ def test_the_pipeline_list_is_read_anonymously(self):+ from scripts import ci_scope+ environ = {"CI_API_V4_URL": "https://gitgud.io/api/v4", "CI_PROJECT_ID": "42", "CI_JOB_TOKEN": "不该被带上"}+ with mock.patch("peach.peach_proxy.read_url", return_value=b'[{"sha": "a", "status": "success"}]') as fetch:+ self.assertEqual(ci_scope.successful_pipelines(environ), [{"sha": "a", "status": "success"}])+ url = fetch.call_args.args[0]+ self.assertTrue(url.startswith("https://gitgud.io/api/v4/projects/42/pipelines?"), url)+ self.assertIn("ref=master", url)+ self.assertIn("status=success", url)+ headers = {key.lower(): value for key, value in fetch.call_args.kwargs["headers"].items()}+ self.assertFalse({"job-token", "private-token", "authorization"} & headers.keys())++class GitLabReleaseTests(unittest.TestCase):SHA = "1" * 40