Peach: Local-first video metadata scraper and media manager.
"""GitLab 发布的主线与流水线判据;认证和 HTTP 传输复用 glab。"""
from __future__ import annotations
import json
import os
import re
import shutil
import subprocess
import sys
from pathlib import Path
from urllib.parse import quote, urlencode
from urllib.request import Request
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from peach.user_agent import USER_AGENT
from peach.peach_proxy import read_url
REQUIRED_TEST_JOBS = frozenset({"verify-linux", "verify-windows", "verify-macos", "wheel-smoke"})
#: 版本提交的主线流水线里演练的标签作业(ADR-0121):全部成功才打标签。规则没匹配上、演练没跑,
#: 也不能当作通过。三平台验收是必需任务,演练规则在版本提交上把它们改成全量。
REHEARSAL_JOBS = frozenset({"windows-media", "build-windows", "build-macos", "smoke-windows", "docker-build", "seal-release"})
TERMINAL_STATUSES = frozenset({"success", "failed", "canceled", "skipped"})
#: 必需任务落在这些状态时,整条流水线等多久都不会转绿;重试会生成同名的新任务。
RED_JOB_STATUSES = frozenset({"failed", "canceled"})
class GitLabForge:
def __init__(self, host: str, command):
if not re.fullmatch(r"[a-zA-Z0-9.-]+(?::[0-9]+)?", host):
raise ValueError("GitLab 主机必须是域名,可附端口")
self.host = host
self.command = command
def api(self, repo: str, endpoint: str, *, public: bool = True):
"""`public=False` 给只有维护者凭据才读得到的端点:匿名重试注定再被拒,直接抛出。"""
executable = os.environ.get("PEACH_GLAB") or shutil.which("glab")
if not executable:
raise ValueError("GitLab 发布需要官方 glab;可用 PEACH_GLAB 指定其路径")
path = f"projects/{quote(repo, safe='')}/{endpoint}"
try:
return json.loads(self.command(executable, "api", "--hostname", self.host,
"--method", "GET", path))
except subprocess.CalledProcessError as error:
if not public or not any(status in (error.stderr or "") for status in ("401", "403")):
raise
# 公开项目的只读元数据可匿名取得,不扩大本机细粒度凭据权限。
request = Request(f"https://{self.host}/api/v4/{path}",
headers={"Accept": "application/json", "User-Agent": USER_AGENT})
return json.loads(read_url(request.full_url, headers=dict(request.header_items()), timeout=60))
def remote_url(self, repo: str) -> str:
# 使用已配置的 Git 认证;PAT 和 Runner token 不进入命令参数。
return f"https://{self.host}/{repo}.git"
def release_url(self, repo: str, tag: str) -> str:
return f"https://{self.host}/{repo}/-/releases/{quote(tag, safe='')}"
def master_sha(self, repo: str) -> str:
return self.api(repo, "repository/branches/master")["commit"]["id"]
def require_free_tag(self, repo: str, tag: str) -> None:
query = urlencode({"search": f"^{re.escape(tag)}$", "per_page": 100})
if any(item["name"] == tag for item in self.api(repo, f"repository/tags?{query}")):
raise ValueError(f"{tag} 已存在,不覆盖;发布新版本请先推进 __version__")
def require_ancestor(self, repo: str, sha: str) -> None:
remote = self.master_sha(repo)
# Git 对提交图作精确判定;比较文件差异不能证明祖先关系。
self.command("git", "fetch", "--no-tags", self.remote_url(repo), "refs/heads/master")
fetched = self.command("git", "rev-parse", "FETCH_HEAD")
if fetched != remote:
raise ValueError("检查期间 GitLab master 已更新,请重新执行")
if self.command("git", "merge-base", remote, sha) != sha:
raise ValueError("发布提交不属于远端 master 历史")
def test_runs(self, repo: str, sha: str) -> list[dict]:
query = urlencode({"sha": sha, "ref": "master", "source": "push", "per_page": 100})
pipelines = self.api(repo, f"pipelines?{query}")
matching = [item for item in pipelines if item["sha"] == sha
and item["ref"] == "master" and item["source"] == "push"]
if not matching:
return []
pipeline = max(matching, key=lambda item: item["id"])
status = pipeline["status"]
if status == "success":
self.require_test_jobs(repo, pipeline["id"], sha)
# 统一 CI 结果字段供唯一发布入口消费,保留真实流水线 ID 与链接。
run = {"id": pipeline["id"], "head_sha": sha, "head_branch": "master", "event": "push",
"status": "completed" if status in TERMINAL_STATUSES else "in_progress",
"conclusion": "success" if status == "success" else status,
"html_url": pipeline["web_url"]}
if status not in TERMINAL_STATUSES:
# 流水线整体是 running 时也可能有任务排着没人接,所以只要没结束就看任务。
jobs = self.jobs(repo, pipeline["id"])
# 任何一个不许失败的任务红了,整条就不会转绿:不只看必需的四个,演练里的构建也算。
run["failed_jobs"] = sorted(
name for name, job in jobs.items()
if job["status"] in RED_JOB_STATUSES and not job.get("allow_failure"))
run["queued_jobs"] = self.queued_jobs(repo, jobs.values())
return [run]
def jobs(self, repo: str, pipeline: int) -> dict[str, dict]:
"""流水线里每个任务名最新的一次;重试过的任务以最后一次为准。"""
jobs = []
for page in range(1, 11):
batch = self.api(repo, f"pipelines/{pipeline}/jobs?per_page=100&page={page}")
jobs.extend(batch)
if len(batch) < 100:
break
else:
raise ValueError("流水线任务超过可核对上限,发布检查未完成")
selected = {}
for job in jobs:
if job["name"] not in selected or job["id"] > selected[job["name"]]["id"]:
selected[job["name"]] = job
return selected
def queued_jobs(self, repo: str, jobs) -> list[dict]:
"""排着队、还没有 runner 接走的任务,附带 runner 侧能确定的事实。
只看 `pending`:`created` 在等前一阶段,`manual`、`scheduled`、`waiting_for_resource`
都在等人或等别的条件,不是没人接。确有在线 runner 带齐这些 tags 的任务不列出,
它只是在排队,交给总超时。`runners` 是 `offline`(带这些 tags 的 runner 都已登记但
不在线)或 `unknown`(runner 列表读不到,或一个带这些 tags 的都查不到——项目接口
未必列出共享 runner,查不到不能当成没有)。
"""
result = []
states: dict[tuple[str, ...], str] = {}
for job in jobs:
if job.get("status") != "pending" or job.get("started_at") or job.get("runner"):
continue
tags = tuple(sorted(job.get("tag_list") or ()))
if tags not in states:
states[tags] = self.runner_state(repo, tags)
if states[tags] == "online":
continue
queued = job.get("queued_duration")
result.append({"id": job["id"], "name": job["name"], "tags": list(tags),
"queued": float(queued) if isinstance(queued, (int, float)) else None,
"runners": states[tags]})
return result
def runner_state(self, repo: str, tags: tuple[str, ...]) -> str:
"""`online`、`offline` 或 `unknown`;读 runner 要维护者权限,读不到就不下结论。"""
if not tags:
return "unknown"
query = urlencode({"tag_list": ",".join(tags), "per_page": 100})
try:
runners = self.api(repo, f"runners?{query}", public=False)
except (subprocess.CalledProcessError, OSError, ValueError):
return "unknown"
if not isinstance(runners, list) or not runners:
return "unknown"
if any(runner.get("status") == "online" and not runner.get("paused")
and runner.get("active", True) for runner in runners):
return "online"
return "offline"
def require_test_jobs(self, repo: str, pipeline: int, sha: str) -> None:
selected = self.jobs(repo, pipeline)
bad = [name for name in sorted(REQUIRED_TEST_JOBS)
if name not in selected or selected[name]["status"] != "success"
or selected[name].get("allow_failure")
or selected[name].get("commit", {}).get("id") != sha]
if bad:
raise ValueError("GitLab 发布验证缺少同一提交的必需成功任务:" + "、".join(bad))
def require_rehearsal(self, repo: str, pipeline: int) -> None:
"""版本提交的流水线里,演练的标签作业必须都在且成功;缺一个都不打标签。"""
selected = self.jobs(repo, pipeline)
bad = [name for name in sorted(REHEARSAL_JOBS)
if name not in selected or selected[name]["status"] != "success"]
if bad:
raise ValueError("版本提交的发布演练没有全部通过,不打标签:" + "、".join(bad)
+ "。检查 .gitlab-ci.yml 的演练规则是否匹配版本提交,修好后再跑一次 --ship --apply")