Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

9.3 KB
Raw Blame
"""核对受保护演练的产物清单;标签发布同一份已验收字节。"""
from __future__ import annotations

import argparse
import json
import os
from pathlib import Path, PurePosixPath
import subprocess
import re
import sys
import tempfile

if not __package__:
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
    __package__ = "scripts"
from . import ci_r2_artifacts as artifacts
from .release_forge import GitLabForge, REQUIRED_TEST_JOBS, REHEARSAL_JOBS
from .release_r2 import require_capacity

GROUPS = ("windows-release", "macos-release", "docker-release")
RECEIPT = Path("build/ci-artifacts/release-candidate.json")


def forge() -> GitLabForge:
    def command(*args):
        return subprocess.check_output(args, text=True, encoding="utf-8", stderr=subprocess.PIPE)
    return GitLabForge(os.environ["CI_SERVER_HOST"], command)


def verified_jobs(client, repo: str, build: dict) -> dict[str, int]:
    selected = client.jobs(repo, int(build["pipeline"]))
    required = (REQUIRED_TEST_JOBS | REHEARSAL_JOBS) - {"seal-release"}
    bad = [name for name in sorted(required) if name not in selected
           or selected[name]["status"] != "success" or selected[name].get("allow_failure")
           or selected[name].get("commit", {}).get("id") != build["commit"]]
    if bad:
        raise ValueError("演练缺少同一提交的成功验证:" + "、".join(bad))
    return {name: selected[name]["id"] for name in sorted(required)}


def listing(root: Path) -> list[dict]:
    environment = artifacts.rclone_environment()
    with tempfile.TemporaryDirectory(prefix="peach-release-capacity-") as directory:
        config = Path(directory) / "rclone.conf"
        config.touch()
        return json.loads(subprocess.check_output([artifacts.rclone_executable(root), "lsjson",
            f"peach:{environment['PEACH_R2_BUCKET']}", "--recursive", "--files-only", "--config", str(config)],
            env=environment, text=True, encoding="utf-8"))


def manifests(root: Path, build: dict) -> dict:
    result = {}
    for group in GROUPS:
        manifest = artifacts.remote_json(root, artifacts.checkpoint(build, group))
        if manifest is None:
            raise ValueError("缺少完整上传的演练产物:" + group)
        artifacts.validate_manifest(manifest, group, build)
        result[group] = manifest
    return result


def publication_items(groups: dict) -> list[dict]:
    result = {}
    for group in ("windows-release", "macos-release"):
        for item in groups[group]["files"]:
            path = PurePosixPath(item["path"])
            if path.suffix not in {".zip", ".exe", ".dmg"}:
                continue
            row = {"name": path.name, "size": item["size"], "sha256": item["sha256"]}
            if path.name in result and result[path.name] != row:
                raise ValueError("发行文件同名但内容不一致")
            result[path.name] = row
    return list(result.values())


def check_capacity(root: Path, build: dict | None = None, groups: dict | None = None) -> dict:
    build = build or artifacts.identity()
    groups = groups or manifests(root, build)
    rows = listing(root)
    limit = int(os.environ.get("PEACH_R2_MAX_BYTES", "8000000000"))
    projected = require_capacity(rows, publication_items(groups), build["tag"], limit)
    return {"current_bytes": sum(row["Size"] for row in rows if not row.get("IsDir")),
            "projected_bytes": projected, "limit_bytes": limit}


def preflight(root: Path) -> dict:
    """构建前取得真实桶占用;精确发行峰值在清单齐备后、安装冒烟前核对。"""
    artifacts.identity()
    rows = listing(root)
    limit = int(os.environ.get("PEACH_R2_MAX_BYTES", "8000000000"))
    current = sum(row["Size"] for row in rows if not row.get("IsDir"))
    if limit <= 0 or current >= limit:
        raise ValueError(f"发布桶当前占用 {current} 字节,预算 {limit};先审阅暂存清理方案")
    scopes = {}
    for row in rows:
        parts = row["Path"].split("/")
        if len(parts) >= 4 and parts[0] == "_ci":
            scope = "/".join(parts[:4])
            scopes[scope] = scopes.get(scope, 0) + row["Size"]
    return {"current_bytes": current, "limit_bytes": limit, "staging_bytes": scopes,
            "publication_peak": "构建清单齐备后精确核对;不按未知新版体积伪造剩余容量"}


def seal(root: Path) -> dict:
    build = artifacts.identity()
    if os.environ.get("CI_COMMIT_BRANCH") != "master" or os.environ.get("CI_COMMIT_TAG"):
        raise ValueError("只有受保护主线的发布演练可以登记候选")
    client = forge()
    jobs = verified_jobs(client, os.environ["CI_PROJECT_PATH"], build)
    groups = manifests(root, build)
    capacity = check_capacity(root, build, groups)
    path = root / "build/ci-artifacts/docker-image.json"
    (root / "build/ci-artifacts").mkdir(parents=True, exist_ok=True)
    (root / "build/ci-artifacts/docker-release.json").write_text(json.dumps(groups["docker-release"]), encoding="utf-8")
    artifacts.transfer(root, "docker-release", "pull")
    docker = json.loads(path.read_text(encoding="utf-8"))
    validate_docker(docker, build)
    candidate = {"format": 1, "identity": build, "jobs": jobs, "groups": groups, "docker": docker}
    artifacts.remote_json(root, artifacts.prefix(build) + "/release-candidate.json", candidate)
    return {"sealed": True, "identity": build, "capacity": capacity}


def validate_docker(docker: dict, build: dict) -> None:
    if (docker.get("commit") != build["commit"] or docker.get("pipeline") != build["pipeline"]
            or docker.get("version") != build["tag"][1:]
            or docker.get("image") != f"peach-ci:{build['tag'][1:]}-{build['pipeline']}"
            or not re.fullmatch(r"sha256:[0-9a-f]{64}", str(docker.get("image_id", "")))):
        raise ValueError("Docker 候选镜像不属于同一提交与演练")


def validate(root: Path) -> dict:
    target = artifacts.identity()
    if os.environ.get("CI_COMMIT_TAG") != target["tag"]:
        raise ValueError("候选晋级只接受受保护版本标签")
    client, repo = forge(), os.environ["CI_PROJECT_PATH"]
    runs = client.test_runs(repo, target["commit"])
    if not runs or runs[0]["status"] != "completed" or runs[0]["conclusion"] != "success":
        raise ValueError("同一提交的主线发布演练尚未成功")
    source = {**target, "pipeline": str(runs[0]["id"])}
    client.require_rehearsal(repo, int(source["pipeline"]))
    candidate = artifacts.remote_json(root, artifacts.prefix(source) + "/release-candidate.json")
    if not isinstance(candidate, dict) or candidate.get("format") != 1 or candidate.get("identity") != source:
        raise ValueError("候选清单不属于当前项目、提交、版本与演练")
    if candidate.get("jobs") != verified_jobs(client, repo, source) or set(candidate.get("groups", {})) != set(GROUPS):
        raise ValueError("候选的验证记录与演练最新成功任务不一致")
    for group, manifest in candidate["groups"].items():
        artifacts.validate_manifest(manifest, group, source)
    validate_docker(candidate.get("docker", {}), source)
    receipt = {"target": target, "candidate": candidate}
    path = root / RECEIPT
    if path.exists() and json.loads(path.read_text(encoding="utf-8")) != receipt:
        raise ValueError("本标签已选择另一份候选,保留原回执并停止")
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    return receipt


def pull(root: Path) -> dict:
    receipt = validate(root)
    candidate = receipt["candidate"]
    for group in GROUPS:
        manifest = candidate["groups"][group]
        path = root / "build/ci-artifacts" / (group + ".json")
        path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
        artifacts.transfer(root, group, "pull", source=candidate["identity"])
    return {"promoted": True, "source_pipeline": candidate["identity"]["pipeline"], "commit": candidate["identity"]["commit"]}


def cleanup(root: Path) -> dict:
    receipt = json.loads((root / RECEIPT).read_text(encoding="utf-8"))
    target = artifacts.identity()
    if receipt.get("target") != target:
        raise ValueError("回收回执不属于当前标签流水线")
    client, repo = forge(), os.environ["CI_PROJECT_PATH"]
    runs = client.test_runs(repo, target["commit"])
    if not runs or runs[0]["status"] != "completed" or runs[0]["conclusion"] != "success":
        raise ValueError("回收前未取得成功演练")
    source = {**target, "pipeline": str(runs[0]["id"])}
    if (receipt["candidate"].get("identity") != source
            or receipt["candidate"].get("jobs") != verified_jobs(client, repo, source)):
        raise ValueError("回收回执与成功演练不一致")
    client.require_rehearsal(repo, int(source["pipeline"]))
    return artifacts.cleanup(root, source=source)


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("mode", choices=("preflight", "check-capacity", "seal", "validate", "pull", "cleanup"))
    args = parser.parse_args()
    functions = {"preflight": preflight, "check-capacity": check_capacity,
                 "seal": seal, "validate": validate, "pull": pull, "cleanup": cleanup}
    result = functions[args.mode](Path.cwd().resolve())
    print(json.dumps(result, ensure_ascii=False))


if __name__ == "__main__":
    main()