Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

19.5 KB
Raw Blame
"""通过 R2 传递演练大产物并晋级至标签,GitLab 只保存带 SHA-256 的清单。"""
from __future__ import annotations

import argparse
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import re
import shutil
import subprocess
import sys
import tempfile

if not __package__:
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
    __package__ = "scripts"
from .release_r2 import rclone_environment, require_capacity

GROUPS = {
    "windows-media": ("build/ffmpeg-windows/ffmpeg", "build/ffmpeg-windows/ffmpeg-corresponding-source.zip",
                      "build/untrunc-windows/untrunc", "build/untrunc-windows/untrunc-corresponding-source.zip"),
    "windows-release": ("release",),
    "macos-release": ("dist/macos.noindex/arm64", "dist/macos.noindex/x86_64"),
    "docker-release": ("build/ci-artifacts/docker-image.json",),
}


def rehearsal_tag() -> str:
    """版本提交演练(ADR-0121)还没有标签:用提交里的版本号代替,流水线规则保证它是版本提交。"""
    source = (Path(__file__).resolve().parents[1] / "src/peach/__init__.py").read_text(encoding="utf-8")
    found = re.search(r'^__version__ = "([0-9]+\.[0-9]+\.[0-9]+)"$', source, re.M)
    return "v" + found.group(1) if found else ""


def identity() -> dict:
    result = {key: os.environ.get("CI_" + variable, "") for key, variable in
              (("project", "PROJECT_ID"), ("pipeline", "PIPELINE_ID"), ("commit", "COMMIT_SHA"), ("tag", "COMMIT_TAG"))}
    if os.environ.get("PEACH_RELEASE_REHEARSAL") == "1" and not result["tag"]:
        result["tag"] = rehearsal_tag()
    if (not all(re.fullmatch(r"[1-9][0-9]*", result[key]) for key in ("project", "pipeline"))
            or not re.fullmatch(r"[0-9a-f]{40}", result["commit"])
            or not re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+", result["tag"])
            or os.environ.get("CI_COMMIT_REF_PROTECTED") != "true"):
        raise ValueError("R2 暂存仅允许受保护版本标签与明确的项目、流水线和提交")
    return result


def prefix(build: dict, group: str | None = None, producer_job: str | None = None) -> str:
    value = f"_ci/{build['project']}/{build['pipeline']}/{build['commit']}"
    if group is not None:
        if group not in GROUPS or not isinstance(producer_job, str) or not re.fullmatch(r"[1-9][0-9]*", producer_job):
            raise ValueError("产物暂存需要明确的生成作业 ID")
        return value + "/" + group + "/" + producer_job
    return value


def allowed_path(name: str, group: str) -> bool:
    parts = PurePosixPath(name).parts
    if (not parts or any(part in {".", ".."} or not re.fullmatch(r"[A-Za-z0-9_.-]+", part) for part in parts)
            or PurePosixPath(name).is_absolute() or str(PurePosixPath(name)) != name):
        return False
    if group == "macos-release":
        return (len(parts) == 4 and "/".join(parts[:3]) in GROUPS[group]
                and (parts[-1] in {"build-info.json", "SHA256SUMS.txt"} or parts[-1].endswith(".dmg")
                     or bool(re.fullmatch(r"(?:ffmpeg|untrunc)-macos-(?:arm64|x86_64)-corresponding-source.zip", parts[-1]))))
    if group == "windows-release":
        return len(parts) == 2 and parts[0] == "release" and (
            bool(re.fullmatch(r"Peach-[0-9]+\.[0-9]+\.[0-9]+-windows-x64[-A-Za-z0-9.]*\.(zip|exe)", parts[-1]))
            or parts[-1] in {"ffmpeg-corresponding-source.zip", "untrunc-corresponding-source.zip",
                              "RELEASE_NOTES.md", "DOWNLOADS.md", "version.txt", "ci_windows.ps1",
                              "smoke_desktop.py", "smoke_update_tasks.py", "smoke_split_update.py"})
    if group == "docker-release":
        return name == "build/ci-artifacts/docker-image.json"
    return any(name == base or name.startswith(base + "/") for base in GROUPS[group])


def local_path(root: Path, name: str, group: str) -> Path:
    if not allowed_path(name, group):
        raise ValueError("产物路径不属于本任务输出目录")
    path = root / name
    if not path.resolve().is_relative_to(root) or any(parent.is_symlink() for parent in (path, *path.parents) if parent != root and parent.is_relative_to(root)):
        raise ValueError("产物路径包含软链接或越出工作目录")
    return path


def checksum(path: Path) -> str:
    with path.open("rb") as stream:
        return hashlib.file_digest(stream, "sha256").hexdigest()


def rclone_executable(root: Path) -> str:
    suffix = ".exe" if os.name == "nt" else ""
    path = root / "build/release-tools" / ("rclone" + suffix)
    if not path.is_file() or path.is_symlink():
        raise ValueError("CI 中转仅使用项目内已核对官方 SHA-256 的 rclone")
    return str(path)


def inventory(root: Path, group: str) -> list[dict]:
    paths = []
    for base in GROUPS[group]:
        source = root / base
        if not source.exists():
            raise ValueError(f"缺少构建产物目录或文件:{base}")
        paths.extend(source.rglob("*") if source.is_dir() else [source])
    items = []
    for path in sorted(paths):
        name = path.relative_to(root).as_posix()
        if group == "macos-release" and not allowed_path(name, group):
            continue
        if path.is_symlink():
            raise ValueError("CI 产物不允许软链接")
        if not path.is_file():
            continue
        local_path(root, name, group)
        items.append({"path": name, "size": path.stat().st_size, "sha256": checksum(path)})
    if not items or len({item["path"].casefold() for item in items}) != len(items):
        raise ValueError("产物清单为空或包含跨平台同名文件")
    if group == "macos-release":
        build = identity()
        for arch in ("arm64", "x86_64"):
            folder = root / "dist/macos.noindex" / arch
            image = folder / f"Peach-{build['tag'][1:]}-macos-{arch}.dmg"
            # 镜像随包带 FFmpeg 与 untrunc,许可证要求同时提供对应源码;清单逐个列出这三份。
            listed = {image.name} | {f"{tool}-macos-{arch}-corresponding-source.zip" for tool in ("ffmpeg", "untrunc")}
            present = {PurePosixPath(item["path"]).name for item in items
                       if item["path"].startswith(f"dist/macos.noindex/{arch}/")}
            if present != listed | {"build-info.json", "SHA256SUMS.txt"}:
                raise ValueError("每种 macOS 架构须包含 DMG、两份对应源码、构建身份与 SHA-256 清单")
            info = json.loads((folder / "build-info.json").read_text(encoding="utf-8"))
            sums = {}
            for line in (folder / "SHA256SUMS.txt").read_text(encoding="utf-8").splitlines():
                digest, _, name = line.partition("  ")
                sums[name] = digest
            if (info.get("commit") != build["commit"] or info.get("version") != build["tag"][1:]
                    or info.get("architecture") != arch or info.get("dirty") is not False
                    or set(sums) != listed
                    or any(sums[name] != checksum(folder / name) for name in listed)):
                raise ValueError("macOS 构建身份或镜像 SHA-256 不符")
    return items


def validate_manifest(manifest: dict, group: str, build: dict) -> list[dict]:
    if manifest.get("format") != 1 or manifest.get("identity") != build or manifest.get("group") != group:
        raise ValueError("产物清单不属于当前版本、提交或流水线")
    prefix(build, group, manifest.get("producer_job"))
    items = manifest.get("files")
    if not isinstance(items, list) or not items:
        raise ValueError("产物清单缺少文件")
    names = set()
    for item in items:
        name = item.get("path", "")
        if (not allowed_path(name, group) or name.casefold() in names
                or type(item.get("size")) is not int or item["size"] < 0
                or not re.fullmatch(r"[0-9a-f]{64}", item.get("sha256", ""))):
            raise ValueError("产物清单的路径、大小或 SHA-256 无效")
        names.add(name.casefold())
    return items


def checkpoint(build: dict, group: str) -> str:
    if group not in GROUPS:
        raise ValueError("未知的产物组")
    return prefix(build) + "/completed/" + group + ".json"


def upload_backup(root: Path, build: dict, group: str) -> Path | None:
    value = os.environ.get("CI_BUILDS_DIR")
    if not value:
        return None
    if group not in GROUPS or any(not re.fullmatch(r"[1-9][0-9]*", build[key]) for key in ("project", "pipeline")):
        raise ValueError("无效的构建续传身份")
    base = Path(value).resolve() / ".peach-upload-resume"
    path = base / build["project"] / build["pipeline"] / build["commit"] / group
    if not path.resolve().is_relative_to(base.resolve()) or path.resolve().is_relative_to(root.resolve()):
        raise ValueError("续传目录必须位于 CI 构建根的独立命名空间")
    if path.is_symlink() or any(parent.is_symlink() for parent in path.parents if parent.is_relative_to(base)):
        raise ValueError("续传目录不能包含软链接")
    return path


def preserve_upload(root: Path, manifest: dict) -> None:
    group, build = manifest["group"], manifest["identity"]
    destination = upload_backup(root, build, group)
    if destination is None:
        return
    destination.mkdir(parents=True, exist_ok=True)
    for item in manifest["files"]:
        source = local_path(root, item["path"], group)
        target = local_path(destination, item["path"], group)
        target.parent.mkdir(parents=True, exist_ok=True)
        shutil.copyfile(source, target)
    temporary = destination / "upload.json.tmp"
    temporary.write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n", encoding="utf-8")
    temporary.replace(destination / "upload.json")


def discard_upload(root: Path, build: dict, group: str) -> None:
    path = upload_backup(root, build, group)
    if path is not None and path.exists():
        shutil.rmtree(path)


def remote_json(root: Path, key: str, payload: dict | None = None) -> dict | None:
    """只在受保护 CI 的暂存命名空间读取或登记完整对象;网络失败不当作不存在。"""
    if not re.fullmatch(r"_ci/[1-9][0-9]*/[1-9][0-9]*/[0-9a-f]{40}/(?:completed/[a-z-]+|release-candidate)\.json", key):
        raise ValueError("无效的 CI 元数据路径")
    environment = rclone_environment()
    executable = rclone_executable(root.resolve(strict=True))
    bucket = f"peach:{environment['PEACH_R2_BUCKET']}"
    with tempfile.TemporaryDirectory(prefix="peach-ci-metadata-") as directory:
        temporary = Path(directory)
        configuration = temporary / "rclone.conf"
        configuration.touch()
        common = {"env": environment, "text": True, "encoding": "utf-8"}
        if payload is None:
            listing = json.loads(subprocess.check_output([executable, "lsjson", bucket, "--recursive", "--files-only",
                "--config", str(configuration)], **common))
            if not any(row["Path"] == key for row in listing):
                return None
            return json.loads(subprocess.check_output([executable, "cat", bucket + "/" + key,
                "--config", str(configuration)], **common))
        source = temporary / "metadata.json"
        source.write_text(json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=2) + "\n", encoding="utf-8")
        subprocess.run([executable, "copyto", str(source), bucket + "/" + key,
            "--checksum", "--immutable", "--config", str(configuration)], env=environment, check=True)
        return payload


def restore(root: Path, group: str) -> dict:
    """恢复同一受保护提交与流水线已完整上传的构建,供失败作业重试复用。"""
    build = identity()
    manifest = remote_json(root, checkpoint(build, group))
    result = {"restored": manifest is not None, "uploaded": manifest is not None, "group": group}
    destination = root / "build/ci-artifacts"
    destination.mkdir(parents=True, exist_ok=True)
    if manifest is not None:
        validate_manifest(manifest, group, build)
        (destination / (group + ".json")).write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
        result.update(transfer(root, group, "pull"))
        discard_upload(root, build, group)
    else:
        backup = upload_backup(root, build, group)
        if backup is not None and (backup / "upload.json").is_file():
            manifest = json.loads((backup / "upload.json").read_text(encoding="utf-8"))
            items = validate_manifest(manifest, group, build)
            for item in items:
                path = local_path(backup, item["path"], group)
                if not path.is_file() or path.stat().st_size != item["size"] or checksum(path) != item["sha256"]:
                    raise ValueError("本机续传文件的大小或 SHA-256 不符")
            for item in items:
                target = local_path(root, item["path"], group)
                target.parent.mkdir(parents=True, exist_ok=True)
                shutil.copyfile(local_path(backup, item["path"], group), target)
            (destination / (group + ".json")).write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
            result["restored"] = True
    (destination / (group + "-resume.json")).write_text(json.dumps(result) + "\n", encoding="utf-8")
    return result


def transfer(root: Path, group: str, mode: str, *, source: dict | None = None) -> dict:
    root = root.resolve(strict=True)
    current = identity()
    build = current if source is None else source
    if source is not None and (mode != "pull" or any(build[key] != current[key] for key in ("project", "commit", "tag"))):
        raise ValueError("晋级产物必须属于同一项目、版本与提交")
    environment = rclone_environment()
    executable = rclone_executable(root)
    manifest_path = root / "build/ci-artifacts" / (group + ".json")
    if mode == "push":
        producer_job = os.environ.get("CI_JOB_ID", "")
        if manifest_path.exists():
            previous = json.loads(manifest_path.read_text(encoding="utf-8"))
            validate_manifest(previous, group, build)
            if previous["files"] == inventory(root, group):
                producer_job = previous["producer_job"]
    else:
        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
        items = validate_manifest(manifest, group, build)
        producer_job = manifest["producer_job"]
    storage_prefix = prefix(build, group, producer_job)
    remote = f"peach:{environment['PEACH_R2_BUCKET']}/{storage_prefix}"
    with tempfile.TemporaryDirectory(prefix="peach-ci-r2-") as directory:
        temporary = Path(directory).resolve()
        configuration = temporary / "rclone.conf"
        configuration.touch()

        def command(*args, read=False):
            args = [executable, *args, "--config", str(configuration)]
            if read:
                return subprocess.check_output(args, env=environment, text=True, encoding="utf-8")
            subprocess.run(args, env=environment, check=True)

        if mode == "push":
            actual = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True, encoding="utf-8").strip()
            dirty = subprocess.check_output(["git", "status", "--porcelain"], cwd=root, text=True, encoding="utf-8").strip()
            if actual != build["commit"] or dirty:
                raise ValueError("暂存需要当前提交的干净构建目录")
            items = inventory(root, group)
            manifest = {"format": 1, "identity": build, "group": group, "producer_job": producer_job, "files": items}
            preserve_upload(root, manifest)
            existing = json.loads(command("lsjson", f"peach:{environment['PEACH_R2_BUCKET']}", "--recursive", "--files-only", read=True))
            require_capacity(existing, [{"name": item["path"], "size": item["size"]} for item in items],
                             storage_prefix, int(environment.get("PEACH_R2_MAX_BYTES", "8000000000")))
            listing = temporary / "files.txt"
            listing.write_text("".join(item["path"] + "\n" for item in items), encoding="utf-8")
            command("copy", str(root), remote, "--files-from-raw", str(listing), "--checksum", "--immutable")
            manifest_path.parent.mkdir(parents=True, exist_ok=True)
            manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
            remote_json(root, checkpoint(build, group), manifest)
            discard_upload(root, build, group)
        else:
            for item in items:
                local_path(root, item["path"], group)
            listing = temporary / "files.txt"
            listing.write_text("".join(item["path"] + "\n" for item in items), encoding="utf-8")
            command("copy", remote, str(temporary / "download"), "--files-from-raw", str(listing), "--checksum")
            for item in items:
                source = temporary / "download" / item["path"]
                if not source.is_file() or source.stat().st_size != item["size"] or checksum(source) != item["sha256"]:
                    raise ValueError(f"R2 暂存产物的大小或 SHA-256 不符:{item['path']}")
            for item in items:
                target = local_path(root, item["path"], group)
                target.parent.mkdir(parents=True, exist_ok=True)
                shutil.copyfile(temporary / "download" / item["path"], target)
    return {"group": group, "mode": mode, "files": len(items), "bytes": sum(item["size"] for item in items), "commit": build["commit"]}


def cleanup(root: Path, *, source: dict | None = None) -> dict:
    """回收指定流水线的暂存前缀;晋级入口在全部发布成功后选择来源演练。"""
    current = identity()
    build = current if source is None else source
    if source is not None and any(build[key] != current[key] for key in ("project", "commit", "tag")):
        raise ValueError("回收范围必须属于同一项目、版本与提交")
    rehearsal = os.environ.get("PEACH_RELEASE_REHEARSAL") == "1" and not os.environ.get("CI_COMMIT_TAG")
    environment = rclone_environment()
    executable = rclone_executable(root.resolve(strict=True))
    remote = f"peach:{environment['PEACH_R2_BUCKET']}/{prefix(build)}"
    with tempfile.TemporaryDirectory(prefix="peach-ci-r2-cleanup-") as directory:
        configuration = Path(directory) / "rclone.conf"
        configuration.touch()
        if not rehearsal:
            index = json.loads(subprocess.check_output(
                [executable, "cat", f"peach:{environment['PEACH_R2_BUCKET']}/releases.json", "--config", str(configuration)],
                env=environment, text=True, encoding="utf-8"))
            if not any(item.get("tag_name") == build["tag"] and item.get("commit_sha") == build["commit"]
                       for item in index):
                raise ValueError("公开版本索引尚未登记当前提交,保留暂存产物")
        subprocess.run([executable, "purge", remote, "--config", str(configuration)], env=environment, check=True)
    return {"cleaned_pipeline": build["pipeline"], "commit": build["commit"]}


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("mode", choices=("push", "pull", "restore", "cleanup"))
    parser.add_argument("group", nargs="?", choices=tuple(GROUPS))
    args = parser.parse_args()
    if args.mode != "cleanup" and not args.group:
        parser.error("上传和下载须指定产物组")
    result = (cleanup(Path.cwd()) if args.mode == "cleanup" else restore(Path.cwd(), args.group)
              if args.mode == "restore" else transfer(Path.cwd(), args.group, args.mode))
    print(json.dumps(result, ensure_ascii=False))


if __name__ == "__main__":
    main()