Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

15.0 KB
Raw Blame
"""An approval delay or publish-job rerun cannot renew a release exception."""

# allow: no-sut-import — the SUT is the standalone publication deadline guard.

from datetime import datetime, timedelta, timezone
import importlib.util
import os
from pathlib import Path
import subprocess
import sys

import pytest
import yaml


ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / ".github/scripts/check_release_deadline.py"
spec = importlib.util.spec_from_file_location("release_deadline", SCRIPT)
deadline = importlib.util.module_from_spec(spec)
spec.loader.exec_module(deadline)
GUARD = 'python3 .github/scripts/check_release_deadline.py --deadline "$RELEASE_DEADLINE"'
BUILD_DEADLINE = "${{ needs.build.outputs.release_deadline }}"
INPUT_DEADLINE = "${{ inputs.release_deadline }}"
DECISION_ARTIFACT = "${{ needs.build.outputs.release_decision_artifact_id }}"
VERIFIED_DEADLINE = "${{ steps.security-decision.outputs.release_deadline }}"


def workflow(name):
    return yaml.safe_load((ROOT / f".github/workflows/{name}.yml").read_text())


def step_named(job, name):
    return next(step for step in job["steps"] if step.get("name") == name)


@pytest.mark.parametrize(
    "instant,allowed",
    [
        ("2026-10-12T23:59:59+00:00", True),
        ("2026-10-13T00:00:00+00:00", False),
        ("2026-10-14T12:00:00+00:00", False),
        ("2026-10-12T20:00:00-04:00", False),
        ("2026-10-13T01:59:59+02:00", True),
    ],
)
def test_approval_and_rerun_time_is_checked_in_utc(instant, allowed):
    now = datetime.fromisoformat(instant)
    if allowed:
        deadline.check_deadline("2026-10-13", now)
    else:
        with pytest.raises(ValueError, match="expired"):
            deadline.check_deadline("2026-10-13", now)


@pytest.mark.parametrize(
    "value",
    [
        "",
        "null",
        "None",
        "0",
        "20261013",
        "2026-10-13T00:00:00",
        "*",
        "none\n",
    ],
)
def test_missing_or_malformed_transport_never_means_unlimited(value):
    with pytest.raises(ValueError):
        deadline.check_deadline(
            value, datetime(2026, 9, 27, tzinfo=timezone.utc)
        )


def test_explicit_no_exception_result_does_not_expire():
    deadline.check_deadline("none", datetime(2099, 1, 1, tzinfo=timezone.utc))


def test_cli_uses_current_utc_time_after_approval(monkeypatch, capsys):
    class FrozenDateTime(datetime):
        @classmethod
        def now(cls, tz=None):
            assert tz == timezone.utc
            return datetime(2026, 10, 13, tzinfo=tz)

    monkeypatch.setattr(deadline, "datetime", FrozenDateTime)
    monkeypatch.setattr(sys, "argv", [str(SCRIPT), "--deadline", "2026-10-13"])
    assert deadline.main() == 1
    assert "::error::Release exception expired" in capsys.readouterr().out


def test_original_deadline_reaches_every_publication_path():
    gate = workflow("release-gate")
    # PyYAML reads the unquoted GitHub Actions 'on' key as YAML 1.1 True.
    exported = gate[True]["workflow_call"]["outputs"]["release_deadline"]
    assert exported["value"] == (
        "${{ jobs.check-code-scanning-alerts.outputs.release_deadline }}"
    )
    assert (
        gate["jobs"]["check-code-scanning-alerts"]["outputs"][
            "release_deadline"
        ]
        == "${{ steps.alert-check.outputs.release_deadline }}"
    )

    jobs = workflow("release")["jobs"]
    assert jobs["build"]["outputs"]["release_deadline"] == (
        "${{ needs.release-gate.outputs.release_deadline }}"
    )
    for caller in ("prerelease-docker", "publish-docker"):
        expected = (
            BUILD_DEADLINE
            if caller == "prerelease-docker"
            else "${{ needs.prerelease-docker.outputs.release_deadline }}"
        )
        assert jobs[caller]["with"]["release_deadline"] == expected
        called_name = jobs[caller]["uses"].split("/")[-1].removesuffix(".yml")
        contract = workflow(called_name)[True]["workflow_call"]["inputs"][
            "release_deadline"
        ]
        assert contract["required"] is True
        assert "default" not in contract

    dispatch = step_named(jobs["trigger-pypi"], "Trigger PyPI publish workflow")
    assert dispatch["env"]["RELEASE_DECISION_ARTIFACT_ID"] == DECISION_ARTIFACT
    assert (
        "release_decision_artifact_id: process.env.RELEASE_DECISION_ARTIFACT_ID"
        in dispatch["with"]["script"]
    )
    recovery = step_named(
        jobs["monitor-pypi"], "Create issue on PyPI publish failure"
    )
    assert recovery["env"]["RELEASE_DECISION_ARTIFACT_ID"] == DECISION_ARTIFACT
    recovery_script = recovery["with"]["script"]
    assert "Re-run failed publisher jobs" in recovery_script
    assert "preserve the original tested artifact and dispatch payload" in (
        recovery_script
    )
    assert "Expired artifacts or security decisions" in recovery_script
    assert "re-run monitor-pypi and its dependent jobs" in recovery_script
    assert "client_payload[" not in recovery_script

    build = jobs["build"]
    assert build.get("name", "build") == "build"
    assert "needs.release-gate.result == 'success'" in build["if"]
    record = step_named(build, "Record release security decision")
    assert (
        record["env"]["RELEASE_DEADLINE"]
        == "${{ needs.release-gate.outputs.release_deadline }}"
    )
    upload = step_named(build, "Upload immutable release security decision")
    assert upload["with"]["overwrite"] is False
    assert upload["with"]["if-no-files-found"] == "error"
    assert (
        upload["with"]["name"]
        == "release-security-decision-${{ github.run_attempt }}"
    )
    assert (
        build["outputs"]["release_decision_artifact_id"]
        == "${{ steps.security-decision.outputs.artifact-id }}"
    )


@pytest.mark.parametrize("job_id", ["build-amd64", "build-arm64"])
def test_prerelease_push_checks_deadline_after_approval(job_id):
    job = workflow("prerelease-docker")["jobs"][job_id]
    assert job["environment"] == "release"
    steps = job["steps"]
    build_index = next(
        i for i, step in enumerate(steps) if step.get("id") == "build"
    )
    build = steps[build_index]
    assert build["with"]["push"] is False
    assert build["with"]["load"] is True
    guard = next(s for s in steps if 'docker push "$IMAGE"' in s.get("run", ""))
    assert steps.index(guard) > build_index
    assert guard["run"].index(GUARD) < guard["run"].index(
        'docker push "$IMAGE"'
    )
    assert guard["env"]["RELEASE_DEADLINE"] == INPUT_DEADLINE
    assert "if" not in guard and not guard.get("continue-on-error")


def test_pypi_checks_after_its_own_approval_and_before_both_upload_targets():
    jobs = workflow("publish")["jobs"]
    verifier = jobs["build-package"]
    verify = step_named(verifier, "Verify original release security decision")
    assert "release_security_decision.py verify" in verify["run"]
    assert (
        verify["env"]["RELEASE_RUN_ID"]
        == "${{ github.event.client_payload.run_id }}"
    )
    assert '--run-id "$RELEASE_RUN_ID"' in verify["run"]
    assert (
        verify["env"]["RELEASE_DECISION_ARTIFACT_ID"]
        == "${{ github.event.client_payload.release_decision_artifact_id }}"
    )
    assert "if" not in verify and not verify.get("continue-on-error")
    assert verifier["outputs"]["release_deadline"] == VERIFIED_DEADLINE

    job = jobs["publish"]
    assert job["environment"] == "release"
    assert job["needs"] == "build-package"
    assert job["permissions"] == {
        "contents": "read",
        "actions": "read",
        "id-token": "write",
    }
    assert "github.event.client_payload.release_deadline" not in str(job)
    steps = job["steps"]
    # Main revalidates the package source after approval. Its narrowly scoped
    # policy checkout must coexist with the original security-decision deadline.
    checkouts = [
        step
        for step in steps
        if step.get("uses", "").startswith("actions/checkout@")
    ]
    assert len(checkouts) == 1
    checkout = checkouts[0]
    assert checkout["with"]["persist-credentials"] is False
    assert (
        checkout["with"]["sparse-checkout"]
        == ".github/scripts/release_package.py"
    )
    revalidate = step_named(job, "Revalidate source release run after approval")
    assert steps.index(checkout) < steps.index(revalidate)
    guard = step_named(
        job, "Check release exception deadline before publishing"
    )
    assert (
        guard["env"]["RELEASE_DEADLINE"]
        == "${{ needs.build-package.outputs.release_deadline }}"
    )
    assert "if" not in guard and not guard.get("continue-on-error")
    guard_index = steps.index(guard)
    assert steps.index(revalidate) < guard_index
    assert steps[guard_index + 1]["name"] == "Publish to Test PyPI"
    assert steps[guard_index + 2]["name"] == "Publish to PyPI"


@pytest.mark.parametrize(
    "value,allowed",
    [
        ("none", True),
        ("tomorrow", True),
        ("today", False),
        ("yesterday", False),
        ("future-timestamp", True),
        ("expired-timestamp", False),
        ("2026-10-30T12:00:00", False),
        ("", False),
        ("null", False),
        ("20261013", False),
        ("none\n", False),
    ],
)
def test_pypi_executes_its_guard_without_checkout(tmp_path, value, allowed):
    today = datetime.now(timezone.utc).date()
    relative = {
        "tomorrow": today + timedelta(days=1),
        "today": today,
        "yesterday": today - timedelta(days=1),
    }
    if value in relative:
        value = relative[value].isoformat()
    if value in {"future-timestamp", "expired-timestamp"}:
        instant = datetime.now(timezone.utc) + timedelta(
            hours=1 if value == "future-timestamp" else -1
        )
        value = instant.isoformat().replace("+00:00", "Z")
    guard = step_named(
        workflow("publish")["jobs"]["publish"],
        "Check release exception deadline before publishing",
    )
    result = subprocess.run(
        ["bash", "-e", "-o", "pipefail", "-c", guard["run"]],
        cwd=tmp_path,
        env={**os.environ, "RELEASE_DEADLINE": value},
        capture_output=True,
        text=True,
        timeout=10,
    )
    assert (result.returncode == 0) is allowed, result.stdout + result.stderr


@pytest.mark.parametrize(
    "workflow_name,job_id,step_name,publish_command",
    [
        (
            "docker-publish",
            "promote",
            "Promote (retag) prerelease manifest to release tags",
            "docker buildx imagetools create",
        ),
        (
            "prerelease-docker",
            "create-manifest",
            "Create and push multi-platform manifest",
            "docker buildx imagetools create",
        ),
        (
            "release",
            "create-release",
            "Create GitHub Release",
            "gh release create",
        ),
    ],
)
def test_tag_publication_is_guarded_after_approval(
    workflow_name, job_id, step_name, publish_command
):
    job = workflow(workflow_name)["jobs"][job_id]
    assert job["environment"] == "release"
    publish = step_named(job, step_name)
    assert not publish.get("continue-on-error")
    expected = (
        "${{ needs.publish-docker.outputs.release_deadline }}"
        if workflow_name == "release"
        else "${{ steps.native-deadline.outputs.release_deadline }}"
    )
    assert publish["env"]["RELEASE_DEADLINE"] == expected
    if workflow_name != "release":
        bind = next(s for s in job["steps"] if s.get("id") == "native-deadline")
        assert bind["env"]["RELEASE_DEADLINE"] == INPUT_DEADLINE
        assert "--additional-deadline" in bind["run"]
        assert "--write-output" in bind["run"]
        assert job["steps"].index(bind) < job["steps"].index(publish)
    run = publish["run"]
    assert run.index(GUARD) < run.index(publish_command)
    assert run.count(GUARD) == run.count(publish_command)
    checkout = next(
        (
            s
            for s in job["steps"]
            if s.get("uses", "").startswith("actions/checkout@")
        ),
        None,
    )
    assert checkout is not None, (
        f"{workflow_name}/{job_id} must check out the deadline checker"
    )
    assert job["steps"].index(checkout) < job["steps"].index(publish)
    assert checkout.get("with", {}).get("ref") in (None, "${{ github.sha }}"), (
        f"{workflow_name}/{job_id} must check out the caller's commit"
    )
    if "sparse-checkout" in checkout.get("with", {}):
        assert (
            str(SCRIPT.relative_to(ROOT)) in checkout["with"]["sparse-checkout"]
        )


@pytest.mark.parametrize("expired", [False, True])
def test_expired_approval_stops_actual_promotion_shell(tmp_path, expired):
    """Execute the real promotion step; replace only Docker's remote effect."""
    job = workflow("docker-publish")["jobs"]["promote"]
    publish = step_named(
        job, "Promote (retag) prerelease manifest to release tags"
    )
    docker = tmp_path / "docker"
    docker.write_text('#!/bin/sh\ntouch "$PUBLISHED_MARKER"\n')
    docker.chmod(0o755)
    marker = tmp_path / "published"
    day = datetime.now(timezone.utc).date() + timedelta(
        days=-1 if expired else 1
    )
    result = subprocess.run(
        ["bash", "-e", "-o", "pipefail", "-c", publish["run"]],
        cwd=ROOT,
        env={
            **os.environ,
            "PATH": f"{tmp_path}:{os.environ['PATH']}",
            "PUBLISHED_MARKER": str(marker),
            "RELEASE_DEADLINE": day.isoformat(),
            "DOCKER_USERNAME": "test",
            "SOURCE_TAG": "prerelease-v1.0.0-abcdef0",
            "VERSION": "1.0.0",
            "MAJOR_MINOR": "1.0",
        },
        capture_output=True,
        text=True,
        timeout=10,
    )
    assert (result.returncode == 0) is not expired, (
        result.stdout + result.stderr
    )
    assert marker.exists() is not expired


@pytest.mark.parametrize("arch", ["amd64", "arm64"])
@pytest.mark.parametrize("expired", [False, True])
def test_native_build_crossing_deadline_cannot_push(tmp_path, arch, expired):
    today = datetime.now(timezone.utc).date()
    expiry = today + timedelta(days=-1 if expired else 1)
    # The pre-build check passed, then a long build/runner pause crossed expiry.
    started = datetime.combine(
        expiry - timedelta(days=1), datetime.min.time(), timezone.utc
    )
    deadline.check_deadline(expiry.isoformat(), started)
    job = workflow("prerelease-docker")["jobs"][f"build-{arch}"]
    push = step_named(job, f"Push {arch.upper()} image before the deadline")
    docker = tmp_path / "docker"
    docker.write_text('#!/bin/sh\ntouch "$PUBLISHED_MARKER"\n')
    docker.chmod(0o755)
    marker = tmp_path / "published"
    result = subprocess.run(
        ["bash", "-e", "-o", "pipefail", "-c", push["run"]],
        cwd=ROOT,
        env={
            **os.environ,
            "PATH": f"{tmp_path}:{os.environ['PATH']}",
            "PUBLISHED_MARKER": str(marker),
            "RELEASE_DEADLINE": expiry.isoformat(),
            "IMAGE": f"test/local-deep-research:prerelease-v1.0.0-abcdef0-{arch}",
        },
        capture_output=True,
        text=True,
        timeout=10,
    )
    assert (result.returncode == 0) is not expired, (
        result.stdout + result.stderr
    )
    assert marker.exists() is not expired