Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

14.2 KB
Raw Blame
"""Only an authentic release build may supply PyPI's security deadline."""

# allow: no-sut-import — the SUT is the standalone release decision verifier.

from argparse import Namespace
from copy import deepcopy
from datetime import datetime, timezone
import hashlib
import importlib.util
import io
import json
from pathlib import Path
import subprocess
import sys
from unittest.mock import patch
import zipfile

import pytest


SCRIPTS = Path(__file__).resolve().parents[2] / ".github/scripts"
spec = importlib.util.spec_from_file_location(
    "security_decision", SCRIPTS / "release_security_decision.py"
)
decision = importlib.util.module_from_spec(spec)
with patch.object(sys, "path", [str(SCRIPTS), *sys.path]):
    spec.loader.exec_module(decision)

REPOSITORY = "LearningCircuit/local-deep-research"
SHA = "a" * 40
ROOT = f"/repos/{REPOSITORY}"
ARTIFACT = f"{ROOT}/actions/artifacts/500"
ATTEMPT = f"{ROOT}/actions/runs/42/attempts/1"
JOBS = f"{ATTEMPT}/jobs?per_page=100&page=1"


@pytest.fixture(autouse=True)
def clock(monkeypatch):
    class FrozenDateTime(datetime):
        @classmethod
        def now(cls, tz=None):
            assert tz == timezone.utc
            return cls(2026, 10, 12, 12, tzinfo=tz)

    monkeypatch.setattr(decision, "datetime", FrozenDateTime)


@pytest.fixture
def record():
    return {
        "version": 1,
        "repository": REPOSITORY,
        "sha": SHA,
        "run_id": 42,
        "run_attempt": 1,
        "release_deadline": "2026-10-13",
    }


def archive_bytes(record, filename=decision.RECORD_NAME):
    stream = io.BytesIO()
    with zipfile.ZipFile(stream, "w", zipfile.ZIP_DEFLATED) as zipped:
        zipped.writestr(filename, json.dumps(record))
    return stream.getvalue()


@pytest.fixture
def responses(record, monkeypatch):
    blob = archive_bytes(record)
    responses = {
        ARTIFACT: {
            "id": 500,
            "name": "release-security-decision-1",
            "expired": False,
            "size_in_bytes": len(blob),
            "digest": "sha256:" + hashlib.sha256(blob).hexdigest(),
            "created_at": "2026-10-12T10:04:55Z",
            "workflow_run": {"id": 42, "head_sha": SHA},
        },
        ATTEMPT: {
            "id": 42,
            "run_attempt": 1,
            "path": ".github/workflows/release.yml",
            "head_sha": SHA,
            "event": "push",
            "status": "in_progress",
            "repository": {"full_name": REPOSITORY},
            "head_repository": {"full_name": REPOSITORY},
        },
        f"{ROOT}/compare/{SHA}...heads/main": {"status": "identical"},
        JOBS: {
            "jobs": [
                {
                    "name": "build",
                    "run_id": 42,
                    "run_attempt": 1,
                    "status": "completed",
                    "conclusion": "success",
                    "started_at": "2026-10-12T10:00:00Z",
                    "completed_at": "2026-10-12T10:05:00Z",
                }
            ]
        },
        f"{ARTIFACT}/zip": blob,
    }

    def read(endpoint):
        response = responses[endpoint]
        return (
            response
            if isinstance(response, bytes)
            else json.dumps(response).encode()
        )

    monkeypatch.setattr(decision, "api_bytes", read)
    return responses


def replace_archive(responses, record, filename=decision.RECORD_NAME):
    blob = archive_bytes(record, filename)
    responses[f"{ARTIFACT}/zip"] = blob
    responses[ARTIFACT]["digest"] = "sha256:" + hashlib.sha256(blob).hexdigest()
    responses[ARTIFACT]["size_in_bytes"] = len(blob)


def verify():
    return decision.verify_decision(REPOSITORY, SHA, "500", "42")


@pytest.fixture
def cli(monkeypatch, tmp_path):
    output = tmp_path / "output"
    monkeypatch.setenv("GITHUB_OUTPUT", str(output))
    monkeypatch.setattr(
        sys,
        "argv",
        [
            "verifier",
            "verify",
            "--repository",
            REPOSITORY,
            "--sha",
            SHA,
            "--artifact-id",
            "500",
            "--run-id",
            "42",
        ],
    )
    return output


def test_successful_build_authorizes_while_parent_waits_for_pypi(responses):
    assert responses[ATTEMPT]["status"] == "in_progress"
    assert verify() == "2026-10-13"


def test_partial_rerun_keeps_original_artifact_attempt(responses):
    # Attempt 1 built successfully, but a later job failed. The current
    # attempt reruns only the monitor, so it has no new build to authorize.
    responses[ATTEMPT].update(status="completed", conclusion="failure")
    later_attempt = f"{ROOT}/actions/runs/42/attempts/2"
    responses[later_attempt] = {
        **responses[ATTEMPT],
        "run_attempt": 2,
        "status": "in_progress",
        "conclusion": None,
    }
    responses[f"{ROOT}/actions/runs/42"] = responses[later_attempt]
    responses[f"{later_attempt}/jobs?per_page=100&page=1"] = {
        "jobs": [
            {
                "name": "monitor-pypi",
                "run_id": 42,
                "run_attempt": 2,
                "status": "in_progress",
                "conclusion": None,
            }
        ]
    }
    with patch.object(decision, "api_bytes", wraps=decision.api_bytes) as api:
        assert verify() == "2026-10-13"
    api.assert_any_call(ATTEMPT)
    api.assert_any_call(JOBS)


def test_no_exception_sentinel_requires_an_authentic_decision(
    responses, record
):
    record["release_deadline"] = "none"
    replace_archive(responses, record)
    assert verify() == "none"


@pytest.mark.parametrize(
    "value,allowed",
    [
        ("2026-10-12T12:00:00Z", False),
        ("2026-10-12T12:00:00.000001Z", True),
        ("2026-10-12T12:00:01Z", True),
    ],
)
def test_bound_decision_retains_exact_grace_expiry(
    responses, record, value, allowed
):
    record["release_deadline"] = value
    replace_archive(responses, record)
    if allowed:
        assert verify() == value
    else:
        with pytest.raises(ValueError, match="expired"):
            verify()


@pytest.mark.parametrize(
    "field,value",
    [
        ("id", 501),
        ("expired", True),
        ("expired", None),
        ("size_in_bytes", 1000000),
        ("name", "other-artifact"),
        ("name", "release-security-decision-none"),
        ("workflow_run", None),
    ],
)
def test_wrong_or_missing_artifact_is_rejected(responses, field, value):
    responses[ARTIFACT][field] = value
    with pytest.raises(ValueError):
        verify()


def test_artifact_for_another_commit_is_rejected(responses):
    responses[ARTIFACT]["workflow_run"]["head_sha"] = "b" * 40
    with pytest.raises(ValueError, match="commit"):
        verify()


@pytest.mark.parametrize(
    "field,value",
    [
        ("path", ".github/workflows/attacker.yml"),
        ("event", "pull_request"),
        ("head_sha", "b" * 40),
        ("run_attempt", 2),
        ("run_attempt", True),
        ("id", 43),
        ("repository", {"full_name": "other/repository"}),
        ("head_repository", {"full_name": "attacker/fork"}),
    ],
)
def test_wrong_workflow_run_or_fork_is_rejected(responses, field, value):
    responses[ATTEMPT][field] = value
    with pytest.raises(ValueError, match="expected release workflow"):
        verify()


@pytest.mark.parametrize("status", ["diverged", "behind", None])
def test_unreviewed_commit_cannot_authorize_publication(responses, status):
    responses[f"{ROOT}/compare/{SHA}...heads/main"]["status"] = status
    with pytest.raises(ValueError, match="not on main"):
        verify()


@pytest.mark.parametrize(
    "field,value",
    [
        ("conclusion", "failure"),
        ("conclusion", "skipped"),
        ("status", "in_progress"),
        ("run_attempt", 2),
        ("run_id", 43),
    ],
)
def test_failed_or_unfinished_build_cannot_authorize(responses, field, value):
    responses[JOBS]["jobs"][0][field] = value
    with pytest.raises(ValueError, match="did not succeed"):
        verify()


@pytest.mark.parametrize("count", [0, 2])
def test_missing_or_ambiguous_build_fails_closed(responses, count):
    responses[JOBS]["jobs"] *= count
    with pytest.raises(ValueError, match="Expected one"):
        verify()


def test_all_job_pages_are_checked(responses):
    build = deepcopy(responses[JOBS]["jobs"])
    responses[JOBS]["jobs"] = [{"name": f"scan-{n}"} for n in range(100)]
    responses[f"{ATTEMPT}/jobs?per_page=100&page=2"] = {"jobs": build}
    assert verify() == "2026-10-13"


@pytest.mark.parametrize(
    "created", ["2026-10-12T09:59:59Z", "2026-10-12T10:05:01Z"]
)
def test_artifact_from_outside_build_window_fails(responses, created):
    responses[ARTIFACT]["created_at"] = created
    with pytest.raises(ValueError, match="during the release build"):
        verify()


def test_tampered_download_fails_digest_verification(responses):
    responses[f"{ARTIFACT}/zip"] += b"tampered"
    with pytest.raises(ValueError, match="digest mismatch"):
        verify()


def test_missing_digest_fails_closed(responses):
    del responses[ARTIFACT]["digest"]
    with pytest.raises(ValueError, match="digest mismatch"):
        verify()


@pytest.mark.parametrize(
    "field,value",
    [
        ("version", True),
        ("sha", "b" * 40),
        ("repository", "attacker/fork"),
        ("run_id", 43),
        ("run_attempt", 2),
        ("run_attempt", True),
        ("release_deadline", None),
        ("release_deadline", ""),
    ],
)
def test_record_must_match_its_provenance(responses, record, field, value):
    record[field] = value
    replace_archive(responses, record)
    with pytest.raises(ValueError):
        verify()


@pytest.mark.parametrize(
    "filename", ["../release-security-decision.json", "other.json"]
)
def test_unexpected_archive_member_is_rejected(responses, record, filename):
    replace_archive(responses, record, filename)
    with pytest.raises(ValueError, match="archive contents"):
        verify()


@pytest.mark.parametrize("artifact_id", ["", "none", "0", "../500", "500\n"])
def test_missing_or_malformed_artifact_id_fails_before_api(
    monkeypatch, artifact_id
):
    def forbidden(*args):
        pytest.fail("API must not run for invalid artifact IDs")

    monkeypatch.setattr(decision, "api_bytes", forbidden)
    with pytest.raises(ValueError):
        decision.verify_decision(REPOSITORY, SHA, artifact_id, "42")


def test_record_writer_binds_context_and_deadline(tmp_path, record):
    path = tmp_path / decision.RECORD_NAME
    decision.record_decision(
        Namespace(
            repository=REPOSITORY,
            sha=SHA,
            run_id="42",
            run_attempt="1",
            deadline="2026-10-13",
            output=path,
        )
    )
    assert json.loads(path.read_text()) == record


def test_writer_does_not_record_expired_permission(tmp_path):
    path = tmp_path / decision.RECORD_NAME
    with pytest.raises(ValueError, match="expired"):
        decision.record_decision(
            Namespace(
                repository=REPOSITORY,
                sha=SHA,
                run_id="42",
                run_attempt="1",
                deadline="2026-10-12",
                output=path,
            )
        )
    assert not path.exists()


def test_cli_publishes_only_verified_output(responses, cli):
    assert decision.main() == 0
    assert cli.read_text() == "release_deadline=2026-10-13\n"


@pytest.mark.parametrize("override", ["none", "2099-01-01"])
@pytest.mark.parametrize(
    "recorded_deadline,allowed",
    [("2026-10-12", False), ("2026-10-13", True)],
)
def test_cli_uses_recorded_deadline_despite_deadline_environment(
    responses, record, cli, monkeypatch, override, recorded_deadline, allowed
):
    record["release_deadline"] = recorded_deadline
    replace_archive(responses, record)
    # The environment is an attempted override, not a supported input.
    monkeypatch.setenv("RELEASE_DEADLINE", override)
    assert decision.main() == (0 if allowed else 1)
    if allowed:
        assert cli.read_text() == f"release_deadline={recorded_deadline}\n"
    else:
        assert not cli.exists()


@pytest.mark.parametrize("override", ["none", "2099-01-01"])
def test_verify_cli_rejects_a_deadline_argument(cli, monkeypatch, override):
    monkeypatch.setattr(sys, "argv", [*sys.argv, "--deadline", override])
    with patch.object(decision, "api_bytes") as api:
        with pytest.raises(SystemExit) as exc:
            decision.main()
    assert exc.value.code == 2
    api.assert_not_called()
    assert not cli.exists()


def test_cli_api_error_never_emits_permission(monkeypatch, cli, capsys):
    def fail(endpoint):
        raise subprocess.CalledProcessError(
            1, ["gh", "api"], stderr=b"HTTP 403\n::warning::injected"
        )

    monkeypatch.setattr(decision, "api_bytes", fail)
    assert decision.main() == 1
    assert not cli.exists()
    assert "HTTP 403%0A::warning::injected" in capsys.readouterr().out


def test_api_transport_uses_only_get(monkeypatch):
    def run(args, **kwargs):
        assert args[:4] == ["gh", "api", "--method", "GET"]
        assert kwargs["check"] and kwargs["timeout"] == 60
        return subprocess.CompletedProcess(args, 0, stdout=b"{}")

    monkeypatch.setattr(decision.subprocess, "run", run)
    assert decision.api_object(ARTIFACT) == {}


def test_authentic_decision_for_same_commit_but_another_run_is_rejected(
    responses,
):
    # This artifact is otherwise authentic and has a still-valid deadline.
    # It cannot authorize a package produced by a different release run.
    with pytest.raises(ValueError, match="tested package release run"):
        decision.verify_decision(REPOSITORY, SHA, "500", "43")


@pytest.mark.parametrize("run_id", ["", "none", "0", "../42", "42\n"])
def test_invalid_expected_run_id_is_rejected_before_api(monkeypatch, run_id):
    def forbidden(*args):
        pytest.fail("API must not run for invalid release run IDs")

    monkeypatch.setattr(decision, "api_bytes", forbidden)
    with pytest.raises(ValueError):
        decision.verify_decision(REPOSITORY, SHA, "500", run_id)


def test_verify_cli_requires_the_original_release_run_id(cli, monkeypatch):
    args = sys.argv.copy()
    index = args.index("--run-id")
    del args[index : index + 2]
    monkeypatch.setattr(sys, "argv", args)
    with pytest.raises(SystemExit) as exc:
        decision.main()
    assert exc.value.code == 2
    assert not cli.exists()