Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

9.2 KB
Raw Blame
"""Exercise the release workflow's PyPI dispatch and monitor contract."""

# allow: no-sut-import — these tests execute workflow shell with a stub gh CLI.

from __future__ import annotations

import json
import os
import shutil
import subprocess
from uuid import UUID
from pathlib import Path

import pytest
import yaml


ROOT = Path(__file__).resolve().parents[2]
RELEASE = ROOT / ".github" / "workflows" / "release.yml"
PUBLISH = ROOT / ".github" / "workflows" / "publish.yml"
TITLE = "PyPI release 123 artifact 456 dispatch 11111111-2222-4333-8444-555555555555"
OLDER_TITLE = "PyPI release 123 artifact 456 dispatch aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"


def _jobs(path: Path) -> dict:
    return yaml.safe_load(path.read_text(encoding="utf-8"))["jobs"]


def _run(title: str, run_id: int, conclusion: str | None = None) -> dict:
    return {
        "databaseId": run_id,
        "displayTitle": title,
        "createdAt": "2026-09-29T12:00:10Z",
        "status": "completed" if conclusion is not None else "in_progress",
        "conclusion": conclusion,
    }


def _monitor(
    tmp_path: Path, lists: list[list[dict]], views: dict[int, list[dict]]
):
    step = next(
        step
        for step in _jobs(RELEASE)["monitor-pypi"]["steps"]
        if step.get("name") == "Wait for PyPI publish workflow to complete"
    )
    script = step["run"]
    assert script.count("max_wait=6300") == 1
    # Advance Bash's wall clock without waiting for the real 30s sleeps.
    script = "sleep() { SECONDS=$((SECONDS + 1)); }\n" + script.replace(
        "max_wait=6300", "max_wait=3"
    )

    fixture = tmp_path / "fixture.json"
    fixture.write_text(
        json.dumps({"lists": lists, "views": views}), encoding="utf-8"
    )
    state = tmp_path / "state.json"
    state.write_text(
        json.dumps({"list_calls": 0, "view_ids": []}), encoding="utf-8"
    )
    gh = tmp_path / "gh"
    gh.write_text(
        """#!/usr/bin/env python3
import json
import os
import sys
from pathlib import Path

fixture = json.loads(Path(os.environ['GH_FIXTURE']).read_text())
path = Path(os.environ['GH_STATE'])
state = json.loads(path.read_text())
if sys.argv[1:3] == ['run', 'list']:
    index = state['list_calls']
    state['list_calls'] += 1
    result = fixture['lists'][min(index, len(fixture['lists']) - 1)]
elif sys.argv[1:3] == ['run', 'view']:
    run_id = sys.argv[3]
    state['view_ids'].append(run_id)
    attempts = fixture['views'][run_id]
    index = state['view_ids'].count(run_id) - 1
    result = attempts[min(index, len(attempts) - 1)]
else:
    raise SystemExit(f'unexpected gh arguments: {sys.argv[1:]}')
path.write_text(json.dumps(state))
print(json.dumps(result))
""",
        encoding="utf-8",
    )
    gh.chmod(0o755)

    output = tmp_path / "github-output"
    env = {
        **os.environ,
        "PATH": f"{tmp_path}{os.pathsep}{os.environ['PATH']}",
        "GH_FIXTURE": str(fixture),
        "GH_STATE": str(state),
        "GITHUB_OUTPUT": str(output),
        "DISPATCH_TIME": "2026-09-29T12:00:00Z",
        "PUBLISH_TITLE": TITLE,
        "RELEASE_TAG": "v2.0.0",
    }
    result = subprocess.run(
        ["bash", "-c", script],
        check=False,
        capture_output=True,
        text=True,
        env=env,
        timeout=10,
    )
    return (
        result,
        output.read_text(encoding="utf-8"),
        json.loads(state.read_text()),
    )


def test_dispatch_has_unique_title_bound_to_payload_and_monitor():
    release = _jobs(RELEASE)
    trigger = release["trigger-pypi"]
    dispatch = next(
        step for step in trigger["steps"] if step.get("id") == "dispatch"
    )
    script = dispatch["with"]["script"]
    publish_name = yaml.safe_load(PUBLISH.read_text(encoding="utf-8"))[
        "run-name"
    ]

    assert "require('node:crypto').randomUUID()" in script
    assert "dispatch_id: dispatchId" in script
    assert "core.setOutput('publish_title', publishTitle)" in script
    assert "await github.rest.repos.createDispatchEvent" in script
    assert trigger["outputs"]["publish_title"] == (
        "${{ steps.dispatch.outputs.publish_title }}"
    )
    assert "${{ github.event.client_payload.dispatch_id }}" in publish_name
    assert "${{ github.event.client_payload.run_id }}" in publish_name
    assert "${{ github.event.client_payload.artifact_id }}" in publish_name
    monitor_step = next(
        step
        for step in release["monitor-pypi"]["steps"]
        if step.get("name") == "Wait for PyPI publish workflow to complete"
    )
    assert monitor_step["env"]["PUBLISH_TITLE"] == (
        "${{ needs.trigger-pypi.outputs.publish_title }}"
    )


@pytest.mark.skipif(
    shutil.which("node") is None, reason="Node.js is unavailable"
)
def test_each_trigger_attempt_emits_a_distinct_matching_dispatch_title():
    trigger = _jobs(RELEASE)["trigger-pypi"]
    script = next(
        step["with"]["script"]
        for step in trigger["steps"]
        if step.get("id") == "dispatch"
    )
    harness = """
const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;
const dispatches = [];
const outputs = {};
const github = {
  rest: { repos: { createDispatchEvent: async (payload) => dispatches.push(payload) } }
};
const context = { repo: { owner: 'LearningCircuit', repo: 'local-deep-research' }, runId: 123 };
const core = { setOutput: (key, value) => { outputs[key] = value; } };
(async () => {
  await new AsyncFunction('github', 'context', 'core', 'require', process.env.DISPATCH_SCRIPT)(
    github, context, core, require
  );
  console.log(JSON.stringify({dispatches, outputs}));
})().catch((error) => { console.error(error); process.exitCode = 1; });
"""
    env = {
        **os.environ,
        "DISPATCH_SCRIPT": script,
        "RELEASE_TAG": "v2.0.0",
        "RELEASE_SHA": "a" * 40,
        "PACKAGE_ARTIFACT_ID": "456",
        "PACKAGE_MANIFEST_SHA256": "b" * 64,
    }
    attempts = []
    for _ in range(2):
        result = subprocess.run(
            ["node", "-e", harness],
            check=True,
            capture_output=True,
            text=True,
            env=env,
            timeout=10,
        )
        attempts.append(json.loads(result.stdout.splitlines()[-1]))

    ids = []
    for attempt in attempts:
        payload = attempt["dispatches"][0]["client_payload"]
        dispatch_id = payload["dispatch_id"]
        assert UUID(dispatch_id).version == 4
        assert attempt["outputs"]["publish_title"] == (
            f"PyPI release 123 artifact 456 dispatch {dispatch_id}"
        )
        ids.append(dispatch_id)
    assert ids[0] != ids[1]


@pytest.mark.skipif(
    shutil.which("jq") is None, reason="workflow shell needs jq"
)
def test_unrelated_newer_success_cannot_hide_intended_failure(tmp_path):
    unrelated = _run(OLDER_TITLE, 900, "success")
    intended = _run(TITLE, 901, "failure")

    result, output, state = _monitor(
        tmp_path, [[unrelated, intended]], {901: [intended]}
    )

    assert result.returncode == 1, result.stderr
    assert output == "pypi_result=failure\n"
    assert state["view_ids"] == ["901"]


@pytest.mark.skipif(
    shutil.which("jq") is None, reason="workflow shell needs jq"
)
def test_monitor_waits_for_its_dispatch_while_other_run_succeeds(tmp_path):
    unrelated = _run(OLDER_TITLE, 900, "success")

    result, output, state = _monitor(tmp_path, [[unrelated]], {})

    assert result.returncode == 1, result.stderr
    assert output == "pypi_result=timed_out\n"
    assert state["view_ids"] == []


@pytest.mark.skipif(
    shutil.which("jq") is None, reason="workflow shell needs jq"
)
def test_selected_run_stays_pinned_across_polls(tmp_path):
    pending = _run(TITLE, 901)
    failed = _run(TITLE, 901, "failure")
    unrelated = _run(OLDER_TITLE, 902, "success")

    result, output, state = _monitor(
        tmp_path,
        [[pending], [unrelated]],
        {901: [pending, failed]},
    )

    assert result.returncode == 1, result.stderr
    assert output == "pypi_result=failure\n"
    assert state["list_calls"] == 1
    assert state["view_ids"] == ["901", "901"]


@pytest.mark.skipif(
    shutil.which("jq") is None, reason="workflow shell needs jq"
)
def test_matching_success_ignores_unrelated_failure(tmp_path):
    intended = _run(TITLE, 901, "success")
    unrelated = _run(OLDER_TITLE, 900, "failure")

    result, output, state = _monitor(
        tmp_path, [[unrelated, intended]], {901: [intended]}
    )

    assert result.returncode == 0, result.stderr
    assert output == "pypi_result=success\n"
    assert state["view_ids"] == ["901"]


@pytest.mark.skipif(
    shutil.which("jq") is None, reason="workflow shell needs jq"
)
def test_ambiguous_dispatch_identity_fails_closed(tmp_path):
    first = _run(TITLE, 901, "success")
    replay = _run(TITLE, 902, "success")

    result, output, state = _monitor(tmp_path, [[first, replay]], {})

    assert result.returncode == 1, result.stderr
    assert output == "pypi_result=ambiguous\n"
    assert state["view_ids"] == []


@pytest.mark.skipif(
    shutil.which("jq") is None, reason="workflow shell needs jq"
)
def test_run_view_identity_mismatch_fails_closed(tmp_path):
    intended = _run(TITLE, 901)
    wrong = _run(TITLE, 902, "success")

    result, output, state = _monitor(tmp_path, [[intended]], {901: [wrong]})

    assert result.returncode == 1, result.stderr
    assert output == "pypi_result=identity_mismatch\n"
    assert state["view_ids"] == ["901"]