Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

37.9 KB
Raw Blame
#!/usr/bin/env python3

from __future__ import annotations

import argparse
import contextlib
import io
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import subprocess

import reconcile_release_convergence as subject


WORKFLOW_ID = 42


def release(tag: str, published_at: str, *, prerelease: bool, immutable: bool = True):
    return {
        "id": abs(hash(tag)),
        "tag_name": tag,
        "draft": False,
        "prerelease": prerelease,
        "immutable": immutable,
        "published_at": published_at,
    }


def run(
    run_id: int,
    tag: str,
    created_at: str,
    *,
    status: str = "completed",
    conclusion: str = "failure",
):
    return {
        "id": run_id,
        "workflow_id": WORKFLOW_ID,
        "path": subject.CONVERGENCE_PATH,
        "event": "workflow_dispatch",
        "head_branch": "main",
        "display_title": f"Release convergence {tag} source 1234",
        "created_at": created_at,
        "status": status,
        "conclusion": conclusion,
    }


class TransientReadTests(unittest.TestCase):
    def result(self, code=0, stdout="{}", stderr=""):
        return subprocess.CompletedProcess([], code, stdout=stdout, stderr=stderr)

    def test_pages_discards_partial_output_before_retry(self):
        github = subject.GitHub("rcourtman/Pulse", "gh")
        results = [
            self.result(1, '[{"id": 1}]', "gh: gateway timeout (HTTP 504)"),
            self.result(0, '[{"id": 1}]\n[{"id": 2}]'),
        ]
        with patch.object(subject.subprocess, "run", side_effect=results) as command, patch("time.sleep") as sleep:
            self.assertEqual([[{"id": 1}], [{"id": 2}]], github.pages("repos/example/releases"))
        self.assertEqual(2, command.call_count)
        sleep.assert_called_once_with(1)

    def test_api_retries_transient_errors_with_bounded_backoff(self):
        github = subject.GitHub("rcourtman/Pulse", "gh")
        for status in (502, 503, 504):
            with self.subTest(status=status):
                failure = self.result(1, stderr=f"gh: unavailable (HTTP {status})")
                with patch.object(subject.subprocess, "run", side_effect=[failure, failure, self.result()]) as command, patch("time.sleep") as sleep:
                    self.assertEqual({}, github.api("repos/example"))
                self.assertEqual(3, command.call_count)
                self.assertEqual([1, 2], [call.args[0] for call in sleep.call_args_list])

    def test_exhausted_reads_fail_closed(self):
        github = subject.GitHub("rcourtman/Pulse", "gh")
        with patch.object(subject.subprocess, "run", return_value=self.result(1, stderr="gh: timeout (HTTP 504)")) as command, patch("time.sleep") as sleep:
            with self.assertRaisesRegex(subject.ReconciliationError, "HTTP 504"):
                github.pages("repos/example/releases")
        self.assertEqual(3, command.call_count)
        self.assertEqual(2, sleep.call_count)

    def test_nontransient_errors_are_not_retried(self):
        github = subject.GitHub("rcourtman/Pulse", "gh")
        for detail in ("HTTP 401", "HTTP 403", "HTTP 404", "HTTP 429", "terminal escape sequences refused"):
            with self.subTest(detail=detail):
                with patch.object(subject.subprocess, "run", return_value=self.result(1, stderr=detail)) as command, patch("time.sleep") as sleep:
                    error = subject.GitHubNotFound if detail == "HTTP 404" else subject.ReconciliationError
                    with self.assertRaises(error):
                        github.api("repos/example")
                self.assertEqual(1, command.call_count)
                sleep.assert_not_called()

    def test_mutations_are_not_retried(self):
        github = subject.GitHub("rcourtman/Pulse", "gh")
        for payload in (None, {"ref": "main"}):
            with self.subTest(payload=payload):
                with patch.object(subject.subprocess, "run", return_value=self.result(1, stderr="HTTP 504")) as command, patch("time.sleep") as sleep:
                    with self.assertRaises(subject.ReconciliationError):
                        github.post("repos/example/actions/dispatches", payload)
                self.assertEqual(1, command.call_count)
                sleep.assert_not_called()


class FailedReadIdentityTests(unittest.TestCase):
    def test_header_bearing_api_failure_names_the_resource_without_retry(self):
        github = subject.GitHub("rcourtman/Pulse", "gh")
        endpoint = "repos/rcourtman/Pulse/actions/runs/37395942598"
        for status in (401, 403, 404):
            with self.subTest(status=status):
                result = subprocess.CompletedProcess([], 1, stdout="", stderr=f"HTTP {status}")
                with patch.object(subject.subprocess, "run", return_value=result) as command, patch("time.sleep") as sleep:
                    with self.assertRaises(subject.ReconciliationError) as failure:
                        github.api(endpoint)
                self.assertIn(f"GET {endpoint}", str(failure.exception))
                self.assertNotIn("Accept:", str(failure.exception))
                if status == 404:
                    self.assertIsInstance(failure.exception, subject.GitHubNotFound)
                self.assertEqual(1, command.call_count)
                sleep.assert_not_called()

    def test_paginated_failure_omits_read_filters(self):
        github = subject.GitHub("rcourtman/Pulse", "gh")
        resource = "repos/rcourtman/Pulse/actions/workflows/release-convergence.yml/runs"
        result = subprocess.CompletedProcess([], 1, stdout="partial", stderr="HTTP 403")
        with patch.object(subject.subprocess, "run", return_value=result) as command, patch("time.sleep") as sleep:
            with self.assertRaises(subject.ReconciliationError) as failure:
                github.pages(resource + "?event=workflow_dispatch&private_filter=fixture-only")
        self.assertIn(f"GET {resource}", str(failure.exception))
        self.assertNotIn("private_filter", str(failure.exception))
        self.assertNotIn("fixture-only", str(failure.exception))
        self.assertEqual(1, command.call_count)
        sleep.assert_not_called()

    def test_unrecognised_resource_has_no_endpoint_disclosure(self):
        for endpoint in (
            "repos/another/private-repository/customer-fixture",
            "repos/rcourtman/Pulse/../customer-fixture",
            "repos/rcourtman/Pulse/" + "x" * 257,
            "repos/rcourtman/Pulse/customer-fixture\nforged diagnostic",
        ):
            with self.subTest(endpoint=endpoint):
                self.assertEqual("GitHub API read", subject.api_read_subject(endpoint))

    def test_discovery_stops_at_the_identified_failed_read_without_mutation(self):
        github = subject.GitHub("rcourtman/Pulse", "gh")
        results = [
            subprocess.CompletedProcess([], 0, stdout='{"default_branch": "main"}', stderr=""),
            subprocess.CompletedProcess([], 1, stdout="", stderr="HTTP 404"),
        ]
        with patch.object(subject.subprocess, "run", side_effect=results) as command, patch("time.sleep") as sleep:
            with self.assertRaisesRegex(subject.GitHubNotFound, "GET repos/rcourtman/Pulse/actions/workflows/release-convergence.yml"):
                subject.discover(github)
        self.assertEqual(2, command.call_count)
        self.assertTrue(all("POST" not in call.args[0] for call in command.call_args_list))
        sleep.assert_not_called()


class LatestFailedRunsTests(unittest.TestCase):
    def select(self, releases, runs):
        return subject.latest_failed_runs(
            releases, runs, workflow_id=WORKFLOW_ID, default_branch="main"
        )

    def test_selects_failed_heads_of_stable_and_preview_channels(self):
        releases = [
            release("v6.4.1", "2026-09-01T00:00:00Z", prerelease=False),
            release("v6.5.0-rc.1", "2026-09-02T00:00:00Z", prerelease=True),
        ]
        runs = [
            run(10, "v6.4.1", "2026-09-01T00:01:00Z"),
            run(11, "v6.5.0-rc.1", "2026-09-02T00:01:00Z"),
        ]
        self.assertEqual([10, 11], self.select(releases, runs))

    def test_does_not_fall_back_behind_a_mutable_channel_head(self):
        releases = [
            release("v6.4.1", "2026-09-01T00:00:00Z", prerelease=False),
            release(
                "v6.4.2", "2026-09-02T00:00:00Z", prerelease=False, immutable=False
            ),
        ]
        self.assertEqual([], self.select(releases, [run(10, "v6.4.1", "2026-09-01T00:01:00Z")]))

    def test_ignores_newer_companion_chart_releases(self):
        releases = [
            release("v6.5.0-rc.1", "2026-09-02T00:00:00Z", prerelease=True),
            release(
                "helm-chart-6.5.0-rc.1",
                "2026-09-02T00:01:00Z",
                prerelease=True,
            ),
        ]
        self.assertEqual(
            [10],
            self.select(
                releases, [run(10, "v6.5.0-rc.1", "2026-09-02T00:02:00Z")]
            ),
        )

    def test_newer_success_or_active_run_clears_old_debt(self):
        releases = [release("v6.5.0-rc.1", "2026-09-02T00:00:00Z", prerelease=True)]
        failed = run(10, "v6.5.0-rc.1", "2026-09-02T00:01:00Z")
        success = run(
            11,
            "v6.5.0-rc.1",
            "2026-09-02T00:02:00Z",
            conclusion="success",
        )
        self.assertEqual([], self.select(releases, [failed, success]))
        active = run(
            12,
            "v6.5.0-rc.1",
            "2026-09-02T00:03:00Z",
            status="in_progress",
            conclusion="",
        )
        self.assertEqual([], self.select(releases, [failed, active]))

    def test_ignores_wrong_workflow_branch_and_title(self):
        releases = [release("v6.5.0-rc.1", "2026-09-02T00:00:00Z", prerelease=True)]
        wrong_workflow = run(10, "v6.5.0-rc.1", "2026-09-02T00:01:00Z")
        wrong_workflow["workflow_id"] = 99
        wrong_branch = run(11, "v6.5.0-rc.1", "2026-09-02T00:02:00Z")
        wrong_branch["head_branch"] = "release/v6.5"
        wrong_title = run(12, "v6.5.0-rc.1", "2026-09-02T00:03:00Z")
        wrong_title["display_title"] += " injected"
        self.assertEqual(
            [], self.select(releases, [wrong_workflow, wrong_branch, wrong_title])
        )


class MarkerTests(unittest.TestCase):
    def test_marker_binds_release_source_and_all_customer_digests(self):
        current_release = {
            "id": 88,
            "target_commitish": "a" * 40,
        }
        marker = {
            "schema_version": 1,
            "tag": "v6.5.0-rc.1",
            "release_id": "88",
            "target_commitish": "a" * 40,
            "source_release_run_id": "1234",
            "convergence_run_id": "5678",
            "r2_prefix": "packet",
            "server_image_digest": "sha256:" + "b" * 64,
            "control_plane_image_digest": "sha256:" + "c" * 64,
            "helm_chart_digest": "sha256:" + "d" * 64,
        }
        self.assertEqual(
            5678,
            subject.validate_marker(
                marker,
                release=current_release,
                tag="v6.5.0-rc.1",
                source_run_id=1234,
            ),
        )
        marker["server_image_digest"] = "sha256:bad"
        with self.assertRaisesRegex(subject.ReconciliationError, "server digest"):
            subject.validate_marker(
                marker,
                release=current_release,
                tag="v6.5.0-rc.1",
                source_run_id=1234,
            )


class FakeGitHub:
    repository = "rcourtman/Pulse"

    def __init__(self, *, current_controls: bool = False, committed: bool = True):
        self.run_id = 100
        self.source_run_id = 200
        self.owner_run_id = 90
        self.title = "Release convergence v6.5.0-rc.1 source 200"
        self.old_sha = "a" * 40
        self.main_sha = self.old_sha if current_controls else "b" * 40
        self.committed = committed
        self.posts = []
        self.runs = [self.run(self.run_id, self.old_sha)]
        self.marker = {
            "schema_version": 1,
            "tag": "v6.5.0-rc.1",
            "release_id": "300",
            "target_commitish": "c" * 40,
            "source_release_run_id": "200",
            "convergence_run_id": "90",
            "r2_prefix": "packet",
            "server_image_digest": "sha256:" + "d" * 64,
            "control_plane_image_digest": "sha256:" + "e" * 64,
            "helm_chart_digest": "sha256:" + "f" * 64,
        }

    def run(self, run_id, head_sha, *, attempt=1, status="completed", conclusion="failure"):
        return {
            "id": run_id,
            "workflow_id": 42,
            "path": subject.CONVERGENCE_PATH,
            "event": "workflow_dispatch",
            "repository": {"full_name": self.repository},
            "display_title": self.title,
            "head_branch": "main",
            "head_sha": head_sha,
            "status": status,
            "conclusion": conclusion,
            "run_attempt": attempt,
            "created_at": f"2026-09-02T00:{run_id % 60:02d}:00Z",
        }

    def api(self, endpoint):
        if endpoint == f"repos/{self.repository}":
            return {"default_branch": "main"}
        suffix = endpoint.removeprefix(f"repos/{self.repository}/")
        if suffix == "actions/workflows/release-convergence.yml":
            return {"id": 42}
        if suffix == "actions/workflows/create-release.yml":
            return {"id": 43}
        if suffix == "commits/main":
            return {"sha": self.main_sha}
        if suffix == "releases/tags/v6.5.0-rc.1":
            if not self.committed:
                return {"tag_name": "v6.5.0-rc.1", "draft": True}
            return {
                "id": 300,
                "tag_name": "v6.5.0-rc.1",
                "target_commitish": "c" * 40,
                "draft": False,
                "immutable": True,
                "published_at": "2026-09-02T00:00:00Z",
                "prerelease": True,
            }
        if suffix.startswith("actions/runs/"):
            run_id = int(suffix.rsplit("/", 1)[1])
            if run_id == self.owner_run_id:
                owner = self.run(self.owner_run_id, self.old_sha, conclusion="success")
                owner["status"] = "completed"
                return owner
            if run_id == self.source_run_id:
                return {
                    "id": self.source_run_id,
                    "workflow_id": 43,
                    "path": subject.CREATE_RELEASE_PATH,
                    "event": "workflow_dispatch",
                    "repository": {"full_name": self.repository},
                    "status": "in_progress",
                }
            return next(value for value in self.runs if value["id"] == run_id)
        raise AssertionError(endpoint)

    def pages(self, endpoint):
        self.assert_endpoint = endpoint
        return [{"workflow_runs": self.runs}]

    def download_marker(self, tag, directory):
        self.download = (tag, directory)
        return self.marker

    def post(self, endpoint, payload=None):
        self.posts.append((endpoint, payload))
        return True

    def unchanged_credential_containment_block(self, run_id):
        self.containment_probe = run_id
        return False


class DecisionSummaryTests(unittest.TestCase):
    def test_local_execution_needs_no_summary_file(self):
        with patch.dict(os.environ, {}, clear=True), contextlib.redirect_stdout(io.StringIO()) as output:
            subject.report_decision("No action.")
        self.assertEqual("No action.\n", output.getvalue())

    def test_summary_appends_escaped_decisions_without_claiming_delivery(self):
        with tempfile.TemporaryDirectory() as directory:
            path = Path(directory) / "summary.md"
            path.write_text("Existing summary\n", encoding="utf-8")
            with patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": str(path)}):
                subject.report_decision("held <not delivered>")
                subject.report_decision("second decision")
            summary = path.read_text(encoding="utf-8")
        self.assertTrue(summary.startswith("Existing summary\n"))
        self.assertIn("held &lt;not delivered&gt;", summary)
        self.assertIn("second decision", summary)
        self.assertIn("not evidence of successful release convergence", summary)

    def test_empty_discovery_does_not_claim_absent_debt(self):
        args = argparse.Namespace(repository="rcourtman/Pulse", run_id=None,
                                  latest=True, max_attempts=5, dry_run=True)
        with patch.object(subject, "parse_args", return_value=args), \
                patch.object(subject, "discover", return_value=[]), \
                patch.object(subject, "report_decision") as report:
            self.assertEqual(0, subject.main())
        message = report.call_args.args[0]
        self.assertIn("No failed run selected for retry", message)
        self.assertIn("missing convergence runs are not qualified", message)

    def discover_with_releases(self, releases):
        github = FakeGitHub()
        github.pages = lambda endpoint: (
            [releases] if "/releases?" in endpoint
            else [{"workflow_runs": github.runs}]
        )
        with tempfile.TemporaryDirectory() as directory:
            path = Path(directory) / "summary.md"
            with patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": str(path)}), \
                    contextlib.redirect_stdout(io.StringIO()) as output:
                selected = subject.discover(github)
            summary = path.read_text() if path.exists() else ""
        self.assertEqual([], github.posts)
        return selected, output.getvalue(), summary

    def test_mutable_stable_debt_visible_even_when_preview_retry_selected(self):
        selected, output, summary = self.discover_with_releases([
            release("v6.4.1", "2026-09-01T00:00:00Z",
                    prerelease=False, immutable=False),
            release("v6.5.0-rc.1", "2026-09-02T00:00:00Z", prerelease=True),
        ])
        self.assertEqual([100], selected)
        for text in (output, summary):
            self.assertIn("Current stable head v6.4.1", text)
            self.assertIn("continuity debt remains", text)
            self.assertIn("No retry or fallback", text)

    def test_unknown_immutability_reports_debt_without_older_fallback(self):
        for unknown in (None, "true", 1, False):
            with self.subTest(immutable=unknown):
                head = release("v6.5.0-rc.2", "2026-09-03T00:00:00Z",
                               prerelease=True)
                head["immutable"] = unknown
                selected, output, summary = self.discover_with_releases([
                    release("v6.5.0-rc.1", "2026-09-02T00:00:00Z", prerelease=True),
                    head,
                ])
                self.assertEqual([], selected)
                self.assertIn("Current preview head v6.5.0-rc.2", output)
                self.assertIn("no confirmed immutable activation commit", summary)

    def test_old_mutable_draft_and_companion_do_not_report_current_debt(self):
        draft = release("v6.6.0", "2026-09-04T00:00:00Z",
                        prerelease=False, immutable=False)
        draft["draft"] = True
        selected, output, summary = self.discover_with_releases([
            release("v6.4.0-rc.1", "2026-09-01T00:00:00Z",
                    prerelease=True, immutable=False),
            release("v6.5.0-rc.1", "2026-09-02T00:00:00Z", prerelease=True),
            release("helm-chart-6.5.0", "2026-09-03T00:00:00Z",
                    prerelease=False, immutable=False),
            draft,
        ])
        self.assertEqual([100], selected)
        self.assertEqual("", output)
        self.assertEqual("", summary)

    def test_credential_hold_is_visible_without_retry(self):
        github = FakeGitHub(current_controls=True)
        github.unchanged_credential_containment_block = lambda run_id: True
        with tempfile.TemporaryDirectory() as directory:
            path = Path(directory) / "summary.md"
            with patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": str(path)}):
                subject.reconcile(github, github.run_id, 5)
            summary = path.read_text(encoding="utf-8")
        self.assertEqual([], github.posts)
        self.assertIn("held by unchanged private credential containment", summary)
        self.assertIn("no unattended retry was dispatched", summary)


class JobLogTests(unittest.TestCase):
    def test_uses_captured_sanitised_job_reader_with_private_auth(self):
        github = subject.GitHub("rcourtman/Pulse", "gh", mutate=False)
        log = "Require credential containment\tcheck\tcredential containment gate: BLOCKED\n"
        with patch.object(subject.subprocess, "run", return_value=subprocess.CompletedProcess(
            [], 0, stdout=log, stderr=""
        )) as command, contextlib.redirect_stdout(io.StringIO()) as output:
            self.assertEqual(log, github.job_log(
                subject.PRIVATE_REPOSITORY, 123, token="test-private-token"
            ))
        self.assertEqual("", output.getvalue())
        args, kwargs = command.call_args
        self.assertEqual(["gh", "run", "view", "--repo", subject.PRIVATE_REPOSITORY,
                          "--job", "123", "--log"], args[0])
        self.assertTrue(kwargs["capture_output"])
        self.assertEqual("test-private-token", kwargs["env"]["GH_TOKEN"])

    def test_unavailable_job_logs_fail_closed(self):
        github = subject.GitHub("rcourtman/Pulse", "gh", mutate=False)
        with patch.object(subject.subprocess, "run", return_value=subprocess.CompletedProcess(
            [], 1, stdout="", stderr="log unavailable"
        )):
            with self.assertRaisesRegex(subject.ReconciliationError, "log unavailable"):
                github.job_log(subject.PRIVATE_REPOSITORY, 123)


class CredentialContainmentTests(unittest.TestCase):
    def github(
        self,
        *,
        current_head=None,
        containment="failure",
        containment_log=subject.CREDENTIAL_BLOCK_MARKER,
        containment_state_changed=False,
        extra_public_job=None,
        verdict="failure",
    ):
        private_run_id = 700
        paid_job_id = 800
        private_head = "d" * 40
        current_head = current_head or private_head
        github = subject.GitHub(
            "rcourtman/Pulse", "gh", mutate=False, private_token="private-token"
        )

        def pages(endpoint, *, token=""):
            if endpoint == (
                "repos/rcourtman/Pulse/actions/runs/100/jobs?per_page=100"
            ):
                self.assertEqual("", token)
                return [
                    {
                        "jobs": [
                            {
                                "id": paid_job_id,
                                "name": subject.PAID_RUNTIME_JOB,
                                "status": "completed",
                                "conclusion": "failure",
                            },
                            {
                                "id": paid_job_id + 1,
                                "name": subject.CONVERGENCE_VERDICT_JOB,
                                "status": "completed",
                                "conclusion": verdict,
                            },
                            *([extra_public_job] if extra_public_job else []),
                        ]
                    }
                ]
            if endpoint == (
                f"repos/rcourtman/Pulse/check-runs/{paid_job_id}/annotations?per_page=100"
            ):
                self.assertEqual("", token)
                return [
                    [
                        {
                            "message": "private Pro live promotion failed: "
                            f"https://github.com/rcourtman/pulse-pro/actions/runs/{private_run_id}"
                        }
                    ]
                ]
            if endpoint == (
                f"repos/{subject.PRIVATE_REPOSITORY}/actions/runs/{private_run_id}/jobs?per_page=100"
            ):
                self.assertEqual("private-token", token)
                return [
                    {
                        "jobs": [
                            {
                                "id": 900,
                                "name": subject.CREDENTIAL_CONTAINMENT_JOB,
                                "conclusion": containment,
                            }
                        ]
                    }
                ]
            raise AssertionError(endpoint)

        def api(endpoint, *, token=""):
            self.assertEqual("private-token", token)
            if endpoint == f"repos/{subject.PRIVATE_REPOSITORY}/actions/runs/{private_run_id}":
                return {
                    "repository": {"full_name": subject.PRIVATE_REPOSITORY},
                    "path": subject.PRIVATE_PROMOTION_PATH,
                    "event": "workflow_dispatch",
                    "status": "completed",
                    "conclusion": "failure",
                    "head_sha": private_head,
                }
            if endpoint == f"repos/{subject.PRIVATE_REPOSITORY}/commits/main":
                return {"sha": current_head}
            content_prefix = f"repos/{subject.PRIVATE_REPOSITORY}/contents/"
            if endpoint.startswith(content_prefix):
                path, ref = endpoint.removeprefix(content_prefix).split("?ref=", 1)
                blob = "1" * 40 if path == subject.CREDENTIAL_CONTAINMENT_PATHS[0] else "2" * 40
                if containment_state_changed and ref == current_head:
                    blob = "3" * 40
                return {"type": "file", "sha": blob}
            raise AssertionError(endpoint)

        github.pages = pages
        github.api = api
        github.job_log = lambda repository, job_id, token="": (
            containment_log
            if (repository, job_id, token)
            == (subject.PRIVATE_REPOSITORY, 900, "private-token")
            else self.fail((repository, job_id, token))
        )
        return github

    def test_recognises_unchanged_private_credential_block(self):
        self.assertTrue(self.github().unchanged_credential_containment_block(100))

    def test_private_change_rearms_convergence(self):
        github = self.github(
            current_head="e" * 40, containment_state_changed=True
        )
        self.assertFalse(github.unchanged_credential_containment_block(100))

    def test_unrelated_private_change_does_not_rearm_convergence(self):
        github = self.github(current_head="e" * 40)
        self.assertTrue(github.unchanged_credential_containment_block(100))

    def test_other_private_failure_remains_retriable(self):
        github = self.github(containment="success")
        self.assertFalse(github.unchanged_credential_containment_block(100))

    def test_other_public_surface_failure_remains_retriable(self):
        github = self.github(
            extra_public_job={
                "id": 802,
                "name": "Converge Helm Pages / release",
                "status": "completed",
                "conclusion": "failure",
            }
        )
        self.assertFalse(github.unchanged_credential_containment_block(100))

    def test_interrupted_public_job_remains_retriable(self):
        github = self.github(
            extra_public_job={
                "id": 802,
                "name": "Release global customer-promotion lease",
                "status": "completed",
                "conclusion": "cancelled",
            }
        )
        self.assertFalse(github.unchanged_credential_containment_block(100))

    def test_missing_failed_aggregate_verdict_remains_retriable(self):
        github = self.github(verdict="success")
        self.assertFalse(github.unchanged_credential_containment_block(100))

    def test_duplicate_failed_aggregate_verdict_remains_retriable(self):
        github = self.github(
            extra_public_job={
                "id": 802,
                "name": subject.CONVERGENCE_VERDICT_JOB,
                "status": "completed",
                "conclusion": "failure",
            }
        )
        self.assertFalse(github.unchanged_credential_containment_block(100))

    def test_malformed_public_job_remains_retriable(self):
        github = self.github(
            extra_public_job={
                "id": 802,
                "status": "completed",
                "conclusion": "failure",
            }
        )
        self.assertFalse(github.unchanged_credential_containment_block(100))

    def test_containment_job_error_without_block_marker_remains_retriable(self):
        github = self.github(containment_log="checkout failed")
        self.assertFalse(github.unchanged_credential_containment_block(100))

    def test_missing_private_token_cannot_weaken_retry(self):
        github = subject.GitHub("rcourtman/Pulse", "gh", mutate=False)
        github.pages = lambda endpoint: self.fail(endpoint)
        self.assertFalse(github.unchanged_credential_containment_block(100))


class ReconciliationTests(unittest.TestCase):
    def test_recovery_receipts_require_successful_submission(self):
        cases = [
            ({}, "Dispatched fresh convergence controls"),
            ({"current_controls": True}, "Re-ran current-control convergence"),
            ({"committed": False}, "Renewed pre-commit convergence owner"),
        ]
        for options, receipt in cases:
            for code in (0, 1):
                with self.subTest(options=options, code=code):
                    github = FakeGitHub(**options)
                    transport = subject.GitHub(github.repository, "gh")
                    github.post = transport.post
                    result = subprocess.CompletedProcess(
                        [], code, stdout="", stderr="rejected" if code else ""
                    )
                    output = io.StringIO()
                    with patch.object(subject.subprocess, "run", return_value=result) as command, contextlib.redirect_stdout(output):
                        if code:
                            with self.assertRaises(subject.ReconciliationError):
                                subject.reconcile(github, github.run_id, 5)
                        else:
                            subject.reconcile(github, github.run_id, 5)
                    command.assert_called_once()
                    if code:
                        self.assertNotIn(receipt, output.getvalue())
                    else:
                        self.assertIn(receipt, output.getvalue())
                    self.assertNotIn("DRY RUN", output.getvalue())

    def test_post_reports_submission_only_after_success(self):
        for payload in (None, {"ref": "main"}):
            for code in (0, 1):
                with self.subTest(payload=payload, code=code):
                    github = subject.GitHub("rcourtman/Pulse", "gh")
                    result = subprocess.CompletedProcess([], code, stdout="", stderr="failed" if code else "")
                    with patch.object(subject.subprocess, "run", return_value=result) as command:
                        if code:
                            with self.assertRaises(subject.ReconciliationError):
                                github.post("repos/rcourtman/Pulse/test", payload)
                        else:
                            self.assertIs(True, github.post("repos/rcourtman/Pulse/test", payload))
                    command.assert_called_once()

    def test_dry_run_receipts_never_claim_a_mutation(self):
        cases = [
            ({}, "Would dispatch", "Dispatched"),
            ({"current_controls": True}, "Would re-run", "Re-ran"),
            ({"committed": False}, "Would renew", "Renewed"),
        ]
        for options, expected, forbidden in cases:
            with self.subTest(options=options):
                github = FakeGitHub(**options)
                transport = subject.GitHub(github.repository, "gh", mutate=False)
                github.post = transport.post
                output = io.StringIO()
                with patch.object(subject.subprocess, "run") as command, contextlib.redirect_stdout(output):
                    subject.reconcile(github, github.run_id, 5)
                command.assert_not_called()
                self.assertIn("DRY RUN: " + expected, output.getvalue())
                self.assertNotIn(forbidden, output.getvalue())
                self.assertEqual([], github.posts)

    def test_stale_failed_run_dispatches_current_controls_with_bound_inputs(self):
        github = FakeGitHub()
        subject.reconcile(github, github.run_id, 5)
        self.assertEqual(1, len(github.posts))
        endpoint, payload = github.posts[0]
        self.assertTrue(endpoint.endswith("release-convergence.yml/dispatches"))
        self.assertEqual("main", payload["ref"])
        self.assertEqual("v6.5.0-rc.1", payload["inputs"]["tag"])
        self.assertEqual("200", payload["inputs"]["source_release_run_id"])
        self.assertEqual("true", payload["inputs"]["prerelease"])

    def test_current_failed_run_uses_exact_rerun(self):
        github = FakeGitHub(current_controls=True)
        subject.reconcile(github, github.run_id, 5)
        self.assertEqual(
            [(f"repos/{github.repository}/actions/runs/{github.run_id}/rerun", None)],
            github.posts,
        )

    def test_newer_sibling_prevents_duplicate_mutation(self):
        github = FakeGitHub()
        github.runs.append(github.run(101, github.main_sha, status="in_progress", conclusion=""))
        subject.reconcile(github, github.run_id, 5)
        self.assertEqual([], github.posts)

    def test_exhausted_current_controls_are_a_stable_noop(self):
        github = FakeGitHub(current_controls=True)
        github.runs[0]["run_attempt"] = 3
        github.runs.append(github.run(101, github.old_sha, attempt=2))
        output = io.StringIO()
        with contextlib.redirect_stdout(output):
            subject.reconcile(github, 101, 5)
        self.assertEqual([], github.posts)
        self.assertIn(
            "remains failed after the 5-attempt retry budget", output.getvalue()
        )

    def test_repaired_controls_rearm_exhausted_stale_debt(self):
        github = FakeGitHub()
        github.runs[0]["run_attempt"] = 5
        subject.reconcile(github, github.run_id, 5)
        self.assertEqual(1, len(github.posts))
        endpoint, payload = github.posts[0]
        self.assertTrue(endpoint.endswith("release-convergence.yml/dispatches"))
        self.assertEqual("main", payload["ref"])

    def test_attempt_budget_resets_for_repaired_controls(self):
        github = FakeGitHub()
        github.runs[0]["run_attempt"] = 5
        github.runs.append(github.run(101, github.main_sha))
        subject.reconcile(github, 101, 5)
        self.assertEqual(
            [(f"repos/{github.repository}/actions/runs/101/rerun", None)],
            github.posts,
        )

    def test_unchanged_credential_block_is_not_retried(self):
        github = FakeGitHub(current_controls=True)
        github.unchanged_credential_containment_block = lambda run_id: True
        subject.reconcile(github, github.run_id, 5)
        self.assertEqual([], github.posts)

    @staticmethod
    def delete_owner_run(github, error):
        fake_api = github.api

        def api(endpoint):
            if endpoint.endswith(f"/actions/runs/{github.owner_run_id}"):
                raise error
            return fake_api(endpoint)

        github.api = api

    def test_deleted_activation_owner_is_terminal_for_fresh_controls(self):
        # v6.5.0's owner run was deleted from Actions on 13 Oct 2026; the
        # immutable marker still names it. Deletion implies a terminal run.
        github = FakeGitHub()
        self.delete_owner_run(github, subject.GitHubNotFound("GitHub object was not found"))
        with contextlib.redirect_stdout(io.StringIO()) as output:
            subject.reconcile(github, github.run_id, 5)
        self.assertIn("owner run 90 for committed v6.5.0-rc.1 was deleted", output.getvalue())
        self.assertEqual(1, len(github.posts))
        endpoint, payload = github.posts[0]
        self.assertTrue(endpoint.endswith("/release-convergence.yml/dispatches"))
        self.assertEqual("200", payload["inputs"]["source_release_run_id"])

    def test_deleted_activation_owner_reruns_current_controls(self):
        github = FakeGitHub(current_controls=True)
        self.delete_owner_run(github, subject.GitHubNotFound("GitHub object was not found"))
        with contextlib.redirect_stdout(io.StringIO()):
            subject.reconcile(github, github.run_id, 5)
        self.assertEqual(
            [(f"repos/{github.repository}/actions/runs/{github.run_id}/rerun", None)],
            github.posts,
        )

    def test_unreadable_activation_owner_is_not_treated_as_deleted(self):
        github = FakeGitHub()
        self.delete_owner_run(github, subject.ReconciliationError("GitHub command failed"))
        with self.assertRaises(subject.ReconciliationError):
            subject.reconcile(github, github.run_id, 5)
        self.assertEqual([], github.posts)

    def test_one_channel_failure_does_not_starve_the_other(self):
        args = argparse.Namespace(repository="rcourtman/Pulse", run_id=None,
                                  latest=True, max_attempts=5, dry_run=True)
        attempted = []

        def reconcile(github, run_id, max_attempts):
            attempted.append(run_id)
            if run_id == 1:
                raise subject.ReconciliationError("stable evidence is gone")

        stderr = io.StringIO()
        with patch.object(subject, "parse_args", return_value=args), \
                patch.object(subject, "discover", return_value=[1, 2]), \
                patch.object(subject, "reconcile", side_effect=reconcile), \
                contextlib.redirect_stderr(stderr):
            self.assertEqual(1, subject.main())
        self.assertEqual([1, 2], attempted)
        self.assertIn("failed for run 1: stable evidence is gone", stderr.getvalue())

    def test_precommit_owner_is_renewed_without_using_committed_budget(self):
        github = FakeGitHub(committed=False)
        github.runs[0]["run_attempt"] = 6
        subject.reconcile(github, github.run_id, 5)
        self.assertEqual(
            [(f"repos/{github.repository}/actions/runs/{github.run_id}/rerun", None)],
            github.posts,
        )


if __name__ == "__main__":
    unittest.main()