Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

27.1 KB
Raw Blame
#!/usr/bin/env python3
from __future__ import annotations

import os
import json
import pathlib
import re
import subprocess
import sys
import tempfile
import unittest


ROOT = pathlib.Path(__file__).resolve().parents[3]
sys.path.insert(0, str(ROOT / "scripts"))

from shard_go_tests import _test_regex, allocate_cpu_plan, build_plan, write_plan


class ReleasePreflightTest(unittest.TestCase):
    def test_backend_cpu_plan_reserves_non_api_capacity_without_oversubscription(
        self,
    ) -> None:
        widths, non_api = allocate_cpu_plan([3624, 15, 16], 8)
        self.assertEqual(widths, [4, 1, 1])
        self.assertEqual(non_api, 2)
        self.assertEqual(sum(widths) + non_api, 8)

        # Stable releases use a four-vCPU hosted runner. Keep the measured
        # three-way API partition there as 1/1/1 plus one non-API worker;
        # falling back to two equal-count shards exceeded the 45-minute
        # watchdog in stable v6.4.0 run 33220050114.
        widths, non_api = allocate_cpu_plan([3624, 15, 16], 4)
        self.assertEqual(widths, [1, 1, 1])
        self.assertEqual(non_api, 1)
        self.assertEqual(sum(widths) + non_api, 4)

        widths, non_api = allocate_cpu_plan([10, 10], 4)
        self.assertEqual(widths, [1, 1])
        self.assertEqual(non_api, 2)
        self.assertEqual(sum(widths) + non_api, 4)

        widths, non_api = allocate_cpu_plan([10], 1)
        self.assertEqual(widths, [1])
        self.assertEqual(non_api, 0)

        for counts, vcpus in [([], 8), ([1], 0), ([1, 1], 1), ([0], 1)]:
            with self.subTest(counts=counts, vcpus=vcpus):
                with self.assertRaises(ValueError):
                    allocate_cpu_plan(counts, vcpus)

    def run_script(
        self, *args: str, env: dict[str, str] | None = None
    ) -> subprocess.CompletedProcess[str]:
        command_env = os.environ.copy()
        if env:
            command_env.update(env)
        return subprocess.run(
            [str(ROOT / "scripts/run-release-preflight.sh"), *args],
            cwd=ROOT,
            env=command_env,
            check=False,
            capture_output=True,
            text=True,
        )

    def test_plan_resolves_exact_sha_and_wsl_transport(self) -> None:
        sha = subprocess.check_output(
            ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
        ).strip()
        result = self.run_script(
            "--profile",
            "rehearsal",
            "--host",
            "test-worker",
            "--wsl-distro",
            "Ubuntu",
            "--plan",
        )
        self.assertEqual(result.returncode, 0, result.stderr)
        self.assertIn(f"SHA:     {sha}", result.stdout)
        self.assertIn("Runtime: WSL Ubuntu", result.stdout)

    def test_missing_optional_worker_is_a_non_gate(self) -> None:
        configured_host = subprocess.run(
            ["git", "config", "--get", "pulse.releasePreflightHost"],
            cwd=ROOT,
            check=False,
            capture_output=True,
            text=True,
        )
        if configured_host.returncode == 0:
            self.skipTest("repository has a local release-preflight host configured")
        result = self.run_script(
            "--profile",
            "release",
            "--if-configured",
            env={
                "PULSE_RELEASE_PREFLIGHT_HOST": "",
                "PULSE_RELEASE_PREFLIGHT_WSL_DISTRO": "",
                "GIT_CONFIG_NOSYSTEM": "1",
            },
        )
        self.assertEqual(result.returncode, 0, result.stderr)
        self.assertIn("canonical hosted checks", result.stdout)

    def test_worker_rejects_non_exact_sha_before_touching_worker_state(self) -> None:
        result = subprocess.run(
            [str(ROOT / "scripts/release-preflight-worker.sh"), "HEAD", "rehearsal"],
            cwd=ROOT,
            check=False,
            capture_output=True,
            text=True,
        )
        self.assertEqual(result.returncode, 2)
        self.assertIn("40-character Git commit id", result.stderr)

    def test_dispatch_helpers_select_the_matching_profiles(self) -> None:
        dry_run = (ROOT / "scripts/trigger-release-dry-run.sh").read_text()
        release = (ROOT / "scripts/trigger-release.sh").read_text()
        workflow = (ROOT / ".github/workflows/release-dry-run.yml").read_text()
        self.assertIn("--profile rehearsal", dry_run)
        self.assertIn("--profile release", release)
        self.assertIn("--if-configured", dry_run)
        self.assertIn("--if-configured", release)
        self.assertIn('PULSE_E2E_DIAGNOSTIC: "1"', workflow)

    def assert_release_dry_run_diagnostic_contract(
        self, workflow: str, diagnostic: str
    ) -> None:
        installed_browser = re.search(
            r"npx playwright install --with-deps ([a-z-]+)", workflow
        )
        diagnostic_step = re.search(
            r"      - name: Run integration diagnostics\n(?P<body>.*?)(?=\n      - name:)",
            workflow,
            flags=re.DOTALL,
        )
        self.assertIsNotNone(diagnostic_step)
        step_body = diagnostic_step.group("body")
        diagnostic_command = re.search(
            r"npx playwright test tests/00-diagnostic\.spec\.ts (?P<args>[^\n]+)",
            step_body,
        )
        self.assertIsNotNone(installed_browser)
        self.assertIsNotNone(diagnostic_command)
        self.assertIn('PULSE_E2E_DIAGNOSTIC: "1"', step_body)
        self.assertLess(
            step_body.index("set -o pipefail"),
            step_body.index("npx playwright test tests/00-diagnostic.spec.ts"),
        )
        self.assertIn("2>&1 | tee diagnostic-evidence/playwright-diagnostic.log", step_body)
        selected_browser = re.search(
            r"--project(?:=|\s+)([a-z-]+)", diagnostic_command.group("args")
        )
        self.assertIsNotNone(
            selected_browser,
            "release diagnostic must select the one browser installed by the job",
        )
        self.assertEqual(selected_browser.group(1), installed_browser.group(1))
        self.assertIn("--retries=0", diagnostic_command.group("args"))

        self.assertRegex(
            diagnostic,
            r"test\.describe\.configure\(\{\s*retries:\s*0\s*\}\);",
        )
        self.assertIn("test.setTimeout(120_000)", diagnostic)

        for required_diagnostic in (
            "API readiness failed:",
            "dependencies.monitor",
            "dependencies.scheduler",
            "dependencies.websocket",
            "/api/security/status",
            "hasAuthentication",
            "Rendered UI readiness failed:",
            "Pulse app shell did not render within 20s",
            "effectiveRenderedOpacity",
            "stayed transparent",
            "Pulse Setup Wizard",
            "Paste your bootstrap token",
            "diagnostic-evidence",
            "release-dry-run-rendered-readiness-chromium.png",
            "testInfo.attach(",
            "DIAGNOSTIC EVIDENCE ERROR:",
            "if (!readinessFailed)",
        ):
            self.assertIn(required_diagnostic, diagnostic)

        self.assertIn("Collect integration diagnostic runtime evidence", workflow)
        self.assertIn("Upload integration diagnostic evidence", workflow)
        self.assertIn("if-no-files-found: error", workflow)
        self.assertIn("tests/integration/test-results/", workflow)
        self.assertNotIn("expect(true).toBe(true)", diagnostic)
        self.assertNotIn("await page.waitForTimeout(3000)", diagnostic)

    def test_release_dry_run_diagnostic_contract(self) -> None:
        workflow = (ROOT / ".github/workflows/release-dry-run.yml").read_text()
        diagnostic = (
            ROOT / "tests/integration/tests/00-diagnostic.spec.ts"
        ).read_text()

        self.assert_release_dry_run_diagnostic_contract(workflow, diagnostic)

        legacy_workflow = workflow.replace(
            " --project=chromium --retries=0", "", 1
        )
        with self.assertRaises(AssertionError):
            self.assert_release_dry_run_diagnostic_contract(
                legacy_workflow, diagnostic
            )

        retrying_diagnostic = diagnostic.replace(
            "test.describe.configure({ retries: 0 });", "", 1
        )
        with self.assertRaises(AssertionError):
            self.assert_release_dry_run_diagnostic_contract(
                workflow, retrying_diagnostic
            )

        unconditional_pass = f"{diagnostic}\nexpect(true).toBe(true);\n"
        with self.assertRaises(AssertionError):
            self.assert_release_dry_run_diagnostic_contract(
                workflow, unconditional_pass
            )

        evidence_failure_ignored = diagnostic.replace(
            "if (!readinessFailed) {", "if (false) {", 1
        )
        with self.assertRaises(AssertionError):
            self.assert_release_dry_run_diagnostic_contract(
                workflow, evidence_failure_ignored
            )

    def test_worker_has_no_publication_or_signing_authority(self) -> None:
        worker = (ROOT / "scripts/release-preflight-worker.sh").read_text()
        runner = (ROOT / "scripts/run-release-preflight.sh").read_text()
        self.assertIn("unset GH_TOKEN GITHUB_TOKEN", worker)
        self.assertIn("export GITHUB_ACTIONS=true", worker)
        self.assertIn("export CI=true", worker)
        self.assertIn('export GOTMPDIR="$GO_TMP_DIR"', worker)
        self.assertIn('rm -rf "$GO_TMP_DIR"', worker)
        self.assertIn('sub(/^toolchain /, "")', worker)
        self.assertNotIn("docker push", worker)
        self.assertNotIn("gh release", worker)
        self.assertIn("mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble", worker)
        self.assertIn(
            'git show "${SOURCE_SHA}:scripts/release-preflight-worker.sh"', runner
        )
        self.assertIn("is not reachable from a fetched origin branch", runner)

    def test_worker_matches_ci_toolchain_and_publishes_smoke_on_a_free_port(self) -> None:
        worker = (ROOT / "scripts/release-preflight-worker.sh").read_text()
        # Non-login ssh shells never source profile hooks, so an installed mise
        # toolchain must be activated explicitly or a stale system Node wins.
        self.assertIn('"$HOME/.local/bin/mise" activate bash --shims', worker)
        # GitHub runners leave GOTMPDIR unset; unix-socket fixtures sit at the
        # sun_path limit under /tmp, so the worker must not nest test temp dirs.
        self.assertIn('GO_TMP_DIR="${PULSE_RELEASE_PREFLIGHT_GO_TMP_DIR:-}"', worker)
        # Smoke stacks publish on a verified-free host port pair and every probe
        # and the browser use that port; a worker may host Pulse on 7655.
        self.assertIn("smoke_port_pair_is_free", worker)
        self.assertIn("PULSE_RELEASE_PREFLIGHT_E2E_PORT", worker)
        self.assertIn('export PULSE_E2E_BASE_URL="http://localhost:${PULSE_E2E_PORT}"', worker)
        self.assertIn('--env "PLAYWRIGHT_BASE_URL=${PULSE_E2E_BASE_URL}"', worker)
        self.assertNotIn("localhost:7655", worker)

    def test_worker_serializes_resource_intensive_test_suites(self) -> None:
        worker = (ROOT / "scripts/release-preflight-worker.sh").read_text()
        scheduling_block = re.search(
            r"# Static frontend checks.*?run_backend\n",
            worker,
            flags=re.DOTALL,
        )
        self.assertIsNotNone(scheduling_block)
        block = scheduling_block.group(0)

        self.assertIn("run_frontend_static_quality &", block)
        self.assertIn("run_integration_prep &", block)
        self.assertNotIn("run_frontend_tests &", block)
        self.assertNotIn("run_backend &", block)
        self.assertLess(block.index("done\n"), block.index("run_frontend_tests\n"))
        self.assertLess(block.index("run_frontend_tests\n"), block.index("run_backend\n"))

    def test_backend_preserves_exit_and_collects_after_failure(self):
        worker = (ROOT / 'scripts/release-preflight-worker.sh').read_text()
        function = re.search(r'^run_backend\(\) \(\n.*?^\)', worker, re.M | re.S).group(0)
        for profile in ('release', 'rehearsal'):
            for code in (0, 17):
                for telemetry_code in (0, 9):
                    with self.subTest(profile=profile, code=code, telemetry_code=telemetry_code):
                        # Stub only operations: execute the real backend shell function
                        # under errexit, without any product tests or data mutation.
                        script = f'''set -euo pipefail
PROFILE={profile}
TEST_DATA_DIR=unused
ACTUAL_GO=go-test-fixture
rm() {{ :; }}
mkdir() {{ :; }}
python3() {{ echo "snapshot $2"; return {telemetry_code}; }}
phase() {{ echo "phase $1"; return {code}; }}
{function}
run_backend
echo CONTINUED
'''
                        result = subprocess.run(['bash', '-c', script], text=True, capture_output=True)
                        self.assertEqual(result.returncode, code, result.stderr)
                        self.assertIn('snapshot before', result.stdout)
                        self.assertIn('snapshot after', result.stdout)
                        self.assertIn(f'RELEASE_BACKEND_EXIT {code}', result.stdout)
                        self.assertEqual('CONTINUED' in result.stdout, code == 0)
                        self.assertIn('backend-serial' if profile == 'rehearsal' else 'backend-race-sharded', result.stdout)


    def test_rehearsal_stream_uses_real_decoder_and_preserves_failure(self):
        worker = (ROOT / 'scripts/release-preflight-worker.sh').read_text()
        function = re.search(r'^run_backend\(\) \(\n.*?^\)', worker, re.M | re.S).group(0)
        for code in (0, 17):
            script = f'''set -euo pipefail
PROFILE=rehearsal
TEST_DATA_DIR=unused
ACTUAL_GO=fixture
REHEARSAL_BACKEND_TIMEOUT=30m
rm() {{ :; }}
mkdir() {{ :; }}
python3() {{
  if [ "$1" = ./scripts/release-resource-snapshot.py ]; then
    echo "snapshot $2"
  else
    command python3 "$@"
  fi
}}
env() {{
  [ "$*" = 'PULSE_DATA_DIR=unused go test -json -p 1 -timeout 30m ./...' ] || return 42
  printf '%s\\n' '{{"Action":"output","Output":"synthetic verdict\\n"}}'
  return {code}
}}
phase() {{ shift; "$@"; }}
{function}
run_backend
echo CONTINUED
'''
            result = subprocess.run(['bash', '-c', script], cwd=ROOT, text=True, capture_output=True)
            self.assertEqual(result.returncode, code, result.stderr)
            self.assertIn('synthetic verdict\n', result.stdout)
            self.assertIn('snapshot after', result.stdout)
            self.assertIn(f'RELEASE_BACKEND_EXIT {code}', result.stdout)
            self.assertEqual('CONTINUED' in result.stdout, code == 0)

    def test_browser_mount_identity_and_locked_cli_follow_the_docker_daemon(self) -> None:
        worker = (ROOT / "scripts/release-preflight-worker.sh").read_text()
        functions = []
        for name in ("playwright_container_user", "run_playwright"):
            match = re.search(rf"^{name}\(\) \{{\n.*?^\}}", worker, re.MULTILINE | re.DOTALL)
            self.assertIsNotNone(match, name)
            functions.append(match.group(0))
        for options, info_rc, valid, expected_user in (
            (["name=seccomp,profile=builtin", "name=rootless"], 0, True, "0:0"),
            (["name=seccomp,profile=builtin"], 0, True, f"{os.getuid()}:{os.getgid()}"),
            ([], 42, False, None),
            ({"rootless": True}, 0, False, None),
            (None, 0, False, None),
        ):
            with self.subTest(options=options, info_rc=info_rc), tempfile.TemporaryDirectory() as raw:
                temp = pathlib.Path(raw)
                docker = temp / "docker"
                docker.write_text(
                    f"#!{sys.executable}\n"
                    "import json,os,pathlib,sys\n"
                    "if sys.argv[1] == 'info':\n"
                    "    print(os.environ['TEST_SECURITY_OPTIONS'])\n"
                    "    raise SystemExit(int(os.environ['TEST_INFO_RC']))\n"
                    "pathlib.Path(os.environ['TEST_CONTAINER_STARTED']).touch()\n"
                    "print(json.dumps(sys.argv[1:]))\n"
                )
                docker.chmod(0o755)
                started = temp / "started"
                script = "set -euo pipefail\n" + "\n".join(functions) + "\n"
                script += 'PLAYWRIGHT_CONTAINER_USER="$(playwright_container_user)"\n'
                script += "run_playwright --version\n"
                script += "run_playwright test tests/00-diagnostic.spec.ts --project=chromium\n"
                result = subprocess.run(
                    ["bash", "-c", script], capture_output=True, text=True, check=False,
                    env={**os.environ, "PATH": f"{temp}:{os.environ['PATH']}",
                         "TEST_SECURITY_OPTIONS": json.dumps(options), "TEST_INFO_RC": str(info_rc),
                         "TEST_CONTAINER_STARTED": str(started),
                         "REPOSITORY_DIR": str(temp / "private workspace"),
                         "PLAYWRIGHT_IMAGE": "locked-playwright-image",
                         "PULSE_E2E_BASE_URL": "http://127.0.0.1:27655"},
                )
                self.assertEqual(valid, result.returncode == 0, result.stderr)
                self.assertEqual(valid, started.exists())
                if not valid:
                    continue
                calls = [json.loads(line) for line in result.stdout.splitlines()]
                self.assertEqual(2, len(calls))
                for args in calls:
                    self.assertEqual(expected_user, args[args.index("--user") + 1])
                    self.assertEqual(f"{temp}/private workspace/tests/integration:/work",
                                     args[args.index("--volume") + 1])
                    self.assertIn("node", args)
                    self.assertIn("/work/node_modules/@playwright/test/cli.js", args)
                    self.assertNotIn("npx", args)
                self.assertEqual("--version", calls[0][-1])
                self.assertEqual(["test", "tests/00-diagnostic.spec.ts", "--project=chromium"], calls[1][-3:])
        prep = worker[worker.index("run_integration_prep() {"):worker.index("# Static frontend checks")]
        self.assertLess(prep.index("phase playwright-image"), prep.index("phase playwright-runtime"))
        self.assertIn("phase playwright-runtime run_playwright --version", prep)

    def test_api_shard_plan_is_deterministic_complete_and_disjoint(self) -> None:
        test_names = [
            f"TestReleaseCase{index:04d}"
            for index in [8, 1, 14, 3, 16, 0, 7, 11, 2, 15, 5, 13, 4, 10, 6, 12, 9]
        ]
        plan = build_plan(test_names, shard_count=2, batch_size=4)
        reordered_plan = build_plan(list(reversed(test_names)), shard_count=2, batch_size=4)
        self.assertNotEqual(plan, reordered_plan)

        assigned = [
            name
            for shard in plan["shards"]
            for batch in shard["batches"]
            for name in batch
        ]
        self.assertEqual(assigned, test_names)
        self.assertEqual(len(assigned), len(set(assigned)))
        self.assertEqual(
            [name for batch in plan["shards"][0]["batches"] for name in batch],
            test_names[:9],
        )
        self.assertEqual(
            [name for batch in plan["shards"][1]["batches"] for name in batch],
            test_names[9:],
        )

        with tempfile.TemporaryDirectory() as directory:
            manifest_path = write_plan(plan, pathlib.Path(directory))
            manifest = manifest_path.read_text()
            self.assertIn('"test_count": 17', manifest)
            self.assertTrue((pathlib.Path(directory) / "shard-01-batch-01.regex").is_file())
            self.assertTrue((pathlib.Path(directory) / "shard-02-batch-01.regex").is_file())

    def test_api_shard_plan_rejects_invalid_or_duplicate_names(self) -> None:
        with self.assertRaisesRegex(ValueError, "invalid top-level"):
            build_plan(["TestGood", "BenchmarkNotATest"], 2, 10)
        with self.assertRaisesRegex(ValueError, "duplicate top-level"):
            build_plan(["TestDuplicate", "TestDuplicate"], 2, 10)
        with self.assertRaisesRegex(ValueError, "cannot exceed"):
            build_plan(["TestOnly"], 2, 10)
        with self.assertRaisesRegex(ValueError, "safe per-argument ceiling"):
            build_plan(["TestOnly"], 1, 10, max_regex_bytes=120_001)
        with self.assertRaisesRegex(ValueError, "one value per shard"):
            build_plan(["TestOne", "TestTwo"], 2, 10, shard_weights=[1])
        with self.assertRaisesRegex(ValueError, "positive integers"):
            build_plan(["TestOne", "TestTwo"], 2, 10, shard_weights=[1, 0])
        with self.assertRaisesRegex(ValueError, "one fewer value"):
            build_plan(
                ["TestOne", "TestTwo"],
                2,
                10,
                shard_boundaries=["TestOne", "TestTwo"],
            )
        with self.assertRaisesRegex(ValueError, "unknown shard boundary"):
            build_plan(
                ["TestOne", "TestTwo"],
                2,
                10,
                shard_boundaries=["TestMissing"],
            )
        with self.assertRaisesRegex(ValueError, "mutually exclusive"):
            build_plan(
                ["TestOne", "TestTwo"],
                2,
                10,
                shard_weights=[1, 1],
                shard_boundaries=["TestOne"],
            )

    def test_api_shard_plan_supports_measured_contiguous_weights(self) -> None:
        test_names = [f"TestReleaseCase{index:04d}" for index in range(20)]
        plan = build_plan(
            test_names,
            shard_count=3,
            batch_size=100,
            shard_weights=[6, 1, 1],
        )

        shards = [
            [name for batch in shard["batches"] for name in batch]
            for shard in plan["shards"]
        ]
        self.assertEqual([len(shard) for shard in shards], [15, 3, 2])
        self.assertEqual([name for shard in shards for name in shard], test_names)
        self.assertEqual(plan["shard_weights"], [6, 1, 1])

    def test_api_shard_plan_supports_named_contiguous_boundaries(self) -> None:
        test_names = [f"TestReleaseCase{index:04d}" for index in range(8)]
        plan = build_plan(
            test_names,
            shard_count=3,
            batch_size=100,
            shard_boundaries=[test_names[3], test_names[5]],
        )

        shards = [
            [name for batch in shard["batches"] for name in batch]
            for shard in plan["shards"]
        ]
        self.assertEqual([len(shard) for shard in shards], [4, 2, 2])
        self.assertEqual([name for shard in shards for name in shard], test_names)
        self.assertEqual(
            plan["shard_boundaries"], [test_names[3], test_names[5]]
        )

    def test_api_shard_plan_bounds_one_shard_regex_argument_bytes(self) -> None:
        test_names = [
            f"Test{index:04d}{index * 7919:08d}{index * 104729:010d}{'X' * 64}"
            for index in range(3736)
        ]
        max_regex_bytes = 64 * 1024
        plan = build_plan(
            test_names,
            shard_count=1,
            batch_size=10000,
            max_regex_bytes=max_regex_bytes,
        )
        batches = plan["shards"][0]["batches"]
        self.assertGreater(len(batches), 1)
        self.assertEqual(
            [name for batch in batches for name in batch],
            test_names,
        )

        with tempfile.TemporaryDirectory() as directory:
            write_plan(plan, pathlib.Path(directory))
            regex_files = sorted(pathlib.Path(directory).glob("*.regex"))
            self.assertEqual(len(regex_files), len(batches))
            for regex_file in regex_files:
                self.assertLessEqual(
                    len(regex_file.read_text().strip().encode()),
                    max_regex_bytes,
                )

        with self.assertRaisesRegex(ValueError, "exceeds max regex bytes"):
            build_plan(
                ["TestNameThatCannotFit"],
                shard_count=1,
                batch_size=1,
                max_regex_bytes=8,
            )

    def test_api_shard_regex_compresses_shared_names_without_changing_membership(
        self,
    ) -> None:
        test_names = [
            "TestHandleCharts_Success",
            "TestHandleCharts_MethodNotAllowed",
            "TestHandleStorageCharts_Success",
            "TestHandleStorageCharts_MethodNotAllowed",
            "TestRouterPublicPathsInventory",
        ]
        regex = _test_regex(test_names)
        naive = "^(?:" + "|".join(test_names) + ")$"
        self.assertLess(len(regex.encode()), len(naive.encode()))

        compiled = re.compile(regex)
        for test_name in test_names:
            self.assertIsNotNone(compiled.fullmatch(test_name))
        for excluded in [
            "TestHandleCharts",
            "TestHandleCharts_Failure",
            "TestRouterPublicPathsInventoryExtra",
        ]:
            self.assertIsNone(compiled.fullmatch(excluded))

    def test_release_workflow_uses_independent_bundle_and_backend_lanes(self) -> None:
        workflow = (ROOT / ".github/workflows/create-release.yml").read_text()
        backend = (ROOT / "scripts/run-release-backend-tests.sh").read_text()
        self.assertIn("frontend_bundle:", workflow)
        bundle_job = workflow[
            workflow.index("  frontend_bundle:") : workflow.index("  frontend_checks:")
        ]
        # Every release channel qualifies on a fresh GitHub-hosted VM, so the
        # bundle and backend lanes must not depend on the retired PVE runners.
        self.assertIn("runs-on: ubuntu-24.04", bundle_job)
        self.assertNotIn("self-hosted", bundle_job)
        self.assertNotIn("pulse-pve-", bundle_job)
        self.assertIn("./scripts/run-release-backend-tests.sh", workflow)
        backend_job = workflow[
            workflow.index("  backend_tests:") : workflow.index("  integration_tests:")
        ]
        self.assertIn("runs-on: ubuntu-24.04", backend_job)
        self.assertNotIn("self-hosted", backend_job)
        self.assertNotIn("pulse-pve-", backend_job)
        self.assertNotIn("      - frontend_checks\n    if:", backend_job)
        self.assertIn("go test -c -race", backend)
        self.assertIn("python3 scripts/shard_go_tests.py", backend)
        self.assertIn('--max-regex-bytes "$MAX_REGEX_BYTES"', backend)
        self.assertIn(
            'MEMORY_WAIT_SECONDS="${PULSE_BACKEND_TEST_MEMORY_WAIT_SECONDS:-120}"',
            backend,
        )
        self.assertIn("shard_admission_required_kib", backend)
        self.assertIn("Degrading to $cpu_shards API shard(s)", backend)
        self.assertIn('if [ "$VCPUS" -ge 4 ]; then', backend)
        self.assertIn("cpu_shards=3", backend)
        self.assertIn(
            'API_SHARD_TIMEOUT="${PULSE_BACKEND_API_SHARD_TIMEOUT:-45m}"',
            backend,
        )
        self.assertIn("RESERVED_OTHER_PACKAGE_PROCS", backend)
        self.assertIn(
            'GOMAXPROCS=1 PULSE_DATA_DIR="$package_root"', backend
        )
        self.assertIn(
            'go test -race -p "$OTHER_PACKAGE_PROCS" -timeout 30m', backend
        )
        self.assertIn('-test.timeout "$API_SHARD_TIMEOUT"', backend)
        self.assertIn(
            "TestWebSocketOriginAllowsTrustedForwardedHostedOriginIPv6Loopback,"
            "TestServerInfoEndpointMethodNotAllowed",
            backend,
        )
        self.assertIn("Completed internal/api shard", backend)
        self.assertIn("export GITHUB_ACTIONS=true", backend)
        self.assertIn("export CI=true", backend)


if __name__ == "__main__":
    unittest.main()