Skip to content

CLI for SidePulse

Python

Smoke-test the published artifact and guard release versions

Peter Kuhar27cee987de14
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index a71049a..b1e62d1 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -21,6 +21,17 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: "3.12"
+ # Publishing v0.2.0 from a tree that still says 0.1.0 produces a release
+ # nobody can reference correctly, and PyPI uploads cannot be replaced.
+ - name: Verify tag matches package version
+ run: |
+ tag="${GITHUB_REF_NAME#v}"
+ pkg="$(python -c "import tomllib, pathlib; print(tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project']['version'])")"
+ if [ "$tag" != "$pkg" ]; then
+ echo "::error::tag $GITHUB_REF_NAME implies version $tag, but pyproject.toml declares $pkg"
+ exit 1
+ fi
+ echo "tag and package version agree: $pkg"
- name: Build sdist and wheel
run: |
python -m pip install --upgrade build
@@ -49,3 +60,46 @@ jobs:
name: dist
path: dist/
- uses: pypa/gh-action-pypi-publish@release/v1
+
+ # Everything above tests the artifact built here. This installs what PyPI
+ # actually serves, on a clean runner, and runs the real commands against it.
+ smoke-test:
+ name: Smoke test PyPI release / ${{ matrix.os }}
+ needs: publish
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [macos-latest, ubuntu-latest]
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+ - name: Install test runner
+ run: python -m pip install --upgrade pip pytest
+
+ # PyPI takes a little while to serve a fresh upload from every CDN edge.
+ - name: Wait for the release to become installable
+ run: |
+ version="${GITHUB_REF_NAME#v}"
+ for attempt in $(seq 1 30); do
+ if python -m pip download --no-deps --no-cache-dir \
+ --dest "$RUNNER_TEMP/probe" "sidepulse==$version" >/dev/null 2>&1; then
+ echo "sidepulse==$version installable after $attempt attempt(s)"
+ exit 0
+ fi
+ echo "not on PyPI yet (attempt $attempt/30); waiting 20s"
+ sleep 20
+ done
+ echo "::error::sidepulse==$version never became installable"
+ exit 1
+
+ # Installs from PyPI instead of the checkout. The checkout is still
+ # present, and is what the resource and module lists are compared
+ # against -- it is the same commit the release was built from.
+ - name: Smoke test the published artifact
+ run: |
+ export SIDEPULSE_INSTALL_SPEC="sidepulse==${GITHUB_REF_NAME#v}"
+ echo "installing $SIDEPULSE_INSTALL_SPEC"
+ python -m pytest tests/test_environment.py -v
diff --git a/README.md b/README.md
index f3c8f02..5af679f 100644
--- a/README.md
+++ b/README.md
@@ -579,3 +579,18 @@ to turn that skip into a failure, which is how CI runs them.
The clean-room install tests take ~15s because they build a virtualenv and
install into it. Set `SIDEPULSE_SKIP_CLEAN_INSTALL=1` to skip them while
iterating; CI always runs them.
+
+### Releasing
+
+`git tag v0.1.0 && git push --tags` runs the full suite, checks the tag against
+the version in `pyproject.toml`, builds, publishes, and then reinstalls the
+release **from PyPI** on clean macOS and Linux runners to re-run the clean-room
+tests against the artifact users actually download.
+
+Bump the version in both `pyproject.toml` and `src/sidepulse/__init__.py` — a
+test fails if they disagree, and the tag guard fails if the tag disagrees with
+either. To point the clean-room tests at any other build:
+
+```sh
+SIDEPULSE_INSTALL_SPEC='sidepulse==0.1.0' python3 -m pytest tests/test_environment.py -v
+```
diff --git a/tests/test_environment.py b/tests/test_environment.py
index 6fd5533..80f8256 100644
--- a/tests/test_environment.py
+++ b/tests/test_environment.py
@@ -10,6 +10,11 @@ These tests build an empty virtualenv, run ``pip install .`` against nothing
but ``pyproject.toml``, and then exercise the result. A dependency that is not
declared is simply absent, so the failure is real rather than theoretical.
+The same tests double as a post-publish smoke test. Set
+``SIDEPULSE_INSTALL_SPEC`` to a requirement string (``sidepulse==0.1.0``) and
+they install that from PyPI instead of the local tree, which checks the
+artifact users actually download rather than the one built here.
+
They are slower than the rest of the suite (a few seconds with a warm wheel
cache, longer on a cold one). Set ``SIDEPULSE_SKIP_CLEAN_INSTALL=1`` to skip
them while iterating locally; CI always runs them.
@@ -19,6 +24,7 @@ from __future__ import annotations
import json
import os
+import re
import subprocess
import sys
import tempfile
@@ -46,6 +52,29 @@ NETWORK_FAILURE_MARKERS = (
)
+def install_spec() -> str:
+ """What to hand to pip: the local tree by default, or a published release."""
+ return os.environ.get("SIDEPULSE_INSTALL_SPEC") or str(REPO_ROOT)
+
+
+def is_local_install() -> bool:
+ return not os.environ.get("SIDEPULSE_INSTALL_SPEC")
+
+
+def expected_version() -> str:
+ """The version the install is expected to report.
+
+ Taken from the requirement pin when smoke-testing a release, otherwise
+ from the source tree being installed.
+ """
+ spec = os.environ.get("SIDEPULSE_INSTALL_SPEC")
+ if spec and "==" in spec:
+ return spec.split("==", 1)[1].strip()
+ text = (REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")
+ match = re.search(r'^version\s*=\s*"([^"]+)"', text, re.MULTILINE)
+ return match.group(1) if match else ""
+
+
@unittest.skipIf(
os.environ.get("SIDEPULSE_SKIP_CLEAN_INSTALL") == "1",
"clean-install tests disabled via SIDEPULSE_SKIP_CLEAN_INSTALL",
@@ -72,22 +101,29 @@ class CleanInstallTests(unittest.TestCase):
raise unittest.SkipTest(f"could not create virtualenv: {create.stderr}")
cls.python = cls.venv / "bin" / "python"
+ cls.install_spec = install_spec()
install = subprocess.run(
[
str(cls.python), "-m", "pip", "install",
- "--quiet", "--disable-pip-version-check", str(REPO_ROOT),
+ "--quiet", "--disable-pip-version-check", cls.install_spec,
],
capture_output=True, text=True, timeout=900,
)
if install.returncode != 0:
output = install.stderr + install.stdout
- if any(marker in output for marker in NETWORK_FAILURE_MARKERS):
+ # A published release that cannot be installed is a real failure,
+ # not an environment quirk, so only the local case degrades to a
+ # skip when there is no index.
+ if is_local_install() and any(
+ marker in output for marker in NETWORK_FAILURE_MARKERS
+ ):
cls._tmp.cleanup()
raise unittest.SkipTest(f"no package index reachable: {output[-500:]}")
cls._tmp.cleanup()
raise AssertionError(
- f"`pip install .` failed on a clean environment:\n{output}"
+ f"`pip install {cls.install_spec}` failed on a clean "
+ f"environment:\n{output}"
)
@classmethod
@@ -154,6 +190,29 @@ class CleanInstallTests(unittest.TestCase):
f"tests imported {result.stdout} instead of the installed package",
)
+ def test_installed_version_is_the_expected_one(self):
+ """Post-publish, this proves PyPI is serving the release we tagged."""
+ result = self.run_python(
+ """
+ import sys
+ from importlib.metadata import version
+ import sidepulse
+ sys.stdout.write(f"{version('sidepulse')} {sidepulse.__version__}")
+ """
+ )
+ self.assertEqual(0, result.returncode, result.stderr)
+ metadata_version, dunder_version = result.stdout.split()
+ self.assertEqual(
+ expected_version(),
+ metadata_version,
+ f"installed {self.install_spec} reports version {metadata_version}",
+ )
+ self.assertEqual(
+ metadata_version,
+ dunder_version,
+ "wheel metadata and sidepulse.__version__ disagree",
+ )
+
def test_status_bar_imports_after_plain_install(self):
"""The regression test for the ScriptingBridge dependency.
@@ -332,9 +391,23 @@ class CleanInstallTests(unittest.TestCase):
self.assertNotIn(DEPENDENCY_ERROR_MARKER, stdout + stderr)
return
- self.fail(
- "status-bar exited immediately instead of running.\n"
- f"returncode={process.returncode}\nstdout={stdout}\nstderr={stderr}"
+ output = stdout + stderr
+ if DEPENDENCY_ERROR_MARKER in output:
+ self.fail(
+ "status-bar died on missing dependencies from a clean "
+ f"install:\n{output}"
+ )
+ if os.environ.get("SIDEPULSE_REQUIRE_UI_TESTS") == "1":
+ self.fail(
+ "status-bar exited immediately instead of running.\n"
+ f"returncode={process.returncode}\nstdout={stdout}\nstderr={stderr}"
+ )
+ # Exited for some other reason -- most likely no window server, which
+ # is an environment limitation rather than a packaging defect. The
+ # dependency check above still ran, which is what this test is for.
+ self.skipTest(
+ f"status-bar could not run a GUI here (rc={process.returncode}): "
+ f"{output.strip()[:300]}"
)
diff --git a/tests/test_packaging.py b/tests/test_packaging.py
index 85b427a..84edb5a 100644
--- a/tests/test_packaging.py
+++ b/tests/test_packaging.py
@@ -371,6 +371,40 @@ class EntryPointTests(unittest.TestCase):
)
+class VersionTests(unittest.TestCase):
+ """The version is declared twice and must not drift.
+
+ A published wheel whose metadata disagrees with ``sidepulse.__version__``
+ makes bug reports impossible to place.
+ """
+
+ def test_pyproject_and_dunder_version_agree(self):
+ import sidepulse
+
+ declared = load_pyproject()["project"]["version"]
+ self.assertEqual(
+ declared,
+ sidepulse.__version__,
+ "pyproject.toml version and sidepulse.__version__ disagree; "
+ "update both when releasing",
+ )
+
+ def test_installed_metadata_matches_source(self):
+ """Guards against testing a stale install of the package."""
+ from importlib.metadata import version
+
+ try:
+ installed = version("sidepulse")
+ except PackageNotFoundError:
+ self.skipTest("sidepulse is not installed in this environment")
+ self.assertEqual(
+ load_pyproject()["project"]["version"],
+ installed,
+ "the installed sidepulse is a different version than this "
+ "source tree; reinstall with `pip install -e .`",
+ )
+
+
class PackageDataTests(unittest.TestCase):
def test_declared_package_data_exists(self):
data = load_pyproject()