Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(ci): make the test-count ratchet count what a clean checkout collects

Juan Denis2ad6720bd007
diff --git a/backend/tests/BASELINE_COUNT b/backend/tests/BASELINE_COUNT
index c0ec9d83..d9cce571 100644
--- a/backend/tests/BASELINE_COUNT
+++ b/backend/tests/BASELINE_COUNT
@@ -1 +1 @@
-4530
+4518
diff --git a/backend/tests/check_test_count.py b/backend/tests/check_test_count.py
index 86077da5..502ebcc7 100644
--- a/backend/tests/check_test_count.py
+++ b/backend/tests/check_test_count.py
@@ -11,6 +11,10 @@ green number. This checks the collected-test count against a checked-in floor
- Lowering it requires editing ``BASELINE_COUNT`` in the same commit — an
explicit, reviewable statement that tests were intentionally removed, instead
of a silent regression.
+- Collection runs with ``SERVERKIT_CLEAN_COLLECT=1`` so the number is the one a
+ CLEAN CHECKOUT collects. A dev box carries gitignored extension copies that
+ add parametrised cases CI can never see; ``--update`` used to bake those into
+ the floor and leave the ratchet permanently red.
Usage:
python tests/check_test_count.py # verify (CI mode) — exit 1 if below
@@ -41,11 +45,19 @@ def collect_count():
# whatever this machine has lying around (dev-data/, sibling checkouts),
# which is how the floor was once --update'd ~282 items above anything a
# clean checkout can collect, turning the ratchet permanently red.
+ # SERVERKIT_CLEAN_COLLECT makes the count machine-independent: the only
+ # module whose parametrisation varies with what is on disk
+ # (test_builtin_extension_drift.py, which pairs builtin-extensions/<ext>
+ # against the mostly-gitignored live copies under app/plugins/) restricts
+ # itself to git-TRACKED copies when it is set. Without it, --update on a dev
+ # box wrote a floor no clean checkout could reach -- 4431 -> 4417, and again
+ # 4530 -> 4516, both times 14 cases from four gitignored extensions.
+ env = dict(os.environ, SERVERKIT_CLEAN_COLLECT='1')
proc = subprocess.run(
[sys.executable, '-m', 'pytest', 'tests', '--collect-only', '-q',
'-p', 'no:cacheprovider'],
cwd=os.path.dirname(HERE), # backend/
- capture_output=True, text=True,
+ capture_output=True, text=True, env=env,
)
out = proc.stdout + proc.stderr
if proc.returncode != 0:
diff --git a/backend/tests/test_builtin_extension_drift.py b/backend/tests/test_builtin_extension_drift.py
index 58a654ed..3908336b 100644
--- a/backend/tests/test_builtin_extension_drift.py
+++ b/backend/tests/test_builtin_extension_drift.py
@@ -18,9 +18,15 @@ legitimately have one side only — cloudflare-ops and localkit ship source with
no materialised live copy, serverkit-git uses a different layout — and most live
copies are gitignored, so a fresh clone sees fewer pairs than a dev machine.
`test_drift_check_is_not_vacuous` stops that from quietly reducing to nothing.
+
+That machine-dependence is also why the test-count ratchet runs collection with
+`SERVERKIT_CLEAN_COLLECT=1` — see `CLEAN_COLLECT` below.
"""
import hashlib
+import os
+import subprocess
+import sys
from pathlib import Path
import pytest
@@ -29,6 +35,35 @@ REPO = Path(__file__).resolve().parent.parent.parent
SOURCE_ROOT = REPO / 'builtin-extensions'
LIVE_ROOT = REPO / 'backend' / 'app' / 'plugins'
+# Set by tests/check_test_count.py. This module is the one place where the
+# collected-test COUNT depends on what a machine happens to have on disk: most
+# live copies under app/plugins/ are gitignored, so a dev box parametrises over
+# pairs no fresh clone can see. --update'ing the floor from such a box wrote a
+# number CI could never reach -- twice (4431 -> 4417, then 4530 -> 4516). When
+# this is set, only git-TRACKED live copies are paired, so the ratchet counts
+# the same tests everywhere. Normal runs are unaffected and still check the
+# gitignored copies on the machine that has them.
+CLEAN_COLLECT = os.environ.get('SERVERKIT_CLEAN_COLLECT') == '1'
+
+
+def _tracked_live_copies():
+ """Directory names under app/plugins/ that git tracks at least one file in."""
+ try:
+ proc = subprocess.run(
+ ['git', 'ls-files', '--', 'backend/app/plugins'],
+ cwd=str(REPO), capture_output=True, text=True,
+ )
+ except OSError: # no git on PATH
+ return None
+ if proc.returncode != 0: # not a git checkout (release tarball, Docker build)
+ return None
+ names = set()
+ for entry in proc.stdout.splitlines():
+ parts = Path(entry).parts if entry else ()
+ if len(parts) > 3:
+ names.add(parts[3])
+ return names
+
def _normalised_hash(path):
"""Content hash ignoring line endings — the repo has mixed CRLF/LF."""
@@ -51,10 +86,13 @@ def _paired_extensions():
"""Extensions whose source and live copies BOTH exist on this machine."""
if not SOURCE_ROOT.is_dir():
return []
+ tracked = _tracked_live_copies() if CLEAN_COLLECT else None
paired = []
for ext_dir in sorted(p for p in SOURCE_ROOT.iterdir() if p.is_dir()):
source = ext_dir / 'backend'
live = LIVE_ROOT / ext_dir.name
+ if tracked is not None and ext_dir.name not in tracked:
+ continue
if source.is_dir() and live.is_dir():
paired.append((ext_dir.name, source, live))
return paired
@@ -121,3 +159,39 @@ def test_drift_check_is_not_vacuous():
'app/plugins/<ext> copy, so the drift check compared nothing. Either '
'the layout moved or the tracked serverkit-email copy is gone.'
)
+
+
+def test_clean_collect_drops_untracked_live_copies(monkeypatch):
+ """The floor-inflation guard, proved.
+
+ Under ``SERVERKIT_CLEAN_COLLECT=1`` a live copy git does not track must not
+ produce a parametrised case, or ``check_test_count.py --update`` bakes a
+ dev-only number into BASELINE_COUNT and the ratchet goes red on CI for a
+ suite that lost nothing (it did, twice: 4431 -> 4417 and 4530 -> 4516).
+ """
+ module = sys.modules[__name__]
+ # Force the unfiltered view first, so the test means the same thing whether
+ # or not the suite itself was started with SERVERKIT_CLEAN_COLLECT set.
+ monkeypatch.setattr(module, 'CLEAN_COLLECT', False)
+ baseline = {ext for ext, _, _ in _paired_extensions()}
+ assert baseline, 'nothing paired — see test_drift_check_is_not_vacuous'
+
+ monkeypatch.setattr(module, 'CLEAN_COLLECT', True)
+ monkeypatch.setattr(module, '_tracked_live_copies', lambda: set())
+ assert _paired_extensions() == []
+
+ monkeypatch.setattr(module, '_tracked_live_copies', lambda: baseline)
+ assert {ext for ext, _, _ in _paired_extensions()} == baseline
+
+ # A non-git checkout must not silently drop every pair.
+ monkeypatch.setattr(module, '_tracked_live_copies', lambda: None)
+ assert {ext for ext, _, _ in _paired_extensions()} == baseline
+
+
+def test_ratchet_still_asks_for_a_clean_collect():
+ """Deleting the env var in check_test_count.py re-opens the same hole."""
+ ratchet = (Path(__file__).parent / 'check_test_count.py').read_text(encoding='utf-8')
+ assert "SERVERKIT_CLEAN_COLLECT='1'" in ratchet, (
+ 'check_test_count.py no longer sets SERVERKIT_CLEAN_COLLECT, so its '
+ 'count is machine-dependent again.'
+ )