Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

test(ratchet): census hand-rolled subprocess stubs in the suite

Juan Denisc7fd2f2033ba
diff --git a/backend/tests/STUB_ADOPTION_CEILING b/backend/tests/STUB_ADOPTION_CEILING
new file mode 100644
index 00000000..6f4247a6
--- /dev/null
+++ b/backend/tests/STUB_ADOPTION_CEILING
@@ -0,0 +1 @@
+26
diff --git a/backend/tests/hand_rolled_subprocess_stub_census.py b/backend/tests/hand_rolled_subprocess_stub_census.py
new file mode 100644
index 00000000..e8684c1b
--- /dev/null
+++ b/backend/tests/hand_rolled_subprocess_stub_census.py
@@ -0,0 +1,151 @@
+"""Count hand-rolled subprocess stubs in the test suite (plan 82 §G1).
+
+tests/subprocess_stub.py is the shared stubbing kit: scripted commands,
+hard-failing UnscriptedCommand, sudo/basename normalization, and the
+popen_guard hook so a stubbed exec still obeys the sbin-PATH rule. A test
+that instead monkeypatches ``subprocess.run``/``Popen``/... by hand gets
+none of that — its execs are invisible to the runtime guard and an
+unscripted command can fabricate returncode=0.
+
+Every other convergence in this repo has a census + ceiling; this one did
+not, so adoption stalled roughly half-way. Same contract as the others:
+the count may only go down. New tests use the ``fake_subprocess`` fixture.
+
+Counted: patches whose target is the ``subprocess`` module itself —
+``monkeypatch.setattr(subprocess, 'run', ...)``,
+``monkeypatch.setattr('subprocess.run', ...)``, ``patch('subprocess.run')``,
+``patch.object(subprocess, 'run')``. Patching a service door
+(``patch.object(mod, 'run_privileged')``) is fine and not counted.
+
+Usage (from ``backend/``)::
+
+ python tests/hand_rolled_subprocess_stub_census.py # report
+ python tests/hand_rolled_subprocess_stub_census.py --update # set ceiling
+"""
+
+import ast
+import os
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+BACKEND = os.path.dirname(HERE)
+CEILING_FILE = os.path.join(HERE, 'STUB_ADOPTION_CEILING')
+
+# The kit itself, its own tests, and this census's harness are exempt: they
+# must touch subprocess directly to do their jobs.
+EXEMPT = {
+ 'tests/subprocess_stub.py',
+ 'tests/test_subprocess_stub.py',
+ 'tests/popen_guard.py',
+ 'tests/test_popen_guard.py',
+ 'tests/conftest.py',
+ 'tests/hand_rolled_subprocess_stub_census.py',
+ 'tests/test_stub_adoption_ratchet.py',
+}
+
+
+def _is_subprocess_target(node):
+ """Does this patch-target argument point at the subprocess module?"""
+ if isinstance(node, ast.Name) and node.id == 'subprocess':
+ return True
+ if isinstance(node, ast.Attribute):
+ # e.g. some_module.subprocess
+ return node.attr == 'subprocess'
+ if isinstance(node, ast.Constant) and isinstance(node.value, str):
+ value = node.value
+ return value == 'subprocess' or value.startswith('subprocess.')
+ return False
+
+
+def count_file(path):
+ """[(lineno, description), ...] for every hand-rolled subprocess patch."""
+ try:
+ tree = ast.parse(open(path, encoding='utf-8').read(), filename=path)
+ except (SyntaxError, UnicodeDecodeError):
+ return []
+ hits = []
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call) or not node.args:
+ continue
+ func = node.func
+ # monkeypatch.setattr(...) / patch(...) / mock.patch(...) /
+ # patch.object(...) / mock.patch.object(...)
+ if isinstance(func, ast.Attribute):
+ name = func.attr
+ elif isinstance(func, ast.Name):
+ name = func.id
+ else:
+ continue
+ if name not in ('setattr', 'patch', 'object'):
+ continue
+ if name == 'object':
+ # only patch.object / mock.patch.object, not any .object(...)
+ base = func.value if isinstance(func, ast.Attribute) else None
+ base_name = (base.attr if isinstance(base, ast.Attribute)
+ else base.id if isinstance(base, ast.Name) else '')
+ if base_name != 'patch':
+ continue
+ if _is_subprocess_target(node.args[0]):
+ hits.append((node.lineno, f'{name}({ast.unparse(node.args[0])}, ...)'))
+ return hits
+
+
+def census():
+ """{relative path: [(lineno, desc), ...]} over tracked test files."""
+ from tracked_files import is_tracked, tracked_py_files
+ tracked = tracked_py_files(BACKEND)
+ found = {}
+ for name in sorted(os.listdir(HERE)):
+ if not name.endswith('.py'):
+ continue
+ path = os.path.join(HERE, name)
+ rel = os.path.relpath(path, BACKEND).replace(os.sep, '/')
+ if rel in EXEMPT or not is_tracked(path, tracked):
+ continue
+ hits = count_file(path)
+ if hits:
+ found[rel] = hits
+ return found
+
+
+def total(found=None):
+ found = census() if found is None else found
+ return sum(len(v) for v in found.values())
+
+
+def read_ceiling():
+ with open(CEILING_FILE, encoding='utf-8') as fh:
+ return int(fh.read().strip())
+
+
+def write_ceiling(value):
+ with open(CEILING_FILE, 'w', encoding='utf-8', newline='\n') as fh:
+ fh.write(f'{value}\n')
+
+
+def main(argv):
+ found = census()
+ count = total(found)
+ if '--update' in argv:
+ write_ceiling(count)
+ print(f'STUB_ADOPTION_CEILING set to {count}')
+ return 0
+
+ ceiling = read_ceiling()
+ print(f'hand-rolled subprocess stubs in tests: {count} (ceiling {ceiling})')
+ for path in sorted(found, key=lambda p: -len(found[p])):
+ print(f' {len(found[path]):3d} {path}')
+ if count > ceiling:
+ print(f'\nFAIL: {count - ceiling} new hand-rolled subprocess stub(s).')
+ print('Use the shared fake_subprocess fixture (tests/subprocess_stub.py) '
+ 'instead — it hard-fails unscripted commands and keeps the '
+ 'sbin-PATH guard in force.')
+ return 1
+ if count < ceiling:
+ print(f'\n{ceiling - count} fewer than the ceiling — run with --update '
+ 'to lock the improvement in.')
+ return 0
+
+
+if __name__ == '__main__':
+ import sys
+ sys.exit(main(sys.argv[1:]))
diff --git a/backend/tests/test_stub_adoption_ratchet.py b/backend/tests/test_stub_adoption_ratchet.py
new file mode 100644
index 00000000..612c53f3
--- /dev/null
+++ b/backend/tests/test_stub_adoption_ratchet.py
@@ -0,0 +1,93 @@
+"""The hand-rolled-subprocess-stub ratchet (plan 82 §G1).
+
+tests/subprocess_stub.py's own docstring notes that every hand-rolled
+subprocess stub is an exec the runtime sbin guard never sees, and an
+unscripted command a blanket except can turn into a fabricated success.
+Adoption of the shared kit stalled about half-way, and unlike every other
+convergence in this repo there was no census holding the line — this is it.
+"""
+
+from hand_rolled_subprocess_stub_census import (
+ census, count_file, read_ceiling, total)
+
+
+def test_the_count_is_at_or_below_the_ceiling():
+ found = census()
+ count = total(found)
+ ceiling = read_ceiling()
+ assert count <= ceiling, (
+ f'{count} hand-rolled subprocess stubs in tests, ceiling is {ceiling}.\n'
+ f'Use the shared fake_subprocess fixture (tests/subprocess_stub.py) '
+ f'instead — it hard-fails unscripted commands and keeps the sbin-PATH '
+ f'guard in force.\n'
+ f'Files: ' + ', '.join(f'{p} ({len(found[p])})' for p in sorted(found))
+ )
+
+
+def test_the_ceiling_is_not_stale():
+ """Slack of 5 is room for an in-flight branch, not a workstream."""
+ count, ceiling = total(), read_ceiling()
+ assert ceiling - count <= 5, (
+ f'ceiling {ceiling} is {ceiling - count} above the actual {count}; '
+ f'run `python tests/hand_rolled_subprocess_stub_census.py --update`'
+ )
+
+
+class TestTheCensusCountsTheRightThings:
+ """A miscounting ratchet reads green either way."""
+
+ def _count(self, tmp_path, source):
+ path = tmp_path / 'sample.py'
+ path.write_text(source, encoding='utf-8')
+ return len(count_file(str(path)))
+
+ def test_counts_monkeypatch_on_the_module_object(self, tmp_path):
+ assert self._count(tmp_path, (
+ 'import subprocess\n'
+ 'def test_x(monkeypatch):\n'
+ ' monkeypatch.setattr(subprocess, "run", lambda *a, **k: None)\n'
+ )) == 1
+
+ def test_counts_string_target_forms(self, tmp_path):
+ assert self._count(tmp_path, (
+ 'from unittest.mock import patch\n'
+ 'def test_x(monkeypatch):\n'
+ ' monkeypatch.setattr("subprocess.run", lambda *a, **k: None)\n'
+ ' with patch("subprocess.check_output"):\n'
+ ' pass\n'
+ )) == 2
+
+ def test_counts_patch_object_on_subprocess(self, tmp_path):
+ assert self._count(tmp_path, (
+ 'import subprocess\n'
+ 'from unittest.mock import patch\n'
+ 'def test_x():\n'
+ ' with patch.object(subprocess, "run"):\n'
+ ' pass\n'
+ )) == 1
+
+ def test_does_not_count_patching_a_service_door(self, tmp_path):
+ """Stubbing run_privileged/run_checked at a module seam is the
+ blessed pattern, not a violation."""
+ assert self._count(tmp_path, (
+ 'from unittest.mock import patch\n'
+ 'from app.services import postfix_service as mod\n'
+ 'def test_x(monkeypatch):\n'
+ ' monkeypatch.setattr(mod, "run_privileged", lambda *a, **k: None)\n'
+ ' with patch.object(mod, "write_privileged_file"):\n'
+ ' pass\n'
+ )) == 0
+
+ def test_does_not_count_the_shared_fixture(self, tmp_path):
+ assert self._count(tmp_path, (
+ 'def test_x(fake_subprocess):\n'
+ ' fake_subprocess.script(["ufw"], stdout="ok")\n'
+ )) == 0
+
+
+def test_the_kit_itself_is_exempt_not_invisible():
+ """subprocess_stub.py must patch subprocess to exist; it is exempted by
+ path, not by the counter failing to see such patterns."""
+ found = census()
+ assert 'tests/subprocess_stub.py' not in found
+ assert 'tests/conftest.py' not in found