Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(test): passive cgroup-v2 runtime baselines for full installs

Juan Denisff3c92891752
diff --git a/.github/workflows/measurements-ci.yml b/.github/workflows/measurements-ci.yml
index 9ba5c28d..f7be0558 100644
--- a/.github/workflows/measurements-ci.yml
+++ b/.github/workflows/measurements-ci.yml
@@ -10,8 +10,12 @@ on:
- 'scripts/measure-repository.py'
- 'scripts/update-readme-measurements.py'
- 'scripts/profile-api.py'
+ - 'scripts/measure-runtime.py'
+ - 'scripts/test/measure-installed-runtime.py'
+ - 'scripts/test/report.py'
- 'scripts/test/test_repository_measurements.py'
- 'scripts/test/test_profile_api.py'
+ - 'scripts/test/test_measure_runtime.py'
- '.github/workflows/measurements-ci.yml'
pull_request:
branches: [main]
@@ -22,8 +26,12 @@ on:
- 'scripts/measure-repository.py'
- 'scripts/update-readme-measurements.py'
- 'scripts/profile-api.py'
+ - 'scripts/measure-runtime.py'
+ - 'scripts/test/measure-installed-runtime.py'
+ - 'scripts/test/report.py'
- 'scripts/test/test_repository_measurements.py'
- 'scripts/test/test_profile_api.py'
+ - 'scripts/test/test_measure_runtime.py'
- '.github/workflows/measurements-ci.yml'
jobs:
@@ -38,5 +46,6 @@ jobs:
run: |
python -m unittest discover -s scripts/test -p test_repository_measurements.py
python -m unittest discover -s scripts/test -p test_profile_api.py
+ python -m unittest discover -s scripts/test -p test_measure_runtime.py
- name: Check translated README tables against reviewed snapshots
run: python scripts/update-readme-measurements.py
diff --git a/backend/app/services/test_sandbox_service.py b/backend/app/services/test_sandbox_service.py
index fc862721..8d7c4b24 100644
--- a/backend/app/services/test_sandbox_service.py
+++ b/backend/app/services/test_sandbox_service.py
@@ -247,6 +247,19 @@ class TestSandboxService:
'detail': detail,
'duration_s': round(time.monotonic() - started, 1),
}
+ runtime_path = work_dir / f'{distro}.runtime.json'
+ if runtime_path.is_file():
+ try:
+ measurement = json.loads(runtime_path.read_text())
+ # Raw samples stay in the per-run artifact; compact evidence
+ # travels with the existing results API and log summary.
+ results[distro]['runtime_measurement'] = {
+ key: value for key, value in measurement.items() if key != 'samples'
+ }
+ except (OSError, ValueError):
+ results[distro]['runtime_measurement'] = {
+ 'status': 'failed', 'reason': 'runtime artifact unreadable',
+ }
run.results = results
db.session.commit()
@@ -345,6 +358,33 @@ class TestSandboxService:
timeout=15)
if probe['success']:
log.write('healthy\n')
- return True, 'install + health OK'
+ detail = cls._measure_runtime(container, log)
+ return True, f'install + health OK; runtime {detail}'
time.sleep(2)
return False, 'install OK but health endpoint never came up'
+
+ @staticmethod
+ def _measure_runtime(container, log):
+ """Observe after install; measurement failure is distinct from install failure."""
+ log.write('\n===== post-install runtime baseline (60s settle + 120s sample) =====\n')
+ log.flush()
+ measured = run_checked(
+ ['docker', 'exec', container, '/opt/serverkit/venv/bin/python',
+ '/src/scripts/test/measure-installed-runtime.py',
+ '--output', '/tmp/serverkit-runtime.json'],
+ stdout=log, stderr=subprocess.STDOUT, timeout=240)
+ artifact = Path(log.name).with_suffix('.runtime.json')
+ copied = run_checked(
+ ['docker', 'cp', f'{container}:/tmp/serverkit-runtime.json', str(artifact)],
+ timeout=30)
+ if not copied['success']:
+ artifact.write_text(json.dumps({
+ 'status': 'failed', 'reason': 'runtime artifact could not be collected',
+ }))
+ if measured['success'] and copied['success']:
+ try:
+ if json.loads(artifact.read_text()).get('status') == 'measured':
+ return 'measured (see log/artifact)'
+ except (OSError, ValueError):
+ pass
+ return 'unavailable/failed (see log/artifact)'
diff --git a/backend/tests/test_sandbox_runtime.py b/backend/tests/test_sandbox_runtime.py
new file mode 100644
index 00000000..db0fd1bf
--- /dev/null
+++ b/backend/tests/test_sandbox_runtime.py
@@ -0,0 +1,44 @@
+"""Resource observation must survive sandbox teardown without inventing success."""
+
+import json
+from pathlib import Path
+
+from app.services import test_sandbox_service as sandbox
+
+
+def test_collects_runtime_artifact_before_container_cleanup(tmp_path, monkeypatch):
+ calls = []
+
+ def run(cmd, **kwargs):
+ calls.append(cmd)
+ if cmd[:2] == ['docker', 'cp']:
+ Path(cmd[-1]).write_text(json.dumps({'status': 'measured', 'samples': []}))
+ return {'success': True}
+
+ monkeypatch.setattr(sandbox, 'run_checked', run)
+ with (tmp_path / 'ubuntu24.log').open('w') as log:
+ result = sandbox.TestSandboxService._measure_runtime('test-container', log)
+ assert result.startswith('measured')
+ assert calls[0][:3] == ['docker', 'exec', 'test-container']
+ assert calls[1][2] == 'test-container:/tmp/serverkit-runtime.json'
+ assert json.loads((tmp_path / 'ubuntu24.runtime.json').read_text())['status'] == 'measured'
+
+
+def test_missing_artifact_is_explicit_failure(tmp_path, monkeypatch):
+ monkeypatch.setattr(sandbox, 'run_checked', lambda *args, **kwargs: {'success': False})
+ with (tmp_path / 'rocky9.log').open('w') as log:
+ result = sandbox.TestSandboxService._measure_runtime('test-container', log)
+ assert result.startswith('unavailable/failed')
+ assert json.loads((tmp_path / 'rocky9.runtime.json').read_text())['status'] == 'failed'
+
+
+def test_successful_copy_of_unavailable_report_is_not_a_measurement(tmp_path, monkeypatch):
+ def run(cmd, **kwargs):
+ if cmd[:2] == ['docker', 'cp']:
+ Path(cmd[-1]).write_text(json.dumps({'status': 'unavailable'}))
+ return {'success': True}
+
+ monkeypatch.setattr(sandbox, 'run_checked', run)
+ with (tmp_path / 'legacy.log').open('w') as log:
+ result = sandbox.TestSandboxService._measure_runtime('test-container', log)
+ assert result.startswith('unavailable/failed')
diff --git a/docs/METRICS.md b/docs/METRICS.md
index 85a53481..bf77b602 100644
--- a/docs/METRICS.md
+++ b/docs/METRICS.md
@@ -96,6 +96,77 @@ construction; background jobs and streamed body iteration are not included.
## Measure memory, images and browser loading separately
+### Passive Linux runtime sampler
+
+`scripts/measure-runtime.py` reads Linux **cgroup v2** counters for explicitly
+selected systemd units or Docker containers. It uses Python's standard library,
+does not restart services or generate HTTP traffic, and saves timestamped raw
+samples plus median/p95/sampled-maximum summaries. Run it on the host, outside
+the services being measured. It rejects overlapping scopes and reports missing
+counters or replaced cgroups instead of inventing zero usage.
+
+For an installed host panel, first verify the unit actually runs the backend
+(not a oneshot wrapper that starts Docker). Record the revision actually running:
+
+```bash
+python3 scripts/measure-runtime.py --unit serverkit.service --unit nginx.service \
+ --scenario idle-dashboard-closed --revision "$RUNNING_SERVERKIT_REVISION" \
+ --notes 'Record DB engine, enabled extensions, app count, warmup and other host activity here' \
+ --duration 600 --interval 5 --output runtime-idle.json
+```
+
+For the panel container, use `--container serverkit` instead of `--unit`.
+Repeat `--unit` / `--container` to include separate supporting services (for
+example Docker, containerd, or an external panel database); results remain per
+target. Newly created application/build containers are **not automatically
+included**. The JSON output is created exclusively so an earlier baseline cannot
+be accidentally overwritten. Ctrl+C saves a partial report with exit code 130;
+failed samples return exit code 1. Optional unavailable counters stay null.
+
+- Memory is `memory.current`, including descendants, charged file cache and
+ kernel memory. It is **not process RSS or Docker CLI's cache-adjusted number**.
+ Raw `memory.stat` is retained; its fields overlap, so do not sum them all.
+- CPU is the change in `cpu.stat` usage divided by actual elapsed time; 100%
+ means one logical CPU, and values can exceed 100% on multicore systems.
+- Swap, task counts, cumulative per-device I/O and memory-event counters are
+ retained where available. Device counters can overlap with stacked storage;
+ inspect deltas per device rather than summing them blindly. Memory events are
+ lifetime counters; compare the first and last samples for events in the run.
+- Peaks are **sampled**, not exact instantaneous high-water marks. Host available
+ memory provides context but includes unrelated workloads. Record machine
+ specs, cgroup limits and background activity alongside any published result.
+
+Definitions follow the [Linux cgroup-v2 documentation](https://docs.kernel.org/admin-guide/cgroup-v2.html).
+Start with a 10-minute quiet baseline after startup settles, then repeat with the
+dashboard open and a fixed set of apps. On a disposable test server, separately
+observe a deployment and backup while also measuring application response times.
+Use the same hardware, limits, versions, data and workload before/after a fix.
+This sampler measures resource usage, not application latency or throughput.
+
+### Automatic observations in full distro installs
+
+The VM harness and Test Sandbox **full** mode invoke
+`scripts/test/measure-installed-runtime.py` after the health check, before other
+API tests. It settles for 60 seconds, then observes for 120 seconds at 5-second
+intervals. The panel and active nginx/Docker/containerd units are recorded
+separately. Each run saves a runtime artifact and an explicit measured,
+unavailable, failed or interrupted status; install success does not imply that
+measurement succeeded. No arbitrary RAM/CPU pass threshold is enforced yet.
+
+VM results include `runtime.json` and a table in the HTML report. Test Sandbox
+keeps `<distro>.runtime.json` next to its log, includes compact measurement data
+in the existing results API, and prints the summary into the log. The VM overlay
+archive SHA-256 identifies uncommitted source; full container installs identify
+the installed checkout/release (which can differ from the local installer).
+
+Quick/provision distro probes do not run ServerKit, so they cannot produce a
+panel runtime baseline. Full systemd containers share the host kernel; parallel
+installs and startup jobs can affect these short observations. Use an isolated
+VM/VPS and longer repeated runs for performance claims. Cgroup-v1 environments
+are explicitly unavailable rather than silently measured with different rules.
+
+### Other measurement boundaries
+
- **Memory:** record the exact process/container, workload, uptime and sampling
interval. Process RSS and Docker memory accounting are different measurements.
Include workers and enabled services; do not describe one idle sample as a
diff --git a/scripts/measure-runtime.py b/scripts/measure-runtime.py
new file mode 100644
index 00000000..452d7b19
--- /dev/null
+++ b/scripts/measure-runtime.py
@@ -0,0 +1,249 @@
+#!/usr/bin/env python3
+"""Passively sample explicit Linux cgroup-v2 services/containers; stdlib only.
+
+Run on the host, outside the measured groups. No restarts, config changes, HTTP
+requests, environment inspection, or load generation. Memory includes cache and
+descendants; CPU 100% means one fully occupied logical CPU. Missing data is null.
+"""
+
+import argparse
+from datetime import datetime, timezone
+import json
+import math
+import os
+from pathlib import Path
+import platform
+import subprocess
+import sys
+import time
+
+
+CGROUP_ROOT = Path('/sys/fs/cgroup')
+
+
+def command(args):
+ return subprocess.run(args, check=True, capture_output=True, text=True,
+ timeout=15).stdout.strip()
+
+
+def group_path(group, root=CGROUP_ROOT):
+ relative = Path(group.lstrip('/'))
+ if not group.startswith('/') or '..' in relative.parts or not relative.parts:
+ raise ValueError('target must be a non-root cgroup')
+ path = (root / relative).resolve()
+ if not path.is_relative_to(root.resolve()) or path == root.resolve():
+ raise ValueError('target is outside the cgroup mount')
+ return path
+
+
+def resolve_target(kind, name):
+ if not name or name.startswith('-'):
+ raise ValueError('invalid target name')
+ metadata = {'kind': kind, 'name': name}
+ if kind == 'unit':
+ group = command(['systemctl', 'show', '--property=ControlGroup', '--value', name])
+ else:
+ # Inspect selected fields only: a full Docker inspect exposes environment secrets.
+ details = command(['docker', 'inspect', '--type=container', '--format',
+ '{{.State.Pid}} {{.Image}} {{.State.StartedAt}}', name]).split()
+ pid = int(details[0])
+ if pid <= 0:
+ raise ValueError('container is not running')
+ metadata.update(image_id=details[1], container_started_at=details[2])
+ entries = Path(f'/proc/{pid}/cgroup').read_text().splitlines()
+ group = next((line[3:] for line in entries if line.startswith('0::')), '')
+ path = group_path(group)
+ metadata.update(cgroup=str(path), inode=path.stat().st_ino)
+ metadata['limits'] = {}
+ for filename in ('memory.max', 'memory.swap.max', 'cpu.max', 'cpuset.cpus.effective'):
+ try:
+ metadata['limits'][filename] = (path / filename).read_text().strip()
+ except OSError:
+ metadata['limits'][filename] = None
+ return metadata
+
+
+def validate_scopes(targets):
+ paths = [Path(target['cgroup']) for target in targets]
+ for index, path in enumerate(paths):
+ for other in paths[:index]:
+ if path.is_relative_to(other) or other.is_relative_to(path):
+ raise ValueError('targets overlap; select disjoint service/container cgroups')
+ # Avoid attributing the sampler's own Python interpreter to ServerKit.
+ entries = Path('/proc/self/cgroup').read_text().splitlines()
+ own_group = next((line[3:] for line in entries if line.startswith('0::')), '')
+ if own_group and own_group != '/':
+ own_path = group_path(own_group)
+ if any(own_path.is_relative_to(path) for path in paths):
+ raise ValueError('run the sampler outside the measured service/container')
+
+
+def key_values(text):
+ return {key: int(value) for key, value in (line.split() for line in text.splitlines())}
+
+
+def io_counters(text):
+ return {parts[0]: {key: int(value) for key, value in
+ (item.split('=', 1) for item in parts[1:])}
+ for parts in (line.split() for line in text.splitlines()) if parts}
+
+
+def read_target(target):
+ path = Path(target['cgroup'])
+ result = {'unavailable': [], 'error': None}
+ files = {
+ 'memory_bytes': ('memory.current', int),
+ 'swap_bytes': ('memory.swap.current', int),
+ 'memory_stat': ('memory.stat', key_values),
+ 'memory_events': ('memory.events', key_values),
+ 'cpu_stat': ('cpu.stat', key_values),
+ 'io_by_device': ('io.stat', io_counters),
+ 'tasks': ('pids.current', int),
+ }
+ for field, (filename, parse) in files.items():
+ try:
+ result[field] = parse((path / filename).read_text())
+ except (OSError, ValueError):
+ result[field] = None
+ result['unavailable'].append(filename)
+ try:
+ if path.stat().st_ino != target['inode']:
+ result['error'] = 'cgroup_replaced'
+ except OSError:
+ result['error'] = 'cgroup_unavailable'
+ if result['memory_bytes'] is None or 'usage_usec' not in (result['cpu_stat'] or {}):
+ result['error'] = result['error'] or 'required_counters_unavailable'
+ # perf_counter: monotonic() has ~15.6 ms ticks on Windows, collapsing short intervals.
+ result['monotonic_seconds'] = time.perf_counter()
+ return result
+
+
+def cpu_percent(previous, current):
+ if not previous or previous['error'] or current['error']:
+ return None
+ elapsed = current['monotonic_seconds'] - previous['monotonic_seconds']
+ delta = current['cpu_stat']['usage_usec'] - previous['cpu_stat']['usage_usec']
+ if elapsed <= 0 or delta < 0:
+ current['error'] = 'cpu_counter_reset_or_invalid_interval'
+ return None
+ return delta / (elapsed * 1_000_000) * 100
+
+
+def host_memory():
+ values = {}
+ for line in Path('/proc/meminfo').read_text().splitlines():
+ key, value = line.split(':', 1)
+ if key in ('MemTotal', 'MemAvailable', 'SwapTotal', 'SwapFree'):
+ values[key] = int(value.split()[0]) * 1024
+ return values
+
+
+def distribution(values):
+ values = sorted(value for value in values if value is not None)
+ def percentile(fraction):
+ position = (len(values) - 1) * fraction
+ lower, upper = math.floor(position), math.ceil(position)
+ return values[lower] + (values[upper] - values[lower]) * (position - lower)
+ return {'count': len(values), 'min': min(values) if values else None,
+ 'p50': percentile(.5) if values else None,
+ 'p95': percentile(.95) if values else None,
+ 'sampled_max': max(values) if values else None}
+
+
+def summarize(samples, targets):
+ summaries = []
+ for index, target in enumerate(targets):
+ rows = [sample['targets'][index] for sample in samples]
+ valid = [row for row in rows if not row['error']]
+ summaries.append({
+ 'name': target['name'], 'kind': target['kind'],
+ 'failed_samples': sum(bool(row['error']) for row in rows),
+ 'unavailable_files': sorted({name for row in rows for name in row['unavailable']}),
+ **{field: distribution([row[field] for row in valid])
+ for field in ('memory_bytes', 'swap_bytes', 'tasks', 'cpu_percent_one_core')},
+ })
+ return summaries
+
+
+def collect(targets, duration, interval):
+ samples, previous, interrupted = [], [None] * len(targets), False
+ started = time.perf_counter()
+ deadline = started + duration
+ try:
+ while True:
+ sample = {'elapsed_seconds': time.perf_counter() - started,
+ 'host_memory_bytes': host_memory(), 'targets': []}
+ for index, target in enumerate(targets):
+ row = read_target(target)
+ row['cpu_percent_one_core'] = cpu_percent(previous[index], row)
+ sample['targets'].append(row)
+ previous[index] = row
+ samples.append(sample)
+ now = time.perf_counter()
+ if now >= deadline:
+ break
+ time.sleep(min(interval, deadline - now))
+ except KeyboardInterrupt:
+ interrupted = True
+ return samples, interrupted
+
+
+def main(argv=None):
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--unit', action='append', default=[], help='systemd unit; repeatable')
+ parser.add_argument('--container', action='append', default=[], help='Docker container; repeatable')
+ parser.add_argument('--duration', type=float, default=600, help='seconds, default 600')
+ parser.add_argument('--interval', type=float, default=5, help='seconds, default 5')
+ parser.add_argument('--scenario', required=True, help='e.g. idle-closed, dashboard-open, deploy')
+ parser.add_argument('--revision', required=True, help='revision actually running, not sampler revision')
+ parser.add_argument('--notes', default='', help='workload, enabled extensions, DB and warmup; no secrets')
+ parser.add_argument('--output', type=Path, required=True)
+ args = parser.parse_args(argv)
+ if not args.unit and not args.container:
+ parser.error('select at least one --unit or --container')
+ if (not all(math.isfinite(value) and value > 0 for value in (args.duration, args.interval))
+ or args.interval > args.duration):
+ parser.error('require finite duration >= interval > 0')
+ if platform.system() != 'Linux' or not (CGROUP_ROOT / 'cgroup.controllers').is_file():
+ parser.error('requires a Linux host with cgroup v2; cgroup v1 is not supported')
+ try:
+ targets = [resolve_target(kind, name) for kind, names in
+ (('unit', args.unit), ('container', args.container)) for name in names]
+ validate_scopes(targets)
+ for target in targets:
+ if read_target(target)['error']:
+ raise ValueError(f"required counters unavailable for {target['name']}")
+ except (OSError, ValueError, subprocess.SubprocessError) as exc:
+ parser.error(f'target discovery failed ({type(exc).__name__}); check names, permissions and cgroup v2')
+ # Exclusive create prevents accidentally overwriting a before/after baseline.
+ with args.output.open('x', encoding='utf-8') as output:
+ report = {
+ 'schema_version': 1, 'started_at': datetime.now(timezone.utc).isoformat(),
+ 'scenario': args.scenario, 'running_revision': args.revision, 'notes': args.notes,
+ 'host': {'os': platform.freedesktop_os_release().get('PRETTY_NAME'),
+ 'kernel': platform.release(), 'architecture': platform.machine(),
+ 'logical_cpus': os.cpu_count(), 'python': platform.python_version(),
+ 'uptime_seconds': float(Path('/proc/uptime').read_text().split()[0])},
+ 'requested_duration_seconds': args.duration, 'interval_seconds': args.interval,
+ 'targets': targets,
+ 'accounting': {
+ 'memory': 'cgroup memory.current; includes cache, kernel and descendants; not process RSS',
+ 'cpu': 'delta usage_usec / actual elapsed time; 100% = one logical CPU',
+ 'io': 'cumulative per-device counters; do not sum stacked devices blindly',
+ 'scope': 'explicit targets only; excludes other services and separately created app containers',
+ 'peaks': 'sampled maxima only; short spikes between samples can be missed',
+ },
+ }
+ print('Sampling explicit targets; Ctrl+C saves a partial report.', file=sys.stderr, flush=True)
+ samples, interrupted = collect(targets, args.duration, args.interval)
+ report.update(samples=samples, interrupted=interrupted,
+ finished_at=datetime.now(timezone.utc).isoformat(),
+ summary=summarize(samples, targets))
+ json.dump(report, output, indent=2, allow_nan=False)
+ output.write('\n')
+ print(json.dumps(report['summary'], indent=2))
+ return 130 if interrupted else int(any(row['failed_samples'] for row in report['summary']))
+
+
+if __name__ == '__main__':
+ raise SystemExit(main())
diff --git a/scripts/test/README.md b/scripts/test/README.md
index 4c24aaaa..a5c663a2 100644
--- a/scripts/test/README.md
+++ b/scripts/test/README.md
@@ -117,12 +117,25 @@ scripts/test/output/<run-id>/
install-status ("OK" or "FAIL")
pytest.log
pytest-report.json
+ runtime.json (post-install resource observation)
sk-test-ubuntu24-<id>/ ...
sk-test-debian12-<id>/ ...
```
## Update & uninstall coverage
+Full VM installs now wait 60 seconds after the health check, then collect a
+120-second passive runtime baseline before pytest. The HTML report shows median
+and sampled-peak cgroup memory, and median/p95 CPU per measured service. Raw data
+is in `runtime.json`; unavailable/failed measurements are shown explicitly and
+do not turn a successful install into a failed install. Test Sandbox full mode
+uses the same sampler and saves `<distro>.runtime.json` beside its log.
+
+These short observations help identify regressions, but parallel VM/container
+work can affect them. Quick/provision checks do not run the panel and cannot
+measure its runtime overhead. See [measurement definitions and longer baseline
+commands](../../docs/METRICS.md#passive-linux-runtime-sampler).
+
Alongside `vm-install.sh`, two more "run inside the VM" scripts exercise the
other lifecycle stages end-to-end (same contract: log to
`/var/log/serverkit-test-*.log`, write `OK`/`FAIL` to `/tmp/serverkit-*-status`):
diff --git a/scripts/test/full-stack-test.ps1 b/scripts/test/full-stack-test.ps1
index 33ea382b..e57eed12 100644
--- a/scripts/test/full-stack-test.ps1
+++ b/scripts/test/full-stack-test.ps1
@@ -197,6 +197,7 @@ function Test-VmInstall {
& $MpExe transfer "${Vm}:/var/log/serverkit-test-install.log" "$VmOut\vm-install.log" 2>&1 | Out-Null
& $MpExe transfer "${Vm}:/tmp/serverkit-install-status" $statusFile 2>&1 | Out-Null
+ & $MpExe transfer "${Vm}:/tmp/serverkit-runtime.json" (Join-Path $VmOut 'runtime.json') 2>&1 | Out-Null
$rawStatus = if (Test-Path $statusFile) { Get-Content $statusFile -Raw -ErrorAction SilentlyContinue } else { $null }
if (-not $rawStatus -or "$rawStatus".Trim() -eq '') {
$fallback = if ($installRC -eq 0) { 'OK' } else { 'FAIL' }
@@ -285,6 +286,8 @@ function Test-VagrantInstall {
| Out-File (Join-Path $VmOut 'vm-install.log') -Encoding utf8
& $VgExe ssh -c "sudo cat /tmp/serverkit-install-status" 2>$null `
| Out-File $statusFile -Encoding ascii
+ & $VgExe ssh -c "sudo cat /tmp/serverkit-runtime.json" 2>$null `
+ | Out-File (Join-Path $VmOut 'runtime.json') -Encoding utf8
# Get-Content -Raw returns $null for empty/missing files; guard before
# calling .Trim() so a failed status capture doesn't crash the run.
$rawStatus = if (Test-Path $statusFile) { Get-Content $statusFile -Raw -ErrorAction SilentlyContinue } else { $null }
diff --git a/scripts/test/measure-installed-runtime.py b/scripts/test/measure-installed-runtime.py
new file mode 100644
index 00000000..9a83b2e5
--- /dev/null
+++ b/scripts/test/measure-installed-runtime.py
@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Record a short post-install baseline, with explicit unavailable/error results.
+
+Used by full distro installs (VMs and systemd containers), never quick/provision
+checks. This is a regression observation, not a production capacity benchmark.
+"""
+
+import argparse
+from datetime import datetime, timezone
+import hashlib
+import importlib.util
+import json
+import math
+import os
+from pathlib import Path
+import platform
+import subprocess
+import sys
+import time
+
+
+SPEC = importlib.util.spec_from_file_location('measure_runtime', Path(__file__).parents[1] / 'measure-runtime.py')
+runtime = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(runtime)
+
+
+def source_identity(source, archive=None):
+ if archive:
+ digest = hashlib.sha256()
+ with archive.open('rb') as stream:
+ for block in iter(lambda: stream.read(1024 * 1024), b''):
+ digest.update(block)
+ return 'local-overlay-archive-sha256:' + digest.hexdigest()
+ try:
+ revision = runtime.command(['git', '-C', str(source), 'rev-parse', 'HEAD'])
+ dirty = runtime.command(['git', '-C', str(source), 'status', '--porcelain', '--untracked-files=no'])
+ return revision + (' (tracked files modified)' if dirty else '')
+ except (OSError, subprocess.SubprocessError):
+ version = source / 'VERSION'
+ return 'release:' + version.read_text().strip() if version.is_file() else 'unknown'
+
+
+def observe(source, duration=120, settle=60, archive=None):
+ result = {'schema_version': 1, 'status': 'unavailable',
+ 'scenario': 'post-install-dashboard-closed',
+ 'started_at': datetime.now(timezone.utc).isoformat(),
+ 'settle_seconds': settle, 'requested_duration_seconds': duration,
+ 'scope_note': 'Selected service cgroups only; other services/apps are excluded. '
+ 'Container tests share the host kernel and may run beside other tests.'}
+ if platform.system() != 'Linux' or not (runtime.CGROUP_ROOT / 'cgroup.controllers').is_file():
+ result['reason'] = 'Linux cgroup v2 unavailable; no runtime measurement taken'
+ return result
+ try:
+ revision = source_identity(source, archive)
+ targets = [runtime.resolve_target('unit', 'serverkit.service')]
+ for unit in ('nginx.service', 'docker.service', 'containerd.service'):
+ try:
+ if runtime.command(['systemctl', 'is-active', unit]) == 'active':
+ targets.append(runtime.resolve_target('unit', unit))
+ except (OSError, ValueError, subprocess.SubprocessError):
+ continue
+ runtime.validate_scopes(targets)
+ # This pause is after builds/health probes, before measurement or pytest.
+ time.sleep(settle)
+ samples, interrupted = runtime.collect(targets, duration, 5)
+ summary = runtime.summarize(samples, targets)
+ failed = any(row['failed_samples'] for row in summary)
+ result.update(status='interrupted' if interrupted else 'failed' if failed else 'measured',
+ running_revision=revision, targets=targets, summary=summary, samples=samples,
+ os=platform.freedesktop_os_release().get('PRETTY_NAME'),
+ kernel=platform.release(), architecture=platform.machine(),
+ logical_cpus=os.cpu_count(),
+ definitions={'memory_bytes': 'memory.current including cache/kernel/descendants',
+ 'cpu_percent_one_core': '100% = one logical CPU',
+ 'sampled_max': 'largest sampled value; short spikes can be missed'})
+ except (OSError, ValueError, subprocess.SubprocessError) as exc:
+ result.update(status='failed', reason=f'Runtime collection failed: {type(exc).__name__}')
+ result['finished_at'] = datetime.now(timezone.utc).isoformat()
+ return result
+
+
+def main(argv=None):
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--source', type=Path, default=Path('/opt/serverkit'))
+ parser.add_argument('--source-archive', type=Path, help='VM harness local overlay archive')
+ parser.add_argument('--output', type=Path, required=True)
+ parser.add_argument('--duration', type=float, default=120)
+ parser.add_argument('--settle', type=float, default=60)
+ args = parser.parse_args(argv)
+ if (not math.isfinite(args.duration) or args.duration < 5
+ or not math.isfinite(args.settle) or args.settle < 0):
+ parser.error('duration must be finite and >= 5; settle finite and >= 0')
+ # A VM reuse explicitly produces a new observation at this harness scratch path.
+ result = observe(args.source, args.duration, args.settle, args.source_archive)
+ args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + '\n', encoding='utf-8')
+ print(json.dumps({key: value for key, value in result.items() if key != 'samples'}, indent=2))
+ return 0 if result['status'] == 'measured' else 1
+
+
+if __name__ == '__main__':
+ raise SystemExit(main())
diff --git a/scripts/test/report.py b/scripts/test/report.py
index e5e83c6f..74ee1afb 100644
--- a/scripts/test/report.py
+++ b/scripts/test/report.py
@@ -23,7 +23,7 @@ def esc(s):
def load_json(path):
try:
- return json.loads(path.read_text(encoding="utf-8"))
+ return json.loads(path.read_text(encoding="utf-8-sig"))
except Exception:
return None
@@ -71,9 +71,35 @@ def vm_summary(vm_dir: Path):
"skip": skip_count,
"install_log": read_text(vm_dir / "install.log"),
"journal": read_text(vm_dir / "journalctl.log", limit=50_000),
+ "runtime": load_json(vm_dir / "runtime.json"),
}
+def runtime_table(measurement):
+ if not measurement:
+ return '<p>Runtime baseline: unavailable (no artifact).</p>'
+ status = measurement.get('status', 'unknown')
+ if status != 'measured':
+ return f"<p>Runtime baseline: {esc(status)}. {esc(measurement.get('reason', ''))}</p>"
+
+ def number(value, divisor=1):
+ return 'unavailable' if value is None else f'{value / divisor:,.2f}'
+
+ rows = []
+ for item in measurement.get('summary', []):
+ memory = item.get('memory_bytes', {})
+ cpu = item.get('cpu_percent_one_core', {})
+ rows.append(f"<tr><td>{esc(item['name'])}</td>"
+ f"<td>{number(memory.get('p50'), 1024 ** 2)}</td>"
+ f"<td>{number(memory.get('sampled_max'), 1024 ** 2)}</td>"
+ f"<td>{number(cpu.get('p50'))}</td><td>{number(cpu.get('p95'))}</td></tr>")
+ return ('<p>Post-install observation: cgroup memory includes cache; CPU 100% = one logical CPU. '
+ 'Sampled peaks can miss short spikes. This is not a capacity benchmark.</p>'
+ '<table><tr><th>Service</th><th>Median MiB</th><th>Sampled peak MiB</th>'
+ '<th>Median CPU %</th><th>p95 CPU %</th></tr>' + ''.join(rows) + '</table>'
+ f"<p>Source: <code>{esc(measurement.get('running_revision'))}</code></p>")
+
+
HTML_TEMPLATE = """<!doctype html>
<html><head><meta charset="utf-8"><title>ServerKit E2E Report — {ts}</title>
{refresh_meta}
@@ -119,6 +145,7 @@ VM_TEMPLATE = """<div class="vm {cls}">
<span>Skipped: <b>{s}</b></span>
</div>
{tests_table}
+ {runtime_table}
<details><summary>Install log</summary><pre>{install_log}</pre></details>
<details><summary>journalctl -u serverkit</summary><pre>{journal}</pre></details>
</div>"""
@@ -203,6 +230,7 @@ def render(run_dir: Path) -> Path:
install="OK" if s["install_ok"] else ("-" if s["overall"] == "RUNNING" else "FAIL"),
p=s["pass"], f=s["fail"], s=s["skip"],
tests_table=tests_table,
+ runtime_table=runtime_table(s.get('runtime')),
install_log=esc(s["install_log"]),
journal=esc(s["journal"]),
))
diff --git a/scripts/test/test_measure_runtime.py b/scripts/test/test_measure_runtime.py
new file mode 100644
index 00000000..3263b215
--- /dev/null
+++ b/scripts/test/test_measure_runtime.py
@@ -0,0 +1,156 @@
+"""Stdlib tests for accounting boundaries, missing data and real file sampling."""
+
+import importlib.util
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+
+SPEC = importlib.util.spec_from_file_location('measure_runtime', Path(__file__).parents[1] / 'measure-runtime.py')
+runtime = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(runtime)
+
+INSTALL_SPEC = importlib.util.spec_from_file_location('measure_installed', Path(__file__).with_name('measure-installed-runtime.py'))
+installed = importlib.util.module_from_spec(INSTALL_SPEC)
+INSTALL_SPEC.loader.exec_module(installed)
+
+REPORT_SPEC = importlib.util.spec_from_file_location('vm_report', Path(__file__).with_name('report.py'))
+vm_report = importlib.util.module_from_spec(REPORT_SPEC)
+REPORT_SPEC.loader.exec_module(vm_report)
+
+
+class RuntimeTests(unittest.TestCase):
+ def setUp(self):
+ self.temp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temp.cleanup)
+ self.path = Path(self.temp.name) / 'service'
+ self.path.mkdir()
+ (self.path / 'memory.current').write_text('104857600\n')
+ (self.path / 'cpu.stat').write_text('usage_usec 500000\nuser_usec 400000\nsystem_usec 100000\n')
+ self.target = {'name': 'serverkit.service', 'kind': 'unit',
+ 'cgroup': str(self.path), 'inode': self.path.stat().st_ino}
+
+ def row(self, usage, clock):
+ return {'cpu_stat': {'usage_usec': usage}, 'monotonic_seconds': clock, 'error': None}
+
+ def test_cpu_normalizes_by_actual_time_and_one_core(self):
+ self.assertEqual(runtime.cpu_percent(self.row(0, 10), self.row(2_000_000, 12)), 100)
+ self.assertEqual(runtime.cpu_percent(self.row(0, 10), self.row(4_000_000, 12)), 200)
+ self.assertEqual(runtime.cpu_percent(self.row(0, 10), self.row(500_000, 20)), 5)
+ self.assertIsNone(runtime.cpu_percent(None, self.row(2_000_000, 12)))
+
+ def test_counter_reset_is_invalid_not_zero_cpu(self):
+ current = self.row(0, 12)
+ self.assertIsNone(runtime.cpu_percent(self.row(100, 10), current))
+ self.assertIsNotNone(current['error'])
+
+ def test_missing_optional_data_stays_null(self):
+ row = runtime.read_target(self.target)
+ self.assertEqual(row['memory_bytes'], 104857600)
+ self.assertIsNone(row['swap_bytes'])
+ self.assertIsNone(row['tasks'])
+ self.assertIsNone(row['io_by_device'])
+ self.assertIn('io.stat', row['unavailable'])
+ self.assertIsNone(row['error'])
+
+ def test_missing_required_data_and_replaced_group_fail(self):
+ (self.path / 'memory.current').unlink()
+ self.assertEqual(runtime.read_target(self.target)['error'], 'required_counters_unavailable')
+ self.target['inode'] += 1
+ self.assertEqual(runtime.read_target(self.target)['error'], 'cgroup_replaced')
+
+ def test_device_io_and_memory_breakdown_preserved(self):
+ (self.path / 'io.stat').write_text('8:0 rbytes=12 wbytes=20 rios=1 wios=2\n253:0 rbytes=12 wbytes=20\n')
+ (self.path / 'memory.stat').write_text('anon 100\nfile 50\nshmem 20\n')
+ row = runtime.read_target(self.target)
+ self.assertEqual(row['io_by_device']['8:0']['wbytes'], 20)
+ self.assertEqual(len(row['io_by_device']), 2)
+ self.assertEqual(row['memory_stat']['file'], 50)
+
+ def test_reject_root_and_traversal(self):
+ for group in ('/', '', '/../../etc', 'relative'):
+ with self.subTest(group=group), self.assertRaises(ValueError):
+ runtime.group_path(group, Path(self.temp.name))
+
+ def test_reject_overlapping_scopes(self):
+ for second in (self.path, self.path / 'child'):
+ with self.subTest(second=second), self.assertRaises(ValueError):
+ runtime.validate_scopes([self.target, {'cgroup': str(second)}])
+
+ def test_real_sampling_and_summary_exclude_failures(self):
+ with patch.object(runtime, 'host_memory', return_value={'MemAvailable': 1024}):
+ samples, interrupted = runtime.collect([self.target], .025, .01)
+ self.assertFalse(interrupted)
+ self.assertGreaterEqual(len(samples), 2)
+ summary = runtime.summarize(samples, [self.target])[0]
+ self.assertEqual(summary['memory_bytes']['p50'], 104857600)
+ self.assertEqual(summary['cpu_percent_one_core']['count'], len(samples) - 1)
+ self.assertEqual(summary['swap_bytes']['count'], 0)
+ self.assertIsNone(summary['swap_bytes']['sampled_max'])
+ samples[-1]['targets'][0]['error'] = 'cgroup_replaced'
+ samples[-1]['targets'][0]['memory_bytes'] = 999999999999
+ summary = runtime.summarize(samples, [self.target])[0]
+ self.assertEqual(summary['failed_samples'], 1)
+ self.assertEqual(summary['memory_bytes']['sampled_max'], 104857600)
+
+ def test_nonfinite_durations_rejected_before_discovery(self):
+ for duration in ('nan', 'inf', '-1'):
+ with self.subTest(duration=duration), patch('sys.stderr'), self.assertRaises(SystemExit) as exc:
+ runtime.main(['--unit', 'serverkit', '--scenario', 'idle', '--revision', 'test',
+ '--duration', duration, '--output', str(self.path / 'out.json')])
+ self.assertEqual(exc.exception.code, 2)
+
+ def test_vm_report_preserves_unavailable_and_escapes_names(self):
+ self.assertIn('unavailable', vm_report.runtime_table(None))
+ self.assertIn('cgroup v2', vm_report.runtime_table({'status': 'unavailable', 'reason': 'cgroup v2'}))
+ rendered = vm_report.runtime_table({'status': 'measured', 'summary': [{
+ 'name': '<script>', 'memory_bytes': {'p50': 1048576, 'sampled_max': 2097152},
+ 'cpu_percent_one_core': {'p50': 0, 'p95': None},
+ }]})
+ self.assertIn('&lt;script&gt;', rendered)
+ self.assertNotIn('<script>', rendered)
+ self.assertIn('1.00', rendered)
+ self.assertIn('unavailable', rendered)
+
+ def test_install_wrapper_marks_unsupported_without_sampling(self):
+ with patch.object(installed.platform, 'system', return_value='Windows'), \
+ patch.object(installed.runtime, 'collect') as collect:
+ result = installed.observe(self.path, duration=5, settle=0)
+ self.assertEqual(result['status'], 'unavailable')
+ collect.assert_not_called()
+
+ def test_overlay_revision_tracks_archive_content_not_installer_git(self):
+ archive = self.path / 'overlay.tar.gz'
+ archive.write_bytes(b'first source')
+ first = installed.source_identity(self.path, archive)
+ archive.write_bytes(b'changed source')
+ self.assertNotEqual(first, installed.source_identity(self.path, archive))
+
+ def test_install_wrapper_keeps_raw_samples_and_failed_status(self):
+ (self.path / 'cgroup.controllers').write_text('cpu memory')
+ row = runtime.read_target(self.target)
+ row.update(error='cgroup_replaced', cpu_percent_one_core=None)
+ with patch.object(installed.platform, 'system', return_value='Linux'), \
+ patch.object(installed.runtime, 'CGROUP_ROOT', self.path), \
+ patch.object(installed, 'source_identity', return_value='revision'), \
+ patch.object(installed.runtime, 'resolve_target', return_value=self.target), \
+ patch.object(installed.runtime, 'command', return_value='inactive'), \
+ patch.object(installed.runtime, 'validate_scopes'), \
+ patch.object(installed.platform, 'freedesktop_os_release', return_value={'PRETTY_NAME': 'Test Linux'}), \
+ patch.object(installed.runtime, 'collect', return_value=([{'targets': [row]}], False)):
+ result = installed.observe(self.path, duration=5, settle=0)
+ self.assertEqual(result['status'], 'failed')
+ self.assertEqual(result['summary'][0]['failed_samples'], 1)
+ self.assertEqual(result['samples'][0]['targets'][0]['error'], 'cgroup_replaced')
+ json.dumps(result, allow_nan=False)
+
+ def test_bom_from_windows_vm_capture_is_readable(self):
+ artifact = self.path / 'runtime.json'
+ artifact.write_text('{"status": "measured"}', encoding='utf-8-sig')
+ self.assertEqual(vm_report.load_json(artifact)['status'], 'measured')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/scripts/test/vm-install.sh b/scripts/test/vm-install.sh
index 15a027c2..9f526ff6 100644
--- a/scripts/test/vm-install.sh
+++ b/scripts/test/vm-install.sh
@@ -148,6 +148,11 @@ log "Step 4/4: waiting for health endpoint"
for i in $(seq 1 30); do
if curl -fsS http://127.0.0.1:5000/api/v1/system/health > /dev/null 2>&1; then
log "Backend healthy after ${i}s"
+ log "Measuring post-install runtime (60s settle + 120s observation)"
+ "$INSTALL_DIR/venv/bin/python" "$SRC/scripts/test/measure-installed-runtime.py" \
+ --source "$INSTALL_DIR" --source-archive /tmp/serverkit-src.tar.gz \
+ --output /tmp/serverkit-runtime.json >> "$LOG" 2>&1 \
+ || log "WARN: runtime baseline unavailable/failed; inspect runtime report"
echo "OK" > /tmp/serverkit-install-status
exit 0
fi