ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
feat(test): passive cgroup-v2 runtime baselines for full installs
Juan Denisff3c92891752
diff --git a/.github/workflows/measurements-ci.yml b/.github/workflows/measurements-ci.ymlindex 9ba5c28d..f7be0558 100644--- a/.github/workflows/measurements-ci.yml+++ b/.github/workflows/measurements-ci.yml@@ -10,8 +10,12 @@ on:- 'scripts/measure-repository.py'- 'scripts/update-readme-measurements.py'- 'scripts/profile-api.py'+ - 'scripts/measure-runtime.py'+ - 'scripts/test/measure-installed-runtime.py'+ - 'scripts/test/report.py'- 'scripts/test/test_repository_measurements.py'- 'scripts/test/test_profile_api.py'+ - 'scripts/test/test_measure_runtime.py'- '.github/workflows/measurements-ci.yml'pull_request:branches: [main]@@ -22,8 +26,12 @@ on:- 'scripts/measure-repository.py'- 'scripts/update-readme-measurements.py'- 'scripts/profile-api.py'+ - 'scripts/measure-runtime.py'+ - 'scripts/test/measure-installed-runtime.py'+ - 'scripts/test/report.py'- 'scripts/test/test_repository_measurements.py'- 'scripts/test/test_profile_api.py'+ - 'scripts/test/test_measure_runtime.py'- '.github/workflows/measurements-ci.yml'jobs:@@ -38,5 +46,6 @@ jobs:run: |python -m unittest discover -s scripts/test -p test_repository_measurements.pypython -m unittest discover -s scripts/test -p test_profile_api.py+ python -m unittest discover -s scripts/test -p test_measure_runtime.py- name: Check translated README tables against reviewed snapshotsrun: python scripts/update-readme-measurements.pydiff --git a/backend/app/services/test_sandbox_service.py b/backend/app/services/test_sandbox_service.pyindex fc862721..8d7c4b24 100644--- a/backend/app/services/test_sandbox_service.py+++ b/backend/app/services/test_sandbox_service.py@@ -247,6 +247,19 @@ class TestSandboxService:'detail': detail,'duration_s': round(time.monotonic() - started, 1),}+ runtime_path = work_dir / f'{distro}.runtime.json'+ if runtime_path.is_file():+ try:+ measurement = json.loads(runtime_path.read_text())+ # Raw samples stay in the per-run artifact; compact evidence+ # travels with the existing results API and log summary.+ results[distro]['runtime_measurement'] = {+ key: value for key, value in measurement.items() if key != 'samples'+ }+ except (OSError, ValueError):+ results[distro]['runtime_measurement'] = {+ 'status': 'failed', 'reason': 'runtime artifact unreadable',+ }run.results = resultsdb.session.commit()@@ -345,6 +358,33 @@ class TestSandboxService:timeout=15)if probe['success']:log.write('healthy\n')- return True, 'install + health OK'+ detail = cls._measure_runtime(container, log)+ return True, f'install + health OK; runtime {detail}'time.sleep(2)return False, 'install OK but health endpoint never came up'++ @staticmethod+ def _measure_runtime(container, log):+ """Observe after install; measurement failure is distinct from install failure."""+ log.write('\n===== post-install runtime baseline (60s settle + 120s sample) =====\n')+ log.flush()+ measured = run_checked(+ ['docker', 'exec', container, '/opt/serverkit/venv/bin/python',+ '/src/scripts/test/measure-installed-runtime.py',+ '--output', '/tmp/serverkit-runtime.json'],+ stdout=log, stderr=subprocess.STDOUT, timeout=240)+ artifact = Path(log.name).with_suffix('.runtime.json')+ copied = run_checked(+ ['docker', 'cp', f'{container}:/tmp/serverkit-runtime.json', str(artifact)],+ timeout=30)+ if not copied['success']:+ artifact.write_text(json.dumps({+ 'status': 'failed', 'reason': 'runtime artifact could not be collected',+ }))+ if measured['success'] and copied['success']:+ try:+ if json.loads(artifact.read_text()).get('status') == 'measured':+ return 'measured (see log/artifact)'+ except (OSError, ValueError):+ pass+ return 'unavailable/failed (see log/artifact)'diff --git a/backend/tests/test_sandbox_runtime.py b/backend/tests/test_sandbox_runtime.pynew file mode 100644index 00000000..db0fd1bf--- /dev/null+++ b/backend/tests/test_sandbox_runtime.py@@ -0,0 +1,44 @@+"""Resource observation must survive sandbox teardown without inventing success."""++import json+from pathlib import Path++from app.services import test_sandbox_service as sandbox+++def test_collects_runtime_artifact_before_container_cleanup(tmp_path, monkeypatch):+ calls = []++ def run(cmd, **kwargs):+ calls.append(cmd)+ if cmd[:2] == ['docker', 'cp']:+ Path(cmd[-1]).write_text(json.dumps({'status': 'measured', 'samples': []}))+ return {'success': True}++ monkeypatch.setattr(sandbox, 'run_checked', run)+ with (tmp_path / 'ubuntu24.log').open('w') as log:+ result = sandbox.TestSandboxService._measure_runtime('test-container', log)+ assert result.startswith('measured')+ assert calls[0][:3] == ['docker', 'exec', 'test-container']+ assert calls[1][2] == 'test-container:/tmp/serverkit-runtime.json'+ assert json.loads((tmp_path / 'ubuntu24.runtime.json').read_text())['status'] == 'measured'+++def test_missing_artifact_is_explicit_failure(tmp_path, monkeypatch):+ monkeypatch.setattr(sandbox, 'run_checked', lambda *args, **kwargs: {'success': False})+ with (tmp_path / 'rocky9.log').open('w') as log:+ result = sandbox.TestSandboxService._measure_runtime('test-container', log)+ assert result.startswith('unavailable/failed')+ assert json.loads((tmp_path / 'rocky9.runtime.json').read_text())['status'] == 'failed'+++def test_successful_copy_of_unavailable_report_is_not_a_measurement(tmp_path, monkeypatch):+ def run(cmd, **kwargs):+ if cmd[:2] == ['docker', 'cp']:+ Path(cmd[-1]).write_text(json.dumps({'status': 'unavailable'}))+ return {'success': True}++ monkeypatch.setattr(sandbox, 'run_checked', run)+ with (tmp_path / 'legacy.log').open('w') as log:+ result = sandbox.TestSandboxService._measure_runtime('test-container', log)+ assert result.startswith('unavailable/failed')diff --git a/docs/METRICS.md b/docs/METRICS.mdindex 85a53481..bf77b602 100644--- a/docs/METRICS.md+++ b/docs/METRICS.md@@ -96,6 +96,77 @@ construction; background jobs and streamed body iteration are not included.## Measure memory, images and browser loading separately+### Passive Linux runtime sampler++`scripts/measure-runtime.py` reads Linux **cgroup v2** counters for explicitly+selected systemd units or Docker containers. It uses Python's standard library,+does not restart services or generate HTTP traffic, and saves timestamped raw+samples plus median/p95/sampled-maximum summaries. Run it on the host, outside+the services being measured. It rejects overlapping scopes and reports missing+counters or replaced cgroups instead of inventing zero usage.++For an installed host panel, first verify the unit actually runs the backend+(not a oneshot wrapper that starts Docker). Record the revision actually running:++```bash+python3 scripts/measure-runtime.py --unit serverkit.service --unit nginx.service \+ --scenario idle-dashboard-closed --revision "$RUNNING_SERVERKIT_REVISION" \+ --notes 'Record DB engine, enabled extensions, app count, warmup and other host activity here' \+ --duration 600 --interval 5 --output runtime-idle.json+```++For the panel container, use `--container serverkit` instead of `--unit`.+Repeat `--unit` / `--container` to include separate supporting services (for+example Docker, containerd, or an external panel database); results remain per+target. Newly created application/build containers are **not automatically+included**. The JSON output is created exclusively so an earlier baseline cannot+be accidentally overwritten. Ctrl+C saves a partial report with exit code 130;+failed samples return exit code 1. Optional unavailable counters stay null.++- Memory is `memory.current`, including descendants, charged file cache and+ kernel memory. It is **not process RSS or Docker CLI's cache-adjusted number**.+ Raw `memory.stat` is retained; its fields overlap, so do not sum them all.+- CPU is the change in `cpu.stat` usage divided by actual elapsed time; 100%+ means one logical CPU, and values can exceed 100% on multicore systems.+- Swap, task counts, cumulative per-device I/O and memory-event counters are+ retained where available. Device counters can overlap with stacked storage;+ inspect deltas per device rather than summing them blindly. Memory events are+ lifetime counters; compare the first and last samples for events in the run.+- Peaks are **sampled**, not exact instantaneous high-water marks. Host available+ memory provides context but includes unrelated workloads. Record machine+ specs, cgroup limits and background activity alongside any published result.++Definitions follow the [Linux cgroup-v2 documentation](https://docs.kernel.org/admin-guide/cgroup-v2.html).+Start with a 10-minute quiet baseline after startup settles, then repeat with the+dashboard open and a fixed set of apps. On a disposable test server, separately+observe a deployment and backup while also measuring application response times.+Use the same hardware, limits, versions, data and workload before/after a fix.+This sampler measures resource usage, not application latency or throughput.++### Automatic observations in full distro installs++The VM harness and Test Sandbox **full** mode invoke+`scripts/test/measure-installed-runtime.py` after the health check, before other+API tests. It settles for 60 seconds, then observes for 120 seconds at 5-second+intervals. The panel and active nginx/Docker/containerd units are recorded+separately. Each run saves a runtime artifact and an explicit measured,+unavailable, failed or interrupted status; install success does not imply that+measurement succeeded. No arbitrary RAM/CPU pass threshold is enforced yet.++VM results include `runtime.json` and a table in the HTML report. Test Sandbox+keeps `<distro>.runtime.json` next to its log, includes compact measurement data+in the existing results API, and prints the summary into the log. The VM overlay+archive SHA-256 identifies uncommitted source; full container installs identify+the installed checkout/release (which can differ from the local installer).++Quick/provision distro probes do not run ServerKit, so they cannot produce a+panel runtime baseline. Full systemd containers share the host kernel; parallel+installs and startup jobs can affect these short observations. Use an isolated+VM/VPS and longer repeated runs for performance claims. Cgroup-v1 environments+are explicitly unavailable rather than silently measured with different rules.++### Other measurement boundaries+- **Memory:** record the exact process/container, workload, uptime and samplinginterval. Process RSS and Docker memory accounting are different measurements.Include workers and enabled services; do not describe one idle sample as adiff --git a/scripts/measure-runtime.py b/scripts/measure-runtime.pynew file mode 100644index 00000000..452d7b19--- /dev/null+++ b/scripts/measure-runtime.py@@ -0,0 +1,249 @@+#!/usr/bin/env python3+"""Passively sample explicit Linux cgroup-v2 services/containers; stdlib only.++Run on the host, outside the measured groups. No restarts, config changes, HTTP+requests, environment inspection, or load generation. Memory includes cache and+descendants; CPU 100% means one fully occupied logical CPU. Missing data is null.+"""++import argparse+from datetime import datetime, timezone+import json+import math+import os+from pathlib import Path+import platform+import subprocess+import sys+import time+++CGROUP_ROOT = Path('/sys/fs/cgroup')+++def command(args):+ return subprocess.run(args, check=True, capture_output=True, text=True,+ timeout=15).stdout.strip()+++def group_path(group, root=CGROUP_ROOT):+ relative = Path(group.lstrip('/'))+ if not group.startswith('/') or '..' in relative.parts or not relative.parts:+ raise ValueError('target must be a non-root cgroup')+ path = (root / relative).resolve()+ if not path.is_relative_to(root.resolve()) or path == root.resolve():+ raise ValueError('target is outside the cgroup mount')+ return path+++def resolve_target(kind, name):+ if not name or name.startswith('-'):+ raise ValueError('invalid target name')+ metadata = {'kind': kind, 'name': name}+ if kind == 'unit':+ group = command(['systemctl', 'show', '--property=ControlGroup', '--value', name])+ else:+ # Inspect selected fields only: a full Docker inspect exposes environment secrets.+ details = command(['docker', 'inspect', '--type=container', '--format',+ '{{.State.Pid}} {{.Image}} {{.State.StartedAt}}', name]).split()+ pid = int(details[0])+ if pid <= 0:+ raise ValueError('container is not running')+ metadata.update(image_id=details[1], container_started_at=details[2])+ entries = Path(f'/proc/{pid}/cgroup').read_text().splitlines()+ group = next((line[3:] for line in entries if line.startswith('0::')), '')+ path = group_path(group)+ metadata.update(cgroup=str(path), inode=path.stat().st_ino)+ metadata['limits'] = {}+ for filename in ('memory.max', 'memory.swap.max', 'cpu.max', 'cpuset.cpus.effective'):+ try:+ metadata['limits'][filename] = (path / filename).read_text().strip()+ except OSError:+ metadata['limits'][filename] = None+ return metadata+++def validate_scopes(targets):+ paths = [Path(target['cgroup']) for target in targets]+ for index, path in enumerate(paths):+ for other in paths[:index]:+ if path.is_relative_to(other) or other.is_relative_to(path):+ raise ValueError('targets overlap; select disjoint service/container cgroups')+ # Avoid attributing the sampler's own Python interpreter to ServerKit.+ entries = Path('/proc/self/cgroup').read_text().splitlines()+ own_group = next((line[3:] for line in entries if line.startswith('0::')), '')+ if own_group and own_group != '/':+ own_path = group_path(own_group)+ if any(own_path.is_relative_to(path) for path in paths):+ raise ValueError('run the sampler outside the measured service/container')+++def key_values(text):+ return {key: int(value) for key, value in (line.split() for line in text.splitlines())}+++def io_counters(text):+ return {parts[0]: {key: int(value) for key, value in+ (item.split('=', 1) for item in parts[1:])}+ for parts in (line.split() for line in text.splitlines()) if parts}+++def read_target(target):+ path = Path(target['cgroup'])+ result = {'unavailable': [], 'error': None}+ files = {+ 'memory_bytes': ('memory.current', int),+ 'swap_bytes': ('memory.swap.current', int),+ 'memory_stat': ('memory.stat', key_values),+ 'memory_events': ('memory.events', key_values),+ 'cpu_stat': ('cpu.stat', key_values),+ 'io_by_device': ('io.stat', io_counters),+ 'tasks': ('pids.current', int),+ }+ for field, (filename, parse) in files.items():+ try:+ result[field] = parse((path / filename).read_text())+ except (OSError, ValueError):+ result[field] = None+ result['unavailable'].append(filename)+ try:+ if path.stat().st_ino != target['inode']:+ result['error'] = 'cgroup_replaced'+ except OSError:+ result['error'] = 'cgroup_unavailable'+ if result['memory_bytes'] is None or 'usage_usec' not in (result['cpu_stat'] or {}):+ result['error'] = result['error'] or 'required_counters_unavailable'+ # perf_counter: monotonic() has ~15.6 ms ticks on Windows, collapsing short intervals.+ result['monotonic_seconds'] = time.perf_counter()+ return result+++def cpu_percent(previous, current):+ if not previous or previous['error'] or current['error']:+ return None+ elapsed = current['monotonic_seconds'] - previous['monotonic_seconds']+ delta = current['cpu_stat']['usage_usec'] - previous['cpu_stat']['usage_usec']+ if elapsed <= 0 or delta < 0:+ current['error'] = 'cpu_counter_reset_or_invalid_interval'+ return None+ return delta / (elapsed * 1_000_000) * 100+++def host_memory():+ values = {}+ for line in Path('/proc/meminfo').read_text().splitlines():+ key, value = line.split(':', 1)+ if key in ('MemTotal', 'MemAvailable', 'SwapTotal', 'SwapFree'):+ values[key] = int(value.split()[0]) * 1024+ return values+++def distribution(values):+ values = sorted(value for value in values if value is not None)+ def percentile(fraction):+ position = (len(values) - 1) * fraction+ lower, upper = math.floor(position), math.ceil(position)+ return values[lower] + (values[upper] - values[lower]) * (position - lower)+ return {'count': len(values), 'min': min(values) if values else None,+ 'p50': percentile(.5) if values else None,+ 'p95': percentile(.95) if values else None,+ 'sampled_max': max(values) if values else None}+++def summarize(samples, targets):+ summaries = []+ for index, target in enumerate(targets):+ rows = [sample['targets'][index] for sample in samples]+ valid = [row for row in rows if not row['error']]+ summaries.append({+ 'name': target['name'], 'kind': target['kind'],+ 'failed_samples': sum(bool(row['error']) for row in rows),+ 'unavailable_files': sorted({name for row in rows for name in row['unavailable']}),+ **{field: distribution([row[field] for row in valid])+ for field in ('memory_bytes', 'swap_bytes', 'tasks', 'cpu_percent_one_core')},+ })+ return summaries+++def collect(targets, duration, interval):+ samples, previous, interrupted = [], [None] * len(targets), False+ started = time.perf_counter()+ deadline = started + duration+ try:+ while True:+ sample = {'elapsed_seconds': time.perf_counter() - started,+ 'host_memory_bytes': host_memory(), 'targets': []}+ for index, target in enumerate(targets):+ row = read_target(target)+ row['cpu_percent_one_core'] = cpu_percent(previous[index], row)+ sample['targets'].append(row)+ previous[index] = row+ samples.append(sample)+ now = time.perf_counter()+ if now >= deadline:+ break+ time.sleep(min(interval, deadline - now))+ except KeyboardInterrupt:+ interrupted = True+ return samples, interrupted+++def main(argv=None):+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument('--unit', action='append', default=[], help='systemd unit; repeatable')+ parser.add_argument('--container', action='append', default=[], help='Docker container; repeatable')+ parser.add_argument('--duration', type=float, default=600, help='seconds, default 600')+ parser.add_argument('--interval', type=float, default=5, help='seconds, default 5')+ parser.add_argument('--scenario', required=True, help='e.g. idle-closed, dashboard-open, deploy')+ parser.add_argument('--revision', required=True, help='revision actually running, not sampler revision')+ parser.add_argument('--notes', default='', help='workload, enabled extensions, DB and warmup; no secrets')+ parser.add_argument('--output', type=Path, required=True)+ args = parser.parse_args(argv)+ if not args.unit and not args.container:+ parser.error('select at least one --unit or --container')+ if (not all(math.isfinite(value) and value > 0 for value in (args.duration, args.interval))+ or args.interval > args.duration):+ parser.error('require finite duration >= interval > 0')+ if platform.system() != 'Linux' or not (CGROUP_ROOT / 'cgroup.controllers').is_file():+ parser.error('requires a Linux host with cgroup v2; cgroup v1 is not supported')+ try:+ targets = [resolve_target(kind, name) for kind, names in+ (('unit', args.unit), ('container', args.container)) for name in names]+ validate_scopes(targets)+ for target in targets:+ if read_target(target)['error']:+ raise ValueError(f"required counters unavailable for {target['name']}")+ except (OSError, ValueError, subprocess.SubprocessError) as exc:+ parser.error(f'target discovery failed ({type(exc).__name__}); check names, permissions and cgroup v2')+ # Exclusive create prevents accidentally overwriting a before/after baseline.+ with args.output.open('x', encoding='utf-8') as output:+ report = {+ 'schema_version': 1, 'started_at': datetime.now(timezone.utc).isoformat(),+ 'scenario': args.scenario, 'running_revision': args.revision, 'notes': args.notes,+ 'host': {'os': platform.freedesktop_os_release().get('PRETTY_NAME'),+ 'kernel': platform.release(), 'architecture': platform.machine(),+ 'logical_cpus': os.cpu_count(), 'python': platform.python_version(),+ 'uptime_seconds': float(Path('/proc/uptime').read_text().split()[0])},+ 'requested_duration_seconds': args.duration, 'interval_seconds': args.interval,+ 'targets': targets,+ 'accounting': {+ 'memory': 'cgroup memory.current; includes cache, kernel and descendants; not process RSS',+ 'cpu': 'delta usage_usec / actual elapsed time; 100% = one logical CPU',+ 'io': 'cumulative per-device counters; do not sum stacked devices blindly',+ 'scope': 'explicit targets only; excludes other services and separately created app containers',+ 'peaks': 'sampled maxima only; short spikes between samples can be missed',+ },+ }+ print('Sampling explicit targets; Ctrl+C saves a partial report.', file=sys.stderr, flush=True)+ samples, interrupted = collect(targets, args.duration, args.interval)+ report.update(samples=samples, interrupted=interrupted,+ finished_at=datetime.now(timezone.utc).isoformat(),+ summary=summarize(samples, targets))+ json.dump(report, output, indent=2, allow_nan=False)+ output.write('\n')+ print(json.dumps(report['summary'], indent=2))+ return 130 if interrupted else int(any(row['failed_samples'] for row in report['summary']))+++if __name__ == '__main__':+ raise SystemExit(main())diff --git a/scripts/test/README.md b/scripts/test/README.mdindex 4c24aaaa..a5c663a2 100644--- a/scripts/test/README.md+++ b/scripts/test/README.md@@ -117,12 +117,25 @@ scripts/test/output/<run-id>/install-status ("OK" or "FAIL")pytest.logpytest-report.json+ runtime.json (post-install resource observation)sk-test-ubuntu24-<id>/ ...sk-test-debian12-<id>/ ...```## Update & uninstall coverage+Full VM installs now wait 60 seconds after the health check, then collect a+120-second passive runtime baseline before pytest. The HTML report shows median+and sampled-peak cgroup memory, and median/p95 CPU per measured service. Raw data+is in `runtime.json`; unavailable/failed measurements are shown explicitly and+do not turn a successful install into a failed install. Test Sandbox full mode+uses the same sampler and saves `<distro>.runtime.json` beside its log.++These short observations help identify regressions, but parallel VM/container+work can affect them. Quick/provision checks do not run the panel and cannot+measure its runtime overhead. See [measurement definitions and longer baseline+commands](../../docs/METRICS.md#passive-linux-runtime-sampler).+Alongside `vm-install.sh`, two more "run inside the VM" scripts exercise theother lifecycle stages end-to-end (same contract: log to`/var/log/serverkit-test-*.log`, write `OK`/`FAIL` to `/tmp/serverkit-*-status`):diff --git a/scripts/test/full-stack-test.ps1 b/scripts/test/full-stack-test.ps1index 33ea382b..e57eed12 100644--- a/scripts/test/full-stack-test.ps1+++ b/scripts/test/full-stack-test.ps1@@ -197,6 +197,7 @@ function Test-VmInstall {& $MpExe transfer "${Vm}:/var/log/serverkit-test-install.log" "$VmOut\vm-install.log" 2>&1 | Out-Null& $MpExe transfer "${Vm}:/tmp/serverkit-install-status" $statusFile 2>&1 | Out-Null+ & $MpExe transfer "${Vm}:/tmp/serverkit-runtime.json" (Join-Path $VmOut 'runtime.json') 2>&1 | Out-Null$rawStatus = if (Test-Path $statusFile) { Get-Content $statusFile -Raw -ErrorAction SilentlyContinue } else { $null }if (-not $rawStatus -or "$rawStatus".Trim() -eq '') {$fallback = if ($installRC -eq 0) { 'OK' } else { 'FAIL' }@@ -285,6 +286,8 @@ function Test-VagrantInstall {| Out-File (Join-Path $VmOut 'vm-install.log') -Encoding utf8& $VgExe ssh -c "sudo cat /tmp/serverkit-install-status" 2>$null `| Out-File $statusFile -Encoding ascii+ & $VgExe ssh -c "sudo cat /tmp/serverkit-runtime.json" 2>$null `+ | Out-File (Join-Path $VmOut 'runtime.json') -Encoding utf8# Get-Content -Raw returns $null for empty/missing files; guard before# calling .Trim() so a failed status capture doesn't crash the run.$rawStatus = if (Test-Path $statusFile) { Get-Content $statusFile -Raw -ErrorAction SilentlyContinue } else { $null }diff --git a/scripts/test/measure-installed-runtime.py b/scripts/test/measure-installed-runtime.pynew file mode 100644index 00000000..9a83b2e5--- /dev/null+++ b/scripts/test/measure-installed-runtime.py@@ -0,0 +1,101 @@+#!/usr/bin/env python3+"""Record a short post-install baseline, with explicit unavailable/error results.++Used by full distro installs (VMs and systemd containers), never quick/provision+checks. This is a regression observation, not a production capacity benchmark.+"""++import argparse+from datetime import datetime, timezone+import hashlib+import importlib.util+import json+import math+import os+from pathlib import Path+import platform+import subprocess+import sys+import time+++SPEC = importlib.util.spec_from_file_location('measure_runtime', Path(__file__).parents[1] / 'measure-runtime.py')+runtime = importlib.util.module_from_spec(SPEC)+SPEC.loader.exec_module(runtime)+++def source_identity(source, archive=None):+ if archive:+ digest = hashlib.sha256()+ with archive.open('rb') as stream:+ for block in iter(lambda: stream.read(1024 * 1024), b''):+ digest.update(block)+ return 'local-overlay-archive-sha256:' + digest.hexdigest()+ try:+ revision = runtime.command(['git', '-C', str(source), 'rev-parse', 'HEAD'])+ dirty = runtime.command(['git', '-C', str(source), 'status', '--porcelain', '--untracked-files=no'])+ return revision + (' (tracked files modified)' if dirty else '')+ except (OSError, subprocess.SubprocessError):+ version = source / 'VERSION'+ return 'release:' + version.read_text().strip() if version.is_file() else 'unknown'+++def observe(source, duration=120, settle=60, archive=None):+ result = {'schema_version': 1, 'status': 'unavailable',+ 'scenario': 'post-install-dashboard-closed',+ 'started_at': datetime.now(timezone.utc).isoformat(),+ 'settle_seconds': settle, 'requested_duration_seconds': duration,+ 'scope_note': 'Selected service cgroups only; other services/apps are excluded. '+ 'Container tests share the host kernel and may run beside other tests.'}+ if platform.system() != 'Linux' or not (runtime.CGROUP_ROOT / 'cgroup.controllers').is_file():+ result['reason'] = 'Linux cgroup v2 unavailable; no runtime measurement taken'+ return result+ try:+ revision = source_identity(source, archive)+ targets = [runtime.resolve_target('unit', 'serverkit.service')]+ for unit in ('nginx.service', 'docker.service', 'containerd.service'):+ try:+ if runtime.command(['systemctl', 'is-active', unit]) == 'active':+ targets.append(runtime.resolve_target('unit', unit))+ except (OSError, ValueError, subprocess.SubprocessError):+ continue+ runtime.validate_scopes(targets)+ # This pause is after builds/health probes, before measurement or pytest.+ time.sleep(settle)+ samples, interrupted = runtime.collect(targets, duration, 5)+ summary = runtime.summarize(samples, targets)+ failed = any(row['failed_samples'] for row in summary)+ result.update(status='interrupted' if interrupted else 'failed' if failed else 'measured',+ running_revision=revision, targets=targets, summary=summary, samples=samples,+ os=platform.freedesktop_os_release().get('PRETTY_NAME'),+ kernel=platform.release(), architecture=platform.machine(),+ logical_cpus=os.cpu_count(),+ definitions={'memory_bytes': 'memory.current including cache/kernel/descendants',+ 'cpu_percent_one_core': '100% = one logical CPU',+ 'sampled_max': 'largest sampled value; short spikes can be missed'})+ except (OSError, ValueError, subprocess.SubprocessError) as exc:+ result.update(status='failed', reason=f'Runtime collection failed: {type(exc).__name__}')+ result['finished_at'] = datetime.now(timezone.utc).isoformat()+ return result+++def main(argv=None):+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument('--source', type=Path, default=Path('/opt/serverkit'))+ parser.add_argument('--source-archive', type=Path, help='VM harness local overlay archive')+ parser.add_argument('--output', type=Path, required=True)+ parser.add_argument('--duration', type=float, default=120)+ parser.add_argument('--settle', type=float, default=60)+ args = parser.parse_args(argv)+ if (not math.isfinite(args.duration) or args.duration < 5+ or not math.isfinite(args.settle) or args.settle < 0):+ parser.error('duration must be finite and >= 5; settle finite and >= 0')+ # A VM reuse explicitly produces a new observation at this harness scratch path.+ result = observe(args.source, args.duration, args.settle, args.source_archive)+ args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + '\n', encoding='utf-8')+ print(json.dumps({key: value for key, value in result.items() if key != 'samples'}, indent=2))+ return 0 if result['status'] == 'measured' else 1+++if __name__ == '__main__':+ raise SystemExit(main())diff --git a/scripts/test/report.py b/scripts/test/report.pyindex e5e83c6f..74ee1afb 100644--- a/scripts/test/report.py+++ b/scripts/test/report.py@@ -23,7 +23,7 @@ def esc(s):def load_json(path):try:- return json.loads(path.read_text(encoding="utf-8"))+ return json.loads(path.read_text(encoding="utf-8-sig"))except Exception:return None@@ -71,9 +71,35 @@ def vm_summary(vm_dir: Path):"skip": skip_count,"install_log": read_text(vm_dir / "install.log"),"journal": read_text(vm_dir / "journalctl.log", limit=50_000),+ "runtime": load_json(vm_dir / "runtime.json"),}+def runtime_table(measurement):+ if not measurement:+ return '<p>Runtime baseline: unavailable (no artifact).</p>'+ status = measurement.get('status', 'unknown')+ if status != 'measured':+ return f"<p>Runtime baseline: {esc(status)}. {esc(measurement.get('reason', ''))}</p>"++ def number(value, divisor=1):+ return 'unavailable' if value is None else f'{value / divisor:,.2f}'++ rows = []+ for item in measurement.get('summary', []):+ memory = item.get('memory_bytes', {})+ cpu = item.get('cpu_percent_one_core', {})+ rows.append(f"<tr><td>{esc(item['name'])}</td>"+ f"<td>{number(memory.get('p50'), 1024 ** 2)}</td>"+ f"<td>{number(memory.get('sampled_max'), 1024 ** 2)}</td>"+ f"<td>{number(cpu.get('p50'))}</td><td>{number(cpu.get('p95'))}</td></tr>")+ return ('<p>Post-install observation: cgroup memory includes cache; CPU 100% = one logical CPU. '+ 'Sampled peaks can miss short spikes. This is not a capacity benchmark.</p>'+ '<table><tr><th>Service</th><th>Median MiB</th><th>Sampled peak MiB</th>'+ '<th>Median CPU %</th><th>p95 CPU %</th></tr>' + ''.join(rows) + '</table>'+ f"<p>Source: <code>{esc(measurement.get('running_revision'))}</code></p>")++HTML_TEMPLATE = """<!doctype html><html><head><meta charset="utf-8"><title>ServerKit E2E Report — {ts}</title>{refresh_meta}@@ -119,6 +145,7 @@ VM_TEMPLATE = """<div class="vm {cls}"><span>Skipped: <b>{s}</b></span></div>{tests_table}+ {runtime_table}<details><summary>Install log</summary><pre>{install_log}</pre></details><details><summary>journalctl -u serverkit</summary><pre>{journal}</pre></details></div>"""@@ -203,6 +230,7 @@ def render(run_dir: Path) -> Path:install="OK" if s["install_ok"] else ("-" if s["overall"] == "RUNNING" else "FAIL"),p=s["pass"], f=s["fail"], s=s["skip"],tests_table=tests_table,+ runtime_table=runtime_table(s.get('runtime')),install_log=esc(s["install_log"]),journal=esc(s["journal"]),))diff --git a/scripts/test/test_measure_runtime.py b/scripts/test/test_measure_runtime.pynew file mode 100644index 00000000..3263b215--- /dev/null+++ b/scripts/test/test_measure_runtime.py@@ -0,0 +1,156 @@+"""Stdlib tests for accounting boundaries, missing data and real file sampling."""++import importlib.util+import json+from pathlib import Path+import tempfile+import unittest+from unittest.mock import patch+++SPEC = importlib.util.spec_from_file_location('measure_runtime', Path(__file__).parents[1] / 'measure-runtime.py')+runtime = importlib.util.module_from_spec(SPEC)+SPEC.loader.exec_module(runtime)++INSTALL_SPEC = importlib.util.spec_from_file_location('measure_installed', Path(__file__).with_name('measure-installed-runtime.py'))+installed = importlib.util.module_from_spec(INSTALL_SPEC)+INSTALL_SPEC.loader.exec_module(installed)++REPORT_SPEC = importlib.util.spec_from_file_location('vm_report', Path(__file__).with_name('report.py'))+vm_report = importlib.util.module_from_spec(REPORT_SPEC)+REPORT_SPEC.loader.exec_module(vm_report)+++class RuntimeTests(unittest.TestCase):+ def setUp(self):+ self.temp = tempfile.TemporaryDirectory()+ self.addCleanup(self.temp.cleanup)+ self.path = Path(self.temp.name) / 'service'+ self.path.mkdir()+ (self.path / 'memory.current').write_text('104857600\n')+ (self.path / 'cpu.stat').write_text('usage_usec 500000\nuser_usec 400000\nsystem_usec 100000\n')+ self.target = {'name': 'serverkit.service', 'kind': 'unit',+ 'cgroup': str(self.path), 'inode': self.path.stat().st_ino}++ def row(self, usage, clock):+ return {'cpu_stat': {'usage_usec': usage}, 'monotonic_seconds': clock, 'error': None}++ def test_cpu_normalizes_by_actual_time_and_one_core(self):+ self.assertEqual(runtime.cpu_percent(self.row(0, 10), self.row(2_000_000, 12)), 100)+ self.assertEqual(runtime.cpu_percent(self.row(0, 10), self.row(4_000_000, 12)), 200)+ self.assertEqual(runtime.cpu_percent(self.row(0, 10), self.row(500_000, 20)), 5)+ self.assertIsNone(runtime.cpu_percent(None, self.row(2_000_000, 12)))++ def test_counter_reset_is_invalid_not_zero_cpu(self):+ current = self.row(0, 12)+ self.assertIsNone(runtime.cpu_percent(self.row(100, 10), current))+ self.assertIsNotNone(current['error'])++ def test_missing_optional_data_stays_null(self):+ row = runtime.read_target(self.target)+ self.assertEqual(row['memory_bytes'], 104857600)+ self.assertIsNone(row['swap_bytes'])+ self.assertIsNone(row['tasks'])+ self.assertIsNone(row['io_by_device'])+ self.assertIn('io.stat', row['unavailable'])+ self.assertIsNone(row['error'])++ def test_missing_required_data_and_replaced_group_fail(self):+ (self.path / 'memory.current').unlink()+ self.assertEqual(runtime.read_target(self.target)['error'], 'required_counters_unavailable')+ self.target['inode'] += 1+ self.assertEqual(runtime.read_target(self.target)['error'], 'cgroup_replaced')++ def test_device_io_and_memory_breakdown_preserved(self):+ (self.path / 'io.stat').write_text('8:0 rbytes=12 wbytes=20 rios=1 wios=2\n253:0 rbytes=12 wbytes=20\n')+ (self.path / 'memory.stat').write_text('anon 100\nfile 50\nshmem 20\n')+ row = runtime.read_target(self.target)+ self.assertEqual(row['io_by_device']['8:0']['wbytes'], 20)+ self.assertEqual(len(row['io_by_device']), 2)+ self.assertEqual(row['memory_stat']['file'], 50)++ def test_reject_root_and_traversal(self):+ for group in ('/', '', '/../../etc', 'relative'):+ with self.subTest(group=group), self.assertRaises(ValueError):+ runtime.group_path(group, Path(self.temp.name))++ def test_reject_overlapping_scopes(self):+ for second in (self.path, self.path / 'child'):+ with self.subTest(second=second), self.assertRaises(ValueError):+ runtime.validate_scopes([self.target, {'cgroup': str(second)}])++ def test_real_sampling_and_summary_exclude_failures(self):+ with patch.object(runtime, 'host_memory', return_value={'MemAvailable': 1024}):+ samples, interrupted = runtime.collect([self.target], .025, .01)+ self.assertFalse(interrupted)+ self.assertGreaterEqual(len(samples), 2)+ summary = runtime.summarize(samples, [self.target])[0]+ self.assertEqual(summary['memory_bytes']['p50'], 104857600)+ self.assertEqual(summary['cpu_percent_one_core']['count'], len(samples) - 1)+ self.assertEqual(summary['swap_bytes']['count'], 0)+ self.assertIsNone(summary['swap_bytes']['sampled_max'])+ samples[-1]['targets'][0]['error'] = 'cgroup_replaced'+ samples[-1]['targets'][0]['memory_bytes'] = 999999999999+ summary = runtime.summarize(samples, [self.target])[0]+ self.assertEqual(summary['failed_samples'], 1)+ self.assertEqual(summary['memory_bytes']['sampled_max'], 104857600)++ def test_nonfinite_durations_rejected_before_discovery(self):+ for duration in ('nan', 'inf', '-1'):+ with self.subTest(duration=duration), patch('sys.stderr'), self.assertRaises(SystemExit) as exc:+ runtime.main(['--unit', 'serverkit', '--scenario', 'idle', '--revision', 'test',+ '--duration', duration, '--output', str(self.path / 'out.json')])+ self.assertEqual(exc.exception.code, 2)++ def test_vm_report_preserves_unavailable_and_escapes_names(self):+ self.assertIn('unavailable', vm_report.runtime_table(None))+ self.assertIn('cgroup v2', vm_report.runtime_table({'status': 'unavailable', 'reason': 'cgroup v2'}))+ rendered = vm_report.runtime_table({'status': 'measured', 'summary': [{+ 'name': '<script>', 'memory_bytes': {'p50': 1048576, 'sampled_max': 2097152},+ 'cpu_percent_one_core': {'p50': 0, 'p95': None},+ }]})+ self.assertIn('<script>', rendered)+ self.assertNotIn('<script>', rendered)+ self.assertIn('1.00', rendered)+ self.assertIn('unavailable', rendered)++ def test_install_wrapper_marks_unsupported_without_sampling(self):+ with patch.object(installed.platform, 'system', return_value='Windows'), \+ patch.object(installed.runtime, 'collect') as collect:+ result = installed.observe(self.path, duration=5, settle=0)+ self.assertEqual(result['status'], 'unavailable')+ collect.assert_not_called()++ def test_overlay_revision_tracks_archive_content_not_installer_git(self):+ archive = self.path / 'overlay.tar.gz'+ archive.write_bytes(b'first source')+ first = installed.source_identity(self.path, archive)+ archive.write_bytes(b'changed source')+ self.assertNotEqual(first, installed.source_identity(self.path, archive))++ def test_install_wrapper_keeps_raw_samples_and_failed_status(self):+ (self.path / 'cgroup.controllers').write_text('cpu memory')+ row = runtime.read_target(self.target)+ row.update(error='cgroup_replaced', cpu_percent_one_core=None)+ with patch.object(installed.platform, 'system', return_value='Linux'), \+ patch.object(installed.runtime, 'CGROUP_ROOT', self.path), \+ patch.object(installed, 'source_identity', return_value='revision'), \+ patch.object(installed.runtime, 'resolve_target', return_value=self.target), \+ patch.object(installed.runtime, 'command', return_value='inactive'), \+ patch.object(installed.runtime, 'validate_scopes'), \+ patch.object(installed.platform, 'freedesktop_os_release', return_value={'PRETTY_NAME': 'Test Linux'}), \+ patch.object(installed.runtime, 'collect', return_value=([{'targets': [row]}], False)):+ result = installed.observe(self.path, duration=5, settle=0)+ self.assertEqual(result['status'], 'failed')+ self.assertEqual(result['summary'][0]['failed_samples'], 1)+ self.assertEqual(result['samples'][0]['targets'][0]['error'], 'cgroup_replaced')+ json.dumps(result, allow_nan=False)++ def test_bom_from_windows_vm_capture_is_readable(self):+ artifact = self.path / 'runtime.json'+ artifact.write_text('{"status": "measured"}', encoding='utf-8-sig')+ self.assertEqual(vm_report.load_json(artifact)['status'], 'measured')+++if __name__ == '__main__':+ unittest.main()diff --git a/scripts/test/vm-install.sh b/scripts/test/vm-install.shindex 15a027c2..9f526ff6 100644--- a/scripts/test/vm-install.sh+++ b/scripts/test/vm-install.sh@@ -148,6 +148,11 @@ log "Step 4/4: waiting for health endpoint"for i in $(seq 1 30); doif curl -fsS http://127.0.0.1:5000/api/v1/system/health > /dev/null 2>&1; thenlog "Backend healthy after ${i}s"+ log "Measuring post-install runtime (60s settle + 120s observation)"+ "$INSTALL_DIR/venv/bin/python" "$SRC/scripts/test/measure-installed-runtime.py" \+ --source "$INSTALL_DIR" --source-archive /tmp/serverkit-src.tar.gz \+ --output /tmp/serverkit-runtime.json >> "$LOG" 2>&1 \+ || log "WARN: runtime baseline unavailable/failed; inspect runtime report"echo "OK" > /tmp/serverkit-install-statusexit 0fi