Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Measure OS runtime memory before tuning Node defaults (#696)

Deeedeefaee20b2
diff --git a/.github/workflows/os-memory.yml b/.github/workflows/os-memory.yml
new file mode 100644
index 000000000..5bc3a9fdf
--- /dev/null
+++ b/.github/workflows/os-memory.yml
@@ -0,0 +1,50 @@
+name: Harness OS runtime memory assessment
+
+on:
+ workflow_dispatch:
+ inputs:
+ image_run_id:
+ description: 'Exact existing image artifact to assess'
+ type: string
+ default: '37149350400'
+ required: true
+ push:
+ branches: ['os-polish/runtime-memory']
+ paths: ['os/tests/memory_*.py', '.github/workflows/os-memory.yml']
+
+permissions:
+ contents: read
+ actions: read
+
+jobs:
+ memory:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v7
+ - uses: actions/download-artifact@v6
+ with:
+ name: programmer-os-x86_64
+ path: os/dist
+ run-id: ${{ inputs.image_run_id || '37149350400' }}
+ github-token: ${{ github.token }}
+ - name: Check the fixture and install VM tools
+ run: |
+ python3 -m py_compile os/tests/memory_guest.py os/tests/memory_vm.py
+ go run github.com/rhysd/actionlint/cmd/[email protected] .github/workflows/os-memory.yml
+ sudo apt-get update -qq
+ sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf python3-pil tesseract-ocr
+ sudo chmod a+rw /dev/kvm
+ - name: Measure alternating profiles on the installed 1 GiB system
+ run: python3 os/tests/memory_vm.py --iso os/dist/*.iso
+ - name: Retain measured evidence without disks
+ if: always()
+ uses: actions/upload-artifact@v6
+ with:
+ name: harness-os-memory
+ path: |
+ os/test-results/memory/
+ !os/test-results/memory/*.qcow2
+ !os/test-results/memory/OVMF_VARS.fd
+ retention-days: 14
+ compression-level: 0
diff --git a/os/DEVELOPMENT.md b/os/DEVELOPMENT.md
index f85ba4c89..ed3c79cd2 100644
--- a/os/DEVELOPMENT.md
+++ b/os/DEVELOPMENT.md
@@ -90,6 +90,21 @@ python3 os/tests/local_ai_vm.py --iso os/dist/IMAGE.iso --probe local-ai
python3 os/tests/local_ai_vm.py --iso os/dist/IMAGE.iso --probe nvidia
```
+The **Harness OS runtime memory assessment** workflow compares Node flags in one
+installed 1 GiB VM. It alternates two default and two candidate rounds, samples
+RSS/PSS and local status latency with four persistent terminal streams, records
+CPU time, then restores the packaged command and checks graphical keyboard input.
+This measures daemon behavior, not model performance or full agent throughput.
+
+In [the October 3 comparison](https://github.com/autonomous-ai/openharness/actions/runs/37154644789),
+the installed Node was 22.23.3. Median process RSS was 117.4/129.2 MiB in the two
+default rounds and 126.1/145.2 MiB with `--optimize-for-size
+--max-semi-space-size=1`. Each round made 200 status requests; p95 response times
+were 1.19–1.25 ms. Memory drifted across this short interleaved run, so it does not
+establish a precise causal difference. It provides no evidence for a saving from
+these flags; the shipped defaults remain unchanged. Retain raw per-round data
+and use representative long-running agent work before adopting a memory limit.
+
## The feedback loop
| Work being tested | Best environment | What it proves |
diff --git a/os/tests/memory_guest.py b/os/tests/memory_guest.py
new file mode 100644
index 000000000..9163b2ade
--- /dev/null
+++ b/os/tests/memory_guest.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Measure candidate Node flags only inside an installed disposable OS guest."""
+import json
+import os
+from pathlib import Path
+import statistics
+import subprocess
+import time
+from urllib.request import urlopen
+
+
+ROOT = Path.home() / 'memory-assessment'
+OVERRIDE = Path.home() / '.config/systemd/user/harness-daemon.service.d/memory-assessment.conf'
+PROFILES = [('default', []), ('small', ['--optimize-for-size', '--max-semi-space-size=1'])]
+
+
+def run(*args, timeout=180):
+ result = subprocess.run(args, capture_output=True, text=True, timeout=timeout)
+ with (ROOT / 'commands.log').open('a') as log:
+ log.write(json.dumps(args) + '\n' + result.stdout + result.stderr + '\n')
+ result.check_returncode()
+ return result.stdout.strip()
+
+
+def api():
+ start = time.perf_counter()
+ with urlopen('http://127.0.0.1:18473/api/status', timeout=5) as response:
+ value = json.load(response)
+ assert value.get('discoveryReady'), value
+ return (time.perf_counter() - start) * 1000
+
+
+def process(pid):
+ memory = {}
+ for line in Path(f'/proc/{pid}/smaps_rollup').read_text().splitlines():
+ parts = line.split()
+ if parts[0] in ('Rss:', 'Pss:', 'Private_Clean:', 'Private_Dirty:'):
+ memory[parts[0][:-1] + '_kib'] = int(parts[1])
+ fields = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()
+ memory['cpu_ticks'] = int(fields[11]) + int(fields[12])
+ return memory
+
+
+def survivor_pids():
+ return {p.name: int(p.read_text()) for p in ROOT.glob('survivor-*.pid')}
+
+
+def main():
+ assert os.getuid() != 0 and not Path('/etc/harness-live').exists()
+ assert Path('/var/lib/harness-os/install.json').is_file()
+ ROOT.mkdir(exist_ok=True)
+ assert not OVERRIDE.exists(), 'Never replace an existing user override'
+ result = {'status': 'running', 'started_at': time.time(), 'rounds': [],
+ 'node': run('node', '--version'), 'clock_ticks_per_second': os.sysconf('SC_CLK_TCK'),
+ 'scope': 'Daemon memory, local status latency and persistent terminal streams; not full agent throughput'}
+ # Keep terminal processes and output moving while the daemon is replaced.
+ (ROOT / 'survivor.py').write_text('''import os,sys,time
+from pathlib import Path
+root=Path.home()/'memory-assessment'
+n=sys.argv[1]
+(root/('survivor-'+n+'.pid')).write_text(str(os.getpid()))
+for i in range(10000):
+ (root/('survivor-'+n+'.count')).write_text(str(i))
+ print('terminal '+n+' output '+str(i)+' '+('x'*120),flush=True)
+ time.sleep(.25)
+''')
+ run('systemctl', '--user', 'stop', 'harness-update.timer')
+ for n in range(4):
+ run('hn', 'new-window', '-d', '-n', f'memory-{n}', f'python3 -u {ROOT}/survivor.py {n}')
+ deadline = time.monotonic() + 15
+ while len(survivor_pids()) != 4:
+ assert time.monotonic() < deadline, 'Terminal streams did not start'
+ time.sleep(.25)
+ original = survivor_pids()
+ try:
+ # Interleave the baseline to expose cache/load drift in one VM.
+ for repetition in range(2):
+ for name, flags in PROFILES:
+ OVERRIDE.parent.mkdir(parents=True, exist_ok=True)
+ OVERRIDE.write_text('[Service]\nExecStart=\nExecStart=/usr/bin/node ' +
+ ' '.join(flags) + ' /usr/lib/harness/cli.mjs start --foreground\n')
+ run('systemctl', '--user', 'daemon-reload')
+ start = time.monotonic()
+ run('systemctl', '--user', 'restart', 'harness-daemon')
+ ready_seconds = time.monotonic() - start
+ pid = int(run('systemctl', '--user', 'show', 'harness-daemon', '-p', 'MainPID', '--value'))
+ command = Path(f'/proc/{pid}/cmdline').read_bytes().replace(b'\0', b' ').decode()
+ assert '/usr/bin/node' in command and all(flag in command for flag in flags)
+ for _ in range(10):
+ api()
+ time.sleep(1)
+ before = process(pid)
+ start = time.monotonic()
+ samples, latencies = [], []
+ for _ in range(10):
+ latencies.extend(api() for _ in range(20))
+ samples.append(process(pid))
+ time.sleep(3)
+ elapsed = time.monotonic() - start
+ assert survivor_pids() == original, 'A terminal owner was replaced'
+ for child in original.values():
+ os.kill(child, 0)
+ ticks = process(pid)['cpu_ticks'] - before['cpu_ticks']
+ ordered = sorted(latencies)
+ row = {'profile': name, 'repetition': repetition, 'flags': flags,
+ 'pid': pid, 'command': command, 'ready_seconds': ready_seconds,
+ 'samples': samples, 'rss_median_kib': statistics.median(s['Rss_kib'] for s in samples),
+ 'pss_median_kib': statistics.median(s['Pss_kib'] for s in samples),
+ 'api_median_ms': statistics.median(latencies), 'api_p95_ms': ordered[int(len(ordered)*.95)],
+ 'api_max_ms': max(latencies), 'cpu_seconds': ticks / os.sysconf('SC_CLK_TCK'),
+ 'measured_seconds': elapsed, 'survivor_pids': original}
+ result['rounds'].append(row)
+ (ROOT / 'receipt.json').write_text(json.dumps(result, indent=2)+'\n')
+ print('HN_MEMORY_ROUND=' + json.dumps({k:v for k,v in row.items() if k!='samples'}), flush=True)
+ result['status'] = 'passed'
+ except BaseException as error:
+ result.update(status='failed', error=repr(error))
+ raise
+ finally:
+ OVERRIDE.unlink(missing_ok=True)
+ run('systemctl', '--user', 'daemon-reload')
+ run('systemctl', '--user', 'restart', 'harness-daemon')
+ result['default_restored'] = '--optimize-for-size' not in run('systemctl', '--user', 'show', 'harness-daemon', '-p', 'ExecStart', '--value')
+ result['finished_at'] = time.time()
+ (ROOT / 'receipt.json').write_text(json.dumps(result, indent=2)+'\n')
+ print('HN_MEMORY_RESULT=' + json.dumps(result), flush=True)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/os/tests/memory_vm.py b/os/tests/memory_vm.py
new file mode 100644
index 000000000..b8e03b819
--- /dev/null
+++ b/os/tests/memory_vm.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+"""Native 1 GiB installed-OS A/B measurement; never changes the published image."""
+import argparse
+import base64
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import time
+from vm import VM, check_graphical_keyboard
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--iso', type=Path, required=True)
+ args = parser.parse_args()
+ if not os.access('/dev/kvm', os.R_OK | os.W_OK):
+ parser.error('Native x86 KVM is required')
+ iso = args.iso.resolve()
+ manifest = json.loads(iso.with_name('manifest.json').read_text())
+ with iso.open('rb') as handle:
+ assert hashlib.file_digest(handle, 'sha256').hexdigest() == manifest['iso']['sha256']
+ folder = Path('os/test-results/memory').resolve()
+ folder.mkdir(parents=True, exist_ok=False)
+ vm = VM(folder, iso, 'uefi', 1024)
+ config = dict(disk='/dev/vda', expected_serial='HN_OS_TEST', confirm_erase='/dev/vda',
+ username='me', hostname='harness', password='test-password-123', encrypt=True, serial_console=True)
+ result = {'status': 'running', 'started_at': time.time(), 'memory_mib': 1024,
+ 'image_source_commit': manifest['source_commit'], 'iso_sha256': manifest['iso']['sha256'],
+ 'test_source_commit': subprocess.check_output(['git','rev-parse','HEAD'], text=True).strip()}
+ try:
+ vm.start(live=True)
+ vm.wait(r'root@[^\r\n]*[#] ', timeout=180)
+ vm.shell_ready = True
+ vm.command('stty -echo')
+ encoded = base64.b64encode(json.dumps(config).encode()).decode()
+ vm.command('printf %s ' + encoded + ' | base64 -d > /tmp/install-config.json')
+ vm.command('nmcli networking off')
+ output, _ = vm.command('harness install --config /tmp/install-config.json --yes-erase-disk', timeout=360)
+ (folder / 'install.log').write_text(output)
+ vm.command('sync')
+ vm.stop()
+ vm.start(live=False)
+ vm.login_installed(config)
+ script = base64.b64encode(Path(__file__).with_name('memory_guest.py').read_bytes()).decode()
+ vm.command(': > /tmp/memory-guest.b64')
+ for start in range(0, len(script), 2000):
+ vm.command('printf %s ' + script[start:start+2000] + ' >> /tmp/memory-guest.b64')
+ vm.command('base64 -d /tmp/memory-guest.b64 > /home/me/memory-guest.py')
+ output, status = vm.command('python3 /home/me/memory-guest.py', timeout=720, check=False)
+ (folder / 'benchmark.log').write_text(output)
+ assert status == 0, 'Memory assessment failed; see benchmark.log'
+ marker = 'HN_MEMORY_RESULT='
+ result['guest'] = json.loads(next(line.split(marker,1)[1] for line in output.splitlines() if marker in line))
+ assert result['guest']['status'] == 'passed' and result['guest']['default_restored']
+ result['restored_keyboard'] = check_graphical_keyboard(vm, 'memory-restored')
+ result['status'] = 'passed'
+ except BaseException as error:
+ result.update(status='failed', error=repr(error))
+ try:
+ vm.screenshot('failure')
+ except Exception:
+ pass
+ raise
+ finally:
+ result['finished_at'] = time.time()
+ (folder / 'receipt.json').write_text(json.dumps(result, indent=2)+'\n')
+ vm.stop()
+
+
+if __name__ == '__main__':
+ main()