The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Measure OS runtime memory before tuning Node defaults (#696)
Deeedeefaee20b2
diff --git a/.github/workflows/os-memory.yml b/.github/workflows/os-memory.ymlnew file mode 100644index 000000000..5bc3a9fdf--- /dev/null+++ b/.github/workflows/os-memory.yml@@ -0,0 +1,50 @@+name: Harness OS runtime memory assessment++on:+ workflow_dispatch:+ inputs:+ image_run_id:+ description: 'Exact existing image artifact to assess'+ type: string+ default: '37149350400'+ required: true+ push:+ branches: ['os-polish/runtime-memory']+ paths: ['os/tests/memory_*.py', '.github/workflows/os-memory.yml']++permissions:+ contents: read+ actions: read++jobs:+ memory:+ runs-on: ubuntu-24.04+ timeout-minutes: 20+ steps:+ - uses: actions/checkout@v7+ - uses: actions/download-artifact@v6+ with:+ name: programmer-os-x86_64+ path: os/dist+ run-id: ${{ inputs.image_run_id || '37149350400' }}+ github-token: ${{ github.token }}+ - name: Check the fixture and install VM tools+ run: |+ python3 -m py_compile os/tests/memory_guest.py os/tests/memory_vm.py+ go run github.com/rhysd/actionlint/cmd/[email protected] .github/workflows/os-memory.yml+ sudo apt-get update -qq+ sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf python3-pil tesseract-ocr+ sudo chmod a+rw /dev/kvm+ - name: Measure alternating profiles on the installed 1 GiB system+ run: python3 os/tests/memory_vm.py --iso os/dist/*.iso+ - name: Retain measured evidence without disks+ if: always()+ uses: actions/upload-artifact@v6+ with:+ name: harness-os-memory+ path: |+ os/test-results/memory/+ !os/test-results/memory/*.qcow2+ !os/test-results/memory/OVMF_VARS.fd+ retention-days: 14+ compression-level: 0diff --git a/os/DEVELOPMENT.md b/os/DEVELOPMENT.mdindex f85ba4c89..ed3c79cd2 100644--- a/os/DEVELOPMENT.md+++ b/os/DEVELOPMENT.md@@ -90,6 +90,21 @@ python3 os/tests/local_ai_vm.py --iso os/dist/IMAGE.iso --probe local-aipython3 os/tests/local_ai_vm.py --iso os/dist/IMAGE.iso --probe nvidia```+The **Harness OS runtime memory assessment** workflow compares Node flags in one+installed 1 GiB VM. It alternates two default and two candidate rounds, samples+RSS/PSS and local status latency with four persistent terminal streams, records+CPU time, then restores the packaged command and checks graphical keyboard input.+This measures daemon behavior, not model performance or full agent throughput.++In [the October 3 comparison](https://github.com/autonomous-ai/openharness/actions/runs/37154644789),+the installed Node was 22.23.3. Median process RSS was 117.4/129.2 MiB in the two+default rounds and 126.1/145.2 MiB with `--optimize-for-size+--max-semi-space-size=1`. Each round made 200 status requests; p95 response times+were 1.19–1.25 ms. Memory drifted across this short interleaved run, so it does not+establish a precise causal difference. It provides no evidence for a saving from+these flags; the shipped defaults remain unchanged. Retain raw per-round data+and use representative long-running agent work before adopting a memory limit.+## The feedback loop| Work being tested | Best environment | What it proves |diff --git a/os/tests/memory_guest.py b/os/tests/memory_guest.pynew file mode 100644index 000000000..9163b2ade--- /dev/null+++ b/os/tests/memory_guest.py@@ -0,0 +1,130 @@+#!/usr/bin/env python3+"""Measure candidate Node flags only inside an installed disposable OS guest."""+import json+import os+from pathlib import Path+import statistics+import subprocess+import time+from urllib.request import urlopen+++ROOT = Path.home() / 'memory-assessment'+OVERRIDE = Path.home() / '.config/systemd/user/harness-daemon.service.d/memory-assessment.conf'+PROFILES = [('default', []), ('small', ['--optimize-for-size', '--max-semi-space-size=1'])]+++def run(*args, timeout=180):+ result = subprocess.run(args, capture_output=True, text=True, timeout=timeout)+ with (ROOT / 'commands.log').open('a') as log:+ log.write(json.dumps(args) + '\n' + result.stdout + result.stderr + '\n')+ result.check_returncode()+ return result.stdout.strip()+++def api():+ start = time.perf_counter()+ with urlopen('http://127.0.0.1:18473/api/status', timeout=5) as response:+ value = json.load(response)+ assert value.get('discoveryReady'), value+ return (time.perf_counter() - start) * 1000+++def process(pid):+ memory = {}+ for line in Path(f'/proc/{pid}/smaps_rollup').read_text().splitlines():+ parts = line.split()+ if parts[0] in ('Rss:', 'Pss:', 'Private_Clean:', 'Private_Dirty:'):+ memory[parts[0][:-1] + '_kib'] = int(parts[1])+ fields = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()+ memory['cpu_ticks'] = int(fields[11]) + int(fields[12])+ return memory+++def survivor_pids():+ return {p.name: int(p.read_text()) for p in ROOT.glob('survivor-*.pid')}+++def main():+ assert os.getuid() != 0 and not Path('/etc/harness-live').exists()+ assert Path('/var/lib/harness-os/install.json').is_file()+ ROOT.mkdir(exist_ok=True)+ assert not OVERRIDE.exists(), 'Never replace an existing user override'+ result = {'status': 'running', 'started_at': time.time(), 'rounds': [],+ 'node': run('node', '--version'), 'clock_ticks_per_second': os.sysconf('SC_CLK_TCK'),+ 'scope': 'Daemon memory, local status latency and persistent terminal streams; not full agent throughput'}+ # Keep terminal processes and output moving while the daemon is replaced.+ (ROOT / 'survivor.py').write_text('''import os,sys,time+from pathlib import Path+root=Path.home()/'memory-assessment'+n=sys.argv[1]+(root/('survivor-'+n+'.pid')).write_text(str(os.getpid()))+for i in range(10000):+ (root/('survivor-'+n+'.count')).write_text(str(i))+ print('terminal '+n+' output '+str(i)+' '+('x'*120),flush=True)+ time.sleep(.25)+''')+ run('systemctl', '--user', 'stop', 'harness-update.timer')+ for n in range(4):+ run('hn', 'new-window', '-d', '-n', f'memory-{n}', f'python3 -u {ROOT}/survivor.py {n}')+ deadline = time.monotonic() + 15+ while len(survivor_pids()) != 4:+ assert time.monotonic() < deadline, 'Terminal streams did not start'+ time.sleep(.25)+ original = survivor_pids()+ try:+ # Interleave the baseline to expose cache/load drift in one VM.+ for repetition in range(2):+ for name, flags in PROFILES:+ OVERRIDE.parent.mkdir(parents=True, exist_ok=True)+ OVERRIDE.write_text('[Service]\nExecStart=\nExecStart=/usr/bin/node ' ++ ' '.join(flags) + ' /usr/lib/harness/cli.mjs start --foreground\n')+ run('systemctl', '--user', 'daemon-reload')+ start = time.monotonic()+ run('systemctl', '--user', 'restart', 'harness-daemon')+ ready_seconds = time.monotonic() - start+ pid = int(run('systemctl', '--user', 'show', 'harness-daemon', '-p', 'MainPID', '--value'))+ command = Path(f'/proc/{pid}/cmdline').read_bytes().replace(b'\0', b' ').decode()+ assert '/usr/bin/node' in command and all(flag in command for flag in flags)+ for _ in range(10):+ api()+ time.sleep(1)+ before = process(pid)+ start = time.monotonic()+ samples, latencies = [], []+ for _ in range(10):+ latencies.extend(api() for _ in range(20))+ samples.append(process(pid))+ time.sleep(3)+ elapsed = time.monotonic() - start+ assert survivor_pids() == original, 'A terminal owner was replaced'+ for child in original.values():+ os.kill(child, 0)+ ticks = process(pid)['cpu_ticks'] - before['cpu_ticks']+ ordered = sorted(latencies)+ row = {'profile': name, 'repetition': repetition, 'flags': flags,+ 'pid': pid, 'command': command, 'ready_seconds': ready_seconds,+ 'samples': samples, 'rss_median_kib': statistics.median(s['Rss_kib'] for s in samples),+ 'pss_median_kib': statistics.median(s['Pss_kib'] for s in samples),+ 'api_median_ms': statistics.median(latencies), 'api_p95_ms': ordered[int(len(ordered)*.95)],+ 'api_max_ms': max(latencies), 'cpu_seconds': ticks / os.sysconf('SC_CLK_TCK'),+ 'measured_seconds': elapsed, 'survivor_pids': original}+ result['rounds'].append(row)+ (ROOT / 'receipt.json').write_text(json.dumps(result, indent=2)+'\n')+ print('HN_MEMORY_ROUND=' + json.dumps({k:v for k,v in row.items() if k!='samples'}), flush=True)+ result['status'] = 'passed'+ except BaseException as error:+ result.update(status='failed', error=repr(error))+ raise+ finally:+ OVERRIDE.unlink(missing_ok=True)+ run('systemctl', '--user', 'daemon-reload')+ run('systemctl', '--user', 'restart', 'harness-daemon')+ result['default_restored'] = '--optimize-for-size' not in run('systemctl', '--user', 'show', 'harness-daemon', '-p', 'ExecStart', '--value')+ result['finished_at'] = time.time()+ (ROOT / 'receipt.json').write_text(json.dumps(result, indent=2)+'\n')+ print('HN_MEMORY_RESULT=' + json.dumps(result), flush=True)+++if __name__ == '__main__':+ main()diff --git a/os/tests/memory_vm.py b/os/tests/memory_vm.pynew file mode 100644index 000000000..b8e03b819--- /dev/null+++ b/os/tests/memory_vm.py@@ -0,0 +1,73 @@+#!/usr/bin/env python3+"""Native 1 GiB installed-OS A/B measurement; never changes the published image."""+import argparse+import base64+import hashlib+import json+import os+from pathlib import Path+import subprocess+import time+from vm import VM, check_graphical_keyboard+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument('--iso', type=Path, required=True)+ args = parser.parse_args()+ if not os.access('/dev/kvm', os.R_OK | os.W_OK):+ parser.error('Native x86 KVM is required')+ iso = args.iso.resolve()+ manifest = json.loads(iso.with_name('manifest.json').read_text())+ with iso.open('rb') as handle:+ assert hashlib.file_digest(handle, 'sha256').hexdigest() == manifest['iso']['sha256']+ folder = Path('os/test-results/memory').resolve()+ folder.mkdir(parents=True, exist_ok=False)+ vm = VM(folder, iso, 'uefi', 1024)+ config = dict(disk='/dev/vda', expected_serial='HN_OS_TEST', confirm_erase='/dev/vda',+ username='me', hostname='harness', password='test-password-123', encrypt=True, serial_console=True)+ result = {'status': 'running', 'started_at': time.time(), 'memory_mib': 1024,+ 'image_source_commit': manifest['source_commit'], 'iso_sha256': manifest['iso']['sha256'],+ 'test_source_commit': subprocess.check_output(['git','rev-parse','HEAD'], text=True).strip()}+ try:+ vm.start(live=True)+ vm.wait(r'root@[^\r\n]*[#] ', timeout=180)+ vm.shell_ready = True+ vm.command('stty -echo')+ encoded = base64.b64encode(json.dumps(config).encode()).decode()+ vm.command('printf %s ' + encoded + ' | base64 -d > /tmp/install-config.json')+ vm.command('nmcli networking off')+ output, _ = vm.command('harness install --config /tmp/install-config.json --yes-erase-disk', timeout=360)+ (folder / 'install.log').write_text(output)+ vm.command('sync')+ vm.stop()+ vm.start(live=False)+ vm.login_installed(config)+ script = base64.b64encode(Path(__file__).with_name('memory_guest.py').read_bytes()).decode()+ vm.command(': > /tmp/memory-guest.b64')+ for start in range(0, len(script), 2000):+ vm.command('printf %s ' + script[start:start+2000] + ' >> /tmp/memory-guest.b64')+ vm.command('base64 -d /tmp/memory-guest.b64 > /home/me/memory-guest.py')+ output, status = vm.command('python3 /home/me/memory-guest.py', timeout=720, check=False)+ (folder / 'benchmark.log').write_text(output)+ assert status == 0, 'Memory assessment failed; see benchmark.log'+ marker = 'HN_MEMORY_RESULT='+ result['guest'] = json.loads(next(line.split(marker,1)[1] for line in output.splitlines() if marker in line))+ assert result['guest']['status'] == 'passed' and result['guest']['default_restored']+ result['restored_keyboard'] = check_graphical_keyboard(vm, 'memory-restored')+ result['status'] = 'passed'+ except BaseException as error:+ result.update(status='failed', error=repr(error))+ try:+ vm.screenshot('failure')+ except Exception:+ pass+ raise+ finally:+ result['finished_at'] = time.time()+ (folder / 'receipt.json').write_text(json.dumps(result, indent=2)+'\n')+ vm.stop()+++if __name__ == '__main__':+ main()