The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Bound memory while staging OS runtime updates (#801)
Dee2bc128201031
diff --git a/os/live_update.py b/os/live_update.pyindex 5a757189e..720cc077e 100644--- a/os/live_update.py+++ b/os/live_update.py@@ -34,6 +34,7 @@ FEEDS = {}FILES = ('harness-tui', 'cli.mjs', 'notify.mjs')LIMIT = 64 * 1024 * 1024+DOWNLOAD_CHUNK = 1024 * 1024RESTART_REQUIRED = Path('/run/harness-os-restart-required')SYSTEM_LOCK = Path('/run/lock/hn-os.lock')BOOT_ID = Path('/proc/sys/kernel/random/boot_id')@@ -142,19 +143,49 @@ def allowed_url(value):return value-def fetch(url, limit):+@contextmanager+def download_response(url, limit):request = Request(allowed_url(url), headers={'User-Agent': 'Harness-Updates/1'})with urlopen(request, timeout=20) as response:allowed_url(response.url)size = response.headers.get('Content-Length')if size and (not size.isdigit() or int(size) > limit):raise ValueError('Update exceeds its download size limit.')+ yield response+++def fetch(url, limit):+ with download_response(url, limit) as response:data = response.read(limit + 1)if len(data) > limit:raise ValueError('Update exceeds its download size limit.')return data+def download(ref, destination):+ """Keep release bytes on disk while the installed workspace remains active."""+ limit = ref.get('size') or LIMIT+ size, checksum = 0, hashlib.sha256()+ created = False+ try:+ with download_response(ref['url'], limit) as response, destination.open('xb') as handle:+ created = True+ while chunk := response.read(min(DOWNLOAD_CHUNK, limit + 1 - size)):+ size += len(chunk)+ if size > limit:+ raise ValueError('Update exceeds its download size limit.')+ checksum.update(chunk)+ handle.write(chunk)+ if checksum.hexdigest() != ref['sha256'].lower() or (+ ref.get('size') is not None and size != ref['size']+ ):+ raise ValueError('Download checksum or size does not match the release.')+ except BaseException:+ if created:+ destination.unlink(missing_ok=True)+ raise++def runtime_platform():"""Match the public hn release keys to the native Linux executable format."""machine = platform.machine()@@ -315,41 +346,36 @@ def check(feeds=None, progress=lambda _: None, force_system=False):# Releases whose tagged commits are already ancestors of that source are# older, even if their public version number is numerically higher.baselines = read(BASE_ID, {}).get('release_baselines', {})- changes, target_versions, errors = {}, dict(base_versions), []+ target_versions, errors = dict(base_versions), []ignored = read(STATE / 'ignored.json', {})- for component, url in (feeds or FEEDS).items():- try:- candidate, refs = release(url, component)- if ignored.get(component) == candidate:- continue- baseline = baselines.get(component, {}).get('version', '0.0.0')- if version(candidate) > max(version(base_versions[component]), version(baseline)):- progress('Downloading hn…' if component == 'hn' else 'Downloading Harness…')- component_files = {}- for name, ref in refs.items():- data = fetch(ref['url'], ref.get('size', LIMIT))- if hashlib.sha256(data).hexdigest() != ref['sha256'].lower() or (- ref.get('size') is not None and len(data) != ref['size']- ):- raise ValueError('Download checksum or size does not match the release.')- if name == 'harness-tui':- verify_hn(data)- component_files[name] = data- changes.update(component_files)- target_versions[component] = candidate- except (ValueError, OSError, KeyError, TypeError) as error:- errors.append(f'{component}: {error}')- # Each component is complete before staging; an unavailable CLI release- # must not hold up an independent hn fix (or leave only half a CLI pair).- if changes:- builds = STATE / 'builds'- builds.mkdir(exist_ok=True)- with tempfile.TemporaryDirectory(prefix='.download-', dir=builds) as temp:- folder = Path(temp)+ builds = STATE / 'builds'+ builds.mkdir(exist_ok=True)+ with tempfile.TemporaryDirectory(prefix='.download-', dir=builds) as temp:+ folder, changes = Path(temp), set()+ for component, url in (feeds or FEEDS).items():+ try:+ candidate, refs = release(url, component)+ if ignored.get(component) == candidate:+ continue+ baseline = baselines.get(component, {}).get('version', '0.0.0')+ if version(candidate) > max(version(base_versions[component]), version(baseline)):+ progress('Downloading hn…' if component == 'hn' else 'Downloading Harness…')+ component_files = set()+ for name, ref in refs.items():+ download(ref, folder / name)+ if name == 'harness-tui':+ with (folder / name).open('rb') as handle:+ verify_hn(handle.read(64))+ component_files.add(name)+ changes.update(component_files)+ target_versions[component] = candidate+ except (ValueError, OSError, KeyError, TypeError) as error:+ errors.append(f'{component}: {error}')+ # Publish only complete components. A failed CLI pair is overwritten+ # from the base before an independent hn update can become ready.+ if changes:for name in FILES:- if name in changes:- (folder / name).write_bytes(changes[name])- else:+ if name not in changes:shutil.copyfile(base / name, folder / name)(folder / name).chmod(0o755 if name == 'harness-tui' else 0o644)with (folder / name).open('rb') as handle:diff --git a/os/tests/test_live_update.py b/os/tests/test_live_update.pyindex b89086458..c1e6a0900 100644--- a/os/tests/test_live_update.py+++ b/os/tests/test_live_update.py@@ -2,6 +2,7 @@ import hashlibimport cursesimport fcntlimport importlib.util+import ioimport jsonfrom pathlib import Pathimport struct@@ -21,6 +22,79 @@ def elf_fixture(machine=62, payload=b''):2, machine, 1, 0, 64, 0, 0, 64, 56, 0, 0, 0, 0) + payload+class Response(io.BytesIO):+ def __init__(self, data, url='https://example.test/runtime', headers=None):+ super().__init__(data)+ self.url, self.headers, self.reads = url, headers or {}, []++ def read(self, size=-1):+ self.reads.append(size)+ return super().read(size)+++class Downloads(unittest.TestCase):+ def test_large_release_is_streamed_in_bounded_reads(self):+ data = b'payload' * 400000+ for declared_size in (True, False):+ with self.subTest(declared_size=declared_size), tempfile.TemporaryDirectory() as temp:+ target = Path(temp) / 'runtime'+ ref = {'url': 'https://example.test/runtime', 'sha256': hashlib.sha256(data).hexdigest()}+ if declared_size:+ ref['size'] = len(data)+ response = Response(data)+ with patch.object(update, 'urlopen', return_value=response):+ update.download(ref, target)+ self.assertEqual(target.read_bytes(), data)+ self.assertGreater(len(response.reads), 2)+ self.assertTrue(all(0 < size <= 1024 * 1024 for size in response.reads))++ def test_invalid_downloads_leave_no_partial_file(self):+ data = b'payload'+ correct = hashlib.sha256(data).hexdigest()+ cases = [+ ({'size': len(data) - 1}, {}), # Oversize without Content-Length.+ ({'size': len(data) + 1}, {}), # Truncated with a correct checksum.+ ({'sha256': '0' * 64}, {}),+ ({}, {'Content-Length': 'invalid'}),+ ({'size': len(data)}, {'Content-Length': str(len(data) + 1)}),+ ]+ for fields, headers in cases:+ with self.subTest(fields=fields, headers=headers), tempfile.TemporaryDirectory() as temp:+ target = Path(temp) / 'runtime'+ ref = dict({'url': 'https://example.test/runtime', 'sha256': correct}, **fields)+ with patch.object(update, 'urlopen', return_value=Response(data, headers=headers)):+ with self.assertRaises(ValueError):+ update.download(ref, target)+ self.assertFalse(target.exists())+ with tempfile.TemporaryDirectory() as temp, patch.object(update, 'LIMIT', len(data) - 1):+ target = Path(temp) / 'runtime'+ with patch.object(update, 'urlopen', return_value=Response(data)), self.assertRaises(ValueError):+ update.download({'url': 'https://example.test/runtime', 'sha256': correct}, target)+ self.assertFalse(target.exists())++ def test_stream_failure_cleans_its_file_but_never_removes_an_existing_file(self):+ ref = {'url': 'https://example.test/runtime', 'sha256': '0' * 64}+ with tempfile.TemporaryDirectory() as temp:+ target = Path(temp) / 'runtime'+ response = Response(b'')+ with (patch.object(response, 'read', side_effect=[b'partial', OSError('connection lost')]),+ patch.object(update, 'urlopen', return_value=response), self.assertRaises(OSError)):+ update.download(ref, target)+ self.assertFalse(target.exists())+ target.write_bytes(b'existing')+ with patch.object(update, 'urlopen', return_value=Response(b'')), self.assertRaises(FileExistsError):+ update.download(ref, target)+ self.assertEqual(target.read_bytes(), b'existing')++ def test_unsafe_redirect_is_rejected_before_creating_a_file(self):+ with tempfile.TemporaryDirectory() as temp:+ target = Path(temp) / 'runtime'+ with (patch.object(update, 'urlopen', return_value=Response(b'', url='http://example.test/runtime')),+ self.assertRaisesRegex(ValueError, 'HTTPS')):+ update.download({'url': 'https://example.test/runtime', 'sha256': '0' * 64}, target)+ self.assertFalse(target.exists())++class FastUpdates(unittest.TestCase):def setUp(self):temporary = tempfile.TemporaryDirectory()@@ -62,11 +136,12 @@ class FastUpdates(unittest.TestCase):update.FEEDS['hn']: json.dumps(dict(version='1.1.0', builds={target: refs['harness-tui']})).encode(),update.FEEDS['cli']: json.dumps(dict(cli=dict(version='1.1.0', cli=refs['cli.mjs'], notify=refs['notify.mjs']))).encode(),}- def fetch(url, limit):+ def fetch(request, timeout):+ url = request.full_urlif fail and fail in url:- return b'broken download'- return manifests[url] if url in manifests else assets[url.rsplit('/', 1)[1]]- return patch.object(update, 'fetch', side_effect=fetch)+ return Response(b'broken download', url)+ return Response(manifests[url] if url in manifests else assets[url.rsplit('/', 1)[1]], url)+ return patch.object(update, 'urlopen', side_effect=fetch)def test_version_and_transport_reject_malformed_and_unsafe_releases(self):for value in ['1.0', '1.0.1-dev.local', '999999999.0.0', '../1.0.0', None]:@@ -116,6 +191,16 @@ class FastUpdates(unittest.TestCase):self.assertEqual((target / 'notify.mjs').read_bytes(), b'old notify.mjs')self.assertTrue(update.read(self.state / 'check.json')['errors'])+ def test_interrupted_download_check_cleans_staging_without_publishing(self):+ def progress(message):+ if message == 'Downloading Harness…':+ raise KeyboardInterrupt()+ with self.feed(), self.assertRaises(KeyboardInterrupt):+ update.check(progress=progress)+ self.assertIsNone(update.prepared())+ self.assertEqual(update.selected(), self.bundled)+ self.assertEqual(list((self.state / 'builds').iterdir()), [])+def test_corrupt_hn_does_not_replace_current_build_and_valid_cli_can_still_stage(self):with self.feed(fail='harness-tui'):self.assertTrue(update.check())@@ -138,7 +223,7 @@ class FastUpdates(unittest.TestCase):self.assertTrue(update.check())self.assertEqual(update.verify(update.prepared())['versions'], {'hn': '1.0.0', 'cli': '1.1.0'})self.assertEqual((update.prepared() / 'harness-tui').read_bytes(), original)- self.assertNotIn('https://example.test/harness-tui', [call.args[0] for call in fetch.call_args_list])+ self.assertNotIn('https://example.test/harness-tui', [call.args[0].full_url for call in fetch.call_args_list])self.assertIn('No hn build is available for linux-arm64', str(update.read(self.state / 'check.json')['errors']))self.assertEqual(update.selected(), self.bundled)