Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Bound memory while staging OS runtime updates (#801)

Dee2bc128201031
diff --git a/os/live_update.py b/os/live_update.py
index 5a757189e..720cc077e 100644
--- a/os/live_update.py
+++ b/os/live_update.py
@@ -34,6 +34,7 @@ FEEDS = {
}
FILES = ('harness-tui', 'cli.mjs', 'notify.mjs')
LIMIT = 64 * 1024 * 1024
+DOWNLOAD_CHUNK = 1024 * 1024
RESTART_REQUIRED = Path('/run/harness-os-restart-required')
SYSTEM_LOCK = Path('/run/lock/hn-os.lock')
BOOT_ID = Path('/proc/sys/kernel/random/boot_id')
@@ -142,19 +143,49 @@ def allowed_url(value):
return value
-def fetch(url, limit):
+@contextmanager
+def download_response(url, limit):
request = Request(allowed_url(url), headers={'User-Agent': 'Harness-Updates/1'})
with urlopen(request, timeout=20) as response:
allowed_url(response.url)
size = response.headers.get('Content-Length')
if size and (not size.isdigit() or int(size) > limit):
raise ValueError('Update exceeds its download size limit.')
+ yield response
+
+
+def fetch(url, limit):
+ with download_response(url, limit) as response:
data = response.read(limit + 1)
if len(data) > limit:
raise ValueError('Update exceeds its download size limit.')
return data
+def download(ref, destination):
+ """Keep release bytes on disk while the installed workspace remains active."""
+ limit = ref.get('size') or LIMIT
+ size, checksum = 0, hashlib.sha256()
+ created = False
+ try:
+ with download_response(ref['url'], limit) as response, destination.open('xb') as handle:
+ created = True
+ while chunk := response.read(min(DOWNLOAD_CHUNK, limit + 1 - size)):
+ size += len(chunk)
+ if size > limit:
+ raise ValueError('Update exceeds its download size limit.')
+ checksum.update(chunk)
+ handle.write(chunk)
+ if checksum.hexdigest() != ref['sha256'].lower() or (
+ ref.get('size') is not None and size != ref['size']
+ ):
+ raise ValueError('Download checksum or size does not match the release.')
+ except BaseException:
+ if created:
+ destination.unlink(missing_ok=True)
+ raise
+
+
def runtime_platform():
"""Match the public hn release keys to the native Linux executable format."""
machine = platform.machine()
@@ -315,41 +346,36 @@ def check(feeds=None, progress=lambda _: None, force_system=False):
# Releases whose tagged commits are already ancestors of that source are
# older, even if their public version number is numerically higher.
baselines = read(BASE_ID, {}).get('release_baselines', {})
- changes, target_versions, errors = {}, dict(base_versions), []
+ target_versions, errors = dict(base_versions), []
ignored = read(STATE / 'ignored.json', {})
- for component, url in (feeds or FEEDS).items():
- try:
- candidate, refs = release(url, component)
- if ignored.get(component) == candidate:
- continue
- baseline = baselines.get(component, {}).get('version', '0.0.0')
- if version(candidate) > max(version(base_versions[component]), version(baseline)):
- progress('Downloading hn…' if component == 'hn' else 'Downloading Harness…')
- component_files = {}
- for name, ref in refs.items():
- data = fetch(ref['url'], ref.get('size', LIMIT))
- if hashlib.sha256(data).hexdigest() != ref['sha256'].lower() or (
- ref.get('size') is not None and len(data) != ref['size']
- ):
- raise ValueError('Download checksum or size does not match the release.')
- if name == 'harness-tui':
- verify_hn(data)
- component_files[name] = data
- changes.update(component_files)
- target_versions[component] = candidate
- except (ValueError, OSError, KeyError, TypeError) as error:
- errors.append(f'{component}: {error}')
- # Each component is complete before staging; an unavailable CLI release
- # must not hold up an independent hn fix (or leave only half a CLI pair).
- if changes:
- builds = STATE / 'builds'
- builds.mkdir(exist_ok=True)
- with tempfile.TemporaryDirectory(prefix='.download-', dir=builds) as temp:
- folder = Path(temp)
+ builds = STATE / 'builds'
+ builds.mkdir(exist_ok=True)
+ with tempfile.TemporaryDirectory(prefix='.download-', dir=builds) as temp:
+ folder, changes = Path(temp), set()
+ for component, url in (feeds or FEEDS).items():
+ try:
+ candidate, refs = release(url, component)
+ if ignored.get(component) == candidate:
+ continue
+ baseline = baselines.get(component, {}).get('version', '0.0.0')
+ if version(candidate) > max(version(base_versions[component]), version(baseline)):
+ progress('Downloading hn…' if component == 'hn' else 'Downloading Harness…')
+ component_files = set()
+ for name, ref in refs.items():
+ download(ref, folder / name)
+ if name == 'harness-tui':
+ with (folder / name).open('rb') as handle:
+ verify_hn(handle.read(64))
+ component_files.add(name)
+ changes.update(component_files)
+ target_versions[component] = candidate
+ except (ValueError, OSError, KeyError, TypeError) as error:
+ errors.append(f'{component}: {error}')
+ # Publish only complete components. A failed CLI pair is overwritten
+ # from the base before an independent hn update can become ready.
+ if changes:
for name in FILES:
- if name in changes:
- (folder / name).write_bytes(changes[name])
- else:
+ if name not in changes:
shutil.copyfile(base / name, folder / name)
(folder / name).chmod(0o755 if name == 'harness-tui' else 0o644)
with (folder / name).open('rb') as handle:
diff --git a/os/tests/test_live_update.py b/os/tests/test_live_update.py
index b89086458..c1e6a0900 100644
--- a/os/tests/test_live_update.py
+++ b/os/tests/test_live_update.py
@@ -2,6 +2,7 @@ import hashlib
import curses
import fcntl
import importlib.util
+import io
import json
from pathlib import Path
import struct
@@ -21,6 +22,79 @@ def elf_fixture(machine=62, payload=b''):
2, machine, 1, 0, 64, 0, 0, 64, 56, 0, 0, 0, 0) + payload
+class Response(io.BytesIO):
+ def __init__(self, data, url='https://example.test/runtime', headers=None):
+ super().__init__(data)
+ self.url, self.headers, self.reads = url, headers or {}, []
+
+ def read(self, size=-1):
+ self.reads.append(size)
+ return super().read(size)
+
+
+class Downloads(unittest.TestCase):
+ def test_large_release_is_streamed_in_bounded_reads(self):
+ data = b'payload' * 400000
+ for declared_size in (True, False):
+ with self.subTest(declared_size=declared_size), tempfile.TemporaryDirectory() as temp:
+ target = Path(temp) / 'runtime'
+ ref = {'url': 'https://example.test/runtime', 'sha256': hashlib.sha256(data).hexdigest()}
+ if declared_size:
+ ref['size'] = len(data)
+ response = Response(data)
+ with patch.object(update, 'urlopen', return_value=response):
+ update.download(ref, target)
+ self.assertEqual(target.read_bytes(), data)
+ self.assertGreater(len(response.reads), 2)
+ self.assertTrue(all(0 < size <= 1024 * 1024 for size in response.reads))
+
+ def test_invalid_downloads_leave_no_partial_file(self):
+ data = b'payload'
+ correct = hashlib.sha256(data).hexdigest()
+ cases = [
+ ({'size': len(data) - 1}, {}), # Oversize without Content-Length.
+ ({'size': len(data) + 1}, {}), # Truncated with a correct checksum.
+ ({'sha256': '0' * 64}, {}),
+ ({}, {'Content-Length': 'invalid'}),
+ ({'size': len(data)}, {'Content-Length': str(len(data) + 1)}),
+ ]
+ for fields, headers in cases:
+ with self.subTest(fields=fields, headers=headers), tempfile.TemporaryDirectory() as temp:
+ target = Path(temp) / 'runtime'
+ ref = dict({'url': 'https://example.test/runtime', 'sha256': correct}, **fields)
+ with patch.object(update, 'urlopen', return_value=Response(data, headers=headers)):
+ with self.assertRaises(ValueError):
+ update.download(ref, target)
+ self.assertFalse(target.exists())
+ with tempfile.TemporaryDirectory() as temp, patch.object(update, 'LIMIT', len(data) - 1):
+ target = Path(temp) / 'runtime'
+ with patch.object(update, 'urlopen', return_value=Response(data)), self.assertRaises(ValueError):
+ update.download({'url': 'https://example.test/runtime', 'sha256': correct}, target)
+ self.assertFalse(target.exists())
+
+ def test_stream_failure_cleans_its_file_but_never_removes_an_existing_file(self):
+ ref = {'url': 'https://example.test/runtime', 'sha256': '0' * 64}
+ with tempfile.TemporaryDirectory() as temp:
+ target = Path(temp) / 'runtime'
+ response = Response(b'')
+ with (patch.object(response, 'read', side_effect=[b'partial', OSError('connection lost')]),
+ patch.object(update, 'urlopen', return_value=response), self.assertRaises(OSError)):
+ update.download(ref, target)
+ self.assertFalse(target.exists())
+ target.write_bytes(b'existing')
+ with patch.object(update, 'urlopen', return_value=Response(b'')), self.assertRaises(FileExistsError):
+ update.download(ref, target)
+ self.assertEqual(target.read_bytes(), b'existing')
+
+ def test_unsafe_redirect_is_rejected_before_creating_a_file(self):
+ with tempfile.TemporaryDirectory() as temp:
+ target = Path(temp) / 'runtime'
+ with (patch.object(update, 'urlopen', return_value=Response(b'', url='http://example.test/runtime')),
+ self.assertRaisesRegex(ValueError, 'HTTPS')):
+ update.download({'url': 'https://example.test/runtime', 'sha256': '0' * 64}, target)
+ self.assertFalse(target.exists())
+
+
class FastUpdates(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
@@ -62,11 +136,12 @@ class FastUpdates(unittest.TestCase):
update.FEEDS['hn']: json.dumps(dict(version='1.1.0', builds={target: refs['harness-tui']})).encode(),
update.FEEDS['cli']: json.dumps(dict(cli=dict(version='1.1.0', cli=refs['cli.mjs'], notify=refs['notify.mjs']))).encode(),
}
- def fetch(url, limit):
+ def fetch(request, timeout):
+ url = request.full_url
if fail and fail in url:
- return b'broken download'
- return manifests[url] if url in manifests else assets[url.rsplit('/', 1)[1]]
- return patch.object(update, 'fetch', side_effect=fetch)
+ return Response(b'broken download', url)
+ return Response(manifests[url] if url in manifests else assets[url.rsplit('/', 1)[1]], url)
+ return patch.object(update, 'urlopen', side_effect=fetch)
def test_version_and_transport_reject_malformed_and_unsafe_releases(self):
for value in ['1.0', '1.0.1-dev.local', '999999999.0.0', '../1.0.0', None]:
@@ -116,6 +191,16 @@ class FastUpdates(unittest.TestCase):
self.assertEqual((target / 'notify.mjs').read_bytes(), b'old notify.mjs')
self.assertTrue(update.read(self.state / 'check.json')['errors'])
+ def test_interrupted_download_check_cleans_staging_without_publishing(self):
+ def progress(message):
+ if message == 'Downloading Harness…':
+ raise KeyboardInterrupt()
+ with self.feed(), self.assertRaises(KeyboardInterrupt):
+ update.check(progress=progress)
+ self.assertIsNone(update.prepared())
+ self.assertEqual(update.selected(), self.bundled)
+ self.assertEqual(list((self.state / 'builds').iterdir()), [])
+
def test_corrupt_hn_does_not_replace_current_build_and_valid_cli_can_still_stage(self):
with self.feed(fail='harness-tui'):
self.assertTrue(update.check())
@@ -138,7 +223,7 @@ class FastUpdates(unittest.TestCase):
self.assertTrue(update.check())
self.assertEqual(update.verify(update.prepared())['versions'], {'hn': '1.0.0', 'cli': '1.1.0'})
self.assertEqual((update.prepared() / 'harness-tui').read_bytes(), original)
- self.assertNotIn('https://example.test/harness-tui', [call.args[0] for call in fetch.call_args_list])
+ self.assertNotIn('https://example.test/harness-tui', [call.args[0].full_url for call in fetch.call_args_list])
self.assertIn('No hn build is available for linux-arm64', str(update.read(self.state / 'check.json')['errors']))
self.assertEqual(update.selected(), self.bundled)