Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Keep Ethernet recoverable without replacing saved settings (#876)

Deebaff05c4ff4b
diff --git a/.github/workflows/os.yml b/.github/workflows/os.yml
index a5c8f9f15..410f3f869 100644
--- a/.github/workflows/os.yml
+++ b/.github/workflows/os.yml
@@ -47,11 +47,11 @@ on:
default: false
type: boolean
checks:
- description: 'Image journey, browser focus, lock/suspend, recovery, Wi-Fi/audio, hardware, private zram, or changed-kernel recovery acceptance'
+ description: 'Image journey, browser focus, lock/suspend, recovery, Wi-Fi/audio, Ethernet recovery, hardware, private zram, or changed-kernel recovery acceptance'
required: false
default: image
type: choice
- options: [image, browser, browser-suspend, session, recovery, essentials, hardware, zram, hardware-update, public-update]
+ options: [image, browser, browser-suspend, session, recovery, essentials, network-recovery, hardware, zram, hardware-update, public-update]
browser_memory:
description: 'With checks=browser, diagnose memory in the UEFI VM only; retain ordinary workload and deadlines'
required: false
@@ -66,12 +66,12 @@ permissions:
actions: read
concurrency:
- group: programmer-os-${{ github.ref }}-${{ inputs.memory_mib || '2048' }}-${{ inputs.live_transport || 'cdrom' }}-${{ inputs.development_update && 'update' || inputs.checks == 'public-update' && 'public-update' || inputs.checks == 'browser-suspend' && 'browser-suspend' || inputs.checks == 'browser' && 'browser' || inputs.checks == 'session' && 'session' || inputs.checks == 'recovery' && 'recovery' || inputs.checks == 'essentials' && 'essentials' || inputs.checks == 'hardware' && 'hardware' || inputs.checks == 'hardware-update' && 'hardware-update' || inputs.dsh && 'dsh' || inputs.workloads && 'workloads' || 'image' }}
+ group: programmer-os-${{ github.ref }}-${{ inputs.memory_mib || '2048' }}-${{ inputs.live_transport || 'cdrom' }}-${{ inputs.development_update && 'update' || inputs.checks == 'public-update' && 'public-update' || inputs.checks == 'browser-suspend' && 'browser-suspend' || inputs.checks == 'browser' && 'browser' || inputs.checks == 'session' && 'session' || inputs.checks == 'recovery' && 'recovery' || inputs.checks == 'essentials' && 'essentials' || inputs.checks == 'network-recovery' && 'network-recovery' || inputs.checks == 'hardware' && 'hardware' || inputs.checks == 'hardware-update' && 'hardware-update' || inputs.dsh && 'dsh' || inputs.workloads && 'workloads' || 'image' }}
cancel-in-progress: false
jobs:
image:
- if: ${{ !inputs.image_run_id && !inputs.publish_run_id && !inputs.development_update && inputs.checks != 'zram' && inputs.checks != 'hardware-update' && inputs.checks != 'browser-suspend' && inputs.checks != 'public-update' }}
+ if: ${{ !inputs.image_run_id && !inputs.publish_run_id && !inputs.development_update && inputs.checks != 'zram' && inputs.checks != 'hardware-update' && inputs.checks != 'browser-suspend' && inputs.checks != 'public-update' && inputs.checks != 'network-recovery' }}
runs-on: ubuntu-24.04
timeout-minutes: 50
steps:
@@ -159,7 +159,7 @@ jobs:
machine:
needs: image
- if: ${{ always() && !inputs.publish_run_id && !inputs.development_update && inputs.checks != 'essentials' && inputs.checks != 'hardware' && inputs.checks != 'zram' && inputs.checks != 'hardware-update' && inputs.checks != 'public-update' && (needs.image.result == 'success' || inputs.image_run_id != '' || inputs.checks == 'browser-suspend') && !cancelled() }}
+ if: ${{ always() && !inputs.publish_run_id && !inputs.development_update && inputs.checks != 'essentials' && inputs.checks != 'hardware' && inputs.checks != 'zram' && inputs.checks != 'hardware-update' && inputs.checks != 'public-update' && inputs.checks != 'network-recovery' && (needs.image.result == 'success' || inputs.image_run_id != '' || inputs.checks == 'browser-suspend') && !cancelled() }}
runs-on: ubuntu-24.04
timeout-minutes: 65
strategy:
@@ -469,6 +469,47 @@ jobs:
os/test-results/essentials/*.wav
retention-days: 14
+ network-recovery:
+ if: ${{ inputs.checks == 'network-recovery' && !inputs.publish_run_id && !inputs.development_update }}
+ runs-on: ubuntu-24.04
+ timeout-minutes: 15
+ strategy:
+ fail-fast: false
+ matrix:
+ profile: [automatic, saved]
+ steps:
+ - uses: actions/checkout@v7
+ - name: Require an existing baseline image
+ env:
+ OS_IMAGE_RUN: ${{ inputs.image_run_id }}
+ run: test -n "$OS_IMAGE_RUN"
+ - uses: actions/download-artifact@v6
+ with:
+ name: programmer-os-x86_64
+ path: os/dist
+ run-id: ${{ inputs.image_run_id }}
+ github-token: ${{ github.token }}
+ - name: Validate source and prepare native tools
+ run: |
+ make -C os check
+ go run github.com/rhysd/actionlint/cmd/[email protected] .github/workflows/os.yml
+ sudo apt-get update -qq
+ sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf python3-pil tesseract-ocr
+ sudo chmod a+rw /dev/kvm
+ - name: Recover Ethernet through the real network page after sleep
+ run: python3 os/tests/network_recovery_vm.py --iso os/dist/*.iso --network-script os/network.py --profile ${{ matrix.profile }}
+ - name: Retain network recovery evidence
+ if: always()
+ uses: actions/upload-artifact@v6
+ with:
+ name: harness-os-network-recovery-${{ matrix.profile }}
+ path: |
+ os/test-results/network-recovery/*.json
+ os/test-results/network-recovery/*.txt
+ os/test-results/network-recovery/*.log
+ os/test-results/network-recovery/*.png
+ retention-days: 14
+
development-update:
if: ${{ inputs.development_update && !inputs.publish_run_id && inputs.checks != 'hardware-update' && inputs.checks != 'public-update' }}
runs-on: ubuntu-24.04
diff --git a/os/network.py b/os/network.py
index d34dbf32a..a7bf3cb56 100644
--- a/os/network.py
+++ b/os/network.py
@@ -71,8 +71,36 @@ def wired_devices():
result = nmcli('-t', '-e', 'yes', '-f', 'DEVICE,TYPE,STATE', 'device', 'status', wait=3)
if result.returncode:
return []
+ # An unavailable Ethernet device can still recover through explicit
+ # activation, including after networking was disabled across suspend.
return [row for line in result.stdout.splitlines()
- if len(row := fields(line)) == 3 and row[1] == 'ethernet' and row[2] != 'unavailable']
+ if len(row := fields(line)) == 3 and row[1] == 'ethernet']
+
+
+def connect_wired(device, state):
+ if state == 'unavailable':
+ # After a disabled boot and sleep, NM can leave the link down and its
+ # available-profile list empty. Connecting then creates a new DHCP
+ # profile, ignoring saved settings. Raise only the selected link and
+ # let NM discover its profiles before requesting ordinary activation.
+ link = subprocess.run(['/usr/bin/ip', 'link', 'set', 'dev', device, 'up'],
+ capture_output=True, text=True, timeout=3)
+ if link.returncode:
+ return False
+ deadline = time.monotonic() + 10
+ while True:
+ status = nmcli('-g', 'GENERAL.STATE', 'device', 'show', device, wait=2)
+ if status.returncode:
+ return False
+ value = status.stdout.strip().split(' ', 1)[0]
+ if value == '100':
+ return True
+ if value == '30':
+ break
+ if time.monotonic() >= deadline:
+ return False
+ time.sleep(.2)
+ return nmcli('device', 'connect', device, wait=30).returncode == 0
def still_connected(network):
@@ -325,7 +353,7 @@ class NetworkPage:
self.busy(value['ssid'] if kind == 'wifi' else 'Ethernet')
try:
if kind == 'wired':
- if nmcli('device', 'connect', value[0], wait=30).returncode == 0:
+ if connect_wired(value[0], value[2]):
return 0
self.message = 'Could not connect Ethernet. Check the cable and try again.'
elif (value['active'] and still_connected(value)) or connect(value):
diff --git a/os/tests/network_recovery_vm.py b/os/tests/network_recovery_vm.py
new file mode 100644
index 000000000..b7e44fc16
--- /dev/null
+++ b/os/tests/network_recovery_vm.py
@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""Reconnect inactive Ethernet through the real network page in a disposable VM."""
+import argparse
+from functools import partial
+import hashlib
+from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+import json
+import os
+from pathlib import Path
+import shlex
+import subprocess
+import threading
+import time
+
+from session_vm import put
+from update_vm import network_state
+from vm import VM
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--iso', type=Path, required=True)
+ parser.add_argument('--network-script', type=Path, required=True)
+ parser.add_argument('--profile', choices=('automatic', 'saved'), required=True)
+ args = parser.parse_args()
+ if not os.access('/dev/kvm', os.R_OK | os.W_OK):
+ parser.error('Use native x86 KVM for this acceptance check.')
+ iso = args.iso.resolve()
+ manifest = json.loads(iso.with_name('manifest.json').read_text())
+ with iso.open('rb') as handle:
+ assert hashlib.file_digest(handle, 'sha256').hexdigest() == manifest['iso']['sha256']
+ candidate = args.network_script.read_bytes()
+ checksum = hashlib.sha256(candidate).hexdigest()
+ folder = Path('os/test-results/network-recovery').resolve()
+ folder.mkdir(parents=True, exist_ok=False)
+ served = folder / 'served'
+ served.mkdir()
+ proof = 'harness-network-recovery\n'
+ (served / 'proof.txt').write_text(proof)
+ server = ThreadingHTTPServer(('127.0.0.1', 0), partial(SimpleHTTPRequestHandler, directory=str(served)))
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ vm = VM(folder, iso, 'uefi', 2048, video='VGA')
+ result = dict(status='running', started_at=time.time(), checks=[],
+ image_source=manifest['source_commit'], iso_sha256=manifest['iso']['sha256'],
+ candidate_network_sha256=checksum,
+ profile_mode=args.profile,
+ test_source=subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(),
+ limits=['Virtual Ethernet and ACPI only; no physical Wi-Fi acceptance.',
+ 'Only the candidate network.py replaces a product file in the verified image.'])
+ config = dict(disk='/dev/vda', expected_serial='HN_OS_TEST', confirm_erase='/dev/vda',
+ username='me', hostname='harness', password='test-password-123', encrypt=True, serial_console=True)
+
+ def login():
+ vm.login_installed(config)
+ vm.command('export PAGER= LC_ALL=C')
+ vm.command('printf %s ' + shlex.quote(config['password'] + '\n') + ' | sudo -S -v')
+
+ def http():
+ vm.command('curl --noproxy "*" --fail --silent --show-error --max-time 10 '
+ f'http://10.0.2.2:{server.server_port}/proof.txt -o /tmp/network-proof', timeout=15)
+ assert vm.read_file('/tmp/network-proof').decode() == proof
+
+ def profile():
+ vm.command('nmcli -g GENERAL.CON-UUID device show ens5 > /tmp/network-profile')
+ return vm.read_file('/tmp/network-profile').decode().strip()
+
+ def settings(identity):
+ vm.command('nmcli -g connection.id,connection.uuid,connection.interface-name,ipv4.method,ipv4.route-metric '
+ 'connection show uuid ' + shlex.quote(identity) + ' > /tmp/network-settings')
+ return vm.read_file('/tmp/network-settings')
+
+ try:
+ vm.start(live=True)
+ vm.wait(r'root@[^\r\n]*[#] ', timeout=180)
+ vm.shell_ready = True
+ vm.command('stty -echo')
+ put(vm, '/tmp/network-install.json', json.dumps(config))
+ vm.command('nmcli networking off')
+ output, _ = vm.command('harness install --config /tmp/network-install.json --yes-erase-disk', timeout=360)
+ (folder / 'install.log').write_text(output)
+ vm.command('sync')
+ vm.stop()
+ vm.start(live=False)
+ login()
+ vm.command('nm-online -q --timeout=30', timeout=35)
+ http()
+ original_profile = profile()
+ assert original_profile and original_profile != '--'
+ result['original_profile'] = original_profile
+ # NetworkManager's first automatic Ethernet connection is generated in
+ # memory, not a saved user configuration. Exercise that case separately
+ # from a genuinely persisted profile with a nondefault route metric.
+ vm.command('sudo -n find /etc/NetworkManager/system-connections -maxdepth 1 -type f '
+ '> /tmp/saved-network-files')
+ assert not vm.read_file('/tmp/saved-network-files').strip(), 'Baseline unexpectedly has saved network profiles'
+ if args.profile == 'saved':
+ vm.command('sudo -n nmcli connection modify uuid ' + shlex.quote(original_profile)
+ + ' connection.id "Harness recovery fixture" ipv4.route-metric 321')
+ vm.command('sudo -n grep -rlFx ' + shlex.quote('uuid=' + original_profile)
+ + ' /etc/NetworkManager/system-connections > /tmp/saved-network-files')
+ assert vm.read_file('/tmp/saved-network-files').strip(), 'Fixture did not persist its connection'
+ original_settings = settings(original_profile)
+ result['saved_settings_sha256'] = hashlib.sha256(original_settings).hexdigest()
+ result['original_network_sha256'] = hashlib.sha256(vm.read_file('/usr/lib/harness-os/network.py')).hexdigest()
+ put(vm, '/tmp/network-candidate.py', candidate.decode())
+ vm.command('sudo -n install -o root -g root -m 755 /tmp/network-candidate.py /usr/lib/harness-os/network.py')
+ assert hashlib.sha256(vm.read_file('/usr/lib/harness-os/network.py')).hexdigest() == checksum
+ vm.command('mkdir -p ~/projects/network-recovery; printf keep > ~/projects/network-recovery/proof')
+ vm.command('sudo -n nmcli networking off; sync')
+ vm.stop()
+ vm.start(live=False)
+ login()
+ vm.command('test "$(nmcli networking)" = disabled')
+ if args.profile == 'saved':
+ assert settings(original_profile) == original_settings, 'Saved configuration did not survive reboot'
+ network_state(vm, 'disabled-boot')
+ vm.command('sudo -n systemctl suspend --no-block')
+ deadline = time.monotonic() + 30
+ while vm.monitor('query-status')['status'] != 'suspended':
+ assert time.monotonic() < deadline, 'Guest did not reach ACPI suspend'
+ time.sleep(.2)
+ vm.monitor('system_wakeup')
+ deadline = time.monotonic() + 30
+ while True:
+ vm.send('\n')
+ try:
+ vm.wait(r'\[me@harness [^\r\n]*\]\$ ', timeout=2)
+ break
+ except TimeoutError:
+ assert time.monotonic() < deadline, 'Serial did not resume'
+ vm.command('pgrep -u 1000 -x "swaylock|gtklock"')
+ vm.screenshot('resume-locked')
+ vm.type_probe(config['password'])
+ vm.keys('ret')
+ vm.command('for n in $(seq 1 60); do ! pgrep -u 1000 -x "swaylock|gtklock" && exit 0; sleep .25; done; exit 1', timeout=20)
+ vm.command('sudo -n nmcli networking on')
+ _, status = vm.command('nm-online -q --timeout=10', timeout=15, check=False)
+ assert status != 0, 'Selected image did not reproduce the disabled-boot/sleep regression'
+ vm.command('test "$(nmcli -g GENERAL.STATE device show ens5)" = "20 (unavailable)"')
+ network_state(vm, 'before-ui-recovery')
+ result['checks'].append('Encrypted cold boot with networking disabled, real suspend, authenticated unlock and re-enable reproduce unavailable Ethernet')
+ vm.keys('meta_l', 'w')
+ deadline = time.monotonic() + 25
+ while True:
+ vm.command('hn capture-pane -p > /tmp/network-page')
+ content = vm.read_file('/tmp/network-page').decode()
+ if all(word in content for word in ('Connect to Wi-Fi', 'Rescan', 'Ethernet ens5')):
+ break
+ assert time.monotonic() < deadline, 'Super+w did not expose the recoverable Ethernet interface: ' + content
+ time.sleep(.25)
+ (folder / 'network-page.txt').write_text(content)
+ vm.screenshot('ethernet-available-for-selection')
+ # This VM has no wireless radio: Rescan is followed by its Ethernet row.
+ vm.keys('down')
+ vm.screenshot('ethernet-selected')
+ vm.keys('ret')
+ vm.command('nm-online -q --timeout=35', timeout=40)
+ http()
+ connected_profile = profile()
+ assert connected_profile and connected_profile != '--'
+ result['connected_profile'] = connected_profile
+ if args.profile == 'saved':
+ assert connected_profile == original_profile, 'Recovery did not use the saved connection'
+ assert settings(original_profile) == original_settings, 'Recovery changed the saved configuration'
+ vm.command('ip -j -4 route show dev ens5 > /tmp/recovered-routes')
+ routes = json.loads(vm.read_file('/tmp/recovered-routes'))
+ result['recovered_routes'] = routes
+ # NM applies the saved metric to the prefix route unchanged. Its
+ # internet connectivity check can add 20000 to the default route
+ # until the external check completes; that is not lost settings.
+ # https://networkmanager.dev/docs/api/latest/NetworkManager.conf.html#config-connectivity
+ assert any(route.get('dst') == '10.0.2.0/24' and route.get('metric') == 321
+ for route in routes), 'Saved prefix route metric was not applied'
+ assert any(route.get('dst') == 'default' and route.get('metric') in (321, 20321)
+ for route in routes), 'Saved default route metric was not applied'
+ result['active_route_metric'] = 321
+ result['checks'].append('Recovery uses the persisted connection UUID and retains its nondefault route metric')
+ vm.command('test "$(cat ~/projects/network-recovery/proof)" = keep')
+ network_state(vm, 'after-ui-recovery')
+ vm.screenshot('ethernet-connected')
+ result['checks'].append('Super+w shows unavailable Ethernet; keyboard selection reconnects it and transfers exact HTTP bytes')
+ result['checks'].append('The project survives the encrypted reboot and recovery; no NetworkManager restart or shell activation command is used for recovery')
+ result['status'] = 'passed'
+ except BaseException as error:
+ result.update(status='failed', error=str(error))
+ try:
+ vm.screenshot('failure')
+ network_state(vm, 'failure')
+ except Exception:
+ pass
+ raise
+ finally:
+ result['finished_at'] = time.time()
+ (folder / 'receipt.json').write_text(json.dumps(result, indent=2) + '\n')
+ vm.stop()
+ server.shutdown()
+ server.server_close()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/os/tests/test_network.py b/os/tests/test_network.py
index 409d84fc1..a0a9a22f4 100644
--- a/os/tests/test_network.py
+++ b/os/tests/test_network.py
@@ -40,6 +40,52 @@ class Network(unittest.TestCase):
network.nmcli('general')
self.assertEqual(run.call_args.args[0][0], '/usr/bin/nmcli')
+ def test_unavailable_ethernet_remains_selectable_for_explicit_reconnection(self):
+ devices = ('lo:loopback:connected (externally)\nens5:ethernet:unavailable\n'
+ 'eth1:ethernet:disconnected\nwlan0:wifi:unavailable\n')
+ with patch.object(network, 'nmcli', return_value=result(output=devices)):
+ self.assertEqual(network.wired_devices(), [
+ ['ens5', 'ethernet', 'unavailable'], ['eth1', 'ethernet', 'disconnected']])
+
+ def test_ethernet_waits_for_availability_before_selecting_a_profile(self):
+ with patch.object(network.subprocess, 'run', return_value=result()) as link, \
+ patch.object(network, 'nmcli', side_effect=[result(output='20 (unavailable)'),
+ result(output='30 (disconnected)'), result()]) as command, \
+ patch.object(network.time, 'sleep'):
+ self.assertTrue(network.connect_wired('ens5', 'unavailable'))
+ self.assertEqual(link.call_args.args[0], ['/usr/bin/ip', 'link', 'set', 'dev', 'ens5', 'up'])
+ self.assertEqual([call.args for call in command.call_args_list], [
+ ('-g', 'GENERAL.STATE', 'device', 'show', 'ens5'),
+ ('-g', 'GENERAL.STATE', 'device', 'show', 'ens5'),
+ ('device', 'connect', 'ens5')])
+
+ def test_ethernet_does_not_interrupt_automatic_reconnection(self):
+ with patch.object(network.subprocess, 'run', return_value=result()), \
+ patch.object(network, 'nmcli', side_effect=[result(output='70 (connecting)'),
+ result(output='100 (connected)')]) as command, patch.object(network.time, 'sleep'):
+ self.assertTrue(network.connect_wired('ens5', 'unavailable'))
+ self.assertFalse(any(call.args[:2] == ('device', 'connect') for call in command.call_args_list))
+
+ def test_ethernet_does_not_create_a_profile_when_the_link_stays_unavailable(self):
+ with patch.object(network.subprocess, 'run', return_value=result()), \
+ patch.object(network, 'nmcli', return_value=result(output='20 (unavailable)')) as command, \
+ patch.object(network.time, 'monotonic', side_effect=[0, 11]):
+ self.assertFalse(network.connect_wired('ens5', 'unavailable'))
+ command.assert_called_once_with('-g', 'GENERAL.STATE', 'device', 'show', 'ens5', wait=2)
+
+ def test_ethernet_link_failure_stops_before_activation(self):
+ with patch.object(network.subprocess, 'run', return_value=result(1)), \
+ patch.object(network, 'nmcli') as command:
+ self.assertFalse(network.connect_wired('ens5', 'unavailable'))
+ command.assert_not_called()
+
+ def test_disconnected_ethernet_keeps_normal_networkmanager_selection(self):
+ with patch.object(network.subprocess, 'run') as link, \
+ patch.object(network, 'nmcli', return_value=result()) as command:
+ self.assertTrue(network.connect_wired('ens5', 'disconnected'))
+ link.assert_not_called()
+ command.assert_called_once_with('device', 'connect', 'ens5', wait=30)
+
def test_scans_escape_ssids_and_deduplicate_radios_without_hiding_open_networks(self):
rows = 'a\\:network:60:WPA2:wlan0:aa\\:bb:\nopen:90:--:wlan0:bb\\:cc:\na\\:network:88:WPA2:wlan0:cc\\:dd:*\n:80:WPA2:wlan0:dd\\:ee:\n'
with patch.object(network, 'nmcli', return_value=result(output=rows)):
@@ -210,6 +256,17 @@ class Network(unittest.TestCase):
self.assertEqual(page.run(), 0)
self.assertEqual(command.call_args.args, ('device', 'connect', 'eth0'))
+ def test_failed_ethernet_reconnection_keeps_the_network_page_open(self):
+ screen, page = self.page([curses.KEY_DOWN, '\n', '\x1b'], first_use=False)
+ with patch.object(network, 'scan', return_value=[]), \
+ patch.object(network, 'wired_devices', return_value=[['ens5', 'ethernet', 'unavailable']]), \
+ patch.object(network, 'connect_wired', return_value=False) as connect, \
+ patch.object(network, 'nmcli') as command:
+ self.assertEqual(page.run(), 1)
+ connect.assert_called_once_with('ens5', 'unavailable')
+ self.assertIn('Could not connect Ethernet.', '\n'.join(text for _, text in screen.lines))
+ self.assertFalse(any(call.args[:2] == ('connection', 'delete') for call in command.call_args_list))
+
def test_wide_and_control_characters_fit_the_terminal(self):
self.assertEqual(network.fit('網路123', 5), '網路1')
self.assertEqual(network.fit('a\x1b\tb', 6), 'a b ')