The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Validate visible game controls without requiring HTML labels (#697)
Dee06b2e68aff1e
diff --git a/.github/workflows/os.yml b/.github/workflows/os.ymlindex 4e817a139..84159cd86 100644--- a/.github/workflows/os.yml+++ b/.github/workflows/os.yml@@ -26,7 +26,7 @@ on:default: falsetype: booleanworkload_seed_run_id:- description: 'Optional: reuse the generated projects from this workload run; finish its game agent and rerun all acceptance checks'+ description: 'Reuse generated projects and rerun acceptance; also enable workloads to ask the game agent to repair its layout'required: falsetype: stringmemory_mib:@@ -191,7 +191,7 @@ jobs:OS_WORKLOAD_SOURCE_RUN: ${{ inputs.workload_seed_run_id }}OS_MEMORY_MIB: ${{ inputs.memory_mib || '2048' }}OS_LIVE_TRANSPORT: ${{ inputs.live_transport || 'cdrom' }}- run: python3 os/tests/vm.py --iso os/dist/*.iso --memory "$OS_MEMORY_MIB" --live-transport "$OS_LIVE_TRANSPORT" --firmware '${{ matrix.firmware }}' ${{ matrix.encryption }} ${{ matrix.agents }} ${{ matrix.cpu }} ${{ (inputs.workloads || inputs.workload_seed_run_id != '') && matrix.firmware == 'bios' && '--workloads' || '' }} ${{ inputs.dsh && matrix.firmware == 'bios' && '--dsh' || '' }} ${{ inputs.workload_seed_run_id != '' && matrix.firmware == 'bios' && '--workload-seed os/work/workload-seed' || '' }}+ run: python3 os/tests/vm.py --iso os/dist/*.iso --memory "$OS_MEMORY_MIB" --live-transport "$OS_LIVE_TRANSPORT" --firmware '${{ matrix.firmware }}' ${{ matrix.encryption }} ${{ matrix.agents }} ${{ matrix.cpu }} ${{ (inputs.workloads || inputs.workload_seed_run_id != '') && matrix.firmware == 'bios' && '--workloads' || '' }} ${{ inputs.dsh && matrix.firmware == 'bios' && '--dsh' || '' }} ${{ inputs.workload_seed_run_id != '' && matrix.firmware == 'bios' && '--workload-seed os/work/workload-seed' || '' }} ${{ inputs.workloads && inputs.workload_seed_run_id != '' && matrix.firmware == 'bios' && '--workload-repair-game' || '' }}- name: Exercise password lock and actual suspend/resumeif: ${{ inputs.checks == 'session' }}run: python3 os/tests/session_vm.py --iso os/dist/*.iso --firmware '${{ matrix.firmware }}' --session-file os/root/usr/lib/harness-os/session --theme-directory os/root/usr/share/plymouth/themes/harnessdiff --git a/os/DEVELOPMENT.md b/os/DEVELOPMENT.mdindex ed3c79cd2..d8a20b128 100644--- a/os/DEVELOPMENT.md+++ b/os/DEVELOPMENT.md@@ -135,6 +135,10 @@ a clean USB install to verify that upgrades have not hidden an installation bug.exercises a constrained machine; `memory_mib=4096` and `live_transport=usb`exercise automatic copy-to-RAM. `workloads=true` or `dsh=true` runs the relevantreal-agent exercises. See the [measured evidence](README.md#real-programmer-exercises).+- `workload_seed_run_id` preserves generated projects and reruns their acceptance+ checks without another model turn. Enable `workloads=true` as well only when+ asking the agent to repair the existing game's layout. The host reads control+ labels from captured pixels, so a canvas legend does not need HTML duplicates.- `python3 os/tools/run-vm.py --iso PATH` creates a persistent virtual disk andopens the installer. `--installed` subsequently boots that disk without the ISO.`--directory PATH` keeps independent test machines separate.diff --git a/os/tests/vm.py b/os/tests/vm.pyindex 16442ef93..0ac9ff53e 100644--- a/os/tests/vm.py+++ b/os/tests/vm.py@@ -549,6 +549,7 @@ def main():parser.add_argument('--workloads', action='store_true', help='Opt in to real free-model project builds and browser acceptance after --agents')parser.add_argument('--dsh', action='store_true', help='Exercise three real DSH agents and shared viewers after --agents')parser.add_argument('--workload-seed', type=Path, help='Previous workload artifact: preserve its generated projects and rerun acceptance')+ parser.add_argument('--workload-repair-game', action='store_true', help='With a seed, explicitly ask the agent to repair game layout before rechecking')parser.add_argument('--live-only', action='store_true', help='Development probe: stop after the live-session checks, without installing')args = parser.parse_args()if args.workloads and not args.agents:@@ -559,6 +560,8 @@ def main():parser.error('Run the two long model suites separately.')if args.workload_seed and not args.workloads:parser.error('--workload-seed requires --workloads')+ if args.workload_repair_game and not args.workload_seed:+ parser.error('--workload-repair-game requires --workload-seed')folder = (args.output or Path(__file__).resolve().parents[1] / 'test-results' / (args.firmware + ('-encrypted' if args.encrypt else '-plain'))).resolve()folder.mkdir(parents=True, exist_ok=False)result = {'firmware': args.firmware, 'encrypted': args.encrypt, 'memory_mib': args.memory,@@ -806,6 +809,11 @@ pacman --noconfirm -U /tmp/hn-os-recovery-probe-1-1-any.pkg.tar.zstfor name in ['terminal-tool', 'website', 'game', 'fullstack']:archive.add(seeds[0] / name, arcname='workloads/seed/' + name)result['workload_project_source_run_id'] = os.environ.get('OS_WORKLOAD_SOURCE_RUN')+ if args.workload_repair_game:+ marker = tarfile.TarInfo('workloads/repair-game')+ marker.size = 0+ archive.addfile(marker, io.BytesIO())+ result['workload_game_repair_requested'] = args.workload_repair_gameencoded = base64.b64encode(package.getvalue()).decode()vm.command(': > /tmp/hn-workloads.b64')for offset in range(0, len(encoded), 2000):@@ -825,6 +833,18 @@ pacman --noconfirm -U /tmp/hn-os-recovery-probe-1-1-any.pkg.tar.zstarchive.extractall(destination, filter='data')(destination / '.local/state/harness-os/workloads').rename(destination / 'reports')vm.command('test "$(cat ~/.local/state/harness-os/workloads/status)" = 0')+ # Validate what the player sees, including canvas-rendered labels.+ # Tesseract is on the test host; no OCR package enters the guest/ISO.+ for size in ['1024x768', '1280x800']:+ frame = destination / 'reports' / f'game-{size}.png'+ visible = subprocess.check_output(['tesseract', str(frame), 'stdout', '--psm', '11'],+ text=True, stderr=subprocess.DEVNULL, timeout=15)+ frame.with_suffix('.txt').write_text(visible)+ words = visible.lower()+ required = ['move', 'space', 'pause', 'restart', 'enter', 'start']+ missing = [word for word in required if not re.search(r'\b' + word + r'\b', words)]+ if missing:+ raise RuntimeError(f'Game controls are not readable at {size}: {missing}; inspect {frame}')result['checks'].append('Free OpenCode built four projects; independent CLI, keyboard browser, game and persistent API checks passed')if args.dsh:package = io.BytesIO()diff --git a/os/tests/workloads/check-browser.mjs b/os/tests/workloads/check-browser.mjsindex 69ad3cfaa..609512ef5 100644--- a/os/tests/workloads/check-browser.mjs+++ b/os/tests/workloads/check-browser.mjs@@ -79,23 +79,20 @@ try {await available('http://127.0.0.1:18883');await page.goto('http://127.0.0.1:18883');await page.waitForFunction(() => window.harnessGame?.stats);+ await page.keyboard.press('Enter');for (const viewport of [{ width: 1024, height: 768 }, { width: 1280, height: 800 }]) {await page.setViewportSize(viewport);+ await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))));+ await page.screenshot({ path: join(report, `game-${viewport.width}x${viewport.height}.png`) });assert.ok(await page.evaluate(() => {const board = document.querySelector('canvas').getBoundingClientRect();- // TASK.txt requires a visible keyboard legend, not a CSS class name.- // Find its smallest containing element using the required controls.- const controls = element => /space/i.test(element.textContent) &&- /pause/i.test(element.textContent) && /restart/i.test(element.textContent);- const element = [...document.querySelectorAll('body *')].find(element =>- controls(element) && ![...element.children].some(controls));- if (!element) return false;- const legend = element.getBoundingClientRect();+ // The legend may be drawn in the canvas. The host independently+ // reads the saved pixels with OCR instead of requiring HTML text.return board.width > 200 && board.height > 200 && board.top >= 0 &&- board.bottom <= innerHeight && legend.width > 0 && legend.height > 0 &&- legend.top >= 0 && legend.bottom <= innerHeight &&- document.documentElement.scrollHeight <= innerHeight + 1;- }), 'The whole game and keyboard legend fit the laptop viewport');+ board.left >= 0 && board.right <= innerWidth && board.bottom <= innerHeight &&+ document.documentElement.scrollHeight <= innerHeight + 1 &&+ document.documentElement.scrollWidth <= innerWidth + 1;+ }), 'The whole game fits the laptop viewport');}await page.keyboard.press('Enter');await page.screenshot({ path: join(report, 'game.png') });diff --git a/os/tests/workloads/run.sh b/os/tests/workloads/run.shindex 80add2957..7bae540ea 100644--- a/os/tests/workloads/run.sh+++ b/os/tests/workloads/run.sh@@ -9,7 +9,7 @@ REPORT="$HOME/.local/state/harness-os/workloads"mkdir -p "$WORK" "$REPORT"if [[ -d "$INPUTS/seed" ]]; thencp -a "$INPUTS/seed/." "$WORK/"- printf '%s\n' 'Reusing previously generated projects; only the interrupted game agent runs again.' > "$REPORT/reuse.txt"+ printf '%s\n' 'Reusing previously generated projects; acceptance runs again. The game agent runs only when explicitly requested.' > "$REPORT/reuse.txt"fitrap 'printf "%s\n" "$?" > "$REPORT/status"' EXIT/usr/bin/opencode --version > "$REPORT/opencode-version.txt" || exit 1@@ -23,7 +23,7 @@ for scenario in terminal-tool website game fullstack; do{"permission":{"question":"deny","task":"deny","external_directory":{"*":"deny","$HOME/Projects/**":"allow","/tmp/opencode/**":"allow","/usr/share/harness-os/**":"allow"}}}EOFprintf '\nChecking %s with the upstream default model\n' "$scenario"- if [[ -d "$INPUTS/seed" && "$scenario" != game ]]; then+ if [[ -d "$INPUTS/seed" && ( "$scenario" != game || ! -e "$INPUTS/repair-game" ) ]]; thenstatus=0printf '%s\n' 'Prior completed model turn reused; project checks run again.' > "$REPORT/$scenario-agent-reused.txt"# Dependencies are deliberately absent from the retained source artifact.@@ -33,7 +33,7 @@ EOFelseprompt='Read TASK.txt, implement the complete task, run its tests, and fix failures. Work now without questions or subagents. Browser acceptance runs separately with sandboxed system Chromium; finish after your unit tests and do not launch a browser or long-running server.'if [[ -d "$INPUTS/seed" ]]; then- prompt='The existing Signal Run game already passes its unit tests and independent keyboard/pause/restart/state checks. Perform ONE remaining task: adjust styles.css so the entire canvas and compact keyboard legend fit within a 1280x800 or 1024x768 browser viewport without vertical scrolling. Let the stage shrink into the remaining viewport height. Preserve simulation and bridge behavior. Do not rewrite the game, broaden features, audit edge cases, or add test frameworks. Run node --test test.mjs once, fix only regressions from your CSS change, then give your final answer and stop. A separate browser tester will verify layout. No questions or subagents.'+ prompt='Read TASK.txt and the existing game. Perform ONE task: make the entire game and a compact, readable keyboard legend fit within both 1280x800 and 1024x768 viewports without scrolling or clipping. The legend may be drawn in the canvas or HTML; ensure all move, Space pause, R restart and Enter start controls are visible. Change only the layout or drawing necessary for that result. Preserve simulation and bridge behavior. Do not rewrite the game, broaden features, audit unrelated edge cases, or add test frameworks. Run node --test test.mjs, fix regressions from your change, then give your final answer and stop. An independent browser and rendered-pixel checker verifies layout. No questions or subagents.'fi(cd "$project" || exit 1