Minecraft: Java Edition 26.3 rewritten from scratch in Rust, checked against vanilla with 281 differential harnesses. Get it with the Horizon launcher.
Rust differential-testinggame-enginegamedevlinuxminecraftminecraft-javareimplementationrustvoxelvulkanwgpu
#!/usr/bin/env python3
"""End-to-end check of agent control: play Horizon with horizon-ctl.
A script creates a world through the title menus, finds a tree it can walk to, breaks a log,
checks the inventory, crafts planks in the inventory screen, places one and reads it back. It
runs through the CLI (one horizon-ctl process per command) and through `horizon-ctl mcp` with a
small JSON-RPC client, and saves a screenshot of each run. The ui scenario drives the menus by
widget id instead: the title screen, the options (a slider set by id) and the inventory's slots;
the options scenario reads and sets options.txt values by key; the events scenario waits for
states (wait-for) and reads the event log; the consume scenario eats and drinks consumables (a
golden apple, milk, honey, chorus fruit) and checks their effects, remainders and teleport, then
holds an eye of ender in use and the sprint key with and without blindness; the use-screen
scenario opens the inventory and the chat over an item in use and checks the use is released, not
eaten behind the GUI; the pick scenario
middle-clicks slots (AbstractContainerScreen's cloning): a full stack in creative, from the hotbar
and from an item tab, and nothing in survival;
the suggestions scenario checks slash-command Tab completion on single player's connected client
after a fresh launch; the no-ai scenario turns No AI on in a scratch game home and checks that
horizon-ctl, its MCP server and the game's control socket all refuse.
python3 tools/agent_control_e2e.py # both modes
python3 tools/agent_control_e2e.py --mode mcp # one of them
python3 tools/agent_control_e2e.py --bin-dir target/fast --out /tmp/agent-e2e
python3 tools/agent_control_e2e.py --scenario ui # menus by widget id only (a minute)
Needs vscreen on Linux (the desktop); on macOS the game renders offscreen, so it also runs in a
locked session or over SSH. It needs the prepared resources in .local/horizon-client and a build of
horizon-client and horizon-ctl (built with the fast profile unless --bin-dir is given). Games
started here have no audio (HORIZON_NO_AUDIO=1).
"""
import argparse
import base64
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
from pathlib import Path
try:
from . import lab
except ImportError:
import lab
ROOT = Path(__file__).resolve().parents[1]
LAVAPIPE = Path('/run/opengl-driver/share/vulkan/icd.d/lvp_icd.x86_64.json')
LOG_KINDS = ['oak_log', 'birch_log', 'spruce_log', 'jungle_log', 'acacia_log', 'dark_oak_log',
'mangrove_log', 'cherry_log']
# Positional arguments of the commands the scenario uses, for the CLI.
POSITIONAL = {
'key': ['key', 'action'],
'goto': ['x', 'y', 'z'], 'break': ['x', 'y', 'z'], 'place': ['x', 'y', 'z'], 'block': ['x', 'y', 'z'],
'find_block': ['block', 'radius'], 'entities': ['radius', 'type'], 'click_slot': ['slot'],
'click_button': ['label'], 'type': ['text'], 'wait': ['ticks'], 'wait_ready': ['seconds'],
'command': ['command'], 'look_at_block': ['x', 'y', 'z'], 'click': ['id'], 'set': ['id', 'value'],
'give': ['item', 'count'], 'options_get': ['keys'], 'options_set': ['key', 'value'],
'wait_for': ['condition', 'value'], 'open': ['world', 'seed', 'mode'], 'select': ['slot'],
'summon': ['kind', 'x', 'y', 'z'], 'interact_entity': ['id'], 'effect': ['id', 'seconds', 'amplifier'],
'xp': ['levels'], 'options': ['page'], 'lan': ['action'], 'world_backup': ['folder'],
'world_rename': ['folder', 'name'], 'world_delete': ['folder'], 'mouse': ['button', 'action'],
}
# Booleans the game takes as bare words.
BARE = {'click_slot': ['right', 'shift'], 'click': ['right', 'middle', 'shift'], 'summon': ['baby']}
class Failed(Exception):
pass
def game_env(screen, home=None):
"""The environment of a game: lab.game_env with HORIZON_HOME `home` (a scratch folder when None), no live-count
ping, and on Linux the virtual screen `screen` (macOS and Windows have none)."""
extra = {'HORIZON_TELEMETRY_URL': 'off'} # no live-count ping from test games
if home is not None:
extra['HORIZON_HOME'] = home
if sys.platform in ('darwin', 'win32'):
return lab.game_env(**extra)
if not shutil.which('vscreen') and os.environ.get('DISPLAY'):
return lab.game_env(**extra)
env = lab.game_env(VSCREEN=screen, **extra)
subprocess.run(['vscreen', 'up'], env=env, check=False, capture_output=True)
text = subprocess.run(['vscreen', 'env'], env=env, capture_output=True, text=True, check=True).stdout
env.pop('DISPLAY', None)
for match in re.finditer(r'(\w+)=(\S+)', text.split('unset')[0]):
env[match.group(1)] = match.group(2)
if LAVAPIPE.exists():
env['VK_ICD_FILENAMES'] = str(LAVAPIPE)
return env
class Cli:
"""One horizon-ctl process per command, as a shell script would."""
name = 'cli'
def __init__(self, ctl, socket, cwd, env):
self.base = [str(ctl), '--socket', str(socket)]
self.cwd, self.env = cwd, env
def __call__(self, tool, **args):
cmd = tool.replace('_', '-')
positional = [str(args.pop(k)) for k in POSITIONAL.get(tool, []) if k in args]
options = []
for k, v in args.items():
if v is True:
options.append(k if k in BARE.get(tool, []) else '--' + k)
elif v is not False and v is not None:
options += ['--' + k, str(v)]
# Options first: free text (type, command) runs to the end of the line.
words = [cmd] + (options + positional if tool in ('type', 'command') else positional + options)
if tool == 'screenshot':
words = ['screenshot', str(args.get('path', self.cwd / 'shot.png'))]
result = subprocess.run(self.base + words, cwd=self.cwd, env=self.env, capture_output=True, text=True,
timeout=600)
out = result.stdout.strip().splitlines()
reply = json.loads(out[-1]) if out else {'error': result.stderr.strip()}
log(self.name, words, reply)
if result.returncode or 'error' in reply:
raise Failed(f'{" ".join(words)}: exit {result.returncode}: {reply}')
return reply
def close(self):
pass
class Mcp:
"""`horizon-ctl mcp` driven over JSON-RPC, as an AI agent's client would."""
name = 'mcp'
def __init__(self, ctl, socket, cwd, env):
self.proc = subprocess.Popen([str(ctl), '--socket', str(socket), 'mcp'], cwd=cwd, env=env,
stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
self.next_id = 0
self.cwd = cwd
init = self.rpc('initialize', {'protocolVersion': '2025-06-18', 'capabilities': {},
'clientInfo': {'name': 'agent_control_e2e', 'version': '1'}})
assert init['serverInfo']['name'] == 'horizon-ctl', init
self.notify('notifications/initialized')
self.tools = {t['name']: t for t in self.rpc('tools/list', {})['tools']}
print(f'[mcp] {len(self.tools)} tools: {", ".join(sorted(self.tools))}')
def notify(self, method):
self.proc.stdin.write(json.dumps({'jsonrpc': '2.0', 'method': method}) + '\n')
self.proc.stdin.flush()
def rpc(self, method, params):
self.next_id += 1
self.proc.stdin.write(json.dumps({'jsonrpc': '2.0', 'id': self.next_id, 'method': method,
'params': params}) + '\n')
self.proc.stdin.flush()
reply = json.loads(self.proc.stdout.readline())
assert reply['id'] == self.next_id, reply
if 'error' in reply:
raise Failed(f'{method}: {reply["error"]}')
return reply['result']
def __call__(self, tool, **args):
if tool not in self.tools:
raise Failed(f'no MCP tool {tool}')
result = self.rpc('tools/call', {'name': tool, 'arguments': args})
texts = [c for c in result['content'] if c['type'] == 'text']
reply = json.loads(texts[-1]['text']) if texts else {}
images = [c for c in result['content'] if c['type'] == 'image']
if images:
reply['_image'] = base64.b64decode(images[0]['data'])
log(self.name, [tool, args], {k: v for k, v in reply.items() if k != '_image'})
if result['isError']:
raise Failed(f'{tool} {args}: {reply}')
return reply
def close(self):
self.proc.stdin.close()
self.proc.wait(timeout=30)
def log(mode, words, reply):
text = json.dumps(reply)
print(f'[{mode}] {words if isinstance(words, list) else words} -> {text[:400]}{"..." if len(text) > 400 else ""}')
def trunk_bases(d):
"""Logs with no log under them, nearest first: (distance, pos, log id)."""
bases = []
for kind in LOG_KINDS:
found = d('find_block', block=kind, radius=40)
for n in found['nearest']:
x, y, z = n['pos']
below = d('block', x=x, y=y - 1, z=z)['block']
if not below.endswith('_log') and not below.endswith('_wood'):
bases.append((n['distance'], (x, y, z), found['block']))
return sorted(bases)
def scenario(d, out):
# 1. A world, made through the title screen's menus.
screen = d('screen')
assert screen['screen'] == 'title', screen
assert any(b['label'] == 'Singleplayer' for b in screen['buttons']), screen
d('click_button', label='Singleplayer')
if d('screen')['menu'].get('mode') != 'create-world': # with no worlds, Singleplayer opens it itself
d('click_button', label='Create New World')
d('type', field='name', replace=True, text='Agent World')
d('type', field='seed', text='5') # spawn near a walkable oak
d('click_button', label='Create New World')
# The world opens a few frames later: in the frame it is prepared off the UI thread behind the
# title screen (world_loading.rs), on the server thread once the server has it.
d('wait_for', condition='loaded', timeout=400)
d('wait_ready', seconds=400)
state = d('state')
assert state['gamemode'] == 'survival' and state['player']['health'] > 0, state
d('command', command='time set day')
# The spawn is random within the spawn radius, on the first full top face below the
# MOTION_BLOCKING height (PlayerSpawnFinder.getLevelRespawnPos): a tree's leaves too, too high
# for goto's drops.
x, y, z = state['player']['block_pos']
ground = y - 1
while ground > y - 16 and d('block', x=x, y=ground, z=z)['block'].endswith(('_leaves', 'air')):
ground -= 1
if ground < y - 1:
d('command', command=f'tp @s {x + 0.5} {ground + 1} {z + 0.5}')
d('wait', ticks=5)
# 2. Walk to a tree, the nearest trunk the path finder can reach, and 3. break the log at the
# bottom of the trunk (it must end up in the inventory). Low leaves can hide every face of that
# log from where the walk stops: then the next tree.
for _, base, log_id in trunk_bases(d)[:8]:
x, y, z = base
try:
d('goto', x=x + 0.5, y=y, z=z + 0.5, near=2.5, timeout=60)
except Failed as e:
print(f' cannot reach {log_id} at {base}: {e}')
continue
try:
broke = d('break', x=x, y=y, z=z)
break
except Failed as e:
if 'no face' not in str(e):
raise
print(f' {log_id} at {base} is hidden: {e}')
else:
raise Failed('no reachable tree near spawn')
assert broke['block_before'].startswith(log_id), broke
assert broke['drops'].get(log_id, 0) >= 1, broke
# On a client of a server (single player's server thread) the drop and its pickup both come from
# the server a few ticks later: wait for the logs in the inventory, or walk to the drop.
for _ in range(20):
if broke['picked_up'].get(log_id, 0) >= 1 or d('inventory')['counts'].get(log_id, 0) >= 1:
break
items = d('entities', radius=8, type='item')['entities']
drop = next((e for e in items if e['item']['item'] == log_id), None)
if drop is not None:
d('goto', x=drop['pos'][0], y=drop['pos'][1], z=drop['pos'][2], near=1.0, timeout=20) # pickup reaches a block; the drop may lie under the trunk
d('wait', ticks=10)
break
d('wait', ticks=1)
inventory = d('inventory')
assert inventory['counts'].get(log_id, 0) >= 1, inventory
# 4. Craft planks in the inventory screen's 2x2 grid.
planks_id = log_id.replace('_log', '_planks')
d('open_inventory')
screen = d('screen')
assert screen['screen'] == 'inventory' and screen['craft_size'] == 2, screen
log_slot = next(s['slot'] for s in screen['slots'] if s.get('item') == log_id and s['slot'] < 36)
d('click_slot', slot=log_slot) # pick up the logs
d('click_slot', slot=100, right=True) # put one in the crafting grid
d('click_slot', slot=log_slot) # put the rest back
d('wait', ticks=2) # the server crafts the result (single player's server thread)
result = next(s for s in d('screen')['slots'] if s['slot'] == 200)
assert result.get('item') == planks_id and result['count'] == 4, result
d('click_slot', slot=200, shift=True) # craft into the inventory
d('close')
d('wait', ticks=2)
inventory = d('inventory')
assert inventory['counts'].get(planks_id, 0) >= 4, inventory
# 5. Place a plank beside the player and read it back.
p = d('state')['player']['block_pos']
hotbar = d('inventory')['hotbar']
if not any(s.get('item') == planks_id for s in hotbar):
raise Failed(f'{planks_id} did not land in the hotbar: {hotbar}')
target = placed = None
for dx, dz in [(2, 0), (-2, 0), (0, 2), (0, -2), (2, 1), (1, 2), (-2, -1), (-1, -2)]:
at = (p[0] + dx, p[1], p[2] + dz)
here = d('block', x=at[0], y=at[1], z=at[2])['block']
below = d('block', x=at[0], y=at[1] - 1, z=at[2])['block']
if here not in ('minecraft:air', 'minecraft:short_grass') or below in ('minecraft:air', 'minecraft:water'):
continue
# The tree's trunk above the broken log can hide a spot's faces: try the next one.
try:
placed = d('place', x=at[0], y=at[1], z=at[2], item=planks_id)
except Failed as e:
if 'no visible face' in str(e):
continue
raise
target = at
break
if target is None:
raise Failed(f'no free spot beside {p} to place a block')
assert placed['changed'], placed
back = d('block', x=target[0], y=target[1], z=target[2])
assert back['block'] == planks_id, back
d('look_at_block', x=target[0], y=target[1], z=target[2])
shot = d('screenshot', path=str(out / f'{d.name}.png'))
if '_image' in shot:
(out / f'{d.name}.png').write_bytes(shot['_image'])
seen = d('looking_at')
assert seen['kind'] == 'block' and seen['block']['pos'] == list(target), seen
# 6. Drop a plank: on a client of a server (single player's server thread) the server must
# throw it, not only the window's own world, so ask the server how many planks are left.
d('select', slot=planks_id)
planks_before = d('inventory')['counts'].get(planks_id, 0)
d('drop')
d('wait', ticks=5)
found = d('command', command=f'clear @s {planks_id} 0')['chat']
assert any(f'Found {planks_before - 1} ' in line for line in found), (planks_before, found)
return {'tree': base, 'log': log_id, 'placed': target}
def widgets(d):
"""The ui tree as {id: widget}."""
return {w['id']: w for w in d('ui')['widgets']}
def ui_scenario(d, out):
# 1. Create a world through the title menus, by widget id.
tree = widgets(d)
assert tree['singleplayer']['kind'] == 'button', tree
d('click', id='singleplayer')
if 'game' not in widgets(d): # with no worlds, Singleplayer opens Create New World itself
d('click', id='create-new-world')
tree = widgets(d)
assert tree['game']['kind'] == 'button' and any(w['kind'] == 'field' for w in tree.values()), tree
d('type', field='name', replace=True, text='UI World')
d('click', id='create-new-world')
d('wait_for', condition='loaded', timeout=400) # a few frames later, as in the survival scenario
d('wait_ready', seconds=400)
# 2. Options from the pause menu: a slider set by id moves its value and label.
d('pause')
d('click', id='options')
fov = widgets(d)['fov']
assert fov['kind'] == 'slider' and abs(fov['value'] - 0.5) < 0.01, fov
d('set', id='fov', value=1)
moved = widgets(d)['fov']
assert moved['value'] > 0.99 and moved['label'] != fov['label'], moved
d('set', id='fov', value=0.5)
try:
d('click', id='no-such-widget')
raise AssertionError('clicking an unknown id succeeded')
except Failed as e:
assert 'no widget' in str(e), e
try:
d('set', id='done', value=0.5)
raise AssertionError('set accepted a button')
except Failed as e:
assert 'set moves sliders' in str(e), e
d('click', id='done')
assert d('click', id='back-to-game')['screen'] == 'playing'
# 3. The inventory's slots are widgets too, with their stacks.
d('give', item='oak_log', count=2)
d('wait', ticks=2) # single player's server thread gives them
d('open_inventory')
tree = widgets(d)
log = next(w for w in tree.values() if w['kind'] == 'slot' and (w.get('stack') or {}).get('item') == 'minecraft:oak_log')
assert log['stack']['count'] == 2 and 'slot-200' in tree, log
d('click', id=log['id'])
d('click', id='slot-100', right=True)
d('click', id=log['id'])
d('wait', ticks=2)
assert widgets(d)['slot-200']['stack']['item'] == 'minecraft:oak_planks'
d('close')
shot = d('screenshot', path=str(out / f'{d.name}-ui.png'))
if '_image' in shot:
(out / f'{d.name}-ui.png').write_bytes(shot['_image'])
return {'widgets': len(tree)}
def options_scenario(d, out):
# Every option, typed, from the title screen; then set, refused, applied and saved.
options = d('options_get')['options']
assert len(options) > 100, len(options)
assert options['fov'] == {'type': 'int', 'min': 30, 'max': 110, 'value': 70}, options['fov']
assert options['renderClouds']['choices'] == ['false', 'fast', 'true'], options['renderClouds']
assert d('options_set', key='fov', value='90')['option']['value'] == 90
try:
d('options_set', key='fov', value='200')
raise AssertionError('fov 200 was accepted')
except Failed as e:
assert '"max":110' in str(e), e
d('options_set', key='renderClouds', value='fast')
d('options_set', key='key_key.jump', value='key.keyboard.j')
got = d('options_get', keys='fov renderClouds key_key.jump')['options']
assert [got[k]['value'] for k in ('fov', 'renderClouds', 'key_key.jump')] == [90, 'fast', 'key.keyboard.j'], got
saved = (d.cwd / 'options.txt').read_text()
assert 'fov:0.5' in saved and 'renderClouds:"fast"' in saved and 'key_key.jump:key.keyboard.j' in saved, saved
# The options screen shows it: FOV 90 sits at (90 - 30) / 80 of its slider.
d('click', id='options')
assert abs(widgets(d)['fov']['value'] - 0.75) < 0.01
# Applied at once: a GUI scale of 2 on the 1280x720 window makes the GUI 640x360.
d('options_set', key='guiScale', value='2')
assert d('ui')['gui_size'] == [640, 360], 'the GUI scale did not apply'
d('options_set', key='guiScale', value='0')
return {'options': len(options)}
def events_scenario(d, out):
# Waits instead of polling, and the event log of what happened meanwhile.
start = d('events')['last']
d('click', id='singleplayer')
if 'game' not in widgets(d):
d('click', id='create-new-world')
d('type', field='name', replace=True, text='Events')
d('click', id='create-new-world')
d('wait_for', condition='loaded', timeout=400)
seq = d('events')['last'] # an event can happen before the wait starts
d('give', item='oak_log', count=3)
d('wait_for', condition='event', value='item', timeout=5, since=seq)
seq = d('events')['last']
d('command', command='say hello agents')
d('wait_for', condition='chat', value='hello agents', timeout=5, since=seq)
d('open_inventory')
d('wait_for', condition='screen', value='inventory', timeout=5)
d('close')
seq = d('events')['last']
d('command', command='kill @s')
d('wait_for', condition='event', value='death', timeout=10, since=seq)
d('wait', ticks=25) # the death screen's buttons wake after 20 ticks
d('respawn')
d('wait_for', condition='event', value='respawn', timeout=10, since=seq)
try:
d('wait_for', condition='screen', value='advancements', timeout=0.5)
raise AssertionError('the wait did not time out')
except Failed as e:
assert 'timed out' in str(e) and 'timeout' in str(e), e
events = d('events', since=start)['events']
kinds = [e['event'] for e in events]
for kind in ['world', 'screen', 'item', 'chat', 'death', 'respawn']:
assert kind in kinds, f'no {kind} event in {kinds}'
logs = next(e for e in events if e['event'] == 'item' and e['item'] == 'minecraft:oak_log')
assert logs['delta'] == 3, logs
assert all(b['seq'] > a['seq'] for a, b in zip(events, events[1:]))
return {'events': len(events)}
def consume_scenario(d, out):
# Consumables finish as vanilla's Consumable.onConsume: always-edible food at full hunger with its
# effects, milk and honey (no food or drink sounds) with their effects and use remainders, and
# chorus fruit's random teleport.
d('click', id='singleplayer')
if 'game' not in widgets(d):
d('click', id='create-new-world')
d('type', field='name', replace=True, text='Consume')
d('click', id='create-new-world')
d('wait_for', condition='loaded', timeout=400)
def consume(item, seconds):
d('command', command='clear @s')
d('give', item=item, count=1)
d('use_item', hold=seconds)
d('wait', ticks=5)
state = d('state')
return state, {e['id'] for e in state['player']['effects']}
def held(state):
return state['held'] and state['held']['item']
state, effects = consume('golden_apple', 2.2)
assert state['player']['food'] == 20 and {'regeneration', 'absorption'} <= effects, effects
assert held(state) is None, state['held']
d('command', command='effect give @s minecraft:speed 60')
state, effects = consume('milk_bucket', 2.2)
assert effects == set(), effects
assert held(state) == 'minecraft:bucket', state['held']
d('command', command='effect give @s minecraft:poison 60')
d('command', command='effect give @s minecraft:speed 60')
state, effects = consume('honey_bottle', 2.6)
assert effects == {'speed'}, effects
assert held(state) == 'minecraft:glass_bottle', state['held']
before = d('state')['player']['position']
state, _ = consume('chorus_fruit', 2.2)
after = state['player']['position']
moved = [b - a for a, b in zip(before, after)]
assert held(state) is None, state['held']
assert moved != [0, 0, 0] and abs(moved[0]) <= 8 and abs(moved[2]) <= 8, moved
d('screenshot', path=str(out / f'consume-{d.name}.png'))
# EnderEyeItem.use: startUsingItem (getUseDuration 0) whether or not a stronghold is found, so a held
# button keeps the eye in use, slowing the player, and throws at most one eye until it lets go.
d('command', command='clear @s')
d('give', item='ender_eye', count=3)
d('command', command='tp @s ~ ~ ~ ~ -90')
d('mouse', button='right', action='down')
d('wait', ticks=12)
eye = d('state')
d('mouse', button='right', action='up')
d('wait', ticks=3)
released = d('state')
assert eye['using'] is not None, eye['using']
assert held(eye) == 'minecraft:ender_eye' and eye['held']['count'] >= 2, eye['held']
assert released['using'] is None, released['using']
# LocalPlayer.isSprintingPossible: blindness (Player.isMobilityRestricted) keeps the sprint key
# from starting a sprint, and the client reads it from the effect the server sent (#4831).
d('command', command='clear @s')
x, y, z = d('state')['player']['block_pos']
# A stone tunnel, so no fluid or falling block reaches the strip.
for box, block in [((x - 2, y - 1, z, x + 2, y + 4, z + 40), 'stone'),
((x - 1, y, z, x + 1, y + 3, z + 40), 'air')]:
reply = d('command', command='fill {} {} {} {} {} {} minecraft:{}'.format(*box, block))
assert any('filled' in line for line in reply.get('chat', [])), reply
def hold_sprint():
d('command', command=f'tp @s {x + 0.5} {y} {z + 0.5} 0 0')
d('wait', ticks=10)
d('key', key='w', action='down')
d('key', key='ctrl', action='down')
d('wait', ticks=20)
return d('state')['player']['sprinting']
def sprints():
sprinting = hold_sprint()
d('stop')
return sprinting
assert hold_sprint(), 'the sprint key did not start a sprint'
# The server's effect, sent to the client as ClientboundUpdateMobEffectPacket.
d('command', command='effect give @s minecraft:blindness 60')
d('wait', ticks=5)
assert not d('state')['player']['sprinting'], 'blindness did not stop a running sprint'
d('stop')
assert not sprints(), 'the sprint key started a sprint while blind'
d('command', command='effect clear @s minecraft:blindness')
assert sprints(), 'no sprint once blindness is cleared'
return {'teleported': [round(m, 2) for m in moved], 'eye_use_ticks': eye['using']['ticks']}
def use_screen_scenario(d, out):
"""An item in use is released by the keybind screen that opens over it.
Minecraft.handleKeybinds opens the inventory (and Gui.handleKeybinds the chat) inside the
call whose end releases a running use, after Gui.setScreen's KeyMapping.releaseAll cleared
key.use. With the button still held, opening the screen must therefore release the use in
that tick: the golden apple is not eaten behind the GUI, and no effect is applied. The
second half opens the chat the same way; the last part checks that eating without a screen
still works. This covers the bin's E/T call sites and the cleared held button, which the
interaction-input parity harness (the release sentence itself) cannot see.
"""
# The title screen may still be loading when the game answers the socket.
deadline = time.time() + 120
while 'singleplayer' not in widgets(d):
if time.time() > deadline:
raise Failed('the title screen did not open')
time.sleep(0.5)
d('click', id='singleplayer')
deadline = time.time() + 120
while not ({'game', 'create-new-world'} & set(widgets(d))):
if time.time() > deadline:
raise Failed('the world list did not open')
time.sleep(0.5)
if 'game' not in widgets(d):
d('click', id='create-new-world')
d('type', field='name', replace=True, text='UseScreen')
d('click', id='create-new-world')
d('wait_for', condition='loaded', timeout=400)
d('command', command='gamemode survival')
d('command', command='clear @s')
d('give', item='golden_apple', count=64)
d('select', slot=1)
d('wait', ticks=2)
def apples(state):
held = state['held']
assert held and held['item'] == 'minecraft:golden_apple', held
return held['count']
assert apples(d('state')) == 64
d('mouse', button='right', action='down')
d('wait', ticks=8)
assert d('state')['using'] is not None, 'the use did not start'
d('key', key='e', action='tap')
d('wait', ticks=2)
assert d('state')['screen'] == 'inventory', d('state')['screen']
# Two seconds is longer than the 1.6 s use: without the release the apple is eaten here.
d('wait', ticks=40)
state = d('state')
assert state['using'] is None, state['using']
assert apples(state) == 64, state['held']
assert state['player']['effects'] == [], state['player']['effects']
d('screenshot', path=str(out / f'use-screen-inventory-{d.name}.png'))
# The mouse can only be released with no screen open: close it first.
d('key', key='e', action='tap')
d('wait', ticks=2)
assert d('state')['screen'] == 'playing', d('state')['screen']
d('mouse', button='right', action='up')
# The chat opens from Gui.handleKeybinds, in the same handleKeybinds call.
d('mouse', button='right', action='down')
d('wait', ticks=8)
assert d('state')['using'] is not None, 'the use did not start'
d('key', key='t', action='tap')
d('wait', ticks=40)
state = d('state')
assert state['screen'] == 'chat', state['screen']
assert state['using'] is None, state['using']
assert apples(state) == 64, state['held']
assert state['player']['effects'] == [], state['player']['effects']
d('screenshot', path=str(out / f'use-screen-chat-{d.name}.png'))
d('key', key='esc', action='tap')
d('wait', ticks=2)
assert d('state')['screen'] == 'playing', d('state')['screen']
d('mouse', button='right', action='up')
# Eating with no screen still finishes: the release only happens for the screens above.
d('command', command='clear @s')
d('give', item='golden_apple', count=1)
d('use_item', hold=2.2)
d('wait', ticks=5)
state = d('state')
effects = {e['id'] for e in state['player']['effects']}
assert {'regeneration', 'absorption'} <= effects, effects
assert state['held'] is None, state['held']
return {'apples_after_screen': 64, 'effects': sorted(effects)}
def pick_scenario(d, out):
# The pick block button (middle) over a container screen clones only with infinite materials:
# AbstractContainerScreen.mouseClicked sends CLONE, which fills the empty hand with a full stack
# of the slot's item (#1048: it clicked like the left button and took the slot's stack).
d('click', id='singleplayer')
if 'game' not in widgets(d):
d('click', id='create-new-world')
d('type', field='name', replace=True, text='Pick')
d('click', id='create-new-world')
d('wait_for', condition='loaded', timeout=400)
d('command', command='gamemode creative')
d('command', command='clear @s')
d('give', item='oak_log', count=2)
def carried():
return d('inventory')['carried']
# 1. An item tab: the hotbar slot's stack is cloned, the slot keeps its 2 logs.
assert d('open_inventory')['screen'] == 'creative'
tree = widgets(d)
assert tree['slot-0']['stack'] == {'item': 'minecraft:oak_log', 'count': 2}, tree['slot-0']
d('click', id='slot-0', middle=True)
assert carried() == {'item': 'minecraft:oak_log', 'count': 64}, carried()
assert widgets(d)['slot-0']['stack']['count'] == 2
d('close')
d('command', command='clear @s')
d('give', item='oak_log', count=2)
# 2. The tab's own display slot gives a full stack too, not one item.
d('open_inventory')
shown = widgets(d)['slot-900']['stack']
d('click', id='slot-900', middle=True)
assert carried() == {'item': shown['item'], 'count': 64}, (shown, carried())
d('close')
# 3. Survival: the middle button over a slot does nothing.
d('command', command='gamemode survival')
d('command', command='clear @s')
d('give', item='oak_log', count=2)
assert d('open_inventory')['screen'] == 'inventory'
d('click', id='slot-0', middle=True)
# The clone of step 2 lived only on this client: ItemPickerMenu's carried stack is the
# inventoryMenu's, and AbstractContainerMenu.removed clears it only for a ServerPlayer. With a
# stack in hand, mouseReleased sends CLONE, the server finds the carried stack it was told about
# wrong and sends the empty one back (broadcastChanges). On the server thread that takes a
# round trip, as in vanilla; in the frame, closing the menu already put the clone away.
for _ in range(20):
if carried() is None:
break
d('wait', ticks=1)
assert carried() is None, carried()
assert widgets(d)['slot-0']['stack'] == {'item': 'minecraft:oak_log', 'count': 2}
d('close')
return {'cloned': shown['item']}
def suggestions_scenario(d, out):
# Start from the title screen: single player's client joins through memory,
# without opening a server world and building its command registries.
d('click', id='singleplayer')
if 'game' not in widgets(d):
d('click', id='create-new-world')
d('type', field='name', replace=True, text='Suggestions')
d('click', id='game-mode') # survival -> hardcore -> creative (commands on)
d('click', id='game-mode')
d('type', field='seed', text='5')
d('click', id='world')
d('click', id='world-type') # superflat: this checks chat, without a spawn search
d('click', id='create-new-world')
d('wait_for', condition='loaded', timeout=400)
d('wait_ready', seconds=400)
cases = [
('/giv', '/give'),
('/fill ^', '/fill ^ ^'),
('/give @s diamond_sw', '/give @s minecraft:diamond_sword'),
('/setblock ~ ~ ~ diamond_blo', '/setblock ~ ~ ~ minecraft:diamond_block'),
('/fill ~ ~ ~ ~1 ~1 ~1 diamond_blo', '/fill ~ ~ ~ ~1 ~1 ~1 minecraft:diamond_block'),
('/setblock ~ ~ ~ oak_stairs[fa', '/setblock ~ ~ ~ oak_stairs[facing='),
('/clear @s diamond_sw', '/clear @s minecraft:diamond_sword'),
('/execute run give @s diamond_sw', '/execute run give @s minecraft:diamond_sword'),
]
for text, expected in cases:
d('key', key='t', action='tap')
d('type', text=text)
if text == '/give @s diamond_sw':
shot = d('screenshot', path=str((out / f'{d.name}-suggestions.png').resolve()))
if '_image' in shot:
(out / f'{d.name}-suggestions.png').write_bytes(shot['_image'])
d('key', key='tab', action='tap')
actual = d('screen')['chat_input']
assert actual == expected, (text, expected, actual)
d('key', key='esc', action='tap')
if d('screen')['screen'] == 'chat': # Escape first dismisses suggestions
d('key', key='esc', action='tap')
return {'completions': len(cases)}
def manage_scenario(d, out):
# The commands that close the audit's gaps: status (F3), entities by right-click, effects and xp,
# LAN status, options pages by name, then from the title screen the worlds list, Make Backup,
# rename and a delete that needs the player's confirmation.
d('open', world='Manage', seed=7)
d('wait_for', condition='loaded', timeout=400)
assert d('status')['player'] is not None
d('give', item='bucket')
d('select', slot='bucket')
pos = d('state')['player']['position']
cow = d('summon', kind='cow', x=pos[0], y=pos[1], z=pos[2] + 2)['id']
d('wait', ticks=5)
d('interact_entity', id=cow)
items = {s.get('item') for s in d('inventory')['hotbar'] if s}
assert 'minecraft:milk_bucket' in items, items
d('effect', id='speed', seconds=30, amplifier=1)
effects = json.dumps(d('effects'))
assert 'speed' in effects, effects
d('xp', levels=5)
assert d('lan', action='status') is not None
d('options', page='horizon')
assert 'graphics' in widgets(d), widgets(d).keys()
d('close')
d('pause')
d('click', id='save-and-quit-to-title')
d('wait_for', condition='screen', value='title', timeout=60)
worlds = {w['folder']: w for w in d('worlds')['worlds']}
assert worlds['Manage']['mode'] == 'survival', worlds
assert d('world_backup', folder='Manage')['bytes'] > 0
assert (d.cwd / 'backups').is_dir() and any((d.cwd / 'backups').glob('*_Manage.zip'))
d('world_rename', folder='Manage', name='Renamed Base')
assert {w['folder']: w for w in d('worlds')['worlds']}['Manage']['name'] == 'Renamed Base'
try:
d('world_delete', folder='Manage')
raise AssertionError('deleted without --confirm')
except Failed as e:
assert 'confirm' in str(e), e
assert (d.cwd / 'saves/Manage').is_dir()
d('world_delete', folder='Manage', confirm=True)
assert not (d.cwd / 'saves/Manage').exists()
assert 'Manage' not in {w['folder'] for w in d('worlds')['worlds']}
assert d('servers')['servers'] == []
return {'worlds': len(worlds)}
def pause_save_scenario(d, out):
# IntegratedServer.tickServer: the tick single player pauses saves everything ("Saving and pausing
# game...", #3280), so the world is on disk while the pause menu is open.
d('open', world='Paused', seed=7)
d('wait_for', condition='loaded', timeout=400)
d('wait_ready', seconds=400)
level = d.cwd / 'saves/Paused/level.dat'
d('wait', ticks=40)
playing = level.stat().st_mtime_ns
d('wait', ticks=40)
assert level.stat().st_mtime_ns == playing, 'level.dat was saved without a pause'
d('pause')
deadline = time.time() + 15
while level.stat().st_mtime_ns == playing and time.time() < deadline:
time.sleep(0.25)
saved = level.stat().st_mtime_ns != playing
shot = d('screenshot', path=str(out / f'{d.name}-pause-save.png'))
if '_image' in shot:
(out / f'{d.name}-pause-save.png').write_bytes(shot['_image'])
assert saved, 'pausing single player did not save level.dat'
return {'saved_on_pause': saved}
SCENARIOS = {'survival': scenario, 'ui': ui_scenario, 'options': options_scenario, 'events': events_scenario,
'consume': consume_scenario, 'use-screen': use_screen_scenario, 'pick': pick_scenario,
'suggestions': suggestions_scenario, 'manage': manage_scenario, 'pause-save': pause_save_scenario}
REFUSAL = "AI features are off: turn off No AI in the launcher's settings"
def scratch(mode, resources):
"""A scratch game home with the prepared resources, and the game's environment for it."""
work = Path(tempfile.mkdtemp(prefix=f'hzac-{mode}-'))
(work / '.local').mkdir()
(work / '.local/horizon-client').mkdir()
lab.link_dir(resources, work / '.local/horizon-client' / lab.VERSION)
# A first launch shows the accessibility onboarding before the title (Gui.addInitialScreens): these
# scenarios start at the title, as a player does after it.
(work / 'options.txt').write_text('onboardAccessibility:false\n')
# horizon-ctl reads No AI in the game home: this one, not the player's.
env = game_env(os.environ.get('VSCREEN', 'agent-control'), home=work)
return work, env
def no_ai_check(mode, ctl, resources):
"""No AI on: horizon-ctl refuses (exit 4, kind denied), so does its MCP server, and the game
started with --control-socket never opens it."""
work, env = scratch(mode, resources)
socket = work / 'c.sock'
try:
(work / 'horizon-extras.txt').write_text('fps:true\nnoAi:true\n')
if mode == 'cli':
for words in (['state'], ['launch', '--world', 'NoAi'], ['logs']):
r = subprocess.run([str(ctl), '--socket', str(socket), *words], cwd=work, env=env,
capture_output=True, text=True, timeout=60)
reply = json.loads(r.stdout.strip().splitlines()[-1])
log(mode, words, reply)
assert r.returncode == 4 and reply == {'error': REFUSAL, 'kind': 'denied'}, (r.returncode, reply)
client = ctl.with_name('horizon-client')
args = [str(client), '--control-socket', str(socket), '--world', 'NoAi', '--world-frames', '30']
if sys.platform == 'darwin':
args.append('--offscreen')
r = subprocess.run(args, cwd=work, env=env, capture_output=True, text=True, timeout=600)
text = r.stdout + r.stderr + ''.join(
p.read_text(errors='replace') for p in (work / 'logs').glob('latest.log'))
assert r.returncode == 0, f'game exit {r.returncode}: {text[-2000:]}'
assert 'control socket not opened' in text and REFUSAL in text, text[-2000:]
assert not socket.exists(), 'the game opened its control socket under No AI'
else:
r = subprocess.run([str(ctl), '--socket', str(socket), 'mcp'], cwd=work, env=env, input='',
capture_output=True, text=True, timeout=60)
log(mode, ['mcp'], {'exit': r.returncode, 'stderr': r.stderr.strip()})
assert r.returncode == 4 and r.stderr.strip() == f'horizon-ctl mcp: {REFUSAL}', r
# Turned on during a session: the next call refuses.
(work / 'horizon-extras.txt').write_text('noAi:false\n')
m = Mcp(ctl, socket, work, env)
(work / 'horizon-extras.txt').write_text('noAi:true\n')
try:
m('state')
raise AssertionError('state ran under No AI')
except Failed as e:
assert REFUSAL in str(e) and 'denied' in str(e), e
m.close()
print(f'[{mode}] PASS: no-ai')
return True
except (Failed, AssertionError, json.JSONDecodeError, IndexError, subprocess.TimeoutExpired) as e:
print(f'[{mode}] FAIL: no-ai: {e!r}')
return False
finally:
shutil.rmtree(work, ignore_errors=True)
def setting_check(mode, ctl, out, resources):
"""Agent Control turned on in Horizon Settings > AI Agent: the game the player starts (no
--control) opens the default endpoint, the page shows the agent connected, Copy Connection
Details copies the socket, and turning the switch off closes the endpoint at once."""
work, env = scratch(mode, resources)
socket = work / 'c.sock'
env['HORIZON_CONTROL'] = str(socket) # the default endpoint, this home's
(work / 'horizon-extras.txt').write_text('agentControl:true\n')
args = [str(ctl.with_name('horizon-client'))]
if sys.platform == 'darwin':
args.append('--offscreen')
game = subprocess.Popen(args, cwd=work, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
d = None
try:
deadline = time.time() + 300
while not socket.exists():
assert game.poll() is None, f'the game exited ({game.returncode})'
assert time.time() < deadline, 'Agent Control on, but no control socket'
time.sleep(0.5)
d = (Cli if mode == 'cli' else Mcp)(ctl, socket, work, env)
assert d('state')['screen'] == 'title' # the player's own game, at its title screen
d('options', page='horizon')
tree = widgets(d)
assert {'ai-agent', 'no-ai'} <= set(tree), sorted(tree)
d('screenshot', path=str(out / f'{mode}-horizon-settings.png'))
d('options', page='agent')
tree = widgets(d)
switch = next(w for w in tree.values() if w.get('label', '').startswith('Agent Control'))
assert switch['label'] == 'Agent Control: On', switch
d('screenshot', path=str(out / f'{mode}-ai-agent.png'))
copy = next(w for w in tree.values() if w.get('label') == 'Copy Connection Details')
d('click', id=copy['id'])
# Off: the endpoint goes away and the next command can't reach the game.
try:
d('click', id=switch['id'])
except Failed:
pass # the reply may be lost with the socket
deadline = time.time() + 30
while socket.exists():
assert time.time() < deadline, 'Agent Control off, but the socket stayed'
time.sleep(0.5)
assert 'agentControl:false' in (work / 'horizon-extras.txt').read_text()
assert game.poll() is None, 'the game exited when Agent Control went off'
print(f'[{mode}] PASS: setting')
return True
except (Failed, AssertionError, StopIteration, json.JSONDecodeError, subprocess.TimeoutExpired) as e:
print(f'[{mode}] FAIL: setting: {e!r}')
return False
finally:
if d is not None:
try:
d.close()
except Exception:
pass
game.terminate()
try:
game.wait(30)
except subprocess.TimeoutExpired:
game.kill()
shutil.rmtree(work, ignore_errors=True)
def run(mode, ctl, out, resources, which='survival'):
if which == 'no-ai':
return no_ai_check(mode, ctl, resources)
if which == 'setting':
return setting_check(mode, ctl, out, resources)
work, env = scratch(mode, resources)
socket = work / 'c.sock'
driver = (Cli if mode == 'cli' else Mcp)(ctl, socket, work, env)
started = time.time()
try:
driver('launch', wait=False, offscreen=sys.platform == 'darwin')
summary = SCENARIOS[which](driver, out)
print(f'[{mode}] PASS in {time.time() - started:.0f} s: {summary}')
return True
except (Failed, AssertionError, StopIteration) as e:
print(f'[{mode}] FAIL: {e!r}')
log_file = socket.with_suffix('.log')
if log_file.exists():
print('game log tail:\n' + '\n'.join(log_file.read_text(errors='replace').splitlines()[-15:]))
return False
finally:
try:
driver('quit')
except Exception:
pass
driver.close()
time.sleep(2)
shutil.rmtree(work, ignore_errors=True)
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--mode', choices=['cli', 'mcp', 'both'], default='both')
parser.add_argument('--scenario', choices=[*SCENARIOS, 'no-ai', 'setting', 'all'], default='all')
parser.add_argument('--bin-dir', help='prebuilt horizon-client and horizon-ctl (default: build with the fast profile)')
parser.add_argument('--out', default=str(ROOT / '.local/agent-control-e2e'), help='screenshots go here')
args = parser.parse_args()
if args.bin_dir:
bins = Path(args.bin_dir).resolve()
else:
subprocess.run(['cargo', 'build', '--locked', '--profile', 'fast', '-p', 'horizon-client', '-p', 'horizon-client-lib', '--bins'],
cwd=ROOT, check=True)
meta = json.loads(subprocess.run(['cargo', 'metadata', '--format-version', '1', '--no-deps'], cwd=ROOT,
capture_output=True, text=True, check=True).stdout)
bins = Path(meta['target_directory']) / 'fast'
resources = (lab.RESOURCES).resolve()
if not (resources / 'horizon-resources.json').exists():
sys.exit(f'no prepared resources in {resources}')
out = Path(args.out).resolve()
out.mkdir(parents=True, exist_ok=True)
modes = ['cli', 'mcp'] if args.mode == 'both' else [args.mode]
scenarios = [*SCENARIOS, 'no-ai', 'setting'] if args.scenario == 'all' else [args.scenario]
results = {f'{m}/{s}': run(m, bins / 'horizon-ctl', out, resources, s) for m in modes for s in scenarios}
print(json.dumps(results))
sys.exit(0 if all(results.values()) else 1)
if __name__ == '__main__':
main()