Minecraft: Java Edition 26.3 rewritten from scratch in Rust, checked against vanilla with 281 differential harnesses. Get it with the Horizon launcher.
Rust differential-testinggame-enginegamedevlinuxminecraftminecraft-javareimplementationrustvoxelvulkanwgpu
import json
import sys
import unittest
from tools.perf_bench import HISTOGRAM_EDGES_MS, OPTIONS, frames, rnd, summary
class ComparisonConditionsTests(unittest.TestCase):
def test_rejects_incompatible_or_unknown_conditions_before_printing(self):
import contextlib
import io
import tempfile
from pathlib import Path
from unittest import mock
import tools.perf_bench as pb
conditions = {"scenario": "flight", "seed": "1234", "render_distance": 8, "simulation_distance": 12,
"window": [1920, 1080], "vsync": False, "offscreen": False}
differences = {"scenario": "other", "seed": "5678", "simulation_distance": 8, "window": [1280, 720],
"vsync": True, "offscreen": True, "fps_cap": 60}
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
reports = root / "benchmarks/perf"
reports.mkdir(parents=True)
def write(label, values):
(reports / f"{label}-rd8.json").write_text(
json.dumps({"conditions": values, "metrics": {}}), encoding="utf-8")
write("first", conditions)
with mock.patch.object(pb, "ROOT", root):
for key, value in differences.items():
with self.subTest(key=key):
write("second", dict(conditions, **{key: value}))
out = io.StringIO()
with contextlib.redirect_stdout(out), self.assertRaisesRegex(ValueError, key):
pb.table(["first", "second"])
self.assertEqual(out.getvalue(), "")
incomplete = dict(conditions)
del incomplete["window"]
write("second", incomplete)
with contextlib.redirect_stdout(io.StringIO()), self.assertRaisesRegex(ValueError, "window"):
pb.table(["first", "second"])
stderr, stdout = io.StringIO(), io.StringIO()
with mock.patch("sys.argv", ["perf_bench.py", "--table", "first", "second"]), \
contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr), \
self.assertRaises(SystemExit) as exit_error:
pb.main()
self.assertEqual(exit_error.exception.code, 2)
self.assertEqual(stdout.getvalue(), "")
self.assertIn("window missing in second", stderr.getvalue())
self.assertNotIn("Traceback", stderr.getvalue())
write("second", dict(conditions, fps_cap=None))
with contextlib.redirect_stdout(io.StringIO()), self.assertRaisesRegex(ValueError, "fps_cap"):
pb.table(["first", "second"])
def test_matching_reports_and_single_incomplete_report_remain_readable(self):
import contextlib
import io
import tempfile
from pathlib import Path
from unittest import mock
import tools.perf_bench as pb
conditions = {"scenario": "flight", "seed": "1234", "render_distance": 8, "simulation_distance": 12,
"window": [1920, 1080], "vsync": False, "offscreen": False}
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
reports = root / "benchmarks/perf"
reports.mkdir(parents=True)
for label, values in (("first", conditions), ("second", conditions),
("single", {"render_distance": 8})):
(reports / f"{label}-rd8.json").write_text(
json.dumps({"conditions": values, "metrics": {}}), encoding="utf-8")
with mock.patch.object(pb, "ROOT", root), contextlib.redirect_stdout(io.StringIO()) as out:
pb.table(["first", "second"])
pb.table(["single"])
self.assertIn("| 8 |", out.getvalue())
class PerfBenchStatsTests(unittest.TestCase):
def test_lows_average_the_slowest_frames(self):
times = [10.0] * 990 + [100.0] * 9 + [1000.0]
s = frames("village", 0, times)
self.assertEqual(s["frames"], 1000)
self.assertEqual(s["avg_fps"], rnd(1000 * 1000 / (9900 + 900 + 1000)))
# Slowest 1%: 9 frames of 100 ms and one of 1000 ms.
self.assertEqual(s["low_1pct_fps"], rnd(1000 * 10 / 1900))
# Slowest 0.1%: the single 1000 ms frame.
self.assertEqual(s["low_01pct_fps"], 1.0)
self.assertEqual((s["p50_ms"], s["p99_ms"], s["p999_ms"], s["max_ms"]), (10.0, 10.0, 100.0, 1000.0))
self.assertEqual(sum(s["histogram"]), 1000)
self.assertEqual(len(s["histogram"]), len(HISTOGRAM_EDGES_MS))
self.assertEqual(s["histogram"][HISTOGRAM_EDGES_MS.index(10)], 990)
self.assertEqual(s["histogram"][HISTOGRAM_EDGES_MS.index(100)], 9)
self.assertEqual(s["histogram"][-1], 1, "1000 ms and over")
def test_same_figures_as_horizon_bench(self):
# crates/horizon-bench's frame_figures_and_histogram test, value for value.
s = frames("flight", 0, [2.0] * 998 + [20.0, 1500.0])
self.assertEqual((s["seconds"], s["low_1pct_fps"], s["low_01pct_fps"]), (3.516, 6.51, 0.667))
self.assertEqual((s["histogram"][2], s["histogram"][13], s["histogram"][23]), (998, 1, 1))
self.assertNotIn("cpu_p50_ms", s)
r = summary([2.0, 4.0, 6.0])
self.assertEqual(r, {"n": 3, "mean": 4.0, "stdev": 2.0, "min": 2.0, "max": 6.0, "ci95": 4.969})
self.assertEqual(summary([5.0])["ci95"], None)
self.assertIsNone(summary([None]))
def test_a_short_flight_still_has_lows(self):
s = frames("x", 0, [5.0, 20.0])
self.assertEqual((s["low_1pct_fps"], s["low_01pct_fps"]), (50.0, 50.0))
self.assertIsNone(frames("x", 0, []))
json.dumps(s)
def test_window_sizes_name_their_reports(self):
from tools.perf_bench import SIZES, parse_sizes, size_label
self.assertEqual(parse_sizes("1280x720,1920X1080"), [(1280, 720), (1920, 1080)])
self.assertEqual(SIZES, [(1280, 720), (1920, 1080)])
self.assertEqual(size_label("horizon-m4", 1280, 720), "horizon-m4")
self.assertEqual(size_label("horizon-m4", 1920, 1080), "horizon-m4-1080p")
self.assertEqual(size_label("horizon-m4", 2560, 1080), "horizon-m4-2560x1080")
def test_one_options_file_sets_the_render_distance(self):
self.assertEqual(OPTIONS["renderDistance"].format(distance=16), "16")
self.assertEqual(OPTIONS["graphicsPreset"], '"custom"')
def test_table_distinguishes_missing_scene_load_from_zero(self):
import contextlib
import io
import tempfile
from pathlib import Path
from unittest import mock
import tools.perf_bench as pb
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
reports = root / "benchmarks" / "perf"
reports.mkdir(parents=True)
# Comparable runs: table() refuses reports whose conditions differ or are missing.
conditions = {"scenario": "flight", "seed": "1234", "render_distance": 8, "simulation_distance": 12,
"window": [1920, 1080], "vsync": False, "offscreen": False}
for label, metrics in (
("missing", {}),
("zero", {"chunks.village.load_ms": {"mean": 0}}),
("measured", {"chunks.village.load_ms": {"mean": 1000},
"chunks.forest.load_ms": {"mean": 3000}}),
):
(reports / f"{label}-rd8.json").write_text(
json.dumps({"conditions": conditions, "metrics": metrics}), encoding="utf-8")
output = io.StringIO()
with mock.patch.object(pb, "ROOT", root), contextlib.redirect_stdout(output):
pb.table(["missing", "zero", "measured"])
row = next(line for line in output.getvalue().splitlines() if line.startswith("| 8 |"))
self.assertEqual([cell.strip().split(", ")[1] for cell in row.split("|")[2:-1]], ["–", "0.0", "2.0"])
if __name__ == "__main__":
unittest.main()
class WaitLoadedTests(unittest.TestCase):
"""wait_loaded on a scripted game with a fake clock: one status per 50 ms poll."""
def run_script(self, statuses, min_chunks=0):
import tools.perf_bench as pb
clock = [0.0]
it = iter(statuses)
def control(_command, timeout=None):
return next(it)
real_time, real_sleep = pb.time.time, pb.time.sleep
pb.time.time = lambda: clock[0]
pb.time.sleep = lambda s: clock.__setitem__(0, clock[0] + s)
try:
return pb.wait_loaded(control, lambda p: True, 60, min_chunks)
finally:
pb.time.time, pb.time.sleep = real_time, real_sleep
@staticmethod
def status(chunks, meshed, stages=None):
s = {"player": {"position": [0, 0, 0]}, "pending_chunks": 0, "screen": "game",
"chunks": chunks, "generated_chunks": chunks, "meshed_sections": meshed}
if stages is not None:
s["streaming"] = {"queued": 0, "generating": 0, "protos": 296, "decorating": 0,
"finishing": 0, "lighting": 0, "meshing": stages}
return s
def test_stages_end_the_load_before_the_remesh_tail(self):
# 20 polls (1 s) of meshing work, then idle stages while blocks keep remeshing
# at 400 meshes/s, with a short burst of work every 2 s.
loading = [self.status(900, 1000 * i, stages=500) for i in range(20)]
tail = [self.status(901, 20000 + 20 * i, stages=40 if i % 40 == 0 else 0) for i in range(400)]
self.assertAlmostEqual(self.run_script(loading + tail), 1.0, places=6)
def test_old_builds_wait_for_meshing_to_go_quiet(self):
busy = [self.status(901, 1000 * i) for i in range(20)]
quiet = [self.status(901, 20000 + i) for i in range(40)]
self.assertAlmostEqual(self.run_script(busy + quiet), 1.0, places=6)
def test_a_quiet_start_is_not_a_load(self):
# Right after a teleport Horizon reports steady counts and nothing queued for 1.5 s before it loads:
# with a target the clock runs on until the chunks are there and the meshing has gone quiet.
idle = [self.status(3, 0, stages=0) for _ in range(30)]
loading = [self.status(30 * (i + 1), 500 * i, stages=500) for i in range(20)]
tail = [self.status(600, 10000, stages=0) for _ in range(60)]
took = self.run_script(idle + loading + tail, min_chunks=500)
self.assertGreater(took, 2.4)
self.assertLess(took, 4.0)
# Without one (the old call), the quiet start ends the wait at once.
self.assertLess(self.run_script(idle + loading + tail), 1.6)
class PortabilityTests(unittest.TestCase):
"""What the driver needs from the OS also works where it differs (Windows: no ps, no load average,
no Unix sockets in Python, nvidia-smi without per-process memory)."""
def test_resident_memory_of_this_process(self):
import os
from tools.perf_bench import rss_bytes
self.assertGreater(rss_bytes(os.getpid()), 1 << 20)
def test_load_average_is_a_number_or_absent(self):
from tools.perf_bench import load_average
self.assertTrue(load_average() is None or load_average() >= 0)
def test_report_without_a_load_average(self):
import argparse
import tools.perf_bench as pb
run = {"startup": {"title_s": 1.0, "world_s": 5.0}, "memory": {}, "load_before": None,
"scenes": {"village": {"load_s": 2.0, "frame_ms": [5.0] * 100, "meshes_load": 10}}}
args = argparse.Namespace(offscreen=False, settle=2.0, label="x")
r = pb.report(args, 8, [run], {"name": "horizon-client"})
self.assertNotIn("load_avg_1m", r["conditions"])
self.assertEqual(r["metrics"]["fps.village.avg_fps"]["mean"], 200.0)
self.assertEqual(pb.report(args, 8, [dict(run, load_before=1.25)], {})["conditions"]["load_avg_1m"], 1.25)
self.assertTrue(r["hardware"]["os"] and r["hardware"]["cpu_threads"])
def test_nvidia_memory_not_available(self):
from unittest import mock
import subprocess
import tools.perf_bench as pb
out = subprocess.CompletedProcess([], 0, stdout="1234, [N/A]\n4321, 812\n")
with mock.patch.object(pb.shutil, "which", return_value="nvidia-smi"), \
mock.patch.object(pb.subprocess, "run", return_value=out):
self.assertIsNone(pb.nvidia_memory_mb(1234))
self.assertEqual(pb.nvidia_memory_mb(4321), 812.0)
self.assertIsNone(pb.nvidia_memory_mb(99))
@unittest.skipUnless(sys.platform == "win32", "the console's OEM code page is Windows'")
def test_windows_version_reads_the_console_code_page(self):
# pr_ready runs the tests with PYTHONUTF8=1, and `cmd /C ver` answers in the OEM code page:
# decoding it as UTF-8 left `stdout` None (issue #5201).
import os
import subprocess
from pathlib import Path
root = Path(__file__).resolve().parents[1]
out = subprocess.run(
[sys.executable, "-c",
"from tools.perf_bench import windows_version; print(windows_version())"],
capture_output=True, text=True, encoding="utf-8", errors="replace", check=True,
env={**os.environ, "PYTHONUTF8": "1"}, cwd=root)
self.assertIn("Windows", out.stdout)
def test_control_over_the_tcp_endpoint(self):
# The game's TCP transport (Windows, or an endpoint path ending in .json): the file names a port and
# a token, and the first line must be `auth TOKEN`.
import socket
import tempfile
import threading
from pathlib import Path
from tools.perf_bench import Control, control_path
self.assertIn(control_path().suffix, (".sock", ".json"))
server = socket.create_server(("127.0.0.1", 0))
lines = []
def serve():
for _ in range(2):
conn, _ = server.accept()
with conn, conn.makefile("rwb") as f:
auth = f.readline().decode().strip()
lines.append(auth)
if auth != "auth secret":
f.write(b'{"error":"bad or missing token"}\n')
f.flush()
continue
f.write(b'{"ok":true}\n')
f.flush()
lines.append(f.readline().decode().strip())
f.write(b'{"screen":"title"}\n')
f.flush()
thread = threading.Thread(target=serve, daemon=True)
thread.start()
with tempfile.TemporaryDirectory() as tmp, server:
path = Path(tmp) / "control.json"
port = server.getsockname()[1]
path.write_text(json.dumps({"transport": "tcp", "port": port, "token": "secret"}))
self.assertEqual(Control(path)("status", timeout=10), {"screen": "title"})
path.write_text(json.dumps({"transport": "tcp", "port": port, "token": "wrong"}))
with self.assertRaises(OSError):
Control(path)("status", timeout=10)
thread.join(10)
self.assertEqual(lines, ["auth secret", "status", "auth wrong"])
def test_a_release_build_reports_its_own_version(self):
import tempfile
from pathlib import Path
from tools.perf_bench import release_version
with tempfile.TemporaryDirectory() as tmp:
log = Path(tmp) / "horizon-rd8.log"
log.write_text("Horizon v0.2.97\nload assets: 1.29s\n")
self.assertEqual(release_version(log), "v0.2.97")
log.write_text("Horizon nightly-2026-10-28-0c02067\n")
self.assertEqual(release_version(log), "nightly-2026-10-28-0c02067")
for dev in ("Horizon 0.1.0\n", "Horizon v0.2.97 (client 0.1.0)\n"):
log.write_text(dev)
self.assertIsNone(release_version(log))
self.assertIsNone(release_version(Path(tmp) / "missing.log"))
def test_vanilla_gets_the_manifest_jvm_flags(self):
from tools.perf_bench import manifest_jvm_flags
manifest = {"arguments": {"jvm": [
{"rules": [{"action": "allow", "os": {"name": "osx"}}], "value": ["-XstartOnFirstThread"]},
{"rules": [{"action": "allow", "os": {"arch": "x86"}}], "value": "-Xss1M"},
"-XX:StackShadowPages=32", "--enable-native-access=ALL-UNNAMED",
"-Djava.library.path=${natives_directory}/java", "-cp", "${classpath}"]}}
self.assertEqual(manifest_jvm_flags(manifest), ["-XX:StackShadowPages=32"])
self.assertEqual(manifest_jvm_flags({}), [])