3D engine that AI agents drive
Keep the shadows of closed rooms dark (part of #197)
Nguyen Xuan An33048fff1cf7
diff --git a/bench/reference/baseline.json b/bench/reference/baseline.jsonindex a270563b..3b9fa202 100644--- a/bench/reference/baseline.json+++ b/bench/reference/baseline.json@@ -26,6 +26,10 @@"cycles": 7.7,"eevee": 0.4},+ "dark-shaft": {+ "cycles": 129.3,+ "eevee": 67.2+ },"fog": {"cycles": 58.3,"eevee": 1.4@@ -67,8 +71,8 @@"eevee": 41.9},"mountain-valley": {- "cycles": 30.5,- "eevee": 11.8+ "cycles": 35.9,+ "eevee": 39.2},"outdoor-hdri": {"cycles": 12.3,@@ -81,10 +85,6 @@"temple-hall": {"cycles": 57.7,"eevee": 1.1- },- "dark-shaft": {- "cycles": 129.3,- "eevee": 67.2}},"graphics": {@@ -141,31 +141,31 @@"gpu-vs-cpu": 0.0066},"dark-shaft": {- "gpu-vs-cycles": 0.143,- "gpu-vs-eevee": 0.1831,- "cpu-vs-cycles": 0.1437,- "cpu-vs-eevee": 0.1825,+ "gpu-vs-cycles": 0.0619,+ "gpu-vs-eevee": 0.2808,+ "cpu-vs-cycles": 0.0618,+ "cpu-vs-eevee": 0.2807,"eevee-vs-cycles": 0.2653,- "gpu-vs-cpu": 0.0047,+ "gpu-vs-cpu": 0.0024,"region-pixels": 0.099,- "region-gpu-vs-cycles": 0.1273,- "region-gpu-vs-eevee": 0.2312,+ "region-gpu-vs-cycles": 0.1858,+ "region-gpu-vs-eevee": 0.2997,"region-eevee-vs-cycles": 0.2643},"fog": {- "gpu-vs-cycles": 0.2242,- "gpu-vs-eevee": 0.1079,- "cpu-vs-cycles": 0.2239,- "cpu-vs-eevee": 0.1084,+ "gpu-vs-cycles": 0.2257,+ "gpu-vs-eevee": 0.106,+ "cpu-vs-cycles": 0.2254,+ "cpu-vs-eevee": 0.1066,"eevee-vs-cycles": 0.2662,"gpu-vs-cpu": 0.0089},"foliage": {- "gpu-vs-cycles": 0.129,- "gpu-vs-eevee": 0.091,- "cpu-vs-cycles": 0.1287,- "cpu-vs-eevee": 0.0906,- "eevee-vs-cycles": 0.0864,+ "gpu-vs-cycles": 0.1181,+ "gpu-vs-eevee": 0.0846,+ "cpu-vs-cycles": 0.1178,+ "cpu-vs-eevee": 0.0843,+ "eevee-vs-cycles": 0.0888,"gpu-vs-cpu": 0.01},"glass": {@@ -233,11 +233,11 @@"gpu-vs-cpu": 0.0061},"mountain-valley": {- "gpu-vs-cycles": 0.3478,- "gpu-vs-eevee": 0.2426,- "cpu-vs-cycles": 0.3466,- "cpu-vs-eevee": 0.2412,- "eevee-vs-cycles": 0.1828,+ "gpu-vs-cycles": 0.349,+ "gpu-vs-eevee": 0.2391,+ "cpu-vs-cycles": 0.3478,+ "cpu-vs-eevee": 0.2378,+ "eevee-vs-cycles": 0.1879,"gpu-vs-cpu": 0.0193},"outdoor-hdri": {@@ -257,15 +257,15 @@"gpu-vs-cpu": 0.0032},"temple-hall": {- "gpu-vs-cycles": 0.1436,- "gpu-vs-eevee": 0.1542,- "cpu-vs-cycles": 0.1441,- "cpu-vs-eevee": 0.1537,+ "gpu-vs-cycles": 0.0659,+ "gpu-vs-eevee": 0.2394,+ "cpu-vs-cycles": 0.0659,+ "cpu-vs-eevee": 0.2393,"eevee-vs-cycles": 0.2341,- "gpu-vs-cpu": 0.0039,+ "gpu-vs-cpu": 0.0016,"region-pixels": 0.0958,- "region-gpu-vs-cycles": 0.2305,- "region-gpu-vs-eevee": 0.2681,+ "region-gpu-vs-cycles": 0.2805,+ "region-gpu-vs-eevee": 0.3061,"region-eevee-vs-cycles": 0.2455}}diff --git a/bench/reference/depth-sheets.py b/bench/reference/depth-sheets.pynew file mode 100644index 00000000..c843fdd2--- /dev/null+++ b/bench/reference/depth-sheets.py@@ -0,0 +1,51 @@+# Blind comparison sheets for issue #197: the Cycles image on top, two 67js images below it in random order, labelled only 1 and 2.+# uv run --no-project --python 3.12 --with numpy --with pillow python -I bench/reference/depth-sheets.py <cycles directory> <before directory> <after directory> <output directory> <scene id>,<scene id>+# <cycles directory> holds <id>-cycles.npy, the other two hold <id>-gpu.png. The file key.txt in the output directory says which image is which.+import random+import sys+from pathlib import Path++import numpy as np+from PIL import Image, ImageDraw++GAMMA = 2.2+WIDTH = 960+HEIGHT = 540+LABEL_HEIGHT = 30+++def tone_map_like_67js(linear):+ color = np.maximum(linear, 0)+ mapped = np.clip((color * (2.51 * color + 0.03)) / (color * (2.43 * color + 0.59) + 0.14), 0, 1)+ return mapped ** (1 / GAMMA)+++def labelled(image, label):+ canvas = Image.new("RGB", (image.width, image.height + LABEL_HEIGHT), (24, 24, 24))+ canvas.paste(image, (0, LABEL_HEIGHT))+ ImageDraw.Draw(canvas).text((10, 9), label, fill=(235, 235, 235))+ return canvas+++def main():+ cycles_directory, before_directory, after_directory, output_directory = (Path(argument) for argument in sys.argv[1:5])+ scene_ids = sys.argv[5].split(",")+ output_directory.mkdir(parents=True, exist_ok=True)+ generator = random.Random()+ key_lines = []+ for scene_id in scene_ids:+ cycles = Image.fromarray((tone_map_like_67js(np.load(cycles_directory / f"{scene_id}-cycles.npy")) * 255).astype(np.uint8)).resize((WIDTH, HEIGHT), Image.LANCZOS)+ before = Image.open(before_directory / f"{scene_id}-gpu.png").convert("RGB")+ after = Image.open(after_directory / f"{scene_id}-gpu.png").convert("RGB")+ pair = [("before", before), ("after", after)]+ generator.shuffle(pair)+ sheet = Image.new("RGB", (WIDTH * 2, (HEIGHT + LABEL_HEIGHT) * 2), (24, 24, 24))+ sheet.paste(labelled(cycles, "Cycles"), (WIDTH // 2, 0))+ sheet.paste(labelled(pair[0][1], "1"), (0, HEIGHT + LABEL_HEIGHT))+ sheet.paste(labelled(pair[1][1], "2"), (WIDTH, HEIGHT + LABEL_HEIGHT))+ sheet.save(output_directory / f"{scene_id}.png")+ key_lines.append(f"{scene_id}: 1 = {pair[0][0]}, 2 = {pair[1][0]}")+ (output_directory / "key.txt").write_text("\n".join(key_lines) + "\n")+++main()diff --git a/bench/reference/harness.js b/bench/reference/harness.jsindex c0561f76..7e5e0651 100644--- a/bench/reference/harness.js+++ b/bench/reference/harness.js@@ -151,10 +151,11 @@ async function probeScene(scene, id, drawnParts, mode) {return { parts: placed.parts, report: placed.report, seconds: (performance.now() - started) / 1000 };}-async function renderReferenceScene(id, { describe, ambientOcclusion = 0, ambientOcclusionRadius = 0.5, bake = null, cascades = 0, backgroundOnly = false, probes = false, ambient = null }) {+async function renderReferenceScene(id, { describe, ambientOcclusion = 0, ambientOcclusionRadius = 0.5, bake = null, cascades = 0, backgroundOnly = false, probes = false, ambient = null, fill = null }) {const scene = await SCENES[id]();// A measurement switch: 0 removes the world's light from every surface, to show how much of a picture comes from it.if (ambient !== null) scene.environment = { ...scene.environment, ambientIntensity: ambient };+ if (fill !== null) scene.environment = { ...scene.environment, fillIntensity: fill };const baked = bake ? await bakeScene(scene, bake) : null;const probed = probes ? await probeScene(scene, id, baked ? baked.parts : scene.parts, probes) : null;const canvas = document.querySelector('canvas');diff --git a/bench/reference/images/dark-shaft.jpg b/bench/reference/images/dark-shaft.jpgindex a986a57b..50e597ff 100644Binary files a/bench/reference/images/dark-shaft.jpg and b/bench/reference/images/dark-shaft.jpg differdiff --git a/bench/reference/images/fog.jpg b/bench/reference/images/fog.jpgindex 582f894e..91628a7a 100644Binary files a/bench/reference/images/fog.jpg and b/bench/reference/images/fog.jpg differdiff --git a/bench/reference/images/foliage.jpg b/bench/reference/images/foliage.jpgindex cc324d18..d450374f 100644Binary files a/bench/reference/images/foliage.jpg and b/bench/reference/images/foliage.jpg differdiff --git a/bench/reference/images/mountain-valley.jpg b/bench/reference/images/mountain-valley.jpgindex 187b8bdf..5d74de43 100644Binary files a/bench/reference/images/mountain-valley.jpg and b/bench/reference/images/mountain-valley.jpg differdiff --git a/bench/reference/images/temple-hall.jpg b/bench/reference/images/temple-hall.jpgindex 67e88491..7ddeee85 100644Binary files a/bench/reference/images/temple-hall.jpg and b/bench/reference/images/temple-hall.jpg differdiff --git a/bench/reference/measure-depth.py b/bench/reference/measure-depth.pynew file mode 100644index 00000000..525d8e89--- /dev/null+++ b/bench/reference/measure-depth.py@@ -0,0 +1,157 @@+# Measures shadow depth: how dark the darks are, how far a surface falls from lit to shadowed, how wide shadow edges are and how much+# a scene darkens where objects meet other surfaces. It compares a 67js render with the Cycles render of the same scene (issue #197).+# uv run --no-project --python 3.12 --with numpy --with pillow --with scipy python -I bench/reference/measure-depth.py <work directory> <scene id>,<scene id> [--candidate <directory with 67js pngs>] [--json <file>]+# Images are compared in display space after the tone curve of src/post.js, the same space score.py uses.+# All measures are on luminance. The surface and edge measures are proxies: the renders carry no object ids, so a surface is a+# tile of one hue, a shadow edge is a luminance step of one stop or more, and contact darkening is read against the neighbourhood.+import json+import sys+from pathlib import Path++import numpy as np+from PIL import Image+from scipy import ndimage++GAMMA = 2.2+LUMA = np.array([0.2126, 0.7152, 0.0722])+PERCENTILES = (1, 5, 25, 50)+TILE = 24+NEIGHBOURHOOD = 31+EDGE_STEP_STOPS = 1.0+FLOOR = 1e-3+KEYS = ["p1", "p5", "p25", "p50", "surface_stops", "edge_median_px", "edge_p90_px", "contact_stops"]+++def tone_map_like_67js(linear):+ color = np.maximum(linear, 0)+ mapped = np.clip((color * (2.51 * color + 0.03)) / (color * (2.43 * color + 0.59) + 0.14), 0, 1)+ return mapped ** (1 / GAMMA)+++def load_cycles(path):+ return tone_map_like_67js(np.load(path))+++def load_png(path):+ return np.asarray(Image.open(path).convert("RGB"), dtype=np.float32) / 255+++def display_luminance(display):+ return (display ** GAMMA) @ LUMA+++def percentile_measures(luminance):+ values = np.percentile(luminance, PERCENTILES)+ return {f"p{percentile}": float(value) for percentile, value in zip(PERCENTILES, values)}+++def surface_contrast(display, luminance):+ """Largest lit to shadow step inside one surface: for each tile of one hue, p95 over p5 luminance in stops; the 90th percentile over tiles."""+ height, width, _ = display.shape+ chroma = display / np.maximum(display.sum(axis=2, keepdims=True), 1e-4)+ stops = []+ for top in range(0, height - TILE + 1, TILE):+ for left in range(0, width - TILE + 1, TILE):+ tile_chroma = chroma[top:top + TILE, left:left + TILE].reshape(-1, 3)+ if tile_chroma.std(axis=0).max() > 0.02:+ continue+ tile = luminance[top:top + TILE, left:left + TILE].ravel()+ high, low = np.percentile(tile, [95, 5])+ if high < 0.02:+ continue+ stops.append(np.log2((high + FLOOR) / (low + FLOOR)))+ if not stops:+ return 0.0+ return float(np.percentile(stops, 90))+++def edge_widths(luminance):+ """Width in pixels of horizontal luminance steps of one stop or more, as the run of pixels between 10 and 90 percent of the step."""+ logged = np.log2(luminance + FLOOR)+ gradient = ndimage.gaussian_filter1d(logged, 0.7, axis=1, order=1)+ widths = []+ height, width = logged.shape+ for row in range(0, height, 2):+ line = logged[row]+ slope = np.abs(gradient[row])+ peaks = np.where((slope[1:-1] > slope[:-2]) & (slope[1:-1] >= slope[2:]) & (slope[1:-1] > 0.15))[0] + 1+ for peak in peaks:+ window = line[max(peak - 40, 0):min(peak + 40, width)]+ step = window.max() - window.min()+ if step < EDGE_STEP_STOPS:+ continue+ low_level = window.min() + 0.1 * step+ high_level = window.min() + 0.9 * step+ inside = np.count_nonzero((window > low_level) & (window < high_level))+ if inside:+ widths.append(inside)+ if not widths:+ return {"edges": 0, "median": 0.0, "p90": 0.0}+ return {"edges": len(widths), "median": float(np.median(widths)), "p90": float(np.percentile(widths, 90))}+++def contact_darkening(luminance):+ """Stops below the neighbourhood mean (in stops) at the darkest 2 percent of pixels: creases and contact shadows read as a dip."""+ logged = np.log2(luminance + FLOOR)+ neighbourhood = ndimage.uniform_filter(logged, size=NEIGHBOURHOOD, mode="nearest")+ return float(-np.percentile(logged - neighbourhood, 2))+++def measure(display):+ luminance = display_luminance(display)+ edge = edge_widths(luminance)+ return {+ **percentile_measures(luminance),+ "surface_stops": surface_contrast(display, luminance),+ "edge_median_px": edge["median"],+ "edge_p90_px": edge["p90"],+ "contact_stops": contact_darkening(luminance),+ }+++GAP_LABELS = ["p1", "p5", "p25", "p50", "surface", "edge", "contact"]+++def gap_row(cycles, ours):+ """67js minus Cycles: luminance percentiles in stops (positive is lighter than Cycles), surface and contact in stops, edge width in pixels."""+ stops = [float(np.log2((ours[key] + FLOOR) / (cycles[key] + FLOOR))) for key in ("p1", "p5", "p25", "p50")]+ return stops + [ours["surface_stops"] - cycles["surface_stops"], ours["edge_median_px"] - cycles["edge_median_px"], ours["contact_stops"] - cycles["contact_stops"]]+++def main():+ work = Path(sys.argv[1])+ scene_ids = sys.argv[2].split(",")+ candidates = [Path(argument) for argument in sys.argv[sys.argv.index("--candidate") + 1:] if not argument.startswith("--")] if "--candidate" in sys.argv else [work]+ json_path = sys.argv[sys.argv.index("--json") + 1] if "--json" in sys.argv else None+ summary = "--summary" in sys.argv+ cycles_measures = {scene_id: measure(load_cycles(work / f"{scene_id}-cycles.npy")) for scene_id in scene_ids}+ results = {}+ for candidate in candidates:+ print(f"== {candidate.name}")+ if summary:+ print(f"{'scene':<20}" + "".join(f"{label:>10}" for label in GAP_LABELS))+ else:+ print(f"{'scene':<20}{'':<8}" + "".join(f"{key:>15}" for key in KEYS))+ gaps = []+ for scene_id in scene_ids:+ cycles = cycles_measures[scene_id]+ ours = measure(load_png(candidate / f"{scene_id}-gpu.png"))+ results.setdefault(candidate.name, {})[scene_id] = {"cycles": cycles, "67js": ours}+ if summary:+ gap = gap_row(cycles, ours)+ gaps.append(gap)+ if len(candidates) == 1:+ print(f"{scene_id:<20}" + "".join(f"{value:>10.2f}" for value in gap))+ continue+ print(f"{scene_id:<20}{'cycles':<8}" + "".join(f"{cycles[key]:>15.4f}" for key in KEYS))+ print(f"{'':<20}{'67js':<8}" + "".join(f"{ours[key]:>15.4f}" for key in KEYS))+ if summary:+ mean = np.mean(np.abs(np.array(gaps)), axis=0)+ print(f"{'mean absolute gap':<20}" + "".join(f"{value:>10.2f}" for value in mean))+ signed = np.mean(np.array(gaps), axis=0)+ print(f"{'mean signed gap':<20}" + "".join(f"{value:>10.2f}" for value in signed))+ if json_path:+ Path(json_path).write_text(json.dumps(results, indent=1))+++main()diff --git a/bench/reference/render-gpu.mjs b/bench/reference/render-gpu.mjsnew file mode 100644index 00000000..250766a4--- /dev/null+++ b/bench/reference/render-gpu.mjs@@ -0,0 +1,52 @@+// Draws chosen reference scenes with 67js on the GPU only, into a directory of your choice, with measurement switches. No Blender, no scoring.+// node bench/reference/render-gpu.mjs <output directory> <scene id>,<scene id> [--ambient <scale>] [--fill <scale>]+// [--ambient-occlusion <strength>] [--ambient-occlusion-radius <units>]+// Files are <output directory>/<scene id>-gpu.png, the names measure-depth.py and score.py read. Used to find what makes shadows light (issue #197).+import { mkdir, writeFile } from 'node:fs/promises';+import { constants, setPriority } from 'node:os';+import { resolve } from 'node:path';+import { chromium } from 'playwright';+import { createServer } from 'vite';++const CHROME = process.env.CHROME_PATH ?? 'C:/Program Files/Google/Chrome/Application/chrome.exe';+const WIDTH = 960;+const HEIGHT = 540;+const argumentsGiven = process.argv.slice(2);+const outputDirectory = resolve(argumentsGiven[0]);+const sceneIds = argumentsGiven[1].split(',');+// Ids of the grand options end in -a, -b or -c and live in their own scene set.+const setOf = id => (/-[abc]$/.test(id) ? 'grand' : null);++function numberAfter(flag) {+ const index = argumentsGiven.indexOf(flag);+ return index >= 0 ? Number(argumentsGiven[index + 1]) : null;+}++const options = {+ ambient: numberAfter('--ambient'),+ fill: numberAfter('--fill'),+ ambientOcclusion: numberAfter('--ambient-occlusion') ?? 0,+ ambientOcclusionRadius: numberAfter('--ambient-occlusion-radius') ?? 0.5,+};++setPriority(0, constants.priority.PRIORITY_BELOW_NORMAL);+await mkdir(outputDirectory, { recursive: true });+const server = await createServer({ server: { host: '127.0.0.1', port: 0 }, logLevel: 'silent' });+await server.listen();+const serverUrl = `http://127.0.0.1:${server.httpServer.address().port}`;+const browser = await chromium.launch({ executablePath: CHROME, headless: true, args: ['--enable-webgl', '--ignore-gpu-blocklist', '--disable-renderer-backgrounding'] });+try {+ const page = await browser.newPage({ viewport: { width: WIDTH, height: HEIGHT } });+ page.on('pageerror', error => console.log('PAGE ERROR', error.message));+ for (const id of sceneIds) {+ await page.goto(`${serverUrl}/bench/reference/harness.html${setOf(id) ? `?set=${setOf(id)}` : ''}`, { timeout: 180000 });+ await page.waitForFunction(() => window.renderReferenceScene, null, { timeout: 180000 });+ const result = await page.evaluate(([sceneId, renderOptions]) => window.renderReferenceScene(sceneId, { describe: false, ...renderOptions }), [id, options]);+ if (result.glError !== 0) throw new Error(`${id} gave WebGL error ${result.glError}.`);+ await writeFile(`${outputDirectory}/${id}-gpu.png`, Buffer.from(result.image, 'base64'));+ console.log(`${id} drawn`);+ }+} finally {+ await browser.close();+ await server.close();+}diff --git a/changes/issue-197.md b/changes/issue-197.mdnew file mode 100644index 00000000..943c492d--- /dev/null+++ b/changes/issue-197.md@@ -0,0 +1 @@+- Keep the shadows of a closed room dark (part of issue #197, which stays open for An's blind pick). A world far dimmer than the key light (under 0.01 of it, fading out by 0.06) now gives a quarter of its light where the sun is blocked, so a sealed hall no longer glows from a sky it cannot see. Against Cycles, `dark-shaft` went from 0.143 to 0.062 FLIP and `temple-hall` from 0.144 to 0.066, the three `temple-hall` options from 0.253, 0.364 and 0.275 to 0.157, 0.152 and 0.102, and the darkest 1 percent of the grand and dark scenes from 1.16 stops too light to 0.68 stops. Outdoor scenes did not move. `bench/reference/measure-depth.py` measures the darks, `render-gpu.mjs` draws chosen scenes with measurement switches and `depth-sheets.py` makes the blind sheets.diff --git a/src/shaders.js b/src/shaders.jsindex 5bfa8a60..796cb2db 100644--- a/src/shaders.js+++ b/src/shaders.js@@ -868,6 +868,10 @@ ${featureCode("linkArea")} vec3 lightNormal = areaLightNormals[index].xyz;float occlusion = 1.0;if (USES_OCCLUSION_MAP) occlusion = mix(1.0, texture(occlusionMap, mapUV(hasOcclusionMap)).r, occlusionStrength);occlusion *= splatCavity;+ if (ENVIRONMENT_ENABLED) {+ float room = 1.0 - smoothstep(0.01, 0.06, dot(environmentDiffuse(vec3(0, 1, 0)), vec3(1)) / max(dot(keyColor, vec3(1)), 0.003));+ occlusion *= mix(1.0, mix(0.25, 1.0, visibility), room * (1.0 - metallic));+ }// Leaf and grass cards keep a faint sheen; full sky reflection washed grazing cards grey-blue.float skyReflection = canopyShaded ? 0.3 : 1.0;if (ENVIRONMENT_ENABLED) {