3D engine that AI agents drive
WIP #506: reflection measurement switches (--no-glossy, material ids, reflection-share.py) and grid glossy reflection of bounced light (unverified on a/b/c)
Nguyen Xuan An655c9e44b1b3
diff --git a/bench/reference/harness.js b/bench/reference/harness.jsindex 15445b0e..666446f4 100644--- a/bench/reference/harness.js+++ b/bench/reference/harness.js@@ -66,6 +66,12 @@ const WATER_REFLECTION = { off: false, ssr: 'ssr' }[new URLSearchParams(locationconst FLOOR_SKY_PARAMETER = new URLSearchParams(location.search).get('floorSky');const FLOOR_SKY_EXPERIMENT = FLOOR_SKY_PARAMETER !== null && FLOOR_SKY_PARAMETER !== '0';const floorSkyReports = {};+// ?glossy=0 is a measurement switch (issue #506): every material loses its metallic and specular, so the frame shows diffuse light and emission only.+const GLOSSY_OFF = new URLSearchParams(location.search).get('glossy') === '0';++function removeGlossy(scene) {+ for (const part of scene.parts) part.material = { ...part.material, metallic: 0, specular: 0, clearcoat: 0 };+}function withFloorSky(id, parts, scene) {const environment = FLOOR_SKY_PARAMETER === '0' ? { ...scene.environment, floorSkyTrace: false } : scene.environment;@@ -229,6 +235,7 @@ async function probeScene(scene, id, drawnParts, mode) {async function renderReferenceScene(id, { describe, ambientOcclusion = 0, ambientOcclusionRadius = 0.5, bake = null, cascades = 0, backgroundOnly = false, probes = false, ambient = null, fill = null, key = null, viewTransform = null, bloom = null }) {const scene = await SCENES[id]();+ if (GLOSSY_OFF) removeGlossy(scene);// A view transform (agx, neutral) replaces the ACES fit in the shaders, as environment.viewTransform does in a recipe.if (viewTransform) scene.environment = { ...scene.environment, viewTransform };// A measurement switch: 0 removes the world's light from every surface, to show how much of a picture comes from it.diff --git a/bench/reference/reflection-share.py b/bench/reference/reflection-share.pynew file mode 100644index 00000000..3e6ede60--- /dev/null+++ b/bench/reference/reflection-share.py@@ -0,0 +1,137 @@+# How much of a Cycles frame is reflection, and how far 67js is from the frame with and without it (issue #506).+# uv run --no-project --python 3.12 --with flip-evaluator --with pillow --with numpy python -I bench/reference/reflection-share.py \+# <cycles directory> <frames directory> <output directory> <scene id>,<scene id> [--labels a,b]+# The Cycles directory holds <id>-cycles.npy, and optionally <id>-cycles-noglossy.npy (render-blender.py --no-glossy) and <id>-materials.npy and+# <id>-materials.json (--material-ids). The frames directory holds the 67js frames as <id>-<label>.png for each label (default before,after; a label may also be a subdirectory holding <id>.png).+# Writes <output>/metrics.json and one sheet per scene: each 67js frame, Cycles without glossy, Cycles full.+import json+import sys+from pathlib import Path++import flip_evaluator+import numpy as np+from PIL import Image, ImageDraw++GAMMA = 2.2+PANEL_WIDTH = 480+PANEL_HEIGHT = 270+LABEL_HEIGHT = 22+LUMINANCE_WEIGHTS = np.array([0.2126, 0.7152, 0.0722], dtype=np.float32)+++def tone_map_like_67js(linear):+ color = np.maximum(linear, 0)+ mapped = np.clip((color * (2.51 * color + 0.03)) / (color * (2.43 * color + 0.59) + 0.14), 0, 1)+ return mapped ** (1 / GAMMA)+++def load_png(path):+ return np.asarray(Image.open(path).convert("RGB"), dtype=np.float32) / 255+++def flip_score(reference, test):+ _, mean_error, _ = flip_evaluator.evaluate(reference.astype(np.float32), test.astype(np.float32), "LDR", inputsRGB=True)+ return float(mean_error)+++def frame_statistics(display):+ luminance = display @ LUMINANCE_WEIGHTS+ linear = display ** GAMMA+ return {+ "mean": round(float(luminance.mean()), 4),+ "p20": round(float(np.percentile(luminance, 20)), 4),+ "rb": round(float(linear[:, :, 0].sum() / max(linear[:, :, 2].sum(), 1e-9)), 3),+ }+++def reflection_by_material(full, without, cycles_directory, scene_id):+ """Signed and absolute linear luminance the reflections add, per material, as a share of the whole frame's luminance."""+ difference = (full - without) @ LUMINANCE_WEIGHTS+ total = float((full @ LUMINANCE_WEIGHTS).sum())+ result = {+ "reflectionShareSigned": round(float(difference.sum()) / total, 4),+ "reflectionShareAbsolute": round(float(np.abs(difference).sum()) / total, 4),+ }+ ids_path = cycles_directory / f"{scene_id}-materials.npy"+ if not ids_path.exists():+ return result+ identifiers = np.load(ids_path)+ names = ["sky"] + json.loads((cycles_directory / f"{scene_id}-materials.json").read_text())+ per_material = {}+ for index, name in enumerate(names):+ mask = identifiers == index+ if not mask.any():+ continue+ per_material[name] = {+ "pixels": round(float(mask.mean()), 4),+ "frameLuminance": round(float((full @ LUMINANCE_WEIGHTS)[mask].sum()) / total, 4),+ "reflectionSigned": round(float(difference[mask].sum()) / total, 4),+ "reflectionAbsolute": round(float(np.abs(difference[mask]).sum()) / total, 4),+ }+ result["byMaterial"] = per_material+ return result+++def panel(display, label):+ image = Image.fromarray(np.round(np.clip(display, 0, 1) * 255).astype(np.uint8)).resize((PANEL_WIDTH, PANEL_HEIGHT), Image.LANCZOS)+ canvas = Image.new("RGB", (PANEL_WIDTH, PANEL_HEIGHT + LABEL_HEIGHT), (20, 20, 20))+ canvas.paste(image, (0, LABEL_HEIGHT))+ ImageDraw.Draw(canvas).text((6, 5), label, fill=(235, 235, 235))+ return canvas+++def sheet(panels):+ canvas = Image.new("RGB", (PANEL_WIDTH * len(panels), PANEL_HEIGHT + LABEL_HEIGHT))+ for index, item in enumerate(panels):+ canvas.paste(item, (index * PANEL_WIDTH, 0))+ return canvas+++def main():+ arguments = sys.argv[1:]+ labels = ["before", "after"]+ if "--labels" in arguments:+ flag_index = arguments.index("--labels")+ labels = arguments[flag_index + 1].split(",")+ del arguments[flag_index:flag_index + 2]+ cycles_directory, frames_directory, output_directory = (Path(item) for item in arguments[:3])+ scene_ids = arguments[3].split(",")+ output_directory.mkdir(parents=True, exist_ok=True)+ metrics = {}+ sheets = []+ for scene_id in scene_ids:+ full_linear = np.load(cycles_directory / f"{scene_id}-cycles.npy")+ full = tone_map_like_67js(full_linear)+ without_path = cycles_directory / f"{scene_id}-cycles-noglossy.npy"+ without_linear = np.load(without_path) if without_path.exists() else None+ without = tone_map_like_67js(without_linear) if without_linear is not None else None+ entry = {"cycles-full": frame_statistics(full)}+ if without is not None:+ entry["cycles-noglossy"] = frame_statistics(without)+ entry["flip-full-vs-noglossy"] = round(flip_score(full, without), 4)+ entry.update(reflection_by_material(full_linear, without_linear, cycles_directory, scene_id))+ panels = []+ for label in labels:+ frame = load_png(frames_directory / f"{scene_id}-{label}.png") if (frames_directory / f"{scene_id}-{label}.png").exists() else load_png(frames_directory / label / f"{scene_id}.png")+ entry[label] = {"flip-vs-full": round(flip_score(full, frame), 4), **frame_statistics(frame)}+ title = f"{scene_id} {label} FLIP to Cycles full {entry[label]['flip-vs-full']:.4f}"+ if without is not None:+ entry[label]["flip-vs-noglossy"] = round(flip_score(without, frame), 4)+ title += f", no-glossy {entry[label]['flip-vs-noglossy']:.4f}"+ panels.append(panel(frame, title))+ if without is not None:+ panels.append(panel(without, f"Cycles no glossy FLIP to full {entry['flip-full-vs-noglossy']:.4f}"))+ panels.append(panel(full, "Cycles full"))+ metrics[scene_id] = entry+ scene_sheet = sheet(panels)+ scene_sheet.save(output_directory / f"sheet-{scene_id}.png")+ sheets.append(scene_sheet)+ print(scene_id, json.dumps({key: value for key, value in entry.items() if key != "byMaterial"}))+ combined = Image.new("RGB", (max(item.width for item in sheets), sheets[0].height * len(sheets)))+ for index, item in enumerate(sheets):+ combined.paste(item, (0, index * item.height))+ combined.save(output_directory / "sheet.png")+ (output_directory / "metrics.json").write_text(json.dumps(metrics, indent=1))+++main()diff --git a/bench/reference/render-blender.py b/bench/reference/render-blender.pyindex 3136ef92..02372e5e 100644--- a/bench/reference/render-blender.py+++ b/bench/reference/render-blender.py@@ -7,6 +7,11 @@# ... -- <work directory> --view-transform <AgX|Khronos PBR Neutral|...> [scene ids] Cycles only, with that Blender view transform, into <id>-cycles-<name>.png (16-bit sRGB).# ... -- <work directory> --max-bounces <n> [--floor-color <v>] [scene ids] Cycles only, with at most n light bounces (0 is direct light only)# and/or the part named "floor" painted a grey of v (0 is black), into <id>-cycles-bounces<n>-floor<v>.npy. They show how much light the ground bounces.+# ... -- <work directory> --no-glossy [scene ids] Cycles only, with every material's metallic, specular and coat at 0 (metals draw as diffuse+# in their base colour, no reflections), into <id>-cycles-noglossy.npy. It shows how much of a Cycles picture is reflection (issue #506).+# ... -- <work directory> --material-ids [scene ids] Workbench, flat colours: <id>-materials.npy holds the material index under each pixel (0 is the sky,+# n is the nth name in <id>-materials.json), so a difference between two renders can be split by material.+# ... -- <work directory> --print-materials [scene ids] prints each material's Principled values as built, one JSON line per scene.# Light units were measured on a white plane: sun strength equals irradiance, a point or spot light has 4 pi watts for each# unit of 67js intensity, and an area light has pi watts for each unit.import base64@@ -922,6 +927,62 @@ def limit_bounces(scene, bounces):scene.cycles.transmission_bounces = bounces+def remove_glossy(scene):+ """A measurement switch: no reflections. Metallic, specular level and coat go to 0 and lose their links, so a metal draws as a diffuse surface of its base colour."""+ for material in bpy.data.materials:+ if not material.use_nodes:+ continue+ for node in material.node_tree.nodes:+ if node.type != "BSDF_PRINCIPLED":+ continue+ for input_name in ("Metallic", "Specular IOR Level", "Coat Weight"):+ socket = node.inputs[input_name]+ for link in list(socket.links):+ material.node_tree.links.remove(link)+ socket.default_value = 0+++MATERIAL_ID_STEPS = 64+++def render_material_ids(scene, work_directory, name):+ """Flat Workbench render with one red level per material, saved as an index map and the list of names in index order (1 first)."""+ names = sorted({slot.material.name for item in scene.objects if item.type == "MESH" for slot in item.material_slots if slot.material})+ for index, material_name in enumerate(names, start=1):+ bpy.data.materials[material_name].diffuse_color = (index / MATERIAL_ID_STEPS, 0, 0, 1)+ scene.render.engine = "BLENDER_WORKBENCH"+ scene.display.shading.light = "FLAT"+ scene.display.shading.color_type = "MATERIAL"+ scene.display.render_aa = "OFF"+ scene.view_settings.view_transform = "Raw"+ scene.world.color = (0, 0, 0)+ render_to_numpy(scene, work_directory, f"{name}-levels")+ levels = np.load(os.path.join(work_directory, f"{name}-levels.npy"))[:, :, 0]+ os.remove(os.path.join(work_directory, f"{name}-levels.npy"))+ np.save(os.path.join(work_directory, f"{name}.npy"), np.rint(levels * MATERIAL_ID_STEPS).astype(np.int16))+ with open(os.path.join(work_directory, f"{name}.json"), "w", encoding="utf-8") as file:+ json.dump(names, file)+++def material_values():+ """The Principled values of every material as built: what a material import check compares with the 67js description."""+ values = {}+ for material in bpy.data.materials:+ if not material.use_nodes:+ continue+ for node in material.node_tree.nodes:+ if node.type == "BSDF_PRINCIPLED":+ values[material.name] = {+ "color": [round(channel, 4) for channel in node.inputs["Base Color"].default_value[:3]],+ "roughness": round(node.inputs["Roughness"].default_value, 4),+ "metallic": round(node.inputs["Metallic"].default_value, 4),+ "specularLevel": round(node.inputs["Specular IOR Level"].default_value, 4),+ "coat": round(node.inputs["Coat Weight"].default_value, 4),+ "linked": [socket.name for socket in node.inputs if socket.links and socket.name in ("Base Color", "Roughness", "Metallic", "Specular IOR Level")],+ }+ return values++def paint_floor(scene, grey):"""A measurement switch: the part named floor becomes a plain grey, so the light it bounces is known."""floor = scene.objects["floor"]@@ -975,6 +1036,15 @@ def main():flag_index = arguments.index("--floor-color")floor_grey = float(arguments[flag_index + 1])del arguments[flag_index:flag_index + 2]+ no_glossy = "--no-glossy" in arguments+ if no_glossy:+ arguments.remove("--no-glossy")+ material_ids = "--material-ids" in arguments+ if material_ids:+ arguments.remove("--material-ids")+ print_materials = "--print-materials" in arguments+ if print_materials:+ arguments.remove("--print-materials")view_transform = Noneif "--view-transform" in arguments:flag_index = arguments.index("--view-transform")@@ -991,6 +1061,19 @@ def main():with open(os.path.join(scene_directory, file_name), encoding="utf-8") as file:description = json.load(file)scene = build_scene(description, root)+ if print_materials:+ print("MATERIALS " + json.dumps({scene_id: material_values()}))+ continue+ if material_ids:+ render_material_ids(scene, work_directory, f"{scene_id}-materials")+ print(f"RENDERED {scene_id} material ids")+ continue+ if no_glossy:+ remove_glossy(scene)+ configure_cycles(scene)+ seconds = render_to_numpy(scene, work_directory, f"{scene_id}-cycles-noglossy")+ print(f"RENDERED {scene_id} as cycles without glossy: {seconds:.1f} s")+ continueif view_transform is not None:configure_cycles(scene)seconds = render_with_view_transform(scene, work_directory, f"{scene_id}-cycles-{view_transform.lower().replace(' ', '-')}", view_transform)diff --git a/bench/reference/run.mjs b/bench/reference/run.mjsindex f9537511..836e13a8 100644--- a/bench/reference/run.mjs+++ b/bench/reference/run.mjs@@ -3,6 +3,7 @@// --bake [<id>,<id>] [--bake-preset <name>] bakes bounced light into the named scenes (every scene when no ids follow) before 67js draws them;// Blender still gets the plain scene.// --probes [<id>,<id>] adds reflection probes (src/reflection-probes.js) to the named scenes, on top of --bake when both are given.+// --no-glossy draws the scenes with metallic 0 and specular 0 on every material (and, in the harness, no sky reflection), the 67js side of render-blender.py --no-glossy (issue #506).// --ambient <scale> multiplies the light the world gives every surface (0 removes it); with --skip-blender it measures what the world adds.// --path-tracer [--samples <n>] [--noise-threshold <n>] [--scene <id>] draws the scenes with the offline path tracer (src/path-tracer.js) instead, keeps the// Cycles renders of an earlier run, and scores them into path-tracer.json (--path-tracer-backend gpu draws them with the GPU tracer and@@ -117,7 +118,7 @@ const sceneIdsGiven = lookIds ?? (sceneFlag >= 0 ? argumentsGiven[sceneFlag + 1]// The fresh bake.json lands in the work directory and is checked against the committed one (compare-scores.mjs); --update-baseline replaces// the committed bake.json and images/<id>-baked.jpg with it.const bakeScoreRun = bakedSceneIds !== null && !sceneSet && !pathTracerFlag;-const bakeVariantFlags = ['--sky-only', '--bake-preset', '--bake-grid', '--bake-backend', '--bake-samples', '--bake-density', '--probes', '--ambient', '--ambient-occlusion', '--cascades', '--no-object-probes', '--object-probes', '--no-water', '--ssr-water', '--no-ground-bounce', '--floor-sky', '--floor-sky-rooms', '--no-floor-sky'];+const bakeVariantFlags = ['--no-glossy', '--sky-only', '--bake-preset', '--bake-grid', '--bake-backend', '--bake-samples', '--bake-density', '--probes', '--ambient', '--ambient-occlusion', '--cascades', '--no-object-probes', '--object-probes', '--no-water', '--ssr-water', '--no-ground-bounce', '--floor-sky', '--floor-sky-rooms', '--no-floor-sky'];const bakeScoreIsBaseline = bakeScoreRun && argumentsGiven.includes('--update-baseline') && !bakeVariantFlags.some(flag => argumentsGiven.includes(flag));const onlyScenes = sceneIdsGiven ?? (bakeScoreRun ? bakedSceneIds : null);const skipCpu = skipCpuFlag || bakeScoreRun || Boolean(viewTransform);@@ -131,7 +132,7 @@ async function renderBackend(backend, serverUrl, graphics) {page.on('console', message => message.text().includes('GL_') && console.log('PAGE', message.text().slice(0, 200)));// Each scene gets a fresh page, so a large scene leaves no GL state or memory behind for the next one.const openHarness = async () => {- const harnessQuery = [sceneSet ? `set=${sceneSet}` : null, argumentsGiven.includes('--no-water') ? 'water=off' : argumentsGiven.includes('--ssr-water') ? 'water=ssr' : null, argumentsGiven.includes('--no-ground-bounce') ? 'groundBounce=off' : null, floorSkyStrength ? `floorSky=${floorSkyStrength}` : argumentsGiven.includes('--no-floor-sky') ? 'floorSky=0' : null, argumentsGiven.includes('--floor-sky-rooms') ? 'floorSkyRooms=1' : null, sunBounceQuery].filter(Boolean).join('&');+ const harnessQuery = [sceneSet ? `set=${sceneSet}` : null, argumentsGiven.includes('--no-glossy') ? 'glossy=0' : null, argumentsGiven.includes('--no-water') ? 'water=off' : argumentsGiven.includes('--ssr-water') ? 'water=ssr' : null, argumentsGiven.includes('--no-ground-bounce') ? 'groundBounce=off' : null, floorSkyStrength ? `floorSky=${floorSkyStrength}` : argumentsGiven.includes('--no-floor-sky') ? 'floorSky=0' : null, argumentsGiven.includes('--floor-sky-rooms') ? 'floorSkyRooms=1' : null, sunBounceQuery].filter(Boolean).join('&');await page.goto(`${serverUrl}/bench/reference/harness.html${harnessQuery ? `?${harnessQuery}` : ''}`, { timeout: PAGE_LOAD_TIMEOUT });await page.waitForFunction(() => window.renderReferenceScene, null, { timeout: PAGE_LOAD_TIMEOUT });};diff --git a/src/irradiance-grid.js b/src/irradiance-grid.jsindex 2daa68be..4f92b724 100644--- a/src/irradiance-grid.js+++ b/src/irradiance-grid.js@@ -326,14 +326,15 @@ registerShaderFeatureCode('irradianceGrid', Object.freeze({'uniform vec3 irradianceGridSize;',// The grid stores light for the cosine lobe already, so this is the same unit as environmentDiffuse. The x axis is// lerped by hand because three nodes share a row of texels (one per colour channel); y and z use the hardware filter.- 'vec4 irradianceGridSample(vec3 position, vec3 normal) {',+ // The lookup sits half a cell off the surface along its normal; a reflection reads the same node with the lobe turned to the reflected direction.+ 'vec4 irradianceGridAlong(vec3 position, vec3 normal, vec3 lobe) {',' vec3 grid = clamp((position + normal * irradianceGridOrigin.w * 0.5 - irradianceGridOrigin.xyz) / irradianceGridOrigin.w, vec3(0.0), irradianceGridSize - 1.0);',' float left = floor(grid.x);',' float right = min(left + 1.0, irradianceGridSize.x - 1.0);',' float blend = grid.x - left;',' vec2 across = (grid.yz + 0.5) / irradianceGridSize.yz;',' float rowWidth = 4.0 * irradianceGridSize.x;',- ' vec4 direction = vec4(1.0, normal);',+ ' vec4 direction = vec4(1.0, lobe);',' vec4 result = vec4(0.0);',' for (int channel = 0; channel < 4; channel++) {',' float slot = float(channel) + 0.5;',@@ -343,10 +344,13 @@ registerShaderFeatureCode('irradianceGrid', Object.freeze({' }',' return max(result, vec4(0.0));','}',+ 'vec4 irradianceGridSample(vec3 position, vec3 normal) { return irradianceGridAlong(position, normal, normal); }','',].join('\n'),irradianceGridOcclusion: ' vec4 irradianceGridValue = irradianceGridEnabled > 0.5 ? irradianceGridSample(worldPosition, normal) : vec4(0.0, 0.0, 0.0, 1.0);\n occlusion *= min(irradianceGridValue.a, 1.0);\n',- irradianceGrid: ' if (irradianceGridEnabled > 0.5) radiance += color * (1.0 - metallic) * irradianceGridValue.rgb;\n',+ // A glossy surface also mirrors the bounced light: the grid read along the reflected direction, scaled by the Fresnel reflectance+ // and fading out as the lobe widens (the cosine-weighted grid already blurs it).+ irradianceGrid: ' if (irradianceGridEnabled > 0.5) {\n radiance += color * (1.0 - metallic) * irradianceGridValue.rgb;\n radiance += reflectance * irradianceGridAlong(worldPosition, normal, reflected).rgb * (1.0 - roughness * roughness);\n }\n',}));const rendererStates = new WeakMap();diff --git a/tests/irradiance-grid.test.js b/tests/irradiance-grid.test.jsindex ee64f782..b60d9808 100644--- a/tests/irradiance-grid.test.js+++ b/tests/irradiance-grid.test.js@@ -89,6 +89,7 @@ test('placing the grid wraps each material once and keeps reading the live matertest('the shader carries the grid code only for a renderer that asks for the feature', () => {assert.match(createFragmentShader(['irradianceGrid']), /irradianceGridSample\(worldPosition, normal\)/);assert.match(createFragmentShader(['irradianceGrid']), /occlusion \*= min\(irradianceGridValue\.a, 1\.0\)/);+ assert.match(createFragmentShader(['irradianceGrid']), /irradianceGridAlong\(worldPosition, normal, reflected\)/);assert.doesNotMatch(createFragmentShader([]), /irradianceGrid/);});