Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

WIP #506: reflection measurement switches (--no-glossy, material ids, reflection-share.py) and grid glossy reflection of bounced light (unverified on a/b/c)

Nguyen Xuan An655c9e44b1b3
diff --git a/bench/reference/harness.js b/bench/reference/harness.js
index 15445b0e..666446f4 100644
--- a/bench/reference/harness.js
+++ b/bench/reference/harness.js
@@ -66,6 +66,12 @@ const WATER_REFLECTION = { off: false, ssr: 'ssr' }[new URLSearchParams(location
const FLOOR_SKY_PARAMETER = new URLSearchParams(location.search).get('floorSky');
const FLOOR_SKY_EXPERIMENT = FLOOR_SKY_PARAMETER !== null && FLOOR_SKY_PARAMETER !== '0';
const floorSkyReports = {};
+// ?glossy=0 is a measurement switch (issue #506): every material loses its metallic and specular, so the frame shows diffuse light and emission only.
+const GLOSSY_OFF = new URLSearchParams(location.search).get('glossy') === '0';
+
+function removeGlossy(scene) {
+ for (const part of scene.parts) part.material = { ...part.material, metallic: 0, specular: 0, clearcoat: 0 };
+}
function withFloorSky(id, parts, scene) {
const environment = FLOOR_SKY_PARAMETER === '0' ? { ...scene.environment, floorSkyTrace: false } : scene.environment;
@@ -229,6 +235,7 @@ async function probeScene(scene, id, drawnParts, mode) {
async function renderReferenceScene(id, { describe, ambientOcclusion = 0, ambientOcclusionRadius = 0.5, bake = null, cascades = 0, backgroundOnly = false, probes = false, ambient = null, fill = null, key = null, viewTransform = null, bloom = null }) {
const scene = await SCENES[id]();
+ if (GLOSSY_OFF) removeGlossy(scene);
// A view transform (agx, neutral) replaces the ACES fit in the shaders, as environment.viewTransform does in a recipe.
if (viewTransform) scene.environment = { ...scene.environment, viewTransform };
// A measurement switch: 0 removes the world's light from every surface, to show how much of a picture comes from it.
diff --git a/bench/reference/reflection-share.py b/bench/reference/reflection-share.py
new file mode 100644
index 00000000..3e6ede60
--- /dev/null
+++ b/bench/reference/reflection-share.py
@@ -0,0 +1,137 @@
+# How much of a Cycles frame is reflection, and how far 67js is from the frame with and without it (issue #506).
+# uv run --no-project --python 3.12 --with flip-evaluator --with pillow --with numpy python -I bench/reference/reflection-share.py \
+# <cycles directory> <frames directory> <output directory> <scene id>,<scene id> [--labels a,b]
+# The Cycles directory holds <id>-cycles.npy, and optionally <id>-cycles-noglossy.npy (render-blender.py --no-glossy) and <id>-materials.npy and
+# <id>-materials.json (--material-ids). The frames directory holds the 67js frames as <id>-<label>.png for each label (default before,after; a label may also be a subdirectory holding <id>.png).
+# Writes <output>/metrics.json and one sheet per scene: each 67js frame, Cycles without glossy, Cycles full.
+import json
+import sys
+from pathlib import Path
+
+import flip_evaluator
+import numpy as np
+from PIL import Image, ImageDraw
+
+GAMMA = 2.2
+PANEL_WIDTH = 480
+PANEL_HEIGHT = 270
+LABEL_HEIGHT = 22
+LUMINANCE_WEIGHTS = np.array([0.2126, 0.7152, 0.0722], dtype=np.float32)
+
+
+def tone_map_like_67js(linear):
+ color = np.maximum(linear, 0)
+ mapped = np.clip((color * (2.51 * color + 0.03)) / (color * (2.43 * color + 0.59) + 0.14), 0, 1)
+ return mapped ** (1 / GAMMA)
+
+
+def load_png(path):
+ return np.asarray(Image.open(path).convert("RGB"), dtype=np.float32) / 255
+
+
+def flip_score(reference, test):
+ _, mean_error, _ = flip_evaluator.evaluate(reference.astype(np.float32), test.astype(np.float32), "LDR", inputsRGB=True)
+ return float(mean_error)
+
+
+def frame_statistics(display):
+ luminance = display @ LUMINANCE_WEIGHTS
+ linear = display ** GAMMA
+ return {
+ "mean": round(float(luminance.mean()), 4),
+ "p20": round(float(np.percentile(luminance, 20)), 4),
+ "rb": round(float(linear[:, :, 0].sum() / max(linear[:, :, 2].sum(), 1e-9)), 3),
+ }
+
+
+def reflection_by_material(full, without, cycles_directory, scene_id):
+ """Signed and absolute linear luminance the reflections add, per material, as a share of the whole frame's luminance."""
+ difference = (full - without) @ LUMINANCE_WEIGHTS
+ total = float((full @ LUMINANCE_WEIGHTS).sum())
+ result = {
+ "reflectionShareSigned": round(float(difference.sum()) / total, 4),
+ "reflectionShareAbsolute": round(float(np.abs(difference).sum()) / total, 4),
+ }
+ ids_path = cycles_directory / f"{scene_id}-materials.npy"
+ if not ids_path.exists():
+ return result
+ identifiers = np.load(ids_path)
+ names = ["sky"] + json.loads((cycles_directory / f"{scene_id}-materials.json").read_text())
+ per_material = {}
+ for index, name in enumerate(names):
+ mask = identifiers == index
+ if not mask.any():
+ continue
+ per_material[name] = {
+ "pixels": round(float(mask.mean()), 4),
+ "frameLuminance": round(float((full @ LUMINANCE_WEIGHTS)[mask].sum()) / total, 4),
+ "reflectionSigned": round(float(difference[mask].sum()) / total, 4),
+ "reflectionAbsolute": round(float(np.abs(difference[mask]).sum()) / total, 4),
+ }
+ result["byMaterial"] = per_material
+ return result
+
+
+def panel(display, label):
+ image = Image.fromarray(np.round(np.clip(display, 0, 1) * 255).astype(np.uint8)).resize((PANEL_WIDTH, PANEL_HEIGHT), Image.LANCZOS)
+ canvas = Image.new("RGB", (PANEL_WIDTH, PANEL_HEIGHT + LABEL_HEIGHT), (20, 20, 20))
+ canvas.paste(image, (0, LABEL_HEIGHT))
+ ImageDraw.Draw(canvas).text((6, 5), label, fill=(235, 235, 235))
+ return canvas
+
+
+def sheet(panels):
+ canvas = Image.new("RGB", (PANEL_WIDTH * len(panels), PANEL_HEIGHT + LABEL_HEIGHT))
+ for index, item in enumerate(panels):
+ canvas.paste(item, (index * PANEL_WIDTH, 0))
+ return canvas
+
+
+def main():
+ arguments = sys.argv[1:]
+ labels = ["before", "after"]
+ if "--labels" in arguments:
+ flag_index = arguments.index("--labels")
+ labels = arguments[flag_index + 1].split(",")
+ del arguments[flag_index:flag_index + 2]
+ cycles_directory, frames_directory, output_directory = (Path(item) for item in arguments[:3])
+ scene_ids = arguments[3].split(",")
+ output_directory.mkdir(parents=True, exist_ok=True)
+ metrics = {}
+ sheets = []
+ for scene_id in scene_ids:
+ full_linear = np.load(cycles_directory / f"{scene_id}-cycles.npy")
+ full = tone_map_like_67js(full_linear)
+ without_path = cycles_directory / f"{scene_id}-cycles-noglossy.npy"
+ without_linear = np.load(without_path) if without_path.exists() else None
+ without = tone_map_like_67js(without_linear) if without_linear is not None else None
+ entry = {"cycles-full": frame_statistics(full)}
+ if without is not None:
+ entry["cycles-noglossy"] = frame_statistics(without)
+ entry["flip-full-vs-noglossy"] = round(flip_score(full, without), 4)
+ entry.update(reflection_by_material(full_linear, without_linear, cycles_directory, scene_id))
+ panels = []
+ for label in labels:
+ frame = load_png(frames_directory / f"{scene_id}-{label}.png") if (frames_directory / f"{scene_id}-{label}.png").exists() else load_png(frames_directory / label / f"{scene_id}.png")
+ entry[label] = {"flip-vs-full": round(flip_score(full, frame), 4), **frame_statistics(frame)}
+ title = f"{scene_id} {label} FLIP to Cycles full {entry[label]['flip-vs-full']:.4f}"
+ if without is not None:
+ entry[label]["flip-vs-noglossy"] = round(flip_score(without, frame), 4)
+ title += f", no-glossy {entry[label]['flip-vs-noglossy']:.4f}"
+ panels.append(panel(frame, title))
+ if without is not None:
+ panels.append(panel(without, f"Cycles no glossy FLIP to full {entry['flip-full-vs-noglossy']:.4f}"))
+ panels.append(panel(full, "Cycles full"))
+ metrics[scene_id] = entry
+ scene_sheet = sheet(panels)
+ scene_sheet.save(output_directory / f"sheet-{scene_id}.png")
+ sheets.append(scene_sheet)
+ print(scene_id, json.dumps({key: value for key, value in entry.items() if key != "byMaterial"}))
+ combined = Image.new("RGB", (max(item.width for item in sheets), sheets[0].height * len(sheets)))
+ for index, item in enumerate(sheets):
+ combined.paste(item, (0, index * item.height))
+ combined.save(output_directory / "sheet.png")
+ (output_directory / "metrics.json").write_text(json.dumps(metrics, indent=1))
+
+
+main()
diff --git a/bench/reference/render-blender.py b/bench/reference/render-blender.py
index 3136ef92..02372e5e 100644
--- a/bench/reference/render-blender.py
+++ b/bench/reference/render-blender.py
@@ -7,6 +7,11 @@
# ... -- <work directory> --view-transform <AgX|Khronos PBR Neutral|...> [scene ids] Cycles only, with that Blender view transform, into <id>-cycles-<name>.png (16-bit sRGB).
# ... -- <work directory> --max-bounces <n> [--floor-color <v>] [scene ids] Cycles only, with at most n light bounces (0 is direct light only)
# and/or the part named "floor" painted a grey of v (0 is black), into <id>-cycles-bounces<n>-floor<v>.npy. They show how much light the ground bounces.
+# ... -- <work directory> --no-glossy [scene ids] Cycles only, with every material's metallic, specular and coat at 0 (metals draw as diffuse
+# in their base colour, no reflections), into <id>-cycles-noglossy.npy. It shows how much of a Cycles picture is reflection (issue #506).
+# ... -- <work directory> --material-ids [scene ids] Workbench, flat colours: <id>-materials.npy holds the material index under each pixel (0 is the sky,
+# n is the nth name in <id>-materials.json), so a difference between two renders can be split by material.
+# ... -- <work directory> --print-materials [scene ids] prints each material's Principled values as built, one JSON line per scene.
# Light units were measured on a white plane: sun strength equals irradiance, a point or spot light has 4 pi watts for each
# unit of 67js intensity, and an area light has pi watts for each unit.
import base64
@@ -922,6 +927,62 @@ def limit_bounces(scene, bounces):
scene.cycles.transmission_bounces = bounces
+def remove_glossy(scene):
+ """A measurement switch: no reflections. Metallic, specular level and coat go to 0 and lose their links, so a metal draws as a diffuse surface of its base colour."""
+ for material in bpy.data.materials:
+ if not material.use_nodes:
+ continue
+ for node in material.node_tree.nodes:
+ if node.type != "BSDF_PRINCIPLED":
+ continue
+ for input_name in ("Metallic", "Specular IOR Level", "Coat Weight"):
+ socket = node.inputs[input_name]
+ for link in list(socket.links):
+ material.node_tree.links.remove(link)
+ socket.default_value = 0
+
+
+MATERIAL_ID_STEPS = 64
+
+
+def render_material_ids(scene, work_directory, name):
+ """Flat Workbench render with one red level per material, saved as an index map and the list of names in index order (1 first)."""
+ names = sorted({slot.material.name for item in scene.objects if item.type == "MESH" for slot in item.material_slots if slot.material})
+ for index, material_name in enumerate(names, start=1):
+ bpy.data.materials[material_name].diffuse_color = (index / MATERIAL_ID_STEPS, 0, 0, 1)
+ scene.render.engine = "BLENDER_WORKBENCH"
+ scene.display.shading.light = "FLAT"
+ scene.display.shading.color_type = "MATERIAL"
+ scene.display.render_aa = "OFF"
+ scene.view_settings.view_transform = "Raw"
+ scene.world.color = (0, 0, 0)
+ render_to_numpy(scene, work_directory, f"{name}-levels")
+ levels = np.load(os.path.join(work_directory, f"{name}-levels.npy"))[:, :, 0]
+ os.remove(os.path.join(work_directory, f"{name}-levels.npy"))
+ np.save(os.path.join(work_directory, f"{name}.npy"), np.rint(levels * MATERIAL_ID_STEPS).astype(np.int16))
+ with open(os.path.join(work_directory, f"{name}.json"), "w", encoding="utf-8") as file:
+ json.dump(names, file)
+
+
+def material_values():
+ """The Principled values of every material as built: what a material import check compares with the 67js description."""
+ values = {}
+ for material in bpy.data.materials:
+ if not material.use_nodes:
+ continue
+ for node in material.node_tree.nodes:
+ if node.type == "BSDF_PRINCIPLED":
+ values[material.name] = {
+ "color": [round(channel, 4) for channel in node.inputs["Base Color"].default_value[:3]],
+ "roughness": round(node.inputs["Roughness"].default_value, 4),
+ "metallic": round(node.inputs["Metallic"].default_value, 4),
+ "specularLevel": round(node.inputs["Specular IOR Level"].default_value, 4),
+ "coat": round(node.inputs["Coat Weight"].default_value, 4),
+ "linked": [socket.name for socket in node.inputs if socket.links and socket.name in ("Base Color", "Roughness", "Metallic", "Specular IOR Level")],
+ }
+ return values
+
+
def paint_floor(scene, grey):
"""A measurement switch: the part named floor becomes a plain grey, so the light it bounces is known."""
floor = scene.objects["floor"]
@@ -975,6 +1036,15 @@ def main():
flag_index = arguments.index("--floor-color")
floor_grey = float(arguments[flag_index + 1])
del arguments[flag_index:flag_index + 2]
+ no_glossy = "--no-glossy" in arguments
+ if no_glossy:
+ arguments.remove("--no-glossy")
+ material_ids = "--material-ids" in arguments
+ if material_ids:
+ arguments.remove("--material-ids")
+ print_materials = "--print-materials" in arguments
+ if print_materials:
+ arguments.remove("--print-materials")
view_transform = None
if "--view-transform" in arguments:
flag_index = arguments.index("--view-transform")
@@ -991,6 +1061,19 @@ def main():
with open(os.path.join(scene_directory, file_name), encoding="utf-8") as file:
description = json.load(file)
scene = build_scene(description, root)
+ if print_materials:
+ print("MATERIALS " + json.dumps({scene_id: material_values()}))
+ continue
+ if material_ids:
+ render_material_ids(scene, work_directory, f"{scene_id}-materials")
+ print(f"RENDERED {scene_id} material ids")
+ continue
+ if no_glossy:
+ remove_glossy(scene)
+ configure_cycles(scene)
+ seconds = render_to_numpy(scene, work_directory, f"{scene_id}-cycles-noglossy")
+ print(f"RENDERED {scene_id} as cycles without glossy: {seconds:.1f} s")
+ continue
if view_transform is not None:
configure_cycles(scene)
seconds = render_with_view_transform(scene, work_directory, f"{scene_id}-cycles-{view_transform.lower().replace(' ', '-')}", view_transform)
diff --git a/bench/reference/run.mjs b/bench/reference/run.mjs
index f9537511..836e13a8 100644
--- a/bench/reference/run.mjs
+++ b/bench/reference/run.mjs
@@ -3,6 +3,7 @@
// --bake [<id>,<id>] [--bake-preset <name>] bakes bounced light into the named scenes (every scene when no ids follow) before 67js draws them;
// Blender still gets the plain scene.
// --probes [<id>,<id>] adds reflection probes (src/reflection-probes.js) to the named scenes, on top of --bake when both are given.
+// --no-glossy draws the scenes with metallic 0 and specular 0 on every material (and, in the harness, no sky reflection), the 67js side of render-blender.py --no-glossy (issue #506).
// --ambient <scale> multiplies the light the world gives every surface (0 removes it); with --skip-blender it measures what the world adds.
// --path-tracer [--samples <n>] [--noise-threshold <n>] [--scene <id>] draws the scenes with the offline path tracer (src/path-tracer.js) instead, keeps the
// Cycles renders of an earlier run, and scores them into path-tracer.json (--path-tracer-backend gpu draws them with the GPU tracer and
@@ -117,7 +118,7 @@ const sceneIdsGiven = lookIds ?? (sceneFlag >= 0 ? argumentsGiven[sceneFlag + 1]
// The fresh bake.json lands in the work directory and is checked against the committed one (compare-scores.mjs); --update-baseline replaces
// the committed bake.json and images/<id>-baked.jpg with it.
const bakeScoreRun = bakedSceneIds !== null && !sceneSet && !pathTracerFlag;
-const bakeVariantFlags = ['--sky-only', '--bake-preset', '--bake-grid', '--bake-backend', '--bake-samples', '--bake-density', '--probes', '--ambient', '--ambient-occlusion', '--cascades', '--no-object-probes', '--object-probes', '--no-water', '--ssr-water', '--no-ground-bounce', '--floor-sky', '--floor-sky-rooms', '--no-floor-sky'];
+const bakeVariantFlags = ['--no-glossy', '--sky-only', '--bake-preset', '--bake-grid', '--bake-backend', '--bake-samples', '--bake-density', '--probes', '--ambient', '--ambient-occlusion', '--cascades', '--no-object-probes', '--object-probes', '--no-water', '--ssr-water', '--no-ground-bounce', '--floor-sky', '--floor-sky-rooms', '--no-floor-sky'];
const bakeScoreIsBaseline = bakeScoreRun && argumentsGiven.includes('--update-baseline') && !bakeVariantFlags.some(flag => argumentsGiven.includes(flag));
const onlyScenes = sceneIdsGiven ?? (bakeScoreRun ? bakedSceneIds : null);
const skipCpu = skipCpuFlag || bakeScoreRun || Boolean(viewTransform);
@@ -131,7 +132,7 @@ async function renderBackend(backend, serverUrl, graphics) {
page.on('console', message => message.text().includes('GL_') && console.log('PAGE', message.text().slice(0, 200)));
// Each scene gets a fresh page, so a large scene leaves no GL state or memory behind for the next one.
const openHarness = async () => {
- const harnessQuery = [sceneSet ? `set=${sceneSet}` : null, argumentsGiven.includes('--no-water') ? 'water=off' : argumentsGiven.includes('--ssr-water') ? 'water=ssr' : null, argumentsGiven.includes('--no-ground-bounce') ? 'groundBounce=off' : null, floorSkyStrength ? `floorSky=${floorSkyStrength}` : argumentsGiven.includes('--no-floor-sky') ? 'floorSky=0' : null, argumentsGiven.includes('--floor-sky-rooms') ? 'floorSkyRooms=1' : null, sunBounceQuery].filter(Boolean).join('&');
+ const harnessQuery = [sceneSet ? `set=${sceneSet}` : null, argumentsGiven.includes('--no-glossy') ? 'glossy=0' : null, argumentsGiven.includes('--no-water') ? 'water=off' : argumentsGiven.includes('--ssr-water') ? 'water=ssr' : null, argumentsGiven.includes('--no-ground-bounce') ? 'groundBounce=off' : null, floorSkyStrength ? `floorSky=${floorSkyStrength}` : argumentsGiven.includes('--no-floor-sky') ? 'floorSky=0' : null, argumentsGiven.includes('--floor-sky-rooms') ? 'floorSkyRooms=1' : null, sunBounceQuery].filter(Boolean).join('&');
await page.goto(`${serverUrl}/bench/reference/harness.html${harnessQuery ? `?${harnessQuery}` : ''}`, { timeout: PAGE_LOAD_TIMEOUT });
await page.waitForFunction(() => window.renderReferenceScene, null, { timeout: PAGE_LOAD_TIMEOUT });
};
diff --git a/src/irradiance-grid.js b/src/irradiance-grid.js
index 2daa68be..4f92b724 100644
--- a/src/irradiance-grid.js
+++ b/src/irradiance-grid.js
@@ -326,14 +326,15 @@ registerShaderFeatureCode('irradianceGrid', Object.freeze({
'uniform vec3 irradianceGridSize;',
// The grid stores light for the cosine lobe already, so this is the same unit as environmentDiffuse. The x axis is
// lerped by hand because three nodes share a row of texels (one per colour channel); y and z use the hardware filter.
- 'vec4 irradianceGridSample(vec3 position, vec3 normal) {',
+ // The lookup sits half a cell off the surface along its normal; a reflection reads the same node with the lobe turned to the reflected direction.
+ 'vec4 irradianceGridAlong(vec3 position, vec3 normal, vec3 lobe) {',
' vec3 grid = clamp((position + normal * irradianceGridOrigin.w * 0.5 - irradianceGridOrigin.xyz) / irradianceGridOrigin.w, vec3(0.0), irradianceGridSize - 1.0);',
' float left = floor(grid.x);',
' float right = min(left + 1.0, irradianceGridSize.x - 1.0);',
' float blend = grid.x - left;',
' vec2 across = (grid.yz + 0.5) / irradianceGridSize.yz;',
' float rowWidth = 4.0 * irradianceGridSize.x;',
- ' vec4 direction = vec4(1.0, normal);',
+ ' vec4 direction = vec4(1.0, lobe);',
' vec4 result = vec4(0.0);',
' for (int channel = 0; channel < 4; channel++) {',
' float slot = float(channel) + 0.5;',
@@ -343,10 +344,13 @@ registerShaderFeatureCode('irradianceGrid', Object.freeze({
' }',
' return max(result, vec4(0.0));',
'}',
+ 'vec4 irradianceGridSample(vec3 position, vec3 normal) { return irradianceGridAlong(position, normal, normal); }',
'',
].join('\n'),
irradianceGridOcclusion: ' vec4 irradianceGridValue = irradianceGridEnabled > 0.5 ? irradianceGridSample(worldPosition, normal) : vec4(0.0, 0.0, 0.0, 1.0);\n occlusion *= min(irradianceGridValue.a, 1.0);\n',
- irradianceGrid: ' if (irradianceGridEnabled > 0.5) radiance += color * (1.0 - metallic) * irradianceGridValue.rgb;\n',
+ // A glossy surface also mirrors the bounced light: the grid read along the reflected direction, scaled by the Fresnel reflectance
+ // and fading out as the lobe widens (the cosine-weighted grid already blurs it).
+ irradianceGrid: ' if (irradianceGridEnabled > 0.5) {\n radiance += color * (1.0 - metallic) * irradianceGridValue.rgb;\n radiance += reflectance * irradianceGridAlong(worldPosition, normal, reflected).rgb * (1.0 - roughness * roughness);\n }\n',
}));
const rendererStates = new WeakMap();
diff --git a/tests/irradiance-grid.test.js b/tests/irradiance-grid.test.js
index ee64f782..b60d9808 100644
--- a/tests/irradiance-grid.test.js
+++ b/tests/irradiance-grid.test.js
@@ -89,6 +89,7 @@ test('placing the grid wraps each material once and keeps reading the live mater
test('the shader carries the grid code only for a renderer that asks for the feature', () => {
assert.match(createFragmentShader(['irradianceGrid']), /irradianceGridSample\(worldPosition, normal\)/);
assert.match(createFragmentShader(['irradianceGrid']), /occlusion \*= min\(irradianceGridValue\.a, 1\.0\)/);
+ assert.match(createFragmentShader(['irradianceGrid']), /irradianceGridAlong\(worldPosition, normal, reflected\)/);
assert.doesNotMatch(createFragmentShader([]), /irradianceGrid/);
});