Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Measure the metal scene mean against Cycles over many seeds (#959)

Nguyen Xuan An7ca4dcdc2281
diff --git a/bench/cycles-parity/mean-ratio-seeds.py b/bench/cycles-parity/mean-ratio-seeds.py
new file mode 100644
index 00000000..7513013d
--- /dev/null
+++ b/bench/cycles-parity/mean-ratio-seeds.py
@@ -0,0 +1,131 @@
+"""Mean ratio of Cook against Cycles over many seeds, with a standard error, plus relMSE with and without the worst pixels.
+
+ uv run --no-project --with numpy python bench/cycles-parity/mean-ratio-seeds.py \
+ --reference "<cycles glob>" --candidate "<cook glob>" [--parts <partsPrefix>] \
+ [--rel-mse-reference <image.npy> --rel-mse-skip-worst 100] [--label name]
+
+Each image is one seed. The per-seed channel means are averaged on each side, the ratio is candidate over reference, and the
+standard error of the ratio follows from the standard errors of the two means (delta method, the two sides independent).
+Fireflies make a single image mean heavy tailed, so the standard error comes from the spread across seeds, never from the pixels.
+"""
+import glob
+import json
+import math
+import sys
+
+import numpy
+
+sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+
+REL_MSE_EPSILON = 1e-2
+
+
+def read_flag(argument_list, flag_name, fallback=None):
+ if flag_name in argument_list:
+ return argument_list[argument_list.index(flag_name) + 1]
+ return fallback
+
+
+def load_images(pattern):
+ paths = sorted(path for part in pattern.split(",") for path in glob.glob(part))
+ if not paths:
+ raise ValueError(f"No image matches {pattern!r}.")
+ return paths, [numpy.load(path).astype(numpy.float64) for path in paths]
+
+
+def mean_and_standard_error(values):
+ values = numpy.asarray(values, dtype=numpy.float64)
+ count = values.shape[0]
+ mean = values.mean(axis=0)
+ if count < 2:
+ return mean, numpy.full_like(mean, numpy.nan)
+ return mean, values.std(axis=0, ddof=1) / math.sqrt(count)
+
+
+def ratio_with_error(candidate_means, reference_means):
+ candidate_mean, candidate_error = mean_and_standard_error(candidate_means)
+ reference_mean, reference_error = mean_and_standard_error(reference_means)
+ ratio = candidate_mean / reference_mean
+ relative_variance = (candidate_error / candidate_mean) ** 2 + (reference_error / reference_mean) ** 2
+ return ratio, ratio * numpy.sqrt(relative_variance)
+
+
+def format_channels(values, digits=5):
+ return "[" + ", ".join(f"{value:.{digits}f}" for value in values) + "]"
+
+
+def per_pixel_rel_mse(reference, candidate):
+ return (((candidate - reference) ** 2) / (reference ** 2 + REL_MSE_EPSILON)).mean(axis=-1).ravel()
+
+
+def rel_mse_values(reference, candidate, skip_worst):
+ per_pixel = per_pixel_rel_mse(reference, candidate)
+ kept = numpy.sort(per_pixel)[: per_pixel.size - skip_worst] if skip_worst > 0 else per_pixel
+ return float(per_pixel.mean()), float(kept.mean())
+
+
+def rel_mse_report(reference_images, candidate_images, rel_mse_reference, skip_worst):
+ report = {}
+ for side, images in (("reference", reference_images), ("candidate", candidate_images)):
+ full = []
+ trimmed = []
+ for image in images:
+ full_value, trimmed_value = rel_mse_values(rel_mse_reference, image, skip_worst)
+ full.append(full_value)
+ trimmed.append(trimmed_value)
+ report[side] = {
+ "relMse": [float(numpy.mean(full)), float(numpy.std(full, ddof=1) / math.sqrt(len(full))) if len(full) > 1 else None],
+ f"relMseWithout{skip_worst}Worst": [float(numpy.mean(trimmed)), float(numpy.std(trimmed, ddof=1) / math.sqrt(len(trimmed))) if len(trimmed) > 1 else None],
+ "imageCount": len(images),
+ }
+ return report
+
+
+def part_report(reference_images, candidate_images, parts_prefix):
+ part_indices = numpy.load(f"{parts_prefix}-parts.npy")
+ with open(f"{parts_prefix}-parts.json", encoding="utf-8") as handle:
+ part_names = json.load(handle)
+ rows = []
+ for index, part_name in enumerate(part_names, start=1):
+ mask = part_indices == index
+ if int(mask.sum()) == 0:
+ continue
+ reference_means = [image[mask].mean(axis=0) for image in reference_images]
+ candidate_means = [image[mask].mean(axis=0) for image in candidate_images]
+ ratio, error = ratio_with_error(candidate_means, reference_means)
+ rows.append({"part": part_name, "pixels": int(mask.sum()), "ratio": ratio.tolist(), "standardError": error.tolist()})
+ return rows
+
+
+def main():
+ argument_list = sys.argv[1:]
+ label = read_flag(argument_list, "--label", "")
+ reference_paths, reference_images = load_images(read_flag(argument_list, "--reference"))
+ candidate_paths, candidate_images = load_images(read_flag(argument_list, "--candidate"))
+ reference_means = [image.reshape(-1, 3).mean(axis=0) for image in reference_images]
+ candidate_means = [image.reshape(-1, 3).mean(axis=0) for image in candidate_images]
+ ratio, error = ratio_with_error(candidate_means, reference_means)
+ result = {
+ "label": label,
+ "referenceImages": len(reference_paths),
+ "candidateImages": len(candidate_paths),
+ "referenceMean": mean_and_standard_error(reference_means)[0].tolist(),
+ "candidateMean": mean_and_standard_error(candidate_means)[0].tolist(),
+ "meanRatio": ratio.tolist(),
+ "meanRatioStandardError": error.tolist(),
+ }
+ print(f"{label}: ratio {format_channels(ratio)} +- {format_channels(error)} ({len(candidate_paths)} cand, {len(reference_paths)} ref)")
+ parts_prefix = read_flag(argument_list, "--parts")
+ if parts_prefix is not None:
+ result["parts"] = part_report(reference_images, candidate_images, parts_prefix)
+ for row in result["parts"]:
+ print(f" part {row['part']:<10} px {row['pixels']:>7} ratio {format_channels(row['ratio'], 4)} +- {format_channels(row['standardError'], 4)}")
+ rel_mse_path = read_flag(argument_list, "--rel-mse-reference")
+ if rel_mse_path is not None:
+ skip_worst = int(read_flag(argument_list, "--rel-mse-skip-worst", "100"))
+ result["relMse"] = rel_mse_report(reference_images, candidate_images, numpy.load(rel_mse_path).astype(numpy.float64), skip_worst)
+ print(f" relMSE {json.dumps(result['relMse'])}")
+ print("JSON " + json.dumps(result))
+
+
+main()
diff --git a/bench/cycles-parity/metal-mean-suite.sh b/bench/cycles-parity/metal-mean-suite.sh
new file mode 100644
index 00000000..cc5eab6e
--- /dev/null
+++ b/bench/cycles-parity/metal-mean-suite.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# Mean radiance of Cook against Cycles on the metal scene over many seeds, and the splits that find where a gap comes from (#959).
+# bash bench/cycles-parity/metal-mean-suite.sh <dataDirectory> <stage>...
+# Stages: build, main, relmse, splits. Needs a GPU box with Blender 5.3 (BLENDER_BIN, default ~/blender/blender) and the built addon.
+# Every image is <dataDirectory>/<side>-<tag>-s<seed>.npy; existing images are kept, so a stage can be repeated.
+set -uo pipefail
+
+repository=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+data=$(mkdir -p "$1" && cd "$1" && pwd)
+shift
+blender_binary=${BLENDER_BIN:-$HOME/blender/blender}
+uv_binary=${UV_BIN:-$HOME/.local/bin/uv}
+scene="$data/scenes/metal.json"
+seed_count=${SEED_COUNT:-12}
+cd "$repository"
+
+render_cycles() { # tag samples seed extra...
+ local tag=$1 samples=$2 seed=$3
+ shift 3
+ local prefix="$data/cycles-$tag-s$seed"
+ [ -f "$prefix.npy" ] && return 0
+ "$blender_binary" -b --factory-startup --python bench/cycles-parity/render-cycles.py -- "$scene" "$prefix" --samples "$samples" --seed "$seed" "$@" > "$prefix.log" 2>&1 || echo "cycles $tag $seed failed" >&2
+}
+
+render_cook() { # kernel tag samples seed extra...
+ local kernel=$1 tag=$2 samples=$3 seed=$4
+ shift 4
+ local prefix="$data/cook$kernel-$tag-s$seed"
+ [ -f "$prefix.npy" ] && return 0
+ node bench/cycles-parity/trace-native.mjs "$scene" "$prefix" --kernel "$kernel" --samples "$samples" --seed "$seed" "$@" > "$prefix.log" 2>&1 || echo "cook $kernel $tag $seed failed" >&2
+}
+
+report() { # label referenceGlob candidateGlob extra...
+ local label=$1 reference=$2 candidate=$3
+ shift 3
+ "$uv_binary" run --no-project --with numpy python bench/cycles-parity/mean-ratio-seeds.py --label "$label" --reference "$reference" --candidate "$candidate" "$@" | grep -av '^JSON'
+}
+
+stage_build() {
+ (cd packages/render-native && node build.mjs) 2>&1 | tail -2
+ mkdir -p "$data/scenes"
+ node bench/cycles-parity/export-scene.mjs "$data" --scene metal 2>&1 | tail -2
+ ls "$data/scenes" 2>&1
+ "$blender_binary" --version | head -1
+}
+
+stage_main() {
+ for seed in $(seq 1 "$seed_count"); do
+ render_cycles m1024 1024 "$seed"
+ render_cook cycles m1024 1024 "$seed"
+ render_cook legacy m1024 1024 "$seed"
+ done
+ render_cycles parts4096 4096 1 --part-ids --keep-exr
+ report "metal 1024 spp x$seed_count seeds, cycles kernel" "$data/cycles-m1024-s*.npy" "$data/cookcycles-m1024-s*.npy" --parts "$data/cycles-parts4096-s1"
+ report "metal 1024 spp x$seed_count seeds, legacy kernel" "$data/cycles-m1024-s*.npy" "$data/cooklegacy-m1024-s*.npy" --parts "$data/cycles-parts4096-s1"
+ report "control: Cycles seeds 1-6 against seeds 7-12" "$data/cycles-m1024-s[789].npy,$data/cycles-m1024-s1[012].npy" "$data/cycles-m1024-s[123456].npy" 2>&1 || true
+}
+
+stage_relmse() {
+ render_cycles ref4096 4096 1
+ render_cycles ref4096 4096 2
+ for seed in 1 2 3 4; do
+ render_cycles r256 256 "$seed"
+ render_cook cycles r256 256 "$seed"
+ render_cook legacy r256 256 "$seed"
+ done
+ local reference="$data/cycles-ref4096-s1.npy"
+ report "256 spp relMSE vs Cycles 4096 seed 1, cycles kernel" "$data/cycles-r256-s[234].npy" "$data/cookcycles-r256-s*.npy" --rel-mse-reference "$reference" --rel-mse-skip-worst 100
+ report "256 spp relMSE vs Cycles 4096 seed 1, legacy kernel" "$data/cycles-r256-s[234].npy" "$data/cooklegacy-r256-s*.npy" --rel-mse-reference "$reference" --rel-mse-skip-worst 100
+ report "4096 spp seed 2 against seed 1 (noise floor of the reference)" "$data/cycles-ref4096-s1.npy" "$data/cycles-ref4096-s2.npy" --rel-mse-reference "$data/cycles-ref4096-s1.npy" --rel-mse-skip-worst 100
+}
+
+split_pair() { # tag samples seeds cookFlags -- cyclesFlags
+ local tag=$1 samples=$2 seeds=$3
+ shift 3
+ local cook_flags=()
+ while [ $# -gt 0 ] && [ "$1" != "--" ]; do cook_flags+=("$1"); shift; done
+ shift
+ for seed in $(seq 1 "$seeds"); do
+ render_cycles "$tag" "$samples" "$seed" "$@"
+ render_cook cycles "$tag" "$samples" "$seed" "${cook_flags[@]}"
+ done
+ report "split $tag" "$data/cycles-$tag-s*.npy" "$data/cookcycles-$tag-s*.npy" --parts "$data/cycles-parts4096-s1"
+}
+
+stage_splits() {
+ split_pair b0 512 6 --bounces 0 -- --bounces 0
+ split_pair b1 512 6 --bounces 1 -- --bounces 1
+ split_pair b2 512 6 --bounces 2 -- --bounces 2
+ split_pair b4 512 6 --bounces 4 -- --bounces 4
+ split_pair clamp10 512 6 --clamp-indirect 10 --clamp-direct 10 -- --cycles-property sample_clamp_indirect=10 --cycles-property sample_clamp_direct=10
+}
+
+for stage in "$@"; do
+ echo "== stage $stage"
+ "stage_$stage"
+done
diff --git a/docs/cycles-parity.md b/docs/cycles-parity.md
index 8c5bc760..34cd712e 100644
--- a/docs/cycles-parity.md
+++ b/docs/cycles-parity.md
@@ -1465,3 +1465,48 @@ Each instance costs 416 bytes packed (160 for the instance and 256 for its mater
Pictures: scenes under the old bounds are byte for byte the same. The PNG SHA-256 of three scenes (the 20,000 triangle cap scene with copies, a field of 3,000 shared cards and the same cards merged) on the job export of `cycles` instanced, `cycles` flattened and `legacy` are equal on `main` and on this change, and the refusal of the scene the flattened and legacy routes cannot take has the same words (`bench/cycles-parity/native-identity-pngs.mjs`, `[measured]`). Over the old bound, `tests/native-instancing-limit.test.js` traces 85,537 parts (the last 20,000 red, all past index 65,536) and the same cards merged into two meshes: 3,381 red pixels in both, mean ratio within 2% per channel, tiles within 5%, and a relative mean squared error of 1.5e-7, against 1.4e-3 between two seeds of the merged picture.
Not measured: Chrome's memory for a page that builds 500,000 parts (the numbers above are Node's), and a scene whose copies are emissive (each emissive copy adds light tree emitters and emissive records).
+
+## Mean radiance on the `metal` scene (#959)
+
+Question: does Cook match Cycles' mean on `metal` within 0.1 percent per channel, and is the error outside the fireflies at or below Cycles' own? `[measured]` on a rented RTX 4090 (Blender 5.3.0 Cycles on OptiX, Cook on Vulkan), `960 x 540`, `bounces = 8`, box filter, no clamp, no denoise, `filter_glossy = 1`, Sobol-Burley on Cook's side.
+
+A single image's mean is heavy tailed (fireflies), so the standard error comes from the spread across seeds: [`mean-ratio-seeds.py`](../bench/cycles-parity/mean-ratio-seeds.py) averages each side's per-seed channel means and combines the two standard errors (delta method). [`metal-mean-suite.sh`](../bench/cycles-parity/metal-mean-suite.sh) renders everything below.
+
+### Mean ratio, 24 seeds a side at 1024 spp (Cook over Cycles, R / G / B, plus or minus one standard error)
+
+| Kernel | Mean ratio | Standard error | Within 0.1 percent |
+| --- | --- | --- | --- |
+| `cycles` | 0.99921 / 0.99930 / 0.99946 | 0.00021 / 0.00019 / 0.00015 | yes, off by 0.079 / 0.070 / 0.054 percent |
+| `legacy` | 0.99856 / 0.99753 / 0.99629 | 0.00028 / 0.00025 / 0.00019 | no, off by 0.14 / 0.25 / 0.37 percent |
+
+Control: Cycles seeds 7 to 12 over seeds 1 to 6 gives 1.0005 / 1.0005 / 1.0004, plus or minus 0.0004, so the comparison itself resolves about 0.04 percent with six seeds. With 24 a side the one-standard-error band of the `cycles` kernel is 0.02 percent, and its 95 percent band reaches 0.12 percent on red, so "within 0.1 percent" is met by the estimate and not by a strict 95 percent bound.
+
+Per part (the noisy ones are fireflies on the chrome, gold and steel; the floor and copper have no fireflies, their error is the bias): `cycles` kernel floor 0.9991 / 0.9992 / 0.9994 (52 percent of the pixels), copper 0.9993 / 0.9994 / 0.9996, chrome 0.995 +- 0.003, gold 0.9995 +- 0.0008, steel 1.0007 +- 0.0012. The `legacy` kernel floor is 0.9977 / 0.9947 / 0.9910 and copper 0.9945 / 0.9911 / 0.9904.
+
+### Error outside the fireflies, 256 spp against Cycles at 4096 spp seed 1 (relMSE without the 100 worst pixels)
+
+| Image | relMSE | relMSE without the 100 worst pixels | Images |
+| --- | --- | --- | --- |
+| Cycles 256 spp (its own noise, seeds 2 to 4) | 0.0486 +- 0.0239 | 0.000382 +- 0.0000013 | 3 |
+| Cook `cycles` kernel 256 spp | 0.0211 +- 0.0095 | 0.000184 +- 0.0000015 | 4 |
+| Cook `legacy` kernel 256 spp | 0.0930 +- 0.0585 | 0.000713 +- 0.0000038 | 4 |
+
+The `cycles` kernel is at 0.48 times Cycles' own error, the `legacy` kernel at 1.87 times. (Both rows share the noise of the 4096 spp reference, which is about 0.00005 in the same measure.)
+
+### Splits
+
+Each pair is Cook `cycles` against Cycles, 6 seeds, 512 spp `[measured]`.
+
+| Split | Mean ratio R / G / B | Floor ratio (no fireflies) |
+| --- | --- | --- |
+| direct light only (`bounces = 0`) | 0.99901 / 0.99913 / 0.99935 | 0.9991 / 0.9992 / 0.9994 |
+| 1 bounce | 0.99892 / 0.99907 / 0.99933 | 0.9991 / 0.9992 / 0.9994 |
+| 2 bounces | 0.99888 / 0.99904 / 0.99932 | 0.9991 / 0.9992 / 0.9994 |
+| 4 bounces | 0.99889 / 0.99904 / 0.99932 | 0.9991 / 0.9992 / 0.9994 |
+| both sides clamp at 10 (direct and indirect) | 1.00117 / 1.00079 / 1.00017 | 0.9991 / 0.9992 / 0.9994 |
+
+The floor's 0.06 to 0.09 percent deficit is already there with direct light only and does not change with the bounce count, so it is not a glossy, multiscatter or bounce-depth effect `[measured]`; it sits in how the sun and sky light the floor (sun disk or sky table, one scale for all channels) `[inferred, not isolated]`. With a clamp of 10 on both sides, the means move apart by up to 1.9 percent on gold: the two clamps are not the same operation, and the default (no clamp) is the one compared above. Two further splits were tried and are not valid: `filter_glossy = 0` on the Cycles side was refused by the render script, and Cook's `nee-only` against Cycles' `--light-mis off` differ by 15 to 20 percent because they do not switch off the same thing.
+
+### What this says about #959
+
+The `cycles` kernel meets both open boxes of #959 on `metal`: the mean is within 0.1 percent per channel, and the error without the 100 worst pixels is below Cycles' own. The `legacy` kernel meets the second and not the first (blue 0.37 percent low). Not proven: the 0.06 to 0.09 percent floor deficit has no isolated cause.