Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(render): a surface reflects the colour it actually has

esenginee50d79804e32
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bfb4dcb4b..a7f8d56b7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,15 @@ published separately; it ships inside the editor.
### Added
+- **A bounce carries the colour of what it came off.** Baked light reflected a
+ neutral grey whatever it hit, so a red wall lit a white floor white — most of
+ the difference between a bounce and a flat ambient term. A surface's albedo now
+ comes from both of the things a shader multiplies: the base colour factor (for
+ an imported model, the material's own, which the import writes onto the
+ component) and the mean of its base colour texture, decoded out of sRGB first
+ so the average is of light rather than of what a screen shows. A surface with
+ neither is still a neutral grey, and says so rather than guessing quietly.
+
- **Bake Lighting.** The trigger the bake was missing: a command that collects
the open scene, solves it, writes the atlas beside the scene file, and records
on each object where in that atlas it reads. Sixteen lights is what a frame
diff --git a/desktop b/desktop
index c37b81210..f6d571b4a 160000
--- a/desktop
+++ b/desktop
@@ -1 +1 @@
-Subproject commit c37b81210bf8392e6b6283889208481e8b246992
+Subproject commit f6d571b4a049daa15a78149df14ac3589430cc15
diff --git a/pipeline/src/assets/lightmapBake.ts b/pipeline/src/assets/lightmapBake.ts
index 6e899d7e4..9a64549e0 100644
--- a/pipeline/src/assets/lightmapBake.ts
+++ b/pipeline/src/assets/lightmapBake.ts
@@ -9,6 +9,7 @@
* the world the transforms come from, and this holds the meshes they name.
*/
import { readFileSync } from 'node:fs';
+import { PNG } from 'pngjs';
import { bakeLightmap, decodeMesh, MeshChannel,
type BakeSurface, type BakeLight, type BakeOptions } from 'esengine';
import { encodeRgbaPng } from './png';
@@ -21,7 +22,15 @@ export interface SceneBakeSurface {
label: string;
/** Column-major 4x4 world transform. */
transform: number[];
- albedo?: [number, number, number];
+ /**
+ * The surface's base colour factor — for an imported model this IS the
+ * material's `baseColorFactor`, which the import writes onto the component.
+ */
+ baseColor?: [number, number, number];
+ /** Absolute path of the base colour texture, whose average completes the
+ * albedo. A bounce off a red wall has to arrive red, and the colour is in
+ * the texture as often as it is in the factor. */
+ baseColorTexture?: string;
}
export interface SceneBakeInput {
@@ -41,6 +50,58 @@ export interface SceneBakeResult {
warnings: string[];
}
+/** sRGB to linear, one channel. A texture stores what a screen shows, and an
+ * average taken before this is decoded is brighter than the surface is. */
+function toLinear(v: number): number {
+ const c = v / 255;
+ return c <= 0.04045 ? c / 12.92 : ((c + 0.055) / 1.055) ** 2.4;
+}
+
+/** A texture's mean colour in linear light, or null when it cannot be read. */
+function averageOf(file: string): [number, number, number] | null {
+ const png = PNG.sync.read(readFileSync(file));
+ let r = 0, g = 0, b = 0;
+ const pixels = png.width * png.height;
+ if (pixels === 0) return null;
+ for (let i = 0; i < pixels; i++) {
+ r += toLinear(png.data[i * 4]);
+ g += toLinear(png.data[i * 4 + 1]);
+ b += toLinear(png.data[i * 4 + 2]);
+ }
+ return [r / pixels, g / pixels, b / pixels];
+}
+
+/**
+ * What fraction of each channel this surface reflects.
+ *
+ * The factor and the texture BOTH carry it — a shader multiplies them, and a
+ * bounce that used only one would drop the colour whenever the other held it.
+ * A surface with neither reflects a neutral grey, which is a guess, so it says so.
+ */
+function albedoOf(s: SceneBakeSurface, cache: Map<string, [number, number, number] | null>,
+ warnings: string[]): [number, number, number] | undefined {
+ const factor = s.baseColor;
+ if (!s.baseColorTexture) {
+ if (!factor) {
+ warnings.push(`${s.label}: no base colour, so light bounces off it as neutral grey`);
+ }
+ return factor;
+ }
+ if (!cache.has(s.baseColorTexture)) {
+ try {
+ cache.set(s.baseColorTexture, averageOf(s.baseColorTexture));
+ } catch {
+ cache.set(s.baseColorTexture, null);
+ warnings.push(`${s.label}: its base colour texture could not be read, so light`
+ + ' bounces off it by its colour factor alone');
+ }
+ }
+ const mean = cache.get(s.baseColorTexture) ?? null;
+ if (!mean) return factor;
+ const f = factor ?? [1, 1, 1];
+ return [mean[0] * f[0], mean[1] * f[1], mean[2] * f[2]];
+}
+
/**
* Bakes what the caller placed, skipping what cannot receive light and saying so.
*
@@ -52,6 +113,9 @@ export function bakeSceneLightmap(input: SceneBakeInput): SceneBakeResult {
const warnings: string[] = [];
const surfaces: BakeSurface[] = [];
const slot: number[] = [];
+ // One decode per texture however many objects share it: a bake reads these
+ // once and a scene reuses the same few across most of its surfaces.
+ const averages = new Map<string, [number, number, number] | null>();
for (let i = 0; i < input.surfaces.length; i++) {
const s = input.surfaces[i];
@@ -68,7 +132,7 @@ export function bakeSceneLightmap(input: SceneBakeInput): SceneBakeResult {
continue;
}
slot.push(i);
- surfaces.push({ mesh, transform: s.transform, albedo: s.albedo });
+ surfaces.push({ mesh, transform: s.transform, albedo: albedoOf(s, averages, warnings) });
}
if (surfaces.length === 0) {
diff --git a/pipeline/tests/lightmap-bake.test.ts b/pipeline/tests/lightmap-bake.test.ts
index 8a2417997..b3fa50b62 100644
--- a/pipeline/tests/lightmap-bake.test.ts
+++ b/pipeline/tests/lightmap-bake.test.ts
@@ -14,6 +14,7 @@ import path from 'node:path';
import { encodeMesh, unwrapLightmapUV, MeshChannel, MeshChannelType,
type MeshData, type BakeLight } from 'esengine';
import { bakeSceneLightmap, type SceneBakeSurface } from '../src/assets/lightmapBake';
+import { encodeRgbaPng } from '../src/assets/png';
let dir = '';
@@ -51,16 +52,23 @@ beforeAll(async () => {
dir = await mkdtemp(path.join(tmpdir(), 'estella-bake-'));
await writeFile(path.join(dir, 'unwrapped.esmesh'), encodeMesh(unwrapLightmapUV(floor(4)).mesh));
await writeFile(path.join(dir, 'raw.esmesh'), encodeMesh(floor(4)));
+ // A texture that is HALF white and half black: its mean is a middle grey no
+ // single pixel of it is, so an average that read one texel would be wrong.
+ const half = new Uint8Array(4 * 4 * 4).fill(255);
+ for (let i = 8; i < 16; i++) { half[i * 4] = 0; half[i * 4 + 1] = 0; half[i * 4 + 2] = 0; }
+ await writeFile(path.join(dir, 'half.png'), encodeRgbaPng(4, 4, half));
});
afterAll(async () => {
await rm(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });
});
-const surface = (file: string, label: string, x = 0): SceneBakeSurface => ({
+const surface = (file: string, label: string, x = 0,
+ baseColor: [number, number, number] = [0.8, 0.8, 0.8]): SceneBakeSurface => ({
meshFile: path.join(dir, file),
label,
transform: [...IDENTITY.slice(0, 12), x, 0, 0, 1],
+ baseColor,
});
describe('baking a scene', () => {
@@ -97,6 +105,45 @@ describe('baking a scene', () => {
expect(result.warnings.join(' ')).toContain('nothing in this scene');
});
+ it('takes a surface\'s colour from its factor AND its texture', () => {
+ // A shader multiplies the two, so a bounce that read only one drops the
+ // colour whenever the other holds it. Read back through the warning the
+ // guess would have produced: with a texture there is nothing to guess.
+ const withTexture = bakeSceneLightmap({
+ surfaces: [{
+ ...surface('unwrapped.esmesh', 'Floor'),
+ baseColor: undefined,
+ baseColorTexture: path.join(dir, 'half.png'),
+ }],
+ lights: LAMP,
+ options: SMALL,
+ });
+ expect(withTexture.warnings.join(' ')).not.toContain('neutral grey');
+
+ // And an unreadable one falls back to the factor, saying so rather than
+ // pretending the surface is white.
+ const broken = bakeSceneLightmap({
+ surfaces: [{
+ ...surface('unwrapped.esmesh', 'Floor'),
+ baseColorTexture: path.join(dir, 'raw.esmesh'),
+ }],
+ lights: LAMP,
+ options: SMALL,
+ });
+ expect(broken.warnings.join(' ')).toContain('could not be read');
+ });
+
+ it('says when it had to guess what a surface reflects', () => {
+ // Neutral grey is a guess, and a bounce off a red wall that arrives grey
+ // is the kind of wrong that looks like the bake simply being dull.
+ const result = bakeSceneLightmap({
+ surfaces: [{ ...surface('unwrapped.esmesh', 'Floor'), baseColor: undefined }],
+ lights: LAMP,
+ options: SMALL,
+ });
+ expect(result.warnings.join(' ')).toContain('neutral grey');
+ });
+
it('writes a PNG the editor can adopt as an asset', () => {
const result = bakeSceneLightmap({
surfaces: [surface('unwrapped.esmesh', 'Floor')], lights: LAMP, options: SMALL,
diff --git a/sdk/tests/lightmap-bake.test.ts b/sdk/tests/lightmap-bake.test.ts
index d77015bbf..d9033066e 100644
--- a/sdk/tests/lightmap-bake.test.ts
+++ b/sdk/tests/lightmap-bake.test.ts
@@ -196,6 +196,29 @@ describe('a baked lightmap', () => {
expect(brightestOf(lit, 1)).toBeGreaterThan(0);
});
+ it('carries the colour of what the light bounced off', () => {
+ // The reason a bounce takes an albedo at all. A white panel over a red
+ // floor comes back red, and over a white floor it does not — colour
+ // bleeding is most of what separates a bounce from a flat ambient term.
+ const ground = baked(floor(8));
+ const panel = baked(ceiling(2));
+ const lights: BakeLight[] = [
+ { kind: 'point', position: [0, 6, 0], color: [1, 1, 1], intensity: 8, radius: 40 },
+ ];
+ const under = (albedo: [number, number, number]): [number, number, number] => {
+ const r = bakeLightmap([
+ { mesh: ground, transform: IDENTITY, albedo },
+ { mesh: panel, transform: translated(0, 3, 0), albedo: [1, 1, 1] },
+ ], lights, { ...LIT, bounces: 1, samples: 64 });
+ return brightestChannels(r, 1);
+ };
+ const [rr, rg] = under([1, 0.1, 0.1]);
+ const [wr, wg] = under([1, 1, 1]);
+ expect(rr).toBeGreaterThan(0);
+ expect(rr / Math.max(rg, 1)).toBeGreaterThan(3);
+ expect(wr / Math.max(wg, 1)).toBeLessThan(1.2);
+ });
+
it('gives each object its own patch of the atlas', () => {
const a = baked(floor(4));
const b = baked(floor(4));
@@ -243,6 +266,27 @@ function brightestOf(result: ReturnType<typeof bakeLightmap>, surface: number):
return best;
}
+/** The channels of the brightest texel in one surface's patch. */
+function brightestChannels(result: ReturnType<typeof bakeLightmap>,
+ surface: number): [number, number, number] {
+ const [su, sv, ou, ov] = result.scaleOffset[surface];
+ let best = -1;
+ let out: [number, number, number] = [0, 0, 0];
+ const x0 = Math.floor(ou * result.size), x1 = Math.ceil((ou + su) * result.size);
+ const y0 = Math.floor(ov * result.size), y1 = Math.ceil((ov + sv) * result.size);
+ for (let y = y0; y < y1; y++) {
+ for (let x = x0; x < x1; x++) {
+ const at = (y * result.size + x) * 4;
+ const sum = result.pixels[at] + result.pixels[at + 1] + result.pixels[at + 2];
+ if (sum > best) {
+ best = sum;
+ out = [result.pixels[at], result.pixels[at + 1], result.pixels[at + 2]];
+ }
+ }
+ }
+ return out;
+}
+
/** Brightness of the floor at world z, on its centre line. */
function sampleFloor(result: ReturnType<typeof bakeLightmap>, mesh: MeshData, z: number): number {
const uv = uvAtWorld(mesh, 0, 0, z);
diff --git a/tools/releaseNotes.mjs b/tools/releaseNotes.mjs
index d30244ffb..c56a3ac15 100644
--- a/tools/releaseNotes.mjs
+++ b/tools/releaseNotes.mjs
@@ -28,6 +28,11 @@ export const NOTES_FLOOR = 'v0.65.0';
* the reader recognises anyway.
*/
export const NOTED = {
+ 'feat(lighting): a bounce carries the colour of what it came off':
+ { note: 'A bounce carries the colour of what it came off.' },
+ 'feat(render): a surface reflects the colour it actually has':
+ { note: 'A bounce carries the colour of what it came off.' },
+
'feat(lighting): Bake Lighting, and the atlas the scene reads it from':
{ note: 'Bake Lighting.' },
'chore(editor): the bake reaches the menu':