Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Add a render smoke for a part-masked grade and a keyed plate (part of #707)

Nguyen Xuan An39d7169d4f7f
diff --git a/package.json b/package.json
index a8f11d2f..b93dc762 100644
--- a/package.json
+++ b/package.json
@@ -178,6 +178,7 @@
"smoke:agent-vegetation": "node scripts/smoke-agent-vegetation.mjs",
"smoke:agent-render-jobs": "node scripts/smoke-agent-render-jobs.mjs",
"smoke:agent-post-pass": "node scripts/smoke-agent-post-pass.mjs",
+ "smoke:agent-compositing": "node scripts/smoke-agent-compositing.mjs",
"smoke:agent-material-snippet": "node scripts/smoke-agent-material-snippet.mjs",
"smoke:agent-texture-sequence": "node scripts/smoke-agent-texture-sequence.mjs",
"smoke:render-passes": "node scripts/smoke-render-passes.mjs",
diff --git a/scripts/smoke-agent-compositing.mjs b/scripts/smoke-agent-compositing.mjs
new file mode 100644
index 00000000..c8643541
--- /dev/null
+++ b/scripts/smoke-agent-compositing.mjs
@@ -0,0 +1,102 @@
+import assert from 'node:assert/strict';
+import { mkdtemp } from 'node:fs/promises';
+import { tmpdir, constants, setPriority } from 'node:os';
+import { join, resolve } from 'node:path';
+import sharp from 'sharp';
+import { Client } from '@modelcontextprotocol/client';
+import { StdioClientTransport } from '@modelcontextprotocol/client/stdio';
+import { createAgentPreviewFixture } from './agent-preview-fixture.mjs';
+
+// Proves two compositing steps on real pixels through the MCP server (#707): a grade masked to one part changes the
+// pixels of that part and leaves the rest of the frame alone, and a luma key over a background replaces the dark
+// backdrop with the background colour while the lit part stays as drawn.
+
+setPriority(0, constants.priority.PRIORITY_BELOW_NORMAL);
+const WIDTH = 320;
+const HEIGHT = 240;
+const HERO = 'glow';
+const directory = await mkdtemp(join(tmpdir(), '67js-mcp-compositing-'));
+const cli = resolve(process.argv[2] ?? 'bin/67js-mcp.mjs');
+let diagnostics = '';
+
+const transport = new StdioClientTransport({ command: process.execPath,
+ args: [cli, '--backend', 'cpu', '--memory', '128', '--jobs', join(directory, 'jobs')], cwd: directory, stderr: 'pipe' });
+transport.stderr.on('data', bytes => { diagnostics = (diagnostics + bytes).slice(-8192); });
+const client = new Client({ name: '67js-compositing-proof', version: '1' });
+
+async function call(name, argumentsValue = {}) {
+ const result = await client.callTool({ name, arguments: argumentsValue }, { timeout: 60000 });
+ assert.equal(result.isError, undefined, JSON.stringify(result.content).slice(0, 600));
+ return result;
+}
+
+async function pixels(sessionId, pass) {
+ const result = await call('scene_look', { sessionId, width: WIDTH, height: HEIGHT, ...(pass ? { pass } : {}) });
+ const image = result.content.find(item => item.type === 'image');
+ const { data, info } = await sharp(Buffer.from(image.data, 'base64')).removeAlpha().raw().toBuffer({ resolveWithObject: true });
+ return { data, channels: info.channels, text: result.content.filter(item => item.type === 'text').map(item => item.text).join('\n') };
+}
+
+function pixelAt(frame, index) {
+ const offset = index * frame.channels;
+ return [frame.data[offset], frame.data[offset + 1], frame.data[offset + 2]];
+}
+
+function distance(first, second) {
+ return Math.abs(first[0] - second[0]) + Math.abs(first[1] - second[1]) + Math.abs(first[2] - second[2]);
+}
+
+async function setEffects(sessionId, revision, effects) {
+ const result = (await call('scene_apply', { sessionId, expectedRevision: revision, operations: [{ type: 'set-effects', effects }] })).structuredContent;
+ assert.equal(result.renderSucceeded, true, result.renderError);
+ return result.revision;
+}
+
+try {
+ await client.connect(transport);
+ const { sessionId } = (await call('scene_create', { recipe: createAgentPreviewFixture() })).structuredContent;
+ const plain = await pixels(sessionId);
+
+ // Pixels of the hero part are the ones the unmasked grade changes most; the mask must confine the grade to them.
+ let revision = 0;
+ const gain = [0.2, 1, 3];
+ revision = await setEffects(sessionId, revision, { colorBalance: { gain } });
+ const everywhere = await pixels(sessionId);
+ revision = await setEffects(sessionId, revision, { colorBalance: { gain, mask: { objects: [HERO] } } });
+ const masked = await pixels(sessionId);
+
+ let changedByMask = 0;
+ let changedElsewhereByMask = 0;
+ let sameAsGradeEverywhere = 0;
+ const total = WIDTH * HEIGHT;
+ for (let index = 0; index < total; index++) {
+ const base = pixelAt(plain, index);
+ const graded = pixelAt(everywhere, index);
+ const limited = pixelAt(masked, index);
+ if (distance(base, limited) > 12) changedByMask++;
+ if (distance(base, graded) > 12 && distance(base, limited) <= 12) changedElsewhereByMask++;
+ if (distance(graded, limited) <= 12) sameAsGradeEverywhere++;
+ }
+ assert.ok(changedByMask > 200, `the masked grade should change the hero pixels, changed ${changedByMask}`);
+ assert.ok(changedElsewhereByMask > changedByMask, `the mask should spare more pixels than it grades: graded ${changedByMask}, spared ${changedElsewhereByMask}`);
+ assert.ok(sameAsGradeEverywhere < total, 'the masked frame should differ from the frame graded everywhere');
+
+ // A luma key over a flat background: the dark backdrop turns into the background, the bright emissive part stays.
+ const background = [0.9, 0.1, 0.1];
+ await setEffects(sessionId, revision, { composite: { background: { color: background }, key: { luma: { low: 0.3, high: 0.6 } } } });
+ const keyed = await pixels(sessionId);
+ let backgroundPixels = 0;
+ let keptPixels = 0;
+ for (let index = 0; index < total; index++) {
+ const keyedPixel = pixelAt(keyed, index);
+ if (keyedPixel[0] > keyedPixel[1] + 60 && keyedPixel[0] > keyedPixel[2] + 60) backgroundPixels++;
+ if (distance(pixelAt(plain, index), keyedPixel) <= 12) keptPixels++;
+ }
+ assert.ok(backgroundPixels > 200, `the key should show the background colour, found ${backgroundPixels} pixels`);
+ assert.ok(keptPixels > 200 && keptPixels < total, `some pixels should stay as drawn, kept ${keptPixels}`);
+
+ console.log(JSON.stringify({ hero: HERO, changedByMask, changedElsewhereByMask, backgroundPixels, keptPixels, total }));
+} finally {
+ await client.close();
+ if (diagnostics.trim()) console.error(diagnostics.trim().split('\n').slice(-5).join('\n'));
+}