Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(agent): the two capabilities the dogfood runs actually asked for

yhhfdaa148ed85a
diff --git a/desktop/scripts/editor-capabilities-e2e.mjs b/desktop/scripts/editor-capabilities-e2e.mjs
index 5a1efcbad..eebfe9405 100644
--- a/desktop/scripts/editor-capabilities-e2e.mjs
+++ b/desktop/scripts/editor-capabilities-e2e.mjs
@@ -42,7 +42,8 @@ const failures = await withEditor(async (ed) => {
const tools = (await ed.rpc('tools/list', {}, 30000)).result?.tools ?? [];
const names = tools.map((t) => t.name);
- for (const cap of ['create_prefab', 'configure_physics_body', 'wire_ui_event', 'create_behavior']) {
+ for (const cap of ['create_hud_text', 'playtest', 'create_prefab', 'configure_physics_body',
+ 'wire_ui_event', 'create_behavior']) {
check(names.includes(cap), `tools/list is missing the capability ${cap}`);
}
console.log(`tools/list OK — ${names.length} entries, capabilities among them`);
@@ -112,6 +113,46 @@ const failures = await withEditor(async (ed) => {
check(Array.isArray(rows) && rows.length === 2, `bindings read back: ${JSON.stringify(rows)}`);
console.log('wire_ui_event OK — two wires, neither dropped');
+ // — create_hud_text: on the SCREEN, anchored, and not in the world —
+ const hud = json(await ed.call('create_hud_text', {
+ text: 'SCORE 0', at: 'top-left', margin: 16, fontSize: 28, color: '#ffcc00ff',
+ }, 60000));
+ check(typeof hud?.entity === 'number', `create_hud_text gave no entity: ${JSON.stringify(hud)}`);
+ check(typeof hud?.canvas === 'number', `create_hud_text found/made no canvas: ${JSON.stringify(hud)}`);
+
+ const label = json(await ed.call('get_inspector', { entity: hud.entity }, 30000));
+ const lc = new Map((label?.components ?? []).map((c) => [c.type, c]));
+ check(lc.has('UINode') && lc.has('Text'), `the label is not a UI text: ${[...lc.keys()]}`);
+ const lf = (type, key) => (lc.get(type)?.fields ?? []).find((f) => f.key === key)?.value;
+ check(String(lf('Text', 'content')) === 'SCORE 0', `content: ${JSON.stringify(lf('Text', 'content'))}`);
+ // Absolute, or the insets below are decoration.
+ check(String(lf('UINode', 'position')).toLowerCase().includes('absolute') || lf('UINode', 'position') === 1,
+ `position did not become Absolute: ${JSON.stringify(lf('UINode', 'position'))}`);
+ // The trap this exists for: a dimension left Auto IGNORES its number.
+ const inset = lf('UINode', 'insetTop');
+ const insetValue = typeof inset === 'object' && inset ? inset.value : inset;
+ const insetUnit = typeof inset === 'object' && inset ? inset.unit : undefined;
+ check(insetValue === 16, `insetTop.value: ${JSON.stringify(inset)}`);
+ check(insetUnit !== 2, `insetTop stayed Auto, so its 16 does nothing: ${JSON.stringify(inset)}`);
+ // And no warning, because nothing wrote a layout-owned field.
+ check(!hud.warnings, `create_hud_text tripped a layout warning: ${JSON.stringify(hud.warnings)}`);
+ console.log('create_hud_text OK — anchored UI text, insets in px, no layout-owned write');
+
+ // A second one reuses the canvas the first made rather than stacking another.
+ const hud2 = json(await ed.call('create_hud_text', { text: 'LIVES 3', at: 'top-right' }, 60000));
+ check(hud2?.canvas === hud.canvas, `the second label made its own canvas: ${hud2?.canvas} vs ${hud.canvas}`);
+ check(hud2?.createdCanvas === false, 'the second label reports creating a canvas');
+ console.log('create_hud_text OK — one canvas, both labels under it');
+
+ // — playtest: run it and come back with a picture —
+ const played = json(await ed.call('playtest', { frames: 10, cols: 24, probe: '1 + 1' }, 120000));
+ check(played?.enteredPlay === true, `playtest did not enter play: ${JSON.stringify(played?.enteredPlay)}`);
+ check(String(played?.probe ?? '').includes('2'), `probe did not evaluate: ${JSON.stringify(played?.probe)}`);
+ check(typeof played?.picture === 'string' && played.picture.length > 20,
+ `playtest brought back no text picture: ${JSON.stringify(played?.picture)?.slice(0, 120)}`);
+ console.log(`playtest OK — entered play, probed, ${String(played.picture).length} chars of picture`);
+ await ed.call('toggle_play', {}, 60000);
+
// A capability's failure names the step, not just the message.
const bad = await ed.call('configure_physics_body', { entity: 999999, body: 'dynamic' }, 30000).catch((e) => ({ text: String(e) }));
check(/step /.test(String(bad?.text ?? '')), `a failing capability did not name its step: ${JSON.stringify(bad)}`);
diff --git a/desktop/shared/capabilityCatalog.mjs b/desktop/shared/capabilityCatalog.mjs
index ed47d705b..6eff3ec82 100644
--- a/desktop/shared/capabilityCatalog.mjs
+++ b/desktop/shared/capabilityCatalog.mjs
@@ -39,7 +39,140 @@ const obj = (properties, required = []) => ({ type: 'object', properties, requir
* runs: a capability that writes a file is irreversible however undoable its
* other steps are, because that is the one an undo cannot rescue.
*/
+/** Where on the screen a HUD element sits, as insets from the edges it hugs. */
+const HUD_ANCHORS = {
+ 'top-left': { insetTop: 1, insetLeft: 1, align: 'Left', verticalAlign: 'Top' },
+ top: { insetTop: 1, insetLeft: 0, insetRight: 0, align: 'Center', verticalAlign: 'Top' },
+ 'top-right': { insetTop: 1, insetRight: 1, align: 'Right', verticalAlign: 'Top' },
+ left: { insetLeft: 1, insetTop: 0, insetBottom: 0, align: 'Left', verticalAlign: 'Middle' },
+ center: { insetLeft: 0, insetRight: 0, insetTop: 0, insetBottom: 0, align: 'Center', verticalAlign: 'Middle' },
+ right: { insetRight: 1, insetTop: 0, insetBottom: 0, align: 'Right', verticalAlign: 'Middle' },
+ 'bottom-left': { insetBottom: 1, insetLeft: 1, align: 'Left', verticalAlign: 'Bottom' },
+ bottom: { insetBottom: 1, insetLeft: 0, insetRight: 0, align: 'Center', verticalAlign: 'Bottom' },
+ 'bottom-right': { insetBottom: 1, insetRight: 1, align: 'Right', verticalAlign: 'Bottom' },
+};
+
export const CAPABILITIES = [
+ {
+ name: 'create_hud_text',
+ effect: 'undoable',
+ description:
+ 'Put a label on the SCREEN — score, lives, a wave counter — anchored to a corner or edge and staying there at '
+ + 'any window size. This is not the same job as a label in the world, and doing it with world coordinates is the '
+ + 'single most repeated way a HUD ends up invisible: the world origin is the middle of the view, so screen-style '
+ + 'coordinates put a HUD hundreds of units above the top edge, and `Transform.position` on a UI node is OWNED BY '
+ + 'THE LAYOUT — it is accepted, overwritten at the next relayout, and nothing reports it. '
+ + '`at` is one of top-left, top, top-right, left, center, right, bottom-left, bottom, bottom-right. '
+ + 'Reuses the scene\'s Canvas or makes one. For a label that belongs IN the world (a nameplate over a unit), use '
+ + 'a Sprite-side Text instead — there Transform.position is exactly how it is placed.',
+ schema: obj({
+ text: { type: 'string' },
+ at: { type: 'string', description: 'corner or edge (default top-left)' },
+ margin: { type: 'number', description: 'px from the edges it hugs (default 24)' },
+ fontSize: { type: 'number' },
+ color: { type: 'string', description: '"#rrggbbaa"' },
+ name: { type: 'string', description: 'entity name (defaults to the text)' },
+ canvas: { type: 'number', description: 'an existing Canvas entity to put it under' },
+ }, ['text']),
+ async run(input, call) {
+ const at = input.at ?? 'top-left';
+ const anchor = HUD_ANCHORS[at];
+ if (!anchor) {
+ throw new Error(`create_hud_text: \`at\` must be one of ${Object.keys(HUD_ANCHORS).join(', ')}`);
+ }
+ const margin = input.margin ?? 24;
+
+ let canvas = input.canvas;
+ if (canvas == null) {
+ // A Canvas is a root in every scene that has one, and `kind` cannot tell
+ // one from any other UI node — so the roots get asked, and only the roots.
+ const tree = await call('get_scene_tree', {});
+ const roots = Array.isArray(tree) ? tree : tree?.nodes ?? tree?.children ?? [];
+ for (const node of roots) {
+ const entity = await call('get_entity', { id: node.id });
+ if ((entity?.components ?? []).some((c) => (c.type ?? c) === 'Canvas')) { canvas = node.id; break; }
+ }
+ }
+
+ const fields = {
+ 'UINode.position': 'Absolute',
+ 'Text.content': input.text,
+ 'Text.align': anchor.align,
+ 'Text.verticalAlign': anchor.verticalAlign,
+ };
+ // Writing a dimension's `value` gives it a px unit; a dimension left Auto
+ // IGNORES the number, which is how three labels end up stacked at one point.
+ for (const side of ['insetTop', 'insetRight', 'insetBottom', 'insetLeft']) {
+ if (anchor[side] === undefined) continue;
+ fields[`UINode.${side}.value`] = anchor[side] * margin;
+ }
+ if (input.fontSize !== undefined) fields['Text.fontSize'] = input.fontSize;
+ if (input.color !== undefined) fields['Text.color'] = input.color;
+
+ const ops = [];
+ if (canvas == null) ops.push({ op: 'create', ref: 'canvas', template: 'canvas', name: 'Canvas' });
+ ops.push({
+ op: 'create',
+ ref: 'label',
+ template: 'ui-text',
+ name: input.name ?? input.text,
+ parent: canvas == null ? '$canvas' : canvas,
+ fields,
+ });
+
+ const built = await call('apply_scene_ops', { ops, label: 'Create HUD text' });
+ return {
+ entity: built.refs.label,
+ canvas: canvas ?? built.refs.canvas,
+ at,
+ createdCanvas: canvas == null,
+ ...(built.warnings ? { warnings: built.warnings } : {}),
+ };
+ },
+ },
+
+ {
+ name: 'playtest',
+ effect: 'irreversible',
+ description:
+ 'Run the game and LOOK at it: enters Play if it is not already running, sends the input you name, steps a fixed '
+ + 'number of frames, then reads the world and returns a text picture of the screen. '
+ + 'This is the loop that tells you whether what you built works, and skipping the looking half is how a run ends '
+ + 'with every call reporting success against an empty screen. The picture comes back as TEXT (a coarse colour '
+ + 'grid), so it is readable whether or not you can see images. '
+ + '`probe` is an expression evaluated in the running game — `find`, `get`, `set`, `resource` are in scope. '
+ + '`input` is a list of { kind, x, y, button, code } as play_input takes them.',
+ schema: obj({
+ frames: { type: 'number', description: 'frames to step after the input (default 30)' },
+ input: { type: 'array', description: 'play_input specs, sent in order before stepping' },
+ probe: { type: 'string', description: 'an expression to evaluate in the running game afterwards' },
+ cols: { type: 'number', description: 'text-picture width in cells (default 48)' },
+ rows: { type: 'number', description: 'text-picture height in cells' },
+ }),
+ async run(input, call) {
+ const before = await call('get_play_state', {});
+ const wasPlaying = before?.playing === true;
+ if (!wasPlaying) await call('toggle_play', {});
+
+ for (const one of input.input ?? []) await call('play_input', one);
+
+ const stepped = await call('step', { frames: input.frames ?? 30 });
+ const probe = input.probe === undefined ? undefined : await call('play_probe', { code: input.probe });
+ const picture = await call('screenshot', {
+ format: 'grid',
+ ...(input.cols !== undefined ? { cols: input.cols } : {}),
+ ...(input.rows !== undefined ? { rows: input.rows } : {}),
+ });
+
+ return {
+ enteredPlay: !wasPlaying,
+ stepped,
+ ...(probe !== undefined ? { probe } : {}),
+ picture,
+ };
+ },
+ },
+
{
name: 'create_prefab',
effect: 'irreversible',
diff --git a/desktop/src/engine/sceneOps.ts b/desktop/src/engine/sceneOps.ts
index 83d466fce..f497d3994 100644
--- a/desktop/src/engine/sceneOps.ts
+++ b/desktop/src/engine/sceneOps.ts
@@ -96,6 +96,12 @@ function layoutOwnedWarning(entity: EntityId, component: string, key: string): s
+ 'where Transform.position is exactly how it is placed.';
}
+/** The component types a template puts on the entity it creates. */
+function rootComponentTypes(prefab: PrefabData): string[] {
+ const root = prefab.entities.find((e) => e.prefabEntityId === prefab.rootEntityId) ?? prefab.entities[0];
+ return (root?.components ?? []).map((c) => c.type);
+}
+
/**
* Split a `"Component.key"` field path. Component names never contain a dot, but
* keys do (`Transform.position.x`), so only the FIRST dot separates them.
@@ -197,15 +203,28 @@ export async function applySceneOps(
// not on entity 3" — true, and no help at all when the same op
// supports `x`/`y`, which quietly supply the Transform this one
// just dropped. Say which side of that the caller landed on.
- const declared = new Set(specs.map((c) => (typeof c === 'string' ? c : c.type)));
+ //
+ // A TEMPLATE brings its own components, so it is asked what they
+ // are: `specs` is not about it, and its default would refuse
+ // `{template: 'ui-text', fields: {'Text.content': …}}`.
+ const declared = new Set(
+ op.template
+ ? rootComponentTypes(prefab)
+ : specs.map((c) => (typeof c === 'string' ? c : c.type)),
+ );
const missing = [...new Set(Object.keys(op.fields).map((k) => k.split('.')[0]))]
.filter((c) => !declared.has(c));
if (missing.length > 0) {
throw new Error(
- `fields write to ${missing.map((c) => `"${c}"`).join(', ')}, which this create does `
- + `not declare (it declares ${[...declared].map((c) => `"${c}"`).join(', ')}). Naming any `
- + '`components` replaces the default ["Transform"] — list every component your fields '
- + `write to${missing.includes('Transform') ? ', or give x/y instead of Transform.position' : ''}.`,
+ op.template
+ ? `fields write to ${missing.map((c) => `"${c}"`).join(', ')}, which the template `
+ + `"${op.template}" does not put on its root (it has `
+ + `${[...declared].map((c) => `"${c}"`).join(', ')}). Add the component in a later op, `
+ + 'or build the entity from `components` instead of a template.'
+ : `fields write to ${missing.map((c) => `"${c}"`).join(', ')}, which this create does `
+ + `not declare (it declares ${[...declared].map((c) => `"${c}"`).join(', ')}). Naming any `
+ + '`components` replaces the default ["Transform"] — list every component your fields '
+ + `write to${missing.includes('Transform') ? ', or give x/y instead of Transform.position' : ''}.`,
);
}
setFields(id, op.fields);
diff --git a/desktop/tests/capabilities.test.ts b/desktop/tests/capabilities.test.ts
index 5e6071ffd..746b00efb 100644
--- a/desktop/tests/capabilities.test.ts
+++ b/desktop/tests/capabilities.test.ts
@@ -31,7 +31,9 @@ function fakeDriver(answers: Record<string, unknown> = {}) {
};
driver.js = vi.fn(async (code: string) => {
calls.push({ method: 'js', args: [code] });
- return answers.js;
+ // Several tools ride `js`; a test that needs to tell them apart answers with
+ // a function of the code rather than one value for all of them.
+ return typeof answers.js === 'function' ? (answers.js as (c: string) => unknown)(code) : answers.js;
});
driver.op = vi.fn(async (op: string, input: unknown) => {
calls.push({ method: op, args: [input] });
@@ -217,3 +219,97 @@ describe('create_behavior', () => {
expect(calls.map((c) => c.method)).toEqual(['js']);
});
});
+
+describe('create_hud_text', () => {
+ it('reuses the scene canvas rather than stacking another', async () => {
+ const { driver, calls } = fakeDriver({
+ getSceneTree: [{ id: 1, name: 'World' }, { id: 2, name: 'Canvas' }],
+ getEntity: undefined,
+ applyOps: { refs: { label: 9 }, created: [9], applied: 1 },
+ });
+ // Only entity 2 answers as a Canvas.
+ let asked = 0;
+ const orig = driver as unknown as (m: string, a: unknown[]) => Promise<unknown>;
+ const spy = (async (m: string, a: unknown[]) => {
+ if (m === 'getEntity') { asked++; return { components: asked === 2 ? [{ type: 'Canvas' }] : [{ type: 'Sprite' }] }; }
+ return orig(m, a);
+ }) as never;
+ (spy as unknown as { js: unknown }).js = (driver as unknown as { js: unknown }).js;
+ (spy as unknown as { op: unknown }).op = (driver as unknown as { op: unknown }).op;
+
+ const res = await runTool(capability('create_hud_text'), spy, { text: 'SCORE 0' });
+ expect(res.isError).toBeFalsy();
+ expect(JSON.parse(text(res))).toMatchObject({ canvas: 2, createdCanvas: false });
+ const ops = (calls.find((c) => c.method === 'applyOps')!.args[0] as Array<{ op: string; template?: string }>);
+ expect(ops.some((o) => o.template === 'canvas')).toBe(false);
+ });
+
+ it('makes a canvas when the scene has none', async () => {
+ const { driver, calls } = fakeDriver({
+ getSceneTree: [{ id: 1, name: 'World' }],
+ getEntity: { components: [{ type: 'Sprite' }] },
+ applyOps: { refs: { canvas: 4, label: 5 }, created: [4, 5], applied: 2 },
+ });
+ const res = await runTool(capability('create_hud_text'), driver, { text: 'SCORE 0' });
+ expect(JSON.parse(text(res))).toMatchObject({ canvas: 4, entity: 5, createdCanvas: true });
+ const ops = (calls.find((c) => c.method === 'applyOps')!.args[0] as Array<{ template?: string }>);
+ expect(ops[0].template).toBe('canvas');
+ });
+
+ it('anchors by insets in px, never by a layout-owned Transform', async () => {
+ const { driver, calls } = fakeDriver({
+ getSceneTree: [], applyOps: { refs: { canvas: 1, label: 2 }, created: [1, 2], applied: 2 },
+ });
+ await runTool(capability('create_hud_text'), driver, { text: 'LIVES', at: 'bottom-right', margin: 12 });
+
+ const ops = calls.find((c) => c.method === 'applyOps')!.args[0] as Array<{ fields?: Record<string, unknown> }>;
+ const fields = ops[1].fields!;
+ expect(fields['UINode.position']).toBe('Absolute');
+ expect(fields['UINode.insetBottom.value']).toBe(12);
+ expect(fields['UINode.insetRight.value']).toBe(12);
+ expect(fields['UINode.insetTop.value']).toBeUndefined();
+ expect(Object.keys(fields).some((k) => k.startsWith('Transform.'))).toBe(false);
+ });
+
+ it('refuses a corner nobody has, naming the ones that exist', async () => {
+ const { driver } = fakeDriver();
+ const res = await runTool(capability('create_hud_text'), driver, { text: 'x', at: 'middle-ish' });
+ expect(res.isError).toBe(true);
+ expect(text(res)).toMatch(/must be one of top-left/);
+ });
+});
+
+describe('playtest', () => {
+ it('enters play, sends input, steps, probes and brings back a picture', async () => {
+ const { driver, calls } = fakeDriver({
+ playState: { playing: false },
+ step: { world: 'play', frames: 10 },
+ play_probe: '42',
+ screenshot: 'AAAA\nBBBB',
+ });
+ const res = await runTool(capability('playtest'), driver, {
+ frames: 10, probe: 'score', input: [{ kind: 'keyDown', code: 'Space' }],
+ });
+
+ expect(calls.map((c) => c.method)).toEqual([
+ 'playState', 'play', 'play_input', 'step', 'play_probe', 'screenshot',
+ ]);
+ expect(JSON.parse(text(res))).toMatchObject({ enteredPlay: true, probe: '42', picture: 'AAAA\nBBBB' });
+ });
+
+ it('does not toggle play on a game already running', async () => {
+ const { driver, calls } = fakeDriver({ playState: { playing: true }, screenshot: 'x' });
+ const res = await runTool(capability('playtest'), driver, {});
+
+ expect(calls.map((c) => c.method)).not.toContain('play');
+ expect(JSON.parse(text(res)).enteredPlay).toBe(false);
+ });
+
+ it('always asks for the picture as TEXT, so a model that cannot see still sees', async () => {
+ const { driver, calls } = fakeDriver({ playState: { playing: true }, screenshot: 'x' });
+ await runTool(capability('playtest'), driver, {});
+
+ const shot = calls.find((c) => c.method === 'screenshot')!;
+ expect((shot.args[0] as { format: string }).format).toBe('grid');
+ });
+});
diff --git a/desktop/tests/scene-ops.test.ts b/desktop/tests/scene-ops.test.ts
index a26c35183..d1d9a0847 100644
--- a/desktop/tests/scene-ops.test.ts
+++ b/desktop/tests/scene-ops.test.ts
@@ -55,7 +55,19 @@ vi.mock('@/engine/EditorSession', () => ({
vi.mock('@/engine/entitySources', () => ({
prefabFromSpecs: (name: string, specs: { type: string }[]) => ({ name, specs }),
sourceById: (id: string) => {
- if (id === 'ui-image') return { build: async () => ({ name: 'Image', specs: [] }) };
+ if (id === 'ui-image') {
+ return {
+ build: async () => ({
+ name: 'Image',
+ specs: [],
+ rootEntityId: 'root',
+ entities: [{
+ prefabEntityId: 'root',
+ components: [{ type: 'Transform' }, { type: 'UINode' }, { type: 'UIVisual' }],
+ }],
+ }),
+ };
+ }
// A project prefab source: it carries the ref that makes the result an
// instance instead of a copy.
if (id === 'prefab:assets/prefabs/Bar.esprefab') {
@@ -139,6 +151,24 @@ describe('applySceneOps', () => {
expect(calls).toContainEqual(['create', 'Line', null]);
});
+ it('reads a TEMPLATE own components, rather than the default it never used', async () => {
+ // A template create supplies no `components`, so the declaration check has
+ // to ask the template — not the default it never used.
+ calls.length = 0;
+ await applySceneOps([{
+ op: 'create', ref: 'img', template: 'ui-image', name: 'Icon',
+ fields: { 'UINode.insetTop.value': 16 },
+ }]);
+ expect(calls.some((c) => c[0] === 'create')).toBe(true);
+ expect(calls.some((c) => c[0] === 'setField' && c[2] === 'UINode' && c[3] === 'insetTop.value' && c[4] === 16)).toBe(true);
+ });
+
+ it('names the template when a field writes to something it does not carry', async () => {
+ await expect(applySceneOps([{
+ op: 'create', template: 'ui-image', fields: { 'RigidBody.gravityScale': 0 },
+ }])).rejects.toThrow(/template "ui-image" does not put on its root/);
+ });
+
it('resolves an entity template before mutating, and rejects an unknown one', async () => {
await applySceneOps([{ op: 'create', ref: 'img', template: 'ui-image', name: 'Icon' }]);
expect(calls).toContainEqual(['create', 'Image', null]);