A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(agent): the two capabilities the dogfood runs actually asked for
yhhfdaa148ed85a
diff --git a/desktop/scripts/editor-capabilities-e2e.mjs b/desktop/scripts/editor-capabilities-e2e.mjsindex 5a1efcbad..eebfe9405 100644--- a/desktop/scripts/editor-capabilities-e2e.mjs+++ b/desktop/scripts/editor-capabilities-e2e.mjs@@ -42,7 +42,8 @@ const failures = await withEditor(async (ed) => {const tools = (await ed.rpc('tools/list', {}, 30000)).result?.tools ?? [];const names = tools.map((t) => t.name);- for (const cap of ['create_prefab', 'configure_physics_body', 'wire_ui_event', 'create_behavior']) {+ for (const cap of ['create_hud_text', 'playtest', 'create_prefab', 'configure_physics_body',+ 'wire_ui_event', 'create_behavior']) {check(names.includes(cap), `tools/list is missing the capability ${cap}`);}console.log(`tools/list OK — ${names.length} entries, capabilities among them`);@@ -112,6 +113,46 @@ const failures = await withEditor(async (ed) => {check(Array.isArray(rows) && rows.length === 2, `bindings read back: ${JSON.stringify(rows)}`);console.log('wire_ui_event OK — two wires, neither dropped');+ // — create_hud_text: on the SCREEN, anchored, and not in the world —+ const hud = json(await ed.call('create_hud_text', {+ text: 'SCORE 0', at: 'top-left', margin: 16, fontSize: 28, color: '#ffcc00ff',+ }, 60000));+ check(typeof hud?.entity === 'number', `create_hud_text gave no entity: ${JSON.stringify(hud)}`);+ check(typeof hud?.canvas === 'number', `create_hud_text found/made no canvas: ${JSON.stringify(hud)}`);++ const label = json(await ed.call('get_inspector', { entity: hud.entity }, 30000));+ const lc = new Map((label?.components ?? []).map((c) => [c.type, c]));+ check(lc.has('UINode') && lc.has('Text'), `the label is not a UI text: ${[...lc.keys()]}`);+ const lf = (type, key) => (lc.get(type)?.fields ?? []).find((f) => f.key === key)?.value;+ check(String(lf('Text', 'content')) === 'SCORE 0', `content: ${JSON.stringify(lf('Text', 'content'))}`);+ // Absolute, or the insets below are decoration.+ check(String(lf('UINode', 'position')).toLowerCase().includes('absolute') || lf('UINode', 'position') === 1,+ `position did not become Absolute: ${JSON.stringify(lf('UINode', 'position'))}`);+ // The trap this exists for: a dimension left Auto IGNORES its number.+ const inset = lf('UINode', 'insetTop');+ const insetValue = typeof inset === 'object' && inset ? inset.value : inset;+ const insetUnit = typeof inset === 'object' && inset ? inset.unit : undefined;+ check(insetValue === 16, `insetTop.value: ${JSON.stringify(inset)}`);+ check(insetUnit !== 2, `insetTop stayed Auto, so its 16 does nothing: ${JSON.stringify(inset)}`);+ // And no warning, because nothing wrote a layout-owned field.+ check(!hud.warnings, `create_hud_text tripped a layout warning: ${JSON.stringify(hud.warnings)}`);+ console.log('create_hud_text OK — anchored UI text, insets in px, no layout-owned write');++ // A second one reuses the canvas the first made rather than stacking another.+ const hud2 = json(await ed.call('create_hud_text', { text: 'LIVES 3', at: 'top-right' }, 60000));+ check(hud2?.canvas === hud.canvas, `the second label made its own canvas: ${hud2?.canvas} vs ${hud.canvas}`);+ check(hud2?.createdCanvas === false, 'the second label reports creating a canvas');+ console.log('create_hud_text OK — one canvas, both labels under it');++ // — playtest: run it and come back with a picture —+ const played = json(await ed.call('playtest', { frames: 10, cols: 24, probe: '1 + 1' }, 120000));+ check(played?.enteredPlay === true, `playtest did not enter play: ${JSON.stringify(played?.enteredPlay)}`);+ check(String(played?.probe ?? '').includes('2'), `probe did not evaluate: ${JSON.stringify(played?.probe)}`);+ check(typeof played?.picture === 'string' && played.picture.length > 20,+ `playtest brought back no text picture: ${JSON.stringify(played?.picture)?.slice(0, 120)}`);+ console.log(`playtest OK — entered play, probed, ${String(played.picture).length} chars of picture`);+ await ed.call('toggle_play', {}, 60000);+// A capability's failure names the step, not just the message.const bad = await ed.call('configure_physics_body', { entity: 999999, body: 'dynamic' }, 30000).catch((e) => ({ text: String(e) }));check(/step /.test(String(bad?.text ?? '')), `a failing capability did not name its step: ${JSON.stringify(bad)}`);diff --git a/desktop/shared/capabilityCatalog.mjs b/desktop/shared/capabilityCatalog.mjsindex ed47d705b..6eff3ec82 100644--- a/desktop/shared/capabilityCatalog.mjs+++ b/desktop/shared/capabilityCatalog.mjs@@ -39,7 +39,140 @@ const obj = (properties, required = []) => ({ type: 'object', properties, requir* runs: a capability that writes a file is irreversible however undoable its* other steps are, because that is the one an undo cannot rescue.*/+/** Where on the screen a HUD element sits, as insets from the edges it hugs. */+const HUD_ANCHORS = {+ 'top-left': { insetTop: 1, insetLeft: 1, align: 'Left', verticalAlign: 'Top' },+ top: { insetTop: 1, insetLeft: 0, insetRight: 0, align: 'Center', verticalAlign: 'Top' },+ 'top-right': { insetTop: 1, insetRight: 1, align: 'Right', verticalAlign: 'Top' },+ left: { insetLeft: 1, insetTop: 0, insetBottom: 0, align: 'Left', verticalAlign: 'Middle' },+ center: { insetLeft: 0, insetRight: 0, insetTop: 0, insetBottom: 0, align: 'Center', verticalAlign: 'Middle' },+ right: { insetRight: 1, insetTop: 0, insetBottom: 0, align: 'Right', verticalAlign: 'Middle' },+ 'bottom-left': { insetBottom: 1, insetLeft: 1, align: 'Left', verticalAlign: 'Bottom' },+ bottom: { insetBottom: 1, insetLeft: 0, insetRight: 0, align: 'Center', verticalAlign: 'Bottom' },+ 'bottom-right': { insetBottom: 1, insetRight: 1, align: 'Right', verticalAlign: 'Bottom' },+};+export const CAPABILITIES = [+ {+ name: 'create_hud_text',+ effect: 'undoable',+ description:+ 'Put a label on the SCREEN — score, lives, a wave counter — anchored to a corner or edge and staying there at '+ + 'any window size. This is not the same job as a label in the world, and doing it with world coordinates is the '+ + 'single most repeated way a HUD ends up invisible: the world origin is the middle of the view, so screen-style '+ + 'coordinates put a HUD hundreds of units above the top edge, and `Transform.position` on a UI node is OWNED BY '+ + 'THE LAYOUT — it is accepted, overwritten at the next relayout, and nothing reports it. '+ + '`at` is one of top-left, top, top-right, left, center, right, bottom-left, bottom, bottom-right. '+ + 'Reuses the scene\'s Canvas or makes one. For a label that belongs IN the world (a nameplate over a unit), use '+ + 'a Sprite-side Text instead — there Transform.position is exactly how it is placed.',+ schema: obj({+ text: { type: 'string' },+ at: { type: 'string', description: 'corner or edge (default top-left)' },+ margin: { type: 'number', description: 'px from the edges it hugs (default 24)' },+ fontSize: { type: 'number' },+ color: { type: 'string', description: '"#rrggbbaa"' },+ name: { type: 'string', description: 'entity name (defaults to the text)' },+ canvas: { type: 'number', description: 'an existing Canvas entity to put it under' },+ }, ['text']),+ async run(input, call) {+ const at = input.at ?? 'top-left';+ const anchor = HUD_ANCHORS[at];+ if (!anchor) {+ throw new Error(`create_hud_text: \`at\` must be one of ${Object.keys(HUD_ANCHORS).join(', ')}`);+ }+ const margin = input.margin ?? 24;++ let canvas = input.canvas;+ if (canvas == null) {+ // A Canvas is a root in every scene that has one, and `kind` cannot tell+ // one from any other UI node — so the roots get asked, and only the roots.+ const tree = await call('get_scene_tree', {});+ const roots = Array.isArray(tree) ? tree : tree?.nodes ?? tree?.children ?? [];+ for (const node of roots) {+ const entity = await call('get_entity', { id: node.id });+ if ((entity?.components ?? []).some((c) => (c.type ?? c) === 'Canvas')) { canvas = node.id; break; }+ }+ }++ const fields = {+ 'UINode.position': 'Absolute',+ 'Text.content': input.text,+ 'Text.align': anchor.align,+ 'Text.verticalAlign': anchor.verticalAlign,+ };+ // Writing a dimension's `value` gives it a px unit; a dimension left Auto+ // IGNORES the number, which is how three labels end up stacked at one point.+ for (const side of ['insetTop', 'insetRight', 'insetBottom', 'insetLeft']) {+ if (anchor[side] === undefined) continue;+ fields[`UINode.${side}.value`] = anchor[side] * margin;+ }+ if (input.fontSize !== undefined) fields['Text.fontSize'] = input.fontSize;+ if (input.color !== undefined) fields['Text.color'] = input.color;++ const ops = [];+ if (canvas == null) ops.push({ op: 'create', ref: 'canvas', template: 'canvas', name: 'Canvas' });+ ops.push({+ op: 'create',+ ref: 'label',+ template: 'ui-text',+ name: input.name ?? input.text,+ parent: canvas == null ? '$canvas' : canvas,+ fields,+ });++ const built = await call('apply_scene_ops', { ops, label: 'Create HUD text' });+ return {+ entity: built.refs.label,+ canvas: canvas ?? built.refs.canvas,+ at,+ createdCanvas: canvas == null,+ ...(built.warnings ? { warnings: built.warnings } : {}),+ };+ },+ },++ {+ name: 'playtest',+ effect: 'irreversible',+ description:+ 'Run the game and LOOK at it: enters Play if it is not already running, sends the input you name, steps a fixed '+ + 'number of frames, then reads the world and returns a text picture of the screen. '+ + 'This is the loop that tells you whether what you built works, and skipping the looking half is how a run ends '+ + 'with every call reporting success against an empty screen. The picture comes back as TEXT (a coarse colour '+ + 'grid), so it is readable whether or not you can see images. '+ + '`probe` is an expression evaluated in the running game — `find`, `get`, `set`, `resource` are in scope. '+ + '`input` is a list of { kind, x, y, button, code } as play_input takes them.',+ schema: obj({+ frames: { type: 'number', description: 'frames to step after the input (default 30)' },+ input: { type: 'array', description: 'play_input specs, sent in order before stepping' },+ probe: { type: 'string', description: 'an expression to evaluate in the running game afterwards' },+ cols: { type: 'number', description: 'text-picture width in cells (default 48)' },+ rows: { type: 'number', description: 'text-picture height in cells' },+ }),+ async run(input, call) {+ const before = await call('get_play_state', {});+ const wasPlaying = before?.playing === true;+ if (!wasPlaying) await call('toggle_play', {});++ for (const one of input.input ?? []) await call('play_input', one);++ const stepped = await call('step', { frames: input.frames ?? 30 });+ const probe = input.probe === undefined ? undefined : await call('play_probe', { code: input.probe });+ const picture = await call('screenshot', {+ format: 'grid',+ ...(input.cols !== undefined ? { cols: input.cols } : {}),+ ...(input.rows !== undefined ? { rows: input.rows } : {}),+ });++ return {+ enteredPlay: !wasPlaying,+ stepped,+ ...(probe !== undefined ? { probe } : {}),+ picture,+ };+ },+ },+{name: 'create_prefab',effect: 'irreversible',diff --git a/desktop/src/engine/sceneOps.ts b/desktop/src/engine/sceneOps.tsindex 83d466fce..f497d3994 100644--- a/desktop/src/engine/sceneOps.ts+++ b/desktop/src/engine/sceneOps.ts@@ -96,6 +96,12 @@ function layoutOwnedWarning(entity: EntityId, component: string, key: string): s+ 'where Transform.position is exactly how it is placed.';}+/** The component types a template puts on the entity it creates. */+function rootComponentTypes(prefab: PrefabData): string[] {+ const root = prefab.entities.find((e) => e.prefabEntityId === prefab.rootEntityId) ?? prefab.entities[0];+ return (root?.components ?? []).map((c) => c.type);+}+/*** Split a `"Component.key"` field path. Component names never contain a dot, but* keys do (`Transform.position.x`), so only the FIRST dot separates them.@@ -197,15 +203,28 @@ export async function applySceneOps(// not on entity 3" — true, and no help at all when the same op// supports `x`/`y`, which quietly supply the Transform this one// just dropped. Say which side of that the caller landed on.- const declared = new Set(specs.map((c) => (typeof c === 'string' ? c : c.type)));+ //+ // A TEMPLATE brings its own components, so it is asked what they+ // are: `specs` is not about it, and its default would refuse+ // `{template: 'ui-text', fields: {'Text.content': …}}`.+ const declared = new Set(+ op.template+ ? rootComponentTypes(prefab)+ : specs.map((c) => (typeof c === 'string' ? c : c.type)),+ );const missing = [...new Set(Object.keys(op.fields).map((k) => k.split('.')[0]))].filter((c) => !declared.has(c));if (missing.length > 0) {throw new Error(- `fields write to ${missing.map((c) => `"${c}"`).join(', ')}, which this create does `- + `not declare (it declares ${[...declared].map((c) => `"${c}"`).join(', ')}). Naming any `- + '`components` replaces the default ["Transform"] — list every component your fields '- + `write to${missing.includes('Transform') ? ', or give x/y instead of Transform.position' : ''}.`,+ op.template+ ? `fields write to ${missing.map((c) => `"${c}"`).join(', ')}, which the template `+ + `"${op.template}" does not put on its root (it has `+ + `${[...declared].map((c) => `"${c}"`).join(', ')}). Add the component in a later op, `+ + 'or build the entity from `components` instead of a template.'+ : `fields write to ${missing.map((c) => `"${c}"`).join(', ')}, which this create does `+ + `not declare (it declares ${[...declared].map((c) => `"${c}"`).join(', ')}). Naming any `+ + '`components` replaces the default ["Transform"] — list every component your fields '+ + `write to${missing.includes('Transform') ? ', or give x/y instead of Transform.position' : ''}.`,);}setFields(id, op.fields);diff --git a/desktop/tests/capabilities.test.ts b/desktop/tests/capabilities.test.tsindex 5e6071ffd..746b00efb 100644--- a/desktop/tests/capabilities.test.ts+++ b/desktop/tests/capabilities.test.ts@@ -31,7 +31,9 @@ function fakeDriver(answers: Record<string, unknown> = {}) {};driver.js = vi.fn(async (code: string) => {calls.push({ method: 'js', args: [code] });- return answers.js;+ // Several tools ride `js`; a test that needs to tell them apart answers with+ // a function of the code rather than one value for all of them.+ return typeof answers.js === 'function' ? (answers.js as (c: string) => unknown)(code) : answers.js;});driver.op = vi.fn(async (op: string, input: unknown) => {calls.push({ method: op, args: [input] });@@ -217,3 +219,97 @@ describe('create_behavior', () => {expect(calls.map((c) => c.method)).toEqual(['js']);});});++describe('create_hud_text', () => {+ it('reuses the scene canvas rather than stacking another', async () => {+ const { driver, calls } = fakeDriver({+ getSceneTree: [{ id: 1, name: 'World' }, { id: 2, name: 'Canvas' }],+ getEntity: undefined,+ applyOps: { refs: { label: 9 }, created: [9], applied: 1 },+ });+ // Only entity 2 answers as a Canvas.+ let asked = 0;+ const orig = driver as unknown as (m: string, a: unknown[]) => Promise<unknown>;+ const spy = (async (m: string, a: unknown[]) => {+ if (m === 'getEntity') { asked++; return { components: asked === 2 ? [{ type: 'Canvas' }] : [{ type: 'Sprite' }] }; }+ return orig(m, a);+ }) as never;+ (spy as unknown as { js: unknown }).js = (driver as unknown as { js: unknown }).js;+ (spy as unknown as { op: unknown }).op = (driver as unknown as { op: unknown }).op;++ const res = await runTool(capability('create_hud_text'), spy, { text: 'SCORE 0' });+ expect(res.isError).toBeFalsy();+ expect(JSON.parse(text(res))).toMatchObject({ canvas: 2, createdCanvas: false });+ const ops = (calls.find((c) => c.method === 'applyOps')!.args[0] as Array<{ op: string; template?: string }>);+ expect(ops.some((o) => o.template === 'canvas')).toBe(false);+ });++ it('makes a canvas when the scene has none', async () => {+ const { driver, calls } = fakeDriver({+ getSceneTree: [{ id: 1, name: 'World' }],+ getEntity: { components: [{ type: 'Sprite' }] },+ applyOps: { refs: { canvas: 4, label: 5 }, created: [4, 5], applied: 2 },+ });+ const res = await runTool(capability('create_hud_text'), driver, { text: 'SCORE 0' });+ expect(JSON.parse(text(res))).toMatchObject({ canvas: 4, entity: 5, createdCanvas: true });+ const ops = (calls.find((c) => c.method === 'applyOps')!.args[0] as Array<{ template?: string }>);+ expect(ops[0].template).toBe('canvas');+ });++ it('anchors by insets in px, never by a layout-owned Transform', async () => {+ const { driver, calls } = fakeDriver({+ getSceneTree: [], applyOps: { refs: { canvas: 1, label: 2 }, created: [1, 2], applied: 2 },+ });+ await runTool(capability('create_hud_text'), driver, { text: 'LIVES', at: 'bottom-right', margin: 12 });++ const ops = calls.find((c) => c.method === 'applyOps')!.args[0] as Array<{ fields?: Record<string, unknown> }>;+ const fields = ops[1].fields!;+ expect(fields['UINode.position']).toBe('Absolute');+ expect(fields['UINode.insetBottom.value']).toBe(12);+ expect(fields['UINode.insetRight.value']).toBe(12);+ expect(fields['UINode.insetTop.value']).toBeUndefined();+ expect(Object.keys(fields).some((k) => k.startsWith('Transform.'))).toBe(false);+ });++ it('refuses a corner nobody has, naming the ones that exist', async () => {+ const { driver } = fakeDriver();+ const res = await runTool(capability('create_hud_text'), driver, { text: 'x', at: 'middle-ish' });+ expect(res.isError).toBe(true);+ expect(text(res)).toMatch(/must be one of top-left/);+ });+});++describe('playtest', () => {+ it('enters play, sends input, steps, probes and brings back a picture', async () => {+ const { driver, calls } = fakeDriver({+ playState: { playing: false },+ step: { world: 'play', frames: 10 },+ play_probe: '42',+ screenshot: 'AAAA\nBBBB',+ });+ const res = await runTool(capability('playtest'), driver, {+ frames: 10, probe: 'score', input: [{ kind: 'keyDown', code: 'Space' }],+ });++ expect(calls.map((c) => c.method)).toEqual([+ 'playState', 'play', 'play_input', 'step', 'play_probe', 'screenshot',+ ]);+ expect(JSON.parse(text(res))).toMatchObject({ enteredPlay: true, probe: '42', picture: 'AAAA\nBBBB' });+ });++ it('does not toggle play on a game already running', async () => {+ const { driver, calls } = fakeDriver({ playState: { playing: true }, screenshot: 'x' });+ const res = await runTool(capability('playtest'), driver, {});++ expect(calls.map((c) => c.method)).not.toContain('play');+ expect(JSON.parse(text(res)).enteredPlay).toBe(false);+ });++ it('always asks for the picture as TEXT, so a model that cannot see still sees', async () => {+ const { driver, calls } = fakeDriver({ playState: { playing: true }, screenshot: 'x' });+ await runTool(capability('playtest'), driver, {});++ const shot = calls.find((c) => c.method === 'screenshot')!;+ expect((shot.args[0] as { format: string }).format).toBe('grid');+ });+});diff --git a/desktop/tests/scene-ops.test.ts b/desktop/tests/scene-ops.test.tsindex a26c35183..d1d9a0847 100644--- a/desktop/tests/scene-ops.test.ts+++ b/desktop/tests/scene-ops.test.ts@@ -55,7 +55,19 @@ vi.mock('@/engine/EditorSession', () => ({vi.mock('@/engine/entitySources', () => ({prefabFromSpecs: (name: string, specs: { type: string }[]) => ({ name, specs }),sourceById: (id: string) => {- if (id === 'ui-image') return { build: async () => ({ name: 'Image', specs: [] }) };+ if (id === 'ui-image') {+ return {+ build: async () => ({+ name: 'Image',+ specs: [],+ rootEntityId: 'root',+ entities: [{+ prefabEntityId: 'root',+ components: [{ type: 'Transform' }, { type: 'UINode' }, { type: 'UIVisual' }],+ }],+ }),+ };+ }// A project prefab source: it carries the ref that makes the result an// instance instead of a copy.if (id === 'prefab:assets/prefabs/Bar.esprefab') {@@ -139,6 +151,24 @@ describe('applySceneOps', () => {expect(calls).toContainEqual(['create', 'Line', null]);});+ it('reads a TEMPLATE own components, rather than the default it never used', async () => {+ // A template create supplies no `components`, so the declaration check has+ // to ask the template — not the default it never used.+ calls.length = 0;+ await applySceneOps([{+ op: 'create', ref: 'img', template: 'ui-image', name: 'Icon',+ fields: { 'UINode.insetTop.value': 16 },+ }]);+ expect(calls.some((c) => c[0] === 'create')).toBe(true);+ expect(calls.some((c) => c[0] === 'setField' && c[2] === 'UINode' && c[3] === 'insetTop.value' && c[4] === 16)).toBe(true);+ });++ it('names the template when a field writes to something it does not carry', async () => {+ await expect(applySceneOps([{+ op: 'create', template: 'ui-image', fields: { 'RigidBody.gravityScale': 0 },+ }])).rejects.toThrow(/template "ui-image" does not put on its root/);+ });+it('resolves an entity template before mutating, and rejects an unknown one', async () => {await applySceneOps([{ op: 'create', ref: 'img', template: 'ui-image', name: 'Icon' }]);expect(calls).toContainEqual(['create', 'Image', null]);