Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(editor): the agent can edit the asset editors, not just the scene

esengine0f0e09bdfea3
diff --git a/desktop/shared/toolCatalog.mjs b/desktop/shared/toolCatalog.mjs
index a4ec63c5b..d5bad1566 100644
--- a/desktop/shared/toolCatalog.mjs
+++ b/desktop/shared/toolCatalog.mjs
@@ -313,6 +313,55 @@ export const TOOLS = [
.then((text) => window.__estellaEditor.applyOps(JSON.parse(text), ${JSON.stringify(i.label ?? null)}))`
: null),
method: 'applyOps', args: (i) => [i.ops, i.label], root: 'editor' },
+ // — The open asset editors. One set of doors for all eight kinds, because they
+ // share AssetDocument: `open_asset` puts one on screen, these read and write
+ // it. A clip, a timeline, a tileset and a material graph differ in what they
+ // HOLD, not in how they are edited, so a tool set per kind would be the same
+ // three verbs written eight times and drifting apart. —
+ { name: 'list_asset_documents', effect: 'read',
+ description: 'Which asset editors are open right now (animation clip, timeline, tileset, material, material graph, animator/state/behaviour graph), with their file path and whether they have unsaved edits. Use it to learn the `docId` the other two tools take when more than one is open.',
+ schema: obj({}),
+ js: () => 'window.__estellaEditor.assetDocuments()', root: 'editor' },
+ { name: 'get_asset_document', effect: 'read',
+ description: "An open asset editor's document, in full, exactly as the editor holds it — the frames of a clip, the tracks of a timeline, the tiles of a tileset, the nodes of a graph. THIS is how you read those files: read_project_file gives you bytes on disk, which are stale the moment the editor has unsaved edits. Omit `docId` when only one is open.",
+ schema: obj({ docId: { type: 'string' } }),
+ js: (i) => `window.__estellaEditor.getAssetDocument(${JSON.stringify(i.docId ?? null)} ?? undefined)`,
+ root: 'editor' },
+ { name: 'edit_asset_document', effect: 'undoable',
+ description: 'Write fields of an open asset editor\'s document as ONE undo step. `changes` is [{path, value}], where `path` is dotted and addresses the typed asset the same way set_field addresses a component: "frames.0.duration", "tracks.2.keys.5.value", "nodes.3.position.x". '
+ + 'A numeric segment indexes an array. Paths that do not already exist are REFUSED rather than created — these are typed documents whose editors rely on their shape, and inventing a field produces a file that loads as something else. Read it with get_asset_document first. '
+ + 'It goes through the same door the editor\'s own UI writes through, so the panel updates and one Undo takes the whole call back.',
+ schema: obj({
+ changes: { type: 'array', description: '[{ path, value }], applied in order as one undo step' },
+ docId: { type: 'string', description: 'which open document, when more than one is' },
+ label: { type: 'string', description: 'undo-stack label (default "Edit asset")' },
+ }, ['changes']),
+ js: (i) => `window.__estellaEditor.editAssetDocument(${JSON.stringify(i.changes)}, ${JSON.stringify(i.docId ?? null)} ?? undefined, ${JSON.stringify(i.label ?? null)} ?? undefined)`,
+ root: 'editor' },
+
+ // — Tilemaps. Painting is not a field write: a cell is addressed by grid
+ // coordinate, and a stroke is hundreds of them in one undo step. —
+ { name: 'create_tilemap', effect: 'undoable',
+ description: 'Create a tilemap entity from a tileset asset (project-relative .estileset path). `grid` optionally overrides the tileset\'s own cell size/shape: { tileWidth, tileHeight, orientation }. Returns the new entity id.',
+ schema: obj({
+ tilesetPath: { type: 'string' },
+ grid: { type: 'object', description: 'optional { tileWidth, tileHeight, orientation }' },
+ }, ['tilesetPath']),
+ js: (i) => `window.__estellaEditor.createTilemap(${JSON.stringify(i.tilesetPath)}, ${JSON.stringify(i.grid ?? null)} ?? undefined)`,
+ root: 'editor' },
+ { name: 'paint_tiles', effect: 'undoable',
+ description: 'Paint cells of a tilemap layer in ONE undo step. `edits` is [{x, y, tileId}] in GRID coordinates (not pixels); tileId 0 clears a cell. `source` is the tilemap entity id. This is the tool for laying out a level — set_field cannot reach a cell.',
+ schema: obj({
+ source: { type: 'number', description: 'the tilemap entity' },
+ edits: { type: 'array', description: '[{ x, y, tileId }] in grid coordinates' },
+ }, ['source', 'edits']),
+ js: (i) => `window.__estellaEditor.paintTiles(${i.source}, ${JSON.stringify(i.edits)})`,
+ root: 'editor' },
+ { name: 'get_tile_collision', effect: 'read',
+ description: 'The collision shapes a tilemap currently generates, for checking that a painted level is walkable before running it.',
+ schema: obj({ source: { type: 'number' } }, ['source']),
+ js: (i) => `window.__estellaEditor.probeTileCollision(${i.source})`, root: 'editor' },
+
{ name: 'refresh_assets',
description: 'Re-scan the project into the asset registry. The editor watches the filesystem, but a batch written from outside (a converter copying twenty sprites in) can outrun the watcher — and until the scan lands those files have no `.meta`, so `set_import_settings` and any @uuid ref to them fail. Cheap and idempotent; call it after writing assets by hand.',
schema: obj({}),
diff --git a/desktop/src/document/assetDocumentOps.ts b/desktop/src/document/assetDocumentOps.ts
new file mode 100644
index 000000000..d19c24a57
--- /dev/null
+++ b/desktop/src/document/assetDocumentOps.ts
@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file assetDocumentOps.ts — reading and writing an OPEN asset document by path.
+ *
+ * The scene has `apply_scene_ops`; the eight asset editors (animation clips,
+ * timelines, tilesets, materials, material graphs, animator/state/behaviour
+ * graphs) had nothing an agent could reach. They did not need eight tool sets:
+ * they all extend {@link AssetDocument}, whose `edit(label, mutate)` is already
+ * the one undoable write door each editor's own UI goes through. What was
+ * missing was a way to name a field of the typed asset from outside — a closure
+ * does not cross a tool call.
+ *
+ * So the address is a dotted path, the same shape `set_field` uses for a
+ * component ("frames.0.duration"), and one call is one undo step. Nothing here
+ * knows what a clip or a tileset IS; it walks whatever the document holds. That
+ * is what keeps a ninth editor from needing a ninth tool.
+ */
+import { AssetDocument } from './AssetDocument';
+
+/** One open editor document, as the agent sees it in a listing. */
+export interface AssetDocumentInfo {
+ docId: string;
+ path: string | null;
+ dirty: boolean;
+}
+
+/** A field write: `"frames.0.duration"` → value. */
+export interface AssetDocumentChange {
+ path: string;
+ value: unknown;
+}
+
+export const openAssetDocuments = (): AssetDocumentInfo[] =>
+ AssetDocument.openDocuments().map((d) => ({ docId: d.docId, path: d.filePath, dirty: d.dirty }));
+
+/**
+ * The document to act on: the named one, or the only one open.
+ *
+ * Refusing when several are open and none was named is deliberate — picking
+ * "the first" would edit whichever editor happened to be constructed first,
+ * which is not a thing the caller can predict or see.
+ */
+function resolve(docId?: string): AssetDocument<unknown> {
+ const open = AssetDocument.openDocuments();
+ if (open.length === 0) throw new Error('no asset document is open (open_asset first)');
+ if (docId) {
+ const found = open.find((d) => d.docId === docId);
+ if (!found) {
+ throw new Error(`no open asset document with docId "${docId}" (open: ${open.map((d) => d.docId).join(', ')})`);
+ }
+ return found;
+ }
+ if (open.length > 1) {
+ throw new Error(`several asset documents are open — name one with docId: ${open.map((d) => d.docId).join(', ')}`);
+ }
+ return open[0];
+}
+
+/** An open document's typed content, exactly as its editor holds it. */
+export function readAssetDocument(docId?: string): { docId: string; path: string | null; asset: unknown } {
+ const doc = resolve(docId);
+ return { docId: doc.docId, path: doc.filePath, asset: doc.asset };
+}
+
+/**
+ * Walk a dotted path and write the leaf, creating nothing.
+ *
+ * Missing segments are an ERROR rather than an autovivified object: the assets
+ * here are typed documents with schemas their editors rely on, and inventing
+ * `frames.7` on a clip with three frames produces a file that loads as
+ * something else. Numeric segments index arrays; everything else is a key.
+ */
+export function setByPath(root: unknown, path: string, value: unknown): void {
+ const parts = path.split('.').filter((p) => p !== '');
+ if (parts.length === 0) throw new Error('empty path');
+
+ let at: unknown = root;
+ for (let i = 0; i < parts.length - 1; i++) {
+ const key = parts[i];
+ if (at === null || typeof at !== 'object') {
+ throw new Error(`"${path}": ${parts.slice(0, i).join('.') || '<root>'} is not an object`);
+ }
+ const next = (at as Record<string, unknown>)[key];
+ if (next === undefined) throw new Error(`"${path}": no ${parts.slice(0, i + 1).join('.')}`);
+ at = next;
+ }
+
+ const last = parts[parts.length - 1];
+ if (at === null || typeof at !== 'object') throw new Error(`"${path}": parent is not an object`);
+ if (Array.isArray(at)) {
+ const index = Number(last);
+ if (!Number.isInteger(index) || index < 0 || index >= at.length) {
+ throw new Error(`"${path}": index ${last} is outside an array of ${at.length}`);
+ }
+ at[index] = value;
+ return;
+ }
+ if (!(last in (at as Record<string, unknown>))) {
+ throw new Error(`"${path}": no such field (present: ${Object.keys(at as object).join(', ')})`);
+ }
+ (at as Record<string, unknown>)[last] = value;
+}
+
+/**
+ * Apply `changes` to an open document as ONE undo step.
+ *
+ * Through the document's own `edit`, so the panel re-renders and the undo entry
+ * is indistinguishable from one the editor's UI made — there is no second way to
+ * write these files.
+ */
+export function editAssetDocument(
+ changes: readonly AssetDocumentChange[],
+ docId?: string,
+ label = 'Edit asset',
+): { docId: string; applied: number } {
+ if (!Array.isArray(changes) || changes.length === 0) throw new Error('`changes` must be a non-empty array');
+ const doc = resolve(docId);
+ if (!doc.asset) throw new Error(`asset document "${doc.docId}" holds nothing`);
+
+ doc.edit(label, (draft) => {
+ changes.forEach((change, i) => {
+ if (!change || typeof change.path !== 'string') throw new Error(`changes[${i}] needs a "path"`);
+ setByPath(draft, change.path, change.value);
+ });
+ });
+ return { docId: doc.docId, applied: changes.length };
+}
diff --git a/desktop/src/main.tsx b/desktop/src/main.tsx
index 3d4d9d62c..2e4e9b667 100644
--- a/desktop/src/main.tsx
+++ b/desktop/src/main.tsx
@@ -71,6 +71,9 @@ import { mcpStatus, subscribeMcp } from './store/McpStore';
import { attachAgentBridge, agentDriving, subscribeAgent } from './store/AgentStore';
import { guardAutomationHook } from './engine/automationGate';
import { activeMode } from './mode/activeMode';
+import {
+ openAssetDocuments, readAssetDocument, editAssetDocument, type AssetDocumentChange,
+} from './document/assetDocumentOps';
// Register the built-in settings (side effect) and replay persisted ones.
import './settings';
import { applySettings } from './store/settingsStore';
@@ -403,6 +406,14 @@ function buildEditorAutomation(): unknown {
*/
documentState,
editorSetup,
+
+ // — The open asset editors (clips, timelines, tilesets, materials, graphs).
+ // One set of doors for all eight, because they share AssetDocument: the
+ // typed asset is read whole and written by dotted path, one undo step. —
+ assetDocuments: () => openAssetDocuments(),
+ getAssetDocument: (docId?: string) => readAssetDocument(docId),
+ editAssetDocument: (changes: AssetDocumentChange[], docId?: string, label?: string) =>
+ editAssetDocument(changes, docId, label),
/** Leave Prefab Mode — the banner's "Back to Scene". Refuses on unsaved prefab
* changes unless `discardChanges` (see openAsset). */
exitPrefabMode: async (discardChanges = false) => {
diff --git a/desktop/tests/asset-document-ops.test.ts b/desktop/tests/asset-document-ops.test.ts
new file mode 100644
index 000000000..6ddfbb2d5
--- /dev/null
+++ b/desktop/tests/asset-document-ops.test.ts
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file Addressing a field of an open asset document from outside.
+ *
+ * The eight asset editors share AssetDocument, whose `edit(label, mutate)`
+ * is already the one undoable write door — but a closure does not cross a
+ * tool call, so the address is a dotted path instead. These pin the part
+ * that has to be strict: a path that does not already exist is refused
+ * rather than created, because these are typed documents whose editors
+ * rely on their shape.
+ */
+import { describe, it, expect } from 'vitest';
+import { setByPath } from '@/document/assetDocumentOps';
+
+const clip = () => ({
+ fps: 12,
+ loop: true,
+ frames: [
+ { duration: 1, sprite: 'a.png' },
+ { duration: 2, sprite: 'b.png' },
+ ],
+ meta: { author: null as string | null },
+});
+
+describe('addressing an asset document by path', () => {
+ it('writes a top-level field', () => {
+ const doc = clip();
+ setByPath(doc, 'fps', 24);
+ expect(doc.fps).toBe(24);
+ });
+
+ it('indexes into an array on the way down', () => {
+ const doc = clip();
+ setByPath(doc, 'frames.1.duration', 5);
+ expect(doc.frames[1].duration).toBe(5);
+ expect(doc.frames[0].duration).toBe(1);
+ });
+
+ it('replaces a whole array element', () => {
+ const doc = clip();
+ setByPath(doc, 'frames.0', { duration: 9, sprite: 'z.png' });
+ expect(doc.frames[0]).toEqual({ duration: 9, sprite: 'z.png' });
+ });
+
+ it('writes a field whose current value is null', () => {
+ const doc = clip();
+ setByPath(doc, 'meta.author', 'me');
+ expect(doc.meta.author).toBe('me');
+ });
+
+ // The strict half. Autovivifying `frames.7` on a clip with two frames produces
+ // a file that loads as something else — refusing is the only honest answer.
+ it('refuses a field the document does not have', () => {
+ expect(() => setByPath(clip(), 'speed', 2)).toThrow(/no such field/);
+ // and says what IS there, so the caller can correct itself
+ expect(() => setByPath(clip(), 'speed', 2)).toThrow(/fps/);
+ });
+
+ it('refuses an index past the end of an array', () => {
+ expect(() => setByPath(clip(), 'frames.7.duration', 1)).toThrow(/no frames\.7/);
+ expect(() => setByPath(clip(), 'frames.2', {})).toThrow(/outside an array of 2/);
+ });
+
+ it('refuses to walk through a leaf', () => {
+ expect(() => setByPath(clip(), 'fps.value', 1)).toThrow(/not an object/);
+ });
+
+ it('refuses an empty path', () => {
+ expect(() => setByPath(clip(), '', 1)).toThrow(/empty path/);
+ expect(() => setByPath(clip(), '..', 1)).toThrow(/empty path/);
+ });
+});