Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(editor): report where you are looking — status bar, checkpoint bar, Outliner

esengineda3d6bfbf924
diff --git a/desktop/src/App.tsx b/desktop/src/App.tsx
index 9964061f2..083dfe852 100644
--- a/desktop/src/App.tsx
+++ b/desktop/src/App.tsx
@@ -8,6 +8,7 @@ import { DockLayout } from '@/layout/DockLayout';
import { ActivityBar } from '@/layout/ActivityBar';
import { ContentDrawer } from '@/layout/ContentDrawer';
import { AgentDrawer } from '@/components/AgentDrawer';
+import { AgentCheckpoint } from '@/components/AgentCheckpoint';
import '@/engine/EditorSession'; // side effect: constructs defaultSession → wires the editor engine
import '@/document/dirtyDocs'; // side effect: registers scene + asset docs on the DirtyRegistry
import { Launcher } from '@/launcher/Launcher';
@@ -297,6 +298,7 @@ export function App() {
<main className="shell__workspace">
<ErrorBoundary label="activitybar"><Perf id="activitybar"><ActivityBar /></Perf></ErrorBoundary>
<DockLayout />
+ <ErrorBoundary label="agentcheckpoint"><AgentCheckpoint /></ErrorBoundary>
<ErrorBoundary label="agentdrawer"><Perf id="agentdrawer"><AgentDrawer /></Perf></ErrorBoundary>
</main>
<ErrorBoundary label="statusbar"><Perf id="statusbar"><StatusBar /></Perf></ErrorBoundary>
diff --git a/desktop/src/components/AgentCheckpoint.tsx b/desktop/src/components/AgentCheckpoint.tsx
new file mode 100644
index 000000000..442c39b56
--- /dev/null
+++ b/desktop/src/components/AgentCheckpoint.tsx
@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file AgentCheckpoint.tsx
+ * @brief The one gesture that takes a whole agent turn back, floating over the
+ * viewport where the result of that turn actually is.
+ *
+ * It appears only when the turn left steps behind. A bar offering to undo zero
+ * steps is a claim the user has to disprove by clicking it, and the kernel
+ * already reports 0 for exactly that reason.
+ *
+ * The revert runs in this window, not through main: the checkpoint is a mark in
+ * THIS EditorHistory, and the agent that made the edits deliberately has no tool
+ * for rolling them back — a model that can undo its own work can undo the user's
+ * and call it a correction.
+ */
+import { useSyncExternalStore } from 'react';
+import { Undo2, Check } from 'lucide-react';
+import { useAgent, dismissCheckpoint } from '@/store/AgentStore';
+import { EditorHistory } from '@/engine/EditorHistory';
+import type { HistoryMark } from '@/engine/EditorHistory';
+import { Toasts } from '@/store/Toasts';
+import { t } from '@/i18n';
+
+export function AgentCheckpoint() {
+ const turns = useAgent((s) => s.turns);
+ const done = useAgent((s) => s.checkpointDone);
+ // Re-read across every edit: what an Undo would now take back changes as the
+ // user keeps working, and the bar has to say so before they press it.
+ const version = useSyncExternalStore(EditorHistory.subscribe, EditorHistory.getVersion);
+
+ const last = turns[turns.length - 1];
+ if (!last || last.reason === null || last.steps === 0 || done === last.id) return null;
+ const mark = last.mark as HistoryMark | null;
+ if (!mark) return null;
+
+ // More steps on the stack than the turn recorded means the user has edited
+ // since. Undo would take those back too, so say so rather than quietly doing it.
+ void version;
+ const now = EditorHistory.stepsSince(mark);
+ const stale = now > last.steps;
+
+ return (
+ <div className={`ag-cp${stale ? ' stale' : ''}`}>
+ <span className="ag-cp-what">{last.prompt}</span>
+ <span className="ag-cp-stat">{t('agent.checkpoint.steps', { count: last.steps })}</span>
+ {stale && <span className="ag-cp-stale">{t('agent.checkpoint.stale')}</span>}
+ <span className="ag-cp-acts">
+ <button
+ type="button"
+ onClick={() => {
+ const undone = EditorHistory.undoToMark(mark);
+ dismissCheckpoint(last.id);
+ Toasts.push(t('agent.checkpoint.undone', { count: undone }), 'info');
+ }}
+ >
+ <Undo2 size={13} strokeWidth={1.9} />{t('agent.checkpoint.undo')}
+ </button>
+ <button type="button" className="keep" onClick={() => dismissCheckpoint(last.id)}>
+ <Check size={13} strokeWidth={2} />{t('agent.checkpoint.keep')}
+ </button>
+ </span>
+ </div>
+ );
+}
diff --git a/desktop/src/components/AgentDrawer.tsx b/desktop/src/components/AgentDrawer.tsx
index 6e92fe985..093d212ed 100644
--- a/desktop/src/components/AgentDrawer.tsx
+++ b/desktop/src/components/AgentDrawer.tsx
@@ -31,6 +31,7 @@ import {
import { useEditorStore } from '@/store/editorStore';
import {
useAgent, sendAgentMessage, stopAgentTurn, confirmAgentCall, resetAgentSession,
+ peekEntities, entitiesInInput,
type AgentTurn, type AgentEntry, type AgentToolEntry,
} from '@/store/AgentStore';
import { t } from '@/i18n';
@@ -45,9 +46,14 @@ function ToolRow({ entry, onConfirm }: { entry: AgentToolEntry; onConfirm: (allo
const [open, setOpen] = useState(false);
const Icon = TOOL_ICON[entry.effect ?? 'read'] ?? Eye;
const arg = summarizeInput(entry.input);
+ const touches = entitiesInInput(entry.input);
return (
<>
- <div className={`ag-call ag-call--${entry.state}`}>
+ <div
+ className={`ag-call ag-call--${entry.state}`}
+ onMouseEnter={() => touches.length && peekEntities(touches)}
+ onMouseLeave={() => touches.length && peekEntities([])}
+ >
<button type="button" className="ag-call-top" onClick={() => setOpen((o) => !o)}>
<span className="ag-call-g"><Icon size={13} strokeWidth={1.8} /></span>
<span className="ag-call-nm">{entry.name}</span>
diff --git a/desktop/src/layout/StatusBar.tsx b/desktop/src/layout/StatusBar.tsx
index 5adba1e9d..a445cde89 100644
--- a/desktop/src/layout/StatusBar.tsx
+++ b/desktop/src/layout/StatusBar.tsx
@@ -1,9 +1,10 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
import { useSyncExternalStore } from 'react';
-import { Gauge, MousePointer2, Boxes, FolderOpen, MemoryStick, Cpu } from 'lucide-react';
+import { Gauge, MousePointer2, Boxes, FolderOpen, MemoryStick, Cpu, Sparkles, Square } from 'lucide-react';
import { useEditorStore } from '@/store/editorStore';
import { useSelection } from '@/store/selectionStore';
+import { useAgent, stopAgentTurn } from '@/store/AgentStore';
import { StatsStore } from '@/engine/StatsStore';
import { EngineHost } from '@/engine/EngineHost';
import { SubsystemIndicator } from './SubsystemIndicator';
@@ -57,6 +58,33 @@ function CursorReadout() {
return <>{cursor ? `${cursor.x}, ${cursor.y}` : '—'}</>;
}
+/**
+ * What the agent is doing, where "what is happening right now" already lives.
+ *
+ * It reports here rather than only in the drawer because the two minutes an
+ * agent works are two minutes you spend watching the VIEWPORT — so progress has
+ * to be somewhere you can see with the drawer closed, and Stop has to be within
+ * reach without reopening it.
+ */
+function AgentSegment() {
+ const phase = useAgent((s) => s.status.phase);
+ const setDrawer = useEditorStore((s) => s.setAgentDrawer);
+ if (phase === 'idle') return null;
+ const waiting = phase === 'awaiting_confirm';
+ return (
+ <span className={`sitem ag-seg${waiting ? ' waiting' : ''}`}>
+ <button type="button" className="ag-seg-open" onClick={() => setDrawer(true)}>
+ <Sparkles size={11} strokeWidth={1.9} />
+ {waiting ? t('agent.status.awaiting') : t('agent.status.running')}
+ </button>
+ <button type="button" className="ag-seg-stop" title={t('agent.stop')} onClick={stopAgentTurn}>
+ <Square size={9} strokeWidth={2.6} />
+ {t('agent.stop')}
+ </button>
+ </span>
+ );
+}
+
// Bottom status strip — live engine telemetry (real FPS / entity count / cursor
// world position) reads in the mono face. Anchors the Content Drawer.
export function StatusBar() {
@@ -87,6 +115,7 @@ export function StatusBar() {
{selectedIds.size ? t('layout.status.selected', { count: selectedIds.size }) : t('layout.status.noSelection')}
</span>
<SelectionReadout />
+ <AgentSegment />
</div>
<span className="sp" />
diff --git a/desktop/src/outliner/OutlinerRow.tsx b/desktop/src/outliner/OutlinerRow.tsx
index 50d882b22..4574eaa56 100644
--- a/desktop/src/outliner/OutlinerRow.tsx
+++ b/desktop/src/outliner/OutlinerRow.tsx
@@ -37,6 +37,10 @@ export interface OutlinerRowProps {
item: OutlinerItem;
/** Entity selection highlight (folders highlight via the folder selection). */
selected: boolean;
+ /** The built-in agent changed this entity in the open conversation. */
+ agentTouched?: boolean;
+ /** The pointer is over a transcript row that names this entity. */
+ agentPeeked?: boolean;
/** Keyboard-focus row (shows a focus ring; distinct from selection). */
cursored?: boolean;
/** Lowercased substring to highlight in the name (the search bare text). */
@@ -95,7 +99,9 @@ function OutlinerRowInner(props: OutlinerRowProps) {
`${isFolder ? ' folder' : ''}` +
`${dropPos === 'on' ? ' drop' : ''}` +
`${dropPos === 'before' ? ' drop-before' : ''}` +
- `${dropPos === 'after' ? ' drop-after' : ''}`
+ `${dropPos === 'after' ? ' drop-after' : ''}` +
+ `${props.agentTouched ? ' agent-touched' : ''}` +
+ `${props.agentPeeked ? ' agent-peek' : ''}`
}
style={{ paddingLeft: depth * 14 }}
role="treeitem"
diff --git a/desktop/src/panels/Outliner.tsx b/desktop/src/panels/Outliner.tsx
index 9164c3bad..bc1b27939 100644
--- a/desktop/src/panels/Outliner.tsx
+++ b/desktop/src/panels/Outliner.tsx
@@ -20,6 +20,7 @@ import { t } from '@/i18n';
import { buildOutlinerItems, collectExpandableKeys, entityKey, folderKey, parseQuery, type OutlinerItem, type SortMode } from '@/outliner/OutlinerModel';
import { useOutliner } from '@/outliner/OutlinerController';
import { OutlinerRow } from '@/outliner/OutlinerRow';
+import { useAgent, touchedEntities } from '@/store/AgentStore';
import { OUTLINER_COLUMNS, TYPE_COLUMN, type OutlinerColumnContext } from '@/outliner/columns';
import { joinFolder, folderParent, folderName, normalizeFolder, isFolderUnder } from '@/outliner/folders';
import type { EntityId } from '@/types';
@@ -164,6 +165,13 @@ export function Outliner() {
);
const flatIds = useMemo(() => entityIds(items), [items]);
const highlight = useMemo(() => parseQuery(query).text, [query]);
+ // What the agent changed, and what the pointer is over in its transcript. The
+ // link the drawer alone cannot make: a tool row naming `id: 7` means nothing
+ // until 7 lights up in the tree you were already looking at.
+ const agentTurns = useAgent((s) => s.turns);
+ const agentPeeked = useAgent((s) => s.peeked);
+ const agentAcked = useAgent((s) => s.checkpointDone);
+ const agentTouched = useMemo(() => touchedEntities(agentTurns, agentAcked), [agentTurns, agentAcked]);
const activeColumns = useMemo(() => OUTLINER_COLUMNS.filter((c) => !hiddenColumns.has(c.id)), [hiddenColumns]);
const columnCtx = useMemo<OutlinerColumnContext>(
() => ({
@@ -656,6 +664,8 @@ export function Outliner() {
selected={it.kind === 'folder' ? selectedFolder === it.path : selectedIds.has(it.id)}
cursored={cursor === it.key}
highlight={highlight}
+ agentTouched={it.kind === 'entity' && agentTouched.has(it.id)}
+ agentPeeked={it.kind === 'entity' && agentPeeked.includes(it.id)}
renaming={renaming === it.key}
dropPos={drop?.key === it.key ? drop.pos : undefined}
prefabRole={
diff --git a/desktop/src/store/AgentStore.ts b/desktop/src/store/AgentStore.ts
index 4f7b62e37..e6c69ff80 100644
--- a/desktop/src/store/AgentStore.ts
+++ b/desktop/src/store/AgentStore.ts
@@ -90,9 +90,22 @@ interface AgentState {
* has to be up before main can call back in — see {@link sendAgentMessage}.
*/
driving: boolean;
+ /**
+ * Entities under the pointer in the transcript, echoed into the Outliner and
+ * the viewport. The link a general-purpose chat UI cannot make: a tool row
+ * naming `id: 7` means nothing until 7 lights up where you already look.
+ */
+ peeked: readonly number[];
+ /** The turn whose checkpoint bar has been answered — see AgentCheckpoint. */
+ checkpointDone: number | null;
}
-export const useAgent = create<AgentState>(() => ({ status: IDLE, turns: [], driving: false }));
+export const useAgent = create<AgentState>(() => ({
+ status: IDLE, turns: [], driving: false, peeked: [], checkpointDone: null,
+}));
+
+export const peekEntities = (ids: readonly number[]): void => useAgent.setState({ peeked: ids });
+export const dismissCheckpoint = (turnId: number): void => useAgent.setState({ checkpointDone: turnId });
/** The run still taking events, if any. Only the last one can be open. */
const openTurn = (turns: readonly AgentTurn[]): AgentTurn | null => {
@@ -286,6 +299,50 @@ export async function resetAgentSession(): Promise<void> {
if (status) useAgent.setState({ status }); // not adoptStatus: reset just ended it
}
+/**
+ * The entity ids a call's arguments name.
+ *
+ * Read off the catalog's own naming convention — `entity`, `id`, `ids` — rather
+ * than a per-tool table, because a table of "which argument of which tool is an
+ * entity" is a second definition of the catalog that would drift from it. The
+ * convention misses a tool that names its argument something else, and that is
+ * the right way to be wrong here: a badge that appears late is a smaller lie
+ * than one that points at the wrong entity.
+ */
+export function entitiesInInput(input: Record<string, unknown>): number[] {
+ const out: number[] = [];
+ for (const key of ['entity', 'id', 'ids', 'entities']) {
+ const value = input[key];
+ if (typeof value === 'number') out.push(value);
+ else if (Array.isArray(value)) out.push(...value.filter((v): v is number => typeof v === 'number'));
+ }
+ return out;
+}
+
+/**
+ * What the conversation has changed and you have not acknowledged yet.
+ *
+ * Answering the checkpoint — Undo or Keep — clears the badges for that turn and
+ * everything before it. The transcript still says what happened; the badge is
+ * about what still wants your eye, and after Undo it did not happen at all.
+ */
+export function touchedEntities(
+ turns: readonly AgentTurn[],
+ acknowledgedUpTo: number | null,
+): ReadonlySet<number> {
+ const touched = new Set<number>();
+ for (const turn of turns) {
+ if (acknowledgedUpTo !== null && turn.id <= acknowledgedUpTo) continue;
+ for (const entry of turn.entries) {
+ // Only calls that ran and changed something: a queued or failed call
+ // touched nothing, and a read never does.
+ if (entry.kind !== 'tool' || entry.state !== 'ok' || entry.effect === 'read') continue;
+ for (const id of entitiesInInput(entry.input)) touched.add(id);
+ }
+ }
+ return touched;
+}
+
/** Whether an agent conversation authorizes the automation hook (main.tsx). */
export const agentDriving = (): boolean => useAgent.getState().driving;
export const subscribeAgent = (fn: () => void): (() => void) => useAgent.subscribe(fn);
diff --git a/desktop/src/theme/agent.css b/desktop/src/theme/agent.css
index e0cb59eeb..e1e21990e 100644
--- a/desktop/src/theme/agent.css
+++ b/desktop/src/theme/agent.css
@@ -355,3 +355,69 @@
@media (prefers-reduced-motion: reduce) {
.ag-spin, .ag-skel, .ag-dots i { animation: none; }
}
+
+/* ── The checkpoint a finished turn leaves behind ────────────────────────────
+ Over the viewport, where the result of the turn is — not in the drawer, which
+ is the place you are NOT looking while you check the work. */
+.ag-cp {
+ position: absolute;
+ left: 50%;
+ bottom: 14px;
+ z-index: 39;
+ transform: translateX(-50%);
+ display: flex;
+ align-items: center;
+ gap: 13px;
+ padding: 7px 8px 7px 13px;
+ border-radius: var(--r-md, 6px);
+ background: var(--raised);
+ border: 1px solid var(--line-strong);
+ box-shadow: 0 16px 40px -12px rgba(0, 0, 0, 0.85);
+ font-size: var(--fs-sm);
+ white-space: nowrap;
+ max-width: min(720px, 70%);
+}
+.ag-cp-what { overflow: hidden; text-overflow: ellipsis; color: var(--text-hi); }
+.ag-cp-stat { font: var(--fs-xs) var(--font-mono); color: var(--text-faint); flex: none; }
+.ag-cp-stale { font-size: var(--fs-xs); color: var(--warn); flex: none; }
+.ag-cp-acts { display: flex; gap: 2px; flex: none; }
+.ag-cp button {
+ display: flex; align-items: center; gap: 6px;
+ padding: 5px 10px;
+ border-radius: var(--r-sm);
+ font-size: var(--fs-sm);
+ color: var(--text-dim);
+ transition: color var(--t-fast), background var(--t-fast);
+}
+.ag-cp button:hover { background: var(--hover); color: var(--text-hi); }
+.ag-cp button.keep { color: var(--text-hi); background: var(--hover); }
+.ag-cp button.keep:hover { background: var(--pressed); }
+
+/* ── The status-bar segment ─────────────────────────────────────────────── */
+.ag-seg { display: inline-flex; align-items: center; gap: 8px; }
+.ag-seg-open { display: inline-flex; align-items: center; gap: 6px; color: var(--text); }
+.ag-seg-open:hover { color: var(--text-hi); }
+.ag-seg.waiting .ag-seg-open { color: var(--warn); }
+.ag-seg-stop {
+ display: inline-flex; align-items: center; gap: 4px;
+ padding: 1px 7px 1px 5px;
+ border-radius: var(--r-sm);
+ background: var(--hover);
+ color: var(--text-dim);
+ font-size: var(--fs-2xs);
+}
+.ag-seg-stop:hover { color: var(--text-hi); background: var(--pressed); }
+
+/* ── Outliner echo ──────────────────────────────────────────────────────────
+ A dot, not a tint: the row already uses background for selection and for the
+ drop target, and a third meaning on the same channel reads as none of them. */
+.row.agent-touched .rname::after {
+ content: '';
+ display: inline-block;
+ width: 4px; height: 4px;
+ margin-left: 7px;
+ border-radius: 50%;
+ background: var(--nebula);
+ vertical-align: middle;
+}
+.row.agent-peek { box-shadow: inset 2px 0 0 var(--nebula); }
diff --git a/desktop/tests/agent-transcript.test.ts b/desktop/tests/agent-transcript.test.ts
index 7a84ebacf..bb27a9591 100644
--- a/desktop/tests/agent-transcript.test.ts
+++ b/desktop/tests/agent-transcript.test.ts
@@ -8,7 +8,10 @@
* claiming they are still running.
*/
import { describe, it, expect } from 'vitest';
-import { applyAgentEvent, type AgentTurn, type AgentToolEntry } from '../src/store/AgentStore';
+import {
+ applyAgentEvent, entitiesInInput, touchedEntities,
+ type AgentTurn, type AgentToolEntry,
+} from '../src/store/AgentStore';
import type { AgentEvent, ToolCall } from '../electron/agent/types';
const fold = (...events: AgentEvent[]): AgentTurn[] =>
@@ -83,10 +86,10 @@ describe('the transcript projection', () => {
{ type: 'tool_start', call: call('save_scene'), effect: 'irreversible' },
{
type: 'awaiting_confirm',
- request: { callId: 'c-save_scene', tool: 'save_scene', reason: 'writes outside the scene', input: {} },
+ request: { callId: 'c-save_scene', tool: 'save_scene', reason: 'irreversible', input: {} },
},
);
- expect(tools(turn)[0]).toMatchObject({ state: 'awaiting', reason: 'writes outside the scene' });
+ expect(tools(turn)[0]).toMatchObject({ state: 'awaiting', reason: 'irreversible' });
});
// The kernel reports a decline as an ordinary failed call, because that is what
@@ -150,3 +153,43 @@ describe('the transcript projection', () => {
expect(applyAgentEvent(before, { type: 'stop', reason: 'tool_use' })).toEqual(before);
});
});
+
+// The Outliner echo. Read off the catalog's own argument names rather than a
+// per-tool table, which would be a second definition of the catalog.
+describe('which entities a turn touched', () => {
+ const toolEntry = (over: Partial<AgentToolEntry>): AgentToolEntry => ({
+ kind: 'tool', id: 'c1', name: 'set_field', input: {}, effect: 'undoable',
+ state: 'ok', summary: 'ok', reason: null, ...over,
+ });
+ const turn = (entries: AgentToolEntry[], id = 0): AgentTurn => ({
+ id, prompt: 'p', entries, inputTokens: 0, outputTokens: 0, steps: 1, mark: { seq: 1 }, reason: 'end_turn',
+ });
+
+ it('reads the ids out of the argument names the catalog uses', () => {
+ expect(entitiesInInput({ entity: 7, component: 'Transform' })).toEqual([7]);
+ expect(entitiesInInput({ id: 3 })).toEqual([3]);
+ expect(entitiesInInput({ ids: [4, 5], path: 'x' })).toEqual([4, 5]);
+ expect(entitiesInInput({ name: 'PauseRoot' })).toEqual([]);
+ });
+
+ it('counts only calls that ran and changed something', () => {
+ expect([...touchedEntities([turn([
+ toolEntry({ id: 'a', input: { entity: 1 } }),
+ toolEntry({ id: 'b', input: { entity: 2 }, state: 'error' }),
+ toolEntry({ id: 'c', input: { entity: 3 }, effect: 'read' }),
+ toolEntry({ id: 'd', input: { entity: 4 }, state: 'queued' }),
+ ])], null)]).toEqual([1]);
+ });
+
+ // The transcript still says what happened; the badge is about what still wants
+ // your eye, and after Undo it did not happen at all.
+ it('forgets a turn once its checkpoint has been answered', () => {
+ const turns = [
+ turn([toolEntry({ id: 'a', input: { entity: 1 } })], 0),
+ turn([toolEntry({ id: 'b', input: { entity: 2 } })], 1),
+ ];
+ expect([...touchedEntities(turns, null)]).toEqual([1, 2]);
+ expect([...touchedEntities(turns, 0)]).toEqual([2]);
+ expect([...touchedEntities(turns, 1)]).toEqual([]);
+ });
+});