Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(editor): AssetDatabase scanner + index + dep graph (REARCH_ASSETS A2)

esengine9ddc4117420d
diff --git a/desktop/electron/assetDb.ts b/desktop/electron/assetDb.ts
new file mode 100644
index 000000000..61cde9df1
--- /dev/null
+++ b/desktop/electron/assetDb.ts
@@ -0,0 +1,151 @@
+/**
+ * @file AssetDatabase scanner (REARCH_ASSETS.md A2). The single project-level
+ * index of "what assets exist": walk the project, read each `.meta`
+ * sidecar (uuid / type / importer), and build a uuid→path registry plus a
+ * dependency graph (which scene/prefab references which asset). The result
+ * is written to `.esengine/cache/assets.json` (the artifact pattern shared
+ * with schemas.json / scripts.mjs) and returned for the editor to load.
+ *
+ * The editor consumes this instead of resolving assets ad-hoc: it feeds the
+ * entries into the engine `Assets` registry (one resolution path), drives the
+ * Content Browser from it, and (later) cooks ship bundles by walking `deps`.
+ *
+ * Pure Node (fs/path), no Electron imports → unit-testable; IPC wiring is in
+ * main.ts. The `.meta` files carry the authored `type`, so this needs no
+ * extension→type table — it reads the type each meta already declares.
+ */
+import { readFile, writeFile, mkdir, readdir } from 'node:fs/promises';
+import path from 'node:path';
+
+/** Local, gitignored cache inside the project (next to workspace.json). */
+const CACHE_DIR = '.esengine/cache';
+const OUTPUT = 'assets.json';
+/** Dirs never scanned for assets — code/build/vcs, not content. */
+const SKIP_DIRS = new Set([
+ 'node_modules', '.git', '.esengine', 'dist', 'dist-electron', 'build', '.vscode', '.cache',
+]);
+const UUID_PREFIX = '@uuid:';
+const UUID_V4 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+/** One asset: stable uuid, current project-relative path, type + importer settings. */
+export interface AssetEntry {
+ uuid: string;
+ path: string;
+ type: string;
+ importer?: Record<string, unknown>;
+}
+
+/**
+ * The project asset index. `entries` is the uuid↔path registry; `deps` maps a
+ * scene/prefab uuid to the asset uuids it references (for delete-detection +
+ * cook). Same `version`/`entries` shape as the engine's AssetManifest, extended.
+ */
+export interface AssetIndex {
+ version: '1.0';
+ entries: AssetEntry[];
+ deps: Record<string, string[]>;
+}
+
+export interface ScanAssetsResult {
+ ok: boolean;
+ /** Absolute path to the written assets.json, or null if not written. */
+ outputPath: string | null;
+ index: AssetIndex;
+ warnings: string[];
+}
+
+/** Recursively yield every `<file>.meta` path (project-relative, forward-slashed). */
+async function* walkMeta(root: string, rel = ''): AsyncGenerator<string> {
+ const abs = rel ? path.join(root, rel) : root;
+ let entries;
+ try {
+ entries = await readdir(abs, { withFileTypes: true });
+ } catch {
+ return; // unreadable dir
+ }
+ for (const e of entries) {
+ if (e.isDirectory()) {
+ if (SKIP_DIRS.has(e.name)) continue;
+ yield* walkMeta(root, rel ? `${rel}/${e.name}` : e.name);
+ } else if (e.name.endsWith('.meta')) {
+ yield rel ? `${rel}/${e.name}` : e.name;
+ }
+ }
+}
+
+/** Collect every `@uuid:<id>` (or bare uuid) referenced in a scene/prefab JSON. */
+function collectRefs(value: unknown, into: Set<string>): void {
+ if (typeof value === 'string') {
+ if (value.startsWith(UUID_PREFIX)) {
+ const id = value.slice(UUID_PREFIX.length).toLowerCase();
+ if (UUID_V4.test(id)) into.add(id);
+ }
+ return;
+ }
+ if (Array.isArray(value)) {
+ for (const v of value) collectRefs(v, into);
+ return;
+ }
+ if (value && typeof value === 'object') {
+ for (const v of Object.values(value as Record<string, unknown>)) collectRefs(v, into);
+ }
+}
+
+/**
+ * Scan `root` for `.meta` sidecars → build the asset index (registry + dep
+ * graph) and (unless `write: false`) write `.esengine/cache/assets.json`.
+ */
+export async function scanAssetDatabase(
+ root: string,
+ opts?: { write?: boolean },
+): Promise<ScanAssetsResult> {
+ const entries: AssetEntry[] = [];
+ const warnings: string[] = [];
+
+ for await (const metaRel of walkMeta(root)) {
+ let meta: { uuid?: unknown; type?: unknown; importer?: unknown };
+ try {
+ meta = JSON.parse(await readFile(path.join(root, metaRel), 'utf8'));
+ } catch (err) {
+ warnings.push(`${metaRel}: ${err instanceof Error ? err.message : String(err)}`);
+ continue;
+ }
+ if (typeof meta?.uuid !== 'string' || typeof meta?.type !== 'string') {
+ warnings.push(`${metaRel}: missing uuid or type`);
+ continue;
+ }
+ entries.push({
+ uuid: meta.uuid.toLowerCase(),
+ path: metaRel.replace(/\.meta$/, ''),
+ type: meta.type,
+ importer: (meta.importer as Record<string, unknown>) ?? {},
+ });
+ }
+
+ entries.sort((a, b) => a.path.localeCompare(b.path));
+
+ // Dependency graph: scenes/prefabs reference assets by uuid.
+ const deps: Record<string, string[]> = {};
+ for (const entry of entries) {
+ if (entry.type !== 'scene' && entry.type !== 'prefab') continue;
+ try {
+ const json = JSON.parse(await readFile(path.join(root, entry.path), 'utf8'));
+ const refs = new Set<string>();
+ collectRefs(json, refs);
+ if (refs.size > 0) deps[entry.uuid] = [...refs].sort();
+ } catch (err) {
+ warnings.push(`${entry.path}: ${err instanceof Error ? err.message : String(err)}`);
+ }
+ }
+
+ const index: AssetIndex = { version: '1.0', entries, deps };
+
+ let outputPath: string | null = null;
+ if (opts?.write !== false) {
+ outputPath = path.join(root, CACHE_DIR, OUTPUT);
+ await mkdir(path.dirname(outputPath), { recursive: true });
+ await writeFile(outputPath, JSON.stringify(index, null, 2) + '\n');
+ }
+
+ return { ok: true, outputPath, index, warnings };
+}
diff --git a/desktop/electron/main.ts b/desktop/electron/main.ts
index 72d891215..4af8c021b 100644
--- a/desktop/electron/main.ts
+++ b/desktop/electron/main.ts
@@ -14,6 +14,7 @@ import {
import { listRecents, addRecent, listTemplates, createFromTemplate } from './launcher';
import { buildProjectScripts } from './buildScripts';
import { extractProjectSchemas } from './extractSchemas';
+import { scanAssetDatabase } from './assetDb';
import { resolveScripts } from '../src/project/format';
import type { WorkspaceState } from '../src/project/format';
@@ -156,6 +157,12 @@ ipcMain.handle('project:extractSchemas', async () => {
return extractProjectSchemas(root, { entry: register, required: manifest.scripts?.register !== undefined });
});
+// Scan the open project's `.meta` sidecars → the asset index (uuid↔path registry
+// + dependency graph) written to .esengine/cache/assets.json. The editor feeds
+// this into the engine Assets registry (one resolution path) and the Content
+// Browser; the cook walks `deps` (REARCH_ASSETS.md A2).
+ipcMain.handle('project:scanAssets', async () => scanAssetDatabase(requireRoot()));
+
ipcMain.handle('recents:list', () => listRecents());
ipcMain.handle('recents:add', (_e, root: string, name: string) => addRecent(root, name));
diff --git a/desktop/electron/preload.ts b/desktop/electron/preload.ts
index 4decf9461..cad4f8861 100644
--- a/desktop/electron/preload.ts
+++ b/desktop/electron/preload.ts
@@ -2,6 +2,7 @@ import { contextBridge, ipcRenderer } from 'electron';
import type { OpenedProject, WorkspaceState, DirEntry, RecentEntry, TemplateEntry } from '../src/project/format';
import type { BuildScriptsResult } from './buildScripts';
import type { ExtractSchemasResult } from './extractSchemas';
+import type { ScanAssetsResult } from './assetDb';
// The privileged bridge the renderer is allowed to touch. Keep this surface small
// and explicit — anything the editor needs from the OS or Node goes through here.
@@ -30,6 +31,8 @@ const api = {
buildScripts: (): Promise<BuildScriptsResult> => ipcRenderer.invoke('project:buildScripts'),
/** Extract the project's component field schemas → .esengine/cache/schemas.json. */
extractSchemas: (): Promise<ExtractSchemasResult> => ipcRenderer.invoke('project:extractSchemas'),
+ /** Scan the project's .meta sidecars → the asset index (registry + dep graph). */
+ scanAssets: (): Promise<ScanAssetsResult> => ipcRenderer.invoke('project:scanAssets'),
},
// New-project templates (launcher New tab).
templates: {
diff --git a/desktop/src/project/ProjectStore.ts b/desktop/src/project/ProjectStore.ts
index a59dc8c4b..60f2f9086 100644
--- a/desktop/src/project/ProjectStore.ts
+++ b/desktop/src/project/ProjectStore.ts
@@ -194,19 +194,21 @@ class ProjectStoreImpl {
}
/**
- * Scan `.meta` sidecars under the project's asset roots into a uuid→path
- * registry, then point the engine `Assets` loader at it + the `estella://`
- * transport. This is the ONE asset-resolution path: `Assets.resolveRef` turns
- * `@uuid:` → path, the backend fetches `estella://project/<path>` (REARCH_ASSETS.md A1).
+ * Load the project's asset index (the main-process AssetDatabase scan,
+ * REARCH_ASSETS.md A2) into a uuid→path registry, then point the engine
+ * `Assets` loader at it + the `estella://` transport. This is the ONE
+ * asset-resolution path: `Assets.resolveRef` turns `@uuid:` → path, the backend
+ * fetches `estella://project/<path>`.
*/
private async buildAssetRegistry(): Promise<void> {
- const st = this.state;
this.uuidToPath.clear();
- if (!st) return;
- // The conventional asset root(s): the top-level dir of the declared scenes /
- // textures dirs (e.g. `assets`), which also holds prefabs/audio/etc.
- const roots = new Set([st.layout.scenes, st.layout.textures].map((d) => d.split('/')[0]));
- for (const root of roots) await this.scanMetaDir(root);
+ if (!this.state) return;
+ try {
+ const { index } = await window.estella.project.scanAssets();
+ for (const e of index.entries) this.uuidToPath.set(e.uuid.toLowerCase(), e.path);
+ } catch (err) {
+ console.warn('[project] asset scan failed', err);
+ }
const assets = EngineHost.getResource(Assets);
if (assets) {
@@ -215,30 +217,6 @@ class ProjectStoreImpl {
}
}
- /** Recursively collect `<file>.meta` → {uuid} under `dir` into uuidToPath. */
- private async scanMetaDir(dir: string): Promise<void> {
- let entries;
- try {
- entries = await window.estella.fs.readDir(dir);
- } catch {
- return; // dir absent
- }
- for (const e of entries) {
- const p = `${dir}/${e.name}`;
- if (e.isDir) {
- await this.scanMetaDir(p);
- continue;
- }
- if (!e.name.endsWith('.meta')) continue;
- try {
- const meta = JSON.parse(await window.estella.fs.read(p)) as { uuid?: string };
- if (meta.uuid) this.uuidToPath.set(meta.uuid.toLowerCase(), p.replace(/\.meta$/, ''));
- } catch {
- // skip a malformed .meta
- }
- }
- }
-
/** Resolve a serialized asset ref to a project-relative path for the engine
* loader: `@uuid:` → path (null if unknown); a plain path passes through. */
private resolveRef(ref: string): string | null {
diff --git a/desktop/tests/asset-db.test.ts b/desktop/tests/asset-db.test.ts
new file mode 100644
index 000000000..cfccea5de
--- /dev/null
+++ b/desktop/tests/asset-db.test.ts
@@ -0,0 +1,93 @@
+/**
+ * @file AssetDatabase scanner (REARCH_ASSETS.md A2). A pure-node step: walk a
+ * project's `.meta` sidecars into a uuid↔path registry + a dependency
+ * graph (which scene/prefab references which asset), written to
+ * `.esengine/cache/assets.json`. Skips code/build/vcs dirs; reads the
+ * `type` each `.meta` already declares (no extension table).
+ */
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, existsSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { scanAssetDatabase, type AssetIndex } from '../electron/assetDb';
+
+let root: string;
+
+const TEX_UUID = '11111111-1111-4111-8111-111111111111';
+const FONT_UUID = '22222222-2222-4222-8222-222222222222';
+const SCENE_UUID = '33333333-3333-4333-8333-333333333333';
+
+function writeAsset(rel: string, type: string, uuid: string, body = ''): void {
+ const abs = path.join(root, rel);
+ mkdirSync(path.dirname(abs), { recursive: true });
+ writeFileSync(abs, body);
+ writeFileSync(`${abs}.meta`, JSON.stringify({ uuid, version: '2.0', type, importer: {} }));
+}
+
+beforeAll(() => {
+ root = mkdtempSync(path.join(tmpdir(), 'estella-assetdb-'));
+ // Two leaf assets + a scene that references them by @uuid:.
+ writeAsset('assets/textures/player.png', 'texture', TEX_UUID, 'PNGDATA');
+ writeAsset('assets/fonts/ui.fnt', 'font', FONT_UUID, 'FNTDATA');
+ writeAsset(
+ 'assets/scenes/main.esscene',
+ 'scene',
+ SCENE_UUID,
+ JSON.stringify({
+ version: '1.0',
+ name: 'main',
+ entities: [
+ {
+ id: 1,
+ name: 'Hero',
+ parent: null,
+ children: [],
+ components: [
+ { type: 'Sprite', data: { texture: `@uuid:${TEX_UUID}` } },
+ { type: 'BitmapText', data: { font: `@uuid:${FONT_UUID}` } },
+ ],
+ },
+ ],
+ }),
+ );
+ // A code dir that must be SKIPPED even though it contains a stray .meta.
+ const nm = path.join(root, 'node_modules', 'pkg');
+ mkdirSync(nm, { recursive: true });
+ writeFileSync(path.join(nm, 'thing.png.meta'), JSON.stringify({ uuid: 'deadbeef', type: 'texture' }));
+});
+
+afterAll(() => {
+ if (root) rmSync(root, { recursive: true, force: true });
+});
+
+describe('scanAssetDatabase (A2)', () => {
+ it('indexes every .meta into uuid↔path entries and writes assets.json', async () => {
+ const res = await scanAssetDatabase(root);
+ expect(res.ok).toBe(true);
+ expect(res.outputPath).toBe(path.join(root, '.esengine/cache/assets.json'));
+ expect(existsSync(res.outputPath!)).toBe(true);
+
+ const byUuid = Object.fromEntries(res.index.entries.map((e) => [e.uuid, e]));
+ expect(byUuid[TEX_UUID].path).toBe('assets/textures/player.png');
+ expect(byUuid[TEX_UUID].type).toBe('texture');
+ expect(byUuid[FONT_UUID].path).toBe('assets/fonts/ui.fnt');
+ expect(byUuid[SCENE_UUID].type).toBe('scene');
+
+ // The written artifact matches the returned index.
+ const onDisk = JSON.parse(readFileSync(res.outputPath!, 'utf8')) as AssetIndex;
+ expect(onDisk.entries.length).toBe(res.index.entries.length);
+ });
+
+ it('builds a dependency graph: the scene depends on the assets it references', async () => {
+ const { index } = await scanAssetDatabase(root, { write: false });
+ expect(index.deps[SCENE_UUID]).toEqual([FONT_UUID, TEX_UUID].sort());
+ // Leaf assets have no deps.
+ expect(index.deps[TEX_UUID]).toBeUndefined();
+ });
+
+ it('skips node_modules / build / vcs dirs', async () => {
+ const { index } = await scanAssetDatabase(root, { write: false });
+ expect(index.entries.some((e) => e.path.includes('node_modules'))).toBe(false);
+ expect(index.entries.length).toBe(3); // texture + font + scene, nothing from node_modules
+ });
+});