A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
* @file externalTools.ts — the third-party tools a project can switch on, found, fetched and checked.
*
* `tools/external-tools.manifest.json` pins each tool to one exact vendor file. A project that
* turns a tool on gets exactly that: it is looked for in the user's tool store, and when it is missing
* the caller is handed a PLAN (what would be fetched, and from where) to put in front of the person
* before anything is downloaded. Nothing here downloads on its own.
*
* Where a file comes from is the release mirror (dl.estella.games) and then the engine release itself,
* the same sources the native runtime templates use. There is no fallback to the vendor's site: the
* release publishes an index naming each file's size and SHA-256 and the vendor file it was made from,
* and a file that does not match the index, or an index that names a different vendor file than the one
* this engine pins, is refused. A wrong or stale mirror therefore costs a failed download, never a
* different tool.
*/
import { createHash, randomUUID } from 'node:crypto';
import { chmodSync, createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
import { once } from 'node:events';
import path from 'node:path';
import { estellaDataDir, releaseAssetBases } from '../../../build-tools/utils/nativeTemplate.js';
import { extractZip } from '../../../build-tools/utils/zip.js';
import manifestJson from '../../../tools/external-tools.manifest.json';
export type ToolPlatform = 'win-x64' | 'linux-x64' | 'linux-arm64' | 'osx-x64' | 'osx-arm64' | 'any';
interface UpstreamPin { url: string; bytes: number; sha256: string }
interface ToolEntry {
name: string;
version: string;
license: string;
licenseFiles: string[];
needs?: string[];
entry?: string;
executables?: Partial<Record<ToolPlatform, string[]>>;
files: Partial<Record<ToolPlatform, { upstream: UpstreamPin }>>;
}
export interface ToolsManifest { formatVersion: number; tools: Record<string, ToolEntry> }
export const TOOLS_INDEX = 'external-tools.json';
/** A base (URL or directory) that holds the index and the archives, replacing the release sources. For CI, an offline share and tests. */
export const TOOLS_BASE_ENV = 'ESTELLA_TOOLS_BASE';
/** Where tools are kept. */
export const TOOLS_STORE_ENV = 'ESTELLA_TOOLS_DIR';
const MARKER = '.estella-tool.json';
export function toolsManifest(): ToolsManifest {
return manifestJson as unknown as ToolsManifest;
}
export function toolIds(manifest = toolsManifest()): string[] {
return Object.keys(manifest.tools);
}
/** The platform key this machine's builds use, or null when no tool is pinned for it. */
export function hostPlatform(platform: NodeJS.Platform = process.platform, arch: string = process.arch): ToolPlatform | null {
const os = platform === 'win32' ? 'win' : platform === 'darwin' ? 'osx' : platform === 'linux' ? 'linux' : null;
const cpu = arch === 'x64' ? 'x64' : arch === 'arm64' ? 'arm64' : null;
if (!os || !cpu) return null;
const key = `${os}-${cpu}`;
return key === 'win-x64' || key === 'linux-x64' || key === 'linux-arm64' || key === 'osx-x64' || key === 'osx-arm64' ? key : null;
}
export function toolStoreDir(env: NodeJS.ProcessEnv = process.env): string {
return env[TOOLS_STORE_ENV] || path.join(estellaDataDir(), 'tools');
}
/** The key of this tool's file for a machine: its own platform's, or the one built for every platform. */
function fileKey(tool: ToolEntry, platform: ToolPlatform): ToolPlatform | null {
if (tool.files[platform]) return platform;
if (tool.files.any) return 'any';
return null;
}
export interface ToolStatus {
id: string;
name: string;
version: string;
license: string;
installed: boolean;
/** Where it is, when it is. */
dir?: string;
/** Why it is not usable, when it is not. */
problem?: string;
/** What it needs, and whether that is in place. */
needs: ToolStatus[];
}
interface Marker { tool: string; version: string; platform: string; upstreamSha256: string; sha256: string }
function readMarker(dir: string): Marker | null {
try { return JSON.parse(readFileSync(path.join(dir, MARKER), 'utf8')) as Marker; } catch { return null; }
}
interface Where { platform?: ToolPlatform | null; manifest?: ToolsManifest; store?: string }
/** Is `id` installed, whole, and the version this engine pins? Looks only at the disk. */
export function toolStatus(id: string, options: Where = {}): ToolStatus {
const manifest = options.manifest ?? toolsManifest();
const tool = manifest.tools[id];
if (!tool) throw new Error(`unknown tool "${id}"`);
const platform = options.platform === undefined ? hostPlatform() : options.platform;
const base = { id, name: tool.name, version: tool.version, license: tool.license };
const needs = (tool.needs ?? []).map((n) => toolStatus(n, options));
if (!platform) return { ...base, installed: false, problem: `no ${tool.name} build is pinned for this machine (${process.platform}/${process.arch})`, needs };
const key = fileKey(tool, platform);
if (!key) return { ...base, installed: false, problem: `no ${tool.name} build is pinned for ${platform}`, needs };
const dir = path.join(options.store ?? toolStoreDir(), id, tool.version, key);
const marker = readMarker(dir);
const pin = tool.files[key]!.upstream;
if (!marker) return { ...base, installed: false, problem: 'not installed', needs };
if (marker.version !== tool.version || marker.upstreamSha256 !== pin.sha256) {
return { ...base, installed: false, dir, problem: 'installed from a different file than this engine pins', needs };
}
const wanted = [...(tool.executables?.[platform] ?? []), ...(tool.entry ? [tool.entry] : [])];
const missing = wanted.filter((f) => !existsSync(path.join(dir, f)));
if (missing.length) return { ...base, installed: false, dir, problem: `incomplete — missing ${missing.join(', ')}`, needs };
return { ...base, installed: true, dir, needs };
}
/** Usable only when it and everything it needs are. */
export function toolReady(status: ToolStatus): boolean {
return status.installed && status.needs.every(toolReady);
}
export interface PlanItem {
id: string;
name: string;
version: string;
platform: ToolPlatform;
license: string;
/** The vendor's file this is made from, so a person can see what it is. */
upstreamUrl: string;
/** The vendor file's size: the download is about this big (the mirror's zip is a little different). */
upstreamBytes: number;
}
export interface InstallPlan {
/** What would be fetched, what it needs first. Empty when everything is in place. */
items: PlanItem[];
/** Where from, in the order tried. */
sources: string[];
}
export function toolSources(env: NodeJS.ProcessEnv = process.env, engineVersion?: string): string[] {
const override = env[TOOLS_BASE_ENV];
// A base given outright needs no release to be named.
return override ? [override.replace(/[\\/]+$/, '')] : (releaseAssetBases(engineVersion ?? defaultEngineVersion(env), env) as string[]);
}
/** The environment variable a host sets to say which engine release it is, when it is not running from the source tree. */
export const ENGINE_VERSION_ENV = 'ESTELLA_ENGINE_VERSION';
/**
* Which release's files to ask the mirror for. A packaged editor has no source tree to read it from, so it says so
* (an `engineVersion` option, or {@link ENGINE_VERSION_ENV}); and when nothing can say, this throws. It once fell back to
* `0.0.0`, which asks the mirror for a release that does not exist and reports a download failure nobody can act on.
*/
function defaultEngineVersion(env: NodeJS.ProcessEnv = process.env): string {
if (env[ENGINE_VERSION_ENV]) return env[ENGINE_VERSION_ENV];
try {
return (JSON.parse(readFileSync(new URL('../../../package.json', import.meta.url), 'utf8')) as { version: string }).version;
} catch {
throw new Error(`cannot tell which engine release this is: pass engineVersion, or set ${ENGINE_VERSION_ENV}`);
}
}
/** What turning `id` on would fetch: the tool and whatever it needs that is not here yet. Touches no network. */
export function installPlan(id: string, options: Where & { env?: NodeJS.ProcessEnv; engineVersion?: string } = {}): InstallPlan {
const manifest = options.manifest ?? toolsManifest();
const platform = options.platform === undefined ? hostPlatform() : options.platform;
const items: PlanItem[] = [];
const seen = new Set<string>();
const visit = (toolId: string): void => {
if (seen.has(toolId)) return;
seen.add(toolId);
const tool = manifest.tools[toolId];
if (!tool) throw new Error(`unknown tool "${toolId}"`);
for (const need of tool.needs ?? []) visit(need);
if (!platform) return;
if (toolStatus(toolId, { ...options, platform }).installed) return;
const key = fileKey(tool, platform);
if (!key) return;
const up = tool.files[key]!.upstream;
items.push({ id: toolId, name: tool.name, version: tool.version, platform: key, license: tool.license, upstreamUrl: up.url, upstreamBytes: up.bytes });
};
visit(id);
return { items, sources: toolSources(options.env ?? process.env, options.engineVersion) };
}
// ---- The index a release publishes -----------------------------------------------------------------
export interface IndexFile { tool: string; version: string; platform: string; file: string; bytes: number; sha256: string; upstreamSha256: string }
export function parseToolsIndex(doc: unknown): IndexFile[] | null {
const d = doc as { kind?: unknown; formatVersion?: unknown; files?: unknown } | null;
if (!d || d.kind !== 'estella-external-tools' || d.formatVersion !== 1 || !Array.isArray(d.files)) return null;
const ok = (d.files as IndexFile[]).filter((f) => typeof f?.tool === 'string' && typeof f.version === 'string' && typeof f.platform === 'string'
&& typeof f.file === 'string' && /^[\w.-]+$/.test(f.file)
&& typeof f.sha256 === 'string' && /^[0-9a-f]{64}$/.test(f.sha256)
&& typeof f.upstreamSha256 === 'string' && /^[0-9a-f]{64}$/.test(f.upstreamSha256)
&& Number.isInteger(f.bytes) && f.bytes > 0);
return ok.length > 0 ? ok : null;
}
type Read = { ok: true; stream: AsyncIterable<Uint8Array> } | { ok: false; error: string };
async function readBase(base: string, name: string, fetchImpl: typeof fetch): Promise<Read> {
if (/^https?:\/\//.test(base)) {
const res = await fetchImpl(`${base}/${name}`);
if (!res.ok || !res.body) return { ok: false, error: `HTTP ${res.status} for ${name}` };
return { ok: true, stream: res.body as unknown as AsyncIterable<Uint8Array> };
}
const file = path.join(base, name);
if (!existsSync(file)) return { ok: false, error: `${file} does not exist` };
const bytes = readFileSync(file);
return { ok: true, stream: (async function* () { yield new Uint8Array(bytes); })() };
}
async function readAll(stream: AsyncIterable<Uint8Array>): Promise<Buffer> {
const parts: Buffer[] = [];
for await (const chunk of stream) parts.push(Buffer.from(chunk));
return Buffer.concat(parts);
}
export interface ToolProgress { tool: string; received: number; total: number }
export interface InstallResult { ok: boolean; installed: string[]; error?: string }
/**
* Fetch and install `id` and what it needs, verified. Never throws: a failure is a message.
* Sources are tried in order; a source that fails any check is skipped for the next.
*/
export async function installTool(id: string, options: Where & {
env?: NodeJS.ProcessEnv;
engineVersion?: string;
fetchImpl?: typeof fetch;
onProgress?: (progress: ToolProgress) => void;
} = {}): Promise<InstallResult> {
const manifest = options.manifest ?? toolsManifest();
const platform = options.platform === undefined ? hostPlatform() : options.platform;
const store = options.store ?? toolStoreDir(options.env);
const fetchImpl = options.fetchImpl ?? fetch;
if (!platform) return { ok: false, installed: [], error: `no tools are pinned for this machine (${process.platform}/${process.arch})` };
const plan = installPlan(id, { ...options, manifest, platform, store });
const installed: string[] = [];
for (const item of plan.items) {
let last = `no source for ${item.name}`;
let done = false;
for (const base of plan.sources) {
const attempt = await installFrom(base, item, manifest, store, platform, fetchImpl, options.onProgress);
if (attempt === null) { done = true; break; }
last = attempt;
if (plan.sources.length > 1) console.warn(`[tools] ${base}: ${last}`);
}
if (!done) return { ok: false, installed, error: last };
installed.push(`${item.id} ${item.version}`);
}
return { ok: true, installed };
}
/** Null on success; otherwise why this source did not work. */
async function installFrom(base: string, item: PlanItem, manifest: ToolsManifest, store: string, platform: ToolPlatform, fetchImpl: typeof fetch, onProgress?: (p: ToolProgress) => void): Promise<string | null> {
const tool = manifest.tools[item.id]!;
const pin = tool.files[item.platform]!.upstream;
const staging = path.join(store, `.incoming-${randomUUID()}`);
try {
const indexRead = await readBase(base, TOOLS_INDEX, fetchImpl);
if (!indexRead.ok) return `no tools index (${indexRead.error})`;
const entries = parseToolsIndex(JSON.parse((await readAll(indexRead.stream)).toString('utf8')));
if (!entries) return 'the tools index is not readable';
const entry = entries.find((e) => e.tool === item.id && e.platform === item.platform && e.version === tool.version);
if (!entry) return `the index has no ${item.name} ${tool.version} for ${item.platform}`;
// The index may name only the vendor file this engine pins. That is what makes the version fixed.
if (entry.upstreamSha256 !== pin.sha256) return `the index's ${item.name} is made from a different vendor file than this engine pins`;
const fileRead = await readBase(base, entry.file, fetchImpl);
if (!fileRead.ok) return `download failed (${fileRead.error})`;
mkdirSync(staging, { recursive: true });
const archive = path.join(staging, entry.file);
const digest = createHash('sha256');
let received = 0;
const sink = createWriteStream(archive);
for await (const chunk of fileRead.stream) {
digest.update(chunk);
received += chunk.byteLength;
onProgress?.({ tool: item.id, received, total: entry.bytes });
if (!sink.write(chunk)) await once(sink, 'drain');
}
await new Promise<void>((resolve, reject) => sink.end((err?: Error) => (err ? reject(err) : resolve())));
if (received !== entry.bytes) return `download is ${received} bytes, the index says ${entry.bytes}`;
const got = digest.digest('hex');
if (got !== entry.sha256) return `checksum mismatch — got ${got.slice(0, 16)}…, expected ${entry.sha256.slice(0, 16)}…`;
const unpacked = path.join(staging, 'files');
mkdirSync(unpacked, { recursive: true });
extractZip(readFileSync(archive), unpacked);
const executables = tool.executables?.[platform] ?? [];
const required = [...executables, ...(tool.entry ? [tool.entry] : []), ...tool.licenseFiles];
const missing = required.filter((f) => !existsSync(path.join(unpacked, f)));
if (missing.length) return `incomplete archive — missing ${missing.join(', ')}`;
if (process.platform !== 'win32') for (const exe of executables) chmodSync(path.join(unpacked, exe), 0o755);
const marker: Marker = { tool: item.id, version: tool.version, platform: item.platform, upstreamSha256: pin.sha256, sha256: got };
writeFileSync(path.join(unpacked, MARKER), JSON.stringify(marker, null, 2));
const dest = path.join(store, item.id, tool.version, item.platform);
rmSync(dest, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });
mkdirSync(path.dirname(dest), { recursive: true });
renameSync(unpacked, dest);
return null;
} catch (err) {
return err instanceof Error ? err.message : String(err);
} finally {
rmSync(staging, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });
}
}
/** Delete an installed tool (every version of it). Takes the id, never a path. */
export function removeTool(id: string, store = toolStoreDir()): boolean {
if (!toolsManifest().tools[id]) return false;
const dir = path.join(store, id);
if (!existsSync(dir) || readdirSync(dir).length === 0) return false;
rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });
return true;
}