Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

13.9 KB
Raw Blame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
 * @file  AOT step: a project's `@compiled` systems, as wasm.
 *
 * The rest of `buildScripts` hands the realm JavaScript. This hands it a module
 * of machine code for the systems the author PROMISED would compile, built with
 * emcc from the C `@estella/compiler` emits.
 *
 * It imports the engine's memory rather than owning one, which is what keeps
 * this cheap: the engine wasm stays prebuilt and only the game's own systems are
 * compiled here — a few hundred bytes, not a link.
 *
 * Never throws. A project with no marked system is not a failure and not a
 * build step; a marked system that will not compile IS one, because that is what
 * the marker means.
 *
 * The mode decides whether it runs at all. `dev` never compiles, because the
 * editor's preview always interprets: a machine with no emsdk builds and runs
 * every project. `release` and `ship` do compile, and there a promise the subset
 * cannot keep is an error rather than a quiet fallback — which is the difference
 * the marker exists to make.
 */
import { existsSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync, readFileSync } from 'node:fs';
import path from 'node:path';
import { brokenPromises, lowerProgram } from '../../../compiler/src/frontend';
import { verifySystem } from '../../../compiler/src/verify';
import { inlineSystem } from '../../../compiler/src/inline';
import { builtinShapes } from '../../../compiler/src/builtins';
import { packLayout } from '../../../compiler/src/abi';
import { CFLAGS, WASM_LINK_FLAGS, emitC, moduleDeclarations } from '../../../compiler/src/codegen';
import { nativeLinkFlags, nativeModuleExt } from '../../../compiler/src/hostCC';
import { AOT_MANIFEST, AOT_WASM } from './aotArtifacts';

/** Where the C and the wasm land, beside the script bundle. */
const CACHE_DIR = '.esengine/cache/aot';
const WASM = AOT_WASM;
const MANIFEST = AOT_MANIFEST;

/** What the runtime needs in order to call one compiled system. */
export interface CompiledSystemInfo {
  /** The system's declared name, as the schedule knows it. */
  name: string;
  /** The exported wasm symbol. */
  symbol: string;
  /** One entry per declared Query: its components, in the order it names them. */
  queries: { comp: string; mut: boolean }[][];
  /**
   * One per declared Res/ResMut, in declaration order. `mut` is what makes the
   * runtime write a mirrored resource back after the call; `fields` is present
   * for a resource the PROJECT declared, because its layout is not something
   * the engine knows — it was derived from the declaration.
   */
  resources: { name: string; mut: boolean; fields?: string[] }[];
  /** Event readers and writers, with the payload layout each one uses. */
  readers: { slot: number; event: string; fields: string[] }[];
  writers: { slot: number; event: string; fields: string[] }[];
  /**
   * Whether the body appends a command record. A road that cannot flush one to
   * the world the interpreter would have flushed it to leaves the system to the
   * interpreter, so this is what that decision is made on.
   */
  commands: boolean;
}

export interface CompiledSystemsManifest {
  /**
   * What this module baked in, so a loader can say WHICH one moved: the engine
   * it was built against, and the project's own component shapes. The fixes
   * differ — rebuild the module, or rebuild the project — so the numbers do too.
   */
  engineAbi: string;
  projectShapes: string;
  /** Which BUILD this is: the artifact bakes the same number, so a loader can
   *  refuse a module paired with another build's sidecar. */
  moduleContract: string;
  systems: CompiledSystemInfo[];
}

/**
 * Which build this is. Not a verbosity setting: it decides whether a `@compiled`
 * marker is a promise anyone is collecting on.
 */
export type BuildMode = 'dev' | 'release' | 'ship';

/**
 * Which machine the systems are compiled FOR.
 *
 * `wasm` imports the engine's memory; `native` is a library a host loads into
 * its own process, where an address is a pointer and not an offset into one
 * block. The same C either way — the header picks the typedef.
 */
export type CompileTarget = 'wasm' | 'native';

/** Everything that differs between the two, in one place rather than in ifs. */
interface TargetPlan {
  /** `sizeof(es_addr_t)` there. It rides the handshake, so an artifact built
   *  for one host is refused by the other rather than reading half a pointer. */
  readonly addressBytes: 4 | 8;
  /** What the module is called in the cache. */
  readonly module: string;
  /** Link flags, plus whatever names the exports on this target. */
  linkFlags(symbols: readonly string[]): string[];
  /** Whether the declaration table is compiled in. It is the DATA half: a wasm
   *  module must not carry it, since its data section would land at an address
   *  the engine already owns, and a library in the host's process is who does. */
  readonly compilesDecls: boolean;
  /** What to say when this machine has no compiler for it. */
  readonly missing: string;
}

const TARGETS: Record<CompileTarget, TargetPlan> = {
  wasm: {
    addressBytes: 4,
    module: WASM,
    linkFlags: (symbols) => [
      // No entry point and no JS glue, so the module's only import is the memory
      // it is given. An empty import section is a property of this command line
      // as much as of the C.
      ...WASM_LINK_FLAGS,
      `-sEXPORTED_FUNCTIONS=${symbols.map((s) => `_${s}`).join(',')}`,
    ],
    compilesDecls: false,
    missing: 'there is no emcc — install the emscripten toolchain and point EMSDK'
      + ' at it (in this repo: `pnpm emsdk:setup`)',
  },
  native: {
    addressBytes: 8,
    module: `systems${nativeModuleExt()}`,
    linkFlags: () => [...nativeLinkFlags()],
    compilesDecls: true,
    missing: 'this machine has no C compiler — install clang or gcc and put it on PATH',
  },
};

export interface BuildCompiledResult {
  ok: boolean;
  /** Absolute path to the built module, or null when there was nothing to
   *  build. Which KIND of module is the target the caller asked for. */
  modulePath: string | null;
  /** With `outputs`: each output's module, by its id (the first is `modulePath`). */
  modules?: Record<string, string>;
  manifest: CompiledSystemsManifest | null;
  errors: string[];
  /** Systems the subset refused that were NOT promised — information, not failure. */
  notes: string[];
}



/** Every `.ts` under `src/`, which is the unit a project's program is. */
function sources(root: string): string[] {
  const out: string[] = [];
  const walk = (dir: string): void => {
    if (!existsSync(dir)) return;
    for (const name of readdirSync(dir)) {
      const p = path.join(dir, name);
      if (statSync(p).isDirectory()) walk(p);
      else if (p.endsWith('.ts') && !p.endsWith('.d.ts')) out.push(p);
    }
  };
  walk(path.join(root, 'src'));
  return out;
}

/** A marker anywhere in a file. Text, not a parse: this only decides whether the
 *  expensive answer is worth computing. */
const anyPromise = (files: string[]): boolean =>
  files.some((f) => readFileSync(f, 'utf8').includes('@compiled'));

/**
 * Whether this project promises any compilation at all.
 *
 * The cheap half of what {@link buildCompiledSystems} answers, exported because
 * the editor's platform catalog must say a target needs emcc BEFORE an export
 * spawns one. Both read this, so they cannot disagree about what was promised.
 */
export function promisesCompilation(root: string): boolean {
  return anyPromise(sources(root));
}

/**
 * Compile the project's `@compiled` systems to `<root>/.esengine/cache/aot/`.
 *
 * The compiler is the caller's to supply, because finding it is the toolchain's
 * job and not this module's; without one a project that promised nothing still
 * builds, and one that promised something says so.
 */
export async function buildCompiledSystems(
  root: string,
  opts: {
    /** dev never compiles; release and ship require what the markers promised. */
    mode: BuildMode;
    /** Which machine this build is for. Absent ⇒ wasm, the road every
     *  browser and mini-game host takes. */
    target?: CompileTarget;
    /** Absolute path to the compiler FOR that target — emcc for wasm, the
     *  host's own cc for native — or null when this machine has none. */
    cc: string | null;
    /** Runs a command; the toolchain's own runner, so env and logging match. */
    run: (cmd: string, args: string[], cwd: string) => Promise<{ code: number; stderr: string }>;
    /**
     * Native only: the machines to build the one C for, each with its own
     * compiler and flags — a cross-compiled package carries one module per ABI.
     * Replaces `cc` and the host's link flags; an empty list means no compiler
     * here, said with `missing`.
     */
    outputs?: ReadonlyArray<{ id: string; cc: string; flags: readonly string[]; module: string }>;
    /** What to say when `outputs` is empty. */
    missing?: string;
  },
): Promise<BuildCompiledResult> {
  const plan = TARGETS[opts.target ?? 'wasm'];
  if (opts.mode === 'dev') {
    // Nothing is compiled and nothing is checked, so a marker costs a dev build
    // nothing at all — not a toolchain, not a second of build time, not a
    // failure on a machine that has neither.
    return { ok: true, modulePath: null, manifest: null, errors: [], notes: [] };
  }
  const files = sources(root);
  if (files.length === 0) {
    return { ok: true, modulePath: null, manifest: null, errors: [], notes: [] };
  }
  // A project with no `@compiled` anywhere promised nothing, so it must not pay
  // for a TypeScript Program over its sources. Nothing is decided here: the parse
  // below still decides, this only skips having nothing to decide.
  if (!anyPromise(files)) {
    return { ok: true, modulePath: null, manifest: null, errors: [], notes: [] };
  }

  let lowered;
  try {
    lowered = lowerProgram(files, builtinShapes());
  } catch (err) {
    return { ok: false, modulePath: null, manifest: null, notes: [], errors: [String(err)] };
  }

  const broken = brokenPromises(lowered);
  if (broken.length > 0) {
    // The marker's whole point: a refusal against a promised system is an error
    // with a line, not a silent fall back to the interpreter.
    return {
      ok: false,
      modulePath: null,
      manifest: null,
      notes: [],
      errors: broken.map((d) => `${d.file}:${d.line}: ${d.system ?? ''} is @compiled but ${d.message}`),
    };
  }

  const promised = new Set(lowered.required);
  const chosen = lowered.module.systems.filter((s) => promised.has(s.name));
  const notes = lowered.diagnostics
    .filter((d) => d.severity === 'note' && d.system)
    .map((d) => `${d.system}: ${d.message}`);
  if (chosen.length === 0) {
    // Nothing was promised, so there is nothing to build and nothing to say.
    return { ok: true, modulePath: null, manifest: null, errors: [], notes };
  }

  const bad = chosen.flatMap((s) => verifySystem(s, lowered.module.comps, lowered.module.fns));
  if (bad.length > 0) {
    return {
      ok: false, modulePath: null, manifest: null, notes,
      errors: bad.map((e) => `${e.system}: ${e.message}`),
    };
  }

  if (opts.outputs ? opts.outputs.length === 0 : !opts.cc) {
    return {
      ok: false, modulePath: null, manifest: null, notes,
      errors: [`${chosen.length} system(s) are marked @compiled but ${opts.outputs ? opts.missing ?? plan.missing : plan.missing}`
        + ', or remove the marker'],
    };
  }

  const layout = packLayout(lowered.module.comps);
  const inlined = chosen.map((s) => inlineSystem(s, lowered.module.fns));
  const c = emitC(lowered.module, layout, inlined, plan.addressBytes);

  const dir = path.join(root, CACHE_DIR);
  rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });
  mkdirSync(dir, { recursive: true });
  writeFileSync(path.join(dir, 'estella_abi.h'), c.header);
  writeFileSync(path.join(dir, 'estella_offsets.h'), c.offsets);
  writeFileSync(path.join(dir, 'systems.c'), c.source);
  const units = [path.join(dir, 'systems.c')];
  if (plan.compilesDecls) {
    writeFileSync(path.join(dir, 'systems_decl.c'), c.decls);
    units.push(path.join(dir, 'systems_decl.c'));
  }

  // One C, as many machines as were asked for: the host's own by default, or one
  // module per output (a cross-compiled package's ABIs), each in its own folder.
  const jobs = opts.outputs
    ? opts.outputs.map((o) => ({ id: o.id, cc: o.cc, flags: [...o.flags], out: path.join(dir, o.id, o.module) }))
    : [{ id: plan.module, cc: opts.cc!, flags: plan.linkFlags(c.symbols), out: path.join(dir, plan.module) }];
  const modules: Record<string, string> = {};
  for (const job of jobs) {
    mkdirSync(path.dirname(job.out), { recursive: true });
    const built = await opts.run(job.cc, [...CFLAGS, '-Wall', '-Wextra', ...job.flags, '-o', job.out, ...units], dir);
    if (built.code !== 0 || !existsSync(job.out)) {
      return { ok: false, modulePath: null, manifest: null, notes,
        errors: [`${opts.outputs ? `${job.id}: ` : ''}${built.stderr.trim()}`] };
    }
    modules[job.id] = job.out;
  }
  const modulePath = jobs[0].out;

  const manifest: CompiledSystemsManifest = {
    engineAbi: c.handshake.engineAbi,
    projectShapes: c.handshake.projectShapes,
    moduleContract: c.handshake.moduleContract,
    // The one mapping from plans to declarations, shared with whatever else
    // installs a module it just compiled — this was a second copy of the C
    // table the artifact carries.
    systems: moduleDeclarations(lowered.module, inlined),
  };
  writeFileSync(path.join(dir, MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`);
  return { ok: true, modulePath, ...(opts.outputs ? { modules } : {}), manifest, errors: [], notes };
}

/** Read back what a previous build wrote, or null. */
export function readCompiledManifest(root: string): CompiledSystemsManifest | null {
  const at = path.join(root, CACHE_DIR, MANIFEST);
  if (!existsSync(at)) return null;
  return JSON.parse(readFileSync(at, 'utf8')) as CompiledSystemsManifest;
}