Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

15.4 KB
Raw Blame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
 * @file    imageFigure.ts
 * @brief   A posed skeleton with plain volumes on it, seen from an angle: a figure drawing's first lines, for a 2D picture to build on.
 *
 * Bones and blocks on purpose. They give gesture, proportion and foreshortening, which coordinates typed blind get wrong, and no look:
 * no face, no skin, no hair, as whatever a model is handed it copies, and a coloured mannequin is what comes back.
 */
import { poseFrames, poseJoints, type HumanoidBuild, type PoseParams } from './figureSkeleton';
import type { ImageLayer } from './imageSource';
import type { FlatFigure } from './imageRaster';

export type { FlatFigure } from './imageRaster';

/** The bones' own height: they are built at it, and sized in the picture. */
const BODY_HEIGHT = 1.6;
const SUPERSAMPLE = 2;
const AROUND = 14;

type V3 = readonly [number, number, number];

export const FIGURE_TAGS = ['torso', 'arm', 'hand', 'leg', 'foot', 'neck', 'head'] as const;
export const FIGURE_JOINTS = [
    'crown', 'head', 'neck', 'chest', 'spine', 'hips',
    'upperArm.L', 'forearm.L', 'hand.L', 'middle4.L', 'upperArm.R', 'forearm.R', 'hand.R', 'middle4.R',
    'thigh.L', 'shin.L', 'foot.L', 'toeTip.L', 'thigh.R', 'shin.R', 'foot.R', 'toeTip.R',
] as const;
const BONES: readonly (readonly [string, string])[] = [
    ['hips', 'spine'], ['spine', 'chest'], ['chest', 'neck'], ['neck', 'head'], ['head', 'crown'],
    ['chest', 'upperArm.L'], ['upperArm.L', 'forearm.L'], ['forearm.L', 'hand.L'], ['hand.L', 'middle4.L'],
    ['chest', 'upperArm.R'], ['upperArm.R', 'forearm.R'], ['forearm.R', 'hand.R'], ['hand.R', 'middle4.R'],
    ['hips', 'thigh.L'], ['thigh.L', 'shin.L'], ['shin.L', 'foot.L'], ['foot.L', 'toeTip.L'],
    ['hips', 'thigh.R'], ['thigh.R', 'shin.R'], ['shin.R', 'foot.R'], ['foot.R', 'toeTip.R'],
];

interface Ring { at: V3; across: V3; deep: V3; a: number; d: number }
interface Volume { tag: (typeof FIGURE_TAGS)[number]; rings: Ring[] }

const plus = (p: V3, v: V3, k: number): V3 => [p[0] + v[0] * k, p[1] + v[1] * k, p[2] + v[2] * k];
const between = (a: V3, b: V3, t: number): V3 => [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, a[2] + (b[2] - a[2]) * t];
const unit = (v: V3): V3 => {
    const len = Math.hypot(v[0], v[1], v[2]) || 1;
    return [v[0] / len, v[1] / len, v[2] / len];
};
const crossed = (a: V3, b: V3): V3 => [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]];

/** The blocks a figure drawing starts from, widths on a body 1.6 tall. */
function volumesOf(frames: ReturnType<typeof poseFrames>, build: HumanoidBuild): Volume[] {
    const f = (name: string) => frames[name]!;
    const [hips, spine, chest, neck, head] = [f('hips'), f('spine'), f('chest'), f('neck'), f('head')];
    const [wHips, wWaist, wChest, wShoulders, wLimbs, wHead] = [build.hips ?? 1, build.waist ?? 1, build.chest ?? 1, build.shoulders ?? 1, build.limbs ?? 1, build.head ?? 1];
    const volumes: Volume[] = [{
        tag: 'torso',
        rings: [
            { at: plus(hips.at, hips.y, -0.1), across: hips.x, deep: hips.z, a: 0.09 * wHips, d: 0.06 },
            { at: plus(hips.at, hips.y, -0.02), across: hips.x, deep: hips.z, a: 0.135 * wHips, d: 0.085 },
            { at: between(hips.at, spine.at, 0.6), across: spine.x, deep: spine.z, a: 0.112 * wWaist, d: 0.07 },
            { at: spine.at, across: spine.x, deep: spine.z, a: 0.1 * wWaist, d: 0.064 },
            { at: chest.at, across: chest.x, deep: chest.z, a: 0.128 * wChest, d: 0.09 },
            { at: between(chest.at, neck.at, 0.62), across: chest.x, deep: chest.z, a: 0.15 * wShoulders, d: 0.078 },
            { at: between(chest.at, neck.at, 0.92), across: chest.x, deep: chest.z, a: 0.075, d: 0.06 },
        ],
    }];
    const round = (at: V3, axis: V3, r: number): Ring => {
        const side = unit(crossed(axis, Math.abs(axis[1]) > 0.9 ? [1, 0, 0] : [0, 1, 0]));
        return { at, across: side, deep: unit(crossed(axis, side)), a: r * wLimbs, d: r * wLimbs };
    };
    const limb = (tag: Volume['tag'], names: readonly string[], radii: readonly number[]): void => {
        const points = names.map((n) => f(n).at);
        const rings: Ring[] = [];
        for (let i = 0; i < points.length; i++) {
            const axis = unit(i + 1 < points.length ? [points[i + 1]![0] - points[i]![0], points[i + 1]![1] - points[i]![1], points[i + 1]![2] - points[i]![2]] : [points[i]![0] - points[i - 1]![0], points[i]![1] - points[i - 1]![1], points[i]![2] - points[i - 1]![2]]);
            if (i === 0) rings.push(round(plus(points[0]!, axis, -radii[0]! * 0.8), axis, radii[0]! * 0.55));
            rings.push(round(points[i]!, axis, radii[i]!));
            if (i === points.length - 1) rings.push(round(plus(points[i]!, axis, radii[i]! * 0.8), axis, radii[i]! * 0.55));
        }
        volumes.push({ tag, rings });
    };
    for (const side of ['L', 'R']) {
        limb('arm', [`upperArm.${side}`, `forearm.${side}`, `hand.${side}`], [0.038, 0.029, 0.021]);
        limb('leg', [`thigh.${side}`, `shin.${side}`, `foot.${side}`], [0.072, 0.046, 0.03]);
        // A mitten and a wedge: where a hand and a foot go, and no fingers to copy.
        const [hand, tip] = [f(`hand.${side}`), f(`middle4.${side}`)];
        volumes.push({ tag: 'hand', rings: [
            { at: hand.at, across: hand.z, deep: hand.y, a: 0.026, d: 0.016 },
            { at: between(hand.at, tip.at, 0.45), across: hand.z, deep: hand.y, a: 0.04, d: 0.017 },
            { at: tip.at, across: hand.z, deep: hand.y, a: 0.022, d: 0.01 },
        ] });
        const [foot, toe] = [f(`foot.${side}`), f(`toeTip.${side}`)];
        const along = unit([toe.at[0] - foot.at[0], toe.at[1] - foot.at[1], toe.at[2] - foot.at[2]]);
        const flat = unit(crossed(along, [1, 0, 0]));
        volumes.push({ tag: 'foot', rings: [
            { at: plus(foot.at, along, -0.05), across: [1, 0, 0], deep: flat, a: 0.022, d: 0.026 },
            { at: foot.at, across: [1, 0, 0], deep: flat, a: 0.034, d: 0.04 },
            { at: between(foot.at, toe.at, 0.7), across: [1, 0, 0], deep: flat, a: 0.04, d: 0.022 },
            { at: toe.at, across: [1, 0, 0], deep: flat, a: 0.022, d: 0.01 },
        ] });
    }
    limb('neck', ['neck', 'head'], [0.042, 0.038]);
    // An egg, turned as the head is turned.
    const centre = between(head.at, f('crown').at, 0.46);
    const rings: Ring[] = [];
    for (let i = 0; i <= 8; i++) {
        const t = (i / 8) * 2 - 1, r = Math.sqrt(Math.max(0.0001, 1 - t * t));
        rings.push({ at: plus(plus(centre, head.y, t * 0.108 * wHead), head.z, 0.006), across: head.x, deep: head.z, a: 0.074 * wHead * r, d: 0.09 * wHead * r });
    }
    volumes.push({ tag: 'head', rings });
    return volumes;
}


const normalized = (v: readonly number[]): [number, number, number] => {
    const len = Math.hypot(v[0]!, v[1]!, v[2]!) || 1;
    return [v[0]! / len, v[1]! / len, v[2]! / len];
};

/** Flattens a `figure` layer. @p project takes a point in the picture before any group's transform, and @p scale is that transform's size. */
export function flattenFigure(layer: ImageLayer, w: number, h: number, project: (p: readonly [number, number]) => readonly [number, number], scale: number): FlatFigure {
    const [ax, ay] = project(layer.at!);
    const s = ((layer.height as number) / BODY_HEIGHT) * scale;
    const yaw = ((layer.yaw ?? 0) * Math.PI) / 180, pitch = ((layer.pitch ?? 0) * Math.PI) / 180;
    const [cy, sy, cp, sp] = [Math.cos(yaw), Math.sin(yaw), Math.cos(pitch), Math.sin(pitch)];
    /** Yaw about the body, then pitch toward the viewer: what the body's axes are in the view. */
    const toView = (p: readonly number[]): [number, number, number] => {
        const [x, y, z] = [p[0]! * cy + p[2]! * sy, p[1]!, -p[0]! * sy + p[2]! * cy];
        return [x, y * cp - z * sp, y * sp + z * cp];
    };
    const fromView = (v: readonly number[]): [number, number, number] => {
        const [y, z] = [v[1]! * cp + v[2]! * sp, -v[1]! * sp + v[2]! * cp];
        return [v[0]! * cy - z * sy, y, v[0]! * sy + z * cy];
    };
    const toPicture = (p: readonly number[]): [number, number] => {
        const v = toView(p);
        return [ax + v[0] * s, ay - v[1] * s];
    };
    const build = (layer.build ?? {}) as HumanoidBuild;
    const pose = layer.pose ?? {};
    const held = { humanoid: build, ...(pose.rotate ? { rotate: pose.rotate } : {}), ...(pose.move ? { move: pose.move } : {}) } as PoseParams;
    // A reach given by two numbers is a place in the picture: where along the line of sight is where the joint already is.
    const reach = Object.entries(pose.reach ?? {});
    let resolved: PoseParams = held;
    if (reach.length) {
        const before = poseJoints(held);
        resolved = {
            ...held,
            reach: Object.fromEntries(reach.map(([name, r]) => {
                if (!before[name]) throw new Error(`no joint "${name}"; there are ${Object.keys(before).join(', ')}`);
                const at = r.at.length === 3 ? r.at : fromView([(r.at[0]! - ax) / s, (ay - r.at[1]!) / s, toView(before[name]!)[2]]);
                return [name, { at, ...(r.pole ? { pole: r.pole } : {}) }];
            })),
        };
    }
    const frames = poseFrames(resolved);
    const joints: Record<string, readonly [number, number]> = {};
    for (const name of FIGURE_JOINTS) {
        const p = toPicture(frames[name]!.at);
        joints[name] = [Math.round(p[0]), Math.round(p[1])];
    }

    const [W, H] = [w * SUPERSAMPLE, h * SUPERSAMPLE];
    const depth = new Float32Array(W * H).fill(-Infinity);
    const tagAt = new Int8Array(W * H).fill(-1);
    const nAt = new Float32Array(W * H * 3);
    for (const volume of volumesOf(frames, build)) {
        const tag = FIGURE_TAGS.indexOf(volume.tag);
        const points: V3[] = [];
        for (const r of volume.rings) {
            for (let i = 0; i < AROUND; i++) {
                const t = (i / AROUND) * Math.PI * 2;
                points.push(plus(plus(r.at, r.across, Math.cos(t) * r.a), r.deep, Math.sin(t) * r.d));
            }
        }
        const quads: [number, number, number, number][] = [];
        for (let k = 0; k + 1 < volume.rings.length; k++) {
            for (let i = 0; i < AROUND; i++) {
                const [a, b] = [k * AROUND + i, k * AROUND + (i + 1) % AROUND];
                quads.push([a, b, b + AROUND, a + AROUND]);
            }
        }
        // Smooth across the block, whichever way the rings wind.
        const middle = volume.rings.reduce<V3>((m, r) => plus(m, r.at, 1 / volume.rings.length), [0, 0, 0]);
        const normals = points.map(() => [0, 0, 0] as [number, number, number]);
        for (const [a, b, c, d] of quads) {
            const [pa, pb, pc] = [points[a]!, points[b]!, points[c]!];
            let n = crossed([pb[0] - pa[0], pb[1] - pa[1], pb[2] - pa[2]], [pc[0] - pa[0], pc[1] - pa[1], pc[2] - pa[2]]);
            const out: V3 = [pa[0] - middle[0], pa[1] - middle[1], pa[2] - middle[2]];
            if (n[0] * out[0] + n[1] * out[1] + n[2] * out[2] < 0) n = [-n[0], -n[1], -n[2]];
            for (const v of [a, b, c, d]) { normals[v]![0] += n[0]; normals[v]![1] += n[1]; normals[v]![2] += n[2]; }
        }
        const picture = points.map((p) => {
            const v = toView(p);
            return [(ax + v[0] * s) * SUPERSAMPLE, (ay - v[1] * s) * SUPERSAMPLE, v[2]] as const;
        });
        const facing = normals.map((n) => toView(unit(n)));
        const triangle = (a: number, b: number, c: number): void => {
            const [p0, p1, p2] = [picture[a]!, picture[b]!, picture[c]!];
            const area = (p1[0] - p0[0]) * (p2[1] - p0[1]) - (p2[0] - p0[0]) * (p1[1] - p0[1]);
            if (area === 0) return;
            const [left, right] = [Math.max(0, Math.floor(Math.min(p0[0], p1[0], p2[0]))), Math.min(W - 1, Math.ceil(Math.max(p0[0], p1[0], p2[0])))];
            const [top, bottom] = [Math.max(0, Math.floor(Math.min(p0[1], p1[1], p2[1]))), Math.min(H - 1, Math.ceil(Math.max(p0[1], p1[1], p2[1])))];
            for (let y = top; y <= bottom; y++) {
                for (let x = left; x <= right; x++) {
                    const [px, py] = [x + 0.5, y + 0.5];
                    const w1 = ((px - p0[0]) * (p2[1] - p0[1]) - (p2[0] - p0[0]) * (py - p0[1])) / area;
                    const w2 = ((p1[0] - p0[0]) * (py - p0[1]) - (px - p0[0]) * (p1[1] - p0[1])) / area;
                    const w0 = 1 - w1 - w2;
                    if (w0 < 0 || w1 < 0 || w2 < 0) continue;
                    const z = w0 * p0[2] + w1 * p1[2] + w2 * p2[2];
                    const i = y * W + x;
                    if (z <= depth[i]!) continue;
                    depth[i] = z;
                    tagAt[i] = tag;
                    for (let k = 0; k < 3; k++) nAt[i * 3 + k] = w0 * facing[a]![k]! + w1 * facing[b]![k]! + w2 * facing[c]![k]!;
                }
            }
        };
        for (const [a, b, c, d] of quads) { triangle(a, b, c); triangle(a, c, d); }
        // Ends closed flat: they are hidden in the block they meet, or small enough not to matter.
        for (const k of [0, volume.rings.length - 1]) {
            for (let i = 1; i + 1 < AROUND; i++) triangle(k * AROUND, k * AROUND + i, k * AROUND + i + 1);
        }
    }
    const mask = new Float32Array(w * h);
    const normals = new Float32Array(w * h * 3);
    const contour = new Float32Array(w * h);
    const parts: Record<string, Float32Array> = Object.fromEntries(FIGURE_TAGS.map((tag) => [tag, new Float32Array(w * h)]));
    const share = 1 / (SUPERSAMPLE * SUPERSAMPLE);
    // How far one block must stand in front of the next, in body units, to count as an edge.
    const step = 0.02;
    for (let y = 0; y < H; y++) {
        for (let x = 0; x < W; x++) {
            const i = y * W + x, o = (y >> 1) * w + (x >> 1);
            const tag = tagAt[i]!;
            const right = x + 1 < W ? depth[i + 1]! : -Infinity, below = y + 1 < H ? depth[i + W]! : -Infinity;
            if (tag >= 0 || right > -Infinity || below > -Infinity) {
                const here = depth[i]!;
                if (Math.abs(here - right) > step || Math.abs(here - below) > step) contour[o] = Math.min(1, contour[o]! + share * 2);
            }
            if (tag < 0) continue;
            mask[o]! += share;
            parts[FIGURE_TAGS[tag]!]![o]! += share;
            for (let k = 0; k < 3; k++) normals[o * 3 + k]! += nAt[i * 3 + k]! * share;
        }
    }
    for (let o = 0; o < mask.length; o++) {
        const len = Math.hypot(normals[o * 3]!, normals[o * 3 + 1]!, normals[o * 3 + 2]!);
        if (len > 0) for (let k = 0; k < 3; k++) normals[o * 3 + k]! /= len;
    }

    // The head's cross on the side it faces: its middle line, and the line the eyes sit on.
    const head = frames.head!, crown = frames.crown!;
    const centre = between(head.at, crown.at, 0.46);
    const wHead = build.head ?? 1;
    const cross = (at: (t: number) => V3): (readonly [number, number])[] => Array.from({ length: 13 }, (_, i) => toPicture(at((i / 12) * Math.PI - Math.PI / 2)));
    const lines = [
        ...BONES.map(([a, b]) => [toPicture(frames[a]!.at), toPicture(frames[b]!.at)]),
        cross((t) => plus(plus(centre, head.y, Math.sin(t) * 0.108 * wHead), head.z, Math.cos(t) * 0.09 * wHead + 0.006)),
        cross((t) => plus(plus(plus(centre, head.y, -0.018 * wHead), head.x, Math.sin(t) * 0.074 * wHead), head.z, Math.cos(t) * 0.088 * wHead + 0.006)),
    ];
    return { mask, parts, normals, contour, joints, lines, light: normalized(layer.light ?? [-0.5, 0.6, 0.6]) };
}

export const figureLight = normalized;