Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

19.2 KB
Raw Blame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
 * Compressed-texture loading (KTX2 / Basis Universal).
 *
 * Decoded textures stay GPU-compressed in VRAM (4–8× smaller than RGBA8), the key
 * constraint on mobile. The actual transcode is done by a wasm side-module behind
 * the {@link BasisTranscoder} seam; this module owns the device
 * capability probe, the format choice, and the WebGL upload — all testable without
 * the wasm.
 *
 * Upload is JS-direct (`gl.compressedTexImage2D`) to mirror the existing
 * `TextureLoader` PNG path, and because WebGL compressed-texture extensions must be
 * enabled JS-side via `getExtension`. The C++ `GfxDevice::compressedTexImage2D`
 * entry backs the non-WebGL2 fallback path instead.
 */
import { TextureContent, type ESEngineModule } from '../wasm';
import { unwrapRestagedPath } from '../assetTypes';
import { applyBoundTextureSampling, handOverNewTexture } from './glTextureUpload';

// =============================================================================
// Format vocabulary
// =============================================================================

/**
 * GPU compressed formats the transcoder can target, mirroring the C++
 * `GfxCompressedFormat`. ASTC is best quality/size, ETC2 is the WebGL2 baseline,
 * S3TC covers desktop GPUs.
 */
export enum CompressedTextureFormat {
    ETC2_RGBA8 = 'etc2-rgba8',
    ASTC_4x4 = 'astc-4x4',
    S3TC_DXT5 = 's3tc-dxt5',
}

const KTX2_IDENTIFIER = [0xab, 0x4b, 0x54, 0x58, 0x20, 0x32, 0x30, 0xbb, 0x0d, 0x0a, 0x1a, 0x0a];

/** True if `bytes` begins with the 12-byte KTX2 file identifier. */
export function isKtx2(bytes: Uint8Array): boolean {
    if (bytes.length < KTX2_IDENTIFIER.length) return false;
    for (let i = 0; i < KTX2_IDENTIFIER.length; i++) {
        if (bytes[i] !== KTX2_IDENTIFIER[i]) return false;
    }
    return true;
}

/** True if `path` names a KTX2 container, including the `.ktx2.bin` a packer
 *  that refuses the real suffix is given (what restaging looks like is
 *  `unwrapRestagedPath`'s to know, not every reader's). */
export function isKtx2Path(path: string): boolean {
    return unwrapRestagedPath(path).endsWith('.ktx2');
}

// =============================================================================
// Transcoder seam
// =============================================================================

export interface TranscodedLevel {
    readonly width: number;
    readonly height: number;
    readonly data: Uint8Array;
}

export interface TranscodeResult {
    readonly width: number;
    readonly height: number;
    /** GPU-ready compressed block data for the requested format, level 0. */
    readonly data: Uint8Array;
    /** Every mip level the file carries, level 0 first; absent means level 0 only. */
    readonly levels?: readonly TranscodedLevel[];
}

export interface RgbaResult {
    readonly width: number;
    readonly height: number;
    /** width*height*4 RGBA8 bytes, level 0. */
    readonly data: Uint8Array;
    /** Every mip level the file carries, level 0 first; absent means level 0 only. */
    readonly levels?: readonly TranscodedLevel[];
}

/**
 * Decodes a KTX2/Basis container into GPU-ready bytes. The concrete implementation
 * is a wasm side-module injected behind this interface, so the loader
 * decision logic stays unit-testable without it.
 */
export interface BasisTranscoder {
    /** Transcode to a device-supported compressed format, or null if it cannot.
     *  A worker-backed transcoder answers with a promise. */
    transcode(ktx2: Uint8Array, target: CompressedTextureFormat): TranscodeResult | null | Promise<TranscodeResult | null>;
    /** Decode to uncompressed RGBA8 — the universal fallback. */
    transcodeToRgba(ktx2: Uint8Array): RgbaResult | null | Promise<RgbaResult | null>;
}

// =============================================================================
// Device capability probe
// =============================================================================

// Minimal shapes for the extension constants we read — robust to lib.dom not
// typing every getExtension overload (some omit the ETC one). The ASTC/ETC
// extensions expose their sRGB variants on the same object; S3TC splits sRGB
// into a separate extension (WEBGL_compressed_texture_s3tc_srgb).
interface AstcExt {
    readonly COMPRESSED_RGBA_ASTC_4x4_KHR: number;
    readonly COMPRESSED_SRGB8_ALPHA8_ASTC_4x4_KHR: number;
}
interface EtcExt {
    readonly COMPRESSED_RGBA8_ETC2_EAC: number;
    readonly COMPRESSED_SRGB8_ALPHA8_ETC2_EAC: number;
}
interface S3tcExt { readonly COMPRESSED_RGBA_S3TC_DXT5_EXT: number }
interface S3tcSrgbExt { readonly COMPRESSED_SRGB_ALPHA_S3TC_DXT5_EXT: number }

export interface CompressedTextureSupport {
    readonly astc: AstcExt | null;
    readonly etc: EtcExt | null;
    readonly s3tc: S3tcExt | null;
    readonly s3tcSrgb: S3tcSrgbExt | null;
}

/**
 * Probes — and, crucially, *enables* — the WebGL compressed-texture extensions.
 * `getExtension` is the only way to turn an extension on, so this must run before
 * any `compressedTexImage2D` with a non-core format.
 */
export function detectCompressedTextureSupport(gl: WebGL2RenderingContext): CompressedTextureSupport {
    return {
        astc: gl.getExtension('WEBGL_compressed_texture_astc') as AstcExt | null,
        etc: gl.getExtension('WEBGL_compressed_texture_etc') as EtcExt | null,
        s3tc: gl.getExtension('WEBGL_compressed_texture_s3tc') as S3tcExt | null,
        s3tcSrgb: gl.getExtension('WEBGL_compressed_texture_s3tc_srgb') as S3tcSrgbExt | null,
    };
}

/**
 * The C++ `GfxCompressedFormat` ordinal for each target, as [unorm, sRGB].
 *
 * Asking the ENGINE which formats it samples is the only question both backends
 * answer; WebGL extensions are one backend's answer, and on the other one that
 * answer is "none", which is how a compressed texture became a white placeholder.
 */
const ENGINE_FORMAT_CODE: Record<CompressedTextureFormat, readonly [number, number]> = {
    [CompressedTextureFormat.ETC2_RGBA8]: [1, 6],
    [CompressedTextureFormat.ASTC_4x4]: [2, 7],
    [CompressedTextureFormat.S3TC_DXT5]: [5, 8],
};

/** The engine's ordinal for `format` under the running colour pipeline. */
export function engineFormatCode(format: CompressedTextureFormat, srgb: boolean): number {
    return ENGINE_FORMAT_CODE[format][srgb ? 1 : 0];
}

/**
 * Quality/size order: ASTC (best), ETC2 (the WebGL2 baseline), S3TC (desktop).
 *
 * ONE list. The GL path and the engine-upload path ask different oracles about
 * the same device; a second ordering would let a texture land in a different
 * format depending on which backend loaded it.
 */
export const TARGET_PREFERENCE: readonly CompressedTextureFormat[] = [
    CompressedTextureFormat.ASTC_4x4,
    CompressedTextureFormat.ETC2_RGBA8,
    CompressedTextureFormat.S3TC_DXT5,
];

/** Best target the ENGINE says it can sample, in {@link TARGET_PREFERENCE} order. */
export function chooseEngineTargetFormat(
    supports: (code: number) => boolean, srgb = false,
): CompressedTextureFormat | null {
    for (const f of TARGET_PREFERENCE) if (supports(engineFormatCode(f, srgb))) return f;
    return null;
}

/**
 * Best available target in {@link TARGET_PREFERENCE} order. null = none.
 * With `srgb` (linear pipeline) a format only qualifies when its sRGB variant is
 * uploadable — S3TC needs the separate s3tc_srgb extension; ASTC/ETC2 sRGB ride
 * the same extension as their UNORM twins. The transcoded block data is
 * identical either way; only the sampling interpretation differs, which is why
 * "can this be uploaded" IS whether an internalformat exists for it.
 */
export function chooseTargetFormat(
    support: CompressedTextureSupport, srgb = false,
): CompressedTextureFormat | null {
    for (const f of TARGET_PREFERENCE) if (glInternalFormat(support, f, srgb) !== null) return f;
    return null;
}

/** Every target this device samples, best first — the capability, not a choice. */
export function supportedTargetFormats(
    supports: (f: CompressedTextureFormat) => boolean,
): CompressedTextureFormat[] {
    return TARGET_PREFERENCE.filter(supports);
}

/** WebGL `internalformat` enum for a chosen format, from its enabling extension. */
export function glInternalFormat(
    support: CompressedTextureSupport, fmt: CompressedTextureFormat, srgb = false,
): number | null {
    switch (fmt) {
        case CompressedTextureFormat.ASTC_4x4:
            if (!support.astc) return null;
            return srgb ? support.astc.COMPRESSED_SRGB8_ALPHA8_ASTC_4x4_KHR
                        : support.astc.COMPRESSED_RGBA_ASTC_4x4_KHR;
        case CompressedTextureFormat.ETC2_RGBA8:
            if (!support.etc) return null;
            return srgb ? support.etc.COMPRESSED_SRGB8_ALPHA8_ETC2_EAC
                        : support.etc.COMPRESSED_RGBA8_ETC2_EAC;
        case CompressedTextureFormat.S3TC_DXT5:
            if (srgb) return support.s3tcSrgb ? support.s3tcSrgb.COMPRESSED_SRGB_ALPHA_S3TC_DXT5_EXT : null;
            return support.s3tc ? support.s3tc.COMPRESSED_RGBA_S3TC_DXT5_EXT : null;
    }
    return null;
}

// =============================================================================
// Upload
// =============================================================================

export interface CompressedUploadOptions {
    readonly filter?: 'linear' | 'nearest';
    readonly wrap?: 'repeat' | 'clamp' | 'mirror';
    /** Linear pipeline: store sRGB-encoded so the sampler linearizes in hardware. */
    readonly srgb?: boolean;
    /** False keeps level 0 alone, however many levels the file carries. */
    readonly mipmaps?: boolean;
    /** Leave out the levels larger than this many texels on a side (texture streaming); 0 keeps them all. */
    readonly maxEdge?: number;
}

/**
 * The chain from the first level no larger than @p maxEdge on a side: what a texture held smaller than its file needs, without decoding
 * or encoding anything. A file whose smallest level is still larger keeps its smallest; 0 keeps the chain as it is.
 */
export function levelsFrom<T extends TranscodeResult | RgbaResult>(r: T, maxEdge: number | undefined): T {
    if (!maxEdge || maxEdge <= 0 || Math.max(r.width, r.height) <= maxEdge) return r;
    const levels = r.levels && r.levels.length > 0 ? r.levels : [{ width: r.width, height: r.height, data: r.data }];
    let from = levels.findIndex((l) => Math.max(l.width, l.height) <= maxEdge);
    if (from < 0) from = levels.length - 1;
    const kept = levels.slice(from);
    return { ...r, width: kept[0]!.width, height: kept[0]!.height, data: kept[0]!.data, levels: kept };
}

export interface UploadedTexture {
    readonly handle: number;
    readonly width: number;
    readonly height: number;
}

/**
 * A texture's sampling from the levels it was given: a chain the file carried is
 * sampled as one, and a single compressed level samples without mips — a mip
 * filter over one level would make it incomplete, and no compressed format can
 * generate the rest. Only uncompressed pixels may have a chain generated.
 */
function applyParams(gl: WebGL2RenderingContext, levels: number, generate: boolean, opts?: CompressedUploadOptions): void {
    applyBoundTextureSampling(gl, { filter: opts?.filter, wrap: opts?.wrap, mipmaps: generate, levels });
}

function levelsOf(r: TranscodeResult | RgbaResult, opts?: CompressedUploadOptions): readonly TranscodedLevel[] {
    const base = { width: r.width, height: r.height, data: r.data };
    if (opts?.mipmaps === false) return [base];
    return r.levels && r.levels.length > 0 ? r.levels : [base];
}

/** S3TC requires each mip's dimensions, scaled to level zero, to align to
 * four texels. Small tail levels remain valid at their original mip index. */
function webglCompressedLevelsSupported(format: CompressedTextureFormat, levels: readonly TranscodedLevel[]): boolean {
    return format !== CompressedTextureFormat.S3TC_DXT5 || levels.every((level, index) =>
        (level.width * 2 ** index) % 4 === 0 && (level.height * 2 ** index) % 4 === 0);
}

/** Upload pre-transcoded compressed blocks via `gl.compressedTexImage2D`. */
export function uploadCompressedTexture(
    gl: WebGL2RenderingContext, module: ESEngineModule,
    support: CompressedTextureSupport, fmt: CompressedTextureFormat,
    t: TranscodeResult, opts?: CompressedUploadOptions,
): UploadedTexture {
    const internalFormat = glInternalFormat(support, fmt, opts?.srgb ?? false);
    if (internalFormat == null) throw new Error(`compressed upload: no GL internalformat for ${fmt}`);
    // gpuBytes is the ACTUAL VRAM size for the eviction budget — a compressed
    // format is 4–8× smaller than the pool's RGBA8 estimate, and billing it at
    // the estimate would squat on most of the budget.
    const levels = levelsOf(t, opts);
    if (!webglCompressedLevelsSupported(fmt, levels)) {
        throw new Error('S3TC texture mip dimensions are not block aligned for WebGL');
    }
    const handle = handOverNewTexture(module, gl, (g) => {
        levels.forEach((l, i) => g.compressedTexImage2D(g.TEXTURE_2D, i, internalFormat, l.width, l.height, 0, l.data));
        applyParams(g, levels.length, false, opts);
    }, {
        width: t.width, height: t.height, content: TextureContent.Asset,
        gpuBytes: levels.reduce((n, l) => n + l.data.byteLength, 0),
    });
    return { handle, width: t.width, height: t.height };
}

/** Fallback: upload decoded RGBA8 via `gl.texImage2D`. */
export function uploadRgbaTexture(
    gl: WebGL2RenderingContext, module: ESEngineModule, r: RgbaResult, opts?: CompressedUploadOptions,
): UploadedTexture {
    const handle = handOverNewTexture(module, gl, (g) => {
        // Linear pipeline: the decoded pixels are sRGB-encoded color, same as
        // the PNG path — store them in an sRGB format so sampling linearizes.
        const internalFormat = opts?.srgb ? g.SRGB8_ALPHA8 : g.RGBA;
        const levels = levelsOf(r, opts);
        levels.forEach((l, i) => g.texImage2D(g.TEXTURE_2D, i, internalFormat, l.width, l.height, 0, g.RGBA, g.UNSIGNED_BYTE, l.data));
        applyParams(g, levels.length, levels.length === 1 && opts?.mipmaps !== false, opts);
    }, { width: r.width, height: r.height, content: TextureContent.Asset });
    return { handle, width: r.width, height: r.height };
}

// =============================================================================
// Orchestration (the compressed-vs-fallback decision — the testable core)
// =============================================================================

/** What a texture ended up as on the GPU. */
export type UploadedGpuFormat = CompressedTextureFormat | 'rgba8';

/**
 * Why an upload landed on the format it did.
 *
 * `uncompressed-payload` is not a fallback — an ordinary image asked for
 * nothing. The other three each name a different thing to change: the DEVICE
 * samples none, the FILE would not transcode, or its size is not whole blocks.
 */
export type TextureUploadReason =
    | 'compressed'
    | 'no-device-format'
    | 'transcode-failed'
    | 'not-block-aligned'
    | 'uncompressed-payload';

/**
 * What one texture upload did with a cooked payload, on THIS device.
 *
 * The second half; the first is what the build shipped
 * (pipeline/src/assets/textureCookDecision.ts). Kept apart because a cook that
 * shipped raw is a build setting and a device that decoded it is a device.
 */
export interface TextureUploadDecision {
    /** What arrived: a transcodable container, or a plain image. */
    readonly payload: 'ktx2' | 'raw';
    /** The best format this device offered for it; null when it offered none. */
    readonly target: CompressedTextureFormat | null;
    readonly effective: UploadedGpuFormat;
    readonly reason: TextureUploadReason;
}

/** The decision a compressed payload came to, given the device's answer and
 *  whether the transcode to it worked. Pure — the three cases hold without a
 *  GL context, a device or the basis module. */
export function compressedUploadDecision(
    target: CompressedTextureFormat | null, transcoded: boolean,
): TextureUploadDecision {
    if (target === null) {
        return { payload: 'ktx2', target: null, effective: 'rgba8', reason: 'no-device-format' };
    }
    if (!transcoded) {
        return { payload: 'ktx2', target, effective: 'rgba8', reason: 'transcode-failed' };
    }
    return { payload: 'ktx2', target, effective: target, reason: 'compressed' };
}

/** The target an engine ordinal names, or null when it names none of the three
 *  (an RGBA upload, or a format no loader targets). */
export function formatFromEngineCode(code: number): CompressedTextureFormat | null {
    for (const f of TARGET_PREFERENCE) {
        const [unorm, srgb] = ENGINE_FORMAT_CODE[f];
        if (code === unorm || code === srgb) return f;
    }
    return null;
}

/**
 * What a native host's own KTX2 upload came to.
 *
 * `format` is the engine ordinal it uploaded, negative when it decoded to RGBA.
 * `blockRefused` separates the two ways that happens — an image that was not
 * whole blocks, or a device that offered nothing. Only the first is the asset's.
 */
export function hostUploadDecision(format: number, blockRefused: boolean): TextureUploadDecision {
    const target = formatFromEngineCode(format);
    if (target) return { payload: 'ktx2', target, effective: target, reason: 'compressed' };
    return {
        payload: 'ktx2', target: null, effective: 'rgba8',
        reason: blockRefused ? 'not-block-aligned' : 'no-device-format',
    };
}

/** A texture that was never a compressed payload — an ordinary image upload. */
export const RAW_PAYLOAD_UPLOAD: TextureUploadDecision = {
    payload: 'raw', target: null, effective: 'rgba8', reason: 'uncompressed-payload',
};

export interface LoadedCompressedTexture extends UploadedTexture {
    /** What this upload came to, and why — the fallback is otherwise an
     *  else-branch that counts nothing. */
    readonly decision: TextureUploadDecision;
}

/**
 * Load a KTX2 buffer into a GPU texture: prefer a device-supported compressed
 * format, fall back to RGBA8 when none is available or the compressed transcode
 * fails. Throws only if even the RGBA decode fails (a corrupt/unsupported file).
 */
export async function loadCompressedTexture(
    gl: WebGL2RenderingContext, module: ESEngineModule,
    transcoder: BasisTranscoder, bytes: Uint8Array, opts?: CompressedUploadOptions,
): Promise<LoadedCompressedTexture> {
    const support = detectCompressedTextureSupport(gl);
    const target = chooseTargetFormat(support, opts?.srgb ?? false);
    const whole = target !== null ? await transcoder.transcode(bytes, target) : null;
    const t = whole ? levelsFrom(whole, opts?.maxEdge) : null;
    const aligned = target !== null && t !== null && webglCompressedLevelsSupported(target, levelsOf(t, opts));
    const decision: TextureUploadDecision = target !== null && t && !aligned
        ? { payload: 'ktx2', target, effective: 'rgba8', reason: 'not-block-aligned' }
        : compressedUploadDecision(target, t !== null);
    if (target !== null && t && aligned) {
        return { ...uploadCompressedTexture(gl, module, support, target, t, opts), decision };
    }
    const decoded = await transcoder.transcodeToRgba(bytes);
    if (!decoded) throw new Error('BasisTranscoder failed to decode KTX2 (compressed and RGBA paths both failed)');
    const rgba = levelsFrom(decoded, opts?.maxEdge);
    return { ...uploadRgbaTexture(gl, module, rgba, opts), decision };
}