A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
* @file textureStreaming.ts
* @brief Holds each mesh texture at the size its views need, within the quality level's budget.
*
* @details The renderer says, for every texture a scene view drew a mesh with, how many texels across it needs for a texel to be about
* a pixel there (MeshPlugin, read through `renderer_takeTextureWants`). Four times a second this turns that into a size per
* texture — the next power of two above the need, never below `minEdge` nor above the file — and moves the textures that are
* the wrong size: decoded again from their file at the new size and put behind the same handle, so every material and
* renderer holding the handle sees it at once. A texture grows as soon as it is needed bigger and shrinks only after it has
* been needed smaller for a while, so a camera passing by does not make it go back and forth.
*
* What it streams: textures decoded from a file that only meshes draw. A texture a sprite, a UI element, a tilemap, a particle
* or a trail also draws is put back at its size, since nothing measures what those need. A mesh texture is loaded no larger
* than 256 texels on a side to begin with (Assets.setTextureStreamingStart); a web build ships a small copy beside it
* (pipeline streamTail.ts), which is what is fetched then, so the file itself is downloaded only once a view needs more. A
* KTX2 file is cut at a level and not decoded again.
*/
import type { App, Plugin } from '../app/app';
import { defineSystem, Schedule } from '../ecs/system';
import { defineResource } from '../ecs/resource';
import { getComponent } from '../ecs/component';
import { playModeOnly } from '../ecs/env';
import { Assets } from '../asset/AssetPlugin';
import { setTextureStreamingStartProvider } from '../asset/Assets';
import { engineApi } from '../ecs/bridge/engineApi';
import { getTextureDimensions } from '../wasm/resourceManager';
import { platformNow } from '../platform';
import { Quality } from './qualityRuntime';
/** What the streamer keeps of one texture. @experimental */
export interface StreamedTexture {
handle: number;
/** The file's own size. */
fullWidth: number;
fullHeight: number;
/** The longer side it is held at now. */
edge: number;
/** The most texels across a view asked for lately; 0 when none has drawn it for `unseenAfter`. */
wanted: number;
/** When a view last drew it, ms. */
seenAt: number;
/** Since when the size it should be has been smaller than the one it is, ms; -1 when it has not. */
smallerSince: number;
}
/** @experimental */
export interface TextureStreamingOptions {
/** No texture is held smaller than this on its longer side. */
minEdge: number;
/** The bytes the streamed textures may hold together; 0 is no cap. */
budgetBytes: number;
/** How long a texture must be needed smaller before it shrinks, ms. */
shrinkAfter: number;
/** How long a texture no view draws keeps the size it was needed at, ms; then it goes to `minEdge`. */
unseenAfter: number;
}
/** @experimental */
export const TEXTURE_STREAMING_DEFAULTS: TextureStreamingOptions = { minEdge: 64, budgetBytes: 0, shrinkAfter: 3000, unseenAfter: 5000 };
const nextPow2 = (n: number): number => 2 ** Math.ceil(Math.log2(Math.max(1, n)));
/** The longer side a texture should be held at: the next power of two above what is wanted, between `minEdge` and the file. @experimental */
export function streamTargetEdge(fullEdge: number, wanted: number, minEdge: number): number {
if (!(wanted > 0)) return Math.min(fullEdge, minEdge);
return Math.min(fullEdge, Math.max(minEdge, nextPow2(wanted)));
}
/** The bytes a texture held at `edge` on its longer side takes, with its chain, at four bytes a texel. @experimental */
export function streamedBytes(fullWidth: number, fullHeight: number, edge: number): number {
const k = Math.min(1, edge / Math.max(fullWidth, fullHeight));
const w = Math.max(1, Math.round(fullWidth * k)), h = Math.max(1, Math.round(fullHeight * k));
return Math.round(w * h * 4 * 4 / 3);
}
/** A size to move a texture to. @experimental */
export interface StreamMove {
handle: number;
edge: number;
}
/**
* What to move now: every texture whose size should change, the growing ones first and the most starved of those first, then the
* shrinking ones. With a budget, the sizes are first brought within it by halving the largest, so a scene that needs more than the
* budget is drawn a level softer rather than over it. Writes `smallerSince` on the entries it judges.
*
* @experimental
*/
export function planStreaming(entries: readonly StreamedTexture[], now: number, options: TextureStreamingOptions): StreamMove[] {
const target = new Map<number, number>();
for (const e of entries) {
const wanted = now - e.seenAt > options.unseenAfter ? 0 : e.wanted;
target.set(e.handle, streamTargetEdge(Math.max(e.fullWidth, e.fullHeight), wanted, options.minEdge));
}
if (options.budgetBytes > 0) {
let total = 0;
for (const e of entries) total += streamedBytes(e.fullWidth, e.fullHeight, target.get(e.handle)!);
while (total > options.budgetBytes) {
let worst: StreamedTexture | null = null, worstBytes = 0;
for (const e of entries) {
const edge = target.get(e.handle)!;
if (edge <= options.minEdge) continue;
const b = streamedBytes(e.fullWidth, e.fullHeight, edge);
if (b > worstBytes) { worst = e; worstBytes = b; }
}
if (!worst) break;
const halved = Math.max(options.minEdge, target.get(worst.handle)! / 2);
total += streamedBytes(worst.fullWidth, worst.fullHeight, halved) - worstBytes;
target.set(worst.handle, halved);
}
}
const grow: Array<StreamMove & { starved: number }> = [];
const shrink: StreamMove[] = [];
for (const e of entries) {
const edge = target.get(e.handle)!;
if (edge > e.edge) {
e.smallerSince = -1;
grow.push({ handle: e.handle, edge, starved: edge / e.edge });
} else if (edge < e.edge) {
if (e.smallerSince < 0) e.smallerSince = now;
if (now - e.smallerSince >= options.shrinkAfter) shrink.push({ handle: e.handle, edge });
} else {
e.smallerSince = -1;
}
}
grow.sort((a, b) => b.starved - a.starved);
return [...grow.map(({ handle, edge }) => ({ handle, edge })), ...shrink];
}
/** What streaming is doing, for a profiler and a check. @experimental */
export class TextureStreaming {
readonly textures = new Map<number, StreamedTexture>();
/** Textures something other than a mesh draws, left at their size. */
readonly held = new Set<number>();
/** Textures that came back larger than asked (a host that cannot cut a file at a level): left alone from then on. */
readonly refused = new Set<number>();
/** Textures being put back at their size because something other than a mesh draws them. */
readonly restoring = new Set<number>();
moves = 0;
failures = 0;
inFlight = 0;
enabled = false;
/** The bytes the streamed textures take now, and would at their files' sizes. */
bytes(): { now: number; full: number } {
let now = 0, full = 0;
for (const t of this.textures.values()) {
now += streamedBytes(t.fullWidth, t.fullHeight, t.edge);
full += streamedBytes(t.fullWidth, t.fullHeight, Math.max(t.fullWidth, t.fullHeight));
}
return { now, full };
}
}
/** @experimental */
export const TextureStreamingRes = defineResource<TextureStreaming>(new TextureStreaming(), 'TextureStreaming');
/** Components whose `texture` field draws a texture no view measures the need of. */
const UNMEASURED = ['Sprite', 'UIVisual', 'TilemapLayer', 'ParticleEmitter', 'TrailRenderer', 'SpriteMask'];
/** @experimental */
export class TextureStreamingPlugin implements Plugin {
name = 'textureStreaming';
build(app: App): void {
const state = new TextureStreaming();
app.insertResource(TextureStreamingRes, state);
const world = app.world;
let last = 0, lastHeld = -Infinity;
const MAX_IN_FLIGHT = 4;
/** The edge a mesh texture starts at while streaming is on. */
const START_EDGE = 256;
// Asked as each texture loads, not first in the frame loop: a scene may start loading before the first frame does, and the
// quality level may be settled after this plugin is built.
const startEdge = (): number => {
const quality = app.hasResource(Quality) ? app.getResource(Quality) : null;
return quality?.profile.textureStreaming && engineApi(app)?.renderer_takeTextureWants ? START_EDGE : 0;
};
setTextureStreamingStartProvider(startEdge);
const parse = (raw: string): Map<number, number> => {
const out = new Map<number, number>();
if (!raw) return out;
for (const pair of raw.split(',')) {
const cut = pair.indexOf(':');
if (cut > 0) out.set(Number(pair.slice(0, cut)), Number(pair.slice(cut + 1)));
}
return out;
};
const refreshHeld = (): void => {
state.held.clear();
for (const name of UNMEASURED) {
const def = getComponent(name);
if (!def) continue;
for (const e of world.getEntitiesWithComponents([def])) {
const t = (world.get(e, def) as { texture?: number }).texture;
if (typeof t === 'number' && t > 0) state.held.add(t);
}
}
};
app.addSystemToSchedule(Schedule.Last, defineSystem([], () => {
const quality = app.hasResource(Quality) ? app.getResource(Quality) : null;
const profile = quality?.profile;
state.enabled = !!profile?.textureStreaming;
const api = engineApi(app);
if (!state.enabled || !api?.renderer_takeTextureWants || !app.hasResource(Assets)) return;
const now = platformNow();
if (now - last < 250) return;
last = now;
if (now - lastHeld > 2000) { refreshHeld(); lastHeld = now; }
const assets = app.getResource(Assets);
for (const [handle, side] of parse(api.renderer_takeTextureWants())) {
if (state.held.has(handle)) { state.textures.delete(handle); continue; }
if (state.refused.has(handle)) continue;
let entry = state.textures.get(handle);
if (!entry) {
if (!assets.textureResizable(handle)) continue;
const size = assets.textureFullSize(handle) ?? getTextureDimensions(handle);
if (!size || size.width <= 0 || size.height <= 0) continue;
const held = getTextureDimensions(handle);
entry = { handle, fullWidth: size.width, fullHeight: size.height, edge: held ? Math.max(held.width, held.height) : Math.max(size.width, size.height), wanted: 0, seenAt: now, smallerSince: -1 };
state.textures.set(handle, entry);
}
entry.wanted = entry.seenAt === now ? Math.max(entry.wanted, side) : side;
entry.seenAt = now;
}
for (const handle of state.held) {
state.textures.delete(handle);
// Loaded small for a mesh and drawn by something no view measures: put back at its size.
const full = assets.textureFullSize(handle), now_ = getTextureDimensions(handle);
if (full && now_ && (now_.width < full.width || now_.height < full.height) && state.inFlight < MAX_IN_FLIGHT && !state.restoring.has(handle)) {
state.restoring.add(handle);
state.inFlight++;
void assets.resizeTexture(handle, 0).catch(() => { state.failures++; })
.finally(() => { state.inFlight--; state.restoring.delete(handle); });
}
}
const options: TextureStreamingOptions = {
...TEXTURE_STREAMING_DEFAULTS,
budgetBytes: Math.max(0, profile!.textureStreamingBudget) * 1024 * 1024,
};
for (const move of planStreaming([...state.textures.values()], now, options)) {
if (state.inFlight >= MAX_IN_FLIGHT) break;
const entry = state.textures.get(move.handle)!;
const full = Math.max(entry.fullWidth, entry.fullHeight);
state.inFlight++;
const was = entry.edge;
entry.edge = move.edge; // claimed now, so the next pass does not ask again while this one decodes
void assets.resizeTexture(move.handle, move.edge >= full ? 0 : move.edge).then((size) => {
if (size && Math.max(size.width, size.height) > move.edge && move.edge < full) {
// It could not be made that small, and asking again would decode it again for nothing.
entry.edge = Math.max(size.width, size.height);
state.refused.add(move.handle);
state.textures.delete(move.handle);
} else if (size) { entry.edge = Math.max(size.width, size.height); state.moves++; }
else { entry.edge = was; state.failures++; state.textures.delete(move.handle); }
}, () => { entry.edge = was; state.failures++; state.textures.delete(move.handle); })
.finally(() => { state.inFlight--; });
}
}, { name: 'TextureStreamingSystem' }), { runIf: playModeOnly });
}
}
/** @experimental */
export const textureStreamingPlugin: Plugin = new TextureStreamingPlugin();