Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(audio): long clips on the Web play from their compressed file instead of decoding to memory

yhh2aefb6f8ca79
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 57329e760..1b02909b5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,10 @@ published separately; it ships inside the editor.
## [Unreleased]
+### Changed
+
+- **Long audio streams on the Web.** Long clips on the Web play from their compressed file instead of being decoded to memory. A clip's import settings have a Load Type: Auto streams clips of 30 seconds or longer and decodes shorter ones, Decompress On Load always decodes, Streaming always streams. A streamed clip takes almost no memory (a three-minute stereo track is about 69 MB decoded), goes through the same bus, volume and fades, and gives up two things the import setting says: a sample-accurate loop point, and a second simultaneous voice of the same clip. The profiler's memory section shows the decoded audio and, when clips stream, how many and the memory they saved (an estimate from their length). The choice rides in the exported game's audio settings, so an exported game and a playable ad stream the same clips Play does.
+
## [0.93.0] - 2026-10-19
### Added
diff --git a/desktop b/desktop
index d57a27e56..ec8b311de 160000
--- a/desktop
+++ b/desktop
@@ -1 +1 @@
-Subproject commit d57a27e560874636a16e761a1cf3a49090ba0f3a
+Subproject commit ec8b311deee1a9e9bf64588019d35b8c2746520b
diff --git a/pipeline/src/assets/cookAssets.ts b/pipeline/src/assets/cookAssets.ts
index 63e8c007d..c11c20cbb 100644
--- a/pipeline/src/assets/cookAssets.ts
+++ b/pipeline/src/assets/cookAssets.ts
@@ -27,7 +27,7 @@ import { packAtlas, decodePngImage, encodePagePng, type AtlasInputImage } from '
// Per-asset texture cook settings (compress opt-out / format / size cap), read
// from the `.meta` `importer` block — the same registry the inspector edits, so a
// texture's ship-time compression is authored per asset, not one global switch.
-import { readTextureCookSettings } from '../project/importSettings';
+import { audioLoadTypeOf, readTextureCookSettings } from '../project/importSettings';
// The compression a build gives one texture, and the one reason it is that.
// The branch below IS this function — a cook that re-derived the choice would
// be a second answer to the question an inspector asks before a build runs.
@@ -428,6 +428,8 @@ export interface CookResult {
* caller needs to read the content it just shipped (e.g. to scan the scenes
* for subsystems the target cannot render). */
includedPaths: string[];
+ /** The shipped audio clips whose import settings choose how they load (not `auto`), by logical path. */
+ audioLoadTypes: Record<string, 'streaming' | 'decompress'>;
/** uuids present in the project but unreachable — culled from the build. */
unused: string[];
/**
@@ -996,9 +998,15 @@ export async function cookAssets(
const logical = byUuid.get(uuid)?.path;
if (logical) inclusion[logical] = how;
}
+ const audioLoadTypes: Record<string, 'streaming' | 'decompress'> = {};
+ for (const uuid of reachable) {
+ const e = byUuid.get(uuid);
+ const type = e?.type === 'audio' ? audioLoadTypeOf(e.importer) : null;
+ if (e && type) audioLoadTypes[e.path] = type;
+ }
return {
ok: failed.length === 0,
- outDir: absOut, manifestPath, included: [...reachable], includedPaths, unused, warnings, failed,
+ outDir: absOut, manifestPath, included: [...reachable], includedPaths, audioLoadTypes, unused, warnings, failed,
inclusion,
};
}
diff --git a/pipeline/src/export/exportGame.ts b/pipeline/src/export/exportGame.ts
index ff38be258..fdd0abcc9 100644
--- a/pipeline/src/export/exportGame.ts
+++ b/pipeline/src/export/exportGame.ts
@@ -36,7 +36,7 @@ import type { DebugChannelConfig, PackagedGameConfig } from 'esengine';
import { packagedDebugChannel } from './debugChannel';
import { packagedHotUpdate } from './hotUpdateConfig';
import {
- DEFAULT_RUNTIME_CONFIG, packagedRuntimeFields, type RuntimeProjectConfig,
+ DEFAULT_RUNTIME_CONFIG, packagedRuntimeFields, withAudioLoadTypes, type RuntimeProjectConfig,
} from '../project/runtimeConfig';
import { engineImportMap, FULL_IMPORT_MAP, LEAN_ENTRY_FILE, FULL_ENTRY_FILE, type EngineImportMap } from '../bundle/importMap';
import { engineInstalls, forcedSideModules, moduleChoices, type EngineInstallPlan } from '../bundle/engineInstalls';
@@ -1197,7 +1197,7 @@ async function produceExport(opts: ExportGameOptions): Promise<ExportGameResult>
...(platform === 'web' && opts.webChannel === 'crazygames' && opts.leaderboardEncryptionKey
? { leaderboardEncryptionKey: opts.leaderboardEncryptionKey } : {}),
...(scriptsFile ? { scripts: scriptsFile } : {}),
- ...packagedRuntimeFields(runtime),
+ ...packagedRuntimeFields(withAudioLoadTypes(runtime, cook.audioLoadTypes)),
...(hotUpdate ? { hotUpdate } : {}),
...(sideModuleDeclarations(projectModules, platform).length > 0
? { sideModules: sideModuleDeclarations(projectModules, platform) } : {}),
diff --git a/pipeline/src/export/exportMiniGame.ts b/pipeline/src/export/exportMiniGame.ts
index 375a46317..50e742ea6 100644
--- a/pipeline/src/export/exportMiniGame.ts
+++ b/pipeline/src/export/exportMiniGame.ts
@@ -33,7 +33,7 @@ import { packagedHotUpdate } from './hotUpdateConfig';
import { loadEsbuild } from '../bundle/esbuildRuntime';
import type { ProjectFeatures, ProjectPackaging } from '../project/format';
import {
- DEFAULT_RUNTIME_CONFIG, packagedRuntimeFields, type RuntimeProjectConfig,
+ DEFAULT_RUNTIME_CONFIG, packagedRuntimeFields, withAudioLoadTypes, type RuntimeProjectConfig,
} from '../project/runtimeConfig';
import { writeFile, mkdir, cp, readFile, rename, rm } from 'node:fs/promises';
import { brotliCompress as brotliCompressCb, constants as zlibConstants } from 'node:zlib';
@@ -481,7 +481,7 @@ export async function exportMiniGame(profile: MiniGameExportProfile, opts: {
// a WebGPU request cannot be honoured here — carrying it would be a config
// field the runtime is guaranteed to ignore.
const { renderBackend: _webgpuNotHere, ...packaged } =
- packagedRuntimeFields(opts.runtime ?? DEFAULT_RUNTIME_CONFIG);
+ packagedRuntimeFields(withAudioLoadTypes(opts.runtime ?? DEFAULT_RUNTIME_CONFIG, cook.audioLoadTypes));
const themeColors = packaged.uiThemeColors;
const runtimeArgs = Object.entries(packaged)
.map(([k, v]) => (k === 'uiThemeColors'
diff --git a/pipeline/src/export/exportPlayable.ts b/pipeline/src/export/exportPlayable.ts
index 88fcea4c3..a86710e46 100644
--- a/pipeline/src/export/exportPlayable.ts
+++ b/pipeline/src/export/exportPlayable.ts
@@ -21,7 +21,7 @@
import { loadEsbuild } from '../bundle/esbuildRuntime';
import { runtimeHostEntry } from '../bundle/runtimeHosts';
import {
- DEFAULT_RUNTIME_CONFIG, packagedRuntimeFields, type RuntimeProjectConfig,
+ DEFAULT_RUNTIME_CONFIG, packagedRuntimeFields, withAudioLoadTypes, type RuntimeProjectConfig,
} from '../project/runtimeConfig';
import { writeFile, mkdir, readFile, rm } from 'node:fs/promises';
import { existsSync } from 'node:fs';
@@ -386,7 +386,7 @@ export async function exportPlayable(opts: {
`window.__GAME_PATHMAP__=${JSON.stringify(pathMap)};` +
span(`scenes/${sceneName}.json`, `window.__GAME_SCENES__=${JSON.stringify(scenes)};`) +
`window.__GAME_FIRST__=${JSON.stringify(sceneName)};` +
- `window.__GAME_RUNTIME__=${JSON.stringify(playableRuntimeFields(opts.runtime ?? DEFAULT_RUNTIME_CONFIG))};` +
+ `window.__GAME_RUNTIME__=${JSON.stringify(playableRuntimeFields(withAudioLoadTypes(opts.runtime ?? DEFAULT_RUNTIME_CONFIG, cook.audioLoadTypes)))};` +
span('game-bundle.js', `window.__GAME_BUNDLE__=${JSON.stringify(pack(Buffer.from(bundle, 'utf8')))};`);
const adProfile = opts.adProfile ?? genericPlayableProfile;
const network = playableAdInjection(adProfile, { title, orientation });
diff --git a/pipeline/src/project/importSettings.ts b/pipeline/src/project/importSettings.ts
index 61a3b797d..efef630c4 100644
--- a/pipeline/src/project/importSettings.ts
+++ b/pipeline/src/project/importSettings.ts
@@ -162,6 +162,18 @@ const DRAGONBONES: ImporterFieldSpec[] = [
];
const AUDIO: ImporterFieldSpec[] = [
+ {
+ key: 'loadType', label: 'Load Type', type: 'enum', default: 'auto', category: 'Audio',
+ options: [
+ { label: 'Auto', value: 'auto' },
+ { label: 'Decompress On Load', value: 'decompress' },
+ { label: 'Streaming', value: 'streaming' },
+ ],
+ tooltip: 'How the clip is held on Web. Auto streams clips of 30 seconds or longer and decodes shorter '
+ + 'ones. Decompress On Load decodes to memory: instant to start, and a 3-minute track costs about 69 MB. '
+ + 'Streaming plays from the compressed file and takes almost no memory, but cannot loop on an exact '
+ + 'sample and plays one instance of the clip at a time.',
+ },
{
key: 'compress', label: 'Compress', type: 'bool', default: true, category: 'Audio',
tooltip: 'Re-encode WAV to MP3 at cook (already-compressed formats pass through). '
@@ -248,6 +260,12 @@ export const IMPORTER_SCHEMAS: Record<string, ImporterFieldSpec[]> = {
video: VIDEO,
};
+/** What the importer asked of a clip when it is not left to the length, or null for `auto`. */
+export function audioLoadTypeOf(importer: Record<string, unknown> | undefined): 'streaming' | 'decompress' | null {
+ const type = importer?.loadType;
+ return type === 'streaming' || type === 'decompress' ? type : null;
+}
+
/** Cook-facing audio import settings, tolerant of hand-edited `.meta` blocks. */
export function readAudioImportSettings(importer: Record<string, unknown> | undefined): {
compress: boolean; bitrateKbps: number;
diff --git a/pipeline/src/project/runtimeConfig.ts b/pipeline/src/project/runtimeConfig.ts
index 4c15d6f58..2cbe749ff 100644
--- a/pipeline/src/project/runtimeConfig.ts
+++ b/pipeline/src/project/runtimeConfig.ts
@@ -166,6 +166,17 @@ export function runtimeConfigOf(
};
}
+/**
+ * The runtime settings with the clips whose import settings choose how they load. Those live in the
+ * assets' own `.meta`, not in the project manifest, so whoever has the asset index adds them here.
+ */
+export function withAudioLoadTypes(
+ rc: RuntimeProjectConfig, loadTypes: Readonly<Record<string, 'streaming' | 'decompress'>> | undefined,
+): RuntimeProjectConfig {
+ if (!loadTypes || Object.keys(loadTypes).length === 0) return rc;
+ return { ...rc, audioConfig: { ...rc.audioConfig, loadTypes: { ...loadTypes } } };
+}
+
/** The settings a shipped build carries, as `game.config.json` spells them. */
export type PackagedRuntimeFields = Pick<
PackagedGameConfig,
@@ -206,6 +217,7 @@ export function packagedRuntimeFields(rc: RuntimeProjectConfig): PackagedRuntime
...(rc.physicsEnabled ? { physicsEnabled: true } : {}),
...(isDefaultPhysics(rc.physicsConfig) ? {} : { physicsConfig: rc.physicsConfig }),
...(rc.audioConfig.buses?.length || rc.audioConfig.maxVoices !== undefined
+ || Object.keys(rc.audioConfig.loadTypes ?? {}).length > 0
? { audioConfig: rc.audioConfig } : {}),
...(rc.achievements.length ? { achievements: rc.achievements } : {}),
...(rc.steamAppId > 0 ? { steamAppId: rc.steamAppId } : {}),
diff --git a/pipeline/tests/runtime-config.test.ts b/pipeline/tests/runtime-config.test.ts
index 1a1fb637c..60c00fa95 100644
--- a/pipeline/tests/runtime-config.test.ts
+++ b/pipeline/tests/runtime-config.test.ts
@@ -11,8 +11,9 @@
*/
import { describe, it, expect } from 'vitest';
import {
- DEFAULT_RUNTIME_CONFIG, packagedRuntimeFields, runtimeConfigOf,
+ DEFAULT_RUNTIME_CONFIG, packagedRuntimeFields, runtimeConfigOf, withAudioLoadTypes,
} from '../src/project/runtimeConfig';
+import { audioLoadTypeOf, importerDefaults } from '../src/project/importSettings';
describe('runtimeConfigOf', () => {
it('carries the same normalized quality policy into exported builds', () => {
@@ -161,3 +162,23 @@ describe('packagedRuntimeFields', () => {
expect(runtimeConfigOf(state).screenFit.designWidth).toBe(640);
});
});
+
+describe('how an audio clip loads', () => {
+ it('is Auto until the importer says otherwise, and a hand-edited value cannot break the cook', () => {
+ expect(importerDefaults('audio').loadType).toBe('auto');
+ expect(audioLoadTypeOf(undefined)).toBeNull();
+ expect(audioLoadTypeOf({ loadType: 'auto' })).toBeNull();
+ expect(audioLoadTypeOf({ loadType: 'streaming' })).toBe('streaming');
+ expect(audioLoadTypeOf({ loadType: 'decompress' })).toBe('decompress');
+ expect(audioLoadTypeOf({ loadType: 'sideways' })).toBeNull();
+ });
+
+ it('rides the packaged audio config only when a clip asked for something', () => {
+ expect(packagedRuntimeFields(DEFAULT_RUNTIME_CONFIG).audioConfig).toBeUndefined();
+ expect(packagedRuntimeFields(withAudioLoadTypes(DEFAULT_RUNTIME_CONFIG, {})).audioConfig).toBeUndefined();
+ const rc = withAudioLoadTypes(DEFAULT_RUNTIME_CONFIG, { 'assets/bgm.mp3': 'streaming' });
+ expect(packagedRuntimeFields(rc).audioConfig).toEqual({ loadTypes: { 'assets/bgm.mp3': 'streaming' } });
+ expect(DEFAULT_RUNTIME_CONFIG.audioConfig.loadTypes).toBeUndefined();
+ });
+});
+
diff --git a/sdk/etc/index.api.md b/sdk/etc/index.api.md
index 2765f2102..b569e23b8 100644
--- a/sdk/etc/index.api.md
+++ b/sdk/etc/index.api.md
@@ -1,7 +1,7 @@
# API surface — esengine/index
<!-- Generated by tools/api-surface.mjs — do not edit. Run --update to accept changes. -->
-Symbols: 115 public · 152 beta · 1620 experimental · 3 deprecated
+Symbols: 115 public · 152 beta · 1621 experimental · 3 deprecated
## ABI_LAYOUT_HASH — const @experimental
```
@@ -1147,6 +1147,7 @@ setBufferBudget: (bytes: number | null) => void
setBusDucking: (target: string, rule: BusDuckRule | null) => boolean
setBusEffects: (busName: string, effects: BusEffectDef[]) => boolean
setBusVolume: (busName: string, volume: number) => void
+setLoadTypes: (types: Readonly<Record<string, AudioLoadType>> | null) => void
setMasterVolume: (volume: number) => void
setMaxVoices: (max: number) => void
setMusicVolume: (volume: number) => void
@@ -1174,6 +1175,8 @@ mixerConfig: AudioMixerConfig | undefined
bytes: number | undefined
duration: number
id: number
+savedBytes: number | undefined
+streamed: boolean | undefined
```
## AudioBufferStats — interface @experimental
@@ -1182,6 +1185,8 @@ bufferBudget: number
bufferBytes: number
bufferCount: number
evictableCount: number
+streamedCount: number
+streamedSavedBytes: number
```
## AudioBus — class @experimental
@@ -1249,6 +1254,11 @@ ComponentDef<AudioListenerData>
enabled: boolean
```
+## AudioLoadType — type @experimental
+```
+'auto' | 'decompress' | 'streaming'
+```
+
## AudioMixer — class @experimental
```
busNames: () => string[]
@@ -1306,6 +1316,7 @@ static prototype: AudioPool
## AudioProjectConfig — interface @experimental
```
buses: AudioBusDecl[] | undefined
+loadTypes: Record<string, "decompress" | "streaming"> | undefined
maxVoices: number | undefined
```
@@ -2116,7 +2127,7 @@ editor: { ms: number; phases: Record<string, number>; parts?: Record<string, num
entities: number | undefined
gpuMs: number | undefined
id: number
-memory: { wasmBytes?: number; jsHeapBytes?: number; vramBytes?: number; } | undefined
+memory: { wasmBytes?: number; jsHeapBytes?: number; vramBytes?: number; audioBytes?: number; audioStreamedCount?: number; audioStreamedSavedBytes?: number; } | undefined
nativeScopes: ScopeCost[] | undefined
quality: QualityReport | undefined
scopes: ScopeCost[]
@@ -4099,6 +4110,9 @@ triangles: number
## FrameStatsReport — interface @experimental
```
+audioBytes: number | undefined
+audioStreamedCount: number | undefined
+audioStreamedSavedBytes: number | undefined
costs: FrameCosts | null
cppCounters: Record<string, number>
cppScopes: Record<string, number>
@@ -6243,8 +6257,8 @@ getFrequencyData: ((out: Uint8Array) => boolean) | undefined
initialize: (options?: AudioBackendInitOptions) => Promise<void>
invalidateSource: ((src: string) => void) | undefined
isReady: boolean
-loadBuffer: (src: string) => Promise<AudioBufferHandle>
-loadBufferFromData: (src: string, data: ArrayBuffer) => Promise<AudioBufferHandle>
+loadBuffer: (src: string, options?: AudioLoadOptions) => Promise<AudioBufferHandle>
+loadBufferFromData: (src: string, data: ArrayBuffer, options?: AudioLoadOptions) => Promise<AudioBufferHandle>
mixer: AudioMixer | null
name: string
play: (buffer: AudioBufferHandle, config: PlayConfig) => AudioHandle
@@ -6540,7 +6554,7 @@ pieces: readonly Vec2[][]
gain: GainNode
inUse: boolean
panner: StereoPannerNode
-source: AudioBufferSourceNode | null
+source: AudioBufferSourceNode | MediaElementAudioSourceNode | null
startTime: number
```
diff --git a/sdk/etc/index.native.api.md b/sdk/etc/index.native.api.md
index b71da9a3a..f8dde197b 100644
--- a/sdk/etc/index.native.api.md
+++ b/sdk/etc/index.native.api.md
@@ -1,7 +1,7 @@
# API surface — esengine/index.native
<!-- Generated by tools/api-surface.mjs — do not edit. Run --update to accept changes. -->
-Symbols: 115 public · 157 beta · 1639 experimental · 3 deprecated
+Symbols: 115 public · 157 beta · 1640 experimental · 3 deprecated
## ABI_LAYOUT_HASH — const @experimental
```
@@ -1147,6 +1147,7 @@ setBufferBudget: (bytes: number | null) => void
setBusDucking: (target: string, rule: BusDuckRule | null) => boolean
setBusEffects: (busName: string, effects: BusEffectDef[]) => boolean
setBusVolume: (busName: string, volume: number) => void
+setLoadTypes: (types: Readonly<Record<string, AudioLoadType>> | null) => void
setMasterVolume: (volume: number) => void
setMaxVoices: (max: number) => void
setMusicVolume: (volume: number) => void
@@ -1174,6 +1175,8 @@ mixerConfig: AudioMixerConfig | undefined
bytes: number | undefined
duration: number
id: number
+savedBytes: number | undefined
+streamed: boolean | undefined
```
## AudioBufferStats — interface @experimental
@@ -1182,6 +1185,8 @@ bufferBudget: number
bufferBytes: number
bufferCount: number
evictableCount: number
+streamedCount: number
+streamedSavedBytes: number
```
## AudioBus — class @experimental
@@ -1249,6 +1254,11 @@ ComponentDef<AudioListenerData>
enabled: boolean
```
+## AudioLoadType — type @experimental
+```
+'auto' | 'decompress' | 'streaming'
+```
+
## AudioMixer — class @experimental
```
busNames: () => string[]
@@ -1306,6 +1316,7 @@ static prototype: AudioPool
## AudioProjectConfig — interface @experimental
```
buses: AudioBusDecl[] | undefined
+loadTypes: Record<string, "decompress" | "streaming"> | undefined
maxVoices: number | undefined
```
@@ -2116,7 +2127,7 @@ editor: { ms: number; phases: Record<string, number>; parts?: Record<string, num
entities: number | undefined
gpuMs: number | undefined
id: number
-memory: { wasmBytes?: number; jsHeapBytes?: number; vramBytes?: number; } | undefined
+memory: { wasmBytes?: number; jsHeapBytes?: number; vramBytes?: number; audioBytes?: number; audioStreamedCount?: number; audioStreamedSavedBytes?: number; } | undefined
nativeScopes: ScopeCost[] | undefined
quality: QualityReport | undefined
scopes: ScopeCost[]
@@ -4107,6 +4118,9 @@ triangles: number
## FrameStatsReport — interface @experimental
```
+audioBytes: number | undefined
+audioStreamedCount: number | undefined
+audioStreamedSavedBytes: number | undefined
costs: FrameCosts | null
cppCounters: Record<string, number>
cppScopes: Record<string, number>
@@ -6446,8 +6460,8 @@ getFrequencyData: ((out: Uint8Array) => boolean) | undefined
initialize: (options?: AudioBackendInitOptions) => Promise<void>
invalidateSource: ((src: string) => void) | undefined
isReady: boolean
-loadBuffer: (src: string) => Promise<AudioBufferHandle>
-loadBufferFromData: (src: string, data: ArrayBuffer) => Promise<AudioBufferHandle>
+loadBuffer: (src: string, options?: AudioLoadOptions) => Promise<AudioBufferHandle>
+loadBufferFromData: (src: string, data: ArrayBuffer, options?: AudioLoadOptions) => Promise<AudioBufferHandle>
mixer: AudioMixer | null
name: string
play: (buffer: AudioBufferHandle, config: PlayConfig) => AudioHandle
@@ -6743,7 +6757,7 @@ pieces: readonly Vec2[][]
gain: GainNode
inUse: boolean
panner: StereoPannerNode
-source: AudioBufferSourceNode | null
+source: AudioBufferSourceNode | MediaElementAudioSourceNode | null
startTime: number
```
diff --git a/sdk/etc/index.node.api.md b/sdk/etc/index.node.api.md
index 4babdc884..bdf396380 100644
--- a/sdk/etc/index.node.api.md
+++ b/sdk/etc/index.node.api.md
@@ -1,7 +1,7 @@
# API surface — esengine/index.node
<!-- Generated by tools/api-surface.mjs — do not edit. Run --update to accept changes. -->
-Symbols: 115 public · 154 beta · 1619 experimental · 3 deprecated
+Symbols: 115 public · 154 beta · 1620 experimental · 3 deprecated
## ANCHOR_AXES — const @experimental
```
@@ -1142,6 +1142,7 @@ setBufferBudget: (bytes: number | null) => void
setBusDucking: (target: string, rule: BusDuckRule | null) => boolean
setBusEffects: (busName: string, effects: BusEffectDef[]) => boolean
setBusVolume: (busName: string, volume: number) => void
+setLoadTypes: (types: Readonly<Record<string, AudioLoadType>> | null) => void
setMasterVolume: (volume: number) => void
setMaxVoices: (max: number) => void
setMusicVolume: (volume: number) => void
@@ -1169,6 +1170,8 @@ mixerConfig: AudioMixerConfig | undefined
bytes: number | undefined
duration: number
id: number
+savedBytes: number | undefined
+streamed: boolean | undefined
```
## AudioBufferStats — interface @experimental
@@ -1177,6 +1180,8 @@ bufferBudget: number
bufferBytes: number
bufferCount: number
evictableCount: number
+streamedCount: number
+streamedSavedBytes: number
```
## AudioBus — class @experimental
@@ -1244,6 +1249,11 @@ ComponentDef<AudioListenerData>
enabled: boolean
```
+## AudioLoadType — type @experimental
+```
+'auto' | 'decompress' | 'streaming'
+```
+
## AudioMixer — class @experimental
```
busNames: () => string[]
@@ -1301,6 +1311,7 @@ static prototype: AudioPool
## AudioProjectConfig — interface @experimental
```
buses: AudioBusDecl[] | undefined
+loadTypes: Record<string, "decompress" | "streaming"> | undefined
maxVoices: number | undefined
```
@@ -2111,7 +2122,7 @@ editor: { ms: number; phases: Record<string, number>; parts?: Record<string, num
entities: number | undefined
gpuMs: number | undefined
id: number
-memory: { wasmBytes?: number; jsHeapBytes?: number; vramBytes?: number; } | undefined
+memory: { wasmBytes?: number; jsHeapBytes?: number; vramBytes?: number; audioBytes?: number; audioStreamedCount?: number; audioStreamedSavedBytes?: number; } | undefined
nativeScopes: ScopeCost[] | undefined
quality: QualityReport | undefined
scopes: ScopeCost[]
@@ -4094,6 +4105,9 @@ triangles: number
## FrameStatsReport — interface @experimental
```
+audioBytes: number | undefined
+audioStreamedCount: number | undefined
+audioStreamedSavedBytes: number | undefined
costs: FrameCosts | null
cppCounters: Record<string, number>
cppScopes: Record<string, number>
@@ -6238,8 +6252,8 @@ getFrequencyData: ((out: Uint8Array) => boolean) | undefined
initialize: (options?: AudioBackendInitOptions) => Promise<void>
invalidateSource: ((src: string) => void) | undefined
isReady: boolean
-loadBuffer: (src: string) => Promise<AudioBufferHandle>
-loadBufferFromData: (src: string, data: ArrayBuffer) => Promise<AudioBufferHandle>
+loadBuffer: (src: string, options?: AudioLoadOptions) => Promise<AudioBufferHandle>
+loadBufferFromData: (src: string, data: ArrayBuffer, options?: AudioLoadOptions) => Promise<AudioBufferHandle>
mixer: AudioMixer | null
name: string
play: (buffer: AudioBufferHandle, config: PlayConfig) => AudioHandle
@@ -6535,7 +6549,7 @@ pieces: readonly Vec2[][]
gain: GainNode
inUse: boolean
panner: StereoPannerNode
-source: AudioBufferSourceNode | null
+source: AudioBufferSourceNode | MediaElementAudioSourceNode | null
startTime: number
```
diff --git a/sdk/etc/minigame.api.md b/sdk/etc/minigame.api.md
index f7fd6e960..55fcee75b 100644
--- a/sdk/etc/minigame.api.md
+++ b/sdk/etc/minigame.api.md
@@ -1,7 +1,7 @@
# API surface — esengine/minigame
<!-- Generated by tools/api-surface.mjs — do not edit. Run --update to accept changes. -->
-Symbols: 115 public · 152 beta · 1641 experimental · 3 deprecated
+Symbols: 115 public · 152 beta · 1642 experimental · 3 deprecated
## ANCHOR_AXES — const @experimental
```
@@ -1142,6 +1142,7 @@ setBufferBudget: (bytes: number | null) => void
setBusDucking: (target: string, rule: BusDuckRule | null) => boolean
setBusEffects: (busName: string, effects: BusEffectDef[]) => boolean
setBusVolume: (busName: string, volume: number) => void
+setLoadTypes: (types: Readonly<Record<string, AudioLoadType>> | null) => void
setMasterVolume: (volume: number) => void
setMaxVoices: (max: number) => void
setMusicVolume: (volume: number) => void
@@ -1169,6 +1170,8 @@ mixerConfig: AudioMixerConfig | undefined
bytes: number | undefined
duration: number
id: number
+savedBytes: number | undefined
+streamed: boolean | undefined
```
## AudioBufferStats — interface @experimental
@@ -1177,6 +1180,8 @@ bufferBudget: number
bufferBytes: number
bufferCount: number
evictableCount: number
+streamedCount: number
+streamedSavedBytes: number
```
## AudioBus — class @experimental
@@ -1244,6 +1249,11 @@ ComponentDef<AudioListenerData>
enabled: boolean
```
+## AudioLoadType — type @experimental
+```
+'auto' | 'decompress' | 'streaming'
+```
+
## AudioMixer — class @experimental
```
busNames: () => string[]
@@ -1301,6 +1311,7 @@ static prototype: AudioPool
## AudioProjectConfig — interface @experimental
```
buses: AudioBusDecl[] | undefined
+loadTypes: Record<string, "decompress" | "streaming"> | undefined
maxVoices: number | undefined
```
@@ -2111,7 +2122,7 @@ editor: { ms: number; phases: Record<string, number>; parts?: Record<string, num
entities: number | undefined
gpuMs: number | undefined
id: number
-memory: { wasmBytes?: number; jsHeapBytes?: number; vramBytes?: number; } | undefined
+memory: { wasmBytes?: number; jsHeapBytes?: number; vramBytes?: number; audioBytes?: number; audioStreamedCount?: number; audioStreamedSavedBytes?: number; } | undefined
nativeScopes: ScopeCost[] | undefined
quality: QualityReport | undefined
scopes: ScopeCost[]
@@ -4094,6 +4105,9 @@ triangles: number
## FrameStatsReport — interface @experimental
```
+audioBytes: number | undefined
+audioStreamedCount: number | undefined
+audioStreamedSavedBytes: number | undefined
costs: FrameCosts | null
cppCounters: Record<string, number>
cppScopes: Record<string, number>
@@ -6531,8 +6545,8 @@ getFrequencyData: ((out: Uint8Array) => boolean) | undefined
initialize: (options?: AudioBackendInitOptions) => Promise<void>
invalidateSource: ((src: string) => void) | undefined
isReady: boolean
-loadBuffer: (src: string) => Promise<AudioBufferHandle>
-loadBufferFromData: (src: string, data: ArrayBuffer) => Promise<AudioBufferHandle>
+loadBuffer: (src: string, options?: AudioLoadOptions) => Promise<AudioBufferHandle>
+loadBufferFromData: (src: string, data: ArrayBuffer, options?: AudioLoadOptions) => Promise<AudioBufferHandle>
mixer: AudioMixer | null
name: string
play: (buffer: AudioBufferHandle, config: PlayConfig) => AudioHandle
@@ -6828,7 +6842,7 @@ pieces: readonly Vec2[][]
gain: GainNode
inUse: boolean
panner: StereoPannerNode
-source: AudioBufferSourceNode | null
+source: AudioBufferSourceNode | MediaElementAudioSourceNode | null
startTime: number
```
diff --git a/sdk/etc/wechat.api.md b/sdk/etc/wechat.api.md
index e07e96d3a..e476cab26 100644
--- a/sdk/etc/wechat.api.md
+++ b/sdk/etc/wechat.api.md
@@ -1,7 +1,7 @@
# API surface — esengine/wechat
<!-- Generated by tools/api-surface.mjs — do not edit. Run --update to accept changes. -->
-Symbols: 115 public · 152 beta · 1632 experimental · 3 deprecated
+Symbols: 115 public · 152 beta · 1633 experimental · 3 deprecated
## ANCHOR_AXES — const @experimental
```
@@ -1142,6 +1142,7 @@ setBufferBudget: (bytes: number | null) => void
setBusDucking: (target: string, rule: BusDuckRule | null) => boolean
setBusEffects: (busName: string, effects: BusEffectDef[]) => boolean
setBusVolume: (busName: string, volume: number) => void
+setLoadTypes: (types: Readonly<Record<string, AudioLoadType>> | null) => void
setMasterVolume: (volume: number) => void
setMaxVoices: (max: number) => void
setMusicVolume: (volume: number) => void
@@ -1169,6 +1170,8 @@ mixerConfig: AudioMixerConfig | undefined
bytes: number | undefined
duration: number
id: number
+savedBytes: number | undefined
+streamed: boolean | undefined
```
## AudioBufferStats — interface @experimental
@@ -1177,6 +1180,8 @@ bufferBudget: number
bufferBytes: number
bufferCount: number
evictableCount: number
+streamedCount: number
+streamedSavedBytes: number
```
## AudioBus — class @experimental
@@ -1244,6 +1249,11 @@ ComponentDef<AudioListenerData>
enabled: boolean
```
+## AudioLoadType — type @experimental
+```
+'auto' | 'decompress' | 'streaming'
+```
+
## AudioMixer — class @experimental
```
busNames: () => string[]
@@ -1301,6 +1311,7 @@ static prototype: AudioPool
## AudioProjectConfig — interface @experimental
```
buses: AudioBusDecl[] | undefined
+loadTypes: Record<string, "decompress" | "streaming"> | undefined
maxVoices: number | undefined
```
@@ -2111,7 +2122,7 @@ editor: { ms: number; phases: Record<string, number>; parts?: Record<string, num
entities: number | undefined
gpuMs: number | undefined
id: number
-memory: { wasmBytes?: number; jsHeapBytes?: number; vramBytes?: number; } | undefined
+memory: { wasmBytes?: number; jsHeapBytes?: number; vramBytes?: number; audioBytes?: number; audioStreamedCount?: number; audioStreamedSavedBytes?: number; } | undefined
nativeScopes: ScopeCost[] | undefined
quality: QualityReport | undefined
scopes: ScopeCost[]
@@ -4094,6 +4105,9 @@ triangles: number
## FrameStatsReport — interface @experimental
```
+audioBytes: number | undefined
+audioStreamedCount: number | undefined
+audioStreamedSavedBytes: number | undefined
costs: FrameCosts | null
cppCounters: Record<string, number>
cppScopes: Record<string, number>
@@ -6245,8 +6259,8 @@ getFrequencyData: ((out: Uint8Array) => boolean) | undefined
initialize: (options?: AudioBackendInitOptions) => Promise<void>
invalidateSource: ((src: string) => void) | undefined
isReady: boolean
-loadBuffer: (src: string) => Promise<AudioBufferHandle>
-loadBufferFromData: (src: string, data: ArrayBuffer) => Promise<AudioBufferHandle>
+loadBuffer: (src: string, options?: AudioLoadOptions) => Promise<AudioBufferHandle>
+loadBufferFromData: (src: string, data: ArrayBuffer, options?: AudioLoadOptions) => Promise<AudioBufferHandle>
mixer: AudioMixer | null
name: string
play: (buffer: AudioBufferHandle, config: PlayConfig) => AudioHandle
@@ -6542,7 +6556,7 @@ pieces: readonly Vec2[][]
gain: GainNode
inUse: boolean
panner: StereoPannerNode
-source: AudioBufferSourceNode | null
+source: AudioBufferSourceNode | MediaElementAudioSourceNode | null
startTime: number
```
diff --git a/sdk/src/app/profileCapture.ts b/sdk/src/app/profileCapture.ts
index 214d3b8ae..0944138f9 100644
--- a/sdk/src/app/profileCapture.ts
+++ b/sdk/src/app/profileCapture.ts
@@ -45,7 +45,11 @@ export interface CapturedFrame {
drawCalls?: number;
triangles?: number;
entities?: number;
- memory?: { wasmBytes?: number; jsHeapBytes?: number; vramBytes?: number };
+ memory?: {
+ wasmBytes?: number; jsHeapBytes?: number; vramBytes?: number;
+ /** Decoded audio resident, the clips streamed instead, and the PCM that saved (estimate). */
+ audioBytes?: number; audioStreamedCount?: number; audioStreamedSavedBytes?: number;
+ };
/** Editor-only. Absent from anything a shipped game records. */
editor?: {
ms: number;
diff --git a/sdk/src/app/profileRecorder.ts b/sdk/src/app/profileRecorder.ts
index 2e61b1527..299f4f88d 100644
--- a/sdk/src/app/profileRecorder.ts
+++ b/sdk/src/app/profileRecorder.ts
@@ -6,6 +6,7 @@
import type { App, Plugin } from './app';
import { Renderer } from '../render/renderer';
import { engineApi } from '../ecs/bridge/engineApi';
+import { Audio } from '../audio/Audio';
import {
PROFILE_CAPTURE_VERSION,
type CaptureSource,
@@ -28,6 +29,13 @@ export interface ProfileRecorderOptions {
const DEFAULT_MAX_FRAMES = 1800;
/** Chrome only; 0 where the runtime does not expose its heap. */
+/** What the app's audio holds, when it has any: decoded bytes, and what streaming kept out of memory. */
+function audioMemory(app: App): { audioBytes?: number; audioStreamedCount?: number; audioStreamedSavedBytes?: number } {
+ if (!app.hasResource(Audio)) return {};
+ const stats = app.getResource(Audio).getBufferStats();
+ return { audioBytes: stats.bufferBytes, audioStreamedCount: stats.streamedCount, audioStreamedSavedBytes: stats.streamedSavedBytes };
+}
+
function jsHeapBytes(): number {
if (typeof performance === 'undefined') return 0;
return (performance as Performance & { memory?: { usedJSHeapSize: number } }).memory?.usedJSHeapSize ?? 0;
@@ -140,6 +148,7 @@ export class ProfileRecorder {
wasmBytes: this.app_.wasmModule?.HEAPU8?.byteLength ?? 0,
jsHeapBytes: jsHeapBytes(),
vramBytes: m?.renderer_getEstimatedGpuBytes?.() ?? 0,
+ ...audioMemory(this.app_),
},
};
this.frames_.push(frame);
diff --git a/sdk/src/audio/Audio.ts b/sdk/src/audio/Audio.ts
index 209bbfb2c..1aaff8f13 100644
--- a/sdk/src/audio/Audio.ts
+++ b/sdk/src/audio/Audio.ts
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
-import type { PlatformAudioBackend, AudioBufferHandle, AudioHandle, PlayConfig, AudioDelivery } from './PlatformAudioBackend';
+import type { PlatformAudioBackend, AudioBufferHandle, AudioHandle, PlayConfig, AudioDelivery, AudioLoadOptions, AudioLoadType } from './PlatformAudioBackend';
import type { AudioMixer, BusDuckRule } from './AudioMixer';
import type { BusEffectDef } from './BusEffects';
import { defineResource } from '../ecs/resource';
@@ -8,6 +8,11 @@ import { RuntimeConfig } from '../defaults';
import { log } from '../util/logger';
import { disposeAll } from '../util/disposeAll';
+/** `./a//b.mp3` and `a/b.mp3` are one clip. */
+function normalizeClipPath(path: string): string {
+ return path.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/{2,}/g, '/');
+}
+
const hostSuspensions = new WeakMap<AudioAPI, { owners: Set<object>; sync(): void }>();
/** Internal host suspension, preserving explicit Audio.suspend/resume intent. */
@@ -55,6 +60,10 @@ export interface AudioBufferStats {
bufferBudget: number;
/** Warm-cache entries (refCount 0) awaiting revive or eviction. */
evictableCount: number;
+ /** Clips played from their compressed file instead of being decoded (unique, not per alias). */
+ streamedCount: number;
+ /** The PCM those clips did not take, estimated from their length: what `bufferBytes` would be if they were decoded. */
+ streamedSavedBytes: number;
}
/**
@@ -124,6 +133,8 @@ export class AudioAPI {
private readonly evictOrder_ = new Set<string>();
private residentBytes_ = 0;
private bufferBudgetOverride_: number | null = null;
+ /** Importer choices by authored path; a clip with none is `auto`. */
+ private loadTypes_ = new Map<string, AudioLoadType>();
private bgmHandle_: AudioHandle | null = null;
private bgmRequest_ = 0;
private playbackEpoch_ = 0;
@@ -260,9 +271,33 @@ export class AudioAPI {
return freed;
}
+ /**
+ * How each clip the importer was told about is held. Keyed by the clip's project-relative path, the way
+ * the asset loader and `play` address it; anything absent is `auto`. A clip already loaded keeps how it
+ * was loaded until it is reloaded.
+ */
+ setLoadTypes(types: Readonly<Record<string, AudioLoadType>> | null): void {
+ this.loadTypes_ = new Map(Object.entries(types ?? {}).map(([path, type]) => [normalizeClipPath(path), type]));
+ }
+
+ private loadOptions_(url: string): AudioLoadOptions | undefined {
+ const loadType = this.loadTypes_.get(normalizeClipPath(url));
+ return loadType ? { loadType } : undefined;
+ }
+
/** Buffer residency counters — the observability side of the budget. */
getBufferStats(): AudioBufferStats {
+ const seen = new Set<SharedAudioBuffer>();
+ let streamedCount = 0;
+ let streamedSavedBytes = 0;
+ for (const entry of this.bufferCache_.values()) {
+ if (!entry.handle.streamed || seen.has(entry.source)) continue;
+ seen.add(entry.source);
+ streamedCount++;
+ streamedSavedBytes += entry.handle.savedBytes ?? 0;
+ }
return {
+ streamedCount, streamedSavedBytes,
bufferCount: this.bufferCache_.size,
bufferBytes: this.residentBytes_,
bufferBudget: this.bufferBudget,
@@ -412,7 +447,8 @@ export class AudioAPI {
return this.preloadFromData(url, data);
}
}
- await this.ensureBuffer_(url, source => this.backend_.loadBuffer(source));
+ const options = this.loadOptions_(url);
+ await this.ensureBuffer_(url, source => (options ? this.backend_.loadBuffer(source, options) : this.backend_.loadBuffer(source)));
}
async preloadAll(urls: string[]): Promise<void> {
@@ -426,7 +462,9 @@ export class AudioAPI {
* for a `url`-delivery backend is the only thing it plays.
*/
async preloadFromData(url: string, data: ArrayBuffer): Promise<void> {
- await this.ensureBuffer_(url, source => this.backend_.loadBufferFromData(source, data));
+ const options = this.loadOptions_(url);
+ await this.ensureBuffer_(url, source => (options
+ ? this.backend_.loadBufferFromData(source, data, options) : this.backend_.loadBufferFromData(source, data)));
}
/**
diff --git a/sdk/src/audio/AudioPool.ts b/sdk/src/audio/AudioPool.ts
index 59786fa39..c670253af 100644
--- a/sdk/src/audio/AudioPool.ts
+++ b/sdk/src/audio/AudioPool.ts
@@ -3,7 +3,8 @@
export interface PooledAudioNode {
gain: GainNode;
panner: StereoPannerNode;
- source: AudioBufferSourceNode | null;
+ /** A decoded clip's buffer source, or a streamed clip's media element source. */
+ source: AudioBufferSourceNode | MediaElementAudioSourceNode | null;
inUse: boolean;
startTime: number;
}
@@ -54,7 +55,9 @@ export class AudioPool {
release(node: PooledAudioNode): void {
if (!node.inUse) return;
if (node.source) {
- try { node.source.stop(); } catch (_) { /* already stopped */ }
+ if ('stop' in node.source) {
+ try { node.source.stop(); } catch (_) { /* already stopped */ }
+ }
node.source.disconnect();
node.source = null;
}
diff --git a/sdk/src/audio/AudioProjectConfig.ts b/sdk/src/audio/AudioProjectConfig.ts
index 20260465e..c800587a7 100644
--- a/sdk/src/audio/AudioProjectConfig.ts
+++ b/sdk/src/audio/AudioProjectConfig.ts
@@ -12,6 +12,7 @@
import type { AudioAPI } from './Audio';
import type { BusDuckRule } from './AudioMixer';
import { parseBusEffects, type BusEffectDef } from './BusEffects';
+import type { AudioLoadType } from './PlatformAudioBackend';
export interface AudioBusDecl {
name: string;
@@ -28,6 +29,12 @@ export interface AudioProjectConfig {
/** How many voices may sound at once before `priority` decides which is
* dropped. 0 or less means no cap. */
maxVoices?: number;
+ /**
+ * Clips whose import settings ask for something other than `auto`, by project-relative path:
+ * `streaming` plays one from its compressed file, `decompress` decodes it at load. Written by the
+ * editor and the exporter from the assets' import settings, never by hand in the project file.
+ */
+ loadTypes?: Record<string, Exclude<AudioLoadType, 'auto'>>;
}
const clamp01 = (v: number) => Math.max(0, Math.min(1, v));
@@ -62,6 +69,14 @@ export function parseAudioProjectConfig(raw: unknown): AudioProjectConfig {
if (typeof maxVoices === 'number' && Number.isFinite(maxVoices)) {
config.maxVoices = Math.floor(maxVoices);
}
+ const rawTypes = (raw as { loadTypes?: unknown } | undefined)?.loadTypes;
+ if (rawTypes && typeof rawTypes === 'object') {
+ const loadTypes: Record<string, 'streaming' | 'decompress'> = {};
+ for (const [path, type] of Object.entries(rawTypes)) {
+ if (path && (type === 'streaming' || type === 'decompress')) loadTypes[path] = type;
+ }
+ if (Object.keys(loadTypes).length > 0) config.loadTypes = loadTypes;
+ }
return config;
}
@@ -74,6 +89,7 @@ export function applyAudioProjectConfig(audio: AudioAPI, config: AudioProjectCon
// Before the buses: the cap is the API's own and applies on every backend,
// while the loop below returns early where there is no mixer graph.
if (config?.maxVoices !== undefined) audio.setMaxVoices(config.maxVoices);
+ if (config?.loadTypes !== undefined) audio.setLoadTypes(config.loadTypes);
for (const bus of config?.buses ?? []) {
if (!audio.ensureBus(bus.name, bus.parent)) return; // no mixer: nothing applies
if (bus.volume !== undefined) audio.setBusVolume(bus.name, bus.volume);
diff --git a/sdk/src/audio/PlatformAudioBackend.ts b/sdk/src/audio/PlatformAudioBackend.ts
index 7dfd451ca..d6ab69eb8 100644
--- a/sdk/src/audio/PlatformAudioBackend.ts
+++ b/sdk/src/audio/PlatformAudioBackend.ts
@@ -18,9 +18,24 @@ export interface AudioHandle {
onEnd?: () => void;
}
+/**
+ * How a clip is held: `decompress` decodes it to PCM at load, `streaming` plays it from its compressed
+ * file, `auto` lets its length decide (see audioStreaming.ts). A backend with one way of playing a clip
+ * ignores it.
+ */
+export type AudioLoadType = 'auto' | 'decompress' | 'streaming';
+
+export interface AudioLoadOptions {
+ loadType?: AudioLoadType;
+}
+
export interface AudioBufferHandle {
readonly id: number;
readonly duration: number;
+ /** Played from the compressed file, not decoded: `bytes` is then 0 and `savedBytes` is what decoding would have cost. */
+ readonly streamed?: boolean;
+ /** The PCM a streamed clip did not take, estimated from its length. */
+ readonly savedBytes?: number;
/**
* Decoded size in bytes, for the audio residency budget. Backends that
* hold real decoded PCM (WebAudio) report it; streaming backends (WeChat
@@ -62,9 +77,9 @@ export interface PlatformAudioBackend {
initialize(options?: AudioBackendInitOptions): Promise<void>;
ensureResumed(): Promise<void>;
/** @param src a RESOLVED source the host can fetch — never a cache key. */
- loadBuffer(src: string): Promise<AudioBufferHandle>;
+ loadBuffer(src: string, options?: AudioLoadOptions): Promise<AudioBufferHandle>;
/** @param src as {@link loadBuffer}; `bytes` backends use `data` and ignore it. */
- loadBufferFromData(src: string, data: ArrayBuffer): Promise<AudioBufferHandle>;
+ loadBufferFromData(src: string, data: ArrayBuffer, options?: AudioLoadOptions): Promise<AudioBufferHandle>;
unloadBuffer(handle: AudioBufferHandle): void;
/** Forget source-address reuse after an authored clip changes. */
invalidateSource?(src: string): void;
diff --git a/sdk/src/audio/WebAudioBackend.ts b/sdk/src/audio/WebAudioBackend.ts
index 0ea4ffcf1..19aad3454 100644
--- a/sdk/src/audio/WebAudioBackend.ts
+++ b/sdk/src/audio/WebAudioBackend.ts
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
-import type { AudioHandle, AudioBufferHandle, PlayConfig, PlatformAudioBackend, AudioBackendInitOptions } from './PlatformAudioBackend';
+import type { AudioHandle, AudioBufferHandle, PlayConfig, PlatformAudioBackend, AudioBackendInitOptions, AudioLoadOptions } from './PlatformAudioBackend';
+import { decodedPcmBytes, estimateAudioSeconds, shouldStream } from './audioStreaming';
import { AudioMixer } from './AudioMixer';
import { AudioPool, type PooledAudioNode } from './AudioPool';
import { getPlatform } from '../platform/base';
@@ -160,14 +161,154 @@ class WebAudioHandle implements AudioHandle {
}
}
+/**
+ * A clip played from its compressed file through a media element, routed into the same pooled gain / pan
+ * and bus as a decoded one, so volume, buses, effects and fades treat it alike.
+ * Gives up a sample-accurate loop point and a second simultaneous voice: playing the clip again ends the first.
+ */
+class WebStreamHandle implements AudioHandle {
+ readonly id: number;
+ onEnd?: () => void;
+
+ private element_: HTMLAudioElement | null;
+ private node_: MediaElementAudioSourceNode | null;
+ private poolNode_: PooledAudioNode | null;
+ private pool_: AudioPool | null;
+ private playing_ = false;
+ private stopped_ = false;
+ private readonly duration_: number;
+ private readonly onRelease_: (handle: WebStreamHandle) => void;
+ private readonly ensureResumed_: () => Promise<void>;
+
+ constructor(
+ id: number, element: HTMLAudioElement, node: MediaElementAudioSourceNode, poolNode: PooledAudioNode,
+ pool: AudioPool, duration: number, onRelease: (handle: WebStreamHandle) => void, ensureResumed: () => Promise<void>,
+ ) {
+ this.id = id;
+ this.element_ = element;
+ this.node_ = node;
+ this.poolNode_ = poolNode;
+ this.pool_ = pool;
+ this.duration_ = duration;
+ this.onRelease_ = onRelease;
+ this.ensureResumed_ = ensureResumed;
+ poolNode.source = node;
+ element.onended = () => {
+ if (this.stopped_ || element.loop) return;
+ this.release_();
+ this.onEnd?.();
+ };
+ }
+
+ /** Start (or restart) the element. A browser that blocks it until a gesture is retried once the context resumes. */
+ begin(startOffset: number): void {
+ const element = this.element_;
+ if (!element) return;
+ if (startOffset > 0) element.currentTime = startOffset;
+ this.playing_ = true;
+ const started = element.play();
+ if (started) {
+ started.catch(async () => {
+ if (this.stopped_ || !this.playing_) return;
+ await this.ensureResumed_().catch(() => undefined);
+ if (!this.stopped_ && this.playing_) await element.play().catch(() => { this.playing_ = false; });
+ });
+ }
+ }
+
+ stop(): void {
+ if (this.stopped_) return;
+ this.release_();
+ }
+
+ private release_(): void {
+ this.stopped_ = true;
+ this.playing_ = false;
+ const element = this.element_;
+ if (element) {
+ element.onended = null;
+ element.pause();
+ element.removeAttribute('src');
+ element.load();
+ }
+ this.node_?.disconnect();
+ if (this.poolNode_ && this.pool_) this.pool_.release(this.poolNode_);
+ this.element_ = null;
+ this.node_ = null;
+ this.poolNode_ = null;
+ this.pool_ = null;
+ this.onRelease_(this);
+ }
+
+ pause(): void {
+ if (!this.playing_ || this.stopped_) return;
+ this.playing_ = false;
+ this.element_?.pause();
+ }
+
+ resume(): void {
+ if (this.playing_ || this.stopped_) return;
+ this.begin(0);
+ }
+
+ /** The host suspended the device: hold the element where it is. It still counts as playing, as a decoded voice does. */
+ holdForSuspend(): boolean {
+ if (!this.playing_ || this.stopped_) return false;
+ this.element_?.pause();
+ return true;
+ }
+
+ /** The device is running again: carry on from where the hold left the element. */
+ continueAfterSuspend(): void {
+ if (this.playing_ && !this.stopped_) this.begin(0);
+ }
+
+ setVolume(volume: number): void {
+ if (this.stopped_) return;
+ this.poolNode_!.gain.gain.value = volume;
+ }
+
+ setPan(pan: number): void {
+ if (this.stopped_) return;
+ this.poolNode_!.panner.pan.value = pan;
+ }
+
+ setLoop(loop: boolean): void {
+ if (this.stopped_ || !this.element_) return;
+ this.element_.loop = loop;
+ }
+
+ setPlaybackRate(rate: number): void {
+ if (this.stopped_ || !this.element_) return;
+ this.element_.playbackRate = rate;
+ }
+
+ get isPlaying(): boolean {
+ return this.playing_;
+ }
+
+ get currentTime(): number {
+ return this.element_ ? this.element_.currentTime : 0;
+ }
+
+ get duration(): number {
+ return this.duration_;
+ }
+}
+
+/** How long to wait for an element's metadata before trusting the header estimate. */
+const METADATA_WAIT_MS = 3000;
+
/** Decoded PCM size of a WebAudio buffer: frames × channels × 4 (Float32). */
function decodedBytes(buf: AudioBuffer): number {
return buf.length * buf.numberOfChannels * 4;
}
interface BufferRecord {
- buffer: AudioBuffer;
+ buffer: AudioBuffer | null;
url: string;
+ /** A streamed clip: the object URL its media element plays, and its length. */
+ stream?: { objectUrl: string; duration: number };
}
interface SourceRecord {
@@ -175,6 +316,36 @@ interface SourceRecord {
bufferId?: number;
}
+const MIME: Record<string, string> = {
+ mp3: 'audio/mpeg', wav: 'audio/wav', ogg: 'audio/ogg', m4a: 'audio/mp4', aac: 'audio/aac', flac: 'audio/flac', webm: 'audio/webm',
+};
+
+function mimeOf(url: string): string {
+ const ext = /\.([a-z0-9]+)(?:[?#]|$)/i.exec(url)?.[1]?.toLowerCase();
+ return (ext && MIME[ext]) || '';
+}
+
+/** The length a media element reports for `objectUrl`, or `estimate` (or 0) when it will not say in time. */
+function probeDuration(objectUrl: string, estimate: number | null): Promise<number> {
+ return new Promise((resolve) => {
+ const element = new Audio();
+ let done = false;
+ const finish = (seconds: number | null): void => {
+ if (done) return;
+ done = true;
+ clearTimeout(timer);
+ element.removeAttribute('src');
+ element.load();
+ resolve(seconds !== null && Number.isFinite(seconds) && seconds > 0 ? seconds : (estimate ?? 0));
+ };
+ const timer = setTimeout(() => finish(null), METADATA_WAIT_MS);
+ element.preload = 'metadata';
+ element.onloadedmetadata = () => finish(element.duration);
+ element.onerror = () => finish(null);
+ element.src = objectUrl;
+ });
+}
+
export class WebAudioBackend implements PlatformAudioBackend {
readonly name = 'WebAudio';
readonly delivery = 'bytes' as const;
@@ -193,6 +364,12 @@ export class WebAudioBackend implements PlatformAudioBackend {
private deviceIntentRevision_ = 0;
private contextGeneration_ = 0;
private analyser_: AnalyserNode | null = null;
+ /** The streamed voice sounding for each streamed clip (one at a time, see WebStreamHandle). */
+ private readonly streams_ = new Map<number, WebStreamHandle>();
+ /** Streams a suspended device paused, to start again when it resumes. */
+ private heldStreams_: WebStreamHandle[] = [];
+ /** Streamed clips unloaded while still sounding: their object URL is revoked when the voice ends. */
+ private readonly retiredStreams_ = new Map<WebStreamHandle, string>();
get mixer(): AudioMixer | null {
return this.mixer_;
@@ -273,7 +450,7 @@ export class WebAudioBackend implements PlatformAudioBackend {
});
}
- async loadBuffer(url: string): Promise<AudioBufferHandle> {
+ async loadBuffer(url: string, options?: AudioLoadOptions): Promise<AudioBufferHandle> {
if (!this.context_) {
throw new Error('AudioContext not initialized');
}
@@ -284,7 +461,7 @@ export class WebAudioBackend implements PlatformAudioBackend {
const inFlight = this.loadingUrls_.get(url);
if (inFlight) return inFlight;
- const promise = this.loadSource_(url, () => getPlatform().readFile(url));
+ const promise = this.loadSource_(url, () => getPlatform().readFile(url), options);
this.loadingUrls_.set(url, promise);
try {
return await promise;
@@ -293,18 +470,26 @@ export class WebAudioBackend implements PlatformAudioBackend {
}
}
- async loadBufferFromData(url: string, data: ArrayBuffer): Promise<AudioBufferHandle> {
- return this.loadSource_(url, async () => data);
+ async loadBufferFromData(url: string, data: ArrayBuffer, options?: AudioLoadOptions): Promise<AudioBufferHandle> {
+ return this.loadSource_(url, async () => data, options);
}
private cachedBuffer_(url: string): AudioBufferHandle | null {
const id = this.sources_.get(url)?.bufferId;
if (id === undefined) return null;
const record = this.buffers_.get(id);
- return record ? { id, duration: record.buffer.duration, bytes: decodedBytes(record.buffer) } : null;
+ return record ? this.handleOf_(id, record) : null;
}
- private async loadSource_(url: string, read: () => Promise<ArrayBuffer>): Promise<AudioBufferHandle> {
+ private handleOf_(id: number, record: BufferRecord): AudioBufferHandle {
+ if (record.stream) {
+ const duration = record.stream.duration;
+ return { id, duration, bytes: 0, streamed: true, savedBytes: decodedPcmBytes(duration) };
+ }
+ return { id, duration: record.buffer!.duration, bytes: decodedBytes(record.buffer!) };
+ }
+
+ private async loadSource_(url: string, read: () => Promise<ArrayBuffer>, options?: AudioLoadOptions): Promise<AudioBufferHandle> {
const context = this.context_;
if (!context) {
throw new Error('AudioContext not initialized');
@@ -321,7 +506,7 @@ export class WebAudioBackend implements PlatformAudioBackend {
try {
const data = await read();
this.assertLoadContext_(context, generation);
- return await this.decodeBuffer_(url, data, source, context, generation);
+ return await this.decodeBuffer_(url, data, source, context, generation, options);
} finally {
source.pending--;
if (this.sources_.get(url) === source && source.pending === 0 && source.bufferId === undefined) {
@@ -338,12 +523,15 @@ export class WebAudioBackend implements PlatformAudioBackend {
private async decodeBuffer_(
url: string, data: ArrayBuffer, source: SourceRecord,
- context: AudioContext, generation: number,
+ context: AudioContext, generation: number, options?: AudioLoadOptions,
): Promise<AudioBufferHandle> {
if (this.sources_.get(url) === source) {
const cached = this.cachedBuffer_(url);
if (cached) return cached;
}
+ if (shouldStream(options?.loadType, data)) {
+ return this.streamBuffer_(url, data, source, context, generation);
+ }
let audioBuffer: AudioBuffer;
try {
@@ -361,6 +549,69 @@ export class WebAudioBackend implements PlatformAudioBackend {
return { id, duration: audioBuffer.duration, bytes: decodedBytes(audioBuffer) };
}
+ /**
+ * Keep a clip as its compressed bytes behind an object URL, and learn its length from the element
+ * (the header estimate when the browser will not say). Nothing is decoded: this is the memory the
+ * clip does not take.
+ */
+ private async streamBuffer_(
+ url: string, data: ArrayBuffer, source: SourceRecord, context: AudioContext, generation: number,
+ ): Promise<AudioBufferHandle> {
+ const objectUrl = URL.createObjectURL(new Blob([data], { type: mimeOf(url) }));
+ const duration = await probeDuration(objectUrl, estimateAudioSeconds(data));
+ if (this.context_ !== context || this.contextGeneration_ !== generation) {
+ URL.revokeObjectURL(objectUrl);
+ throw new Error('AudioContext changed while loading audio');
+ }
+ const id = ++this.nextBufferId_;
+ this.buffers_.set(id, { buffer: null, url, stream: { objectUrl, duration } });
+ if (this.sources_.get(url) === source) source.bufferId = id;
+ return this.handleOf_(id, this.buffers_.get(id)!);
+ }
+
+ private releaseStream_(handle: WebStreamHandle): void {
+ for (const [id, voice] of this.streams_) if (voice === handle) this.streams_.delete(id);
+ this.heldStreams_ = this.heldStreams_.filter((h) => h !== handle);
+ const retired = this.retiredStreams_.get(handle);
+ if (retired !== undefined) {
+ this.retiredStreams_.delete(handle);
+ URL.revokeObjectURL(retired);
+ }
+ }
+
+ private playStream_(bufferId: number, stream: { objectUrl: string; duration: number }, config: PlayConfig): AudioHandle {
+ const context = this.context_!;
+ const pool = this.pool_!;
+ const mixer = this.mixer_!;
+ // One voice per streamed clip: a second play ends the first.
+ this.streams_.get(bufferId)?.stop();
+ const element = new Audio();
+ element.preload = 'auto';
+ element.src = stream.objectUrl;
+ element.loop = config.loop ?? false;
+ element.playbackRate = config.playbackRate ?? 1;
+ const poolNode = pool.acquire();
+ let node: MediaElementAudioSourceNode;
+ try {
+ node = context.createMediaElementSource(element);
+ node.connect(poolNode.gain);
+ poolNode.gain.gain.value = config.volume ?? 1.0;
+ poolNode.panner.pan.value = config.pan ?? 0;
+ const bus = mixer.getBus(config.bus ?? 'sfx') ?? mixer.sfx;
+ poolNode.panner.connect(bus.input);
+ } catch (error) {
+ pool.release(poolNode);
+ throw error;
+ }
+ const handle = new WebStreamHandle(
+ ++this.nextHandleId_, element, node, poolNode, pool, stream.duration,
+ (h) => this.releaseStream_(h), () => this.ensureResumed(),
+ );
+ this.streams_.set(bufferId, handle);
+ handle.begin(config.startOffset ?? 0);
+ return handle;
+ }
+
invalidateSource(url: string): void {
this.sources_.delete(url);
this.loadingUrls_.delete(url);
@@ -370,6 +621,11 @@ export class WebAudioBackend implements PlatformAudioBackend {
const record = this.buffers_.get(handle.id);
if (!record) return;
this.buffers_.delete(handle.id);
+ if (record.stream) {
+ const sounding = this.streams_.get(handle.id);
+ if (sounding) this.retiredStreams_.set(sounding, record.stream.objectUrl);
+ else URL.revokeObjectURL(record.stream.objectUrl);
+ }
const source = this.sources_.get(record.url);
if (source?.bufferId === handle.id) {
delete source.bufferId;
@@ -382,7 +638,9 @@ export class WebAudioBackend implements PlatformAudioBackend {
throw new Error('Audio system not initialized');
}
- const audioBuffer = this.buffers_.get(buffer.id)?.buffer;
+ const record = this.buffers_.get(buffer.id);
+ if (record?.stream) return this.playStream_(buffer.id, record.stream, config);
+ const audioBuffer = record?.buffer;
if (!audioBuffer) {
throw new Error(`Buffer ${buffer.id} not found`);
}
@@ -427,11 +685,16 @@ export class WebAudioBackend implements PlatformAudioBackend {
}
suspend(): void {
+ // A media element runs on its own clock, so a suspended context would leave it playing out of earshot and out of step.
+ for (const voice of this.streams_.values()) if (voice.holdForSuspend()) this.heldStreams_.push(voice);
this.fireDeviceIntent_(true);
}
resume(): void {
this.fireDeviceIntent_(false);
+ const held = this.heldStreams_;
+ this.heldStreams_ = [];
+ for (const voice of held) voice.continueAfterSuspend();
}
dispose(): void {
@@ -444,6 +707,12 @@ export class WebAudioBackend implements PlatformAudioBackend {
document.removeEventListener('keydown', this.resumeHandler_);
this.resumeHandler_ = null;
}
+ for (const voice of [...this.streams_.values()]) voice.stop();
+ for (const record of this.buffers_.values()) if (record.stream) URL.revokeObjectURL(record.stream.objectUrl);
+ for (const objectUrl of this.retiredStreams_.values()) URL.revokeObjectURL(objectUrl);
+ this.retiredStreams_.clear();
+ this.streams_.clear();
+ this.heldStreams_ = [];
this.analyser_ = null;
this.pool_ = null;
this.mixer_ = null;
diff --git a/sdk/src/audio/audioStreaming.ts b/sdk/src/audio/audioStreaming.ts
new file mode 100644
index 000000000..370e495e5
--- /dev/null
+++ b/sdk/src/audio/audioStreaming.ts
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file audioStreaming.ts
+ * @brief When a clip is played from its compressed file instead of being decoded into memory.
+ *
+ * A long clip decoded to PCM costs minutes × 48000 × 2 channels × 4 bytes (a three-minute track
+ * is ~69MB), which is what a phone browser or a playable ad cannot spare. A short effect is the
+ * opposite: decoded it plays with no start-up delay, streamed it waits on the element.
+ *
+ * The importer says `auto`, `decompress` or `streaming`; `auto` means the length decides.
+ * Pure, so a backend and a test agree on the rule.
+ */
+import type { AudioLoadType } from './PlatformAudioBackend';
+
+/** A clip at least this long streams when the importer says `auto`. */
+export const AUTO_STREAM_SECONDS = 30;
+
+/** A compressed file of unknown length this big streams under `auto` — what 30 seconds of a
+ * typical 128 kbps file weighs, rounded up to leave the short effects alone. */
+const UNKNOWN_LENGTH_STREAM_BYTES = 600 * 1024;
+
+/** The PCM a decoded clip costs: what `bytes` would report if it were decoded, and what streaming saves. */
+export function decodedPcmBytes(seconds: number, sampleRate = 48000, channels = 2): number {
+ return Math.round(Math.max(0, seconds) * sampleRate * channels * 4);
+}
+
+const MP3_BITRATES: Record<number, readonly number[]> = {
+ // MPEG-1 Layer III, then MPEG-2/2.5 Layer III, in kbps.
+ 1: [0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0],
+ 2: [0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0],
+};
+
+function id3Size(view: DataView): number {
+ if (view.byteLength < 10 || view.getUint8(0) !== 0x49 || view.getUint8(1) !== 0x44 || view.getUint8(2) !== 0x33) return 0;
+ return 10 + ((view.getUint8(6) & 0x7f) << 21 | (view.getUint8(7) & 0x7f) << 14 | (view.getUint8(8) & 0x7f) << 7 | (view.getUint8(9) & 0x7f));
+}
+
+/** Length of a PCM WAV from its header, or null when it is not a RIFF/WAVE we can read. */
+function wavSeconds(view: DataView): number | null {
+ if (view.byteLength < 44 || view.getUint32(0, false) !== 0x52494646 || view.getUint32(8, false) !== 0x57415645) return null;
+ let byteRate = 0;
+ let off = 12;
+ while (off + 8 <= view.byteLength) {
+ const id = view.getUint32(off, false);
+ const size = view.getUint32(off + 4, true);
+ if (id === 0x666d7420 && off + 20 <= view.byteLength) byteRate = view.getUint32(off + 16, true);
+ if (id === 0x64617461) return byteRate > 0 ? Math.min(size, view.byteLength - (off + 8)) / byteRate : null;
+ off += 8 + size + (size & 1);
+ }
+ return null;
+}
+
+/** Length of a constant-bitrate MP3 from its first frame header and the file size; null if it is not one. */
+function mp3Seconds(view: DataView): number | null {
+ const start = id3Size(view);
+ for (let i = start; i + 4 <= Math.min(view.byteLength, start + 4096); i++) {
+ if (view.getUint8(i) !== 0xff || (view.getUint8(i + 1) & 0xe0) !== 0xe0) continue;
+ const b1 = view.getUint8(i + 1);
+ const b2 = view.getUint8(i + 2);
+ const version = (b1 >> 3) & 3; // 3 = MPEG-1, 2 = MPEG-2, 0 = MPEG-2.5
+ const layer = (b1 >> 1) & 3; // 1 = Layer III
+ const bitrateIndex = b2 >> 4;
+ if (version === 1 || layer !== 1 || bitrateIndex === 0 || bitrateIndex === 15) continue;
+ const kbps = MP3_BITRATES[version === 3 ? 1 : 2]![bitrateIndex]!;
+ return kbps > 0 ? ((view.byteLength - i) * 8) / (kbps * 1000) : null;
+ }
+ return null;
+}
+
+/**
+ * How long a compressed clip plays, read from its header without decoding it.
+ * WAV is exact; an MP3 is exact at a constant bitrate and an estimate at a variable one; any
+ * other container is unknown (null) and the caller falls back to its size.
+ */
+export function estimateAudioSeconds(data: ArrayBuffer): number | null {
+ const view = new DataView(data);
+ return wavSeconds(view) ?? mp3Seconds(view);
+}
+
+/** Whether a clip that arrived as `data` is streamed, given what the importer said. */
+export function shouldStream(loadType: AudioLoadType | undefined, data: ArrayBuffer): boolean {
+ if (loadType === 'streaming') return true;
+ if (loadType === 'decompress') return false;
+ const seconds = estimateAudioSeconds(data);
+ return seconds === null ? data.byteLength >= UNKNOWN_LENGTH_STREAM_BYTES : seconds >= AUTO_STREAM_SECONDS;
+}
diff --git a/sdk/src/audio/index.ts b/sdk/src/audio/index.ts
index 717b9331b..fd95e3f33 100644
--- a/sdk/src/audio/index.ts
+++ b/sdk/src/audio/index.ts
@@ -24,4 +24,4 @@ export { AttenuationModel, calculateAttenuation, calculatePanning, spatialDistan
export { WebAudioBackend } from './WebAudioBackend';
export { MiniGameAudioBackend } from './MiniGameAudioBackend';
export { NativeAudioBackend } from './NativeAudioBackend';
-export type { AudioHandle, AudioBufferHandle, PlayConfig, PlatformAudioBackend, AudioBackendInitOptions, AudioDelivery } from './PlatformAudioBackend';
+export type { AudioHandle, AudioBufferHandle, PlayConfig, PlatformAudioBackend, AudioBackendInitOptions, AudioDelivery, AudioLoadType, AudioLoadOptions } from './PlatformAudioBackend';
diff --git a/sdk/src/core-content.ts b/sdk/src/core-content.ts
index 54715b55e..043e9bdbe 100644
--- a/sdk/src/core-content.ts
+++ b/sdk/src/core-content.ts
@@ -681,6 +681,7 @@ export {
type SoundEventSelectionResult,
applyAudioProjectConfig,
type AudioProjectConfig,
+ type AudioLoadType,
type AudioBusDecl,
type BusEffectDef,
type FilterEffectDef,
diff --git a/sdk/src/runtime/frameStats.ts b/sdk/src/runtime/frameStats.ts
index 00e81a4fc..1891b2b24 100644
--- a/sdk/src/runtime/frameStats.ts
+++ b/sdk/src/runtime/frameStats.ts
@@ -8,6 +8,7 @@
import type { App, FrameCosts } from '../app/app';
import { engineApi } from '../ecs/bridge/engineApi';
import { Quality } from '../render/qualityRuntime';
+import { Audio } from '../audio/Audio';
import type { QualityReport } from '../render/quality';
export interface FrameStatsReport {
@@ -27,6 +28,10 @@ export interface FrameStatsReport {
gpuScopes: Record<string, number>;
wasmBytes: number;
vramBytes: number;
+ /** Decoded audio resident (bytes), and how many clips play from their compressed file instead and the PCM that saved (estimate). */
+ audioBytes?: number;
+ audioStreamedCount?: number;
+ audioStreamedSavedBytes?: number;
quality?: QualityReport;
}
@@ -35,6 +40,12 @@ function jsonMap(json: string | undefined): Record<string, number> {
try { return JSON.parse(json) as Record<string, number>; } catch { return {}; }
}
+function audioStats(app: App): Pick<FrameStatsReport, 'audioBytes' | 'audioStreamedCount' | 'audioStreamedSavedBytes'> {
+ if (!app.hasResource(Audio)) return {};
+ const stats = app.getResource(Audio).getBufferStats();
+ return { audioBytes: stats.bufferBytes, audioStreamedCount: stats.streamedCount, audioStreamedSavedBytes: stats.streamedSavedBytes };
+}
+
export function frameStatsReport(app: App): FrameStatsReport {
const m = engineApi(app);
return {
@@ -51,6 +62,7 @@ export function frameStatsReport(app: App): FrameStatsReport {
gpuScopes: jsonMap(m?.engine_getGpuScopes?.()),
wasmBytes: app.wasmModule?.HEAPU8?.byteLength ?? 0,
vramBytes: m?.renderer_getEstimatedGpuBytes?.() ?? 0,
+ ...audioStats(app),
quality: app.hasResource(Quality) ? app.getResource(Quality)?.report() : undefined,
};
}
diff --git a/sdk/tests/audio-streaming.test.ts b/sdk/tests/audio-streaming.test.ts
new file mode 100644
index 000000000..8d44dbd9f
--- /dev/null
+++ b/sdk/tests/audio-streaming.test.ts
@@ -0,0 +1,237 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { WebAudioBackend } from '../src/audio/WebAudioBackend';
+import { AudioAPI } from '../src/audio/Audio';
+import { AUTO_STREAM_SECONDS, decodedPcmBytes, estimateAudioSeconds, shouldStream } from '../src/audio/audioStreaming';
+import { parseAudioProjectConfig, applyAudioProjectConfig } from '../src/audio/AudioProjectConfig';
+import { setPlatform } from '../src/platform/base';
+import type { PlatformAdapter } from '../src/platform/types';
+
+/** An 8 kHz mono 8-bit WAV of `seconds`: 8000 bytes a second keeps a long clip small. */
+function wav(seconds: number): ArrayBuffer {
+ const data = Math.round(seconds * 8000);
+ const buf = new ArrayBuffer(44 + data);
+ const v = new DataView(buf);
+ const tag = (o: number, s: string) => { for (let i = 0; i < 4; i++) v.setUint8(o + i, s.charCodeAt(i)); };
+ tag(0, 'RIFF'); v.setUint32(4, 36 + data, true); tag(8, 'WAVE'); tag(12, 'fmt ');
+ v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true);
+ v.setUint32(24, 8000, true); v.setUint32(28, 8000, true); v.setUint16(32, 1, true); v.setUint16(34, 8, true);
+ tag(36, 'data'); v.setUint32(40, data, true);
+ return buf;
+}
+
+/** A constant-bitrate MP3: an MPEG-1 Layer III frame header at 128 kbps, then padding to the length. */
+function mp3(seconds: number): ArrayBuffer {
+ const bytes = new Uint8Array(Math.round((seconds * 128000) / 8));
+ bytes.set([0xff, 0xfb, 0x90, 0x00]);
+ return bytes.buffer;
+}
+
+describe('estimating a clip\'s length without decoding it', () => {
+ it('reads a WAV exactly and a constant-bitrate MP3 from its size', () => {
+ expect(estimateAudioSeconds(wav(12))).toBeCloseTo(12, 3);
+ expect(estimateAudioSeconds(mp3(180))).toBeCloseTo(180, 0);
+ });
+
+ it('skips an ID3 tag, and says nothing about a container it does not know', () => {
+ const tagged = new Uint8Array(10 + 1000 + (30 * 128000) / 8);
+ tagged.set([0x49, 0x44, 0x33, 3, 0, 0, 0, 0, 0x07, 0x68]); // 1000-byte tag
+ tagged.set([0xff, 0xfb, 0x90, 0x00], 1010);
+ expect(estimateAudioSeconds(tagged.buffer)).toBeCloseTo(30, 0);
+ expect(estimateAudioSeconds(new Uint8Array(64).buffer)).toBeNull();
+ });
+
+ it('prices the PCM a streamed clip does not take — 69MB for three stereo minutes', () => {
+ expect(decodedPcmBytes(180)).toBe(180 * 48000 * 2 * 4);
+ expect(decodedPcmBytes(180) / 1e6).toBeCloseTo(69.1, 0);
+ });
+});
+
+describe('which clips stream', () => {
+ it('auto streams from the threshold up, and an explicit choice overrides the length', () => {
+ expect(shouldStream('auto', wav(AUTO_STREAM_SECONDS - 1))).toBe(false);
+ expect(shouldStream(undefined, wav(AUTO_STREAM_SECONDS))).toBe(true);
+ expect(shouldStream('streaming', wav(1))).toBe(true);
+ expect(shouldStream('decompress', wav(300))).toBe(false);
+ });
+
+ it('streams an unknown container when the file is large, and decodes it when small', () => {
+ expect(shouldStream('auto', new ArrayBuffer(900 * 1024))).toBe(true);
+ expect(shouldStream('auto', new ArrayBuffer(40 * 1024))).toBe(false);
+ });
+});
+
+// ── the backend ────────────────────────────────────────────────────────────────
+
+interface FakeElement {
+ src: string; loop: boolean; playbackRate: number; currentTime: number; preload: string; duration: number;
+ onended: (() => void) | null; onloadedmetadata: (() => void) | null; onerror: (() => void) | null;
+ play: ReturnType<typeof vi.fn>; pause: ReturnType<typeof vi.fn>; load: ReturnType<typeof vi.fn>;
+ removeAttribute: ReturnType<typeof vi.fn>;
+}
+let elements: FakeElement[] = [];
+let objectUrls: string[] = [];
+let revoked: string[] = [];
+let metadataDuration = 180;
+let contextState = 'running';
+const media: any[] = [];
+
+function install(): any {
+ elements = []; objectUrls = []; revoked = []; media.length = 0; contextState = 'running';
+ class FakeAudio {
+ src = ''; loop = false; playbackRate = 1; currentTime = 0; preload = ''; duration = metadataDuration;
+ onended: (() => void) | null = null; onloadedmetadata: (() => void) | null = null; onerror: (() => void) | null = null;
+ play = vi.fn().mockResolvedValue(undefined); pause = vi.fn(); load = vi.fn(); removeAttribute = vi.fn();
+ constructor() { elements.push(this as unknown as FakeElement); setTimeout(() => this.onloadedmetadata?.(), 0); }
+ }
+ (globalThis as any).Audio = FakeAudio;
+ let n = 0;
+ (globalThis as any).URL.createObjectURL = vi.fn(() => { const u = `blob:clip-${++n}`; objectUrls.push(u); return u; });
+ (globalThis as any).URL.revokeObjectURL = vi.fn((u: string) => { revoked.push(u); });
+ const gain = (c: any) => ({ gain: { value: 1, setTargetAtTime: vi.fn() }, connect: vi.fn(), disconnect: vi.fn(), context: c });
+ const ctx: any = {
+ get state() { return contextState; }, currentTime: 0, destination: {},
+ createGain: vi.fn(() => gain(ctx)), createStereoPanner: vi.fn(() => ({ pan: { value: 0 }, connect: vi.fn(), disconnect: vi.fn() })),
+ createBufferSource: vi.fn(), createAnalyser: vi.fn(() => ({ fftSize: 0, smoothingTimeConstant: 0, connect: vi.fn(), disconnect: vi.fn(), getByteFrequencyData: vi.fn() })),
+ createMediaElementSource: vi.fn((el: unknown) => { const node = { el, connect: vi.fn(), disconnect: vi.fn() }; media.push(node); return node; }),
+ decodeAudioData: vi.fn().mockResolvedValue({ duration: 2, length: 96000, numberOfChannels: 2 }),
+ resume: vi.fn().mockResolvedValue(undefined), suspend: vi.fn().mockResolvedValue(undefined), close: vi.fn(),
+ };
+ (globalThis as any).AudioContext = vi.fn(() => ctx);
+ setPlatform({ name: 'web', readFile: vi.fn(), now: vi.fn() } as unknown as PlatformAdapter);
+ return ctx;
+}
+
+describe('WebAudioBackend streaming', () => {
+ let backend: WebAudioBackend;
+ let ctx: any;
+
+ beforeEach(async () => {
+ metadataDuration = 180;
+ ctx = install();
+ backend = new WebAudioBackend();
+ await backend.initialize();
+ });
+ afterEach(() => backend.dispose());
+
+ it('keeps a long clip as compressed bytes: nothing decoded, the length from the element', async () => {
+ const handle = await backend.loadBufferFromData('bgm.mp3', mp3(180));
+ expect(ctx.decodeAudioData).not.toHaveBeenCalled();
+ expect(handle).toMatchObject({ streamed: true, bytes: 0, duration: 180 });
+ expect(handle.savedBytes).toBe(decodedPcmBytes(180));
+ });
+
+ it('still decodes a short effect, and honours an explicit choice either way', async () => {
+ expect((await backend.loadBufferFromData('click.wav', wav(0.4))).streamed).toBeUndefined();
+ expect(ctx.decodeAudioData).toHaveBeenCalledTimes(1);
+ expect((await backend.loadBufferFromData('tiny.wav', wav(0.4), { loadType: 'streaming' })).streamed).toBe(true);
+ expect((await backend.loadBufferFromData('long.mp3', mp3(120), { loadType: 'decompress' })).streamed).toBeUndefined();
+ });
+
+ it('plays through the pooled gain into the bus, loops, and ends the voice when the element ends', async () => {
+ const handle = await backend.loadBufferFromData('bgm.mp3', mp3(180));
+ const voice = backend.play(handle, { volume: 0.5, loop: true, bus: 'music' });
+ const el = elements.at(-1)!;
+ expect(el.src).toBe(objectUrls[0]);
+ expect(el.loop).toBe(true);
+ expect(el.play).toHaveBeenCalledTimes(1);
+ expect(media).toHaveLength(1);
+ expect(voice.isPlaying).toBe(true);
+ expect(voice.duration).toBe(180);
+ voice.setVolume(0.25);
+ voice.setPlaybackRate(1.5);
+ expect(el.playbackRate).toBe(1.5);
+ voice.pause();
+ expect(el.pause).toHaveBeenCalled();
+ expect(voice.isPlaying).toBe(false);
+ voice.resume();
+ expect(voice.isPlaying).toBe(true);
+
+ voice.setLoop(false);
+ const ended = vi.fn();
+ voice.onEnd = ended;
+ el.onended?.();
+ expect(ended).toHaveBeenCalledTimes(1);
+ expect(voice.isPlaying).toBe(false);
+ });
+
+ it('lets a second play of the same clip end the first, and a different clip play beside it', async () => {
+ const a = await backend.loadBufferFromData('a.mp3', mp3(60));
+ const b = await backend.loadBufferFromData('b.mp3', mp3(60));
+ const first = backend.play(a, {});
+ const second = backend.play(a, {});
+ expect(first.isPlaying).toBe(false);
+ expect(second.isPlaying).toBe(true);
+ const other = backend.play(b, {});
+ expect(second.isPlaying).toBe(true);
+ expect(other.isPlaying).toBe(true);
+ });
+
+ it('holds the object URL until a clip unloaded mid-play has stopped', async () => {
+ const handle = await backend.loadBufferFromData('bgm.mp3', mp3(60));
+ const voice = backend.play(handle, {});
+ backend.unloadBuffer(handle);
+ expect(revoked).toEqual([]);
+ voice.stop();
+ expect(revoked).toEqual([objectUrls[0]]);
+ const idle = await backend.loadBufferFromData('idle.mp3', mp3(60));
+ backend.unloadBuffer(idle);
+ expect(revoked).toHaveLength(2);
+ });
+
+ it('pauses a streamed voice when the device suspends and starts it again on resume', async () => {
+ const handle = await backend.loadBufferFromData('bgm.mp3', mp3(60));
+ const voice = backend.play(handle, {});
+ const el = elements.at(-1)!;
+ backend.suspend();
+ expect(el.pause).toHaveBeenCalled();
+ el.play.mockClear();
+ backend.resume();
+ expect(el.play).toHaveBeenCalledTimes(1);
+ expect(voice.isPlaying).toBe(true);
+ });
+
+ it('retries a start the browser blocked once the context is allowed to run', async () => {
+ const handle = await backend.loadBufferFromData('bgm.mp3', mp3(60));
+ contextState = 'suspended';
+ const blocked = vi.fn().mockRejectedValueOnce(new Error('NotAllowedError')).mockResolvedValue(undefined);
+ const spy = vi.spyOn(globalThis as any, 'Audio').mockImplementation(function (this: any) {
+ Object.assign(this, { src: '', loop: false, playbackRate: 1, currentTime: 0, play: blocked, pause: vi.fn(), load: vi.fn(), removeAttribute: vi.fn() });
+ return this;
+ } as never);
+ backend.play(handle, {});
+ await new Promise((r) => setTimeout(r, 0));
+ await new Promise((r) => setTimeout(r, 0));
+ expect(blocked).toHaveBeenCalledTimes(2);
+ spy.mockRestore();
+ });
+});
+
+describe('what the API reports', () => {
+ it('counts streamed clips and the memory they did not take, and takes importer choices by path', async () => {
+ metadataDuration = 180;
+ install();
+ const backend = new WebAudioBackend();
+ await backend.initialize();
+ const audio = new AudioAPI(backend, backend.mixer);
+ applyAudioProjectConfig(audio, parseAudioProjectConfig({ loadTypes: { 'assets/audio/short.wav': 'streaming', 'assets/audio/long.mp3': 'decompress', junk: 'auto' } }));
+
+ await audio.preloadFromData('assets/audio/bgm.mp3', mp3(180));
+ await audio.preloadFromData('./assets/audio/short.wav', wav(0.4));
+ await audio.preloadFromData('assets/audio/long.mp3', mp3(120));
+ await audio.preloadFromData('assets/audio/click.wav', wav(0.4));
+
+ const stats = audio.getBufferStats();
+ expect(stats.streamedCount).toBe(2);
+ expect(stats.streamedSavedBytes).toBeGreaterThan(decodedPcmBytes(180));
+ expect(stats.bufferBytes).toBeGreaterThan(0);
+ backend.dispose();
+ });
+
+ it('keeps only the choices a person can make, and leaves auto to the length', () => {
+ expect(parseAudioProjectConfig({ loadTypes: { 'a.mp3': 'streaming', 'b.mp3': 'auto', 'c.mp3': 3, '': 'streaming' } }).loadTypes)
+ .toEqual({ 'a.mp3': 'streaming' });
+ expect(parseAudioProjectConfig({}).loadTypes).toBeUndefined();
+ });
+});
diff --git a/tools/releaseNotes.mjs b/tools/releaseNotes.mjs
index dacbf9781..d3e4fa590 100644
--- a/tools/releaseNotes.mjs
+++ b/tools/releaseNotes.mjs
@@ -208,6 +208,8 @@ export const NOTED = {
{ note: 'The preview language menu appears once the string tables are indexed.' },
'fix(editor): Play starts in the preview language even where the runtime picked the system language':
{ note: 'Play starts in the preview language.' },
+ 'feat(audio): long clips on the Web play from their compressed file instead of decoding to memory':
+ { note: 'Long clips on the Web play from their compressed file instead of being decoded to memory.' },
'feat(render): a Tiling & Offset node for material graphs':
{ note: 'Material graphs have a Tiling & Offset node.' },
'feat(editor): an asset header names its kind and size, and a segmented switch is used only when its words fit':