A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(audio): a native host streams long clips - the file stays compressed and each voice decodes as it plays, by the same rule as the web; a 175 s mp3 keeps 2.3 MB instead of 64, and loads in no time instead of a decode on the frame
yhh06a8c6043ea9
diff --git a/native/host/bindings/AudioBindings.cpp b/native/host/bindings/AudioBindings.cppindex de986a611..c3fb714c8 100644--- a/native/host/bindings/AudioBindings.cpp+++ b/native/host/bindings/AudioBindings.cpp@@ -28,19 +28,24 @@ using namespace esengine;namespace eshost {namespace {-// es_audioLoad(ArrayBuffer | TypedArray) -> { id, duration, bytes } | null. The-// SDK hands the compressed clip bytes; the engine decodes + registers them.+// es_audioLoad(ArrayBuffer | TypedArray, stream?) -> { id, duration, bytes, streamed } | null.+// The SDK hands the compressed clip bytes and says whether to stream it; the+// engine decodes + registers them, or keeps the file and decodes as it plays.+// `streamed` is what happened, which is how the SDK tells a host that streams+// from one that predates the second argument and decoded regardless.esjs::Val js_audioLoad(const esjs::Args& a) {if (a.size() < 1) return esjs::null();std::vector<u8> bytes;readByteSource(a[0], bytes);if (bytes.empty()) return esjs::null();- auto r = host().audio.load(bytes.data(), bytes.size());+ const bool stream = a.size() > 1 && esjs::toBool(a[1]);+ auto r = host().audio.load(bytes.data(), bytes.size(), stream);if (r.id < 0) return esjs::null();esjs::Val o = esjs::object();esjs::set(o, "id", esjs::int32(r.id));esjs::set(o, "duration", esjs::number(r.duration));esjs::set(o, "bytes", esjs::int64(r.bytes));+ esjs::set(o, "streamed", esjs::boolean(r.streamed));return o;}@@ -125,7 +130,7 @@ esjs::Val js_audioSpectrum(const esjs::Args& a) {void registerAudioBindings(HostState& h, esjs::Val global) {if (!h.audio.ready()) return;- bindGlobal(global, "es_audioLoad", js_audioLoad, 1);+ bindGlobal(global, "es_audioLoad", js_audioLoad, 2);bindGlobal(global, "es_audioUnload", js_audioUnload, 1);bindGlobal(global, "es_audioPlay", js_audioPlay, 5);bindGlobal(global, "es_audioStop", js_audioStop, 1);diff --git a/native/host/media/native_audio.cpp b/native/host/media/native_audio.cppindex eb88316e7..03a8faa1a 100644--- a/native/host/media/native_audio.cpp+++ b/native/host/media/native_audio.cpp@@ -27,14 +27,23 @@namespace eshost {-// A decoded clip. register_decoded_data() does NOT copy the PCM, so `pcm` must-// outlive every voice playing it — freed only once unloaded AND no voice remains-// (`refs` == 0), or a still-playing sound would read freed memory.+// A loaded clip: its PCM, or — streamed — its compressed file, which every voice+// decodes for itself as it plays. Neither register_decoded_data() nor+// register_encoded_data() copies, so the data must outlive every voice playing it+// — freed only once unloaded AND no voice remains (`refs` == 0), or a+// still-playing sound would read freed memory.struct Buffer {std::string name; // "buf://<id>", the resource-manager keyvoid* pcm = nullptr; // owned; f32 interleaved at the engine's rate/channels+ std::vector<uint8_t> encoded; // streamed: the file as it was loadedint refs = 0; // live voices playing this bufferbool unloaded = false; // SDK dropped it; free when the last voice releases++ void free() {+ if (pcm) ma_free(pcm, nullptr);+ pcm = nullptr;+ encoded = {};+ }};// One playing instance. ma_sound is a node in the engine graph and must not move,@@ -99,7 +108,7 @@ struct AudioEngine::Impl {auto it = buffers.find(bufferId);if (it == buffers.end()) return;if (--it->second.refs <= 0 && it->second.unloaded) {- ma_free(it->second.pcm, nullptr);+ it->second.free();buffers.erase(it);}}@@ -219,7 +228,7 @@ void AudioEngine::shutdown() {if (impl_->ready) ma_engine_uninit(&impl_->engine);closeDevice(impl_);if (impl_->contextReady) ma_context_uninit(&impl_->context);- for (auto& [id, buf] : impl_->buffers) ma_free(buf.pcm, nullptr);+ for (auto& [id, buf] : impl_->buffers) buf.free();impl_->buffers.clear();delete impl_;impl_ = nullptr;@@ -227,12 +236,41 @@ void AudioEngine::shutdown() {bool AudioEngine::ready() const { return impl_ && impl_->ready; }-AudioEngine::Loaded AudioEngine::load(const uint8_t* bytes, size_t n) {- const Loaded fail{-1, 0.0, 0};+AudioEngine::Loaded AudioEngine::load(const uint8_t* bytes, size_t n, bool stream) {+ const Loaded fail{-1, 0.0, 0, false};if (!ready() || !bytes || n == 0) return fail;// Decode straight to the engine's format so playback never resamples.ma_decoder_config dcfg = ma_decoder_config_init(ma_format_f32, impl_->channels, impl_->sampleRate);++ // Streamed: keep the file and hand the manager a view of it. A sound made from+ // an ENCODED registration gets a decoder of its own over these bytes and reads+ // it as the mixer asks, so what stays resident is the file — megabytes for a+ // track whose PCM would be tens of them — and nothing is decoded up front,+ // which is also the stall a long clip used to cost the frame that loaded it.+ if (stream) {+ // Opened once here for two answers: that it decodes at all, and how long it+ // is. A file that fails either way falls through to the decoded path, which+ // reports the failure the way it always has.+ ma_decoder probe;+ if (ma_decoder_init_memory(bytes, n, &dcfg, &probe) == MA_SUCCESS) {+ ma_uint64 frames = 0;+ if (ma_decoder_get_length_in_pcm_frames(&probe, &frames) != MA_SUCCESS) frames = 0;+ ma_decoder_uninit(&probe);++ const int id = impl_->nextBuffer++;+ Buffer& buf = impl_->buffers[id];+ buf.name = "buf://" + std::to_string(id);+ buf.encoded.assign(bytes, bytes + n);+ if (ma_resource_manager_register_encoded_data(+ ma_engine_get_resource_manager(&impl_->engine), buf.name.c_str(),+ buf.encoded.data(), buf.encoded.size()) == MA_SUCCESS) {+ const double seconds = impl_->sampleRate ? static_cast<double>(frames) / impl_->sampleRate : 0.0;+ return Loaded{id, seconds, static_cast<long long>(n), true};+ }+ impl_->buffers.erase(id);+ }+ }ma_uint64 frames = 0;void* pcm = nullptr;if (ma_decode_memory(bytes, n, &dcfg, &frames, &pcm) != MA_SUCCESS || !pcm) return fail;@@ -246,15 +284,15 @@ AudioEngine::Loaded AudioEngine::load(const uint8_t* bytes, size_t n) {if (ma_resource_manager_register_decoded_data(ma_engine_get_resource_manager(&impl_->engine), buf.name.c_str(),pcm, frames, ma_format_f32, impl_->channels, impl_->sampleRate) != MA_SUCCESS) {+ buf.free();impl_->buffers.erase(id);- ma_free(pcm, nullptr);return fail;}const double duration = impl_->sampleRate ? static_cast<double>(frames) / impl_->sampleRate : 0.0;const long long decodedBytes =static_cast<long long>(frames) * impl_->channels * static_cast<long long>(sizeof(float));- return Loaded{id, duration, decodedBytes};+ return Loaded{id, duration, decodedBytes, false};}void AudioEngine::unload(int bufferId) {@@ -267,7 +305,7 @@ void AudioEngine::unload(int bufferId) {ma_engine_get_resource_manager(&impl_->engine), it->second.name.c_str());it->second.unloaded = true;if (it->second.refs <= 0) {- ma_free(it->second.pcm, nullptr);+ it->second.free();impl_->buffers.erase(it);}}@@ -281,7 +319,9 @@ int AudioEngine::play(int bufferId, float volume, float pan, bool loop, float raVoice& voice = impl_->voices[id]; // node-stable: &voice.sound is fixedvoice.bufferId = bufferId;voice.loop = loop;- // Data is already decoded + registered, so init just wraps it (no decode).+ // The data is registered under the name, so init wraps it: the PCM as it is,+ // or — for a streamed clip — a decoder over the file that this voice alone+ // reads, which is why two voices of one track do not share a cursor.// NO_SPATIALIZATION: 2D games pan explicitly; the 3D positioner would fight it.if (ma_sound_init_from_file(&impl_->engine, bit->second.name.c_str(),MA_SOUND_FLAG_NO_SPATIALIZATION, nullptr, nullptr,diff --git a/native/host/media/native_audio.hpp b/native/host/media/native_audio.hppindex 8a2d507ef..78ed24a01 100644--- a/native/host/media/native_audio.hpp+++ b/native/host/media/native_audio.hpp@@ -9,8 +9,11 @@* The SDK's NativeAudioBackend is a thin shell over the es_audio** bindings, which in turn are a thin shell over this.*- * A buffer id names a decoded clip; a voice id names one playing- * instance. The miniaudio header is kept out of this interface (pimpl),+ * A buffer id names a loaded clip; a voice id names one playing+ * instance. A clip is held one of two ways: decoded to PCM, which starts+ * with no delay and costs its whole length in memory (0.37 MB a second),+ * or streamed, which keeps the compressed file and decodes each voice as+ * it plays — what music and anything else long wants. The miniaudio header is kept out of this interface (pimpl),* so host_core.cpp does not pay to compile it.*/#pragma once@@ -36,11 +39,16 @@ public:struct Loaded {int id; ///< < 0 on decode failure.- double duration; ///< Seconds.- long long bytes; ///< Decoded PCM size, for the residency budget.+ double duration; ///< Seconds; 0 when a streamed file does not say.+ long long bytes; ///< What stays resident: the PCM, or the compressed file when streamed.+ bool streamed; ///< Held compressed and decoded while it plays.};- /** Decode a compressed clip to PCM and register it. */- Loaded load(const uint8_t* bytes, size_t n);+ /**+ * Register a compressed clip: decoded to PCM now, or with @p stream kept as it+ * is and decoded by each voice as it plays. A file that cannot be streamed is+ * decoded instead, and `streamed` says which happened.+ */+ Loaded load(const uint8_t* bytes, size_t n, bool stream = false);void unload(int bufferId);/**diff --git a/sdk/etc/index.native.api.md b/sdk/etc/index.native.api.mdindex c90c1f059..e2b061e73 100644--- a/sdk/etc/index.native.api.md+++ b/sdk/etc/index.native.api.md@@ -6712,7 +6712,7 @@ width: number## NativeHostBindings — interface @experimental```-es_audioLoad: ((bytes: ArrayBuffer) => { id: number; duration: number; bytes: number; } | null) | undefined+es_audioLoad: ((bytes: ArrayBuffer, stream?: boolean) => { id: number; duration: number; bytes: number; streamed?: boolean; } | null) | undefinedes_audioPause: ((voiceId: number) => void) | undefinedes_audioPlay: ((bufferId: number, volume: number, pan: number, loop: boolean, rate: number) => number) | undefinedes_audioResume: ((voiceId: number) => void) | undefineddiff --git a/sdk/src/audio/Audio.ts b/sdk/src/audio/Audio.tsindex 1aaff8f13..be919c87b 100644--- a/sdk/src/audio/Audio.ts+++ b/sdk/src/audio/Audio.ts@@ -131,6 +131,10 @@ export class AudioAPI {private readonly bufferSources_ = new Map<string, SharedAudioBuffer>();/** refCount==0 urls in eviction order (oldest first); access re-appends. */private readonly evictOrder_ = new Set<string>();+ /** Urls a play is waiting on, by how many: not evictable until that play has started. */+ private readonly awaitingPlay_ = new Map<string, number>();+ /** Clips already reported as larger than the whole budget. */+ private readonly oversizeReported_ = new Set<string>();private residentBytes_ = 0;private bufferBudgetOverride_: number | null = null;/** Importer choices by authored path; a clip with none is `auto`. */@@ -317,11 +321,28 @@ export class AudioAPI {if (!source.charged) {source.charged = true;this.residentBytes_ += handle.bytes ?? 0;+ this.reportOversize_(url, handle);this.enforceBudget_();}this.evictOrder_.add(url);}+ /**+ * A clip that is larger than the whole budget cannot be kept warm: making room for it empties the cache,+ * and the next clip to load evicts it in turn, so it is decoded again every time it is played. Nothing+ * about that is visible from the game — the sound just starts late, or not at all — so it is said once.+ */+ private reportOversize_(url: string, handle: AudioBufferHandle): void {+ const bytes = handle.bytes ?? 0;+ const budget = this.bufferBudget;+ if (budget === 0 || bytes <= budget || this.oversizeReported_.has(url)) return;+ this.oversizeReported_.add(url);+ const mb = (n: number): string => (n / (1024 * 1024)).toFixed(1);+ log.warn('audio', `${url} takes ${mb(bytes)} MB decoded, more than the whole ${mb(budget)} MB audio cache `+ + '(audioCacheBudget): it is dropped as soon as another clip loads and decoded again on its next play. '+ + "Set its import to 'streaming', or raise the budget");+ }+/** Cache lookup for playback: refreshes the LRU position of an evictable* entry so frequently-played warm sounds aren't the first to go. */private lookupBuffer_(url: string): AudioBufferHandle | undefined {@@ -362,6 +383,8 @@ export class AudioAPI {if (budget === 0) return;for (const url of this.evictOrder_) {if (this.residentBytes_ <= budget) break;+ // A play is about to start on it: evicting it now is a sound that never happens.+ if (this.awaitingPlay_.has(url)) continue;const entry = this.bufferCache_.get(url);if (entry) this.freeBuffer_(url, entry);}@@ -451,6 +474,29 @@ export class AudioAPI {await this.ensureBuffer_(url, source => (options ? this.backend_.loadBuffer(source, options) : this.backend_.loadBuffer(source)));}+ /**+ * Load `url` for a play that is waiting on it and start that play, as one step: the clip cannot be+ * evicted between arriving and being played, which is what a load finishing beside it used to do to a+ * clip larger than the budget. Says so when the clip is not there to play after all, instead of the+ * silence that left a game with no music and nothing in its log.+ */+ private async loadThenPlay_(url: string, what: string, play: (buffer: AudioBufferHandle) => void): Promise<void> {+ this.awaitingPlay_.set(url, (this.awaitingPlay_.get(url) ?? 0) + 1);+ try {+ await this.preload(url);+ if (this.disposed_) return;+ const buffer = this.lookupBuffer_(url);+ if (buffer) play(buffer);+ else log.warn('audio', `${what} did not play: ${url} loaded, but was gone from the audio cache before it could start`);+ } catch (err) {+ log.warn('audio', `Failed to preload ${what}: ${url}`, err);+ } finally {+ const left = (this.awaitingPlay_.get(url) ?? 1) - 1;+ if (left > 0) this.awaitingPlay_.set(url, left);+ else this.awaitingPlay_.delete(url);+ }+ }+async preloadAll(urls: string[]): Promise<void> {await Promise.all(urls.map(url => this.preload(url)));}@@ -506,14 +552,9 @@ export class AudioAPI {const buffer = this.lookupBuffer_(url);if (!buffer) {const pending = this.createDeferredHandle_(playConfig.volume ?? 1);- this.preload(url).then(() => {- if (this.disposed_ || epoch !== this.playbackEpoch_ || pending.cancelled) return;- const buf = this.lookupBuffer_(url);- if (buf) {- pending.resolve(overrides => this.playVoice_(buf, { ...playConfig, ...overrides }));- }- }).catch(err => {- log.warn('audio', `Failed to preload audio: ${url}`, err);+ void this.loadThenPlay_(url, 'audio', buf => {+ if (epoch !== this.playbackEpoch_ || pending.cancelled) return;+ pending.resolve(overrides => this.playVoice_(buf, { ...playConfig, ...overrides }));});return pending;}@@ -556,15 +597,7 @@ export class AudioAPI {if (buffer) {play(buffer);} else {- this.preload(url).then(() => {- if (this.disposed_) return;- const buf = this.lookupBuffer_(url);- if (buf) {- play(buf);- }- }).catch(err => {- log.warn('audio', `Failed to preload BGM: ${url}`, err);- });+ void this.loadThenPlay_(url, 'BGM', play);}}diff --git a/sdk/src/audio/NativeAudioBackend.ts b/sdk/src/audio/NativeAudioBackend.tsindex e7f00150f..dd3193dc0 100644--- a/sdk/src/audio/NativeAudioBackend.ts+++ b/sdk/src/audio/NativeAudioBackend.ts@@ -10,6 +10,10 @@* JS — the no-JIT budget forbids it. `mixer` is therefore null (no JS* DSP graph), like WeChat; per-voice volume / pan / rate / loop are the* full control surface.+ *+ * A clip the importer (or its length, under `auto`) says to stream is+ * handed to the host to keep compressed: the rule is audioStreaming.ts,+ * the one the web backend follows, so the two agree on which clips.*/import type { AudioMixer } from './AudioMixer';import type {@@ -17,6 +21,8 @@ import type {PlatformAudioBackend, PlayConfig,} from './PlatformAudioBackend';import type { NativeAudioBridge } from '../platform/native/bridge';+import type { AudioLoadOptions } from './PlatformAudioBackend';+import { decodedPcmBytes, shouldStream } from './audioStreaming';import { platformReadFile } from '../platform/base';import { scalar } from '../math/scalar';@@ -82,21 +88,27 @@ export class NativeAudioBackend implements PlatformAudioBackend {async initialize(_options?: AudioBackendInitOptions): Promise<void> {}async ensureResumed(): Promise<void> {}- async loadBuffer(url: string): Promise<AudioBufferHandle> {+ async loadBuffer(url: string, options?: AudioLoadOptions): Promise<AudioBufferHandle> {// On native the audio ref resolver hands back a packaged path (not bytes),// so read the file through the same platform channel as every other asset,// then let the native engine decode it.- return this.loadFromBytes_(await platformReadFile(url));+ return this.loadFromBytes_(await platformReadFile(url), options);}- async loadBufferFromData(_url: string, data: ArrayBuffer): Promise<AudioBufferHandle> {- return this.loadFromBytes_(data);+ async loadBufferFromData(_url: string, data: ArrayBuffer, options?: AudioLoadOptions): Promise<AudioBufferHandle> {+ return this.loadFromBytes_(data, options);}- private loadFromBytes_(bytes: ArrayBuffer): AudioBufferHandle {- const decoded = this.audio_.load(bytes);- if (!decoded) throw new Error('[native] audio decode failed');- return { id: decoded.id, duration: decoded.duration, bytes: decoded.bytes };+ private loadFromBytes_(bytes: ArrayBuffer, options?: AudioLoadOptions): AudioBufferHandle {+ const loaded = this.audio_.load(bytes, shouldStream(options?.loadType, bytes));+ if (!loaded) throw new Error('[native] audio decode failed');+ // What the host DID, not what was asked: one that could not stream the file decoded it, and its PCM+ // is then what counts against the budget.+ if (!loaded.streamed) return { id: loaded.id, duration: loaded.duration, bytes: loaded.bytes };+ return {+ id: loaded.id, duration: loaded.duration, bytes: loaded.bytes, streamed: true,+ savedBytes: Math.max(0, decodedPcmBytes(loaded.duration) - loaded.bytes),+ };}unloadBuffer(handle: AudioBufferHandle): void {diff --git a/sdk/src/audio/PlatformAudioBackend.ts b/sdk/src/audio/PlatformAudioBackend.tsindex d6ab69eb8..dbb15c52a 100644--- a/sdk/src/audio/PlatformAudioBackend.ts+++ b/sdk/src/audio/PlatformAudioBackend.ts@@ -20,8 +20,9 @@ export interface AudioHandle {/*** How a clip is held: `decompress` decodes it to PCM at load, `streaming` plays it from its compressed- * file, `auto` lets its length decide (see audioStreaming.ts). A backend with one way of playing a clip- * ignores it.+ * file, `auto` lets its length decide (see audioStreaming.ts) — the same rule on the web and on a native+ * host, so a track that streams in the editor streams on the phone. A backend with one way of playing a+ * clip ignores it.*/export type AudioLoadType = 'auto' | 'decompress' | 'streaming';@@ -32,7 +33,11 @@ export interface AudioLoadOptions {export interface AudioBufferHandle {readonly id: number;readonly duration: number;- /** Played from the compressed file, not decoded: `bytes` is then 0 and `savedBytes` is what decoding would have cost. */+ /**+ * Played from the compressed file, not decoded. `bytes` is then what the backend still keeps of it — nothing+ * where the host's media element reads the file (web), the file itself where the engine decodes from memory+ * (native) — and `savedBytes` is what decoding would have cost on top of that.+ */readonly streamed?: boolean;/** The PCM a streamed clip did not take, estimated from its length. */readonly savedBytes?: number;diff --git a/sdk/src/platform/native/bridge.ts b/sdk/src/platform/native/bridge.tsindex 85ac46955..194c4e660 100644--- a/sdk/src/platform/native/bridge.ts+++ b/sdk/src/platform/native/bridge.ts@@ -128,17 +128,22 @@ export interface NativeTextEditorBridge {* when the host bound the audio primitives; a host without a sound device omits* `NativeBridge.audio` and the audio system stays silent (the Null backend).*- * A `bufferId` names a decoded clip; a `voiceId` names one playing instance of+ * A `bufferId` names a loaded clip; a `voiceId` names one playing instance of* it. Both are non-negative host handles; a failed load or play returns null / -1.*/export interface NativeAudioBridge {/** `bins` byte magnitudes of what is playing (0..Nyquist), or null when the* host has no analyser / nothing has played. The device's AnalyserNode. */spectrum?(bins: number): ArrayBuffer | null;- /** Decode + register a clip from its (compressed) file bytes. Returns the- * buffer handle, its length in seconds and its decoded size in bytes (for the- * residency budget), or null on decode failure. */- load(bytes: ArrayBuffer): { id: number; duration: number; bytes: number } | null;+ /**+ * Register a clip from its (compressed) file bytes: decoded to PCM now, or —+ * with `stream` — kept as the file and decoded by each voice as it plays.+ * Returns the buffer handle, its length in seconds (0 when a streamed file does+ * not say) and what stays resident in bytes, for the residency budget; null on+ * decode failure. `streamed` is what the host did: one that cannot stream the+ * file, or predates the argument, decodes it and leaves this unset.+ */+ load(bytes: ArrayBuffer, stream?: boolean): { id: number; duration: number; bytes: number; streamed?: boolean } | null;unload(bufferId: number): void;/** Start a voice on a loaded buffer. Returns its voice id, or -1 if the buffer* is unknown. */diff --git a/sdk/src/platform/native/hostBridge.ts b/sdk/src/platform/native/hostBridge.tsindex 970db701d..bdc76b1ff 100644--- a/sdk/src/platform/native/hostBridge.ts+++ b/sdk/src/platform/native/hostBridge.ts@@ -137,7 +137,7 @@ export interface NativeHostBindings extends NativeDeviceHost {es_textEditor_write?(value: string, selectionStart: number, selectionEnd: number): void;es_audioSpectrum?(bins: number): ArrayBuffer | null;- es_audioLoad?(bytes: ArrayBuffer): { id: number; duration: number; bytes: number } | null;+ es_audioLoad?(bytes: ArrayBuffer, stream?: boolean): { id: number; duration: number; bytes: number; streamed?: boolean } | null;es_audioUnload?(bufferId: number): void;es_audioPlay?(bufferId: number, volume: number, pan: number, loop: boolean, rate: number): number;es_audioStop?(voiceId: number): void;@@ -379,7 +379,7 @@ function hostAudio(if (failed) throw failure;};return {- load: (bytes) => bindings.es_audioLoad!(bytes),+ load: (bytes, stream) => bindings.es_audioLoad!(bytes, stream === true),unload: (id) => bindings.es_audioUnload!(id),play: (buf, vol, pan, loop, rate) => bindings.es_audioPlay!(buf, vol, pan, loop, rate),stop: (v) => bindings.es_audioStop!(v),diff --git a/sdk/tests/audio-residency.test.ts b/sdk/tests/audio-residency.test.tsindex c7d2cebd5..f25462a64 100644--- a/sdk/tests/audio-residency.test.ts+++ b/sdk/tests/audio-residency.test.ts@@ -9,6 +9,7 @@*/import { describe, it, expect, vi, beforeEach } from 'vitest';import { AudioAPI } from '../src/audio/Audio';+import { log } from '../src/util/logger';import { Assets } from '../src/asset/Assets';import type { Backend } from '../src/asset/Backend';import type { PlatformAudioBackend, AudioBufferHandle, AudioHandle } from '../src/audio/PlatformAudioBackend';@@ -315,3 +316,65 @@ describe('AudioAPI buffer residency', () => {expect(audio.getBufferStats().evictableCount).toBe(1);});});++// A three-minute track decoded is about 64 MB and the default cache is 32: the clip is larger than the whole+// budget, so whatever loads next evicts it. A play that was waiting on it then found nothing and said nothing (#122).+describe('a clip larger than the whole audio cache', () => {+ function sized(bytes: Record<string, number>) {+ const backend = createMockBackend();+ const played: number[] = [];+ let nextId = 100;+ backend.loadBuffer = async (url: string): Promise<AudioBufferHandle> => {+ backend.loads.push(url);+ return { id: ++nextId, duration: 1, bytes: bytes[url] ?? 10 };+ };+ const play = backend.play;+ backend.play = (buffer, config) => { played.push(buffer.id); return play(buffer, config); };+ const audio = new AudioAPI(backend);+ audio.setBufferBudget(32);+ return { audio, backend, played };+ }++ it('still plays when other clips finish loading beside it', async () => {+ const { audio, backend, played } = sized({ 'music/theme.mp3': 64 });+ const warn = vi.spyOn(log, 'warn').mockImplementation(() => {});+ audio.playBGM('music/theme.mp3');+ await Promise.all(['sfx/a.wav', 'sfx/b.wav', 'sfx/c.wav'].map((url) => audio.preload(url)));+ await new Promise((r) => setTimeout(r, 0));+ const said = warn.mock.calls.map((c) => String(c[1]));+ warn.mockRestore();+ expect(backend.loads).toContain('music/theme.mp3');+ expect(played).toHaveLength(1);+ expect(said.filter((m) => m.includes('did not play'))).toEqual([]);+ // Said once, by name and with the numbers: it will be decoded again on its next play.+ expect(said.filter((m) => m.includes('more than the whole'))).toHaveLength(1);+ expect(said.find((m) => m.includes('more than the whole'))).toMatch(/music\/theme\.mp3.*streaming/);+ });++ it('is evictable again once its play has started', async () => {+ const { audio } = sized({ 'music/theme.mp3': 64 });+ const warn = vi.spyOn(log, 'warn').mockImplementation(() => {});+ audio.playBGM('music/theme.mp3');+ await new Promise((r) => setTimeout(r, 0));+ expect(audio.getBufferHandle('music/theme.mp3')).toBeDefined();+ await audio.preload('sfx/a.wav');+ warn.mockRestore();+ // The waiting play is what held it; the cache is back under its budget now.+ expect(audio.getBufferHandle('music/theme.mp3')).toBeUndefined();+ expect(audio.getBufferStats().bufferBytes).toBeLessThanOrEqual(32);+ });++ it('says nothing about a streamed clip, which keeps only its file', async () => {+ const { audio, backend } = sized({});+ backend.loadBuffer = async (url: string): Promise<AudioBufferHandle> => (url.startsWith('music/')+ ? { id: 7, duration: 175, bytes: 3, streamed: true, savedBytes: 64 } : { id: 8 + url.length, duration: 1, bytes: 10 });+ const warn = vi.spyOn(log, 'warn').mockImplementation(() => {});+ audio.playBGM('music/theme.mp3');+ await Promise.all(['sfx/a.wav', 'sfx/b.wav'].map((url) => audio.preload(url)));+ await new Promise((r) => setTimeout(r, 0));+ expect(warn).not.toHaveBeenCalled();+ warn.mockRestore();+ expect(audio.getBufferHandle('music/theme.mp3')).toBeDefined();+ expect(audio.getBufferStats()).toMatchObject({ streamedCount: 1, streamedSavedBytes: 64 });+ });+});diff --git a/sdk/tests/native-audio.test.ts b/sdk/tests/native-audio.test.tsindex fe1ac9d43..f1d057dd0 100644--- a/sdk/tests/native-audio.test.ts+++ b/sdk/tests/native-audio.test.ts@@ -311,3 +311,55 @@ describe('assertNativeHost — audio stays optional', () => {expect(() => assertNativeHost(scope)).not.toThrow();});});++// The same rule the web backend follows (audioStreaming.ts), so a track that streams in the editor streams on the+// phone: before this the native host decoded every clip whole, 0.37 MB a second (#122).+describe('NativeAudioBackend streaming', () => {+ /** A PCM WAV header that says `seconds`, with one byte a second so the file stays small. */+ function wav(seconds: number): ArrayBuffer {+ const view = new DataView(new ArrayBuffer(44 + seconds));+ view.setUint32(0, 0x52494646, false); view.setUint32(8, 0x57415645, false);+ view.setUint32(12, 0x666d7420, false); view.setUint32(16, 16, true);+ view.setUint32(28, 1, true); // byte rate+ view.setUint32(36, 0x64617461, false); view.setUint32(40, seconds, true);+ return view.buffer;+ }++ function streamingHost(canStream = true) {+ const scope = audioScope();+ const asked: boolean[] = [];+ scope.es_audioLoad = (bytes: ArrayBuffer, stream?: boolean) => {+ asked.push(stream === true);+ return stream && canStream+ ? { id: 1, duration: 175, bytes: bytes.byteLength, streamed: true }+ : { id: 1, duration: 175, bytes: 67_057_376 };+ };+ return { backend: new NativeAudioBackend(createHostBridge(scope).audio!), asked };+ }++ it('hands the host a clip the importer says to stream, and reports what that saved', async () => {+ const { backend, asked } = streamingHost();+ const data = new ArrayBuffer(2_400_000);+ const buffer = await backend.loadBufferFromData('music', data, { loadType: 'streaming' });+ expect(asked).toEqual([true]);+ // What stays resident is the file; the PCM it did not take is the rest of what decoding would cost.+ expect(buffer).toMatchObject({ streamed: true, bytes: 2_400_000, duration: 175 });+ expect(buffer.savedBytes).toBe(175 * 48000 * 2 * 4 - 2_400_000);+ });++ it('lets the length decide under auto, and decodes what the importer says to decode', async () => {+ const { backend, asked } = streamingHost();+ await backend.loadBufferFromData('long', wav(40));+ await backend.loadBufferFromData('short', wav(5));+ await backend.loadBufferFromData('forced', wav(40), { loadType: 'decompress' });+ expect(asked).toEqual([true, false, false]);+ });++ it('reports a decoded clip as decoded when the host could not stream it', async () => {+ const { backend, asked } = streamingHost(false);+ const buffer = await backend.loadBufferFromData('music', new ArrayBuffer(2_400_000), { loadType: 'streaming' });+ expect(asked).toEqual([true]);+ expect(buffer.streamed).toBeUndefined();+ expect(buffer.bytes).toBe(67_057_376);+ });+});