Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(audio): a native host streams long clips - the file stays compressed and each voice decodes as it plays, by the same rule as the web; a 175 s mp3 keeps 2.3 MB instead of 64, and loads in no time instead of a decode on the frame

yhh06a8c6043ea9
diff --git a/native/host/bindings/AudioBindings.cpp b/native/host/bindings/AudioBindings.cpp
index de986a611..c3fb714c8 100644
--- a/native/host/bindings/AudioBindings.cpp
+++ b/native/host/bindings/AudioBindings.cpp
@@ -28,19 +28,24 @@ using namespace esengine;
namespace eshost {
namespace {
-// es_audioLoad(ArrayBuffer | TypedArray) -> { id, duration, bytes } | null. The
-// SDK hands the compressed clip bytes; the engine decodes + registers them.
+// es_audioLoad(ArrayBuffer | TypedArray, stream?) -> { id, duration, bytes, streamed } | null.
+// The SDK hands the compressed clip bytes and says whether to stream it; the
+// engine decodes + registers them, or keeps the file and decodes as it plays.
+// `streamed` is what happened, which is how the SDK tells a host that streams
+// from one that predates the second argument and decoded regardless.
esjs::Val js_audioLoad(const esjs::Args& a) {
if (a.size() < 1) return esjs::null();
std::vector<u8> bytes;
readByteSource(a[0], bytes);
if (bytes.empty()) return esjs::null();
- auto r = host().audio.load(bytes.data(), bytes.size());
+ const bool stream = a.size() > 1 && esjs::toBool(a[1]);
+ auto r = host().audio.load(bytes.data(), bytes.size(), stream);
if (r.id < 0) return esjs::null();
esjs::Val o = esjs::object();
esjs::set(o, "id", esjs::int32(r.id));
esjs::set(o, "duration", esjs::number(r.duration));
esjs::set(o, "bytes", esjs::int64(r.bytes));
+ esjs::set(o, "streamed", esjs::boolean(r.streamed));
return o;
}
@@ -125,7 +130,7 @@ esjs::Val js_audioSpectrum(const esjs::Args& a) {
void registerAudioBindings(HostState& h, esjs::Val global) {
if (!h.audio.ready()) return;
- bindGlobal(global, "es_audioLoad", js_audioLoad, 1);
+ bindGlobal(global, "es_audioLoad", js_audioLoad, 2);
bindGlobal(global, "es_audioUnload", js_audioUnload, 1);
bindGlobal(global, "es_audioPlay", js_audioPlay, 5);
bindGlobal(global, "es_audioStop", js_audioStop, 1);
diff --git a/native/host/media/native_audio.cpp b/native/host/media/native_audio.cpp
index eb88316e7..03a8faa1a 100644
--- a/native/host/media/native_audio.cpp
+++ b/native/host/media/native_audio.cpp
@@ -27,14 +27,23 @@
namespace eshost {
-// A decoded clip. register_decoded_data() does NOT copy the PCM, so `pcm` must
-// outlive every voice playing it — freed only once unloaded AND no voice remains
-// (`refs` == 0), or a still-playing sound would read freed memory.
+// A loaded clip: its PCM, or — streamed — its compressed file, which every voice
+// decodes for itself as it plays. Neither register_decoded_data() nor
+// register_encoded_data() copies, so the data must outlive every voice playing it
+// — freed only once unloaded AND no voice remains (`refs` == 0), or a
+// still-playing sound would read freed memory.
struct Buffer {
std::string name; // "buf://<id>", the resource-manager key
void* pcm = nullptr; // owned; f32 interleaved at the engine's rate/channels
+ std::vector<uint8_t> encoded; // streamed: the file as it was loaded
int refs = 0; // live voices playing this buffer
bool unloaded = false; // SDK dropped it; free when the last voice releases
+
+ void free() {
+ if (pcm) ma_free(pcm, nullptr);
+ pcm = nullptr;
+ encoded = {};
+ }
};
// One playing instance. ma_sound is a node in the engine graph and must not move,
@@ -99,7 +108,7 @@ struct AudioEngine::Impl {
auto it = buffers.find(bufferId);
if (it == buffers.end()) return;
if (--it->second.refs <= 0 && it->second.unloaded) {
- ma_free(it->second.pcm, nullptr);
+ it->second.free();
buffers.erase(it);
}
}
@@ -219,7 +228,7 @@ void AudioEngine::shutdown() {
if (impl_->ready) ma_engine_uninit(&impl_->engine);
closeDevice(impl_);
if (impl_->contextReady) ma_context_uninit(&impl_->context);
- for (auto& [id, buf] : impl_->buffers) ma_free(buf.pcm, nullptr);
+ for (auto& [id, buf] : impl_->buffers) buf.free();
impl_->buffers.clear();
delete impl_;
impl_ = nullptr;
@@ -227,12 +236,41 @@ void AudioEngine::shutdown() {
bool AudioEngine::ready() const { return impl_ && impl_->ready; }
-AudioEngine::Loaded AudioEngine::load(const uint8_t* bytes, size_t n) {
- const Loaded fail{-1, 0.0, 0};
+AudioEngine::Loaded AudioEngine::load(const uint8_t* bytes, size_t n, bool stream) {
+ const Loaded fail{-1, 0.0, 0, false};
if (!ready() || !bytes || n == 0) return fail;
// Decode straight to the engine's format so playback never resamples.
ma_decoder_config dcfg = ma_decoder_config_init(ma_format_f32, impl_->channels, impl_->sampleRate);
+
+ // Streamed: keep the file and hand the manager a view of it. A sound made from
+ // an ENCODED registration gets a decoder of its own over these bytes and reads
+ // it as the mixer asks, so what stays resident is the file — megabytes for a
+ // track whose PCM would be tens of them — and nothing is decoded up front,
+ // which is also the stall a long clip used to cost the frame that loaded it.
+ if (stream) {
+ // Opened once here for two answers: that it decodes at all, and how long it
+ // is. A file that fails either way falls through to the decoded path, which
+ // reports the failure the way it always has.
+ ma_decoder probe;
+ if (ma_decoder_init_memory(bytes, n, &dcfg, &probe) == MA_SUCCESS) {
+ ma_uint64 frames = 0;
+ if (ma_decoder_get_length_in_pcm_frames(&probe, &frames) != MA_SUCCESS) frames = 0;
+ ma_decoder_uninit(&probe);
+
+ const int id = impl_->nextBuffer++;
+ Buffer& buf = impl_->buffers[id];
+ buf.name = "buf://" + std::to_string(id);
+ buf.encoded.assign(bytes, bytes + n);
+ if (ma_resource_manager_register_encoded_data(
+ ma_engine_get_resource_manager(&impl_->engine), buf.name.c_str(),
+ buf.encoded.data(), buf.encoded.size()) == MA_SUCCESS) {
+ const double seconds = impl_->sampleRate ? static_cast<double>(frames) / impl_->sampleRate : 0.0;
+ return Loaded{id, seconds, static_cast<long long>(n), true};
+ }
+ impl_->buffers.erase(id);
+ }
+ }
ma_uint64 frames = 0;
void* pcm = nullptr;
if (ma_decode_memory(bytes, n, &dcfg, &frames, &pcm) != MA_SUCCESS || !pcm) return fail;
@@ -246,15 +284,15 @@ AudioEngine::Loaded AudioEngine::load(const uint8_t* bytes, size_t n) {
if (ma_resource_manager_register_decoded_data(
ma_engine_get_resource_manager(&impl_->engine), buf.name.c_str(),
pcm, frames, ma_format_f32, impl_->channels, impl_->sampleRate) != MA_SUCCESS) {
+ buf.free();
impl_->buffers.erase(id);
- ma_free(pcm, nullptr);
return fail;
}
const double duration = impl_->sampleRate ? static_cast<double>(frames) / impl_->sampleRate : 0.0;
const long long decodedBytes =
static_cast<long long>(frames) * impl_->channels * static_cast<long long>(sizeof(float));
- return Loaded{id, duration, decodedBytes};
+ return Loaded{id, duration, decodedBytes, false};
}
void AudioEngine::unload(int bufferId) {
@@ -267,7 +305,7 @@ void AudioEngine::unload(int bufferId) {
ma_engine_get_resource_manager(&impl_->engine), it->second.name.c_str());
it->second.unloaded = true;
if (it->second.refs <= 0) {
- ma_free(it->second.pcm, nullptr);
+ it->second.free();
impl_->buffers.erase(it);
}
}
@@ -281,7 +319,9 @@ int AudioEngine::play(int bufferId, float volume, float pan, bool loop, float ra
Voice& voice = impl_->voices[id]; // node-stable: &voice.sound is fixed
voice.bufferId = bufferId;
voice.loop = loop;
- // Data is already decoded + registered, so init just wraps it (no decode).
+ // The data is registered under the name, so init wraps it: the PCM as it is,
+ // or — for a streamed clip — a decoder over the file that this voice alone
+ // reads, which is why two voices of one track do not share a cursor.
// NO_SPATIALIZATION: 2D games pan explicitly; the 3D positioner would fight it.
if (ma_sound_init_from_file(&impl_->engine, bit->second.name.c_str(),
MA_SOUND_FLAG_NO_SPATIALIZATION, nullptr, nullptr,
diff --git a/native/host/media/native_audio.hpp b/native/host/media/native_audio.hpp
index 8a2d507ef..78ed24a01 100644
--- a/native/host/media/native_audio.hpp
+++ b/native/host/media/native_audio.hpp
@@ -9,8 +9,11 @@
* The SDK's NativeAudioBackend is a thin shell over the es_audio*
* bindings, which in turn are a thin shell over this.
*
- * A buffer id names a decoded clip; a voice id names one playing
- * instance. The miniaudio header is kept out of this interface (pimpl),
+ * A buffer id names a loaded clip; a voice id names one playing
+ * instance. A clip is held one of two ways: decoded to PCM, which starts
+ * with no delay and costs its whole length in memory (0.37 MB a second),
+ * or streamed, which keeps the compressed file and decodes each voice as
+ * it plays — what music and anything else long wants. The miniaudio header is kept out of this interface (pimpl),
* so host_core.cpp does not pay to compile it.
*/
#pragma once
@@ -36,11 +39,16 @@ public:
struct Loaded {
int id; ///< < 0 on decode failure.
- double duration; ///< Seconds.
- long long bytes; ///< Decoded PCM size, for the residency budget.
+ double duration; ///< Seconds; 0 when a streamed file does not say.
+ long long bytes; ///< What stays resident: the PCM, or the compressed file when streamed.
+ bool streamed; ///< Held compressed and decoded while it plays.
};
- /** Decode a compressed clip to PCM and register it. */
- Loaded load(const uint8_t* bytes, size_t n);
+ /**
+ * Register a compressed clip: decoded to PCM now, or with @p stream kept as it
+ * is and decoded by each voice as it plays. A file that cannot be streamed is
+ * decoded instead, and `streamed` says which happened.
+ */
+ Loaded load(const uint8_t* bytes, size_t n, bool stream = false);
void unload(int bufferId);
/**
diff --git a/sdk/etc/index.native.api.md b/sdk/etc/index.native.api.md
index c90c1f059..e2b061e73 100644
--- a/sdk/etc/index.native.api.md
+++ b/sdk/etc/index.native.api.md
@@ -6712,7 +6712,7 @@ width: number
## NativeHostBindings — interface @experimental
```
-es_audioLoad: ((bytes: ArrayBuffer) => { id: number; duration: number; bytes: number; } | null) | undefined
+es_audioLoad: ((bytes: ArrayBuffer, stream?: boolean) => { id: number; duration: number; bytes: number; streamed?: boolean; } | null) | undefined
es_audioPause: ((voiceId: number) => void) | undefined
es_audioPlay: ((bufferId: number, volume: number, pan: number, loop: boolean, rate: number) => number) | undefined
es_audioResume: ((voiceId: number) => void) | undefined
diff --git a/sdk/src/audio/Audio.ts b/sdk/src/audio/Audio.ts
index 1aaff8f13..be919c87b 100644
--- a/sdk/src/audio/Audio.ts
+++ b/sdk/src/audio/Audio.ts
@@ -131,6 +131,10 @@ export class AudioAPI {
private readonly bufferSources_ = new Map<string, SharedAudioBuffer>();
/** refCount==0 urls in eviction order (oldest first); access re-appends. */
private readonly evictOrder_ = new Set<string>();
+ /** Urls a play is waiting on, by how many: not evictable until that play has started. */
+ private readonly awaitingPlay_ = new Map<string, number>();
+ /** Clips already reported as larger than the whole budget. */
+ private readonly oversizeReported_ = new Set<string>();
private residentBytes_ = 0;
private bufferBudgetOverride_: number | null = null;
/** Importer choices by authored path; a clip with none is `auto`. */
@@ -317,11 +321,28 @@ export class AudioAPI {
if (!source.charged) {
source.charged = true;
this.residentBytes_ += handle.bytes ?? 0;
+ this.reportOversize_(url, handle);
this.enforceBudget_();
}
this.evictOrder_.add(url);
}
+ /**
+ * A clip that is larger than the whole budget cannot be kept warm: making room for it empties the cache,
+ * and the next clip to load evicts it in turn, so it is decoded again every time it is played. Nothing
+ * about that is visible from the game — the sound just starts late, or not at all — so it is said once.
+ */
+ private reportOversize_(url: string, handle: AudioBufferHandle): void {
+ const bytes = handle.bytes ?? 0;
+ const budget = this.bufferBudget;
+ if (budget === 0 || bytes <= budget || this.oversizeReported_.has(url)) return;
+ this.oversizeReported_.add(url);
+ const mb = (n: number): string => (n / (1024 * 1024)).toFixed(1);
+ log.warn('audio', `${url} takes ${mb(bytes)} MB decoded, more than the whole ${mb(budget)} MB audio cache `
+ + '(audioCacheBudget): it is dropped as soon as another clip loads and decoded again on its next play. '
+ + "Set its import to 'streaming', or raise the budget");
+ }
+
/** Cache lookup for playback: refreshes the LRU position of an evictable
* entry so frequently-played warm sounds aren't the first to go. */
private lookupBuffer_(url: string): AudioBufferHandle | undefined {
@@ -362,6 +383,8 @@ export class AudioAPI {
if (budget === 0) return;
for (const url of this.evictOrder_) {
if (this.residentBytes_ <= budget) break;
+ // A play is about to start on it: evicting it now is a sound that never happens.
+ if (this.awaitingPlay_.has(url)) continue;
const entry = this.bufferCache_.get(url);
if (entry) this.freeBuffer_(url, entry);
}
@@ -451,6 +474,29 @@ export class AudioAPI {
await this.ensureBuffer_(url, source => (options ? this.backend_.loadBuffer(source, options) : this.backend_.loadBuffer(source)));
}
+ /**
+ * Load `url` for a play that is waiting on it and start that play, as one step: the clip cannot be
+ * evicted between arriving and being played, which is what a load finishing beside it used to do to a
+ * clip larger than the budget. Says so when the clip is not there to play after all, instead of the
+ * silence that left a game with no music and nothing in its log.
+ */
+ private async loadThenPlay_(url: string, what: string, play: (buffer: AudioBufferHandle) => void): Promise<void> {
+ this.awaitingPlay_.set(url, (this.awaitingPlay_.get(url) ?? 0) + 1);
+ try {
+ await this.preload(url);
+ if (this.disposed_) return;
+ const buffer = this.lookupBuffer_(url);
+ if (buffer) play(buffer);
+ else log.warn('audio', `${what} did not play: ${url} loaded, but was gone from the audio cache before it could start`);
+ } catch (err) {
+ log.warn('audio', `Failed to preload ${what}: ${url}`, err);
+ } finally {
+ const left = (this.awaitingPlay_.get(url) ?? 1) - 1;
+ if (left > 0) this.awaitingPlay_.set(url, left);
+ else this.awaitingPlay_.delete(url);
+ }
+ }
+
async preloadAll(urls: string[]): Promise<void> {
await Promise.all(urls.map(url => this.preload(url)));
}
@@ -506,14 +552,9 @@ export class AudioAPI {
const buffer = this.lookupBuffer_(url);
if (!buffer) {
const pending = this.createDeferredHandle_(playConfig.volume ?? 1);
- this.preload(url).then(() => {
- if (this.disposed_ || epoch !== this.playbackEpoch_ || pending.cancelled) return;
- const buf = this.lookupBuffer_(url);
- if (buf) {
- pending.resolve(overrides => this.playVoice_(buf, { ...playConfig, ...overrides }));
- }
- }).catch(err => {
- log.warn('audio', `Failed to preload audio: ${url}`, err);
+ void this.loadThenPlay_(url, 'audio', buf => {
+ if (epoch !== this.playbackEpoch_ || pending.cancelled) return;
+ pending.resolve(overrides => this.playVoice_(buf, { ...playConfig, ...overrides }));
});
return pending;
}
@@ -556,15 +597,7 @@ export class AudioAPI {
if (buffer) {
play(buffer);
} else {
- this.preload(url).then(() => {
- if (this.disposed_) return;
- const buf = this.lookupBuffer_(url);
- if (buf) {
- play(buf);
- }
- }).catch(err => {
- log.warn('audio', `Failed to preload BGM: ${url}`, err);
- });
+ void this.loadThenPlay_(url, 'BGM', play);
}
}
diff --git a/sdk/src/audio/NativeAudioBackend.ts b/sdk/src/audio/NativeAudioBackend.ts
index e7f00150f..dd3193dc0 100644
--- a/sdk/src/audio/NativeAudioBackend.ts
+++ b/sdk/src/audio/NativeAudioBackend.ts
@@ -10,6 +10,10 @@
* JS — the no-JIT budget forbids it. `mixer` is therefore null (no JS
* DSP graph), like WeChat; per-voice volume / pan / rate / loop are the
* full control surface.
+ *
+ * A clip the importer (or its length, under `auto`) says to stream is
+ * handed to the host to keep compressed: the rule is audioStreaming.ts,
+ * the one the web backend follows, so the two agree on which clips.
*/
import type { AudioMixer } from './AudioMixer';
import type {
@@ -17,6 +21,8 @@ import type {
PlatformAudioBackend, PlayConfig,
} from './PlatformAudioBackend';
import type { NativeAudioBridge } from '../platform/native/bridge';
+import type { AudioLoadOptions } from './PlatformAudioBackend';
+import { decodedPcmBytes, shouldStream } from './audioStreaming';
import { platformReadFile } from '../platform/base';
import { scalar } from '../math/scalar';
@@ -82,21 +88,27 @@ export class NativeAudioBackend implements PlatformAudioBackend {
async initialize(_options?: AudioBackendInitOptions): Promise<void> {}
async ensureResumed(): Promise<void> {}
- async loadBuffer(url: string): Promise<AudioBufferHandle> {
+ async loadBuffer(url: string, options?: AudioLoadOptions): Promise<AudioBufferHandle> {
// On native the audio ref resolver hands back a packaged path (not bytes),
// so read the file through the same platform channel as every other asset,
// then let the native engine decode it.
- return this.loadFromBytes_(await platformReadFile(url));
+ return this.loadFromBytes_(await platformReadFile(url), options);
}
- async loadBufferFromData(_url: string, data: ArrayBuffer): Promise<AudioBufferHandle> {
- return this.loadFromBytes_(data);
+ async loadBufferFromData(_url: string, data: ArrayBuffer, options?: AudioLoadOptions): Promise<AudioBufferHandle> {
+ return this.loadFromBytes_(data, options);
}
- private loadFromBytes_(bytes: ArrayBuffer): AudioBufferHandle {
- const decoded = this.audio_.load(bytes);
- if (!decoded) throw new Error('[native] audio decode failed');
- return { id: decoded.id, duration: decoded.duration, bytes: decoded.bytes };
+ private loadFromBytes_(bytes: ArrayBuffer, options?: AudioLoadOptions): AudioBufferHandle {
+ const loaded = this.audio_.load(bytes, shouldStream(options?.loadType, bytes));
+ if (!loaded) throw new Error('[native] audio decode failed');
+ // What the host DID, not what was asked: one that could not stream the file decoded it, and its PCM
+ // is then what counts against the budget.
+ if (!loaded.streamed) return { id: loaded.id, duration: loaded.duration, bytes: loaded.bytes };
+ return {
+ id: loaded.id, duration: loaded.duration, bytes: loaded.bytes, streamed: true,
+ savedBytes: Math.max(0, decodedPcmBytes(loaded.duration) - loaded.bytes),
+ };
}
unloadBuffer(handle: AudioBufferHandle): void {
diff --git a/sdk/src/audio/PlatformAudioBackend.ts b/sdk/src/audio/PlatformAudioBackend.ts
index d6ab69eb8..dbb15c52a 100644
--- a/sdk/src/audio/PlatformAudioBackend.ts
+++ b/sdk/src/audio/PlatformAudioBackend.ts
@@ -20,8 +20,9 @@ export interface AudioHandle {
/**
* How a clip is held: `decompress` decodes it to PCM at load, `streaming` plays it from its compressed
- * file, `auto` lets its length decide (see audioStreaming.ts). A backend with one way of playing a clip
- * ignores it.
+ * file, `auto` lets its length decide (see audioStreaming.ts) — the same rule on the web and on a native
+ * host, so a track that streams in the editor streams on the phone. A backend with one way of playing a
+ * clip ignores it.
*/
export type AudioLoadType = 'auto' | 'decompress' | 'streaming';
@@ -32,7 +33,11 @@ export interface AudioLoadOptions {
export interface AudioBufferHandle {
readonly id: number;
readonly duration: number;
- /** Played from the compressed file, not decoded: `bytes` is then 0 and `savedBytes` is what decoding would have cost. */
+ /**
+ * Played from the compressed file, not decoded. `bytes` is then what the backend still keeps of it — nothing
+ * where the host's media element reads the file (web), the file itself where the engine decodes from memory
+ * (native) — and `savedBytes` is what decoding would have cost on top of that.
+ */
readonly streamed?: boolean;
/** The PCM a streamed clip did not take, estimated from its length. */
readonly savedBytes?: number;
diff --git a/sdk/src/platform/native/bridge.ts b/sdk/src/platform/native/bridge.ts
index 85ac46955..194c4e660 100644
--- a/sdk/src/platform/native/bridge.ts
+++ b/sdk/src/platform/native/bridge.ts
@@ -128,17 +128,22 @@ export interface NativeTextEditorBridge {
* when the host bound the audio primitives; a host without a sound device omits
* `NativeBridge.audio` and the audio system stays silent (the Null backend).
*
- * A `bufferId` names a decoded clip; a `voiceId` names one playing instance of
+ * A `bufferId` names a loaded clip; a `voiceId` names one playing instance of
* it. Both are non-negative host handles; a failed load or play returns null / -1.
*/
export interface NativeAudioBridge {
/** `bins` byte magnitudes of what is playing (0..Nyquist), or null when the
* host has no analyser / nothing has played. The device's AnalyserNode. */
spectrum?(bins: number): ArrayBuffer | null;
- /** Decode + register a clip from its (compressed) file bytes. Returns the
- * buffer handle, its length in seconds and its decoded size in bytes (for the
- * residency budget), or null on decode failure. */
- load(bytes: ArrayBuffer): { id: number; duration: number; bytes: number } | null;
+ /**
+ * Register a clip from its (compressed) file bytes: decoded to PCM now, or —
+ * with `stream` — kept as the file and decoded by each voice as it plays.
+ * Returns the buffer handle, its length in seconds (0 when a streamed file does
+ * not say) and what stays resident in bytes, for the residency budget; null on
+ * decode failure. `streamed` is what the host did: one that cannot stream the
+ * file, or predates the argument, decodes it and leaves this unset.
+ */
+ load(bytes: ArrayBuffer, stream?: boolean): { id: number; duration: number; bytes: number; streamed?: boolean } | null;
unload(bufferId: number): void;
/** Start a voice on a loaded buffer. Returns its voice id, or -1 if the buffer
* is unknown. */
diff --git a/sdk/src/platform/native/hostBridge.ts b/sdk/src/platform/native/hostBridge.ts
index 970db701d..bdc76b1ff 100644
--- a/sdk/src/platform/native/hostBridge.ts
+++ b/sdk/src/platform/native/hostBridge.ts
@@ -137,7 +137,7 @@ export interface NativeHostBindings extends NativeDeviceHost {
es_textEditor_write?(value: string, selectionStart: number, selectionEnd: number): void;
es_audioSpectrum?(bins: number): ArrayBuffer | null;
- es_audioLoad?(bytes: ArrayBuffer): { id: number; duration: number; bytes: number } | null;
+ es_audioLoad?(bytes: ArrayBuffer, stream?: boolean): { id: number; duration: number; bytes: number; streamed?: boolean } | null;
es_audioUnload?(bufferId: number): void;
es_audioPlay?(bufferId: number, volume: number, pan: number, loop: boolean, rate: number): number;
es_audioStop?(voiceId: number): void;
@@ -379,7 +379,7 @@ function hostAudio(
if (failed) throw failure;
};
return {
- load: (bytes) => bindings.es_audioLoad!(bytes),
+ load: (bytes, stream) => bindings.es_audioLoad!(bytes, stream === true),
unload: (id) => bindings.es_audioUnload!(id),
play: (buf, vol, pan, loop, rate) => bindings.es_audioPlay!(buf, vol, pan, loop, rate),
stop: (v) => bindings.es_audioStop!(v),
diff --git a/sdk/tests/audio-residency.test.ts b/sdk/tests/audio-residency.test.ts
index c7d2cebd5..f25462a64 100644
--- a/sdk/tests/audio-residency.test.ts
+++ b/sdk/tests/audio-residency.test.ts
@@ -9,6 +9,7 @@
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { AudioAPI } from '../src/audio/Audio';
+import { log } from '../src/util/logger';
import { Assets } from '../src/asset/Assets';
import type { Backend } from '../src/asset/Backend';
import type { PlatformAudioBackend, AudioBufferHandle, AudioHandle } from '../src/audio/PlatformAudioBackend';
@@ -315,3 +316,65 @@ describe('AudioAPI buffer residency', () => {
expect(audio.getBufferStats().evictableCount).toBe(1);
});
});
+
+// A three-minute track decoded is about 64 MB and the default cache is 32: the clip is larger than the whole
+// budget, so whatever loads next evicts it. A play that was waiting on it then found nothing and said nothing (#122).
+describe('a clip larger than the whole audio cache', () => {
+ function sized(bytes: Record<string, number>) {
+ const backend = createMockBackend();
+ const played: number[] = [];
+ let nextId = 100;
+ backend.loadBuffer = async (url: string): Promise<AudioBufferHandle> => {
+ backend.loads.push(url);
+ return { id: ++nextId, duration: 1, bytes: bytes[url] ?? 10 };
+ };
+ const play = backend.play;
+ backend.play = (buffer, config) => { played.push(buffer.id); return play(buffer, config); };
+ const audio = new AudioAPI(backend);
+ audio.setBufferBudget(32);
+ return { audio, backend, played };
+ }
+
+ it('still plays when other clips finish loading beside it', async () => {
+ const { audio, backend, played } = sized({ 'music/theme.mp3': 64 });
+ const warn = vi.spyOn(log, 'warn').mockImplementation(() => {});
+ audio.playBGM('music/theme.mp3');
+ await Promise.all(['sfx/a.wav', 'sfx/b.wav', 'sfx/c.wav'].map((url) => audio.preload(url)));
+ await new Promise((r) => setTimeout(r, 0));
+ const said = warn.mock.calls.map((c) => String(c[1]));
+ warn.mockRestore();
+ expect(backend.loads).toContain('music/theme.mp3');
+ expect(played).toHaveLength(1);
+ expect(said.filter((m) => m.includes('did not play'))).toEqual([]);
+ // Said once, by name and with the numbers: it will be decoded again on its next play.
+ expect(said.filter((m) => m.includes('more than the whole'))).toHaveLength(1);
+ expect(said.find((m) => m.includes('more than the whole'))).toMatch(/music\/theme\.mp3.*streaming/);
+ });
+
+ it('is evictable again once its play has started', async () => {
+ const { audio } = sized({ 'music/theme.mp3': 64 });
+ const warn = vi.spyOn(log, 'warn').mockImplementation(() => {});
+ audio.playBGM('music/theme.mp3');
+ await new Promise((r) => setTimeout(r, 0));
+ expect(audio.getBufferHandle('music/theme.mp3')).toBeDefined();
+ await audio.preload('sfx/a.wav');
+ warn.mockRestore();
+ // The waiting play is what held it; the cache is back under its budget now.
+ expect(audio.getBufferHandle('music/theme.mp3')).toBeUndefined();
+ expect(audio.getBufferStats().bufferBytes).toBeLessThanOrEqual(32);
+ });
+
+ it('says nothing about a streamed clip, which keeps only its file', async () => {
+ const { audio, backend } = sized({});
+ backend.loadBuffer = async (url: string): Promise<AudioBufferHandle> => (url.startsWith('music/')
+ ? { id: 7, duration: 175, bytes: 3, streamed: true, savedBytes: 64 } : { id: 8 + url.length, duration: 1, bytes: 10 });
+ const warn = vi.spyOn(log, 'warn').mockImplementation(() => {});
+ audio.playBGM('music/theme.mp3');
+ await Promise.all(['sfx/a.wav', 'sfx/b.wav'].map((url) => audio.preload(url)));
+ await new Promise((r) => setTimeout(r, 0));
+ expect(warn).not.toHaveBeenCalled();
+ warn.mockRestore();
+ expect(audio.getBufferHandle('music/theme.mp3')).toBeDefined();
+ expect(audio.getBufferStats()).toMatchObject({ streamedCount: 1, streamedSavedBytes: 64 });
+ });
+});
diff --git a/sdk/tests/native-audio.test.ts b/sdk/tests/native-audio.test.ts
index fe1ac9d43..f1d057dd0 100644
--- a/sdk/tests/native-audio.test.ts
+++ b/sdk/tests/native-audio.test.ts
@@ -311,3 +311,55 @@ describe('assertNativeHost — audio stays optional', () => {
expect(() => assertNativeHost(scope)).not.toThrow();
});
});
+
+// The same rule the web backend follows (audioStreaming.ts), so a track that streams in the editor streams on the
+// phone: before this the native host decoded every clip whole, 0.37 MB a second (#122).
+describe('NativeAudioBackend streaming', () => {
+ /** A PCM WAV header that says `seconds`, with one byte a second so the file stays small. */
+ function wav(seconds: number): ArrayBuffer {
+ const view = new DataView(new ArrayBuffer(44 + seconds));
+ view.setUint32(0, 0x52494646, false); view.setUint32(8, 0x57415645, false);
+ view.setUint32(12, 0x666d7420, false); view.setUint32(16, 16, true);
+ view.setUint32(28, 1, true); // byte rate
+ view.setUint32(36, 0x64617461, false); view.setUint32(40, seconds, true);
+ return view.buffer;
+ }
+
+ function streamingHost(canStream = true) {
+ const scope = audioScope();
+ const asked: boolean[] = [];
+ scope.es_audioLoad = (bytes: ArrayBuffer, stream?: boolean) => {
+ asked.push(stream === true);
+ return stream && canStream
+ ? { id: 1, duration: 175, bytes: bytes.byteLength, streamed: true }
+ : { id: 1, duration: 175, bytes: 67_057_376 };
+ };
+ return { backend: new NativeAudioBackend(createHostBridge(scope).audio!), asked };
+ }
+
+ it('hands the host a clip the importer says to stream, and reports what that saved', async () => {
+ const { backend, asked } = streamingHost();
+ const data = new ArrayBuffer(2_400_000);
+ const buffer = await backend.loadBufferFromData('music', data, { loadType: 'streaming' });
+ expect(asked).toEqual([true]);
+ // What stays resident is the file; the PCM it did not take is the rest of what decoding would cost.
+ expect(buffer).toMatchObject({ streamed: true, bytes: 2_400_000, duration: 175 });
+ expect(buffer.savedBytes).toBe(175 * 48000 * 2 * 4 - 2_400_000);
+ });
+
+ it('lets the length decide under auto, and decodes what the importer says to decode', async () => {
+ const { backend, asked } = streamingHost();
+ await backend.loadBufferFromData('long', wav(40));
+ await backend.loadBufferFromData('short', wav(5));
+ await backend.loadBufferFromData('forced', wav(40), { loadType: 'decompress' });
+ expect(asked).toEqual([true, false, false]);
+ });
+
+ it('reports a decoded clip as decoded when the host could not stream it', async () => {
+ const { backend, asked } = streamingHost(false);
+ const buffer = await backend.loadBufferFromData('music', new ArrayBuffer(2_400_000), { loadType: 'streaming' });
+ expect(asked).toEqual([true]);
+ expect(buffer.streamed).toBeUndefined();
+ expect(buffer.bytes).toBe(67_057_376);
+ });
+});