A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
* @file native_audio.cpp
* @brief AudioEngine (see native_audio.hpp), backed by miniaudio.
* @details This TU owns the miniaudio implementation (like host_core.cpp owns
* stb_image). On iOS it is compiled as Objective-C++ (the CMake build
* adds `-x objective-c++`) because miniaudio's CoreAudio backend touches
* AVAudioSession; on Android it is plain C++ and miniaudio dlopens
* AAudio/OpenSL at runtime. The Obj-C paths are __APPLE__-guarded inside
* miniaudio, so the same source compiles on both.
*/
#define MINIAUDIO_IMPLEMENTATION
#define MA_NO_ENCODING // playback only; no encoders needed
#include "miniaudio.h"
#include "native_audio.hpp"
#include "audio_spectrum.hpp"
#include "audio_watchdog.hpp"
#include <atomic>
#include <chrono>
#include <string>
#include <unordered_map>
#include <vector>
namespace eshost {
// A loaded clip: its PCM, or — streamed — its compressed file, which every voice
// decodes for itself as it plays. Neither register_decoded_data() nor
// register_encoded_data() copies, so the data must outlive every voice playing it
// — freed only once unloaded AND no voice remains (`refs` == 0), or a
// still-playing sound would read freed memory.
struct Buffer {
std::string name; // "buf://<id>", the resource-manager key
void* pcm = nullptr; // owned; f32 interleaved at the engine's rate/channels
std::vector<uint8_t> encoded; // streamed: the file as it was loaded
int refs = 0; // live voices playing this buffer
bool unloaded = false; // SDK dropped it; free when the last voice releases
void free() {
if (pcm) ma_free(pcm, nullptr);
pcm = nullptr;
encoded = {};
}
};
// One playing instance. ma_sound is a node in the engine graph and must not move,
// so voices live in a node-stable unordered_map (never reallocated on rehash).
struct Voice {
ma_sound sound{};
int bufferId = -1;
bool loop = false;
};
// The analyser's tap: a node the voices play into, which passes their frames
// through unchanged and keeps a copy for the FFT. Attaching voices HERE rather
// than straight to the endpoint is what gives the game a spectrum without the
// audio thread ever waiting on it (AudioSpectrum::write is wait-free).
struct SpectrumTap {
ma_node_base base{};
AudioSpectrum* spectrum = nullptr;
ma_uint32 channels = 2;
};
void spectrumTapProcess(ma_node* node, const float** framesIn, ma_uint32* frameCountIn,
float** framesOut, ma_uint32* frameCountOut) {
auto* tap = reinterpret_cast<SpectrumTap*>(node);
const ma_uint32 frames = frameCountIn && frameCountOut
? (*frameCountIn < *frameCountOut ? *frameCountIn : *frameCountOut) : 0;
if (frames > 0 && framesIn && framesIn[0] && framesOut && framesOut[0]) {
const size_t samples = (size_t)frames * tap->channels;
for (size_t i = 0; i < samples; i++) framesOut[0][i] = framesIn[0][i];
if (tap->spectrum) tap->spectrum->write(framesIn[0], frames, tap->channels);
}
if (frameCountOut) *frameCountOut = frames;
if (frameCountIn) *frameCountIn = frames;
}
struct AudioEngine::Impl {
ma_context context{};
ma_device device{};
bool deviceReady = false;
std::atomic<uint64_t> callbacks{0};
AudioWatchdog watchdog;
ma_engine engine{};
SpectrumTap tap{};
bool tapReady = false;
AudioSpectrum spectrum;
bool contextReady = false;
bool ready = false;
ma_uint32 sampleRate = 0;
ma_uint32 channels = 0;
std::unordered_map<int, Buffer> buffers;
std::unordered_map<int, Voice> voices;
int nextBuffer = 1;
int nextVoice = 1;
ma_sound* soundOf(int voiceId) {
auto it = voices.find(voiceId);
return it == voices.end() ? nullptr : &it->second.sound;
}
// Drop one reference to a buffer; free its PCM once it is both unloaded and
// unreferenced. The resource-manager registration is released at unload().
void releaseBuffer(int bufferId) {
auto it = buffers.find(bufferId);
if (it == buffers.end()) return;
if (--it->second.refs <= 0 && it->second.unloaded) {
it->second.free();
buffers.erase(it);
}
}
void destroyVoice(int voiceId) {
auto it = voices.find(voiceId);
if (it == voices.end()) return;
const int bufferId = it->second.bufferId;
ma_sound_uninit(&it->second.sound);
voices.erase(it);
releaseBuffer(bufferId);
}
};
namespace {
double nowMs() {
return std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now().time_since_epoch()).count();
}
void deviceData(ma_device* device, void* out, const void*, ma_uint32 frames) {
auto* impl = static_cast<AudioEngine::Impl*>(device->pUserData);
ma_engine_read_pcm_frames(&impl->engine, out, frames, nullptr);
impl->callbacks.fetch_add(1, std::memory_order_relaxed);
}
bool openDevice(AudioEngine::Impl* impl) {
ma_device_config cfg = ma_device_config_init(ma_device_type_playback);
cfg.playback.format = ma_format_f32;
cfg.playback.channels = impl->channels;
cfg.sampleRate = impl->sampleRate;
cfg.dataCallback = deviceData;
cfg.pUserData = impl;
impl->deviceReady = ma_device_init(&impl->context, &cfg, &impl->device) == MA_SUCCESS;
return impl->deviceReady;
}
void closeDevice(AudioEngine::Impl* impl) {
if (!impl->deviceReady) return;
ma_device_uninit(&impl->device);
impl->deviceReady = false;
}
} // namespace
AudioEngine::~AudioEngine() { shutdown(); }
bool AudioEngine::init() {
if (impl_) return impl_->ready;
impl_ = new Impl();
// Own the context so the iOS audio session is playback-only. miniaudio
// defaults to PlayAndRecord, which opens the microphone — that needs a usage
// permission and, on the simulator, deadlocks AURemoteIO's Initialize. A game
// only plays out; Playback also sounds through the silent switch. Ignored by
// the Android backends.
ma_context_config ctxCfg = ma_context_config_init();
ctxCfg.coreaudio.sessionCategory = ma_ios_session_category_playback;
if (ma_context_init(nullptr, 0, &ctxCfg, &impl_->context) != MA_SUCCESS) {
delete impl_;
impl_ = nullptr;
return false;
}
impl_->contextReady = true;
if (!openDevice(impl_)) {
ma_context_uninit(&impl_->context);
delete impl_;
impl_ = nullptr;
return false;
}
ma_engine_config cfg = ma_engine_config_init();
cfg.pContext = &impl_->context;
cfg.pDevice = &impl_->device;
if (ma_engine_init(&cfg, &impl_->engine) != MA_SUCCESS) {
closeDevice(impl_);
ma_context_uninit(&impl_->context);
delete impl_;
impl_ = nullptr;
return false;
}
impl_->ready = true;
impl_->sampleRate = ma_engine_get_sample_rate(&impl_->engine);
impl_->channels = ma_engine_get_channels(&impl_->engine);
impl_->watchdog.want(true, nowMs());
// The analyser tap, between the voices and the endpoint. Optional: a failure
// here costs the spectrum, not the sound.
static ma_node_vtable vtable{};
vtable.onProcess = spectrumTapProcess;
vtable.onGetRequiredInputFrameCount = nullptr;
vtable.inputBusCount = 1;
vtable.outputBusCount = 1;
vtable.flags = MA_NODE_FLAG_PASSTHROUGH;
ma_node_config tapCfg = ma_node_config_init();
tapCfg.vtable = &vtable;
tapCfg.pInputChannels = &impl_->channels;
tapCfg.pOutputChannels = &impl_->channels;
impl_->tap.spectrum = &impl_->spectrum;
impl_->tap.channels = impl_->channels;
if (ma_node_init(ma_engine_get_node_graph(&impl_->engine), &tapCfg, nullptr,
&impl_->tap.base) == MA_SUCCESS
&& ma_node_attach_output_bus(&impl_->tap.base, 0,
ma_engine_get_endpoint(&impl_->engine), 0) == MA_SUCCESS) {
impl_->tapReady = true;
}
return true;
}
void AudioEngine::shutdown() {
if (!impl_) return;
for (auto& [id, voice] : impl_->voices) ma_sound_uninit(&voice.sound);
impl_->voices.clear();
if (impl_->tapReady) { ma_node_uninit(&impl_->tap.base, nullptr); impl_->tapReady = false; }
if (impl_->deviceReady) ma_device_stop(&impl_->device);
if (impl_->ready) ma_engine_uninit(&impl_->engine);
closeDevice(impl_);
if (impl_->contextReady) ma_context_uninit(&impl_->context);
for (auto& [id, buf] : impl_->buffers) buf.free();
impl_->buffers.clear();
delete impl_;
impl_ = nullptr;
}
bool AudioEngine::ready() const { return impl_ && impl_->ready; }
AudioEngine::Loaded AudioEngine::load(const uint8_t* bytes, size_t n, bool stream) {
const Loaded fail{-1, 0.0, 0, false};
if (!ready() || !bytes || n == 0) return fail;
// Decode straight to the engine's format so playback never resamples.
ma_decoder_config dcfg = ma_decoder_config_init(ma_format_f32, impl_->channels, impl_->sampleRate);
// Streamed: keep the file and hand the manager a view of it. A sound made from
// an ENCODED registration gets a decoder of its own over these bytes and reads
// it as the mixer asks, so what stays resident is the file — megabytes for a
// track whose PCM would be tens of them — and nothing is decoded up front,
// which is also the stall a long clip used to cost the frame that loaded it.
if (stream) {
// Opened once here for two answers: that it decodes at all, and how long it
// is. A file that fails either way falls through to the decoded path, which
// reports the failure the way it always has.
ma_decoder probe;
if (ma_decoder_init_memory(bytes, n, &dcfg, &probe) == MA_SUCCESS) {
ma_uint64 frames = 0;
if (ma_decoder_get_length_in_pcm_frames(&probe, &frames) != MA_SUCCESS) frames = 0;
ma_decoder_uninit(&probe);
const int id = impl_->nextBuffer++;
Buffer& buf = impl_->buffers[id];
buf.name = "buf://" + std::to_string(id);
buf.encoded.assign(bytes, bytes + n);
if (ma_resource_manager_register_encoded_data(
ma_engine_get_resource_manager(&impl_->engine), buf.name.c_str(),
buf.encoded.data(), buf.encoded.size()) == MA_SUCCESS) {
const double seconds = impl_->sampleRate ? static_cast<double>(frames) / impl_->sampleRate : 0.0;
return Loaded{id, seconds, static_cast<long long>(n), true};
}
impl_->buffers.erase(id);
}
}
ma_uint64 frames = 0;
void* pcm = nullptr;
if (ma_decode_memory(bytes, n, &dcfg, &frames, &pcm) != MA_SUCCESS || !pcm) return fail;
// Insert first so the name registered with the manager is the map entry's own
// (node-stable) string — never a moved-from temporary whose SSO buffer relocates.
const int id = impl_->nextBuffer++;
Buffer& buf = impl_->buffers[id];
buf.name = "buf://" + std::to_string(id);
buf.pcm = pcm;
if (ma_resource_manager_register_decoded_data(
ma_engine_get_resource_manager(&impl_->engine), buf.name.c_str(),
pcm, frames, ma_format_f32, impl_->channels, impl_->sampleRate) != MA_SUCCESS) {
buf.free();
impl_->buffers.erase(id);
return fail;
}
const double duration = impl_->sampleRate ? static_cast<double>(frames) / impl_->sampleRate : 0.0;
const long long decodedBytes =
static_cast<long long>(frames) * impl_->channels * static_cast<long long>(sizeof(float));
return Loaded{id, duration, decodedBytes, false};
}
void AudioEngine::unload(int bufferId) {
if (!impl_) return;
auto it = impl_->buffers.find(bufferId);
if (it == impl_->buffers.end() || it->second.unloaded) return;
// Release the manager's own reference; live voices keep their own until they
// end, so the PCM is freed by releaseBuffer() once the last one does.
ma_resource_manager_unregister_data(
ma_engine_get_resource_manager(&impl_->engine), it->second.name.c_str());
it->second.unloaded = true;
if (it->second.refs <= 0) {
it->second.free();
impl_->buffers.erase(it);
}
}
double AudioEngine::time() const {
if (!ready()) return 0.0;
const ma_uint32 rate = ma_engine_get_sample_rate(&impl_->engine);
return rate > 0 ? static_cast<double>(ma_engine_get_time_in_pcm_frames(&impl_->engine)) / rate : 0.0;
}
int AudioEngine::play(int bufferId, float volume, float pan, bool loop, float rate, float loopStart, float start, float end, double at) {
if (!ready()) return -1;
auto bit = impl_->buffers.find(bufferId);
if (bit == impl_->buffers.end() || bit->second.unloaded) return -1;
const int id = impl_->nextVoice++;
Voice& voice = impl_->voices[id]; // node-stable: &voice.sound is fixed
voice.bufferId = bufferId;
voice.loop = loop;
// The data is registered under the name, so init wraps it: the PCM as it is,
// or — for a streamed clip — a decoder over the file that this voice alone
// reads, which is why two voices of one track do not share a cursor.
// NO_SPATIALIZATION: 2D games pan explicitly; the 3D positioner would fight it.
if (ma_sound_init_from_file(&impl_->engine, bit->second.name.c_str(),
MA_SOUND_FLAG_NO_SPATIALIZATION, nullptr, nullptr,
&voice.sound) != MA_SUCCESS) {
impl_->voices.erase(id);
return -1;
}
// Into the tap when there is one, so the analyser sees this voice; straight
// to the endpoint otherwise (identical audio either way).
if (impl_->tapReady) {
ma_node_attach_output_bus(&voice.sound, 0, &impl_->tap.base, 0);
}
ma_sound_set_volume(&voice.sound, volume);
ma_sound_set_pan(&voice.sound, pan);
ma_sound_set_pitch(&voice.sound, rate > 0 ? rate : 1.0f);
ma_sound_set_looping(&voice.sound, loop ? MA_TRUE : MA_FALSE);
if (start > 0.0f || end > 0.0f || (loop && loopStart > 0.0f)) {
// Places in the clip's own frames, which is what the data source counts in.
ma_uint32 sampleRate = 0;
ma_data_source* source = ma_sound_get_data_source(&voice.sound);
if (source && ma_sound_get_data_format(&voice.sound, nullptr, nullptr, &sampleRate, nullptr, 0) == MA_SUCCESS && sampleRate > 0) {
const auto frame = [sampleRate](float seconds) { return static_cast<ma_uint64>(static_cast<double>(seconds) * sampleRate); };
// Round from the loop start and not from the top, and over at the clip's end and not the file's.
if (loop && (loopStart > 0.0f || end > 0.0f)) {
ma_data_source_set_loop_point_in_pcm_frames(source, frame(loopStart), end > 0.0f ? frame(end) : ~static_cast<ma_uint64>(0));
}
// Begun past the silence encoding put before it.
if (start > 0.0f) ma_sound_seek_to_pcm_frame(&voice.sound, frame(start));
}
}
// Held until a moment of the engine's clock, where one is given: started now, it is silent until then.
if (at > 0.0) {
ma_sound_set_start_time_in_pcm_frames(
&voice.sound, static_cast<ma_uint64>(at * ma_engine_get_sample_rate(&impl_->engine)));
}
ma_sound_start(&voice.sound);
bit->second.refs++;
return id;
}
bool AudioEngine::spectrum(uint8_t* out, size_t bins) {
if (!impl_ || !impl_->tapReady) return false;
return impl_->spectrum.read(out, bins);
}
void AudioEngine::stop(int voiceId) {
if (impl_) impl_->destroyVoice(voiceId);
}
void AudioEngine::pause(int voiceId) {
// ma_sound_stop pauses in place (keeps the cursor); start() resumes.
if (ma_sound* s = impl_ ? impl_->soundOf(voiceId) : nullptr) ma_sound_stop(s);
}
void AudioEngine::resume(int voiceId) {
if (ma_sound* s = impl_ ? impl_->soundOf(voiceId) : nullptr) ma_sound_start(s);
}
void AudioEngine::setVolume(int voiceId, float volume) {
if (ma_sound* s = impl_ ? impl_->soundOf(voiceId) : nullptr) ma_sound_set_volume(s, volume);
}
void AudioEngine::setPan(int voiceId, float pan) {
if (ma_sound* s = impl_ ? impl_->soundOf(voiceId) : nullptr) ma_sound_set_pan(s, pan);
}
void AudioEngine::setLoop(int voiceId, bool loop) {
if (!impl_) return;
auto it = impl_->voices.find(voiceId);
if (it == impl_->voices.end()) return;
it->second.loop = loop;
ma_sound_set_looping(&it->second.sound, loop ? MA_TRUE : MA_FALSE);
}
void AudioEngine::setRate(int voiceId, float rate) {
if (ma_sound* s = impl_ ? impl_->soundOf(voiceId) : nullptr) {
ma_sound_set_pitch(s, rate > 0 ? rate : 1.0f);
}
}
AudioEngine::VoiceState AudioEngine::voiceState(int voiceId) {
if (!impl_) return {false, false, 0.0};
auto it = impl_->voices.find(voiceId);
if (it == impl_->voices.end()) return {false, false, 0.0};
float cursor = 0;
ma_sound_get_cursor_in_seconds(&it->second.sound, &cursor);
return {true, ma_sound_is_playing(&it->second.sound) == MA_TRUE, static_cast<double>(cursor)};
}
void AudioEngine::suspendAll() {
if (!ready()) return;
impl_->watchdog.want(false, nowMs());
if (impl_->deviceReady) ma_device_stop(&impl_->device);
}
void AudioEngine::resumeAll() {
if (!ready()) return;
impl_->watchdog.want(true, nowMs());
if (impl_->deviceReady) ma_device_start(&impl_->device);
}
AudioEngine::Health AudioEngine::maintain() {
Health health;
if (!ready()) return health;
const double now = nowMs();
const AudioWatchdog::Action action =
impl_->watchdog.tick(now, impl_->callbacks.load(std::memory_order_relaxed));
health.recovered = impl_->watchdog.takeRecovered();
if (action == AudioWatchdog::Action::None) return health;
health.attempt = impl_->watchdog.failures();
health.silentMs = impl_->watchdog.silentMs(now);
health.rebuilt = action == AudioWatchdog::Action::Rebuild || !impl_->deviceReady;
if (health.rebuilt) {
closeDevice(impl_);
health.started = openDevice(impl_) && ma_device_start(&impl_->device) == MA_SUCCESS;
} else {
ma_device_stop(&impl_->device);
health.started = ma_device_start(&impl_->device) == MA_SUCCESS;
}
return health;
}
void AudioEngine::debugFault(int kind) {
if (!ready()) return;
if (kind == 0) ma_device_stop(&impl_->device);
else closeDevice(impl_);
}
void AudioEngine::pumpEnded(const std::function<void(int)>& onEnded) {
if (!ready()) return;
std::vector<int> ended;
for (auto& [id, voice] : impl_->voices) {
if (!voice.loop && ma_sound_at_end(&voice.sound) == MA_TRUE) ended.push_back(id);
}
for (const int id : ended) {
onEnded(id); // may synchronously play() new voices — safe (iterating a copy)
impl_->destroyVoice(id);
}
}
} // namespace eshost