A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(audio): a device answers getSpectrum, so a visualiser has something to draw
esenginee128d53fa074
diff --git a/native/CMakeLists.txt b/native/CMakeLists.txtindex 2a8bf2895..2b6e3d601 100644--- a/native/CMakeLists.txt+++ b/native/CMakeLists.txt@@ -209,6 +209,7 @@ if(ESTELLA_QUICKJS_DIR)host/bindings/NetBindings.cpphost/bindings/TextEditorBindings.cpphost/media/native_audio.cpp+ host/media/audio_spectrum.cpphost/media/ktx2_decode.cpphost/media/glyph_raster.cpp"${BASIS_TRANSCODER}" "${BASIS_ZSTD}" "${NATIVE_BINDINGS}" "${NATIVE_FN_BINDINGS}")@@ -221,7 +222,8 @@ if(ESTELLA_QUICKJS_DIR)# be Objective-C++ on Apple; it also forces MA_NO_RUNTIME_LINKING there, so# the audio frameworks are linked at build time (the app links the merged# archive, so these also live in native/ios/project.yml).- set_source_files_properties(host/media/native_audio.cpp PROPERTIES+ set_source_files_properties(host/media/native_audio.cpp+ host/media/audio_spectrum.cpp PROPERTIESCOMPILE_OPTIONS "-x;objective-c++")target_link_libraries(estella_js_host esengine estella_physics estella_video estella_spine qjs"-framework UIKit" "-framework QuartzCore" "-framework Foundation"diff --git a/native/host/bindings/AudioBindings.cpp b/native/host/bindings/AudioBindings.cppindex a78e282b2..a14925c5c 100644--- a/native/host/bindings/AudioBindings.cpp+++ b/native/host/bindings/AudioBindings.cpp@@ -21,6 +21,8 @@*/#include "Bindings.hpp"+#include <vector>+using namespace esengine;namespace eshost {@@ -106,6 +108,18 @@ JSValue js_audioResumeAll(JSContext* ctx, JSValueConst, int, JSValueConst*) {} // namespace+// es_audioSpectrum(bins) -> ArrayBuffer of `bins` magnitude bytes, or null when+// nothing has played yet. The device's answer to a WebAudio AnalyserNode.+JSValue js_audioSpectrum(JSContext* ctx, JSValueConst, int argc, JSValueConst* argv) {+ if (argc < 1 || !hostAlive()) return JS_NULL;+ int32_t bins = 0;+ JS_ToInt32(ctx, &bins, argv[0]);+ if (bins <= 0 || bins > 4096) return JS_NULL;+ std::vector<uint8_t> out((size_t)bins, 0);+ if (!host().audio.spectrum(out.data(), out.size())) return JS_NULL;+ return JS_NewArrayBufferCopy(ctx, out.data(), out.size());+}+void registerAudioBindings(HostState& h, JSValue global) {if (!h.audio.ready()) return;bindGlobal(h, global, "es_audioLoad", js_audioLoad, 1);@@ -117,6 +131,7 @@ void registerAudioBindings(HostState& h, JSValue global) {bindGlobal(h, global, "es_audioSetVolume", js_audioSetVolume, 2);bindGlobal(h, global, "es_audioSetPan", js_audioSetPan, 2);bindGlobal(h, global, "es_audioSetLoop", js_audioSetLoop, 2);+ bindGlobal(h, global, "es_audioSpectrum", js_audioSpectrum, 1);bindGlobal(h, global, "es_audioSetRate", js_audioSetRate, 2);bindGlobal(h, global, "es_audioVoiceState", js_audioVoiceState, 1);bindGlobal(h, global, "es_audioSuspendAll", js_audioSuspendAll, 0);diff --git a/native/host/media/audio_spectrum.cpp b/native/host/media/audio_spectrum.cppnew file mode 100644index 000000000..3515b7a15--- /dev/null+++ b/native/host/media/audio_spectrum.cpp@@ -0,0 +1,105 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file audio_spectrum.cpp+ * @brief Implements the device's spectrum analyser (see audio_spectrum.hpp).+ */+#include "audio_spectrum.hpp"++#include <algorithm>+#include <cmath>++namespace eshost {+namespace {++constexpr float kPi = 3.14159265358979323846f;++/** In-place radix-2 Cooley-Tukey FFT. `n` is a power of two. */+void fft(float* re, float* im, size_t n) {+ // Bit-reversal permutation.+ for (size_t i = 1, j = 0; i < n; i++) {+ size_t bit = n >> 1;+ for (; j & bit; bit >>= 1) j ^= bit;+ j ^= bit;+ if (i < j) {+ std::swap(re[i], re[j]);+ std::swap(im[i], im[j]);+ }+ }+ for (size_t len = 2; len <= n; len <<= 1) {+ const float angle = -2.0f * kPi / (float)len;+ const float wr = std::cos(angle);+ const float wi = std::sin(angle);+ for (size_t i = 0; i < n; i += len) {+ float cr = 1.0f, ci = 0.0f;+ for (size_t k = 0; k < len / 2; k++) {+ const size_t a = i + k;+ const size_t b = a + len / 2;+ const float xr = re[b] * cr - im[b] * ci;+ const float xi = re[b] * ci + im[b] * cr;+ re[b] = re[a] - xr;+ im[b] = im[a] - xi;+ re[a] += xr;+ im[a] += xi;+ const float nr = cr * wr - ci * wi;+ ci = cr * wi + ci * wr;+ cr = nr;+ }+ }+ }+}++} // namespace++void AudioSpectrum::write(const float* frames, size_t frameCount, uint32_t channels) {+ if (!frames || frameCount == 0 || channels == 0) return;+ uint32_t index = writeIndex_.load(std::memory_order_relaxed);+ for (size_t f = 0; f < frameCount; f++) {+ float sum = 0.0f;+ for (uint32_t c = 0; c < channels; c++) sum += frames[f * channels + c];+ ring_[index & (kWindow - 1)] = sum / (float)channels;+ index++;+ }+ writeIndex_.store(index, std::memory_order_release);+ everWrote_.store(true, std::memory_order_release);+}++bool AudioSpectrum::read(uint8_t* out, size_t bins) {+ if (!out || bins == 0) return false;+ if (!everWrote_.load(std::memory_order_acquire)) return false;++ re_.assign(kWindow, 0.0f);+ im_.assign(kWindow, 0.0f);+ // Oldest-to-newest, Hann-windowed: the window kills the spectral leakage a+ // hard-cut buffer would smear across every bin.+ const uint32_t end = writeIndex_.load(std::memory_order_acquire);+ for (size_t i = 0; i < kWindow; i++) {+ const float sample = ring_[(end + i) & (kWindow - 1)];+ const float w = 0.5f * (1.0f - std::cos(2.0f * kPi * (float)i / (float)(kWindow - 1)));+ re_[i] = sample * w;+ }+ fft(re_.data(), im_.data(), kWindow);++ // Byte magnitudes over 0..Nyquist, in decibels — the scale the web's+ // getByteFrequencyData uses, so a visualizer looks the same on both.+ constexpr float kMinDb = -90.0f;+ constexpr float kMaxDb = -20.0f;+ const size_t usable = kWindow / 2;+ for (size_t b = 0; b < bins; b++) {+ // Each output bin averages the FFT bins it covers, so a short bar array+ // still reflects the whole band rather than sampling it.+ const size_t from = b * usable / bins;+ const size_t to = std::max(from + 1, (b + 1) * usable / bins);+ float sum = 0.0f;+ for (size_t k = from; k < to && k < usable; k++) {+ sum += std::sqrt(re_[k] * re_[k] + im_[k] * im_[k]);+ }+ const float mag = sum / (float)(to - from) / (float)(kWindow / 4);+ const float db = 20.0f * std::log10(std::max(mag, 1e-7f));+ const float t = (db - kMinDb) / (kMaxDb - kMinDb);+ out[b] = (uint8_t)std::clamp(t * 255.0f, 0.0f, 255.0f);+ }+ return true;+}++} // namespace eshostdiff --git a/native/host/media/audio_spectrum.hpp b/native/host/media/audio_spectrum.hppnew file mode 100644index 000000000..a835b09ab--- /dev/null+++ b/native/host/media/audio_spectrum.hpp@@ -0,0 +1,62 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file audio_spectrum.hpp+ * @brief The analyser behind `AudioAPI.getSpectrum()` on a device: a tap on+ * what the engine is about to play, and the FFT over it.+ * @details The web reads its spectrum from a WebAudio AnalyserNode. There is no+ * such node here, so the host provides the same answer the same way a+ * browser does — window the most recent frames, transform them, and+ * hand back byte magnitudes in the layout getByteFrequencyData uses.+ *+ * Thread-safe by design: the audio thread only ever WRITES into a ring+ * buffer, and the game thread only ever reads a snapshot of it, so no+ * lock sits in the audio callback (where one would be a dropout).+ *+ * @author ESEngine Team+ * @date 2026+ *+ * @copyright Copyright (c) 2026 ESEngine Team+ * Licensed under the Apache License, Version 2.0.+ */+#pragma once++#include <atomic>+#include <cstddef>+#include <cstdint>+#include <vector>++namespace eshost {++/**+ * A rolling window of the engine's output, and the FFT over it.+ *+ * `write` is called from the audio thread with interleaved frames; `read` is+ * called from the game thread and fills `out` with `out.size()` magnitude bytes+ * covering 0 .. Nyquist, exactly like the web's getByteFrequencyData.+ */+class AudioSpectrum {+public:+ /** Samples the FFT runs over. 1024 at 48 kHz is ~21 ms — long enough to+ * resolve a bass line, short enough to look live. */+ static constexpr size_t kWindow = 1024;++ /** Mix `frameCount` interleaved frames down to mono and keep the newest+ * window. Audio thread; wait-free. */+ void write(const float* frames, size_t frameCount, uint32_t channels);++ /** Fill `out` with byte magnitudes. False when nothing has played yet, so a+ * caller can tell "silent" from "unsupported". Game thread. */+ bool read(uint8_t* out, size_t bins);++private:+ /** Power-of-two ring of mono samples the audio thread writes into. */+ float ring_[kWindow]{};+ std::atomic<uint32_t> writeIndex_{0};+ std::atomic<bool> everWrote_{false};+ /** Scratch for the transform — game thread only, so no allocation per read. */+ std::vector<float> re_;+ std::vector<float> im_;+};++} // namespace eshostdiff --git a/native/host/media/native_audio.cpp b/native/host/media/native_audio.cppindex 28a25c3f2..6a69b0053 100644--- a/native/host/media/native_audio.cpp+++ b/native/host/media/native_audio.cpp@@ -16,6 +16,8 @@#include "native_audio.hpp"+#include "audio_spectrum.hpp"+#include <string>#include <unordered_map>#include <vector>@@ -40,9 +42,36 @@ struct Voice {bool loop = false;};+// The analyser's tap: a node the voices play into, which passes their frames+// through unchanged and keeps a copy for the FFT. Attaching voices HERE rather+// than straight to the endpoint is what gives the game a spectrum without the+// audio thread ever waiting on it (AudioSpectrum::write is wait-free).+struct SpectrumTap {+ ma_node_base base{};+ AudioSpectrum* spectrum = nullptr;+ ma_uint32 channels = 2;+};++void spectrumTapProcess(ma_node* node, const float** framesIn, ma_uint32* frameCountIn,+ float** framesOut, ma_uint32* frameCountOut) {+ auto* tap = reinterpret_cast<SpectrumTap*>(node);+ const ma_uint32 frames = frameCountIn && frameCountOut+ ? (*frameCountIn < *frameCountOut ? *frameCountIn : *frameCountOut) : 0;+ if (frames > 0 && framesIn && framesIn[0] && framesOut && framesOut[0]) {+ const size_t samples = (size_t)frames * tap->channels;+ for (size_t i = 0; i < samples; i++) framesOut[0][i] = framesIn[0][i];+ if (tap->spectrum) tap->spectrum->write(framesIn[0], frames, tap->channels);+ }+ if (frameCountOut) *frameCountOut = frames;+ if (frameCountIn) *frameCountIn = frames;+}+struct AudioEngine::Impl {ma_context context{};ma_engine engine{};+ SpectrumTap tap{};+ bool tapReady = false;+ AudioSpectrum spectrum;bool contextReady = false;bool ready = false;ma_uint32 sampleRate = 0;@@ -109,6 +138,27 @@ bool AudioEngine::init() {impl_->ready = true;impl_->sampleRate = ma_engine_get_sample_rate(&impl_->engine);impl_->channels = ma_engine_get_channels(&impl_->engine);++ // The analyser tap, between the voices and the endpoint. Optional: a failure+ // here costs the spectrum, not the sound.+ static ma_node_vtable vtable{};+ vtable.onProcess = spectrumTapProcess;+ vtable.onGetRequiredInputFrameCount = nullptr;+ vtable.inputBusCount = 1;+ vtable.outputBusCount = 1;+ vtable.flags = MA_NODE_FLAG_PASSTHROUGH;+ ma_node_config tapCfg = ma_node_config_init();+ tapCfg.vtable = &vtable;+ tapCfg.pInputChannels = &impl_->channels;+ tapCfg.pOutputChannels = &impl_->channels;+ impl_->tap.spectrum = &impl_->spectrum;+ impl_->tap.channels = impl_->channels;+ if (ma_node_init(ma_engine_get_node_graph(&impl_->engine), &tapCfg, nullptr,+ &impl_->tap.base) == MA_SUCCESS+ && ma_node_attach_output_bus(&impl_->tap.base, 0,+ ma_engine_get_endpoint(&impl_->engine), 0) == MA_SUCCESS) {+ impl_->tapReady = true;+ }return true;}@@ -116,6 +166,7 @@ void AudioEngine::shutdown() {if (!impl_) return;for (auto& [id, voice] : impl_->voices) ma_sound_uninit(&voice.sound);impl_->voices.clear();+ if (impl_->tapReady) { ma_node_uninit(&impl_->tap.base, nullptr); impl_->tapReady = false; }if (impl_->ready) ma_engine_uninit(&impl_->engine);if (impl_->contextReady) ma_context_uninit(&impl_->context);for (auto& [id, buf] : impl_->buffers) ma_free(buf.pcm, nullptr);@@ -188,6 +239,11 @@ int AudioEngine::play(int bufferId, float volume, float pan, bool loop, float raimpl_->voices.erase(id);return -1;}+ // Into the tap when there is one, so the analyser sees this voice; straight+ // to the endpoint otherwise (identical audio either way).+ if (impl_->tapReady) {+ ma_node_attach_output_bus(&voice.sound, 0, &impl_->tap.base, 0);+ }ma_sound_set_volume(&voice.sound, volume);ma_sound_set_pan(&voice.sound, pan);ma_sound_set_pitch(&voice.sound, rate > 0 ? rate : 1.0f);@@ -197,6 +253,11 @@ int AudioEngine::play(int bufferId, float volume, float pan, bool loop, float rareturn id;}+bool AudioEngine::spectrum(uint8_t* out, size_t bins) {+ if (!impl_ || !impl_->tapReady) return false;+ return impl_->spectrum.read(out, bins);+}+void AudioEngine::stop(int voiceId) {if (impl_) impl_->destroyVoice(voiceId);}diff --git a/native/host/media/native_audio.hpp b/native/host/media/native_audio.hppindex 98eba8a0e..4952f1d0a 100644--- a/native/host/media/native_audio.hpp+++ b/native/host/media/native_audio.hpp@@ -43,6 +43,13 @@ public:Loaded load(const uint8_t* bytes, size_t n);void unload(int bufferId);+ /**+ * Fill @p bins byte magnitudes of what is playing, 0..Nyquist — the device's+ * answer to a WebAudio AnalyserNode (see media/audio_spectrum.hpp). False+ * when nothing has played yet or the tap could not be built.+ */+ bool spectrum(uint8_t* out, size_t bins);+/** Start a voice on a loaded buffer. Returns its id, or -1 if unknown. */int play(int bufferId, float volume, float pan, bool loop, float rate);void stop(int voiceId);diff --git a/sdk/etc/index.native.api.md b/sdk/etc/index.native.api.mdindex 024c04b6a..bd6b98a58 100644--- a/sdk/etc/index.native.api.md+++ b/sdk/etc/index.native.api.md@@ -4135,6 +4135,7 @@ es_audioSetLoop: ((voiceId: number, loop: boolean) => void) | undefinedes_audioSetPan: ((voiceId: number, pan: number) => void) | undefinedes_audioSetRate: ((voiceId: number, rate: number) => void) | undefinedes_audioSetVolume: ((voiceId: number, volume: number) => void) | undefined+es_audioSpectrum: ((bins: number) => ArrayBuffer | null) | undefinedes_audioStop: ((voiceId: number) => void) | undefinedes_audioSuspendAll: (() => void) | undefinedes_audioUnload: ((bufferId: number) => void) | undefineddiff --git a/sdk/src/audio/NativeAudioBackend.ts b/sdk/src/audio/NativeAudioBackend.tsindex f5f123ed8..b0491a883 100644--- a/sdk/src/audio/NativeAudioBackend.ts+++ b/sdk/src/audio/NativeAudioBackend.ts@@ -65,6 +65,9 @@ export class NativeAudioBackend implements PlatformAudioBackend {readonly isReady = true;private readonly voices_ = new Map<number, NativeAudioHandle>();+ /** Reused across frames: a visualizer asks every frame, and a fresh view per+ * ask would be garbage the interpreter has to collect at 60 Hz. */+ private spectrumView_: Uint8Array | null = null;private readonly offEnded_: () => void;constructor(private readonly audio_: NativeAudioBridge) {@@ -115,6 +118,22 @@ export class NativeAudioBackend implements PlatformAudioBackend {return handle;}+ /**+ * The host's analyser, in the layout the web's getByteFrequencyData uses —+ * so a visualizer written against the web reads the same bytes here. False+ * when the host has no analyser tap, which the caller treats exactly as it+ * treats a backend with no analysis at all.+ */+ getFrequencyData(out: Uint8Array): boolean {+ const bytes = this.audio_.spectrum?.(out.length);+ if (!bytes) return false;+ if (!this.spectrumView_ || this.spectrumView_.buffer !== bytes) {+ this.spectrumView_ = new Uint8Array(bytes);+ }+ out.set(this.spectrumView_.subarray(0, out.length));+ return true;+ }+suspend(): void { this.audio_.suspendAll(); }resume(): void { this.audio_.resumeAll(); }diff --git a/sdk/src/platform/native/bridge.ts b/sdk/src/platform/native/bridge.tsindex ae23c0471..83d4dadab 100644--- a/sdk/src/platform/native/bridge.ts+++ b/sdk/src/platform/native/bridge.ts@@ -96,6 +96,9 @@ export interface NativeTextEditorBridge {* it. Both are non-negative host handles; a failed load or play returns null / -1.*/export interface NativeAudioBridge {+ /** `bins` byte magnitudes of what is playing (0..Nyquist), or null when the+ * host has no analyser / nothing has played. The device's AnalyserNode. */+ spectrum?(bins: number): ArrayBuffer | null;/** Decode + register a clip from its (compressed) file bytes. Returns the* buffer handle, its length in seconds and its decoded size in bytes (for the* residency budget), or null on decode failure. */diff --git a/sdk/src/platform/native/hostBridge.ts b/sdk/src/platform/native/hostBridge.tsindex 590f17b96..9e5fd177c 100644--- a/sdk/src/platform/native/hostBridge.ts+++ b/sdk/src/platform/native/hostBridge.ts@@ -87,6 +87,7 @@ export interface NativeHostBindings {es_textEditor_blur?(): void;es_textEditor_write?(value: string, selectionStart: number, selectionEnd: number): void;+ es_audioSpectrum?(bins: number): ArrayBuffer | null;es_audioLoad?(bytes: ArrayBuffer): { id: number; duration: number; bytes: number } | null;es_audioUnload?(bufferId: number): void;es_audioPlay?(bufferId: number, volume: number, pan: number, loop: boolean, rate: number): number;@@ -260,6 +261,12 @@ function hostAudio(suspendAll: () => bindings.es_audioSuspendAll!(),resumeAll: () => bindings.es_audioResumeAll!(),onEnded: (cb) => { ended = cb; return () => { if (ended === cb) ended = null; }; },+ // Optional: a host without the analyser tap simply has no spectrum, and+ // a visualizer falls back to flat bars (as it does on any backend+ // without analysis).+ ...(bindings.es_audioSpectrum+ ? { spectrum: (bins: number) => bindings.es_audioSpectrum!(bins) }+ : {}),};}