Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(audio): a device answers getSpectrum, so a visualiser has something to draw

esenginee128d53fa074
diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt
index 2a8bf2895..2b6e3d601 100644
--- a/native/CMakeLists.txt
+++ b/native/CMakeLists.txt
@@ -209,6 +209,7 @@ if(ESTELLA_QUICKJS_DIR)
host/bindings/NetBindings.cpp
host/bindings/TextEditorBindings.cpp
host/media/native_audio.cpp
+ host/media/audio_spectrum.cpp
host/media/ktx2_decode.cpp
host/media/glyph_raster.cpp
"${BASIS_TRANSCODER}" "${BASIS_ZSTD}" "${NATIVE_BINDINGS}" "${NATIVE_FN_BINDINGS}")
@@ -221,7 +222,8 @@ if(ESTELLA_QUICKJS_DIR)
# be Objective-C++ on Apple; it also forces MA_NO_RUNTIME_LINKING there, so
# the audio frameworks are linked at build time (the app links the merged
# archive, so these also live in native/ios/project.yml).
- set_source_files_properties(host/media/native_audio.cpp PROPERTIES
+ set_source_files_properties(host/media/native_audio.cpp
+ host/media/audio_spectrum.cpp PROPERTIES
COMPILE_OPTIONS "-x;objective-c++")
target_link_libraries(estella_js_host esengine estella_physics estella_video estella_spine qjs
"-framework UIKit" "-framework QuartzCore" "-framework Foundation"
diff --git a/native/host/bindings/AudioBindings.cpp b/native/host/bindings/AudioBindings.cpp
index a78e282b2..a14925c5c 100644
--- a/native/host/bindings/AudioBindings.cpp
+++ b/native/host/bindings/AudioBindings.cpp
@@ -21,6 +21,8 @@
*/
#include "Bindings.hpp"
+#include <vector>
+
using namespace esengine;
namespace eshost {
@@ -106,6 +108,18 @@ JSValue js_audioResumeAll(JSContext* ctx, JSValueConst, int, JSValueConst*) {
} // namespace
+// es_audioSpectrum(bins) -> ArrayBuffer of `bins` magnitude bytes, or null when
+// nothing has played yet. The device's answer to a WebAudio AnalyserNode.
+JSValue js_audioSpectrum(JSContext* ctx, JSValueConst, int argc, JSValueConst* argv) {
+ if (argc < 1 || !hostAlive()) return JS_NULL;
+ int32_t bins = 0;
+ JS_ToInt32(ctx, &bins, argv[0]);
+ if (bins <= 0 || bins > 4096) return JS_NULL;
+ std::vector<uint8_t> out((size_t)bins, 0);
+ if (!host().audio.spectrum(out.data(), out.size())) return JS_NULL;
+ return JS_NewArrayBufferCopy(ctx, out.data(), out.size());
+}
+
void registerAudioBindings(HostState& h, JSValue global) {
if (!h.audio.ready()) return;
bindGlobal(h, global, "es_audioLoad", js_audioLoad, 1);
@@ -117,6 +131,7 @@ void registerAudioBindings(HostState& h, JSValue global) {
bindGlobal(h, global, "es_audioSetVolume", js_audioSetVolume, 2);
bindGlobal(h, global, "es_audioSetPan", js_audioSetPan, 2);
bindGlobal(h, global, "es_audioSetLoop", js_audioSetLoop, 2);
+ bindGlobal(h, global, "es_audioSpectrum", js_audioSpectrum, 1);
bindGlobal(h, global, "es_audioSetRate", js_audioSetRate, 2);
bindGlobal(h, global, "es_audioVoiceState", js_audioVoiceState, 1);
bindGlobal(h, global, "es_audioSuspendAll", js_audioSuspendAll, 0);
diff --git a/native/host/media/audio_spectrum.cpp b/native/host/media/audio_spectrum.cpp
new file mode 100644
index 000000000..3515b7a15
--- /dev/null
+++ b/native/host/media/audio_spectrum.cpp
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file audio_spectrum.cpp
+ * @brief Implements the device's spectrum analyser (see audio_spectrum.hpp).
+ */
+#include "audio_spectrum.hpp"
+
+#include <algorithm>
+#include <cmath>
+
+namespace eshost {
+namespace {
+
+constexpr float kPi = 3.14159265358979323846f;
+
+/** In-place radix-2 Cooley-Tukey FFT. `n` is a power of two. */
+void fft(float* re, float* im, size_t n) {
+ // Bit-reversal permutation.
+ for (size_t i = 1, j = 0; i < n; i++) {
+ size_t bit = n >> 1;
+ for (; j & bit; bit >>= 1) j ^= bit;
+ j ^= bit;
+ if (i < j) {
+ std::swap(re[i], re[j]);
+ std::swap(im[i], im[j]);
+ }
+ }
+ for (size_t len = 2; len <= n; len <<= 1) {
+ const float angle = -2.0f * kPi / (float)len;
+ const float wr = std::cos(angle);
+ const float wi = std::sin(angle);
+ for (size_t i = 0; i < n; i += len) {
+ float cr = 1.0f, ci = 0.0f;
+ for (size_t k = 0; k < len / 2; k++) {
+ const size_t a = i + k;
+ const size_t b = a + len / 2;
+ const float xr = re[b] * cr - im[b] * ci;
+ const float xi = re[b] * ci + im[b] * cr;
+ re[b] = re[a] - xr;
+ im[b] = im[a] - xi;
+ re[a] += xr;
+ im[a] += xi;
+ const float nr = cr * wr - ci * wi;
+ ci = cr * wi + ci * wr;
+ cr = nr;
+ }
+ }
+ }
+}
+
+} // namespace
+
+void AudioSpectrum::write(const float* frames, size_t frameCount, uint32_t channels) {
+ if (!frames || frameCount == 0 || channels == 0) return;
+ uint32_t index = writeIndex_.load(std::memory_order_relaxed);
+ for (size_t f = 0; f < frameCount; f++) {
+ float sum = 0.0f;
+ for (uint32_t c = 0; c < channels; c++) sum += frames[f * channels + c];
+ ring_[index & (kWindow - 1)] = sum / (float)channels;
+ index++;
+ }
+ writeIndex_.store(index, std::memory_order_release);
+ everWrote_.store(true, std::memory_order_release);
+}
+
+bool AudioSpectrum::read(uint8_t* out, size_t bins) {
+ if (!out || bins == 0) return false;
+ if (!everWrote_.load(std::memory_order_acquire)) return false;
+
+ re_.assign(kWindow, 0.0f);
+ im_.assign(kWindow, 0.0f);
+ // Oldest-to-newest, Hann-windowed: the window kills the spectral leakage a
+ // hard-cut buffer would smear across every bin.
+ const uint32_t end = writeIndex_.load(std::memory_order_acquire);
+ for (size_t i = 0; i < kWindow; i++) {
+ const float sample = ring_[(end + i) & (kWindow - 1)];
+ const float w = 0.5f * (1.0f - std::cos(2.0f * kPi * (float)i / (float)(kWindow - 1)));
+ re_[i] = sample * w;
+ }
+ fft(re_.data(), im_.data(), kWindow);
+
+ // Byte magnitudes over 0..Nyquist, in decibels — the scale the web's
+ // getByteFrequencyData uses, so a visualizer looks the same on both.
+ constexpr float kMinDb = -90.0f;
+ constexpr float kMaxDb = -20.0f;
+ const size_t usable = kWindow / 2;
+ for (size_t b = 0; b < bins; b++) {
+ // Each output bin averages the FFT bins it covers, so a short bar array
+ // still reflects the whole band rather than sampling it.
+ const size_t from = b * usable / bins;
+ const size_t to = std::max(from + 1, (b + 1) * usable / bins);
+ float sum = 0.0f;
+ for (size_t k = from; k < to && k < usable; k++) {
+ sum += std::sqrt(re_[k] * re_[k] + im_[k] * im_[k]);
+ }
+ const float mag = sum / (float)(to - from) / (float)(kWindow / 4);
+ const float db = 20.0f * std::log10(std::max(mag, 1e-7f));
+ const float t = (db - kMinDb) / (kMaxDb - kMinDb);
+ out[b] = (uint8_t)std::clamp(t * 255.0f, 0.0f, 255.0f);
+ }
+ return true;
+}
+
+} // namespace eshost
diff --git a/native/host/media/audio_spectrum.hpp b/native/host/media/audio_spectrum.hpp
new file mode 100644
index 000000000..a835b09ab
--- /dev/null
+++ b/native/host/media/audio_spectrum.hpp
@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file audio_spectrum.hpp
+ * @brief The analyser behind `AudioAPI.getSpectrum()` on a device: a tap on
+ * what the engine is about to play, and the FFT over it.
+ * @details The web reads its spectrum from a WebAudio AnalyserNode. There is no
+ * such node here, so the host provides the same answer the same way a
+ * browser does — window the most recent frames, transform them, and
+ * hand back byte magnitudes in the layout getByteFrequencyData uses.
+ *
+ * Thread-safe by design: the audio thread only ever WRITES into a ring
+ * buffer, and the game thread only ever reads a snapshot of it, so no
+ * lock sits in the audio callback (where one would be a dropout).
+ *
+ * @author ESEngine Team
+ * @date 2026
+ *
+ * @copyright Copyright (c) 2026 ESEngine Team
+ * Licensed under the Apache License, Version 2.0.
+ */
+#pragma once
+
+#include <atomic>
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+namespace eshost {
+
+/**
+ * A rolling window of the engine's output, and the FFT over it.
+ *
+ * `write` is called from the audio thread with interleaved frames; `read` is
+ * called from the game thread and fills `out` with `out.size()` magnitude bytes
+ * covering 0 .. Nyquist, exactly like the web's getByteFrequencyData.
+ */
+class AudioSpectrum {
+public:
+ /** Samples the FFT runs over. 1024 at 48 kHz is ~21 ms — long enough to
+ * resolve a bass line, short enough to look live. */
+ static constexpr size_t kWindow = 1024;
+
+ /** Mix `frameCount` interleaved frames down to mono and keep the newest
+ * window. Audio thread; wait-free. */
+ void write(const float* frames, size_t frameCount, uint32_t channels);
+
+ /** Fill `out` with byte magnitudes. False when nothing has played yet, so a
+ * caller can tell "silent" from "unsupported". Game thread. */
+ bool read(uint8_t* out, size_t bins);
+
+private:
+ /** Power-of-two ring of mono samples the audio thread writes into. */
+ float ring_[kWindow]{};
+ std::atomic<uint32_t> writeIndex_{0};
+ std::atomic<bool> everWrote_{false};
+ /** Scratch for the transform — game thread only, so no allocation per read. */
+ std::vector<float> re_;
+ std::vector<float> im_;
+};
+
+} // namespace eshost
diff --git a/native/host/media/native_audio.cpp b/native/host/media/native_audio.cpp
index 28a25c3f2..6a69b0053 100644
--- a/native/host/media/native_audio.cpp
+++ b/native/host/media/native_audio.cpp
@@ -16,6 +16,8 @@
#include "native_audio.hpp"
+#include "audio_spectrum.hpp"
+
#include <string>
#include <unordered_map>
#include <vector>
@@ -40,9 +42,36 @@ struct Voice {
bool loop = false;
};
+// The analyser's tap: a node the voices play into, which passes their frames
+// through unchanged and keeps a copy for the FFT. Attaching voices HERE rather
+// than straight to the endpoint is what gives the game a spectrum without the
+// audio thread ever waiting on it (AudioSpectrum::write is wait-free).
+struct SpectrumTap {
+ ma_node_base base{};
+ AudioSpectrum* spectrum = nullptr;
+ ma_uint32 channels = 2;
+};
+
+void spectrumTapProcess(ma_node* node, const float** framesIn, ma_uint32* frameCountIn,
+ float** framesOut, ma_uint32* frameCountOut) {
+ auto* tap = reinterpret_cast<SpectrumTap*>(node);
+ const ma_uint32 frames = frameCountIn && frameCountOut
+ ? (*frameCountIn < *frameCountOut ? *frameCountIn : *frameCountOut) : 0;
+ if (frames > 0 && framesIn && framesIn[0] && framesOut && framesOut[0]) {
+ const size_t samples = (size_t)frames * tap->channels;
+ for (size_t i = 0; i < samples; i++) framesOut[0][i] = framesIn[0][i];
+ if (tap->spectrum) tap->spectrum->write(framesIn[0], frames, tap->channels);
+ }
+ if (frameCountOut) *frameCountOut = frames;
+ if (frameCountIn) *frameCountIn = frames;
+}
+
struct AudioEngine::Impl {
ma_context context{};
ma_engine engine{};
+ SpectrumTap tap{};
+ bool tapReady = false;
+ AudioSpectrum spectrum;
bool contextReady = false;
bool ready = false;
ma_uint32 sampleRate = 0;
@@ -109,6 +138,27 @@ bool AudioEngine::init() {
impl_->ready = true;
impl_->sampleRate = ma_engine_get_sample_rate(&impl_->engine);
impl_->channels = ma_engine_get_channels(&impl_->engine);
+
+ // The analyser tap, between the voices and the endpoint. Optional: a failure
+ // here costs the spectrum, not the sound.
+ static ma_node_vtable vtable{};
+ vtable.onProcess = spectrumTapProcess;
+ vtable.onGetRequiredInputFrameCount = nullptr;
+ vtable.inputBusCount = 1;
+ vtable.outputBusCount = 1;
+ vtable.flags = MA_NODE_FLAG_PASSTHROUGH;
+ ma_node_config tapCfg = ma_node_config_init();
+ tapCfg.vtable = &vtable;
+ tapCfg.pInputChannels = &impl_->channels;
+ tapCfg.pOutputChannels = &impl_->channels;
+ impl_->tap.spectrum = &impl_->spectrum;
+ impl_->tap.channels = impl_->channels;
+ if (ma_node_init(ma_engine_get_node_graph(&impl_->engine), &tapCfg, nullptr,
+ &impl_->tap.base) == MA_SUCCESS
+ && ma_node_attach_output_bus(&impl_->tap.base, 0,
+ ma_engine_get_endpoint(&impl_->engine), 0) == MA_SUCCESS) {
+ impl_->tapReady = true;
+ }
return true;
}
@@ -116,6 +166,7 @@ void AudioEngine::shutdown() {
if (!impl_) return;
for (auto& [id, voice] : impl_->voices) ma_sound_uninit(&voice.sound);
impl_->voices.clear();
+ if (impl_->tapReady) { ma_node_uninit(&impl_->tap.base, nullptr); impl_->tapReady = false; }
if (impl_->ready) ma_engine_uninit(&impl_->engine);
if (impl_->contextReady) ma_context_uninit(&impl_->context);
for (auto& [id, buf] : impl_->buffers) ma_free(buf.pcm, nullptr);
@@ -188,6 +239,11 @@ int AudioEngine::play(int bufferId, float volume, float pan, bool loop, float ra
impl_->voices.erase(id);
return -1;
}
+ // Into the tap when there is one, so the analyser sees this voice; straight
+ // to the endpoint otherwise (identical audio either way).
+ if (impl_->tapReady) {
+ ma_node_attach_output_bus(&voice.sound, 0, &impl_->tap.base, 0);
+ }
ma_sound_set_volume(&voice.sound, volume);
ma_sound_set_pan(&voice.sound, pan);
ma_sound_set_pitch(&voice.sound, rate > 0 ? rate : 1.0f);
@@ -197,6 +253,11 @@ int AudioEngine::play(int bufferId, float volume, float pan, bool loop, float ra
return id;
}
+bool AudioEngine::spectrum(uint8_t* out, size_t bins) {
+ if (!impl_ || !impl_->tapReady) return false;
+ return impl_->spectrum.read(out, bins);
+}
+
void AudioEngine::stop(int voiceId) {
if (impl_) impl_->destroyVoice(voiceId);
}
diff --git a/native/host/media/native_audio.hpp b/native/host/media/native_audio.hpp
index 98eba8a0e..4952f1d0a 100644
--- a/native/host/media/native_audio.hpp
+++ b/native/host/media/native_audio.hpp
@@ -43,6 +43,13 @@ public:
Loaded load(const uint8_t* bytes, size_t n);
void unload(int bufferId);
+ /**
+ * Fill @p bins byte magnitudes of what is playing, 0..Nyquist — the device's
+ * answer to a WebAudio AnalyserNode (see media/audio_spectrum.hpp). False
+ * when nothing has played yet or the tap could not be built.
+ */
+ bool spectrum(uint8_t* out, size_t bins);
+
/** Start a voice on a loaded buffer. Returns its id, or -1 if unknown. */
int play(int bufferId, float volume, float pan, bool loop, float rate);
void stop(int voiceId);
diff --git a/sdk/etc/index.native.api.md b/sdk/etc/index.native.api.md
index 024c04b6a..bd6b98a58 100644
--- a/sdk/etc/index.native.api.md
+++ b/sdk/etc/index.native.api.md
@@ -4135,6 +4135,7 @@ es_audioSetLoop: ((voiceId: number, loop: boolean) => void) | undefined
es_audioSetPan: ((voiceId: number, pan: number) => void) | undefined
es_audioSetRate: ((voiceId: number, rate: number) => void) | undefined
es_audioSetVolume: ((voiceId: number, volume: number) => void) | undefined
+es_audioSpectrum: ((bins: number) => ArrayBuffer | null) | undefined
es_audioStop: ((voiceId: number) => void) | undefined
es_audioSuspendAll: (() => void) | undefined
es_audioUnload: ((bufferId: number) => void) | undefined
diff --git a/sdk/src/audio/NativeAudioBackend.ts b/sdk/src/audio/NativeAudioBackend.ts
index f5f123ed8..b0491a883 100644
--- a/sdk/src/audio/NativeAudioBackend.ts
+++ b/sdk/src/audio/NativeAudioBackend.ts
@@ -65,6 +65,9 @@ export class NativeAudioBackend implements PlatformAudioBackend {
readonly isReady = true;
private readonly voices_ = new Map<number, NativeAudioHandle>();
+ /** Reused across frames: a visualizer asks every frame, and a fresh view per
+ * ask would be garbage the interpreter has to collect at 60 Hz. */
+ private spectrumView_: Uint8Array | null = null;
private readonly offEnded_: () => void;
constructor(private readonly audio_: NativeAudioBridge) {
@@ -115,6 +118,22 @@ export class NativeAudioBackend implements PlatformAudioBackend {
return handle;
}
+ /**
+ * The host's analyser, in the layout the web's getByteFrequencyData uses —
+ * so a visualizer written against the web reads the same bytes here. False
+ * when the host has no analyser tap, which the caller treats exactly as it
+ * treats a backend with no analysis at all.
+ */
+ getFrequencyData(out: Uint8Array): boolean {
+ const bytes = this.audio_.spectrum?.(out.length);
+ if (!bytes) return false;
+ if (!this.spectrumView_ || this.spectrumView_.buffer !== bytes) {
+ this.spectrumView_ = new Uint8Array(bytes);
+ }
+ out.set(this.spectrumView_.subarray(0, out.length));
+ return true;
+ }
+
suspend(): void { this.audio_.suspendAll(); }
resume(): void { this.audio_.resumeAll(); }
diff --git a/sdk/src/platform/native/bridge.ts b/sdk/src/platform/native/bridge.ts
index ae23c0471..83d4dadab 100644
--- a/sdk/src/platform/native/bridge.ts
+++ b/sdk/src/platform/native/bridge.ts
@@ -96,6 +96,9 @@ export interface NativeTextEditorBridge {
* it. Both are non-negative host handles; a failed load or play returns null / -1.
*/
export interface NativeAudioBridge {
+ /** `bins` byte magnitudes of what is playing (0..Nyquist), or null when the
+ * host has no analyser / nothing has played. The device's AnalyserNode. */
+ spectrum?(bins: number): ArrayBuffer | null;
/** Decode + register a clip from its (compressed) file bytes. Returns the
* buffer handle, its length in seconds and its decoded size in bytes (for the
* residency budget), or null on decode failure. */
diff --git a/sdk/src/platform/native/hostBridge.ts b/sdk/src/platform/native/hostBridge.ts
index 590f17b96..9e5fd177c 100644
--- a/sdk/src/platform/native/hostBridge.ts
+++ b/sdk/src/platform/native/hostBridge.ts
@@ -87,6 +87,7 @@ export interface NativeHostBindings {
es_textEditor_blur?(): void;
es_textEditor_write?(value: string, selectionStart: number, selectionEnd: number): void;
+ es_audioSpectrum?(bins: number): ArrayBuffer | null;
es_audioLoad?(bytes: ArrayBuffer): { id: number; duration: number; bytes: number } | null;
es_audioUnload?(bufferId: number): void;
es_audioPlay?(bufferId: number, volume: number, pan: number, loop: boolean, rate: number): number;
@@ -260,6 +261,12 @@ function hostAudio(
suspendAll: () => bindings.es_audioSuspendAll!(),
resumeAll: () => bindings.es_audioResumeAll!(),
onEnded: (cb) => { ended = cb; return () => { if (ended === cb) ended = null; }; },
+ // Optional: a host without the analyser tap simply has no spectrum, and
+ // a visualizer falls back to flat bars (as it does on any backend
+ // without analysis).
+ ...(bindings.es_audioSpectrum
+ ? { spectrum: (bins: number) => bindings.es_audioSpectrum!(bins) }
+ : {}),
};
}