Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

8.1 KB
Raw Blame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
 * @file    NativeAudioBackend.ts
 * @brief   A PlatformAudioBackend over the native host's audio engine (embedded
 *          Dawn + JS engine on iOS/Android).
 * @details A thin adapter over {@link NativeAudioBridge}, exactly as the WeChat
 *          backend is over InnerAudioContext: decode, mixing and output all run
 *          in the host's native engine (miniaudio), so nothing per-sample touches
 *          JS — the no-JIT budget forbids it. `mixer` is therefore null (no JS
 *          DSP graph), like WeChat; per-voice volume / pan / rate / loop are the
 *          full control surface.
 *
 *          A clip the importer (or its length, under `auto`) says to stream is
 *          handed to the host to keep compressed: the rule is audioStreaming.ts,
 *          the one the web backend follows, so the two agree on which clips.
 */
import type { AudioMixer } from './AudioMixer';
import type {
    AudioBackendInitOptions, AudioBufferHandle, AudioHandle,
    PlatformAudioBackend, PlayConfig,
} from './PlatformAudioBackend';
import type { NativeAudioBridge } from '../platform/native/bridge';
import type { AudioLoadOptions } from './PlatformAudioBackend';
import { clipSpan } from './PlatformAudioBackend';
import { decodedPcmBytes, shouldStream } from './audioStreaming';
import { platformReadFile } from '../platform/base';
import { scalar } from '../math/scalar';

class NativeAudioHandle implements AudioHandle {
    onEnd?: () => void;
    private done_ = false;

    constructor(
        readonly id: number,
        private readonly audio_: NativeAudioBridge,
        private readonly duration_: number,
        private readonly onDispose_: () => void,
    ) {}

    stop(): void {
        if (this.done_) return;
        this.done_ = true;
        try { this.audio_.stop(this.id); } finally { this.onDispose_(); }
    }

    /** @internal The host reported this voice ended on its own (pushed through
     *  {@link NativeAudioBridge.onEnded}). Fire onEnd once; a later stop() no-ops. */
    handleEnded_(): void {
        if (this.done_) return;
        this.done_ = true;
        this.onDispose_();
        this.onEnd?.();
    }

    pause(): void { if (!this.done_) this.audio_.pause(this.id); }
    resume(): void { if (!this.done_) this.audio_.resume(this.id); }
    setVolume(volume: number): void { if (!this.done_) this.audio_.setVolume(this.id, volume); }
    setPan(pan: number): void { if (!this.done_) this.audio_.setPan(this.id, scalar.clamp(pan, -1, 1)); }
    setLoop(loop: boolean): void { if (!this.done_) this.audio_.setLoop(this.id, loop); }
    setPlaybackRate(rate: number): void { if (!this.done_) this.audio_.setRate(this.id, rate); }

    get isPlaying(): boolean { return !this.done_ && (this.audio_.voiceState(this.id)?.playing ?? false); }
    get currentTime(): number { return this.done_ ? 0 : this.audio_.voiceState(this.id)?.currentTime ?? 0; }
    get duration(): number { return this.duration_; }
}

export class NativeAudioBackend implements PlatformAudioBackend {
    readonly name = 'native';
    readonly delivery = 'bytes' as const;
    readonly mixer: AudioMixer | null = null;
    readonly isReady = true;

    private readonly voices_ = new Map<number, NativeAudioHandle>();
    /** Where a clip the exporter re-encoded really lies in what the host decoded, by buffer. */
    private readonly spans_ = new Map<number, { lead: number; end: number }>();
    private readonly together_ = new WeakMap<object, number>();
    /** Reused across frames: a visualizer asks every frame, and a fresh view per
     *  ask would be garbage the interpreter has to collect at 60 Hz. */
    private spectrumView_: Uint8Array | null = null;
    private readonly offEnded_: () => void;

    constructor(private readonly audio_: NativeAudioBridge) {
        // The host pushes a voice-ended id (like touch); route it to that handle.
        this.offEnded_ = audio_.onEnded((voiceId) => {
            this.voices_.get(voiceId)?.handleEnded_();
        });
    }

    // The native engine is brought up by the host at boot; nothing to initialize
    // or resume here (no user-gesture requirement, unlike a browser AudioContext).
    async initialize(_options?: AudioBackendInitOptions): Promise<void> {}
    async ensureResumed(): Promise<void> {}

    async loadBuffer(url: string, options?: AudioLoadOptions): Promise<AudioBufferHandle> {
        // On native the audio ref resolver hands back a packaged path (not bytes),
        // so read the file through the same platform channel as every other asset,
        // then let the native engine decode it.
        return this.loadFromBytes_(await platformReadFile(url), options);
    }

    async loadBufferFromData(_url: string, data: ArrayBuffer, options?: AudioLoadOptions): Promise<AudioBufferHandle> {
        return this.loadFromBytes_(data, options);
    }

    private loadFromBytes_(bytes: ArrayBuffer, options?: AudioLoadOptions): AudioBufferHandle {
        const loaded = this.audio_.load(bytes, shouldStream(options?.loadType, bytes));
        if (!loaded) throw new Error('[native] audio decode failed');
        const span = clipSpan(options?.trim, loaded.duration);
        if (span) { this.spans_.set(loaded.id, span); loaded.duration = span.end - span.lead; }
        // What the host DID, not what was asked: one that could not stream the file decoded it, and its PCM
        // is then what counts against the budget.
        if (!loaded.streamed) return { id: loaded.id, duration: loaded.duration, bytes: loaded.bytes };
        return {
            id: loaded.id, duration: loaded.duration, bytes: loaded.bytes, streamed: true,
            savedBytes: Math.max(0, decodedPcmBytes(loaded.duration) - loaded.bytes),
        };
    }

    unloadBuffer(handle: AudioBufferHandle): void {
        this.spans_.delete(handle.id);
        this.audio_.unload(handle.id);
    }

    play(buffer: AudioBufferHandle, config: PlayConfig): AudioHandle {
        const span = this.spans_.get(buffer.id);
        // Voices of one token begin at one moment of the host's own clock, a twentieth of a second off: the calls that
        // start them arrive one after another, and the device may run between two of them.
        let at = 0;
        if (config.together && this.audio_.time) {
            at = this.together_.get(config.together) ?? this.audio_.time() + 0.05;
            this.together_.set(config.together, at);
        }
        const voiceId = this.audio_.play(
            buffer.id,
            config.volume ?? 1,
            scalar.clamp(config.pan ?? 0, -1, 1),
            config.loop ?? false,
            config.playbackRate ?? 1,
            // Places in what the host decoded: the loop start, where the clip begins, and where it ends (0 is its natural end).
            config.loop && config.loopStart && config.loopStart > 0 ? (span?.lead ?? 0) + config.loopStart : span?.lead ?? 0,
            span?.lead ?? 0,
            span?.end ?? 0,
            at,
        );
        const handle = new NativeAudioHandle(
            voiceId, this.audio_, buffer.duration, () => this.voices_.delete(voiceId),
        );
        this.voices_.set(voiceId, handle);
        return handle;
    }

    /**
     * The host's analyser, in the layout the web's getByteFrequencyData uses —
     * so a visualizer written against the web reads the same bytes here. False
     * when the host has no analyser tap, which the caller treats exactly as it
     * treats a backend with no analysis at all.
     */
    getFrequencyData(out: Uint8Array): boolean {
        const bytes = this.audio_.spectrum?.(out.length);
        if (!bytes) return false;
        if (!this.spectrumView_ || this.spectrumView_.buffer !== bytes) {
            this.spectrumView_ = new Uint8Array(bytes);
        }
        out.set(this.spectrumView_.subarray(0, out.length));
        return true;
    }

    suspend(): void { this.audio_.suspendAll(); }
    resume(): void { this.audio_.resumeAll(); }

    dispose(): void {
        for (const handle of this.voices_.values()) handle.stop();
        this.voices_.clear();
        this.offEnded_();
    }
}