Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(editor): audio assets get a waveform inspector + import settings

esengine04cce185fe55
diff --git a/desktop/src/components/AudioWavePreview.tsx b/desktop/src/components/AudioWavePreview.tsx
new file mode 100644
index 000000000..ca58b25dc
--- /dev/null
+++ b/desktop/src/components/AudioWavePreview.tsx
@@ -0,0 +1,159 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file AudioWavePreview.tsx
+ * @brief Waveform + transport for the audio asset inspector: fetches the
+ * clip over estella://, decodes it once (a module-level AudioContext,
+ * decode-only), draws min/max peak columns on a canvas, and drives a
+ * plain <audio> element for play/pause/click-seek with a playhead.
+ */
+import { useEffect, useRef, useState } from 'react';
+import { Play, Pause } from 'lucide-react';
+import { t } from '@/i18n';
+
+let decodeCtx: AudioContext | null = null;
+const decoder = (): AudioContext => (decodeCtx ??= new AudioContext());
+
+interface WaveInfo {
+ peaks: Float32Array; // interleaved min,max per column
+ duration: number;
+ channels: number;
+ sampleRate: number;
+}
+
+const COLUMNS = 400;
+
+async function loadWave(path: string): Promise<WaveInfo> {
+ const bytes = await (await fetch(`estella://project/${path}`)).arrayBuffer();
+ const buffer = await decoder().decodeAudioData(bytes);
+ const peaks = new Float32Array(COLUMNS * 2);
+ const data = buffer.getChannelData(0);
+ const second = buffer.numberOfChannels > 1 ? buffer.getChannelData(1) : null;
+ const step = data.length / COLUMNS;
+ for (let c = 0; c < COLUMNS; c++) {
+ let min = 0;
+ let max = 0;
+ const start = Math.floor(c * step);
+ const end = Math.min(data.length, Math.ceil((c + 1) * step));
+ for (let i = start; i < end; i++) {
+ const v = second ? (data[i] + second[i]) * 0.5 : data[i];
+ if (v < min) min = v;
+ if (v > max) max = v;
+ }
+ peaks[c * 2] = min;
+ peaks[c * 2 + 1] = max;
+ }
+ return { peaks, duration: buffer.duration, channels: buffer.numberOfChannels, sampleRate: buffer.sampleRate };
+}
+
+function drawWave(canvas: HTMLCanvasElement, wave: WaveInfo | null, progress: number): void {
+ const ctx = canvas.getContext('2d');
+ if (!ctx) return;
+ const dpr = window.devicePixelRatio || 1;
+ const w = canvas.clientWidth;
+ const h = canvas.clientHeight;
+ if (canvas.width !== w * dpr || canvas.height !== h * dpr) {
+ canvas.width = w * dpr;
+ canvas.height = h * dpr;
+ }
+ ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
+ ctx.clearRect(0, 0, w, h);
+ if (!wave) return;
+ const styles = getComputedStyle(canvas);
+ const base = styles.getPropertyValue('--star').trim() || '#6ea8ff';
+ const dim = styles.getPropertyValue('--text-faint').trim() || '#666';
+ const mid = h / 2;
+ const colW = w / COLUMNS;
+ const playedCols = Math.floor(progress * COLUMNS);
+ for (let c = 0; c < COLUMNS; c++) {
+ const min = wave.peaks[c * 2];
+ const max = wave.peaks[c * 2 + 1];
+ ctx.fillStyle = c <= playedCols && progress > 0 ? base : dim;
+ const top = mid + min * (mid - 2);
+ const bottom = mid + max * (mid - 2);
+ ctx.fillRect(c * colW, Math.min(top, bottom), Math.max(1, colW - 0.5), Math.max(1, Math.abs(bottom - top)));
+ }
+ if (progress > 0) {
+ ctx.fillStyle = base;
+ ctx.fillRect(progress * w - 0.5, 0, 1, h);
+ }
+}
+
+const fmtTime = (s: number) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, '0')}`;
+
+export function AudioWavePreview({ path }: { path: string }) {
+ const canvasRef = useRef<HTMLCanvasElement>(null);
+ const audioRef = useRef<HTMLAudioElement | null>(null);
+ const [wave, setWave] = useState<WaveInfo | null>(null);
+ const [failed, setFailed] = useState(false);
+ const [playing, setPlaying] = useState(false);
+ const [progress, setProgress] = useState(0);
+
+ useEffect(() => {
+ let alive = true;
+ setWave(null);
+ setFailed(false);
+ setPlaying(false);
+ setProgress(0);
+ void loadWave(path)
+ .then((v) => alive && setWave(v))
+ .catch(() => alive && setFailed(true));
+ const audio = new Audio(`estella://project/${path}`);
+ audioRef.current = audio;
+ const onTime = () => setProgress(audio.duration > 0 ? audio.currentTime / audio.duration : 0);
+ const onEnd = () => { setPlaying(false); setProgress(0); };
+ audio.addEventListener('timeupdate', onTime);
+ audio.addEventListener('ended', onEnd);
+ return () => {
+ alive = false;
+ audio.pause();
+ audio.removeEventListener('timeupdate', onTime);
+ audio.removeEventListener('ended', onEnd);
+ audioRef.current = null;
+ };
+ }, [path]);
+
+ useEffect(() => {
+ if (canvasRef.current) drawWave(canvasRef.current, wave, progress);
+ }, [wave, progress]);
+
+ const toggle = () => {
+ const audio = audioRef.current;
+ if (!audio) return;
+ if (playing) {
+ audio.pause();
+ setPlaying(false);
+ } else {
+ void audio.play();
+ setPlaying(true);
+ }
+ };
+
+ const seek = (e: React.MouseEvent) => {
+ const audio = audioRef.current;
+ const canvas = canvasRef.current;
+ if (!audio || !canvas || !Number.isFinite(audio.duration)) return;
+ const r = canvas.getBoundingClientRect();
+ const p = Math.max(0, Math.min(1, (e.clientX - r.left) / r.width));
+ audio.currentTime = p * audio.duration;
+ setProgress(p);
+ };
+
+ return (
+ <div className="awp">
+ <canvas ref={canvasRef} className="awp-wave" onClick={seek} title={t('det.audioSeekTip')} />
+ <div className="awp-bar">
+ <button type="button" className="awp-play" onClick={toggle} title={t('det.audioPlayTip')}>
+ {playing ? <Pause size={13} /> : <Play size={13} />}
+ </button>
+ <span className="awp-meta">
+ {failed
+ ? t('det.audioDecodeFailed')
+ : wave
+ ? `${fmtTime(wave.duration)} · ${wave.channels} ch · ${(wave.sampleRate / 1000).toFixed(1)} kHz`
+ : '…'}
+ </span>
+ </div>
+ </div>
+ );
+}
diff --git a/desktop/src/i18n/messages/details.ts b/desktop/src/i18n/messages/details.ts
index b0dd72013..ad785fa1e 100644
--- a/desktop/src/i18n/messages/details.ts
+++ b/desktop/src/i18n/messages/details.ts
@@ -164,6 +164,9 @@ export const detailsMessages = defineMessages({
'det.importSaved': { en: 'Import settings saved', zh: '导入设置已保存' },
'det.importSaveFailed': { en: 'Failed to save import settings: {error}', zh: '保存导入设置失败:{error}' },
'det.noImportSettings': { en: 'This asset type has no import settings.', zh: '此资产类型没有导入设置。' },
+ 'det.audioSeekTip': { en: 'Click to seek', zh: '点击跳转播放位置' },
+ 'det.audioPlayTip': { en: 'Play / pause', zh: '播放 / 暂停' },
+ 'det.audioDecodeFailed': { en: 'Could not decode audio', zh: '无法解码音频' },
'det.type': { en: 'Type', zh: '类型' },
'det.metaPath': { en: 'Path', zh: '路径' },
'det.metaDimensions': { en: 'Dimensions', zh: '尺寸' },
diff --git a/desktop/src/panels/Details.tsx b/desktop/src/panels/Details.tsx
index 3a1d7a1b9..e83049f79 100644
--- a/desktop/src/panels/Details.tsx
+++ b/desktop/src/panels/Details.tsx
@@ -30,6 +30,7 @@ import {
type LucideIcon,
} from 'lucide-react';
import { AssetIcon } from '@/components/icons';
+import { AudioWavePreview } from '@/components/AudioWavePreview';
import { Toasts } from '@/store/Toasts';
import { baseName, assetTypeOf, IMAGE_RE } from '@/project/assetMeta';
import { revealAsset } from '@/project/assetReveal';
@@ -2316,15 +2317,19 @@ function GenericAssetInspector({ path }: { path: string }) {
<div className="insp-body">
{/* A compact preview for a quick visual ID — kept short so the editable
Import Settings sit above the fold (the reason to open an asset). */}
- <div className="cb-prev" style={{ height: 108 }}>
- <div className="pv">
- {isImage ? (
- <img src={`estella://project/${path}`} alt="" draggable={false} />
- ) : (
- <AssetIcon type={type} size={44} />
- )}
+ {type === 'audio' ? (
+ <AudioWavePreview path={path} />
+ ) : (
+ <div className="cb-prev" style={{ height: 108 }}>
+ <div className="pv">
+ {isImage ? (
+ <img src={`estella://project/${path}`} alt="" draggable={false} />
+ ) : (
+ <AssetIcon type={type} size={44} />
+ )}
+ </div>
</div>
- </div>
+ )}
{/* Editable import settings first (the reason to select an asset), then
read-only metadata. */}
diff --git a/desktop/src/project/assetImporter.ts b/desktop/src/project/assetImporter.ts
index faa2ed59a..4a60313e4 100644
--- a/desktop/src/project/assetImporter.ts
+++ b/desktop/src/project/assetImporter.ts
@@ -49,6 +49,19 @@ const SPINE: ImporterFieldSpec[] = [
{ key: 'premultiplyAlpha', label: 'Premultiply Alpha', type: 'bool', default: false, category: 'Spine', advanced: true },
];
+const AUDIO: ImporterFieldSpec[] = [
+ {
+ key: 'compress', label: 'Compress', type: 'bool', default: true, category: 'Audio',
+ tooltip: 'Re-encode WAV to MP3 at cook (already-compressed formats pass through). '
+ + 'MP3 has a small encoder delay — disable for seamless-loop clips.',
+ },
+ {
+ key: 'bitrateKbps', label: 'Bitrate', type: 'enum', default: 128, category: 'Audio',
+ options: [96, 128, 192].map((n) => ({ label: `${n} kbps`, value: n })),
+ tooltip: 'MP3 bitrate for cooked WAV sources.',
+ },
+];
+
const SCENELIKE: ImporterFieldSpec[] = [
{
key: 'autoMigrate', label: 'Auto Migrate', type: 'bool', default: true, category: 'Import',
@@ -62,10 +75,23 @@ export const IMPORTER_SCHEMAS: Record<string, ImporterFieldSpec[]> = {
texture: TEXTURE,
sprite: TEXTURE,
spine: SPINE,
+ audio: AUDIO,
scene: SCENELIKE,
prefab: SCENELIKE,
};
+/** Cook-facing audio import settings, tolerant of hand-edited `.meta` blocks. */
+export function readAudioImportSettings(importer: Record<string, unknown> | undefined): {
+ compress: boolean; bitrateKbps: number;
+} {
+ const compress = importer?.compress;
+ const bitrate = importer?.bitrateKbps;
+ return {
+ compress: typeof compress === 'boolean' ? compress : true,
+ bitrateKbps: bitrate === 96 || bitrate === 128 || bitrate === 192 ? bitrate : 128,
+ };
+}
+
const getByPath = (obj: Record<string, unknown>, path: string): unknown =>
path.split('.').reduce<unknown>((o, k) => (o && typeof o === 'object' ? (o as Record<string, unknown>)[k] : undefined), obj);
diff --git a/desktop/src/project/assetTypes.ts b/desktop/src/project/assetTypes.ts
index 2631342c5..b8d307264 100644
--- a/desktop/src/project/assetTypes.ts
+++ b/desktop/src/project/assetTypes.ts
@@ -36,7 +36,7 @@ export const ASSET_TYPES: Record<AssetType, AssetTypeDef> = {
texture: { extensions: ['png', 'webp'], badge: 'TEX', icon: FileImage, tint: '#7fa6c4' },
sprite: { extensions: ['jpg', 'jpeg', 'gif'], badge: 'IMG', icon: Image, tint: '#7fa6c4' },
spine: { extensions: ['atlas', 'skel'], badge: 'SPN', icon: PersonStanding, tint: '#9b8fc0' },
- audio: { extensions: ['ogg', 'mp3', 'wav'], badge: 'AUD', icon: Music, tint: '#7faf9c' },
+ audio: { extensions: ['ogg', 'mp3', 'wav', 'aac', 'flac', 'm4a', 'webm'], badge: 'AUD', icon: Music, tint: '#7faf9c' },
prefab: { extensions: ['esprefab'], badge: 'PFB', icon: Component, tint: '#c2a274' },
// .esmaterial is the real extension (the SDK MaterialAssetLoader only loads it);
// .esmat is tolerated as a legacy alias (cf. electron/importAssets.ts).
diff --git a/desktop/src/theme/inspector.css b/desktop/src/theme/inspector.css
index ebb0db174..615b8158a 100644
--- a/desktop/src/theme/inspector.css
+++ b/desktop/src/theme/inspector.css
@@ -299,3 +299,41 @@
.lt-miss-key { font-family: var(--mono); font-size: var(--fs-xs); color: var(--text-dim); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding-left: 2px; }
.lt-miss-ref { font-size: var(--fs-2xs); color: var(--text-mute); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; }
.lt-error { margin: 8px; padding: 6px 9px; font-size: var(--fs-2xs); line-height: 1.5; font-family: var(--mono); color: var(--text-dim); background: var(--inset); border: 1px solid var(--border); border-radius: var(--r-sm); word-break: break-all; }
+
+/* Audio asset waveform preview (AudioWavePreview) */
+.awp {
+ display: flex;
+ flex-direction: column;
+ border-bottom: 1px solid var(--line);
+ background: var(--inset);
+}
+.awp-wave {
+ width: 100%;
+ height: 84px;
+ display: block;
+ cursor: pointer;
+}
+.awp-bar {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ padding: 4px 8px;
+ border-top: 1px solid var(--line);
+ background: var(--srf-2);
+}
+.awp-play {
+ display: inline-flex;
+ align-items: center;
+ background: transparent;
+ color: var(--star-hi);
+ border: 1px solid var(--star-line);
+ border-radius: var(--r-sm);
+ padding: 3px 6px;
+ cursor: pointer;
+}
+.awp-play:hover { background: var(--star-dim); }
+.awp-meta {
+ color: var(--text-faint);
+ font-size: var(--fs-2xs);
+ font-variant-numeric: tabular-nums;
+}
diff --git a/desktop/tests/asset-importer.test.ts b/desktop/tests/asset-importer.test.ts
index fb518e286..40c0c1888 100644
--- a/desktop/tests/asset-importer.test.ts
+++ b/desktop/tests/asset-importer.test.ts
@@ -30,7 +30,7 @@ describe('importerDefaults (import-time .meta)', () => {
expect(importerDefaults('spine')).toEqual({ scale: 1, defaultSkin: 'default', premultiplyAlpha: false });
expect(importerDefaults('scene')).toEqual({ autoMigrate: true });
expect(importerDefaults('prefab')).toEqual({ autoMigrate: true });
- expect(importerDefaults('audio')).toEqual({});
+ expect(importerDefaults('audio')).toEqual({ compress: true, bitrateKbps: 128 });
});
});
@@ -38,7 +38,7 @@ describe('hasImporterSettings', () => {
it('is true only for types with editable settings', () => {
expect(hasImporterSettings('texture')).toBe(true);
expect(hasImporterSettings('spine')).toBe(true);
- expect(hasImporterSettings('audio')).toBe(false);
+ expect(hasImporterSettings('audio')).toBe(true);
expect(hasImporterSettings('font')).toBe(false);
});
});
@@ -57,7 +57,7 @@ describe('buildImporterComponent (inspector fields)', () => {
expect(byKey['sliceBorder.top'].value).toBe(0); // nested default
});
it('returns null for types without settings', () => {
- expect(buildImporterComponent('audio', {})).toBeNull();
+ expect(buildImporterComponent('audio', {})).not.toBeNull();
});
});
diff --git a/sdk/src/asset/loaders/AudioAssetLoader.ts b/sdk/src/asset/loaders/AudioAssetLoader.ts
index dabe11ec9..3b3f8e155 100644
--- a/sdk/src/asset/loaders/AudioAssetLoader.ts
+++ b/sdk/src/asset/loaders/AudioAssetLoader.ts
@@ -6,7 +6,7 @@ import { log } from '../../logger';
export class AudioAssetLoader implements AssetLoader<AudioResult> {
readonly type = 'audio';
- readonly extensions = ['.mp3', '.wav', '.ogg', '.m4a', '.aac'];
+ readonly extensions = ['.mp3', '.wav', '.ogg', '.m4a', '.aac', '.flac', '.webm'];
/** Same lazy audio accessor Assets uses — needed by unload/invalidate,
* which have no LoadContext. Defaults to "no audio" for bare setups. */
diff --git a/sdk/src/assetTypes.ts b/sdk/src/assetTypes.ts
index f845e2002..6805be0ef 100644
--- a/sdk/src/assetTypes.ts
+++ b/sdk/src/assetTypes.ts
@@ -32,7 +32,7 @@ const ASSET_TYPE_REGISTRY: readonly AssetTypeEntry[] = [
// wechatPackInclude: WeChat's fs denies package reads of unlisted custom
// extensions — KTX2 textures are fs-read (transcoded), not image-decoded.
{ extensions: ['ktx2'], contentType: 'binary', editorType: 'texture', addressableType: 'texture', wechatPackInclude: true, hasTransitiveDeps: false },
- { extensions: ['mp3', 'wav', 'ogg', 'aac', 'flac', 'webm'], contentType: 'audio', editorType: 'audio', addressableType: 'audio', wechatPackInclude: false, hasTransitiveDeps: false },
+ { extensions: ['mp3', 'wav', 'ogg', 'aac', 'flac', 'm4a', 'webm'], contentType: 'audio', editorType: 'audio', addressableType: 'audio', wechatPackInclude: false, hasTransitiveDeps: false },
{ extensions: ['esmaterial'], contentType: 'json', editorType: 'material', addressableType: 'material', wechatPackInclude: true, hasTransitiveDeps: true },
{ extensions: ['esshader'], contentType: 'text', editorType: 'shader', addressableType: null, wechatPackInclude: false, hasTransitiveDeps: false },
{ extensions: ['atlas'], contentType: 'text', editorType: 'spine-atlas', addressableType: 'binary', wechatPackInclude: true, hasTransitiveDeps: true },
@@ -68,6 +68,7 @@ const MIME_MAP: Record<string, string> = {
wav: 'audio/wav',
ogg: 'audio/ogg',
aac: 'audio/aac',
+ m4a: 'audio/mp4',
flac: 'audio/flac',
webm: 'audio/webm',
json: 'application/json',