A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(editor): audio assets get a waveform inspector + import settings
esengine04cce185fe55
diff --git a/desktop/src/components/AudioWavePreview.tsx b/desktop/src/components/AudioWavePreview.tsxnew file mode 100644index 000000000..ca58b25dc--- /dev/null+++ b/desktop/src/components/AudioWavePreview.tsx@@ -0,0 +1,159 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file AudioWavePreview.tsx+ * @brief Waveform + transport for the audio asset inspector: fetches the+ * clip over estella://, decodes it once (a module-level AudioContext,+ * decode-only), draws min/max peak columns on a canvas, and drives a+ * plain <audio> element for play/pause/click-seek with a playhead.+ */+import { useEffect, useRef, useState } from 'react';+import { Play, Pause } from 'lucide-react';+import { t } from '@/i18n';++let decodeCtx: AudioContext | null = null;+const decoder = (): AudioContext => (decodeCtx ??= new AudioContext());++interface WaveInfo {+ peaks: Float32Array; // interleaved min,max per column+ duration: number;+ channels: number;+ sampleRate: number;+}++const COLUMNS = 400;++async function loadWave(path: string): Promise<WaveInfo> {+ const bytes = await (await fetch(`estella://project/${path}`)).arrayBuffer();+ const buffer = await decoder().decodeAudioData(bytes);+ const peaks = new Float32Array(COLUMNS * 2);+ const data = buffer.getChannelData(0);+ const second = buffer.numberOfChannels > 1 ? buffer.getChannelData(1) : null;+ const step = data.length / COLUMNS;+ for (let c = 0; c < COLUMNS; c++) {+ let min = 0;+ let max = 0;+ const start = Math.floor(c * step);+ const end = Math.min(data.length, Math.ceil((c + 1) * step));+ for (let i = start; i < end; i++) {+ const v = second ? (data[i] + second[i]) * 0.5 : data[i];+ if (v < min) min = v;+ if (v > max) max = v;+ }+ peaks[c * 2] = min;+ peaks[c * 2 + 1] = max;+ }+ return { peaks, duration: buffer.duration, channels: buffer.numberOfChannels, sampleRate: buffer.sampleRate };+}++function drawWave(canvas: HTMLCanvasElement, wave: WaveInfo | null, progress: number): void {+ const ctx = canvas.getContext('2d');+ if (!ctx) return;+ const dpr = window.devicePixelRatio || 1;+ const w = canvas.clientWidth;+ const h = canvas.clientHeight;+ if (canvas.width !== w * dpr || canvas.height !== h * dpr) {+ canvas.width = w * dpr;+ canvas.height = h * dpr;+ }+ ctx.setTransform(dpr, 0, 0, dpr, 0, 0);+ ctx.clearRect(0, 0, w, h);+ if (!wave) return;+ const styles = getComputedStyle(canvas);+ const base = styles.getPropertyValue('--star').trim() || '#6ea8ff';+ const dim = styles.getPropertyValue('--text-faint').trim() || '#666';+ const mid = h / 2;+ const colW = w / COLUMNS;+ const playedCols = Math.floor(progress * COLUMNS);+ for (let c = 0; c < COLUMNS; c++) {+ const min = wave.peaks[c * 2];+ const max = wave.peaks[c * 2 + 1];+ ctx.fillStyle = c <= playedCols && progress > 0 ? base : dim;+ const top = mid + min * (mid - 2);+ const bottom = mid + max * (mid - 2);+ ctx.fillRect(c * colW, Math.min(top, bottom), Math.max(1, colW - 0.5), Math.max(1, Math.abs(bottom - top)));+ }+ if (progress > 0) {+ ctx.fillStyle = base;+ ctx.fillRect(progress * w - 0.5, 0, 1, h);+ }+}++const fmtTime = (s: number) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, '0')}`;++export function AudioWavePreview({ path }: { path: string }) {+ const canvasRef = useRef<HTMLCanvasElement>(null);+ const audioRef = useRef<HTMLAudioElement | null>(null);+ const [wave, setWave] = useState<WaveInfo | null>(null);+ const [failed, setFailed] = useState(false);+ const [playing, setPlaying] = useState(false);+ const [progress, setProgress] = useState(0);++ useEffect(() => {+ let alive = true;+ setWave(null);+ setFailed(false);+ setPlaying(false);+ setProgress(0);+ void loadWave(path)+ .then((v) => alive && setWave(v))+ .catch(() => alive && setFailed(true));+ const audio = new Audio(`estella://project/${path}`);+ audioRef.current = audio;+ const onTime = () => setProgress(audio.duration > 0 ? audio.currentTime / audio.duration : 0);+ const onEnd = () => { setPlaying(false); setProgress(0); };+ audio.addEventListener('timeupdate', onTime);+ audio.addEventListener('ended', onEnd);+ return () => {+ alive = false;+ audio.pause();+ audio.removeEventListener('timeupdate', onTime);+ audio.removeEventListener('ended', onEnd);+ audioRef.current = null;+ };+ }, [path]);++ useEffect(() => {+ if (canvasRef.current) drawWave(canvasRef.current, wave, progress);+ }, [wave, progress]);++ const toggle = () => {+ const audio = audioRef.current;+ if (!audio) return;+ if (playing) {+ audio.pause();+ setPlaying(false);+ } else {+ void audio.play();+ setPlaying(true);+ }+ };++ const seek = (e: React.MouseEvent) => {+ const audio = audioRef.current;+ const canvas = canvasRef.current;+ if (!audio || !canvas || !Number.isFinite(audio.duration)) return;+ const r = canvas.getBoundingClientRect();+ const p = Math.max(0, Math.min(1, (e.clientX - r.left) / r.width));+ audio.currentTime = p * audio.duration;+ setProgress(p);+ };++ return (+ <div className="awp">+ <canvas ref={canvasRef} className="awp-wave" onClick={seek} title={t('det.audioSeekTip')} />+ <div className="awp-bar">+ <button type="button" className="awp-play" onClick={toggle} title={t('det.audioPlayTip')}>+ {playing ? <Pause size={13} /> : <Play size={13} />}+ </button>+ <span className="awp-meta">+ {failed+ ? t('det.audioDecodeFailed')+ : wave+ ? `${fmtTime(wave.duration)} · ${wave.channels} ch · ${(wave.sampleRate / 1000).toFixed(1)} kHz`+ : '…'}+ </span>+ </div>+ </div>+ );+}diff --git a/desktop/src/i18n/messages/details.ts b/desktop/src/i18n/messages/details.tsindex b0dd72013..ad785fa1e 100644--- a/desktop/src/i18n/messages/details.ts+++ b/desktop/src/i18n/messages/details.ts@@ -164,6 +164,9 @@ export const detailsMessages = defineMessages({'det.importSaved': { en: 'Import settings saved', zh: '导入设置已保存' },'det.importSaveFailed': { en: 'Failed to save import settings: {error}', zh: '保存导入设置失败:{error}' },'det.noImportSettings': { en: 'This asset type has no import settings.', zh: '此资产类型没有导入设置。' },+ 'det.audioSeekTip': { en: 'Click to seek', zh: '点击跳转播放位置' },+ 'det.audioPlayTip': { en: 'Play / pause', zh: '播放 / 暂停' },+ 'det.audioDecodeFailed': { en: 'Could not decode audio', zh: '无法解码音频' },'det.type': { en: 'Type', zh: '类型' },'det.metaPath': { en: 'Path', zh: '路径' },'det.metaDimensions': { en: 'Dimensions', zh: '尺寸' },diff --git a/desktop/src/panels/Details.tsx b/desktop/src/panels/Details.tsxindex 3a1d7a1b9..e83049f79 100644--- a/desktop/src/panels/Details.tsx+++ b/desktop/src/panels/Details.tsx@@ -30,6 +30,7 @@ import {type LucideIcon,} from 'lucide-react';import { AssetIcon } from '@/components/icons';+import { AudioWavePreview } from '@/components/AudioWavePreview';import { Toasts } from '@/store/Toasts';import { baseName, assetTypeOf, IMAGE_RE } from '@/project/assetMeta';import { revealAsset } from '@/project/assetReveal';@@ -2316,15 +2317,19 @@ function GenericAssetInspector({ path }: { path: string }) {<div className="insp-body">{/* A compact preview for a quick visual ID — kept short so the editableImport Settings sit above the fold (the reason to open an asset). */}- <div className="cb-prev" style={{ height: 108 }}>- <div className="pv">- {isImage ? (- <img src={`estella://project/${path}`} alt="" draggable={false} />- ) : (- <AssetIcon type={type} size={44} />- )}+ {type === 'audio' ? (+ <AudioWavePreview path={path} />+ ) : (+ <div className="cb-prev" style={{ height: 108 }}>+ <div className="pv">+ {isImage ? (+ <img src={`estella://project/${path}`} alt="" draggable={false} />+ ) : (+ <AssetIcon type={type} size={44} />+ )}+ </div></div>- </div>+ )}{/* Editable import settings first (the reason to select an asset), thenread-only metadata. */}diff --git a/desktop/src/project/assetImporter.ts b/desktop/src/project/assetImporter.tsindex faa2ed59a..4a60313e4 100644--- a/desktop/src/project/assetImporter.ts+++ b/desktop/src/project/assetImporter.ts@@ -49,6 +49,19 @@ const SPINE: ImporterFieldSpec[] = [{ key: 'premultiplyAlpha', label: 'Premultiply Alpha', type: 'bool', default: false, category: 'Spine', advanced: true },];+const AUDIO: ImporterFieldSpec[] = [+ {+ key: 'compress', label: 'Compress', type: 'bool', default: true, category: 'Audio',+ tooltip: 'Re-encode WAV to MP3 at cook (already-compressed formats pass through). '+ + 'MP3 has a small encoder delay — disable for seamless-loop clips.',+ },+ {+ key: 'bitrateKbps', label: 'Bitrate', type: 'enum', default: 128, category: 'Audio',+ options: [96, 128, 192].map((n) => ({ label: `${n} kbps`, value: n })),+ tooltip: 'MP3 bitrate for cooked WAV sources.',+ },+];+const SCENELIKE: ImporterFieldSpec[] = [{key: 'autoMigrate', label: 'Auto Migrate', type: 'bool', default: true, category: 'Import',@@ -62,10 +75,23 @@ export const IMPORTER_SCHEMAS: Record<string, ImporterFieldSpec[]> = {texture: TEXTURE,sprite: TEXTURE,spine: SPINE,+ audio: AUDIO,scene: SCENELIKE,prefab: SCENELIKE,};+/** Cook-facing audio import settings, tolerant of hand-edited `.meta` blocks. */+export function readAudioImportSettings(importer: Record<string, unknown> | undefined): {+ compress: boolean; bitrateKbps: number;+} {+ const compress = importer?.compress;+ const bitrate = importer?.bitrateKbps;+ return {+ compress: typeof compress === 'boolean' ? compress : true,+ bitrateKbps: bitrate === 96 || bitrate === 128 || bitrate === 192 ? bitrate : 128,+ };+}+const getByPath = (obj: Record<string, unknown>, path: string): unknown =>path.split('.').reduce<unknown>((o, k) => (o && typeof o === 'object' ? (o as Record<string, unknown>)[k] : undefined), obj);diff --git a/desktop/src/project/assetTypes.ts b/desktop/src/project/assetTypes.tsindex 2631342c5..b8d307264 100644--- a/desktop/src/project/assetTypes.ts+++ b/desktop/src/project/assetTypes.ts@@ -36,7 +36,7 @@ export const ASSET_TYPES: Record<AssetType, AssetTypeDef> = {texture: { extensions: ['png', 'webp'], badge: 'TEX', icon: FileImage, tint: '#7fa6c4' },sprite: { extensions: ['jpg', 'jpeg', 'gif'], badge: 'IMG', icon: Image, tint: '#7fa6c4' },spine: { extensions: ['atlas', 'skel'], badge: 'SPN', icon: PersonStanding, tint: '#9b8fc0' },- audio: { extensions: ['ogg', 'mp3', 'wav'], badge: 'AUD', icon: Music, tint: '#7faf9c' },+ audio: { extensions: ['ogg', 'mp3', 'wav', 'aac', 'flac', 'm4a', 'webm'], badge: 'AUD', icon: Music, tint: '#7faf9c' },prefab: { extensions: ['esprefab'], badge: 'PFB', icon: Component, tint: '#c2a274' },// .esmaterial is the real extension (the SDK MaterialAssetLoader only loads it);// .esmat is tolerated as a legacy alias (cf. electron/importAssets.ts).diff --git a/desktop/src/theme/inspector.css b/desktop/src/theme/inspector.cssindex ebb0db174..615b8158a 100644--- a/desktop/src/theme/inspector.css+++ b/desktop/src/theme/inspector.css@@ -299,3 +299,41 @@.lt-miss-key { font-family: var(--mono); font-size: var(--fs-xs); color: var(--text-dim); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; padding-left: 2px; }.lt-miss-ref { font-size: var(--fs-2xs); color: var(--text-mute); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; }.lt-error { margin: 8px; padding: 6px 9px; font-size: var(--fs-2xs); line-height: 1.5; font-family: var(--mono); color: var(--text-dim); background: var(--inset); border: 1px solid var(--border); border-radius: var(--r-sm); word-break: break-all; }++/* Audio asset waveform preview (AudioWavePreview) */+.awp {+ display: flex;+ flex-direction: column;+ border-bottom: 1px solid var(--line);+ background: var(--inset);+}+.awp-wave {+ width: 100%;+ height: 84px;+ display: block;+ cursor: pointer;+}+.awp-bar {+ display: flex;+ align-items: center;+ gap: 8px;+ padding: 4px 8px;+ border-top: 1px solid var(--line);+ background: var(--srf-2);+}+.awp-play {+ display: inline-flex;+ align-items: center;+ background: transparent;+ color: var(--star-hi);+ border: 1px solid var(--star-line);+ border-radius: var(--r-sm);+ padding: 3px 6px;+ cursor: pointer;+}+.awp-play:hover { background: var(--star-dim); }+.awp-meta {+ color: var(--text-faint);+ font-size: var(--fs-2xs);+ font-variant-numeric: tabular-nums;+}diff --git a/desktop/tests/asset-importer.test.ts b/desktop/tests/asset-importer.test.tsindex fb518e286..40c0c1888 100644--- a/desktop/tests/asset-importer.test.ts+++ b/desktop/tests/asset-importer.test.ts@@ -30,7 +30,7 @@ describe('importerDefaults (import-time .meta)', () => {expect(importerDefaults('spine')).toEqual({ scale: 1, defaultSkin: 'default', premultiplyAlpha: false });expect(importerDefaults('scene')).toEqual({ autoMigrate: true });expect(importerDefaults('prefab')).toEqual({ autoMigrate: true });- expect(importerDefaults('audio')).toEqual({});+ expect(importerDefaults('audio')).toEqual({ compress: true, bitrateKbps: 128 });});});@@ -38,7 +38,7 @@ describe('hasImporterSettings', () => {it('is true only for types with editable settings', () => {expect(hasImporterSettings('texture')).toBe(true);expect(hasImporterSettings('spine')).toBe(true);- expect(hasImporterSettings('audio')).toBe(false);+ expect(hasImporterSettings('audio')).toBe(true);expect(hasImporterSettings('font')).toBe(false);});});@@ -57,7 +57,7 @@ describe('buildImporterComponent (inspector fields)', () => {expect(byKey['sliceBorder.top'].value).toBe(0); // nested default});it('returns null for types without settings', () => {- expect(buildImporterComponent('audio', {})).toBeNull();+ expect(buildImporterComponent('audio', {})).not.toBeNull();});});diff --git a/sdk/src/asset/loaders/AudioAssetLoader.ts b/sdk/src/asset/loaders/AudioAssetLoader.tsindex dabe11ec9..3b3f8e155 100644--- a/sdk/src/asset/loaders/AudioAssetLoader.ts+++ b/sdk/src/asset/loaders/AudioAssetLoader.ts@@ -6,7 +6,7 @@ import { log } from '../../logger';export class AudioAssetLoader implements AssetLoader<AudioResult> {readonly type = 'audio';- readonly extensions = ['.mp3', '.wav', '.ogg', '.m4a', '.aac'];+ readonly extensions = ['.mp3', '.wav', '.ogg', '.m4a', '.aac', '.flac', '.webm'];/** Same lazy audio accessor Assets uses — needed by unload/invalidate,* which have no LoadContext. Defaults to "no audio" for bare setups. */diff --git a/sdk/src/assetTypes.ts b/sdk/src/assetTypes.tsindex f845e2002..6805be0ef 100644--- a/sdk/src/assetTypes.ts+++ b/sdk/src/assetTypes.ts@@ -32,7 +32,7 @@ const ASSET_TYPE_REGISTRY: readonly AssetTypeEntry[] = [// wechatPackInclude: WeChat's fs denies package reads of unlisted custom// extensions — KTX2 textures are fs-read (transcoded), not image-decoded.{ extensions: ['ktx2'], contentType: 'binary', editorType: 'texture', addressableType: 'texture', wechatPackInclude: true, hasTransitiveDeps: false },- { extensions: ['mp3', 'wav', 'ogg', 'aac', 'flac', 'webm'], contentType: 'audio', editorType: 'audio', addressableType: 'audio', wechatPackInclude: false, hasTransitiveDeps: false },+ { extensions: ['mp3', 'wav', 'ogg', 'aac', 'flac', 'm4a', 'webm'], contentType: 'audio', editorType: 'audio', addressableType: 'audio', wechatPackInclude: false, hasTransitiveDeps: false },{ extensions: ['esmaterial'], contentType: 'json', editorType: 'material', addressableType: 'material', wechatPackInclude: true, hasTransitiveDeps: true },{ extensions: ['esshader'], contentType: 'text', editorType: 'shader', addressableType: null, wechatPackInclude: false, hasTransitiveDeps: false },{ extensions: ['atlas'], contentType: 'text', editorType: 'spine-atlas', addressableType: 'binary', wechatPackInclude: true, hasTransitiveDeps: true },@@ -68,6 +68,7 @@ const MIME_MAP: Record<string, string> = {wav: 'audio/wav',ogg: 'audio/ogg',aac: 'audio/aac',+ m4a: 'audio/mp4',flac: 'audio/flac',webm: 'audio/webm',json: 'application/json',