Skip to content

šŸŽ„ Make videos programmatically with React

`@remotion/studio`: Show average audio volume in asset inspector and Convert

JonnyBurgereba1d1ef72d6
diff --git a/packages/bundler/src/shared-bundler-config.ts b/packages/bundler/src/shared-bundler-config.ts
index c50af391f3..0e1eab4073 100644
--- a/packages/bundler/src/shared-bundler-config.ts
+++ b/packages/bundler/src/shared-bundler-config.ts
@@ -61,8 +61,27 @@ export const getResolveConfig = () => ({
'worker.mjs',
),
}),
- // test visual controls before removing this
- '@remotion/studio': require.resolve('@remotion/studio'),
+ // Studio entry points are also passed to the bundler as resolved CJS paths.
+ // Route both those paths and package imports through the same ESM build.
+ ...Object.fromEntries(
+ [
+ '@remotion/studio',
+ '@remotion/studio/internals',
+ '@remotion/studio/previewEntry',
+ '@remotion/studio/renderEntry',
+ ].flatMap((specifier) => {
+ const commonJsPath = require.resolve(specifier);
+ const esmPath = path.join(
+ path.dirname(commonJsPath),
+ 'esm',
+ path.basename(commonJsPath, '.js') + '.mjs',
+ );
+ return [
+ [`${specifier}$`, esmPath],
+ [commonJsPath, esmPath],
+ ];
+ }),
+ ),
[path.join(
require.resolve('@remotion/timeline-utils'),
'..',
diff --git a/packages/convert/app/components/AudioTrackOverview.tsx b/packages/convert/app/components/AudioTrackOverview.tsx
index b1e4f4f850..c000c9713a 100644
--- a/packages/convert/app/components/AudioTrackOverview.tsx
+++ b/packages/convert/app/components/AudioTrackOverview.tsx
@@ -1,6 +1,7 @@
import type {InputAudioTrack} from 'mediabunny';
import {useEffect, useState} from 'react';
import {renderHumanReadableAudioCodec} from '~/lib/render-codec-label';
+import {AudioVolumeRow} from './AudioVolumeRow';
import {PacketList} from './PacketList';
import {TextButtonWithChevron} from './TexrButtonWithChevron';
import {Table, TableBody, TableCell, TableRow} from './ui/table';
@@ -59,6 +60,7 @@ export const AudioTrackOverview: React.FC<{
<TableCell className="font-brand">Sample Rate</TableCell>
<TableCell className="text-right">{sampleRate ?? '...'}</TableCell>
</TableRow>
+ <AudioVolumeRow track={track} />
</TableBody>
</Table>
);
diff --git a/packages/convert/app/components/AudioVolumeRow.tsx b/packages/convert/app/components/AudioVolumeRow.tsx
new file mode 100644
index 0000000000..b1fd0ce1f9
--- /dev/null
+++ b/packages/convert/app/components/AudioVolumeRow.tsx
@@ -0,0 +1,41 @@
+import {
+ formatAverageAudioVolume,
+ getAverageAudioVolume,
+} from '@remotion/timeline-utils';
+import type {InputAudioTrack} from 'mediabunny';
+import {useEffect, useState} from 'react';
+import {TableCell, TableRow} from './ui/table';
+
+export const AudioVolumeRow: React.FC<{readonly track: InputAudioTrack}> = ({
+ track,
+}) => {
+ const [value, setValue] = useState<string | null>(null);
+ useEffect(() => {
+ const controller = new AbortController();
+ setValue('Calculating…');
+ getAverageAudioVolume({track, signal: controller.signal})
+ .then((volume) => {
+ if (!controller.signal.aborted) {
+ setValue(formatAverageAudioVolume(volume));
+ }
+ })
+ .catch(() => {
+ if (!controller.signal.aborted) {
+ setValue('Unavailable');
+ }
+ });
+ return () => controller.abort();
+ }, [track]);
+
+ return (
+ <TableRow>
+ <TableCell
+ className="font-brand"
+ title="RMS level across all channels over the full audio track, including silence."
+ >
+ Average volume
+ </TableCell>
+ <TableCell className="text-right">{value ?? 'Calculating…'}</TableCell>
+ </TableRow>
+ );
+};
diff --git a/packages/convert/app/components/ContainerOverview.tsx b/packages/convert/app/components/ContainerOverview.tsx
index 5145ee8ce6..ad3f559343 100644
--- a/packages/convert/app/components/ContainerOverview.tsx
+++ b/packages/convert/app/components/ContainerOverview.tsx
@@ -14,6 +14,7 @@ import {
renderHumanReadableAudioCodec,
renderHumanReadableVideoCodec,
} from '~/lib/render-codec-label';
+import {AudioVolumeRow} from './AudioVolumeRow';
import {MetadataDisplay} from './MetadataTable';
import {Skeleton} from './ui/skeleton';
@@ -29,6 +30,7 @@ export const ContainerOverview: React.FC<{
readonly metadata: MetadataTags | null;
readonly isAudioOnly: boolean;
readonly sampleRate: number | null;
+ readonly audioTrack: InputAudioTrack | null;
}> = ({
container,
dimensions,
@@ -41,6 +43,7 @@ export const ContainerOverview: React.FC<{
metadata,
isAudioOnly,
sampleRate,
+ audioTrack,
}) => {
return (
<Table className="table-fixed">
@@ -155,6 +158,7 @@ export const ContainerOverview: React.FC<{
</TableCell>
</TableRow>
)}
+ {audioTrack ? <AudioVolumeRow track={audioTrack} /> : null}
{metadata ? <MetadataDisplay metadata={metadata} /> : null}
</TableBody>
</Table>
diff --git a/packages/convert/app/components/Probe.tsx b/packages/convert/app/components/Probe.tsx
index c7c94b9087..ffb2445841 100644
--- a/packages/convert/app/components/Probe.tsx
+++ b/packages/convert/app/components/Probe.tsx
@@ -221,6 +221,9 @@ export const Probe: React.FC<{
metadata={metadata}
isHdr={isHdr}
sampleRate={sampleRate}
+ audioTrack={
+ tracks?.find((track) => track.isAudioTrack()) ?? null
+ }
/>
) : selectedTrack.isVideoTrack() ? (
<VideoTrackOverview
diff --git a/packages/convert/test/audio-volume.test.tsx b/packages/convert/test/audio-volume.test.tsx
new file mode 100644
index 0000000000..33fb1aab1a
--- /dev/null
+++ b/packages/convert/test/audio-volume.test.tsx
@@ -0,0 +1,94 @@
+import {afterEach, expect, test} from 'bun:test';
+import {cleanup, render, screen} from '@testing-library/react';
+import {ALL_FORMATS, BufferSource, Input} from 'mediabunny';
+import {ContainerOverview} from '../app/components/ContainerOverview';
+
+afterEach(cleanup);
+
+test('calculates full-track stereo RMS in Convert, including silence', async () => {
+ // Two seconds: left channel at half scale for one second, then silence;
+ // right channel silent throughout. Mean square = 0.5² / 2 / 2.
+ const frames = 16000;
+ const wav = new Uint8Array(44 + frames * 4);
+ const view = new DataView(wav.buffer);
+ for (const [offset, text] of [
+ [0, 'RIFF'],
+ [8, 'WAVE'],
+ [12, 'fmt '],
+ [36, 'data'],
+ ] as const) {
+ wav.set(new TextEncoder().encode(text), offset);
+ }
+ view.setUint32(4, wav.length - 8, true);
+ view.setUint32(16, 16, true);
+ view.setUint16(20, 1, true);
+ view.setUint16(22, 2, true);
+ view.setUint32(24, 8000, true);
+ view.setUint32(28, 32000, true);
+ view.setUint16(32, 4, true);
+ view.setUint16(34, 16, true);
+ view.setUint32(40, frames * 4, true);
+ for (let frame = 0; frame < frames / 2; frame++) {
+ view.setInt16(44 + frame * 4, frame % 2 === 0 ? 16384 : -16384, true);
+ }
+
+ const input = new Input({
+ formats: ALL_FORMATS,
+ source: new BufferSource(wav),
+ });
+ try {
+ const audioTrack = await input.getPrimaryAudioTrack();
+ const mounted = render(
+ <ContainerOverview
+ dimensions={null}
+ durationInSeconds={2}
+ videoCodec={null}
+ audioCodec="pcm-s16"
+ size={wav.length}
+ frameRate={null}
+ container={null}
+ isHdr={false}
+ metadata={null}
+ isAudioOnly
+ audioTrack={audioTrack}
+ sampleRate={8000}
+ />,
+ );
+ expect(screen.getByText('Average volume')).toBeTruthy();
+ expect(screen.queryByRole('button', {name: 'Calculate'})).toBeNull();
+ expect(await screen.findByText('-12.0 dBFS')).toBeTruthy();
+ mounted.unmount();
+ } finally {
+ input.dispose();
+ }
+
+ wav.fill(0, 44);
+ const silentInput = new Input({
+ formats: ALL_FORMATS,
+ source: new BufferSource(wav),
+ });
+ try {
+ const audioTrack = await silentInput.getPrimaryAudioTrack();
+ render(
+ <ContainerOverview
+ dimensions={null}
+ durationInSeconds={2}
+ videoCodec={null}
+ audioCodec="pcm-s16"
+ size={wav.length}
+ frameRate={null}
+ container={null}
+ isHdr={false}
+ metadata={null}
+ isAudioOnly
+ audioTrack={audioTrack}
+ sampleRate={8000}
+ />,
+ );
+ expect(screen.queryByRole('button', {name: 'Calculate'})).toBeNull();
+ expect(await screen.findByText('āˆ’āˆž dBFS (silent)')).toBeTruthy();
+ } finally {
+ cleanup();
+ silentInput.dispose();
+ }
+});
diff --git a/packages/convert/test/frame-rate.test.tsx b/packages/convert/test/frame-rate.test.tsx
index c32799b94e..b821b9a656 100644
--- a/packages/convert/test/frame-rate.test.tsx
+++ b/packages/convert/test/frame-rate.test.tsx
@@ -25,6 +25,7 @@ const FrameRateProbe: React.FC<{file: File}> = ({file}) => {
metadata={null}
isAudioOnly={false}
sampleRate={null}
+ audioTrack={null}
/>
<div data-testid="timeline-fps">{fps}</div>
</>
diff --git a/packages/core/src/CompositionManagerContext.tsx b/packages/core/src/CompositionManagerContext.tsx
index c48b583229..ebce566abb 100644
--- a/packages/core/src/CompositionManagerContext.tsx
+++ b/packages/core/src/CompositionManagerContext.tsx
@@ -22,6 +22,7 @@ export type BaseMetadata = Pick<
export type AssetPreviewMetadata = BaseMetadata & {
asset: string;
+ src: string;
};
export const getAssetPreviewCompositionId = (asset: string) => `asset:${asset}`;
diff --git a/packages/studio/src/components/AssetAudioVolume.tsx b/packages/studio/src/components/AssetAudioVolume.tsx
new file mode 100644
index 0000000000..792e7b7d11
--- /dev/null
+++ b/packages/studio/src/components/AssetAudioVolume.tsx
@@ -0,0 +1,44 @@
+import {
+ formatAverageAudioVolume,
+ subscribeToWaveformPeaks,
+} from '@remotion/timeline-utils';
+import React, {useEffect, useState} from 'react';
+import {LIGHT_TEXT} from '../helpers/colors';
+
+export const AssetAudioVolume: React.FC<{
+ readonly src: string | null;
+ readonly waveformSampleRate: number | null;
+}> = ({src, waveformSampleRate}) => {
+ const [value, setValue] = useState('Calculating…');
+ useEffect(() => {
+ setValue('Calculating…');
+ if (src === null || waveformSampleRate === null) {
+ return;
+ }
+
+ return subscribeToWaveformPeaks({
+ src,
+ waveformSampleRate,
+ onPeaks: (_peaks, final, averageVolume) => {
+ if (final) {
+ setValue(formatAverageAudioVolume(averageVolume));
+ }
+ },
+ onError: () => setValue('Unavailable'),
+ });
+ }, [src, waveformSampleRate]);
+
+ return (
+ <span
+ style={{
+ fontFamily: 'sans-serif',
+ fontSize: 13,
+ lineHeight: '20px',
+ color: LIGHT_TEXT,
+ }}
+ title="RMS level across all channels over the full audio track, including silence."
+ >
+ {value}
+ </span>
+ );
+};
diff --git a/packages/studio/src/components/CurrentAsset.tsx b/packages/studio/src/components/CurrentAsset.tsx
index 4df55ec986..d3ef3da9e0 100644
--- a/packages/studio/src/components/CurrentAsset.tsx
+++ b/packages/studio/src/components/CurrentAsset.tsx
@@ -7,6 +7,7 @@ import {CURRENT_COLOR, LIGHT_TEXT} from '../helpers/colors';
import {formatMediaDuration} from '../helpers/format-media-duration';
import {getFileManagerName} from '../helpers/get-file-manager-name';
import {getPreviewFileType} from '../helpers/get-preview-file-type';
+import {TIMELINE_FRAME_WIDTH_AT_MAX_ZOOM} from '../helpers/get-timeline-max-zoom';
import {openInRemotionConvert} from '../helpers/open-in-remotion-convert';
import {
renderHumanReadableAudioCodec,
@@ -18,6 +19,7 @@ import {useMediaMetadata} from '../helpers/use-media-metadata';
import {ExpandedFolderIcon} from '../icons/folder';
import {RemotionConvertIcon} from '../icons/remotion-convert';
import {TrashIcon} from '../icons/trash';
+import {AssetAudioVolume} from './AssetAudioVolume';
import {InlineEditableTitle} from './InlineEditableTitle';
import {InspectorInfoHeader} from './InspectorInfoHeader';
import {
@@ -180,6 +182,9 @@ export const AssetInfo: React.FC<{
readonly onAssetClick?: () => void;
readonly readOnlyStudio: boolean;
}> = ({assetName, contentSized = false, onAssetClick, readOnlyStudio}) => {
+ const {currentAssetMetadata} = useContext(Internals.CompositionManager);
+ const volumeMetadata =
+ currentAssetMetadata?.asset === assetName ? currentAssetMetadata : null;
const connectionStatus = useContext(StudioServerConnectionCtx)
.previewServerState.type;
const browserStudioOperations = getBrowserStudioOperations();
@@ -330,6 +335,24 @@ export const AssetInfo: React.FC<{
<span style={assetMetadataValueStyle}>{detail.value}</span>
</InspectorDetailRow>
))}
+ {src ? (
+ <InspectorDetailRow label="Average volume">
+ <span style={assetMetadataValueStyle}>
+ <AssetAudioVolume
+ key={volumeMetadata?.src ?? src}
+ src={volumeMetadata?.src ?? null}
+ waveformSampleRate={
+ volumeMetadata
+ ? Math.ceil(
+ volumeMetadata.fps *
+ TIMELINE_FRAME_WIDTH_AT_MAX_ZOOM,
+ )
+ : null
+ }
+ />
+ </span>
+ </InspectorDetailRow>
+ ) : null}
</div>
)}
</InspectorSection>
diff --git a/packages/studio/src/helpers/get-asset-metadata.ts b/packages/studio/src/helpers/get-asset-metadata.ts
index f442772be3..5ef1f274eb 100644
--- a/packages/studio/src/helpers/get-asset-metadata.ts
+++ b/packages/studio/src/helpers/get-asset-metadata.ts
@@ -71,6 +71,10 @@ export const getAssetPreviewMetadata = ({
return {
asset: canvasContent.asset,
+ src: addAssetCacheBust({
+ src: staticFile(canvasContent.asset),
+ fetchedAt: metadata.fetchedAt,
+ }),
width: dimensions.width,
height: dimensions.height,
fps,
diff --git a/packages/studio/src/test/asset-audio-volume.test.tsx b/packages/studio/src/test/asset-audio-volume.test.tsx
new file mode 100644
index 0000000000..d7e6ee56a9
--- /dev/null
+++ b/packages/studio/src/test/asset-audio-volume.test.tsx
@@ -0,0 +1,88 @@
+import {afterEach, expect, test} from 'bun:test';
+import {createServer} from 'node:http';
+import type {AddressInfo} from 'node:net';
+import {subscribeToWaveformPeaks} from '@remotion/timeline-utils';
+import {cleanup, render, screen} from '@testing-library/react';
+import {AssetAudioVolume} from '../components/AssetAudioVolume';
+
+afterEach(cleanup);
+
+test('automatically shares waveform analysis and reuses cached volume on remount', async () => {
+ // Two seconds: left channel at half scale for one second, then silence;
+ // right channel silent throughout. Mean square = 0.5² / 2 / 2.
+ const frames = 16000;
+ const wav = new Uint8Array(44 + frames * 4);
+ const view = new DataView(wav.buffer);
+ for (const [offset, text] of [
+ [0, 'RIFF'],
+ [8, 'WAVE'],
+ [12, 'fmt '],
+ [36, 'data'],
+ ] as const) {
+ wav.set(new TextEncoder().encode(text), offset);
+ }
+
+ view.setUint32(4, wav.length - 8, true);
+ view.setUint32(16, 16, true);
+ view.setUint16(20, 1, true);
+ view.setUint16(22, 2, true);
+ view.setUint32(24, 8000, true);
+ view.setUint32(28, 32000, true);
+ view.setUint16(32, 4, true);
+ view.setUint16(34, 16, true);
+ view.setUint32(40, frames * 4, true);
+ for (let frame = 0; frame < frames / 2; frame++) {
+ view.setInt16(44 + frame * 4, frame % 2 === 0 ? 16384 : -16384, true);
+ }
+
+ let requests = 0;
+ const server = createServer((request, response) => {
+ requests++;
+ const range = request.headers.range?.match(/bytes=(\d+)-(\d*)/);
+ const start = range ? Number(range[1]) : 0;
+ const end = range?.[2]
+ ? Math.min(Number(range[2]), wav.length - 1)
+ : wav.length - 1;
+ response.writeHead(range ? 206 : 200, {
+ 'Content-Type': 'audio/wav',
+ 'Content-Length': end - start + 1,
+ 'Accept-Ranges': 'bytes',
+ 'Content-Range': `bytes ${start}-${end}/${wav.length}`,
+ });
+ response.end(wav.slice(start, end + 1));
+ });
+ await new Promise<void>((resolve) => server.listen(0, resolve));
+ const src = `http://localhost:${(server.address() as AddressInfo).port}/stereo.wav`;
+ let unsubscribe = () => {};
+ try {
+ const waveform = new Promise<void>((resolve, reject) => {
+ unsubscribe = subscribeToWaveformPeaks({
+ src,
+ waveformSampleRate: 100,
+ onPeaks: (peaks, final, volume) => {
+ if (!final) return;
+ expect(peaks.length).toBe(200);
+ expect(volume!).toBeCloseTo(-12.0412, 3);
+ resolve();
+ },
+ onError: reject,
+ });
+ });
+ const mounted = render(
+ <AssetAudioVolume src={src} waveformSampleRate={100} />,
+ );
+ expect(screen.queryByRole('button', {name: 'Calculate'})).toBeNull();
+ expect(await screen.findByText('-12.0 dBFS')).toBeTruthy();
+ await waveform;
+ expect(requests).toBe(1);
+ mounted.unmount();
+ render(<AssetAudioVolume src={src} waveformSampleRate={100} />);
+ expect(await screen.findByText('-12.0 dBFS')).toBeTruthy();
+ expect(requests).toBe(1);
+ } finally {
+ unsubscribe();
+ cleanup();
+ server.closeAllConnections();
+ await new Promise<void>((resolve) => server.close(() => resolve()));
+ }
+});
diff --git a/packages/timeline-utils/src/audio-waveform-worker.ts b/packages/timeline-utils/src/audio-waveform-worker.ts
index 09393b388f..f890bc5f1a 100644
--- a/packages/timeline-utils/src/audio-waveform-worker.ts
+++ b/packages/timeline-utils/src/audio-waveform-worker.ts
@@ -8,11 +8,17 @@ import {loadWaveformPeaks} from './audio-waveform/load-waveform-peaks';
declare const self: DedicatedWorkerGlobalScope;
-const postPeaks = (requestId: number, peaks: Float32Array, final: boolean) => {
+const postPeaks = (
+ requestId: number,
+ peaks: Float32Array,
+ final: boolean,
+ averageVolume: number | null,
+) => {
// Structured cloning copies the array, so the decoder can keep
// mutating its buffer while the main thread reads the snapshot.
const payload: AudioWaveformWorkerOutgoingMessage = {
type: 'peaks',
+ averageVolume,
requestId,
peaks,
final,
@@ -42,12 +48,12 @@ self.addEventListener(
waveformSampleRate: message.waveformSampleRate,
onProgress: ({peaks, final}) => {
if (!final) {
- postPeaks(message.requestId, peaks, false);
+ postPeaks(message.requestId, peaks, false, null);
}
},
})
- .then((peaks) => {
- postPeaks(message.requestId, peaks, true);
+ .then(({peaks, averageVolume}) => {
+ postPeaks(message.requestId, peaks, true, averageVolume);
})
.catch((error) => {
postError(message.requestId, error);
diff --git a/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts b/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts
index bb4f703152..f0462ce699 100644
--- a/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts
+++ b/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts
@@ -12,6 +12,7 @@ export type AudioWaveformWorkerPeaksMessage = {
readonly requestId: number;
readonly peaks: Float32Array;
readonly final: boolean;
+ readonly averageVolume: number | null;
};
export type AudioWaveformWorkerErrorMessage = {
diff --git a/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.ts b/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.ts
index 0d82d15898..8a80b6d57c 100644
--- a/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.ts
+++ b/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.ts
@@ -7,11 +7,17 @@ import {
} from './waveform-peak-processor';
const DEFAULT_PROGRESS_INTERVAL_IN_MS = 50;
-const peaksCache = new Map<string, Float32Array>();
+export type WaveformResult = {
+ readonly peaks: Float32Array;
+ readonly averageVolume: number | null;
+};
+
+const peaksCache = new Map<string, WaveformResult>();
export {TARGET_SAMPLE_RATE};
type Progress = {
+ readonly averageVolume: number | null;
readonly peaks: Float32Array;
readonly completedPeaks: number;
readonly totalPeaks: number;
@@ -28,7 +34,7 @@ export async function loadWaveformPeaks(
url: string,
signal: AbortSignal,
options?: LoadWaveformPeaksOptions,
-): Promise<Float32Array> {
+): Promise<WaveformResult> {
const waveformSampleRate = options?.waveformSampleRate ?? TARGET_SAMPLE_RATE;
if (!Number.isFinite(waveformSampleRate) || waveformSampleRate <= 0) {
throw new Error('The waveform sample rate must be a positive number.');
@@ -38,9 +44,10 @@ export async function loadWaveformPeaks(
const cached = peaksCache.get(cacheKey);
if (cached) {
emitWaveformProgress({
- peaks: cached,
- completedPeaks: cached.length,
- totalPeaks: cached.length,
+ peaks: cached.peaks,
+ averageVolume: cached.averageVolume,
+ completedPeaks: cached.peaks.length,
+ totalPeaks: cached.peaks.length,
final: true,
onProgress: options?.onProgress,
});
@@ -55,7 +62,7 @@ export async function loadWaveformPeaks(
try {
const audioTrack = await input.getPrimaryAudioTrack();
if (!audioTrack) {
- return new Float32Array(0);
+ return {peaks: new Float32Array(0), averageVolume: null};
}
if (await audioTrack.isLive()) {
@@ -97,7 +104,7 @@ export async function loadWaveformPeaks(
for await (const sample of sink.samples()) {
if (signal.aborted) {
sample.close();
- return new Float32Array(0);
+ return {peaks: new Float32Array(0), averageVolume: null};
}
const startFrame = getAudioSampleStartFrameAtTimelineZero(sample);
@@ -126,15 +133,19 @@ export async function loadWaveformPeaks(
frameCount,
});
const channels = Math.max(1, sample.numberOfChannels);
+ const {sampleRate} = sample;
sample.close();
- processor.processSampleChunk(floats, channels);
+ processor.processSampleChunk(floats, channels, sampleRate);
}
processor.finalize();
- const {peaks} = processor;
- peaksCache.set(cacheKey, peaks);
- return peaks;
+ const result = {
+ peaks: processor.peaks,
+ averageVolume: processor.averageVolume,
+ };
+ peaksCache.set(cacheKey, result);
+ return result;
} finally {
input.dispose();
}
diff --git a/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts b/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts
index 6f2be3091a..31a42534f1 100644
--- a/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts
+++ b/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts
@@ -3,11 +3,15 @@ import type {
AudioWaveformWorkerOutgoingMessage,
} from './audio-waveform-worker-types';
import {TARGET_SAMPLE_RATE} from './constants';
-import {loadWaveformPeaks} from './load-waveform-peaks';
+import {type WaveformResult, loadWaveformPeaks} from './load-waveform-peaks';
import {makeAudioWaveformWorker} from './make-audio-waveform-worker';
type WaveformPeaksListener = {
- readonly onPeaks: (peaks: Float32Array, final: boolean) => void;
+ readonly onPeaks: (
+ peaks: Float32Array,
+ final: boolean,
+ averageVolume: number | null,
+ ) => void;
readonly onError: (error: Error) => void;
};
@@ -23,7 +27,7 @@ type InFlightLoad = {
// All AudioWaveform instances share one decode worker and one peaks cache,
// so remounts (timeline virtualization, zoom window cropping) and multiple
// clips with the same src never re-fetch or re-decode the audio.
-const peaksCache = new Map<string, Float32Array>();
+const peaksCache = new Map<string, WaveformResult>();
const inFlightByCacheKey = new Map<string, InFlightLoad>();
const inFlightByRequestId = new Map<number, InFlightLoad>();
@@ -31,9 +35,14 @@ let worker: Worker | null = null;
let workerFailed = false;
let nextRequestId = 0;
-const emitPeaks = (load: InFlightLoad, peaks: Float32Array, final: boolean) => {
+const emitPeaks = (
+ load: InFlightLoad,
+ peaks: Float32Array,
+ final: boolean,
+ averageVolume: number | null,
+) => {
if (final) {
- peaksCache.set(load.cacheKey, peaks);
+ peaksCache.set(load.cacheKey, {peaks, averageVolume});
load.latestPeaks = null;
inFlightByCacheKey.delete(load.cacheKey);
if (load.requestId !== null) {
@@ -44,7 +53,7 @@ const emitPeaks = (load: InFlightLoad, peaks: Float32Array, final: boolean) => {
}
for (const listener of load.listeners) {
- listener.onPeaks(peaks, final);
+ listener.onPeaks(peaks, final, averageVolume);
}
};
@@ -75,11 +84,11 @@ const startMainThreadLoad = (load: InFlightLoad) => {
// The processor mutates the same array in place; snapshot it so
// React state updates see a new reference.
- emitPeaks(load, peaks.slice(), false);
+ emitPeaks(load, peaks.slice(), false, null);
},
})
- .then((peaks) => {
- emitPeaks(load, peaks, true);
+ .then(({peaks, averageVolume}) => {
+ emitPeaks(load, peaks, true, averageVolume);
})
.catch((error) => {
emitError(
@@ -133,7 +142,7 @@ const getOrCreateWorker = (): Worker | null => {
return;
}
- emitPeaks(load, message.peaks, message.final);
+ emitPeaks(load, message.peaks, message.final, message.averageVolume);
},
);
worker.addEventListener('error', (event) => {
@@ -152,13 +161,17 @@ export const subscribeToWaveformPeaks = ({
}: {
readonly src: string;
readonly waveformSampleRate?: number;
- readonly onPeaks: (peaks: Float32Array, final: boolean) => void;
+ readonly onPeaks: (
+ peaks: Float32Array,
+ final: boolean,
+ averageVolume: number | null,
+ ) => void;
readonly onError: (error: Error) => void;
}): (() => void) => {
const cacheKey = `${waveformSampleRate}:${src}`;
const cached = peaksCache.get(cacheKey);
if (cached) {
- onPeaks(cached, true);
+ onPeaks(cached.peaks, true, cached.averageVolume);
return () => undefined;
}
@@ -168,7 +181,7 @@ export const subscribeToWaveformPeaks = ({
if (existing) {
existing.listeners.add(listener);
if (existing.latestPeaks) {
- onPeaks(existing.latestPeaks, false);
+ onPeaks(existing.latestPeaks, false, null);
}
return () => {
diff --git a/packages/timeline-utils/src/audio-waveform/waveform-peak-processor.ts b/packages/timeline-utils/src/audio-waveform/waveform-peak-processor.ts
index b13950314b..16cee4fce2 100644
--- a/packages/timeline-utils/src/audio-waveform/waveform-peak-processor.ts
+++ b/packages/timeline-utils/src/audio-waveform/waveform-peak-processor.ts
@@ -1,5 +1,6 @@
type Progress = {
readonly peaks: Float32Array;
+ readonly averageVolume: number | null;
readonly completedPeaks: number;
readonly totalPeaks: number;
readonly final: boolean;
@@ -15,11 +16,17 @@ type WaveformPeakProcessorOptions = {
type WaveformPeakProcessor = {
readonly peaks: Float32Array;
- processSampleChunk: (floats: Float32Array, channels: number) => void;
+ readonly averageVolume: number | null;
+ processSampleChunk: (
+ floats: Float32Array,
+ channels: number,
+ sampleRate: number,
+ ) => void;
finalize: () => void;
};
export const emitWaveformProgress = ({
+ averageVolume,
completedPeaks,
final,
onProgress,
@@ -29,6 +36,7 @@ export const emitWaveformProgress = ({
readonly onProgress?: (progress: Progress) => void;
}) => {
onProgress?.({
+ averageVolume,
peaks,
completedPeaks,
totalPeaks,
@@ -44,6 +52,8 @@ export const createWaveformPeakProcessor = ({
now,
}: WaveformPeakProcessorOptions): WaveformPeakProcessor => {
const peaks = new Float32Array(totalPeaks);
+ let energy = 0;
+ let duration = 0;
let peakIndex = 0;
let peakMax = 0;
let sampleInPeak = 0;
@@ -63,6 +73,8 @@ export const createWaveformPeakProcessor = ({
lastProgressAt = timestamp;
lastProgressPeak = peakIndex;
emitWaveformProgress({
+ averageVolume:
+ force && duration > 0 ? 10 * Math.log10(energy / duration) : null,
peaks,
completedPeaks: peakIndex,
totalPeaks,
@@ -73,20 +85,28 @@ export const createWaveformPeakProcessor = ({
return {
peaks,
- processSampleChunk: (floats, channels) => {
+ get averageVolume() {
+ return duration > 0 ? 10 * Math.log10(energy / duration) : null;
+ },
+ processSampleChunk: (floats, channels, sampleRate) => {
const frameCount = Math.floor(floats.length / Math.max(1, channels));
for (let frame = 0; frame < frameCount; frame++) {
// `f32` copies are interleaved, so timing advances per frame.
let framePeak = 0;
+ let frameEnergy = 0;
for (let channel = 0; channel < channels; channel++) {
const sampleIndex = frame * channels + channel;
const abs = Math.abs(floats[sampleIndex] ?? 0);
+ frameEnergy += abs * abs;
if (abs > framePeak) {
framePeak = abs;
}
}
+ energy += frameEnergy / channels / sampleRate;
+ duration += 1 / sampleRate;
+
if (framePeak > peakMax) {
peakMax = framePeak;
}
diff --git a/packages/timeline-utils/src/get-average-audio-volume.ts b/packages/timeline-utils/src/get-average-audio-volume.ts
new file mode 100644
index 0000000000..bb32872325
--- /dev/null
+++ b/packages/timeline-utils/src/get-average-audio-volume.ts
@@ -0,0 +1,57 @@
+import {AudioSampleSink, type InputAudioTrack} from 'mediabunny';
+
+/** RMS level over every decoded channel and sample, relative to full scale. */
+export const getAverageAudioVolume = async ({
+ track,
+ signal,
+}: {
+ track: InputAudioTrack;
+ signal: AbortSignal;
+}): Promise<number | null> => {
+ signal.throwIfAborted();
+ if (!(await track.canDecode())) {
+ return null;
+ }
+
+ let energy = 0;
+ let duration = 0;
+ let lastYield = performance.now();
+ const sink = new AudioSampleSink(track);
+ for await (const sample of sink.samples()) {
+ try {
+ signal.throwIfAborted();
+ const data = new Float32Array(sample.numberOfFrames);
+ let sumOfSquares = 0;
+ for (let channel = 0; channel < sample.numberOfChannels; channel++) {
+ sample.copyTo(data, {planeIndex: channel, format: 'f32-planar'});
+ for (const value of data) {
+ sumOfSquares += value * value;
+ }
+ }
+
+ // Weight by time so changes in sample rate do not skew the result.
+ energy += sumOfSquares / sample.numberOfChannels / sample.sampleRate;
+ duration += sample.numberOfFrames / sample.sampleRate;
+ } finally {
+ sample.close();
+ }
+
+ if (performance.now() - lastYield > 50) {
+ await new Promise<void>((resolve) => setTimeout(resolve, 0));
+ lastYield = performance.now();
+ }
+ }
+
+ signal.throwIfAborted();
+ return duration === 0 ? null : 10 * Math.log10(energy / duration);
+};
+
+export const formatAverageAudioVolume = (volume: number | null) => {
+ if (volume === null) {
+ return 'Unavailable';
+ }
+
+ return volume === -Infinity
+ ? 'āˆ’āˆž dBFS (silent)'
+ : `${volume.toFixed(1)} dBFS`;
+};
diff --git a/packages/timeline-utils/src/index.ts b/packages/timeline-utils/src/index.ts
index b12fa151d8..964ff2e83f 100644
--- a/packages/timeline-utils/src/index.ts
+++ b/packages/timeline-utils/src/index.ts
@@ -47,3 +47,8 @@ export {
export {renderFrameStripToCanvas} from './render-frame-strip-to-canvas';
export type {RenderFrameStripToCanvasOptions} from './render-frame-strip-to-canvas';
export {resizeVideoFrame} from './resize-video-frame';
+
+export {
+ getAverageAudioVolume,
+ formatAverageAudioVolume,
+} from './get-average-audio-volume';
diff --git a/packages/timeline-utils/src/test/load-waveform-peaks-server.test.ts b/packages/timeline-utils/src/test/load-waveform-peaks-server.test.ts
index 864c373f3a..4f517ba2ce 100644
--- a/packages/timeline-utils/src/test/load-waveform-peaks-server.test.ts
+++ b/packages/timeline-utils/src/test/load-waveform-peaks-server.test.ts
@@ -10,7 +10,7 @@ beforeAll(() => {
});
test('loadWaveformPeaks decodes an MP4 audio track with @mediabunny/server', async () => {
- const peaks = await loadWaveformPeaks(
+ const {peaks} = await loadWaveformPeaks(
SAMPLE_MEDIA_URL,
new AbortController().signal,
);
@@ -32,7 +32,7 @@ test('loadWaveformPeaks progress matches completed peak count for MP4 audio', as
let lastCompleted = 0;
let sawFinal = false;
- const peaks = await loadWaveformPeaks(
+ const {peaks} = await loadWaveformPeaks(
SAMPLE_MEDIA_URL,
new AbortController().signal,
{
@@ -51,22 +51,21 @@ test('loadWaveformPeaks progress matches completed peak count for MP4 audio', as
});
test('loadWaveformPeaks caches each requested fidelity separately', async () => {
- const defaultPeaks = await loadWaveformPeaks(
- SAMPLE_MEDIA_URL,
- new AbortController().signal,
- );
- const detailedPeaks = await loadWaveformPeaks(
- SAMPLE_MEDIA_URL,
- new AbortController().signal,
- {waveformSampleRate: 200},
- );
+ const {peaks: defaultPeaks, averageVolume: defaultVolume} =
+ await loadWaveformPeaks(SAMPLE_MEDIA_URL, new AbortController().signal);
+ const {peaks: detailedPeaks, averageVolume: detailedVolume} =
+ await loadWaveformPeaks(SAMPLE_MEDIA_URL, new AbortController().signal, {
+ waveformSampleRate: 200,
+ });
+ expect(defaultVolume).not.toBeNull();
+ expect(detailedVolume!).toBeCloseTo(defaultVolume!, 6);
expect(detailedPeaks.length).toBeGreaterThan(defaultPeaks.length * 1.9);
expect(detailedPeaks.length).toBeLessThan(defaultPeaks.length * 2.1);
});
test('loadWaveformPeaks draws a non-zero first peak for edts.mp4', async () => {
- const peaks = await loadWaveformPeaks(
+ const {peaks} = await loadWaveformPeaks(
EDTS_MEDIA_URL,
new AbortController().signal,
);
diff --git a/packages/timeline-utils/src/test/waveform-peak-processor.test.ts b/packages/timeline-utils/src/test/waveform-peak-processor.test.ts
index 923e42af6d..ba9ee5fef1 100644
--- a/packages/timeline-utils/src/test/waveform-peak-processor.test.ts
+++ b/packages/timeline-utils/src/test/waveform-peak-processor.test.ts
@@ -32,6 +32,7 @@ test('Should emit incremental progress for completed peaks', () => {
processor.processSampleChunk(
new Float32Array([0.1, 0.7, 0.5, 0.2, 0.3, 0.8, 0.4, 0.1]),
2,
+ 48000,
);
expect(progressCalls).toHaveLength(1);
@@ -63,7 +64,7 @@ test('Should flush the last partial peak during finalize', () => {
},
});
- processor.processSampleChunk(new Float32Array([0.2, 0.6, 0.4]), 1);
+ processor.processSampleChunk(new Float32Array([0.2, 0.6, 0.4]), 1, 48000);
processor.finalize();
expect(progressCalls).toHaveLength(2);
@@ -94,9 +95,9 @@ test('Should throttle intermediate progress but always emit the final update', (
});
now = 100;
- processor.processSampleChunk(new Float32Array([0.2]), 1);
+ processor.processSampleChunk(new Float32Array([0.2]), 1, 48000);
now = 120;
- processor.processSampleChunk(new Float32Array([0.4]), 1);
+ processor.processSampleChunk(new Float32Array([0.4]), 1, 48000);
now = 130;
processor.finalize();