š„ Make videos programmatically with React
`@remotion/studio`: Show average audio volume in asset inspector and Convert
JonnyBurgereba1d1ef72d6
diff --git a/packages/bundler/src/shared-bundler-config.ts b/packages/bundler/src/shared-bundler-config.tsindex c50af391f3..0e1eab4073 100644--- a/packages/bundler/src/shared-bundler-config.ts+++ b/packages/bundler/src/shared-bundler-config.ts@@ -61,8 +61,27 @@ export const getResolveConfig = () => ({'worker.mjs',),}),- // test visual controls before removing this- '@remotion/studio': require.resolve('@remotion/studio'),+ // Studio entry points are also passed to the bundler as resolved CJS paths.+ // Route both those paths and package imports through the same ESM build.+ ...Object.fromEntries(+ [+ '@remotion/studio',+ '@remotion/studio/internals',+ '@remotion/studio/previewEntry',+ '@remotion/studio/renderEntry',+ ].flatMap((specifier) => {+ const commonJsPath = require.resolve(specifier);+ const esmPath = path.join(+ path.dirname(commonJsPath),+ 'esm',+ path.basename(commonJsPath, '.js') + '.mjs',+ );+ return [+ [`${specifier}$`, esmPath],+ [commonJsPath, esmPath],+ ];+ }),+ ),[path.join(require.resolve('@remotion/timeline-utils'),'..',diff --git a/packages/convert/app/components/AudioTrackOverview.tsx b/packages/convert/app/components/AudioTrackOverview.tsxindex b1e4f4f850..c000c9713a 100644--- a/packages/convert/app/components/AudioTrackOverview.tsx+++ b/packages/convert/app/components/AudioTrackOverview.tsx@@ -1,6 +1,7 @@import type {InputAudioTrack} from 'mediabunny';import {useEffect, useState} from 'react';import {renderHumanReadableAudioCodec} from '~/lib/render-codec-label';+import {AudioVolumeRow} from './AudioVolumeRow';import {PacketList} from './PacketList';import {TextButtonWithChevron} from './TexrButtonWithChevron';import {Table, TableBody, TableCell, TableRow} from './ui/table';@@ -59,6 +60,7 @@ export const AudioTrackOverview: React.FC<{<TableCell className="font-brand">Sample Rate</TableCell><TableCell className="text-right">{sampleRate ?? '...'}</TableCell></TableRow>+ <AudioVolumeRow track={track} /></TableBody></Table>);diff --git a/packages/convert/app/components/AudioVolumeRow.tsx b/packages/convert/app/components/AudioVolumeRow.tsxnew file mode 100644index 0000000000..b1fd0ce1f9--- /dev/null+++ b/packages/convert/app/components/AudioVolumeRow.tsx@@ -0,0 +1,41 @@+import {+ formatAverageAudioVolume,+ getAverageAudioVolume,+} from '@remotion/timeline-utils';+import type {InputAudioTrack} from 'mediabunny';+import {useEffect, useState} from 'react';+import {TableCell, TableRow} from './ui/table';++export const AudioVolumeRow: React.FC<{readonly track: InputAudioTrack}> = ({+ track,+}) => {+ const [value, setValue] = useState<string | null>(null);+ useEffect(() => {+ const controller = new AbortController();+ setValue('Calculatingā¦');+ getAverageAudioVolume({track, signal: controller.signal})+ .then((volume) => {+ if (!controller.signal.aborted) {+ setValue(formatAverageAudioVolume(volume));+ }+ })+ .catch(() => {+ if (!controller.signal.aborted) {+ setValue('Unavailable');+ }+ });+ return () => controller.abort();+ }, [track]);++ return (+ <TableRow>+ <TableCell+ className="font-brand"+ title="RMS level across all channels over the full audio track, including silence."+ >+ Average volume+ </TableCell>+ <TableCell className="text-right">{value ?? 'Calculatingā¦'}</TableCell>+ </TableRow>+ );+};diff --git a/packages/convert/app/components/ContainerOverview.tsx b/packages/convert/app/components/ContainerOverview.tsxindex 5145ee8ce6..ad3f559343 100644--- a/packages/convert/app/components/ContainerOverview.tsx+++ b/packages/convert/app/components/ContainerOverview.tsx@@ -14,6 +14,7 @@ import {renderHumanReadableAudioCodec,renderHumanReadableVideoCodec,} from '~/lib/render-codec-label';+import {AudioVolumeRow} from './AudioVolumeRow';import {MetadataDisplay} from './MetadataTable';import {Skeleton} from './ui/skeleton';@@ -29,6 +30,7 @@ export const ContainerOverview: React.FC<{readonly metadata: MetadataTags | null;readonly isAudioOnly: boolean;readonly sampleRate: number | null;+ readonly audioTrack: InputAudioTrack | null;}> = ({container,dimensions,@@ -41,6 +43,7 @@ export const ContainerOverview: React.FC<{metadata,isAudioOnly,sampleRate,+ audioTrack,}) => {return (<Table className="table-fixed">@@ -155,6 +158,7 @@ export const ContainerOverview: React.FC<{</TableCell></TableRow>)}+ {audioTrack ? <AudioVolumeRow track={audioTrack} /> : null}{metadata ? <MetadataDisplay metadata={metadata} /> : null}</TableBody></Table>diff --git a/packages/convert/app/components/Probe.tsx b/packages/convert/app/components/Probe.tsxindex c7c94b9087..ffb2445841 100644--- a/packages/convert/app/components/Probe.tsx+++ b/packages/convert/app/components/Probe.tsx@@ -221,6 +221,9 @@ export const Probe: React.FC<{metadata={metadata}isHdr={isHdr}sampleRate={sampleRate}+ audioTrack={+ tracks?.find((track) => track.isAudioTrack()) ?? null+ }/>) : selectedTrack.isVideoTrack() ? (<VideoTrackOverviewdiff --git a/packages/convert/test/audio-volume.test.tsx b/packages/convert/test/audio-volume.test.tsxnew file mode 100644index 0000000000..33fb1aab1a--- /dev/null+++ b/packages/convert/test/audio-volume.test.tsx@@ -0,0 +1,94 @@+import {afterEach, expect, test} from 'bun:test';+import {cleanup, render, screen} from '@testing-library/react';+import {ALL_FORMATS, BufferSource, Input} from 'mediabunny';+import {ContainerOverview} from '../app/components/ContainerOverview';++afterEach(cleanup);++test('calculates full-track stereo RMS in Convert, including silence', async () => {+ // Two seconds: left channel at half scale for one second, then silence;+ // right channel silent throughout. Mean square = 0.5² / 2 / 2.+ const frames = 16000;+ const wav = new Uint8Array(44 + frames * 4);+ const view = new DataView(wav.buffer);+ for (const [offset, text] of [+ [0, 'RIFF'],+ [8, 'WAVE'],+ [12, 'fmt '],+ [36, 'data'],+ ] as const) {+ wav.set(new TextEncoder().encode(text), offset);+ }+ view.setUint32(4, wav.length - 8, true);+ view.setUint32(16, 16, true);+ view.setUint16(20, 1, true);+ view.setUint16(22, 2, true);+ view.setUint32(24, 8000, true);+ view.setUint32(28, 32000, true);+ view.setUint16(32, 4, true);+ view.setUint16(34, 16, true);+ view.setUint32(40, frames * 4, true);+ for (let frame = 0; frame < frames / 2; frame++) {+ view.setInt16(44 + frame * 4, frame % 2 === 0 ? 16384 : -16384, true);+ }++ const input = new Input({+ formats: ALL_FORMATS,+ source: new BufferSource(wav),+ });+ try {+ const audioTrack = await input.getPrimaryAudioTrack();+ const mounted = render(+ <ContainerOverview+ dimensions={null}+ durationInSeconds={2}+ videoCodec={null}+ audioCodec="pcm-s16"+ size={wav.length}+ frameRate={null}+ container={null}+ isHdr={false}+ metadata={null}+ isAudioOnly+ audioTrack={audioTrack}+ sampleRate={8000}+ />,+ );+ expect(screen.getByText('Average volume')).toBeTruthy();+ expect(screen.queryByRole('button', {name: 'Calculate'})).toBeNull();+ expect(await screen.findByText('-12.0 dBFS')).toBeTruthy();+ mounted.unmount();+ } finally {+ input.dispose();+ }++ wav.fill(0, 44);+ const silentInput = new Input({+ formats: ALL_FORMATS,+ source: new BufferSource(wav),+ });+ try {+ const audioTrack = await silentInput.getPrimaryAudioTrack();+ render(+ <ContainerOverview+ dimensions={null}+ durationInSeconds={2}+ videoCodec={null}+ audioCodec="pcm-s16"+ size={wav.length}+ frameRate={null}+ container={null}+ isHdr={false}+ metadata={null}+ isAudioOnly+ audioTrack={audioTrack}+ sampleRate={8000}+ />,+ );+ expect(screen.queryByRole('button', {name: 'Calculate'})).toBeNull();+ expect(await screen.findByText('āā dBFS (silent)')).toBeTruthy();+ } finally {+ cleanup();+ silentInput.dispose();+ }+});diff --git a/packages/convert/test/frame-rate.test.tsx b/packages/convert/test/frame-rate.test.tsxindex c32799b94e..b821b9a656 100644--- a/packages/convert/test/frame-rate.test.tsx+++ b/packages/convert/test/frame-rate.test.tsx@@ -25,6 +25,7 @@ const FrameRateProbe: React.FC<{file: File}> = ({file}) => {metadata={null}isAudioOnly={false}sampleRate={null}+ audioTrack={null}/><div data-testid="timeline-fps">{fps}</div></>diff --git a/packages/core/src/CompositionManagerContext.tsx b/packages/core/src/CompositionManagerContext.tsxindex c48b583229..ebce566abb 100644--- a/packages/core/src/CompositionManagerContext.tsx+++ b/packages/core/src/CompositionManagerContext.tsx@@ -22,6 +22,7 @@ export type BaseMetadata = Pick<export type AssetPreviewMetadata = BaseMetadata & {asset: string;+ src: string;};export const getAssetPreviewCompositionId = (asset: string) => `asset:${asset}`;diff --git a/packages/studio/src/components/AssetAudioVolume.tsx b/packages/studio/src/components/AssetAudioVolume.tsxnew file mode 100644index 0000000000..792e7b7d11--- /dev/null+++ b/packages/studio/src/components/AssetAudioVolume.tsx@@ -0,0 +1,44 @@+import {+ formatAverageAudioVolume,+ subscribeToWaveformPeaks,+} from '@remotion/timeline-utils';+import React, {useEffect, useState} from 'react';+import {LIGHT_TEXT} from '../helpers/colors';++export const AssetAudioVolume: React.FC<{+ readonly src: string | null;+ readonly waveformSampleRate: number | null;+}> = ({src, waveformSampleRate}) => {+ const [value, setValue] = useState('Calculatingā¦');+ useEffect(() => {+ setValue('Calculatingā¦');+ if (src === null || waveformSampleRate === null) {+ return;+ }++ return subscribeToWaveformPeaks({+ src,+ waveformSampleRate,+ onPeaks: (_peaks, final, averageVolume) => {+ if (final) {+ setValue(formatAverageAudioVolume(averageVolume));+ }+ },+ onError: () => setValue('Unavailable'),+ });+ }, [src, waveformSampleRate]);++ return (+ <span+ style={{+ fontFamily: 'sans-serif',+ fontSize: 13,+ lineHeight: '20px',+ color: LIGHT_TEXT,+ }}+ title="RMS level across all channels over the full audio track, including silence."+ >+ {value}+ </span>+ );+};diff --git a/packages/studio/src/components/CurrentAsset.tsx b/packages/studio/src/components/CurrentAsset.tsxindex 4df55ec986..d3ef3da9e0 100644--- a/packages/studio/src/components/CurrentAsset.tsx+++ b/packages/studio/src/components/CurrentAsset.tsx@@ -7,6 +7,7 @@ import {CURRENT_COLOR, LIGHT_TEXT} from '../helpers/colors';import {formatMediaDuration} from '../helpers/format-media-duration';import {getFileManagerName} from '../helpers/get-file-manager-name';import {getPreviewFileType} from '../helpers/get-preview-file-type';+import {TIMELINE_FRAME_WIDTH_AT_MAX_ZOOM} from '../helpers/get-timeline-max-zoom';import {openInRemotionConvert} from '../helpers/open-in-remotion-convert';import {renderHumanReadableAudioCodec,@@ -18,6 +19,7 @@ import {useMediaMetadata} from '../helpers/use-media-metadata';import {ExpandedFolderIcon} from '../icons/folder';import {RemotionConvertIcon} from '../icons/remotion-convert';import {TrashIcon} from '../icons/trash';+import {AssetAudioVolume} from './AssetAudioVolume';import {InlineEditableTitle} from './InlineEditableTitle';import {InspectorInfoHeader} from './InspectorInfoHeader';import {@@ -180,6 +182,9 @@ export const AssetInfo: React.FC<{readonly onAssetClick?: () => void;readonly readOnlyStudio: boolean;}> = ({assetName, contentSized = false, onAssetClick, readOnlyStudio}) => {+ const {currentAssetMetadata} = useContext(Internals.CompositionManager);+ const volumeMetadata =+ currentAssetMetadata?.asset === assetName ? currentAssetMetadata : null;const connectionStatus = useContext(StudioServerConnectionCtx).previewServerState.type;const browserStudioOperations = getBrowserStudioOperations();@@ -330,6 +335,24 @@ export const AssetInfo: React.FC<{<span style={assetMetadataValueStyle}>{detail.value}</span></InspectorDetailRow>))}+ {src ? (+ <InspectorDetailRow label="Average volume">+ <span style={assetMetadataValueStyle}>+ <AssetAudioVolume+ key={volumeMetadata?.src ?? src}+ src={volumeMetadata?.src ?? null}+ waveformSampleRate={+ volumeMetadata+ ? Math.ceil(+ volumeMetadata.fps *+ TIMELINE_FRAME_WIDTH_AT_MAX_ZOOM,+ )+ : null+ }+ />+ </span>+ </InspectorDetailRow>+ ) : null}</div>)}</InspectorSection>diff --git a/packages/studio/src/helpers/get-asset-metadata.ts b/packages/studio/src/helpers/get-asset-metadata.tsindex f442772be3..5ef1f274eb 100644--- a/packages/studio/src/helpers/get-asset-metadata.ts+++ b/packages/studio/src/helpers/get-asset-metadata.ts@@ -71,6 +71,10 @@ export const getAssetPreviewMetadata = ({return {asset: canvasContent.asset,+ src: addAssetCacheBust({+ src: staticFile(canvasContent.asset),+ fetchedAt: metadata.fetchedAt,+ }),width: dimensions.width,height: dimensions.height,fps,diff --git a/packages/studio/src/test/asset-audio-volume.test.tsx b/packages/studio/src/test/asset-audio-volume.test.tsxnew file mode 100644index 0000000000..d7e6ee56a9--- /dev/null+++ b/packages/studio/src/test/asset-audio-volume.test.tsx@@ -0,0 +1,88 @@+import {afterEach, expect, test} from 'bun:test';+import {createServer} from 'node:http';+import type {AddressInfo} from 'node:net';+import {subscribeToWaveformPeaks} from '@remotion/timeline-utils';+import {cleanup, render, screen} from '@testing-library/react';+import {AssetAudioVolume} from '../components/AssetAudioVolume';++afterEach(cleanup);++test('automatically shares waveform analysis and reuses cached volume on remount', async () => {+ // Two seconds: left channel at half scale for one second, then silence;+ // right channel silent throughout. Mean square = 0.5² / 2 / 2.+ const frames = 16000;+ const wav = new Uint8Array(44 + frames * 4);+ const view = new DataView(wav.buffer);+ for (const [offset, text] of [+ [0, 'RIFF'],+ [8, 'WAVE'],+ [12, 'fmt '],+ [36, 'data'],+ ] as const) {+ wav.set(new TextEncoder().encode(text), offset);+ }++ view.setUint32(4, wav.length - 8, true);+ view.setUint32(16, 16, true);+ view.setUint16(20, 1, true);+ view.setUint16(22, 2, true);+ view.setUint32(24, 8000, true);+ view.setUint32(28, 32000, true);+ view.setUint16(32, 4, true);+ view.setUint16(34, 16, true);+ view.setUint32(40, frames * 4, true);+ for (let frame = 0; frame < frames / 2; frame++) {+ view.setInt16(44 + frame * 4, frame % 2 === 0 ? 16384 : -16384, true);+ }++ let requests = 0;+ const server = createServer((request, response) => {+ requests++;+ const range = request.headers.range?.match(/bytes=(\d+)-(\d*)/);+ const start = range ? Number(range[1]) : 0;+ const end = range?.[2]+ ? Math.min(Number(range[2]), wav.length - 1)+ : wav.length - 1;+ response.writeHead(range ? 206 : 200, {+ 'Content-Type': 'audio/wav',+ 'Content-Length': end - start + 1,+ 'Accept-Ranges': 'bytes',+ 'Content-Range': `bytes ${start}-${end}/${wav.length}`,+ });+ response.end(wav.slice(start, end + 1));+ });+ await new Promise<void>((resolve) => server.listen(0, resolve));+ const src = `http://localhost:${(server.address() as AddressInfo).port}/stereo.wav`;+ let unsubscribe = () => {};+ try {+ const waveform = new Promise<void>((resolve, reject) => {+ unsubscribe = subscribeToWaveformPeaks({+ src,+ waveformSampleRate: 100,+ onPeaks: (peaks, final, volume) => {+ if (!final) return;+ expect(peaks.length).toBe(200);+ expect(volume!).toBeCloseTo(-12.0412, 3);+ resolve();+ },+ onError: reject,+ });+ });+ const mounted = render(+ <AssetAudioVolume src={src} waveformSampleRate={100} />,+ );+ expect(screen.queryByRole('button', {name: 'Calculate'})).toBeNull();+ expect(await screen.findByText('-12.0 dBFS')).toBeTruthy();+ await waveform;+ expect(requests).toBe(1);+ mounted.unmount();+ render(<AssetAudioVolume src={src} waveformSampleRate={100} />);+ expect(await screen.findByText('-12.0 dBFS')).toBeTruthy();+ expect(requests).toBe(1);+ } finally {+ unsubscribe();+ cleanup();+ server.closeAllConnections();+ await new Promise<void>((resolve) => server.close(() => resolve()));+ }+});diff --git a/packages/timeline-utils/src/audio-waveform-worker.ts b/packages/timeline-utils/src/audio-waveform-worker.tsindex 09393b388f..f890bc5f1a 100644--- a/packages/timeline-utils/src/audio-waveform-worker.ts+++ b/packages/timeline-utils/src/audio-waveform-worker.ts@@ -8,11 +8,17 @@ import {loadWaveformPeaks} from './audio-waveform/load-waveform-peaks';declare const self: DedicatedWorkerGlobalScope;-const postPeaks = (requestId: number, peaks: Float32Array, final: boolean) => {+const postPeaks = (+ requestId: number,+ peaks: Float32Array,+ final: boolean,+ averageVolume: number | null,+) => {// Structured cloning copies the array, so the decoder can keep// mutating its buffer while the main thread reads the snapshot.const payload: AudioWaveformWorkerOutgoingMessage = {type: 'peaks',+ averageVolume,requestId,peaks,final,@@ -42,12 +48,12 @@ self.addEventListener(waveformSampleRate: message.waveformSampleRate,onProgress: ({peaks, final}) => {if (!final) {- postPeaks(message.requestId, peaks, false);+ postPeaks(message.requestId, peaks, false, null);}},})- .then((peaks) => {- postPeaks(message.requestId, peaks, true);+ .then(({peaks, averageVolume}) => {+ postPeaks(message.requestId, peaks, true, averageVolume);}).catch((error) => {postError(message.requestId, error);diff --git a/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts b/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.tsindex bb4f703152..f0462ce699 100644--- a/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts+++ b/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts@@ -12,6 +12,7 @@ export type AudioWaveformWorkerPeaksMessage = {readonly requestId: number;readonly peaks: Float32Array;readonly final: boolean;+ readonly averageVolume: number | null;};export type AudioWaveformWorkerErrorMessage = {diff --git a/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.ts b/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.tsindex 0d82d15898..8a80b6d57c 100644--- a/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.ts+++ b/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.ts@@ -7,11 +7,17 @@ import {} from './waveform-peak-processor';const DEFAULT_PROGRESS_INTERVAL_IN_MS = 50;-const peaksCache = new Map<string, Float32Array>();+export type WaveformResult = {+ readonly peaks: Float32Array;+ readonly averageVolume: number | null;+};++const peaksCache = new Map<string, WaveformResult>();export {TARGET_SAMPLE_RATE};type Progress = {+ readonly averageVolume: number | null;readonly peaks: Float32Array;readonly completedPeaks: number;readonly totalPeaks: number;@@ -28,7 +34,7 @@ export async function loadWaveformPeaks(url: string,signal: AbortSignal,options?: LoadWaveformPeaksOptions,-): Promise<Float32Array> {+): Promise<WaveformResult> {const waveformSampleRate = options?.waveformSampleRate ?? TARGET_SAMPLE_RATE;if (!Number.isFinite(waveformSampleRate) || waveformSampleRate <= 0) {throw new Error('The waveform sample rate must be a positive number.');@@ -38,9 +44,10 @@ export async function loadWaveformPeaks(const cached = peaksCache.get(cacheKey);if (cached) {emitWaveformProgress({- peaks: cached,- completedPeaks: cached.length,- totalPeaks: cached.length,+ peaks: cached.peaks,+ averageVolume: cached.averageVolume,+ completedPeaks: cached.peaks.length,+ totalPeaks: cached.peaks.length,final: true,onProgress: options?.onProgress,});@@ -55,7 +62,7 @@ export async function loadWaveformPeaks(try {const audioTrack = await input.getPrimaryAudioTrack();if (!audioTrack) {- return new Float32Array(0);+ return {peaks: new Float32Array(0), averageVolume: null};}if (await audioTrack.isLive()) {@@ -97,7 +104,7 @@ export async function loadWaveformPeaks(for await (const sample of sink.samples()) {if (signal.aborted) {sample.close();- return new Float32Array(0);+ return {peaks: new Float32Array(0), averageVolume: null};}const startFrame = getAudioSampleStartFrameAtTimelineZero(sample);@@ -126,15 +133,19 @@ export async function loadWaveformPeaks(frameCount,});const channels = Math.max(1, sample.numberOfChannels);+ const {sampleRate} = sample;sample.close();- processor.processSampleChunk(floats, channels);+ processor.processSampleChunk(floats, channels, sampleRate);}processor.finalize();- const {peaks} = processor;- peaksCache.set(cacheKey, peaks);- return peaks;+ const result = {+ peaks: processor.peaks,+ averageVolume: processor.averageVolume,+ };+ peaksCache.set(cacheKey, result);+ return result;} finally {input.dispose();}diff --git a/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts b/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.tsindex 6f2be3091a..31a42534f1 100644--- a/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts+++ b/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts@@ -3,11 +3,15 @@ import type {AudioWaveformWorkerOutgoingMessage,} from './audio-waveform-worker-types';import {TARGET_SAMPLE_RATE} from './constants';-import {loadWaveformPeaks} from './load-waveform-peaks';+import {type WaveformResult, loadWaveformPeaks} from './load-waveform-peaks';import {makeAudioWaveformWorker} from './make-audio-waveform-worker';type WaveformPeaksListener = {- readonly onPeaks: (peaks: Float32Array, final: boolean) => void;+ readonly onPeaks: (+ peaks: Float32Array,+ final: boolean,+ averageVolume: number | null,+ ) => void;readonly onError: (error: Error) => void;};@@ -23,7 +27,7 @@ type InFlightLoad = {// All AudioWaveform instances share one decode worker and one peaks cache,// so remounts (timeline virtualization, zoom window cropping) and multiple// clips with the same src never re-fetch or re-decode the audio.-const peaksCache = new Map<string, Float32Array>();+const peaksCache = new Map<string, WaveformResult>();const inFlightByCacheKey = new Map<string, InFlightLoad>();const inFlightByRequestId = new Map<number, InFlightLoad>();@@ -31,9 +35,14 @@ let worker: Worker | null = null;let workerFailed = false;let nextRequestId = 0;-const emitPeaks = (load: InFlightLoad, peaks: Float32Array, final: boolean) => {+const emitPeaks = (+ load: InFlightLoad,+ peaks: Float32Array,+ final: boolean,+ averageVolume: number | null,+) => {if (final) {- peaksCache.set(load.cacheKey, peaks);+ peaksCache.set(load.cacheKey, {peaks, averageVolume});load.latestPeaks = null;inFlightByCacheKey.delete(load.cacheKey);if (load.requestId !== null) {@@ -44,7 +53,7 @@ const emitPeaks = (load: InFlightLoad, peaks: Float32Array, final: boolean) => {}for (const listener of load.listeners) {- listener.onPeaks(peaks, final);+ listener.onPeaks(peaks, final, averageVolume);}};@@ -75,11 +84,11 @@ const startMainThreadLoad = (load: InFlightLoad) => {// The processor mutates the same array in place; snapshot it so// React state updates see a new reference.- emitPeaks(load, peaks.slice(), false);+ emitPeaks(load, peaks.slice(), false, null);},})- .then((peaks) => {- emitPeaks(load, peaks, true);+ .then(({peaks, averageVolume}) => {+ emitPeaks(load, peaks, true, averageVolume);}).catch((error) => {emitError(@@ -133,7 +142,7 @@ const getOrCreateWorker = (): Worker | null => {return;}- emitPeaks(load, message.peaks, message.final);+ emitPeaks(load, message.peaks, message.final, message.averageVolume);},);worker.addEventListener('error', (event) => {@@ -152,13 +161,17 @@ export const subscribeToWaveformPeaks = ({}: {readonly src: string;readonly waveformSampleRate?: number;- readonly onPeaks: (peaks: Float32Array, final: boolean) => void;+ readonly onPeaks: (+ peaks: Float32Array,+ final: boolean,+ averageVolume: number | null,+ ) => void;readonly onError: (error: Error) => void;}): (() => void) => {const cacheKey = `${waveformSampleRate}:${src}`;const cached = peaksCache.get(cacheKey);if (cached) {- onPeaks(cached, true);+ onPeaks(cached.peaks, true, cached.averageVolume);return () => undefined;}@@ -168,7 +181,7 @@ export const subscribeToWaveformPeaks = ({if (existing) {existing.listeners.add(listener);if (existing.latestPeaks) {- onPeaks(existing.latestPeaks, false);+ onPeaks(existing.latestPeaks, false, null);}return () => {diff --git a/packages/timeline-utils/src/audio-waveform/waveform-peak-processor.ts b/packages/timeline-utils/src/audio-waveform/waveform-peak-processor.tsindex b13950314b..16cee4fce2 100644--- a/packages/timeline-utils/src/audio-waveform/waveform-peak-processor.ts+++ b/packages/timeline-utils/src/audio-waveform/waveform-peak-processor.ts@@ -1,5 +1,6 @@type Progress = {readonly peaks: Float32Array;+ readonly averageVolume: number | null;readonly completedPeaks: number;readonly totalPeaks: number;readonly final: boolean;@@ -15,11 +16,17 @@ type WaveformPeakProcessorOptions = {type WaveformPeakProcessor = {readonly peaks: Float32Array;- processSampleChunk: (floats: Float32Array, channels: number) => void;+ readonly averageVolume: number | null;+ processSampleChunk: (+ floats: Float32Array,+ channels: number,+ sampleRate: number,+ ) => void;finalize: () => void;};export const emitWaveformProgress = ({+ averageVolume,completedPeaks,final,onProgress,@@ -29,6 +36,7 @@ export const emitWaveformProgress = ({readonly onProgress?: (progress: Progress) => void;}) => {onProgress?.({+ averageVolume,peaks,completedPeaks,totalPeaks,@@ -44,6 +52,8 @@ export const createWaveformPeakProcessor = ({now,}: WaveformPeakProcessorOptions): WaveformPeakProcessor => {const peaks = new Float32Array(totalPeaks);+ let energy = 0;+ let duration = 0;let peakIndex = 0;let peakMax = 0;let sampleInPeak = 0;@@ -63,6 +73,8 @@ export const createWaveformPeakProcessor = ({lastProgressAt = timestamp;lastProgressPeak = peakIndex;emitWaveformProgress({+ averageVolume:+ force && duration > 0 ? 10 * Math.log10(energy / duration) : null,peaks,completedPeaks: peakIndex,totalPeaks,@@ -73,20 +85,28 @@ export const createWaveformPeakProcessor = ({return {peaks,- processSampleChunk: (floats, channels) => {+ get averageVolume() {+ return duration > 0 ? 10 * Math.log10(energy / duration) : null;+ },+ processSampleChunk: (floats, channels, sampleRate) => {const frameCount = Math.floor(floats.length / Math.max(1, channels));for (let frame = 0; frame < frameCount; frame++) {// `f32` copies are interleaved, so timing advances per frame.let framePeak = 0;+ let frameEnergy = 0;for (let channel = 0; channel < channels; channel++) {const sampleIndex = frame * channels + channel;const abs = Math.abs(floats[sampleIndex] ?? 0);+ frameEnergy += abs * abs;if (abs > framePeak) {framePeak = abs;}}+ energy += frameEnergy / channels / sampleRate;+ duration += 1 / sampleRate;+if (framePeak > peakMax) {peakMax = framePeak;}diff --git a/packages/timeline-utils/src/get-average-audio-volume.ts b/packages/timeline-utils/src/get-average-audio-volume.tsnew file mode 100644index 0000000000..bb32872325--- /dev/null+++ b/packages/timeline-utils/src/get-average-audio-volume.ts@@ -0,0 +1,57 @@+import {AudioSampleSink, type InputAudioTrack} from 'mediabunny';++/** RMS level over every decoded channel and sample, relative to full scale. */+export const getAverageAudioVolume = async ({+ track,+ signal,+}: {+ track: InputAudioTrack;+ signal: AbortSignal;+}): Promise<number | null> => {+ signal.throwIfAborted();+ if (!(await track.canDecode())) {+ return null;+ }++ let energy = 0;+ let duration = 0;+ let lastYield = performance.now();+ const sink = new AudioSampleSink(track);+ for await (const sample of sink.samples()) {+ try {+ signal.throwIfAborted();+ const data = new Float32Array(sample.numberOfFrames);+ let sumOfSquares = 0;+ for (let channel = 0; channel < sample.numberOfChannels; channel++) {+ sample.copyTo(data, {planeIndex: channel, format: 'f32-planar'});+ for (const value of data) {+ sumOfSquares += value * value;+ }+ }++ // Weight by time so changes in sample rate do not skew the result.+ energy += sumOfSquares / sample.numberOfChannels / sample.sampleRate;+ duration += sample.numberOfFrames / sample.sampleRate;+ } finally {+ sample.close();+ }++ if (performance.now() - lastYield > 50) {+ await new Promise<void>((resolve) => setTimeout(resolve, 0));+ lastYield = performance.now();+ }+ }++ signal.throwIfAborted();+ return duration === 0 ? null : 10 * Math.log10(energy / duration);+};++export const formatAverageAudioVolume = (volume: number | null) => {+ if (volume === null) {+ return 'Unavailable';+ }++ return volume === -Infinity+ ? 'āā dBFS (silent)'+ : `${volume.toFixed(1)} dBFS`;+};diff --git a/packages/timeline-utils/src/index.ts b/packages/timeline-utils/src/index.tsindex b12fa151d8..964ff2e83f 100644--- a/packages/timeline-utils/src/index.ts+++ b/packages/timeline-utils/src/index.ts@@ -47,3 +47,8 @@ export {export {renderFrameStripToCanvas} from './render-frame-strip-to-canvas';export type {RenderFrameStripToCanvasOptions} from './render-frame-strip-to-canvas';export {resizeVideoFrame} from './resize-video-frame';++export {+ getAverageAudioVolume,+ formatAverageAudioVolume,+} from './get-average-audio-volume';diff --git a/packages/timeline-utils/src/test/load-waveform-peaks-server.test.ts b/packages/timeline-utils/src/test/load-waveform-peaks-server.test.tsindex 864c373f3a..4f517ba2ce 100644--- a/packages/timeline-utils/src/test/load-waveform-peaks-server.test.ts+++ b/packages/timeline-utils/src/test/load-waveform-peaks-server.test.ts@@ -10,7 +10,7 @@ beforeAll(() => {});test('loadWaveformPeaks decodes an MP4 audio track with @mediabunny/server', async () => {- const peaks = await loadWaveformPeaks(+ const {peaks} = await loadWaveformPeaks(SAMPLE_MEDIA_URL,new AbortController().signal,);@@ -32,7 +32,7 @@ test('loadWaveformPeaks progress matches completed peak count for MP4 audio', aslet lastCompleted = 0;let sawFinal = false;- const peaks = await loadWaveformPeaks(+ const {peaks} = await loadWaveformPeaks(SAMPLE_MEDIA_URL,new AbortController().signal,{@@ -51,22 +51,21 @@ test('loadWaveformPeaks progress matches completed peak count for MP4 audio', as});test('loadWaveformPeaks caches each requested fidelity separately', async () => {- const defaultPeaks = await loadWaveformPeaks(- SAMPLE_MEDIA_URL,- new AbortController().signal,- );- const detailedPeaks = await loadWaveformPeaks(- SAMPLE_MEDIA_URL,- new AbortController().signal,- {waveformSampleRate: 200},- );+ const {peaks: defaultPeaks, averageVolume: defaultVolume} =+ await loadWaveformPeaks(SAMPLE_MEDIA_URL, new AbortController().signal);+ const {peaks: detailedPeaks, averageVolume: detailedVolume} =+ await loadWaveformPeaks(SAMPLE_MEDIA_URL, new AbortController().signal, {+ waveformSampleRate: 200,+ });+ expect(defaultVolume).not.toBeNull();+ expect(detailedVolume!).toBeCloseTo(defaultVolume!, 6);expect(detailedPeaks.length).toBeGreaterThan(defaultPeaks.length * 1.9);expect(detailedPeaks.length).toBeLessThan(defaultPeaks.length * 2.1);});test('loadWaveformPeaks draws a non-zero first peak for edts.mp4', async () => {- const peaks = await loadWaveformPeaks(+ const {peaks} = await loadWaveformPeaks(EDTS_MEDIA_URL,new AbortController().signal,);diff --git a/packages/timeline-utils/src/test/waveform-peak-processor.test.ts b/packages/timeline-utils/src/test/waveform-peak-processor.test.tsindex 923e42af6d..ba9ee5fef1 100644--- a/packages/timeline-utils/src/test/waveform-peak-processor.test.ts+++ b/packages/timeline-utils/src/test/waveform-peak-processor.test.ts@@ -32,6 +32,7 @@ test('Should emit incremental progress for completed peaks', () => {processor.processSampleChunk(new Float32Array([0.1, 0.7, 0.5, 0.2, 0.3, 0.8, 0.4, 0.1]),2,+ 48000,);expect(progressCalls).toHaveLength(1);@@ -63,7 +64,7 @@ test('Should flush the last partial peak during finalize', () => {},});- processor.processSampleChunk(new Float32Array([0.2, 0.6, 0.4]), 1);+ processor.processSampleChunk(new Float32Array([0.2, 0.6, 0.4]), 1, 48000);processor.finalize();expect(progressCalls).toHaveLength(2);@@ -94,9 +95,9 @@ test('Should throttle intermediate progress but always emit the final update', (});now = 100;- processor.processSampleChunk(new Float32Array([0.2]), 1);+ processor.processSampleChunk(new Float32Array([0.2]), 1, 48000);now = 120;- processor.processSampleChunk(new Float32Array([0.4]), 1);+ processor.processSampleChunk(new Float32Array([0.4]), 1, 48000);now = 130;processor.finalize();