Skip to content

Create polished demo videos without editing skills. Mac/Windows/Linux

add: TimelineEditor waveformgenerator as a worker. to be extra efficient

Alan Trebugeais30aacaef483b
diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx
index 29b0f25c..cf7005e4 100644
--- a/src/components/video-editor/timeline/TimelineEditor.tsx
+++ b/src/components/video-editor/timeline/TimelineEditor.tsx
@@ -229,7 +229,9 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(
return { previewSpans, hiddenZoomIds };
}, [clipRegions, liveSpanPreviewById, zoomRegions]);
const { shortcuts: keyShortcuts, isMac } = useShortcuts();
- const audioPeaks = useTimelineAudioPeaks(videoPath);
+ const audioPeaks = useTimelineAudioPeaks(videoPath, {
+ enableSourceSidecarFallback: true,
+ });
useEffect(() => {
if (aspectRatio === "native") {
diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx
index d6d13ed1..c7d65898 100644
--- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx
+++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx
@@ -33,6 +33,7 @@ import {
import TimelineAxis from "../axis/TimelineAxis";
import ClipMarkerOverlay from "../overlays/ClipMarkerOverlay";
import PlaybackCursor from "../playhead/PlaybackCursor";
+import { useTimelineAudioPeaks } from "../../hooks/useTimelineAudioPeaks";
const HINT_CLIP = "Press C to split clip";
const HINT_ANNOTATION = "Press A to add annotation";
@@ -234,6 +235,38 @@ interface TimelineCanvasRowsProps {
onZoomRowClick: MouseEventHandler<HTMLDivElement>;
}
+interface AudioItemWithWaveformProps {
+ item: TimelineRenderItem;
+ span: { start: number; end: number };
+ waveformSpan: { start: number; end: number };
+ isSelected: boolean;
+ onSelectAudio?: (id: string | null) => void;
+}
+
+function AudioItemWithWaveform({
+ item,
+ span,
+ waveformSpan,
+ isSelected,
+ onSelectAudio,
+}: AudioItemWithWaveformProps) {
+ const peaks = useTimelineAudioPeaks(item.audioPath ?? null);
+ return (
+ <Item
+ id={item.id}
+ rowId={item.rowId}
+ span={span}
+ isSelected={isSelected}
+ onSelectId={onSelectAudio}
+ variant="audio"
+ waveformPeaks={peaks}
+ waveformSegmentSpan={waveformSpan}
+ >
+ {item.label}
+ </Item>
+ );
+}
+
const TimelineCanvasRows = memo(function TimelineCanvasRows({
items,
videoDurationMs,
@@ -422,17 +455,14 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({
{audioRows.map(({ rowId, items: rowItems }, index) => (
<Row key={rowId} id={rowId} isEmpty={rowItems.length === 0} hint={index === 0 ? HINT_AUDIO : undefined}>
{rowItems.map((item) => (
- <Item
- id={item.id}
+ <AudioItemWithWaveform
key={item.id}
- rowId={item.rowId}
+ item={item}
span={item.span}
+ waveformSpan={liveSpanPreviewById?.[item.id] ?? item.span}
isSelected={selectAllBlocksActive || item.id === selectedAudioId}
- onSelectId={onSelectAudio}
- variant="audio"
- >
- {item.label}
- </Item>
+ onSelectAudio={onSelectAudio}
+ />
))}
</Row>
))}
diff --git a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx
index 367ed1ed..eff26d09 100644
--- a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx
+++ b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx
@@ -61,8 +61,13 @@ function AudioWaveformComponent({
const { peaks: peakData, durationMs } = peaks;
if (durationMs <= 0 || peakData.length === 0) return;
- const visibleStartMs = segmentStartMs ?? range.start;
- const visibleEndMs = segmentEndMs ?? range.end;
+ const rawVisibleStartMs = segmentStartMs ?? range.start;
+ const rawVisibleEndMs = segmentEndMs ?? range.end;
+ const msPerBin = durationMs / peakData.length;
+ const visibleStartMs =
+ msPerBin > 0 ? Math.round(rawVisibleStartMs / msPerBin) * msPerBin : rawVisibleStartMs;
+ const visibleEndMs =
+ msPerBin > 0 ? Math.round(rawVisibleEndMs / msPerBin) * msPerBin : rawVisibleEndMs;
const visibleDurationMs = visibleEndMs - visibleStartMs;
if (visibleDurationMs <= 0) return;
@@ -71,11 +76,14 @@ function AudioWaveformComponent({
ctx.beginPath();
for (let px = 0; px < width; px++) {
const t = visibleStartMs + (px / width) * visibleDurationMs;
- const binIndex = Math.min(
- peakData.length - 1,
- Math.max(0, Math.floor((t / durationMs) * peakData.length)),
+ const exactIndex = Math.max(
+ 0,
+ Math.min(peakData.length - 1, (t / durationMs) * (peakData.length - 1)),
);
- const amplitude = peakData[binIndex];
+ const leftIndex = Math.floor(exactIndex);
+ const rightIndex = Math.min(peakData.length - 1, leftIndex + 1);
+ const mix = exactIndex - leftIndex;
+ const amplitude = peakData[leftIndex] * (1 - mix) + peakData[rightIndex] * mix;
const barHeight = amplitude * midY * 0.85;
ctx.moveTo(px, midY - barHeight);
diff --git a/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts b/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts
new file mode 100644
index 00000000..7ff7f26c
--- /dev/null
+++ b/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts
@@ -0,0 +1,88 @@
+import WorkerConstructor from "./waveform.worker?worker";
+import type { AudioPeaksData } from "../../core/timelineTypes";
+
+const DEFAULT_PEAK_COUNT = 2048;
+
+export class WaveformGenerator {
+ private audioContext: AudioContext;
+ private worker: Worker;
+ private peaksCache = new Map<string, AudioPeaksData>();
+ private pending = new Map<string, Promise<AudioPeaksData>>();
+
+ constructor() {
+ this.audioContext = new (window.AudioContext || (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext)();
+ this.worker = new WorkerConstructor();
+ }
+
+ private computePeaksWithWorker(channelData: Float32Array, samples: number): Promise<Float32Array> {
+ return new Promise((resolve, reject) => {
+ const onMessage = (event: MessageEvent<Float32Array>) => {
+ this.worker.removeEventListener("message", onMessage);
+ this.worker.removeEventListener("error", onError);
+ resolve(event.data);
+ };
+ const onError = (error: ErrorEvent) => {
+ this.worker.removeEventListener("message", onMessage);
+ this.worker.removeEventListener("error", onError);
+ reject(error.error ?? new Error(error.message));
+ };
+
+ this.worker.addEventListener("message", onMessage);
+ this.worker.addEventListener("error", onError);
+ this.worker.postMessage(
+ {
+ channelData,
+ samples,
+ },
+ [channelData.buffer],
+ );
+ });
+ }
+
+ public async generate(url: string, peakCount = DEFAULT_PEAK_COUNT): Promise<AudioPeaksData> {
+ const cacheKey = `${url}::${peakCount}`;
+ const cached = this.peaksCache.get(cacheKey);
+ if (cached) return cached;
+
+ const inflight = this.pending.get(cacheKey);
+ if (inflight) return inflight;
+
+ const request = (async () => {
+ const response = await fetch(url);
+ if (!response.ok) {
+ throw new Error(`Failed to load media: ${response.status}`);
+ }
+
+ const arrayBuffer = await response.arrayBuffer();
+ const decoded = await this.audioContext.decodeAudioData(arrayBuffer);
+ const channelData = decoded.getChannelData(0).slice();
+ const peaks = await this.computePeaksWithWorker(channelData, peakCount);
+
+ let max = 0;
+ for (let i = 0; i < peaks.length; i++) {
+ if (peaks[i] > max) max = peaks[i];
+ }
+ if (max > 0) {
+ for (let i = 0; i < peaks.length; i++) {
+ peaks[i] /= max;
+ }
+ }
+
+ const result: AudioPeaksData = {
+ peaks,
+ durationMs: decoded.duration * 1000,
+ };
+ this.peaksCache.set(cacheKey, result);
+ this.pending.delete(cacheKey);
+ return result;
+ })().catch((error) => {
+ this.pending.delete(cacheKey);
+ throw error;
+ });
+
+ this.pending.set(cacheKey, request);
+ return request;
+ }
+}
+
+export const waveformGenerator = new WaveformGenerator();
diff --git a/src/components/video-editor/timeline/components/waveform/waveform.worker.ts b/src/components/video-editor/timeline/components/waveform/waveform.worker.ts
new file mode 100644
index 00000000..c8a7f8c5
--- /dev/null
+++ b/src/components/video-editor/timeline/components/waveform/waveform.worker.ts
@@ -0,0 +1,31 @@
+self.onmessage = (e: MessageEvent) => {
+ const { channelData, samples } = e.data as {
+ channelData: Float32Array;
+ samples: number;
+ };
+
+ if (!channelData || samples <= 0) {
+ self.postMessage(new Float32Array(0));
+ return;
+ }
+
+ try {
+ const step = Math.max(1, Math.floor(channelData.length / samples));
+ const result = new Float32Array(samples);
+
+ for (let i = 0; i < samples; i++) {
+ const start = i * step;
+ const end = Math.min(start + step, channelData.length);
+ let max = 0;
+ for (let j = start; j < end; j++) {
+ const val = Math.abs(channelData[j]);
+ if (val > max) max = val;
+ }
+ result[i] = max;
+ }
+
+ self.postMessage(result);
+ } catch {
+ self.postMessage(new Float32Array(0));
+ }
+};
diff --git a/src/components/video-editor/timeline/core/timelineTypes.ts b/src/components/video-editor/timeline/core/timelineTypes.ts
index 2222f7cc..4a58c969 100644
--- a/src/components/video-editor/timeline/core/timelineTypes.ts
+++ b/src/components/video-editor/timeline/core/timelineTypes.ts
@@ -32,6 +32,7 @@ export interface TimelineRenderItem {
rowId: string;
span: Span;
label: string;
+ audioPath?: string;
zoomDepth?: number;
zoomMode?: ZoomMode;
speedValue?: number;
diff --git a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts
index 9af79a80..4ca2a8f4 100644
--- a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts
+++ b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts
@@ -1,10 +1,9 @@
import { useEffect, useRef, useState } from "react";
-import { fromFileUrl, resolveVideoUrl } from "../../projectPersistence";
+import { resolveMediaResourceUrl } from "@/lib/exporter/localMediaSource";
+import { fromFileUrl } from "../../projectPersistence";
+import { waveformGenerator } from "../components/waveform/WaveformGenerator";
import type { AudioPeaksData } from "../core/timelineTypes";
-/** Number of peak bins to produce — enough for smooth display at any zoom. */
-const TARGET_PEAK_COUNT = 2048;
-
function buildSidecarAudioCandidates(sourcePath: string): string[] {
const normalized = sourcePath.replace(/\\/g, "/");
const lastSlash = normalized.lastIndexOf("/");
@@ -29,193 +28,69 @@ function extractLocalPathFromMediaServerUrl(input: string): string | null {
(url.hostname === "127.0.0.1" || url.hostname === "localhost") &&
url.pathname === "/video";
if (!isLocalMediaServer) return null;
- const rawPath = url.searchParams.get("path");
- if (!rawPath) return null;
- return rawPath;
+ return url.searchParams.get("path");
} catch {
return null;
}
}
-async function decodePeaksFromMediaUrl(
- mediaUrl: string,
- cancelled: () => boolean,
-): Promise<AudioPeaksData> {
- const response = await fetch(mediaUrl);
- if (!response.ok) {
- throw new Error(`Failed to load media for waveform: ${response.status}`);
- }
- if (cancelled()) throw new Error("cancelled");
- console.debug("[timeline-audio-peaks] fetch ok", {
- url: mediaUrl,
- status: response.status,
- });
-
- const arrayBuffer = await response.arrayBuffer();
- if (cancelled()) throw new Error("cancelled");
- console.debug("[timeline-audio-peaks] bytes loaded", {
- url: mediaUrl,
- byteLength: arrayBuffer.byteLength,
- });
-
- const audioCtx = new OfflineAudioContext(1, 1, 44100);
- const decoded = await audioCtx.decodeAudioData(arrayBuffer);
- if (cancelled()) throw new Error("cancelled");
- console.debug("[timeline-audio-peaks] decode ok", {
- url: mediaUrl,
- durationSec: decoded.duration,
- channelCount: decoded.numberOfChannels,
- sampleRate: decoded.sampleRate,
- sampleFrames: decoded.length,
- });
-
- const channelData = decoded.getChannelData(0);
- const durationMs = decoded.duration * 1000;
- const binSize = Math.max(1, Math.floor(channelData.length / TARGET_PEAK_COUNT));
- const peakCount = Math.ceil(channelData.length / binSize);
- const peaks = new Float32Array(peakCount);
-
- for (let i = 0; i < peakCount; i++) {
- const start = i * binSize;
- const end = Math.min(start + binSize, channelData.length);
- let max = 0;
- for (let j = start; j < end; j++) {
- const abs = Math.abs(channelData[j]);
- if (abs > max) max = abs;
- }
- peaks[i] = max;
- }
-
- let globalMax = 0;
- for (let i = 0; i < peaks.length; i++) {
- if (peaks[i] > globalMax) globalMax = peaks[i];
- }
- if (globalMax > 0) {
- for (let i = 0; i < peaks.length; i++) {
- peaks[i] /= globalMax;
- }
- }
-
- return { peaks, durationMs };
+interface TimelineAudioPeaksOptions {
+ enableSourceSidecarFallback?: boolean;
}
-/**
- * Decode audio from a media file URL and produce a fixed-length array of peak
- * amplitudes suitable for waveform visualisation.
- *
- * Returns `null` while loading or if the file has no decodeable audio.
- */
-export function useTimelineAudioPeaks(fileUrl: string | null | undefined): AudioPeaksData | null {
+export function useTimelineAudioPeaks(
+ mediaResource: string | null | undefined,
+ options: TimelineAudioPeaksOptions = {},
+): AudioPeaksData | null {
const [data, setData] = useState<AudioPeaksData | null>(null);
- const urlRef = useRef(fileUrl);
+ const sourceRef = useRef(mediaResource);
+ const enableSourceSidecarFallback = options.enableSourceSidecarFallback ?? false;
useEffect(() => {
- urlRef.current = fileUrl;
+ sourceRef.current = mediaResource;
setData(null);
- console.debug("[timeline-audio-peaks] reset", { input: fileUrl ?? null });
-
- if (!fileUrl) {
- console.debug("[timeline-audio-peaks] no input source, skipping");
- return;
- }
+ if (!mediaResource) return;
let cancelled = false;
- (async () => {
+ const run = async () => {
+ const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource);
+ const localSourcePath =
+ localPathFromServer ||
+ (/^file:\/\//i.test(mediaResource) ? fromFileUrl(mediaResource) : mediaResource);
+
+ const tryGenerate = async (resource: string): Promise<AudioPeaksData> => {
+ const resolvedUrl = await resolveMediaResourceUrl(resource);
+ return waveformGenerator.generate(resolvedUrl);
+ };
+
try {
- const extractedLocalPath = extractLocalPathFromMediaServerUrl(fileUrl);
- const isRemoteLike = /^(https?:|blob:|data:)/i.test(fileUrl) && !extractedLocalPath;
- const isFileUrl = /^file:\/\//i.test(fileUrl);
- const sourceLocalPath = isRemoteLike
- ? null
- : extractedLocalPath
- ? extractedLocalPath
- : isFileUrl
- ? fromFileUrl(fileUrl)
- : fileUrl;
- console.debug("[timeline-audio-peaks] resolving source", {
- input: fileUrl,
- isRemoteLike,
- isFileUrl,
- sourceLocalPath,
- extractedLocalPath,
- });
- const mediaUrl = isRemoteLike
- ? fileUrl
- : isFileUrl
- ? await resolveVideoUrl(fromFileUrl(fileUrl))
- : await resolveVideoUrl(fileUrl);
- if (cancelled) return;
- console.debug("[timeline-audio-peaks] resolved source", {
- input: fileUrl,
- resolved: mediaUrl,
- });
+ const result = await tryGenerate(mediaResource);
+ if (!cancelled && sourceRef.current === mediaResource) setData(result);
+ return;
+ } catch {
+ // fallthrough
+ }
- let decodedPeaks: AudioPeaksData | null = null;
- try {
- decodedPeaks = await decodePeaksFromMediaUrl(mediaUrl, () => cancelled);
- } catch (primaryError) {
- if (primaryError instanceof Error && primaryError.message === "cancelled") {
- return;
- }
- console.warn("[timeline-audio-peaks] primary decode failed", {
- input: fileUrl,
- url: mediaUrl,
- error: primaryError,
- });
- if (sourceLocalPath) {
- const candidates = buildSidecarAudioCandidates(sourceLocalPath);
- console.debug("[timeline-audio-peaks] trying sidecar candidates", {
- sourceLocalPath,
- candidates,
- });
- for (const candidatePath of candidates) {
- try {
- const candidateUrl = await resolveVideoUrl(candidatePath);
- console.debug("[timeline-audio-peaks] trying sidecar", {
- candidatePath,
- candidateUrl,
- });
- decodedPeaks = await decodePeaksFromMediaUrl(candidateUrl, () => cancelled);
- console.info("[timeline-audio-peaks] sidecar decode succeeded", {
- candidatePath,
- candidateUrl,
- });
- break;
- } catch (sidecarError) {
- if (sidecarError instanceof Error && sidecarError.message === "cancelled") {
- return;
- }
- console.debug("[timeline-audio-peaks] sidecar decode failed", {
- candidatePath,
- error: sidecarError,
- });
- }
- }
- }
- }
+ if (!enableSourceSidecarFallback || !localSourcePath) return;
- if (!cancelled && urlRef.current === fileUrl && decodedPeaks) {
- console.debug("[timeline-audio-peaks] peaks generated", {
- input: fileUrl,
- durationMs: Math.round(decodedPeaks.durationMs),
- peakCount: decodedPeaks.peaks.length,
- });
- setData(decodedPeaks);
+ for (const candidate of buildSidecarAudioCandidates(localSourcePath)) {
+ try {
+ const result = await tryGenerate(candidate);
+ if (!cancelled && sourceRef.current === mediaResource) setData(result);
+ return;
+ } catch {
+ // try next
}
- } catch (error) {
- // File has no audio or decoding failed — leave as null.
- console.warn("[timeline-audio-peaks] failed", {
- input: fileUrl,
- error,
- });
}
- })();
+ };
+
+ void run();
return () => {
cancelled = true;
};
- }, [fileUrl]);
+ }, [mediaResource, enableSourceSidecarFallback]);
return data;
}
diff --git a/src/components/video-editor/timeline/model/timelineModel.ts b/src/components/video-editor/timeline/model/timelineModel.ts
index 0981b85c..b36aabc9 100644
--- a/src/components/video-editor/timeline/model/timelineModel.ts
+++ b/src/components/video-editor/timeline/model/timelineModel.ts
@@ -68,6 +68,7 @@ export function buildTimelineItems(params: {
rowId: getAudioTrackRowId(region.trackIndex ?? 0),
span: { start: region.startMs, end: region.endMs },
label: getAudioLabel(region),
+ audioPath: region.audioPath,
variant: "audio",
}));