Create polished demo videos without editing skills. Mac/Windows/Linux
add: TimelineEditor waveformgenerator as a worker. to be extra efficient
Alan Trebugeais30aacaef483b
diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsxindex 29b0f25c..cf7005e4 100644--- a/src/components/video-editor/timeline/TimelineEditor.tsx+++ b/src/components/video-editor/timeline/TimelineEditor.tsx@@ -229,7 +229,9 @@ const TimelineEditor = forwardRef<TimelineEditorHandle, TimelineEditorProps>(return { previewSpans, hiddenZoomIds };}, [clipRegions, liveSpanPreviewById, zoomRegions]);const { shortcuts: keyShortcuts, isMac } = useShortcuts();- const audioPeaks = useTimelineAudioPeaks(videoPath);+ const audioPeaks = useTimelineAudioPeaks(videoPath, {+ enableSourceSidecarFallback: true,+ });useEffect(() => {if (aspectRatio === "native") {diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsxindex d6d13ed1..c7d65898 100644--- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx+++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx@@ -33,6 +33,7 @@ import {import TimelineAxis from "../axis/TimelineAxis";import ClipMarkerOverlay from "../overlays/ClipMarkerOverlay";import PlaybackCursor from "../playhead/PlaybackCursor";+import { useTimelineAudioPeaks } from "../../hooks/useTimelineAudioPeaks";const HINT_CLIP = "Press C to split clip";const HINT_ANNOTATION = "Press A to add annotation";@@ -234,6 +235,38 @@ interface TimelineCanvasRowsProps {onZoomRowClick: MouseEventHandler<HTMLDivElement>;}+interface AudioItemWithWaveformProps {+ item: TimelineRenderItem;+ span: { start: number; end: number };+ waveformSpan: { start: number; end: number };+ isSelected: boolean;+ onSelectAudio?: (id: string | null) => void;+}++function AudioItemWithWaveform({+ item,+ span,+ waveformSpan,+ isSelected,+ onSelectAudio,+}: AudioItemWithWaveformProps) {+ const peaks = useTimelineAudioPeaks(item.audioPath ?? null);+ return (+ <Item+ id={item.id}+ rowId={item.rowId}+ span={span}+ isSelected={isSelected}+ onSelectId={onSelectAudio}+ variant="audio"+ waveformPeaks={peaks}+ waveformSegmentSpan={waveformSpan}+ >+ {item.label}+ </Item>+ );+}+const TimelineCanvasRows = memo(function TimelineCanvasRows({items,videoDurationMs,@@ -422,17 +455,14 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({{audioRows.map(({ rowId, items: rowItems }, index) => (<Row key={rowId} id={rowId} isEmpty={rowItems.length === 0} hint={index === 0 ? HINT_AUDIO : undefined}>{rowItems.map((item) => (- <Item- id={item.id}+ <AudioItemWithWaveformkey={item.id}- rowId={item.rowId}+ item={item}span={item.span}+ waveformSpan={liveSpanPreviewById?.[item.id] ?? item.span}isSelected={selectAllBlocksActive || item.id === selectedAudioId}- onSelectId={onSelectAudio}- variant="audio"- >- {item.label}- </Item>+ onSelectAudio={onSelectAudio}+ />))}</Row>))}diff --git a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsxindex 367ed1ed..eff26d09 100644--- a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx+++ b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx@@ -61,8 +61,13 @@ function AudioWaveformComponent({const { peaks: peakData, durationMs } = peaks;if (durationMs <= 0 || peakData.length === 0) return;- const visibleStartMs = segmentStartMs ?? range.start;- const visibleEndMs = segmentEndMs ?? range.end;+ const rawVisibleStartMs = segmentStartMs ?? range.start;+ const rawVisibleEndMs = segmentEndMs ?? range.end;+ const msPerBin = durationMs / peakData.length;+ const visibleStartMs =+ msPerBin > 0 ? Math.round(rawVisibleStartMs / msPerBin) * msPerBin : rawVisibleStartMs;+ const visibleEndMs =+ msPerBin > 0 ? Math.round(rawVisibleEndMs / msPerBin) * msPerBin : rawVisibleEndMs;const visibleDurationMs = visibleEndMs - visibleStartMs;if (visibleDurationMs <= 0) return;@@ -71,11 +76,14 @@ function AudioWaveformComponent({ctx.beginPath();for (let px = 0; px < width; px++) {const t = visibleStartMs + (px / width) * visibleDurationMs;- const binIndex = Math.min(- peakData.length - 1,- Math.max(0, Math.floor((t / durationMs) * peakData.length)),+ const exactIndex = Math.max(+ 0,+ Math.min(peakData.length - 1, (t / durationMs) * (peakData.length - 1)),);- const amplitude = peakData[binIndex];+ const leftIndex = Math.floor(exactIndex);+ const rightIndex = Math.min(peakData.length - 1, leftIndex + 1);+ const mix = exactIndex - leftIndex;+ const amplitude = peakData[leftIndex] * (1 - mix) + peakData[rightIndex] * mix;const barHeight = amplitude * midY * 0.85;ctx.moveTo(px, midY - barHeight);diff --git a/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts b/src/components/video-editor/timeline/components/waveform/WaveformGenerator.tsnew file mode 100644index 00000000..7ff7f26c--- /dev/null+++ b/src/components/video-editor/timeline/components/waveform/WaveformGenerator.ts@@ -0,0 +1,88 @@+import WorkerConstructor from "./waveform.worker?worker";+import type { AudioPeaksData } from "../../core/timelineTypes";++const DEFAULT_PEAK_COUNT = 2048;++export class WaveformGenerator {+ private audioContext: AudioContext;+ private worker: Worker;+ private peaksCache = new Map<string, AudioPeaksData>();+ private pending = new Map<string, Promise<AudioPeaksData>>();++ constructor() {+ this.audioContext = new (window.AudioContext || (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext)();+ this.worker = new WorkerConstructor();+ }++ private computePeaksWithWorker(channelData: Float32Array, samples: number): Promise<Float32Array> {+ return new Promise((resolve, reject) => {+ const onMessage = (event: MessageEvent<Float32Array>) => {+ this.worker.removeEventListener("message", onMessage);+ this.worker.removeEventListener("error", onError);+ resolve(event.data);+ };+ const onError = (error: ErrorEvent) => {+ this.worker.removeEventListener("message", onMessage);+ this.worker.removeEventListener("error", onError);+ reject(error.error ?? new Error(error.message));+ };++ this.worker.addEventListener("message", onMessage);+ this.worker.addEventListener("error", onError);+ this.worker.postMessage(+ {+ channelData,+ samples,+ },+ [channelData.buffer],+ );+ });+ }++ public async generate(url: string, peakCount = DEFAULT_PEAK_COUNT): Promise<AudioPeaksData> {+ const cacheKey = `${url}::${peakCount}`;+ const cached = this.peaksCache.get(cacheKey);+ if (cached) return cached;++ const inflight = this.pending.get(cacheKey);+ if (inflight) return inflight;++ const request = (async () => {+ const response = await fetch(url);+ if (!response.ok) {+ throw new Error(`Failed to load media: ${response.status}`);+ }++ const arrayBuffer = await response.arrayBuffer();+ const decoded = await this.audioContext.decodeAudioData(arrayBuffer);+ const channelData = decoded.getChannelData(0).slice();+ const peaks = await this.computePeaksWithWorker(channelData, peakCount);++ let max = 0;+ for (let i = 0; i < peaks.length; i++) {+ if (peaks[i] > max) max = peaks[i];+ }+ if (max > 0) {+ for (let i = 0; i < peaks.length; i++) {+ peaks[i] /= max;+ }+ }++ const result: AudioPeaksData = {+ peaks,+ durationMs: decoded.duration * 1000,+ };+ this.peaksCache.set(cacheKey, result);+ this.pending.delete(cacheKey);+ return result;+ })().catch((error) => {+ this.pending.delete(cacheKey);+ throw error;+ });++ this.pending.set(cacheKey, request);+ return request;+ }+}++export const waveformGenerator = new WaveformGenerator();diff --git a/src/components/video-editor/timeline/components/waveform/waveform.worker.ts b/src/components/video-editor/timeline/components/waveform/waveform.worker.tsnew file mode 100644index 00000000..c8a7f8c5--- /dev/null+++ b/src/components/video-editor/timeline/components/waveform/waveform.worker.ts@@ -0,0 +1,31 @@+self.onmessage = (e: MessageEvent) => {+ const { channelData, samples } = e.data as {+ channelData: Float32Array;+ samples: number;+ };++ if (!channelData || samples <= 0) {+ self.postMessage(new Float32Array(0));+ return;+ }++ try {+ const step = Math.max(1, Math.floor(channelData.length / samples));+ const result = new Float32Array(samples);++ for (let i = 0; i < samples; i++) {+ const start = i * step;+ const end = Math.min(start + step, channelData.length);+ let max = 0;+ for (let j = start; j < end; j++) {+ const val = Math.abs(channelData[j]);+ if (val > max) max = val;+ }+ result[i] = max;+ }++ self.postMessage(result);+ } catch {+ self.postMessage(new Float32Array(0));+ }+};diff --git a/src/components/video-editor/timeline/core/timelineTypes.ts b/src/components/video-editor/timeline/core/timelineTypes.tsindex 2222f7cc..4a58c969 100644--- a/src/components/video-editor/timeline/core/timelineTypes.ts+++ b/src/components/video-editor/timeline/core/timelineTypes.ts@@ -32,6 +32,7 @@ export interface TimelineRenderItem {rowId: string;span: Span;label: string;+ audioPath?: string;zoomDepth?: number;zoomMode?: ZoomMode;speedValue?: number;diff --git a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.tsindex 9af79a80..4ca2a8f4 100644--- a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts+++ b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts@@ -1,10 +1,9 @@import { useEffect, useRef, useState } from "react";-import { fromFileUrl, resolveVideoUrl } from "../../projectPersistence";+import { resolveMediaResourceUrl } from "@/lib/exporter/localMediaSource";+import { fromFileUrl } from "../../projectPersistence";+import { waveformGenerator } from "../components/waveform/WaveformGenerator";import type { AudioPeaksData } from "../core/timelineTypes";-/** Number of peak bins to produce — enough for smooth display at any zoom. */-const TARGET_PEAK_COUNT = 2048;-function buildSidecarAudioCandidates(sourcePath: string): string[] {const normalized = sourcePath.replace(/\\/g, "/");const lastSlash = normalized.lastIndexOf("/");@@ -29,193 +28,69 @@ function extractLocalPathFromMediaServerUrl(input: string): string | null {(url.hostname === "127.0.0.1" || url.hostname === "localhost") &&url.pathname === "/video";if (!isLocalMediaServer) return null;- const rawPath = url.searchParams.get("path");- if (!rawPath) return null;- return rawPath;+ return url.searchParams.get("path");} catch {return null;}}-async function decodePeaksFromMediaUrl(- mediaUrl: string,- cancelled: () => boolean,-): Promise<AudioPeaksData> {- const response = await fetch(mediaUrl);- if (!response.ok) {- throw new Error(`Failed to load media for waveform: ${response.status}`);- }- if (cancelled()) throw new Error("cancelled");- console.debug("[timeline-audio-peaks] fetch ok", {- url: mediaUrl,- status: response.status,- });-- const arrayBuffer = await response.arrayBuffer();- if (cancelled()) throw new Error("cancelled");- console.debug("[timeline-audio-peaks] bytes loaded", {- url: mediaUrl,- byteLength: arrayBuffer.byteLength,- });-- const audioCtx = new OfflineAudioContext(1, 1, 44100);- const decoded = await audioCtx.decodeAudioData(arrayBuffer);- if (cancelled()) throw new Error("cancelled");- console.debug("[timeline-audio-peaks] decode ok", {- url: mediaUrl,- durationSec: decoded.duration,- channelCount: decoded.numberOfChannels,- sampleRate: decoded.sampleRate,- sampleFrames: decoded.length,- });-- const channelData = decoded.getChannelData(0);- const durationMs = decoded.duration * 1000;- const binSize = Math.max(1, Math.floor(channelData.length / TARGET_PEAK_COUNT));- const peakCount = Math.ceil(channelData.length / binSize);- const peaks = new Float32Array(peakCount);-- for (let i = 0; i < peakCount; i++) {- const start = i * binSize;- const end = Math.min(start + binSize, channelData.length);- let max = 0;- for (let j = start; j < end; j++) {- const abs = Math.abs(channelData[j]);- if (abs > max) max = abs;- }- peaks[i] = max;- }-- let globalMax = 0;- for (let i = 0; i < peaks.length; i++) {- if (peaks[i] > globalMax) globalMax = peaks[i];- }- if (globalMax > 0) {- for (let i = 0; i < peaks.length; i++) {- peaks[i] /= globalMax;- }- }-- return { peaks, durationMs };+interface TimelineAudioPeaksOptions {+ enableSourceSidecarFallback?: boolean;}-/**- * Decode audio from a media file URL and produce a fixed-length array of peak- * amplitudes suitable for waveform visualisation.- *- * Returns `null` while loading or if the file has no decodeable audio.- */-export function useTimelineAudioPeaks(fileUrl: string | null | undefined): AudioPeaksData | null {+export function useTimelineAudioPeaks(+ mediaResource: string | null | undefined,+ options: TimelineAudioPeaksOptions = {},+): AudioPeaksData | null {const [data, setData] = useState<AudioPeaksData | null>(null);- const urlRef = useRef(fileUrl);+ const sourceRef = useRef(mediaResource);+ const enableSourceSidecarFallback = options.enableSourceSidecarFallback ?? false;useEffect(() => {- urlRef.current = fileUrl;+ sourceRef.current = mediaResource;setData(null);- console.debug("[timeline-audio-peaks] reset", { input: fileUrl ?? null });-- if (!fileUrl) {- console.debug("[timeline-audio-peaks] no input source, skipping");- return;- }+ if (!mediaResource) return;let cancelled = false;- (async () => {+ const run = async () => {+ const localPathFromServer = extractLocalPathFromMediaServerUrl(mediaResource);+ const localSourcePath =+ localPathFromServer ||+ (/^file:\/\//i.test(mediaResource) ? fromFileUrl(mediaResource) : mediaResource);++ const tryGenerate = async (resource: string): Promise<AudioPeaksData> => {+ const resolvedUrl = await resolveMediaResourceUrl(resource);+ return waveformGenerator.generate(resolvedUrl);+ };+try {- const extractedLocalPath = extractLocalPathFromMediaServerUrl(fileUrl);- const isRemoteLike = /^(https?:|blob:|data:)/i.test(fileUrl) && !extractedLocalPath;- const isFileUrl = /^file:\/\//i.test(fileUrl);- const sourceLocalPath = isRemoteLike- ? null- : extractedLocalPath- ? extractedLocalPath- : isFileUrl- ? fromFileUrl(fileUrl)- : fileUrl;- console.debug("[timeline-audio-peaks] resolving source", {- input: fileUrl,- isRemoteLike,- isFileUrl,- sourceLocalPath,- extractedLocalPath,- });- const mediaUrl = isRemoteLike- ? fileUrl- : isFileUrl- ? await resolveVideoUrl(fromFileUrl(fileUrl))- : await resolveVideoUrl(fileUrl);- if (cancelled) return;- console.debug("[timeline-audio-peaks] resolved source", {- input: fileUrl,- resolved: mediaUrl,- });+ const result = await tryGenerate(mediaResource);+ if (!cancelled && sourceRef.current === mediaResource) setData(result);+ return;+ } catch {+ // fallthrough+ }- let decodedPeaks: AudioPeaksData | null = null;- try {- decodedPeaks = await decodePeaksFromMediaUrl(mediaUrl, () => cancelled);- } catch (primaryError) {- if (primaryError instanceof Error && primaryError.message === "cancelled") {- return;- }- console.warn("[timeline-audio-peaks] primary decode failed", {- input: fileUrl,- url: mediaUrl,- error: primaryError,- });- if (sourceLocalPath) {- const candidates = buildSidecarAudioCandidates(sourceLocalPath);- console.debug("[timeline-audio-peaks] trying sidecar candidates", {- sourceLocalPath,- candidates,- });- for (const candidatePath of candidates) {- try {- const candidateUrl = await resolveVideoUrl(candidatePath);- console.debug("[timeline-audio-peaks] trying sidecar", {- candidatePath,- candidateUrl,- });- decodedPeaks = await decodePeaksFromMediaUrl(candidateUrl, () => cancelled);- console.info("[timeline-audio-peaks] sidecar decode succeeded", {- candidatePath,- candidateUrl,- });- break;- } catch (sidecarError) {- if (sidecarError instanceof Error && sidecarError.message === "cancelled") {- return;- }- console.debug("[timeline-audio-peaks] sidecar decode failed", {- candidatePath,- error: sidecarError,- });- }- }- }- }+ if (!enableSourceSidecarFallback || !localSourcePath) return;- if (!cancelled && urlRef.current === fileUrl && decodedPeaks) {- console.debug("[timeline-audio-peaks] peaks generated", {- input: fileUrl,- durationMs: Math.round(decodedPeaks.durationMs),- peakCount: decodedPeaks.peaks.length,- });- setData(decodedPeaks);+ for (const candidate of buildSidecarAudioCandidates(localSourcePath)) {+ try {+ const result = await tryGenerate(candidate);+ if (!cancelled && sourceRef.current === mediaResource) setData(result);+ return;+ } catch {+ // try next}- } catch (error) {- // File has no audio or decoding failed — leave as null.- console.warn("[timeline-audio-peaks] failed", {- input: fileUrl,- error,- });}- })();+ };++ void run();return () => {cancelled = true;};- }, [fileUrl]);+ }, [mediaResource, enableSourceSidecarFallback]);return data;}diff --git a/src/components/video-editor/timeline/model/timelineModel.ts b/src/components/video-editor/timeline/model/timelineModel.tsindex 0981b85c..b36aabc9 100644--- a/src/components/video-editor/timeline/model/timelineModel.ts+++ b/src/components/video-editor/timeline/model/timelineModel.ts@@ -68,6 +68,7 @@ export function buildTimelineItems(params: {rowId: getAudioTrackRowId(region.trackIndex ?? 0),span: { start: region.startMs, end: region.endMs },label: getAudioLabel(region),+ audioPath: region.audioPath,variant: "audio",}));