Skip to content

The swiss army knife of lossless video/audio editing

improve waveform rendering #260

Mikael Finstad37ae097a2cd3
diff --git a/src/main/ffmpeg.ts b/src/main/ffmpeg.ts
index 1eb8808d..1726113f 100644
--- a/src/main/ffmpeg.ts
+++ b/src/main/ffmpeg.ts
@@ -174,7 +174,11 @@ export async function runFfprobe(args: readonly string[], { timeout = isDev ? 10
}
export async function renderWaveformPng({ filePath, start, duration, color, streamIndex }: {
- filePath: string, start: number, duration: number, color: string, streamIndex: number,
+ filePath: string,
+ start: number,
+ duration: number,
+ color: string,
+ streamIndex: number,
}): Promise<Waveform> {
const args1 = [
'-hide_banner',
@@ -203,31 +207,21 @@ export async function renderWaveformPng({ filePath, start, duration, color, stre
let ps1: ResultPromise<{ encoding: 'buffer' }> | undefined;
let ps2: ResultPromise<{ encoding: 'buffer' }> | undefined;
try {
- ps1 = runFfmpegProcess(args1, { buffer: false }, { logCli: false });
- ps2 = runFfmpegProcess(args2, undefined, { logCli: false });
+ const timeout = 10000;
+ ps1 = runFfmpegProcess(args1, { buffer: false, timeout }, { logCli: false });
+ ps2 = runFfmpegProcess(args2, { timeout }, { logCli: false });
assert(ps1.stdout != null);
assert(ps2.stdin != null);
ps1.stdout.pipe(ps2.stdin);
- const timer = setTimeout(() => {
- ps1?.kill();
- ps2?.kill();
- logger.warn('ffmpeg timed out');
- }, 10000);
-
- let stdout;
- try {
- ({ stdout } = await ps2);
- } finally {
- clearTimeout(timer);
- }
+ const { stdout } = await ps2;
return {
buffer: Buffer.from(stdout),
};
} catch (err) {
- if (ps1) ps1.kill();
- if (ps2) ps2.kill();
+ ps1?.kill();
+ ps2?.kill();
throw err;
}
}
diff --git a/src/renderer/src/hooks/useWaveform.ts b/src/renderer/src/hooks/useWaveform.ts
index f642028a..1620bf62 100644
--- a/src/renderer/src/hooks/useWaveform.ts
+++ b/src/renderer/src/hooks/useWaveform.ts
@@ -1,6 +1,5 @@
import { useState, useRef, useEffect } from 'react';
import sortBy from 'lodash/sortBy';
-import { useThrottle } from '@uidotdev/usehooks';
import { renderWaveformPng, safeCreateBlob } from '../ffmpeg';
import { RenderableWaveform } from '../types';
@@ -18,7 +17,6 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt
audioStream: FFprobeStream | undefined,
ffmpegExtractWindow: number,
}) => {
- const creatingWaveformPromise = useRef<Promise<unknown>>();
const [waveforms, setWaveforms] = useState<RenderableWaveform[]>([]);
const waveformsRef = useRef<RenderableWaveform[]>();
@@ -34,63 +32,81 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt
const waveformStartTime = Math.floor(relevantTime / ffmpegExtractWindow) * ffmpegExtractWindow;
const safeExtractDuration = fileDuration != null ? Math.min(waveformStartTime + ffmpegExtractWindow, fileDuration) - waveformStartTime : undefined;
- const waveformStartTimeThrottled = useThrottle(waveformStartTime, 1000);
+ const waveformStartTimeRef = useRef(waveformStartTime);
+
+ useEffect(() => {
+ waveformStartTimeRef.current = waveformStartTime;
+ }, [waveformStartTime]);
useEffect(() => {
let aborted = false;
(async () => {
- const alreadyHaveWaveformAtTime = (waveformsRef.current ?? []).some((waveform) => waveform.from === waveformStartTimeThrottled);
- const shouldRun = !!filePath && safeExtractDuration != null && audioStream && waveformEnabled && !alreadyHaveWaveformAtTime && !creatingWaveformPromise.current;
- if (!shouldRun) return;
-
- try {
- const promise = renderWaveformPng({ filePath, start: waveformStartTimeThrottled, duration: safeExtractDuration, color: '##000000', streamIndex: audioStream.index });
- creatingWaveformPromise.current = promise;
-
- setWaveforms((currentWaveforms) => {
- const waveformsByCreatedAt = sortBy(currentWaveforms, 'createdAt');
- return [
- // cleanup old
- ...(currentWaveforms.length >= maxWaveforms ? waveformsByCreatedAt.slice(1) : waveformsByCreatedAt),
- // add new
- {
- from: waveformStartTimeThrottled,
- to: waveformStartTimeThrottled + safeExtractDuration,
- duration: safeExtractDuration,
- createdAt: new Date(),
- },
- ];
- });
-
- const { buffer } = await promise;
-
- if (aborted) {
- setWaveforms((currentWaveforms) => currentWaveforms.filter((w) => w.from !== waveformStartTimeThrottled));
- return;
- }
+ if (!filePath || safeExtractDuration == null || !audioStream || !waveformEnabled) {
+ return;
+ }
- setWaveforms((currentWaveforms) => currentWaveforms.map((w) => {
- if (w.from !== waveformStartTimeThrottled) {
- return w;
+ while (!aborted) {
+ const times = [
+ waveformStartTimeRef.current,
+ waveformStartTimeRef.current + ffmpegExtractWindow,
+ waveformStartTimeRef.current - ffmpegExtractWindow,
+ ];
+
+ for (const time of times) {
+ const alreadyHaveWaveformAtTime = (waveformsRef.current ?? []).some((waveform) => waveform.from === time);
+ if (!alreadyHaveWaveformAtTime) {
+ try {
+ const promise = renderWaveformPng({ filePath, start: time, duration: safeExtractDuration, color: '##000000', streamIndex: audioStream.index });
+
+ setWaveforms((currentWaveforms) => {
+ const waveformsByCreatedAt = sortBy(currentWaveforms, 'createdAt');
+ return [
+ // If too many waveforms, cleanup old
+ ...(currentWaveforms.length >= maxWaveforms ? waveformsByCreatedAt.slice(1) : waveformsByCreatedAt),
+ // Add new waveform
+ {
+ from: time,
+ to: time + safeExtractDuration,
+ duration: safeExtractDuration,
+ createdAt: new Date(),
+ },
+ ];
+ });
+
+ const { buffer } = await promise;
+
+ if (aborted) {
+ // remove unfinished waveform
+ setWaveforms((currentWaveforms) => currentWaveforms.filter((w) => w.from !== time));
+ return;
+ }
+
+ setWaveforms((currentWaveforms) => currentWaveforms.map((w) => {
+ if (w.from !== time) {
+ return w;
+ }
+
+ return {
+ ...w,
+ url: URL.createObjectURL(safeCreateBlob(buffer, { type: 'image/png' })),
+ };
+ }));
+ } catch (err) {
+ console.error('Failed to render waveform', err);
+ }
}
+ }
- return {
- ...w,
- url: URL.createObjectURL(safeCreateBlob(buffer, { type: 'image/png' })),
- };
- }));
- } catch (err) {
- console.error('Failed to render waveform', err);
- } finally {
- creatingWaveformPromise.current = undefined;
+ // could be problematic if we spawn ffmpeg processes too often, so throttle it
+ await new Promise((r) => setTimeout(r, 100));
}
})();
return () => {
aborted = true;
};
- }, [audioStream, filePath, safeExtractDuration, waveformEnabled, waveformStartTimeThrottled]);
+ }, [audioStream, ffmpegExtractWindow, filePath, safeExtractDuration, waveformEnabled]);
const lastWaveformsRef = useRef<RenderableWaveform[]>([]);
useEffect(() => {