The swiss army knife of lossless video/audio editing
improve waveform rendering #260
Mikael Finstad37ae097a2cd3
diff --git a/src/main/ffmpeg.ts b/src/main/ffmpeg.tsindex 1eb8808d..1726113f 100644--- a/src/main/ffmpeg.ts+++ b/src/main/ffmpeg.ts@@ -174,7 +174,11 @@ export async function runFfprobe(args: readonly string[], { timeout = isDev ? 10}export async function renderWaveformPng({ filePath, start, duration, color, streamIndex }: {- filePath: string, start: number, duration: number, color: string, streamIndex: number,+ filePath: string,+ start: number,+ duration: number,+ color: string,+ streamIndex: number,}): Promise<Waveform> {const args1 = ['-hide_banner',@@ -203,31 +207,21 @@ export async function renderWaveformPng({ filePath, start, duration, color, strelet ps1: ResultPromise<{ encoding: 'buffer' }> | undefined;let ps2: ResultPromise<{ encoding: 'buffer' }> | undefined;try {- ps1 = runFfmpegProcess(args1, { buffer: false }, { logCli: false });- ps2 = runFfmpegProcess(args2, undefined, { logCli: false });+ const timeout = 10000;+ ps1 = runFfmpegProcess(args1, { buffer: false, timeout }, { logCli: false });+ ps2 = runFfmpegProcess(args2, { timeout }, { logCli: false });assert(ps1.stdout != null);assert(ps2.stdin != null);ps1.stdout.pipe(ps2.stdin);- const timer = setTimeout(() => {- ps1?.kill();- ps2?.kill();- logger.warn('ffmpeg timed out');- }, 10000);-- let stdout;- try {- ({ stdout } = await ps2);- } finally {- clearTimeout(timer);- }+ const { stdout } = await ps2;return {buffer: Buffer.from(stdout),};} catch (err) {- if (ps1) ps1.kill();- if (ps2) ps2.kill();+ ps1?.kill();+ ps2?.kill();throw err;}}diff --git a/src/renderer/src/hooks/useWaveform.ts b/src/renderer/src/hooks/useWaveform.tsindex f642028a..1620bf62 100644--- a/src/renderer/src/hooks/useWaveform.ts+++ b/src/renderer/src/hooks/useWaveform.ts@@ -1,6 +1,5 @@import { useState, useRef, useEffect } from 'react';import sortBy from 'lodash/sortBy';-import { useThrottle } from '@uidotdev/usehooks';import { renderWaveformPng, safeCreateBlob } from '../ffmpeg';import { RenderableWaveform } from '../types';@@ -18,7 +17,6 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioStaudioStream: FFprobeStream | undefined,ffmpegExtractWindow: number,}) => {- const creatingWaveformPromise = useRef<Promise<unknown>>();const [waveforms, setWaveforms] = useState<RenderableWaveform[]>([]);const waveformsRef = useRef<RenderableWaveform[]>();@@ -34,63 +32,81 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioStconst waveformStartTime = Math.floor(relevantTime / ffmpegExtractWindow) * ffmpegExtractWindow;const safeExtractDuration = fileDuration != null ? Math.min(waveformStartTime + ffmpegExtractWindow, fileDuration) - waveformStartTime : undefined;- const waveformStartTimeThrottled = useThrottle(waveformStartTime, 1000);+ const waveformStartTimeRef = useRef(waveformStartTime);++ useEffect(() => {+ waveformStartTimeRef.current = waveformStartTime;+ }, [waveformStartTime]);useEffect(() => {let aborted = false;(async () => {- const alreadyHaveWaveformAtTime = (waveformsRef.current ?? []).some((waveform) => waveform.from === waveformStartTimeThrottled);- const shouldRun = !!filePath && safeExtractDuration != null && audioStream && waveformEnabled && !alreadyHaveWaveformAtTime && !creatingWaveformPromise.current;- if (!shouldRun) return;-- try {- const promise = renderWaveformPng({ filePath, start: waveformStartTimeThrottled, duration: safeExtractDuration, color: '##000000', streamIndex: audioStream.index });- creatingWaveformPromise.current = promise;-- setWaveforms((currentWaveforms) => {- const waveformsByCreatedAt = sortBy(currentWaveforms, 'createdAt');- return [- // cleanup old- ...(currentWaveforms.length >= maxWaveforms ? waveformsByCreatedAt.slice(1) : waveformsByCreatedAt),- // add new- {- from: waveformStartTimeThrottled,- to: waveformStartTimeThrottled + safeExtractDuration,- duration: safeExtractDuration,- createdAt: new Date(),- },- ];- });-- const { buffer } = await promise;-- if (aborted) {- setWaveforms((currentWaveforms) => currentWaveforms.filter((w) => w.from !== waveformStartTimeThrottled));- return;- }+ if (!filePath || safeExtractDuration == null || !audioStream || !waveformEnabled) {+ return;+ }- setWaveforms((currentWaveforms) => currentWaveforms.map((w) => {- if (w.from !== waveformStartTimeThrottled) {- return w;+ while (!aborted) {+ const times = [+ waveformStartTimeRef.current,+ waveformStartTimeRef.current + ffmpegExtractWindow,+ waveformStartTimeRef.current - ffmpegExtractWindow,+ ];++ for (const time of times) {+ const alreadyHaveWaveformAtTime = (waveformsRef.current ?? []).some((waveform) => waveform.from === time);+ if (!alreadyHaveWaveformAtTime) {+ try {+ const promise = renderWaveformPng({ filePath, start: time, duration: safeExtractDuration, color: '##000000', streamIndex: audioStream.index });++ setWaveforms((currentWaveforms) => {+ const waveformsByCreatedAt = sortBy(currentWaveforms, 'createdAt');+ return [+ // If too many waveforms, cleanup old+ ...(currentWaveforms.length >= maxWaveforms ? waveformsByCreatedAt.slice(1) : waveformsByCreatedAt),+ // Add new waveform+ {+ from: time,+ to: time + safeExtractDuration,+ duration: safeExtractDuration,+ createdAt: new Date(),+ },+ ];+ });++ const { buffer } = await promise;++ if (aborted) {+ // remove unfinished waveform+ setWaveforms((currentWaveforms) => currentWaveforms.filter((w) => w.from !== time));+ return;+ }++ setWaveforms((currentWaveforms) => currentWaveforms.map((w) => {+ if (w.from !== time) {+ return w;+ }++ return {+ ...w,+ url: URL.createObjectURL(safeCreateBlob(buffer, { type: 'image/png' })),+ };+ }));+ } catch (err) {+ console.error('Failed to render waveform', err);+ }}+ }- return {- ...w,- url: URL.createObjectURL(safeCreateBlob(buffer, { type: 'image/png' })),- };- }));- } catch (err) {- console.error('Failed to render waveform', err);- } finally {- creatingWaveformPromise.current = undefined;+ // could be problematic if we spawn ffmpeg processes too often, so throttle it+ await new Promise((r) => setTimeout(r, 100));}})();return () => {aborted = true;};- }, [audioStream, filePath, safeExtractDuration, waveformEnabled, waveformStartTimeThrottled]);+ }, [audioStream, ffmpegExtractWindow, filePath, safeExtractDuration, waveformEnabled]);const lastWaveformsRef = useRef<RenderableWaveform[]>([]);useEffect(() => {