The swiss army knife of lossless video/audio editing
improve waveform and keyframes
Mikael Finstadac127e88ab42
diff --git a/src/App.jsx b/src/App.jsxindex b666f390..d890e908 100644--- a/src/App.jsx+++ b/src/App.jsx@@ -821,8 +821,8 @@ const App = memo(() => {const shouldShowKeyframes = keyframesEnabled && !!mainVideoStream && calcShouldShowKeyframes(zoomedDuration);const shouldShowWaveform = calcShouldShowWaveform(zoomedDuration);- const { neighbouringFrames, findNearestKeyFrameTime } = useKeyframes({ keyframesEnabled, filePath, commandedTime, mainVideoStream, detectedFps, ffmpegExtractWindow });- const { waveform } = useWaveform({ filePath, commandedTime, zoomedDuration, waveformEnabled, mainAudioStream, shouldShowWaveform, ffmpegExtractWindow });+ const { neighbouringKeyFrames, findNearestKeyFrameTime } = useKeyframes({ keyframesEnabled, filePath, commandedTime, mainVideoStream, detectedFps, ffmpegExtractWindow });+ const { waveforms } = useWaveform({ filePath, commandedTime, zoomedDuration, waveformEnabled, mainAudioStream, shouldShowWaveform, ffmpegExtractWindow });const resetState = useCallback(() => {const video = videoRef.current;@@ -2425,11 +2425,11 @@ const App = memo(() => {><TimelineshouldShowKeyframes={shouldShowKeyframes}- waveform={waveform}+ waveforms={waveforms}shouldShowWaveform={shouldShowWaveform}waveformEnabled={waveformEnabled}thumbnailsEnabled={thumbnailsEnabled}- neighbouringFrames={neighbouringFrames}+ neighbouringKeyFrames={neighbouringKeyFrames}thumbnails={thumbnailsSorted}getCurrentTime={getCurrentTime}commandedTimeRef={commandedTimeRef}diff --git a/src/Timeline.jsx b/src/Timeline.jsxindex 35675497..69d9d519 100644--- a/src/Timeline.jsx+++ b/src/Timeline.jsx@@ -18,8 +18,7 @@ const currentTimeWidth = 1;const hammerOptions = { recognizers: {} };-const Waveform = memo(({ calculateTimelinePercent, durationSafe, waveform, zoom, timelineHeight }) => {- const imgRef = useRef();+const Waveform = memo(({ waveform, calculateTimelinePercent, durationSafe }) => {const [style, setStyle] = useState({ display: 'none' });const leftPos = calculateTimelinePercent(waveform.from);@@ -32,14 +31,19 @@ const Waveform = memo(({ calculateTimelinePercent, durationSafe, waveform, zoom,position: 'absolute', height: '100%', left: leftPos, width: `${((toTruncated - waveform.from) / durationSafe) * 100}%`,});}-return (- <div style={{ height: timelineHeight, width: `${zoom * 100}%`, position: 'relative' }}>- <img ref={imgRef} src={waveform.url} draggable={false} style={style} alt="" onLoad={onLoad} />- </div>+ <img src={waveform.url} draggable={false} style={style} alt="" onLoad={onLoad} />);});+const Waveforms = memo(({ calculateTimelinePercent, durationSafe, waveforms, zoom, timelineHeight }) => (+ <div style={{ height: timelineHeight, width: `${zoom * 100}%`, position: 'relative' }}>+ {waveforms.map((waveform) => (+ <Waveform key={`${waveform.from}-${waveform.to}`} waveform={waveform} calculateTimelinePercent={calculateTimelinePercent} durationSafe={durationSafe} />+ ))}+ </div>+));+const CommandedTime = memo(({ commandedTimePercent }) => {const color = 'white';const commonStyle = { left: commandedTimePercent, position: 'absolute', zIndex: 4, pointerEvents: 'none' };@@ -54,9 +58,9 @@ const CommandedTime = memo(({ commandedTimePercent }) => {const Timeline = memo(({durationSafe, getCurrentTime, startTimeOffset, playerTime, commandedTime,- zoom, neighbouringFrames, seekAbs, apparentCutSegments,+ zoom, neighbouringKeyFrames, seekAbs, apparentCutSegments,setCurrentSegIndex, currentSegIndexSafe, invertCutSegments, inverseCutSegments, formatTimecode,- waveform, shouldShowWaveform, shouldShowKeyframes, timelineHeight, thumbnails,+ waveforms, shouldShowWaveform, shouldShowKeyframes, timelineHeight, thumbnails,onZoomWindowStartTimeChange, waveformEnabled, thumbnailsEnabled,playing, isFileOpened, onWheel, commandedTimeRef, goToTimecode,}) => {@@ -75,10 +79,11 @@ const Timeline = memo(({const isZoomed = zoom > 1;- const keyframes = neighbouringFrames ? neighbouringFrames.filter(f => f.keyframe) : [];// Don't show keyframes if too packed together (at current zoom)// See https://github.com/mifi/lossless-cut/issues/259- const areKeyframesTooClose = keyframes.length > zoom * 200;+ // todo+ // const areKeyframesTooClose = keyframes.length > zoom * 200;+ const areKeyframesTooClose = false;const calculateTimelinePos = useCallback((time) => (time !== undefined ? Math.min(time / durationSafe, 1) : undefined), [durationSafe]);const calculateTimelinePercent = useCallback((time) => {@@ -218,11 +223,11 @@ const Timeline = memo(({onScroll={onTimelineScroll}ref={timelineScrollerRef}>- {waveformEnabled && shouldShowWaveform && waveform && (- <Waveform+ {waveformEnabled && shouldShowWaveform && waveforms && (+ <WaveformscalculateTimelinePercent={calculateTimelinePercent}durationSafe={durationSafe}- waveform={waveform}+ waveforms={waveforms}zoom={zoom}timelineHeight={timelineHeight}/>@@ -288,7 +293,7 @@ const Timeline = memo(({/>))}- {shouldShowKeyframes && !areKeyframesTooClose && keyframes.map((f) => (+ {shouldShowKeyframes && !areKeyframesTooClose && neighbouringKeyFrames.map((f) => (<div key={f.time} style={{ position: 'absolute', top: 0, bottom: 0, left: `${(f.time / durationSafe) * 100}%`, marginLeft: -1, width: 1, background: 'rgba(0,0,0,0.4)', pointerEvents: 'none' }} />))}</div>diff --git a/src/ffmpeg.js b/src/ffmpeg.jsindex ef9c0b0b..1a8f7d55 100644--- a/src/ffmpeg.js+++ b/src/ffmpeg.js@@ -30,10 +30,17 @@ function getFfPath(cmd) {export const getFfmpegPath = () => getFfPath('ffmpeg');export const getFfprobePath = () => getFfPath('ffprobe');-export async function runFfprobe(args) {+export async function runFfprobe(args, { timeout = isDev ? 10000 : 30000 } = {}) {const ffprobePath = getFfprobePath();console.log(getFfCommandLine('ffprobe', args));- return execa(ffprobePath, args);+ const ps = execa(ffprobePath, args);+ const timer = setTimeout(() => {+ console.warn('killing timed out ffprobe');+ ps.kill();+ }, timeout);+ const ret = await ps;+ clearTimeout(timer);+ return ret;}export function runFfmpeg(args) {@@ -92,7 +99,11 @@ export async function readFrames({ filePath, aroundTime, window, stream }) {}const { stdout } = await runFfprobe(['-v', 'error', ...intervalsArgs, '-show_packets', '-select_streams', stream, '-show_entries', 'packet=pts_time,flags', '-of', 'json', filePath]);const packetsFiltered = JSON.parse(stdout).packets- .map(p => ({ keyframe: p.flags[0] === 'K', time: parseFloat(p.pts_time, 10) }))+ .map(p => ({+ keyframe: p.flags[0] === 'K',+ time: parseFloat(p.pts_time, 10),+ createdAt: new Date(),+ })).filter(p => !Number.isNaN(p.time));return sortBy(packetsFiltered, 'time');@@ -472,7 +483,12 @@ export async function renderWaveformPng({ filePath, aroundTime, window, color })ps2 = execa(ffmpegPath, args2, { encoding: null });ps1.stdout.pipe(ps2.stdin);+ const timer = setTimeout(() => {+ ps1.kill();+ ps2.kill();+ }, 10000);const { stdout } = await ps2;+ clearTimeout(timer);const blob = new Blob([stdout], { type: 'image/png' });@@ -481,6 +497,7 @@ export async function renderWaveformPng({ filePath, aroundTime, window, color })from,aroundTime,to,+ createdAt: new Date(),};} catch (err) {if (ps1) ps1.kill();diff --git a/src/hooks/useKeyframes.js b/src/hooks/useKeyframes.jsindex fc7f8982..0fab97ed 100644--- a/src/hooks/useKeyframes.js+++ b/src/hooks/useKeyframes.js@@ -1,43 +1,61 @@-import { useState, useCallback, useRef, useEffect } from 'react';+import { useState, useCallback, useRef, useEffect, useMemo } from 'react';+import sortBy from 'lodash/sortBy';import useDebounceOld from 'react-use/lib/useDebounce'; // Want to phase out thisimport { readFrames, findNearestKeyFrameTime as ffmpegFindNearestKeyFrameTime } from '../ffmpeg';+const maxKeyframes = 1000;+// const maxKeyframes = 100;export default ({ keyframesEnabled, filePath, commandedTime, mainVideoStream, detectedFps, ffmpegExtractWindow }) => {const readingKeyframesPromise = useRef();- const [neighbouringFrames, setNeighbouringFrames] = useState([]);+ const [neighbouringKeyFramesMap, setNeighbouringKeyFrames] = useState({});+ const neighbouringKeyFrames = useMemo(() => Object.values(neighbouringKeyFramesMap), [neighbouringKeyFramesMap]);- const findNearestKeyFrameTime = useCallback(({ time, direction }) => ffmpegFindNearestKeyFrameTime({ frames: neighbouringFrames, time, direction, fps: detectedFps }), [neighbouringFrames, detectedFps]);+ const findNearestKeyFrameTime = useCallback(({ time, direction }) => ffmpegFindNearestKeyFrameTime({ frames: neighbouringKeyFrames, time, direction, fps: detectedFps }), [neighbouringKeyFrames, detectedFps]);- useEffect(() => {- setNeighbouringFrames([]);- }, [filePath]);+ useEffect(() => setNeighbouringKeyFrames({}), [filePath]);useDebounceOld(() => {- // We still want to calculate keyframes even if not shouldShowKeyframes because maybe we want to be able to step to the closest keyframe- const shouldRun = () => keyframesEnabled && filePath && mainVideoStream && commandedTime != null;+ let aborted = false;- async function run() {- if (!shouldRun() || readingKeyframesPromise.current) return;+ (async () => {+ // See getIntervalAroundTime+ // We still want to calculate keyframes even if not shouldShowKeyframes because maybe we want to be able to step to the closest keyframe+ const shouldRun = keyframesEnabled && filePath && mainVideoStream && commandedTime != null && !readingKeyframesPromise.current;+ if (!shouldRun) return;try {const promise = readFrames({ filePath, aroundTime: commandedTime, stream: mainVideoStream.index, window: ffmpegExtractWindow });readingKeyframesPromise.current = promise;const newFrames = await promise;- if (!shouldRun()) return;+ if (aborted) return;+ const newKeyFrames = newFrames.filter((frame) => frame.keyframe);// console.log(newFrames);- setNeighbouringFrames(newFrames);+ setNeighbouringKeyFrames((existingKeyFramesMap) => {+ let existingFrames = Object.values(existingKeyFramesMap);+ if (existingFrames.length >= maxKeyframes) {+ existingFrames = sortBy(existingFrames, 'createdAt').slice(newKeyFrames.length);+ }+ const toObj = (map) => Object.fromEntries(map.map((frame) => [frame.time, frame]));+ return {+ ...toObj(existingFrames),+ ...toObj(newKeyFrames),+ };+ });} catch (err) {console.error('Failed to read keyframes', err);} finally {readingKeyframesPromise.current = undefined;}- }- run();- }, 500, [keyframesEnabled, filePath, commandedTime, mainVideoStream]);+ })();++ return () => {+ aborted = true;+ };+ }, 500, [keyframesEnabled, filePath, commandedTime, mainVideoStream, ffmpegExtractWindow]);return {- neighbouringFrames, findNearestKeyFrameTime,+ neighbouringKeyFrames, findNearestKeyFrameTime,};};diff --git a/src/hooks/useWaveform.js b/src/hooks/useWaveform.jsindex c10fc4ce..565a35d8 100644--- a/src/hooks/useWaveform.js+++ b/src/hooks/useWaveform.js@@ -1,40 +1,61 @@import { useState, useRef, useEffect } from 'react';+import sortBy from 'lodash/sortBy';import useDebounceOld from 'react-use/lib/useDebounce'; // Want to phase out thisimport { waveformColor } from '../colors';import { renderWaveformPng } from '../ffmpeg';+const maxWaveforms = 100;+// const maxWaveforms = 3; // testing+export default ({ filePath, commandedTime, zoomedDuration, waveformEnabled, mainAudioStream, shouldShowWaveform, ffmpegExtractWindow }) => {const creatingWaveformPromise = useRef();- const [waveform, setWaveform] = useState();-- useEffect(() => {- setWaveform();- }, [filePath]);+ const [waveforms, setWaveforms] = useState([]);useDebounceOld(() => {- const shouldRun = () => filePath && mainAudioStream && commandedTime != null && shouldShowWaveform && waveformEnabled;+ let aborted = false;++ (async () => {+ const alreadyHaveWaveformAtCommandedTime = waveforms.some((waveform) => waveform.from < commandedTime && waveform.to > commandedTime);+ const shouldRun = filePath && mainAudioStream && commandedTime != null && shouldShowWaveform && waveformEnabled && !alreadyHaveWaveformAtCommandedTime && !creatingWaveformPromise.current;+ if (!shouldRun) return;- async function run() {- if (!shouldRun() || creatingWaveformPromise.current) return;try {const promise = renderWaveformPng({ filePath, aroundTime: commandedTime, window: ffmpegExtractWindow, color: waveformColor });creatingWaveformPromise.current = promise;- if (!shouldRun()) return;- const wf = await promise;- setWaveform(wf);+ const newWaveform = await promise;+ if (aborted) return;+ setWaveforms((currentWaveforms) => {+ const waveformsByCreatedAt = sortBy(currentWaveforms, 'createdAt');+ return [+ // cleanup old+ ...(currentWaveforms.length >= maxWaveforms ? waveformsByCreatedAt.slice(1) : waveformsByCreatedAt),+ newWaveform,+ ];+ });} catch (err) {console.error('Failed to render waveform', err);} finally {creatingWaveformPromise.current = undefined;}- }+ })();++ return () => {+ aborted = true;+ };+ }, 500, [filePath, commandedTime, zoomedDuration, waveformEnabled, mainAudioStream, shouldShowWaveform, waveforms, ffmpegExtractWindow]);- run();- }, 500, [filePath, commandedTime, zoomedDuration, waveformEnabled, mainAudioStream, shouldShowWaveform]);+ const lastWaveformsRef = useRef([]);+ useEffect(() => {+ const removedWaveforms = lastWaveformsRef.current.filter((wf) => !waveforms.includes(wf));+ // Cleanup old+ // if (removedWaveforms.length > 0) console.log('cleanup waveforms', removedWaveforms.length);+ removedWaveforms.forEach((waveform) => URL.revokeObjectURL(waveform.url));+ lastWaveformsRef.current = waveforms;+ }, [waveforms]);- // Cleanup old- useEffect(() => () => waveform && URL.revokeObjectURL(waveform.url), [waveform]);+ useEffect(() => setWaveforms([]), [filePath]);+ useEffect(() => () => setWaveforms([]), []);- return { waveform };+ return { waveforms };};