Skip to content

The swiss army knife of lossless video/audio editing

improve waveform and keyframes

Mikael Finstadac127e88ab42
diff --git a/src/App.jsx b/src/App.jsx
index b666f390..d890e908 100644
--- a/src/App.jsx
+++ b/src/App.jsx
@@ -821,8 +821,8 @@ const App = memo(() => {
const shouldShowKeyframes = keyframesEnabled && !!mainVideoStream && calcShouldShowKeyframes(zoomedDuration);
const shouldShowWaveform = calcShouldShowWaveform(zoomedDuration);
- const { neighbouringFrames, findNearestKeyFrameTime } = useKeyframes({ keyframesEnabled, filePath, commandedTime, mainVideoStream, detectedFps, ffmpegExtractWindow });
- const { waveform } = useWaveform({ filePath, commandedTime, zoomedDuration, waveformEnabled, mainAudioStream, shouldShowWaveform, ffmpegExtractWindow });
+ const { neighbouringKeyFrames, findNearestKeyFrameTime } = useKeyframes({ keyframesEnabled, filePath, commandedTime, mainVideoStream, detectedFps, ffmpegExtractWindow });
+ const { waveforms } = useWaveform({ filePath, commandedTime, zoomedDuration, waveformEnabled, mainAudioStream, shouldShowWaveform, ffmpegExtractWindow });
const resetState = useCallback(() => {
const video = videoRef.current;
@@ -2425,11 +2425,11 @@ const App = memo(() => {
>
<Timeline
shouldShowKeyframes={shouldShowKeyframes}
- waveform={waveform}
+ waveforms={waveforms}
shouldShowWaveform={shouldShowWaveform}
waveformEnabled={waveformEnabled}
thumbnailsEnabled={thumbnailsEnabled}
- neighbouringFrames={neighbouringFrames}
+ neighbouringKeyFrames={neighbouringKeyFrames}
thumbnails={thumbnailsSorted}
getCurrentTime={getCurrentTime}
commandedTimeRef={commandedTimeRef}
diff --git a/src/Timeline.jsx b/src/Timeline.jsx
index 35675497..69d9d519 100644
--- a/src/Timeline.jsx
+++ b/src/Timeline.jsx
@@ -18,8 +18,7 @@ const currentTimeWidth = 1;
const hammerOptions = { recognizers: {} };
-const Waveform = memo(({ calculateTimelinePercent, durationSafe, waveform, zoom, timelineHeight }) => {
- const imgRef = useRef();
+const Waveform = memo(({ waveform, calculateTimelinePercent, durationSafe }) => {
const [style, setStyle] = useState({ display: 'none' });
const leftPos = calculateTimelinePercent(waveform.from);
@@ -32,14 +31,19 @@ const Waveform = memo(({ calculateTimelinePercent, durationSafe, waveform, zoom,
position: 'absolute', height: '100%', left: leftPos, width: `${((toTruncated - waveform.from) / durationSafe) * 100}%`,
});
}
-
return (
- <div style={{ height: timelineHeight, width: `${zoom * 100}%`, position: 'relative' }}>
- <img ref={imgRef} src={waveform.url} draggable={false} style={style} alt="" onLoad={onLoad} />
- </div>
+ <img src={waveform.url} draggable={false} style={style} alt="" onLoad={onLoad} />
);
});
+const Waveforms = memo(({ calculateTimelinePercent, durationSafe, waveforms, zoom, timelineHeight }) => (
+ <div style={{ height: timelineHeight, width: `${zoom * 100}%`, position: 'relative' }}>
+ {waveforms.map((waveform) => (
+ <Waveform key={`${waveform.from}-${waveform.to}`} waveform={waveform} calculateTimelinePercent={calculateTimelinePercent} durationSafe={durationSafe} />
+ ))}
+ </div>
+));
+
const CommandedTime = memo(({ commandedTimePercent }) => {
const color = 'white';
const commonStyle = { left: commandedTimePercent, position: 'absolute', zIndex: 4, pointerEvents: 'none' };
@@ -54,9 +58,9 @@ const CommandedTime = memo(({ commandedTimePercent }) => {
const Timeline = memo(({
durationSafe, getCurrentTime, startTimeOffset, playerTime, commandedTime,
- zoom, neighbouringFrames, seekAbs, apparentCutSegments,
+ zoom, neighbouringKeyFrames, seekAbs, apparentCutSegments,
setCurrentSegIndex, currentSegIndexSafe, invertCutSegments, inverseCutSegments, formatTimecode,
- waveform, shouldShowWaveform, shouldShowKeyframes, timelineHeight, thumbnails,
+ waveforms, shouldShowWaveform, shouldShowKeyframes, timelineHeight, thumbnails,
onZoomWindowStartTimeChange, waveformEnabled, thumbnailsEnabled,
playing, isFileOpened, onWheel, commandedTimeRef, goToTimecode,
}) => {
@@ -75,10 +79,11 @@ const Timeline = memo(({
const isZoomed = zoom > 1;
- const keyframes = neighbouringFrames ? neighbouringFrames.filter(f => f.keyframe) : [];
// Don't show keyframes if too packed together (at current zoom)
// See https://github.com/mifi/lossless-cut/issues/259
- const areKeyframesTooClose = keyframes.length > zoom * 200;
+ // todo
+ // const areKeyframesTooClose = keyframes.length > zoom * 200;
+ const areKeyframesTooClose = false;
const calculateTimelinePos = useCallback((time) => (time !== undefined ? Math.min(time / durationSafe, 1) : undefined), [durationSafe]);
const calculateTimelinePercent = useCallback((time) => {
@@ -218,11 +223,11 @@ const Timeline = memo(({
onScroll={onTimelineScroll}
ref={timelineScrollerRef}
>
- {waveformEnabled && shouldShowWaveform && waveform && (
- <Waveform
+ {waveformEnabled && shouldShowWaveform && waveforms && (
+ <Waveforms
calculateTimelinePercent={calculateTimelinePercent}
durationSafe={durationSafe}
- waveform={waveform}
+ waveforms={waveforms}
zoom={zoom}
timelineHeight={timelineHeight}
/>
@@ -288,7 +293,7 @@ const Timeline = memo(({
/>
))}
- {shouldShowKeyframes && !areKeyframesTooClose && keyframes.map((f) => (
+ {shouldShowKeyframes && !areKeyframesTooClose && neighbouringKeyFrames.map((f) => (
<div key={f.time} style={{ position: 'absolute', top: 0, bottom: 0, left: `${(f.time / durationSafe) * 100}%`, marginLeft: -1, width: 1, background: 'rgba(0,0,0,0.4)', pointerEvents: 'none' }} />
))}
</div>
diff --git a/src/ffmpeg.js b/src/ffmpeg.js
index ef9c0b0b..1a8f7d55 100644
--- a/src/ffmpeg.js
+++ b/src/ffmpeg.js
@@ -30,10 +30,17 @@ function getFfPath(cmd) {
export const getFfmpegPath = () => getFfPath('ffmpeg');
export const getFfprobePath = () => getFfPath('ffprobe');
-export async function runFfprobe(args) {
+export async function runFfprobe(args, { timeout = isDev ? 10000 : 30000 } = {}) {
const ffprobePath = getFfprobePath();
console.log(getFfCommandLine('ffprobe', args));
- return execa(ffprobePath, args);
+ const ps = execa(ffprobePath, args);
+ const timer = setTimeout(() => {
+ console.warn('killing timed out ffprobe');
+ ps.kill();
+ }, timeout);
+ const ret = await ps;
+ clearTimeout(timer);
+ return ret;
}
export function runFfmpeg(args) {
@@ -92,7 +99,11 @@ export async function readFrames({ filePath, aroundTime, window, stream }) {
}
const { stdout } = await runFfprobe(['-v', 'error', ...intervalsArgs, '-show_packets', '-select_streams', stream, '-show_entries', 'packet=pts_time,flags', '-of', 'json', filePath]);
const packetsFiltered = JSON.parse(stdout).packets
- .map(p => ({ keyframe: p.flags[0] === 'K', time: parseFloat(p.pts_time, 10) }))
+ .map(p => ({
+ keyframe: p.flags[0] === 'K',
+ time: parseFloat(p.pts_time, 10),
+ createdAt: new Date(),
+ }))
.filter(p => !Number.isNaN(p.time));
return sortBy(packetsFiltered, 'time');
@@ -472,7 +483,12 @@ export async function renderWaveformPng({ filePath, aroundTime, window, color })
ps2 = execa(ffmpegPath, args2, { encoding: null });
ps1.stdout.pipe(ps2.stdin);
+ const timer = setTimeout(() => {
+ ps1.kill();
+ ps2.kill();
+ }, 10000);
const { stdout } = await ps2;
+ clearTimeout(timer);
const blob = new Blob([stdout], { type: 'image/png' });
@@ -481,6 +497,7 @@ export async function renderWaveformPng({ filePath, aroundTime, window, color })
from,
aroundTime,
to,
+ createdAt: new Date(),
};
} catch (err) {
if (ps1) ps1.kill();
diff --git a/src/hooks/useKeyframes.js b/src/hooks/useKeyframes.js
index fc7f8982..0fab97ed 100644
--- a/src/hooks/useKeyframes.js
+++ b/src/hooks/useKeyframes.js
@@ -1,43 +1,61 @@
-import { useState, useCallback, useRef, useEffect } from 'react';
+import { useState, useCallback, useRef, useEffect, useMemo } from 'react';
+import sortBy from 'lodash/sortBy';
import useDebounceOld from 'react-use/lib/useDebounce'; // Want to phase out this
import { readFrames, findNearestKeyFrameTime as ffmpegFindNearestKeyFrameTime } from '../ffmpeg';
+const maxKeyframes = 1000;
+// const maxKeyframes = 100;
export default ({ keyframesEnabled, filePath, commandedTime, mainVideoStream, detectedFps, ffmpegExtractWindow }) => {
const readingKeyframesPromise = useRef();
- const [neighbouringFrames, setNeighbouringFrames] = useState([]);
+ const [neighbouringKeyFramesMap, setNeighbouringKeyFrames] = useState({});
+ const neighbouringKeyFrames = useMemo(() => Object.values(neighbouringKeyFramesMap), [neighbouringKeyFramesMap]);
- const findNearestKeyFrameTime = useCallback(({ time, direction }) => ffmpegFindNearestKeyFrameTime({ frames: neighbouringFrames, time, direction, fps: detectedFps }), [neighbouringFrames, detectedFps]);
+ const findNearestKeyFrameTime = useCallback(({ time, direction }) => ffmpegFindNearestKeyFrameTime({ frames: neighbouringKeyFrames, time, direction, fps: detectedFps }), [neighbouringKeyFrames, detectedFps]);
- useEffect(() => {
- setNeighbouringFrames([]);
- }, [filePath]);
+ useEffect(() => setNeighbouringKeyFrames({}), [filePath]);
useDebounceOld(() => {
- // We still want to calculate keyframes even if not shouldShowKeyframes because maybe we want to be able to step to the closest keyframe
- const shouldRun = () => keyframesEnabled && filePath && mainVideoStream && commandedTime != null;
+ let aborted = false;
- async function run() {
- if (!shouldRun() || readingKeyframesPromise.current) return;
+ (async () => {
+ // See getIntervalAroundTime
+ // We still want to calculate keyframes even if not shouldShowKeyframes because maybe we want to be able to step to the closest keyframe
+ const shouldRun = keyframesEnabled && filePath && mainVideoStream && commandedTime != null && !readingKeyframesPromise.current;
+ if (!shouldRun) return;
try {
const promise = readFrames({ filePath, aroundTime: commandedTime, stream: mainVideoStream.index, window: ffmpegExtractWindow });
readingKeyframesPromise.current = promise;
const newFrames = await promise;
- if (!shouldRun()) return;
+ if (aborted) return;
+ const newKeyFrames = newFrames.filter((frame) => frame.keyframe);
// console.log(newFrames);
- setNeighbouringFrames(newFrames);
+ setNeighbouringKeyFrames((existingKeyFramesMap) => {
+ let existingFrames = Object.values(existingKeyFramesMap);
+ if (existingFrames.length >= maxKeyframes) {
+ existingFrames = sortBy(existingFrames, 'createdAt').slice(newKeyFrames.length);
+ }
+ const toObj = (map) => Object.fromEntries(map.map((frame) => [frame.time, frame]));
+ return {
+ ...toObj(existingFrames),
+ ...toObj(newKeyFrames),
+ };
+ });
} catch (err) {
console.error('Failed to read keyframes', err);
} finally {
readingKeyframesPromise.current = undefined;
}
- }
- run();
- }, 500, [keyframesEnabled, filePath, commandedTime, mainVideoStream]);
+ })();
+
+ return () => {
+ aborted = true;
+ };
+ }, 500, [keyframesEnabled, filePath, commandedTime, mainVideoStream, ffmpegExtractWindow]);
return {
- neighbouringFrames, findNearestKeyFrameTime,
+ neighbouringKeyFrames, findNearestKeyFrameTime,
};
};
diff --git a/src/hooks/useWaveform.js b/src/hooks/useWaveform.js
index c10fc4ce..565a35d8 100644
--- a/src/hooks/useWaveform.js
+++ b/src/hooks/useWaveform.js
@@ -1,40 +1,61 @@
import { useState, useRef, useEffect } from 'react';
+import sortBy from 'lodash/sortBy';
import useDebounceOld from 'react-use/lib/useDebounce'; // Want to phase out this
import { waveformColor } from '../colors';
import { renderWaveformPng } from '../ffmpeg';
+const maxWaveforms = 100;
+// const maxWaveforms = 3; // testing
+
export default ({ filePath, commandedTime, zoomedDuration, waveformEnabled, mainAudioStream, shouldShowWaveform, ffmpegExtractWindow }) => {
const creatingWaveformPromise = useRef();
- const [waveform, setWaveform] = useState();
-
- useEffect(() => {
- setWaveform();
- }, [filePath]);
+ const [waveforms, setWaveforms] = useState([]);
useDebounceOld(() => {
- const shouldRun = () => filePath && mainAudioStream && commandedTime != null && shouldShowWaveform && waveformEnabled;
+ let aborted = false;
+
+ (async () => {
+ const alreadyHaveWaveformAtCommandedTime = waveforms.some((waveform) => waveform.from < commandedTime && waveform.to > commandedTime);
+ const shouldRun = filePath && mainAudioStream && commandedTime != null && shouldShowWaveform && waveformEnabled && !alreadyHaveWaveformAtCommandedTime && !creatingWaveformPromise.current;
+ if (!shouldRun) return;
- async function run() {
- if (!shouldRun() || creatingWaveformPromise.current) return;
try {
const promise = renderWaveformPng({ filePath, aroundTime: commandedTime, window: ffmpegExtractWindow, color: waveformColor });
creatingWaveformPromise.current = promise;
- if (!shouldRun()) return;
- const wf = await promise;
- setWaveform(wf);
+ const newWaveform = await promise;
+ if (aborted) return;
+ setWaveforms((currentWaveforms) => {
+ const waveformsByCreatedAt = sortBy(currentWaveforms, 'createdAt');
+ return [
+ // cleanup old
+ ...(currentWaveforms.length >= maxWaveforms ? waveformsByCreatedAt.slice(1) : waveformsByCreatedAt),
+ newWaveform,
+ ];
+ });
} catch (err) {
console.error('Failed to render waveform', err);
} finally {
creatingWaveformPromise.current = undefined;
}
- }
+ })();
+
+ return () => {
+ aborted = true;
+ };
+ }, 500, [filePath, commandedTime, zoomedDuration, waveformEnabled, mainAudioStream, shouldShowWaveform, waveforms, ffmpegExtractWindow]);
- run();
- }, 500, [filePath, commandedTime, zoomedDuration, waveformEnabled, mainAudioStream, shouldShowWaveform]);
+ const lastWaveformsRef = useRef([]);
+ useEffect(() => {
+ const removedWaveforms = lastWaveformsRef.current.filter((wf) => !waveforms.includes(wf));
+ // Cleanup old
+ // if (removedWaveforms.length > 0) console.log('cleanup waveforms', removedWaveforms.length);
+ removedWaveforms.forEach((waveform) => URL.revokeObjectURL(waveform.url));
+ lastWaveformsRef.current = waveforms;
+ }, [waveforms]);
- // Cleanup old
- useEffect(() => () => waveform && URL.revokeObjectURL(waveform.url), [waveform]);
+ useEffect(() => setWaveforms([]), [filePath]);
+ useEffect(() => () => setWaveforms([]), []);
- return { waveform };
+ return { waveforms };
};