Skip to content

The swiss army knife of lossless video/audio editing

implement streaming detection #2325

Mikael Finstad250218f85fe4
diff --git a/src/main/ffmpeg.ts b/src/main/ffmpeg.ts
index 730dc9fd..8d87fe5b 100644
--- a/src/main/ffmpeg.ts
+++ b/src/main/ffmpeg.ts
@@ -254,56 +254,61 @@ export function mapTimesToSegments(times: number[], includeLast: boolean) {
return segments;
}
-function adjustSegmentsWithOffset({ segments, from }: { segments: { start: number, end: number | undefined }[], from?: number | undefined }) {
- const offset = from ?? 0;
- return segments.map(({ start, end }) => ({ start: start + offset, end: end != null ? end + offset : end }));
+interface DetectedSegment {
+ start: number,
+ end: number,
}
// https://stackoverflow.com/questions/35675529/using-ffmpeg-how-to-do-a-scene-change-detection-with-timecode
-export async function detectSceneChanges({ filePath, minChange, onProgress, from, to }: {
+export async function detectSceneChanges({ filePath, minChange, onProgress, onSegmentDetected, from, to }: {
filePath: string,
minChange: number | string,
onProgress: (p: number) => void,
+ onSegmentDetected: (p: DetectedSegment) => void,
from: number,
to: number,
}) {
const args = [
'-hide_banner',
...getInputSeekArgs({ filePath, from, to }),
- '-filter_complex', `select='gt(scene,${minChange})',metadata=print:file=-`,
+ '-filter_complex', `select='gt(scene,${minChange})',metadata=print:file=-:direct=1`, // direct=1 to flush stdout immediately
'-f', 'null', '-',
];
const process = runFfmpegProcess(args, { buffer: false });
- const times = [0];
-
handleProgress(process, to - from, onProgress);
+
assert(process.stdout != null);
const rl = readline.createInterface({ input: process.stdout });
+
+ let lastTime: number | undefined;
+
rl.on('line', (line) => {
// eslint-disable-next-line unicorn/better-regex
const match = line.match(/^frame:\d+\s+pts:\d+\s+pts_time:([\d.]+)/);
if (!match) return;
const time = parseFloat(match[1]!);
- // @ts-expect-error todo
- if (Number.isNaN(time) || time <= times.at(-1)) return;
- times.push(time);
+ if (!Number.isNaN(time)) {
+ if (lastTime != null && time > lastTime) {
+ onSegmentDetected({ start: from + lastTime, end: from + time });
+ }
+ lastTime = time;
+ }
});
await process;
- const segments = mapTimesToSegments(times, false);
-
- return { detectedSegments: adjustSegmentsWithOffset({ segments, from }), ffmpegArgs: args };
+ return { ffmpegArgs: args };
}
-async function detectIntervals({ filePath, customArgs, onProgress, from, to, matchLineTokens, boundingMode }: {
+async function detectIntervals({ filePath, customArgs, onProgress, onSegmentDetected, from, to, matchLineTokens, boundingMode }: {
filePath: string,
customArgs: string[],
onProgress: (p: number) => void,
+ onSegmentDetected: (p: DetectedSegment) => void,
from: number,
to: number,
- matchLineTokens: (line: string) => { start: number, end: number } | undefined,
+ matchLineTokens: (line: string) => DetectedSegment | undefined,
boundingMode: boolean,
}) {
const args = [
@@ -314,8 +319,7 @@ async function detectIntervals({ filePath, customArgs, onProgress, from, to, mat
];
const process = runFfmpegProcess(args, { buffer: false });
- let segments: { start: number, end: number }[] = [];
- const midpoints: number[] = [];
+ let lastMidpoint: number | undefined;
function customMatcher(line: string) {
const match = matchLineTokens(line);
@@ -323,43 +327,44 @@ async function detectIntervals({ filePath, customArgs, onProgress, from, to, mat
const { start, end } = match;
if (boundingMode) {
- segments.push({ start, end });
+ onSegmentDetected({ start: from + start, end: from + end });
} else {
- midpoints.push(start + ((end - start) / 2));
+ const midpoint = start + ((end - start) / 2);
+
+ onSegmentDetected({ start: from + (lastMidpoint ?? 0), end: from + midpoint });
+ lastMidpoint = midpoint;
}
}
+
handleProgress(process, to - from, onProgress, customMatcher);
await process;
- if (!boundingMode) {
- segments = midpoints.flatMap((time, i) => [
- {
- start: midpoints[i - 1] ?? 0,
- end: time,
- },
- ]);
-
- const lastMidpoint = midpoints.at(-1);
- if (lastMidpoint != null) {
- segments.push({
- start: lastMidpoint,
- end: to - from,
- });
- }
+ if (!boundingMode && lastMidpoint != null) {
+ onSegmentDetected({
+ start: from + lastMidpoint,
+ end: to,
+ });
}
- return { detectedSegments: adjustSegmentsWithOffset({ segments, from }), ffmpegArgs: args };
+ return { ffmpegArgs: args };
}
const mapFilterOptions = (options: Record<string, string>) => Object.entries(options).map(([key, value]) => `${key}=${value}`).join(':');
-export async function blackDetect({ filePath, filterOptions, boundingMode, onProgress, from, to }: {
- filePath: string, filterOptions: Record<string, string>, boundingMode: boolean, onProgress: (p: number) => void, from: number, to: number,
+export async function blackDetect({ filePath, filterOptions, boundingMode, onProgress, onSegmentDetected, from, to }: {
+ filePath: string,
+ filterOptions: Record<string, string>,
+ boundingMode: boolean,
+ onProgress: (p: number) => void,
+ onSegmentDetected: (p: DetectedSegment) => void,
+ from: number,
+ to: number,
}) {
return detectIntervals({
filePath,
onProgress,
+ onSegmentDetected,
from,
to,
boundingMode,
@@ -383,12 +388,18 @@ export async function blackDetect({ filePath, filterOptions, boundingMode, onPro
});
}
-export async function silenceDetect({ filePath, filterOptions, boundingMode, onProgress, from, to }: {
- filePath: string, filterOptions: Record<string, string>, boundingMode: boolean, onProgress: (p: number) => void, from: number, to: number,
+export async function silenceDetect({ filePath, filterOptions, boundingMode, onProgress, onSegmentDetected, from, to }: {
+ filePath: string,
+ filterOptions: Record<string, string>,
+ boundingMode: boolean,
+ onProgress: (p: number) => void,
+ onSegmentDetected: (p: DetectedSegment) => void,
+ from: number, to: number,
}) {
return detectIntervals({
filePath,
onProgress,
+ onSegmentDetected,
from,
to,
boundingMode,
diff --git a/src/renderer/src/hooks/useSegments.ts b/src/renderer/src/hooks/useSegments.ts
index e85f598a..1f4860dd 100644
--- a/src/renderer/src/hooks/useSegments.ts
+++ b/src/renderer/src/hooks/useSegments.ts
@@ -10,7 +10,7 @@ import { handleError, shuffleArray } from '../util';
import { errorToast } from '../swal';
import { showParametersDialog } from '../dialogs/parameters';
import { createNumSegments as createNumSegmentsDialog, createFixedByteSixedSegments as createFixedByteSixedSegmentsDialog, createRandomSegments as createRandomSegmentsDialog, labelSegmentDialog, askForShiftSegments, askForAlignSegments, selectSegmentsByLabelDialog, selectSegmentsByExprDialog, mutateSegmentsByExprDialog, askForSegmentDuration } from '../dialogs';
-import { createSegment, sortSegments, invertSegments, combineOverlappingSegments as combineOverlappingSegments2, combineSelectedSegments as combineSelectedSegments2, isDurationValid, addSegmentColorIndex, filterNonMarkers, makeDurationSegments } from '../segments';
+import { createSegment, sortSegments, invertSegments, combineOverlappingSegments as combineOverlappingSegments2, combineSelectedSegments as combineSelectedSegments2, isDurationValid, addSegmentColorIndex, filterNonMarkers, makeDurationSegments, isInitialSegment } from '../segments';
import { parameters as allFfmpegParameters, FfmpegDialog } from '../ffmpegParameters';
import { maxSegmentsAllowed } from '../util/constants';
import { ParseTimecode, SegmentBase, segmentTagsSchema, SegmentToExport, StateSegment, UpdateSegAtIndex } from '../types';
@@ -100,7 +100,7 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStrea
if (validEdl.length > maxSegmentsAllowed) throw new Error(i18n.t('Tried to create too many segments (max {{maxSegmentsAllowed}}.)', { maxSegmentsAllowed }));
setCutSegments((existingSegments) => {
- const needToAppend = append && existingSegments.length > 1;
+ const needToAppend = append && !(existingSegments.length === 1 && isInitialSegment(existingSegments[0]!));
let newSegments = validEdl.map((segment, i) => createIndexedSegment({ segment, incrementCount: needToAppend || i > 0 }));
if (needToAppend) newSegments = [...existingSegments, ...newSegments];
return newSegments;
@@ -111,7 +111,7 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStrea
name: string,
workingText: string,
errorText: string,
- fn: () => Promise<{ detectedSegments: SegmentBase[], ffmpegArgs: string[] }>,
+ fn: (onSegmentDetected: (seg: SegmentBase) => void) => Promise<{ ffmpegArgs: string[] }>,
}) => {
if (!filePath) return;
if (workingRef.current) return;
@@ -119,11 +119,12 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStrea
setWorking({ text: workingText });
setProgress(0);
- const { detectedSegments, ffmpegArgs } = await fn();
+ // todo throttle?
+ const { ffmpegArgs } = await fn((detectedSegment) => {
+ console.log('Detected', name, detectedSegment);
+ loadCutSegments([detectedSegment], true);
+ });
appendFfmpegCommandLog(ffmpegArgs);
-
- console.log(name, detectedSegments);
- loadCutSegments(detectedSegments, true);
} catch (err) {
if (!(err instanceof Error && err.name === 'AbortError')) handleError(errorText, err);
} finally {
@@ -164,7 +165,7 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStrea
setFfmpegParametersForDialog(dialogType, parameters);
invariant(mode === '1' || mode === '2');
invariant(filePath != null);
- await detectSegments({ name: 'blackScenes', workingText: i18n.t('Detecting black scenes'), errorText: i18n.t('Failed to detect black scenes'), fn: async () => blackDetect({ filePath, filterOptions, boundingMode: mode === '1', onProgress: setProgress, from: start, to: end }) });
+ await detectSegments({ name: 'blackScenes', workingText: i18n.t('Detecting black scenes'), errorText: i18n.t('Failed to detect black scenes'), fn: async (onSegmentDetected) => blackDetect({ filePath, filterOptions, boundingMode: mode === '1', onProgress: setProgress, onSegmentDetected, from: start, to: end }) });
}, [durationSafe, currentCutSeg, getFfmpegParameters, setFfmpegParametersForDialog, filePath, detectSegments, setProgress]);
const detectSilentScenes = useCallback(async () => {
@@ -176,7 +177,7 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStrea
const { mode, ...filterOptions } = parameters;
invariant(mode === '1' || mode === '2');
invariant(filePath != null);
- await detectSegments({ name: 'silentScenes', workingText: i18n.t('Detecting silent scenes'), errorText: i18n.t('Failed to detect silent scenes'), fn: async () => silenceDetect({ filePath, filterOptions, boundingMode: mode === '1', onProgress: setProgress, from: start, to: end }) });
+ await detectSegments({ name: 'silentScenes', workingText: i18n.t('Detecting silent scenes'), errorText: i18n.t('Failed to detect silent scenes'), fn: async (onSegmentDetected) => silenceDetect({ filePath, filterOptions, boundingMode: mode === '1', onProgress: setProgress, onSegmentDetected, from: start, to: end }) });
}, [currentCutSeg, detectSegments, durationSafe, filePath, getFfmpegParameters, setFfmpegParametersForDialog, setProgress]);
const detectSceneChanges = useCallback(async () => {
@@ -189,7 +190,7 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStrea
// eslint-disable-next-line prefer-destructuring
const minChange = parameters['minChange'];
invariant(minChange != null);
- await detectSegments({ name: 'sceneChanges', workingText: i18n.t('Detecting scene changes'), errorText: i18n.t('Failed to detect scene changes'), fn: async () => ffmpegDetectSceneChanges({ filePath, minChange, onProgress: setProgress, from: start, to: end }) });
+ await detectSegments({ name: 'sceneChanges', workingText: i18n.t('Detecting scene changes'), errorText: i18n.t('Failed to detect scene changes'), fn: async (onSegmentDetected) => ffmpegDetectSceneChanges({ filePath, minChange, onProgress: setProgress, onSegmentDetected, from: start, to: end }) });
}, [currentCutSeg, detectSegments, durationSafe, filePath, getFfmpegParameters, setFfmpegParametersForDialog, setProgress]);
const createSegmentsFromKeyframes = useCallback(async () => {
@@ -203,7 +204,7 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStrea
const removeSegments = useCallback((removeSegmentIds: string[]) => {
setCutSegments((existingSegments) => {
- if (existingSegments.length === 1 && existingSegments[0]!.start == null && existingSegments[0]!.end == null) {
+ if (existingSegments.length === 1 && isInitialSegment(existingSegments[0]!)) {
return existingSegments; // We are at initial segment, nothing more we can do (it cannot be removed)
}
@@ -494,7 +495,7 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStrea
const findSegmentsAtCursor = useCallback((currentTime: number) => (
cutSegments.flatMap((segment, index) => {
if (
- (segment.start == null || segment.start <= currentTime)
+ (segment.start <= currentTime)
&& (segment.end != null && segment.end >= currentTime)
) {
return [index];
diff --git a/src/renderer/src/segments.ts b/src/renderer/src/segments.ts
index a7e34f55..b2dde5c1 100644
--- a/src/renderer/src/segments.ts
+++ b/src/renderer/src/segments.ts
@@ -67,7 +67,6 @@ export function partitionIntoOverlappingRanges<T extends SegmentBase>(array: T[]
const item = array[i]!;
const { start } = item;
const prevStart = array[i - 1]!.start;
- if (start == null || prevStart == null) throw new Error();
if (start >= prevStart && start < getMaxEnd(ret[g]!)) {
ret[g]!.push(item);
} else {
@@ -142,7 +141,7 @@ export function invertSegments(sortedCutSegments: ({ start: number, end: number,
if (includeFirstSegment) {
const firstSeg = sortedCutSegments[0]!;
- if (firstSeg.start != null && firstSeg.start > 0) {
+ if (firstSeg.start > 0) {
ret.push({
start: 0,
end: firstSeg.start,
@@ -176,7 +175,7 @@ export function invertSegments(sortedCutSegments: ({ start: number, end: number,
// Filter out zero length resulting segments
// https://github.com/mifi/lossless-cut/issues/909
- return ret.filter(({ start, end }) => end == null || start == null || end > start);
+ return ret.filter(({ start, end }) => end == null || end > start);
}
// because chapters need to be contiguous, we need to insert gaps in-between
@@ -184,9 +183,9 @@ export function convertSegmentsToChapters(sortedSegments: { start: number, end:
if (sortedSegments.length === 0) return [];
if (hasAnySegmentOverlap(sortedSegments)) throw new Error('Segments cannot overlap');
- const invertedSegments = invertSegments(sortedSegments, true, false).map(({ start, end, ...seg }) => {
- invariant(start != null && end != null); // to please typescript
- return { ...seg, start, end };
+ const invertedSegments = invertSegments(sortedSegments, true, false).map(({ end, ...seg }) => {
+ invariant(end != null); // to please typescript
+ return { ...seg, end };
});
// inverted segments will be "gap" segments. Merge together with normal segments
@@ -262,3 +261,5 @@ export function makeDurationSegments(segmentDuration: number, fileDuration: numb
}
return edl;
}
+
+export const isInitialSegment = (seg: SegmentBase) => seg.start === 0 && seg.end == null;