The swiss army knife of lossless video/audio editing
implement streaming detection #2325
Mikael Finstad250218f85fe4
diff --git a/src/main/ffmpeg.ts b/src/main/ffmpeg.tsindex 730dc9fd..8d87fe5b 100644--- a/src/main/ffmpeg.ts+++ b/src/main/ffmpeg.ts@@ -254,56 +254,61 @@ export function mapTimesToSegments(times: number[], includeLast: boolean) {return segments;}-function adjustSegmentsWithOffset({ segments, from }: { segments: { start: number, end: number | undefined }[], from?: number | undefined }) {- const offset = from ?? 0;- return segments.map(({ start, end }) => ({ start: start + offset, end: end != null ? end + offset : end }));+interface DetectedSegment {+ start: number,+ end: number,}// https://stackoverflow.com/questions/35675529/using-ffmpeg-how-to-do-a-scene-change-detection-with-timecode-export async function detectSceneChanges({ filePath, minChange, onProgress, from, to }: {+export async function detectSceneChanges({ filePath, minChange, onProgress, onSegmentDetected, from, to }: {filePath: string,minChange: number | string,onProgress: (p: number) => void,+ onSegmentDetected: (p: DetectedSegment) => void,from: number,to: number,}) {const args = ['-hide_banner',...getInputSeekArgs({ filePath, from, to }),- '-filter_complex', `select='gt(scene,${minChange})',metadata=print:file=-`,+ '-filter_complex', `select='gt(scene,${minChange})',metadata=print:file=-:direct=1`, // direct=1 to flush stdout immediately'-f', 'null', '-',];const process = runFfmpegProcess(args, { buffer: false });- const times = [0];-handleProgress(process, to - from, onProgress);+assert(process.stdout != null);const rl = readline.createInterface({ input: process.stdout });++ let lastTime: number | undefined;+rl.on('line', (line) => {// eslint-disable-next-line unicorn/better-regexconst match = line.match(/^frame:\d+\s+pts:\d+\s+pts_time:([\d.]+)/);if (!match) return;const time = parseFloat(match[1]!);- // @ts-expect-error todo- if (Number.isNaN(time) || time <= times.at(-1)) return;- times.push(time);+ if (!Number.isNaN(time)) {+ if (lastTime != null && time > lastTime) {+ onSegmentDetected({ start: from + lastTime, end: from + time });+ }+ lastTime = time;+ }});await process;- const segments = mapTimesToSegments(times, false);-- return { detectedSegments: adjustSegmentsWithOffset({ segments, from }), ffmpegArgs: args };+ return { ffmpegArgs: args };}-async function detectIntervals({ filePath, customArgs, onProgress, from, to, matchLineTokens, boundingMode }: {+async function detectIntervals({ filePath, customArgs, onProgress, onSegmentDetected, from, to, matchLineTokens, boundingMode }: {filePath: string,customArgs: string[],onProgress: (p: number) => void,+ onSegmentDetected: (p: DetectedSegment) => void,from: number,to: number,- matchLineTokens: (line: string) => { start: number, end: number } | undefined,+ matchLineTokens: (line: string) => DetectedSegment | undefined,boundingMode: boolean,}) {const args = [@@ -314,8 +319,7 @@ async function detectIntervals({ filePath, customArgs, onProgress, from, to, mat];const process = runFfmpegProcess(args, { buffer: false });- let segments: { start: number, end: number }[] = [];- const midpoints: number[] = [];+ let lastMidpoint: number | undefined;function customMatcher(line: string) {const match = matchLineTokens(line);@@ -323,43 +327,44 @@ async function detectIntervals({ filePath, customArgs, onProgress, from, to, matconst { start, end } = match;if (boundingMode) {- segments.push({ start, end });+ onSegmentDetected({ start: from + start, end: from + end });} else {- midpoints.push(start + ((end - start) / 2));+ const midpoint = start + ((end - start) / 2);++ onSegmentDetected({ start: from + (lastMidpoint ?? 0), end: from + midpoint });+ lastMidpoint = midpoint;}}+handleProgress(process, to - from, onProgress, customMatcher);await process;- if (!boundingMode) {- segments = midpoints.flatMap((time, i) => [- {- start: midpoints[i - 1] ?? 0,- end: time,- },- ]);-- const lastMidpoint = midpoints.at(-1);- if (lastMidpoint != null) {- segments.push({- start: lastMidpoint,- end: to - from,- });- }+ if (!boundingMode && lastMidpoint != null) {+ onSegmentDetected({+ start: from + lastMidpoint,+ end: to,+ });}- return { detectedSegments: adjustSegmentsWithOffset({ segments, from }), ffmpegArgs: args };+ return { ffmpegArgs: args };}const mapFilterOptions = (options: Record<string, string>) => Object.entries(options).map(([key, value]) => `${key}=${value}`).join(':');-export async function blackDetect({ filePath, filterOptions, boundingMode, onProgress, from, to }: {- filePath: string, filterOptions: Record<string, string>, boundingMode: boolean, onProgress: (p: number) => void, from: number, to: number,+export async function blackDetect({ filePath, filterOptions, boundingMode, onProgress, onSegmentDetected, from, to }: {+ filePath: string,+ filterOptions: Record<string, string>,+ boundingMode: boolean,+ onProgress: (p: number) => void,+ onSegmentDetected: (p: DetectedSegment) => void,+ from: number,+ to: number,}) {return detectIntervals({filePath,onProgress,+ onSegmentDetected,from,to,boundingMode,@@ -383,12 +388,18 @@ export async function blackDetect({ filePath, filterOptions, boundingMode, onPro});}-export async function silenceDetect({ filePath, filterOptions, boundingMode, onProgress, from, to }: {- filePath: string, filterOptions: Record<string, string>, boundingMode: boolean, onProgress: (p: number) => void, from: number, to: number,+export async function silenceDetect({ filePath, filterOptions, boundingMode, onProgress, onSegmentDetected, from, to }: {+ filePath: string,+ filterOptions: Record<string, string>,+ boundingMode: boolean,+ onProgress: (p: number) => void,+ onSegmentDetected: (p: DetectedSegment) => void,+ from: number, to: number,}) {return detectIntervals({filePath,onProgress,+ onSegmentDetected,from,to,boundingMode,diff --git a/src/renderer/src/hooks/useSegments.ts b/src/renderer/src/hooks/useSegments.tsindex e85f598a..1f4860dd 100644--- a/src/renderer/src/hooks/useSegments.ts+++ b/src/renderer/src/hooks/useSegments.ts@@ -10,7 +10,7 @@ import { handleError, shuffleArray } from '../util';import { errorToast } from '../swal';import { showParametersDialog } from '../dialogs/parameters';import { createNumSegments as createNumSegmentsDialog, createFixedByteSixedSegments as createFixedByteSixedSegmentsDialog, createRandomSegments as createRandomSegmentsDialog, labelSegmentDialog, askForShiftSegments, askForAlignSegments, selectSegmentsByLabelDialog, selectSegmentsByExprDialog, mutateSegmentsByExprDialog, askForSegmentDuration } from '../dialogs';-import { createSegment, sortSegments, invertSegments, combineOverlappingSegments as combineOverlappingSegments2, combineSelectedSegments as combineSelectedSegments2, isDurationValid, addSegmentColorIndex, filterNonMarkers, makeDurationSegments } from '../segments';+import { createSegment, sortSegments, invertSegments, combineOverlappingSegments as combineOverlappingSegments2, combineSelectedSegments as combineSelectedSegments2, isDurationValid, addSegmentColorIndex, filterNonMarkers, makeDurationSegments, isInitialSegment } from '../segments';import { parameters as allFfmpegParameters, FfmpegDialog } from '../ffmpegParameters';import { maxSegmentsAllowed } from '../util/constants';import { ParseTimecode, SegmentBase, segmentTagsSchema, SegmentToExport, StateSegment, UpdateSegAtIndex } from '../types';@@ -100,7 +100,7 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStreaif (validEdl.length > maxSegmentsAllowed) throw new Error(i18n.t('Tried to create too many segments (max {{maxSegmentsAllowed}}.)', { maxSegmentsAllowed }));setCutSegments((existingSegments) => {- const needToAppend = append && existingSegments.length > 1;+ const needToAppend = append && !(existingSegments.length === 1 && isInitialSegment(existingSegments[0]!));let newSegments = validEdl.map((segment, i) => createIndexedSegment({ segment, incrementCount: needToAppend || i > 0 }));if (needToAppend) newSegments = [...existingSegments, ...newSegments];return newSegments;@@ -111,7 +111,7 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStreaname: string,workingText: string,errorText: string,- fn: () => Promise<{ detectedSegments: SegmentBase[], ffmpegArgs: string[] }>,+ fn: (onSegmentDetected: (seg: SegmentBase) => void) => Promise<{ ffmpegArgs: string[] }>,}) => {if (!filePath) return;if (workingRef.current) return;@@ -119,11 +119,12 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStreasetWorking({ text: workingText });setProgress(0);- const { detectedSegments, ffmpegArgs } = await fn();+ // todo throttle?+ const { ffmpegArgs } = await fn((detectedSegment) => {+ console.log('Detected', name, detectedSegment);+ loadCutSegments([detectedSegment], true);+ });appendFfmpegCommandLog(ffmpegArgs);-- console.log(name, detectedSegments);- loadCutSegments(detectedSegments, true);} catch (err) {if (!(err instanceof Error && err.name === 'AbortError')) handleError(errorText, err);} finally {@@ -164,7 +165,7 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStreasetFfmpegParametersForDialog(dialogType, parameters);invariant(mode === '1' || mode === '2');invariant(filePath != null);- await detectSegments({ name: 'blackScenes', workingText: i18n.t('Detecting black scenes'), errorText: i18n.t('Failed to detect black scenes'), fn: async () => blackDetect({ filePath, filterOptions, boundingMode: mode === '1', onProgress: setProgress, from: start, to: end }) });+ await detectSegments({ name: 'blackScenes', workingText: i18n.t('Detecting black scenes'), errorText: i18n.t('Failed to detect black scenes'), fn: async (onSegmentDetected) => blackDetect({ filePath, filterOptions, boundingMode: mode === '1', onProgress: setProgress, onSegmentDetected, from: start, to: end }) });}, [durationSafe, currentCutSeg, getFfmpegParameters, setFfmpegParametersForDialog, filePath, detectSegments, setProgress]);const detectSilentScenes = useCallback(async () => {@@ -176,7 +177,7 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStreaconst { mode, ...filterOptions } = parameters;invariant(mode === '1' || mode === '2');invariant(filePath != null);- await detectSegments({ name: 'silentScenes', workingText: i18n.t('Detecting silent scenes'), errorText: i18n.t('Failed to detect silent scenes'), fn: async () => silenceDetect({ filePath, filterOptions, boundingMode: mode === '1', onProgress: setProgress, from: start, to: end }) });+ await detectSegments({ name: 'silentScenes', workingText: i18n.t('Detecting silent scenes'), errorText: i18n.t('Failed to detect silent scenes'), fn: async (onSegmentDetected) => silenceDetect({ filePath, filterOptions, boundingMode: mode === '1', onProgress: setProgress, onSegmentDetected, from: start, to: end }) });}, [currentCutSeg, detectSegments, durationSafe, filePath, getFfmpegParameters, setFfmpegParametersForDialog, setProgress]);const detectSceneChanges = useCallback(async () => {@@ -189,7 +190,7 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStrea// eslint-disable-next-line prefer-destructuringconst minChange = parameters['minChange'];invariant(minChange != null);- await detectSegments({ name: 'sceneChanges', workingText: i18n.t('Detecting scene changes'), errorText: i18n.t('Failed to detect scene changes'), fn: async () => ffmpegDetectSceneChanges({ filePath, minChange, onProgress: setProgress, from: start, to: end }) });+ await detectSegments({ name: 'sceneChanges', workingText: i18n.t('Detecting scene changes'), errorText: i18n.t('Failed to detect scene changes'), fn: async (onSegmentDetected) => ffmpegDetectSceneChanges({ filePath, minChange, onProgress: setProgress, onSegmentDetected, from: start, to: end }) });}, [currentCutSeg, detectSegments, durationSafe, filePath, getFfmpegParameters, setFfmpegParametersForDialog, setProgress]);const createSegmentsFromKeyframes = useCallback(async () => {@@ -203,7 +204,7 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStreaconst removeSegments = useCallback((removeSegmentIds: string[]) => {setCutSegments((existingSegments) => {- if (existingSegments.length === 1 && existingSegments[0]!.start == null && existingSegments[0]!.end == null) {+ if (existingSegments.length === 1 && isInitialSegment(existingSegments[0]!)) {return existingSegments; // We are at initial segment, nothing more we can do (it cannot be removed)}@@ -494,7 +495,7 @@ function useSegments({ filePath, workingRef, setWorking, setProgress, videoStreaconst findSegmentsAtCursor = useCallback((currentTime: number) => (cutSegments.flatMap((segment, index) => {if (- (segment.start == null || segment.start <= currentTime)+ (segment.start <= currentTime)&& (segment.end != null && segment.end >= currentTime)) {return [index];diff --git a/src/renderer/src/segments.ts b/src/renderer/src/segments.tsindex a7e34f55..b2dde5c1 100644--- a/src/renderer/src/segments.ts+++ b/src/renderer/src/segments.ts@@ -67,7 +67,6 @@ export function partitionIntoOverlappingRanges<T extends SegmentBase>(array: T[]const item = array[i]!;const { start } = item;const prevStart = array[i - 1]!.start;- if (start == null || prevStart == null) throw new Error();if (start >= prevStart && start < getMaxEnd(ret[g]!)) {ret[g]!.push(item);} else {@@ -142,7 +141,7 @@ export function invertSegments(sortedCutSegments: ({ start: number, end: number,if (includeFirstSegment) {const firstSeg = sortedCutSegments[0]!;- if (firstSeg.start != null && firstSeg.start > 0) {+ if (firstSeg.start > 0) {ret.push({start: 0,end: firstSeg.start,@@ -176,7 +175,7 @@ export function invertSegments(sortedCutSegments: ({ start: number, end: number,// Filter out zero length resulting segments// https://github.com/mifi/lossless-cut/issues/909- return ret.filter(({ start, end }) => end == null || start == null || end > start);+ return ret.filter(({ start, end }) => end == null || end > start);}// because chapters need to be contiguous, we need to insert gaps in-between@@ -184,9 +183,9 @@ export function convertSegmentsToChapters(sortedSegments: { start: number, end:if (sortedSegments.length === 0) return [];if (hasAnySegmentOverlap(sortedSegments)) throw new Error('Segments cannot overlap');- const invertedSegments = invertSegments(sortedSegments, true, false).map(({ start, end, ...seg }) => {- invariant(start != null && end != null); // to please typescript- return { ...seg, start, end };+ const invertedSegments = invertSegments(sortedSegments, true, false).map(({ end, ...seg }) => {+ invariant(end != null); // to please typescript+ return { ...seg, end };});// inverted segments will be "gap" segments. Merge together with normal segments@@ -262,3 +261,5 @@ export function makeDurationSegments(segmentDuration: number, fileDuration: numb}return edl;}++export const isInitialSegment = (seg: SegmentBase) => seg.start === 0 && seg.end == null;