The swiss army knife of lossless video/audio editing
retry EPERM (windows antivirus)
Mikael Finstad847be925e5b9
diff --git a/src/edlStore.js b/src/edlStore.jsindex 7ac6ddd3..6835a28b 100644--- a/src/edlStore.js+++ b/src/edlStore.js@@ -5,39 +5,39 @@ import { parseSrt, formatSrt, parseCuesheet, parseXmeml, parseFcpXml, parseCsv,import { askForYouTubeInput, showOpenDialog } from './dialogs';import { getOutPath } from './util';-const fs = window.require('fs-extra');+const { readFile, writeFile } = window.require('fs/promises');const cueParser = window.require('cue-parser');const { basename } = window.require('path');const { dialog } = window.require('@electron/remote');export async function loadCsvSeconds(path) {- return parseCsv(await fs.readFile(path, 'utf-8'), parseCsvTime);+ return parseCsv(await readFile(path, 'utf-8'), parseCsvTime);}export async function loadCsvFrames(path, fps) {if (!fps) throw new Error('The loaded file has an unknown framerate');- return parseCsv(await fs.readFile(path, 'utf-8'), getFrameValParser(fps));+ return parseCsv(await readFile(path, 'utf-8'), getFrameValParser(fps));}export async function loadXmeml(path) {- return parseXmeml(await fs.readFile(path, 'utf-8'));+ return parseXmeml(await readFile(path, 'utf-8'));}export async function loadFcpXml(path) {- return parseFcpXml(await fs.readFile(path, 'utf-8'));+ return parseFcpXml(await readFile(path, 'utf-8'));}export async function loadDvAnalyzerSummaryTxt(path) {- return parseDvAnalyzerSummaryTxt(await fs.readFile(path, 'utf-8'));+ return parseDvAnalyzerSummaryTxt(await readFile(path, 'utf-8'));}export async function loadPbf(path) {- return parsePbf(await fs.readFile(path));+ return parsePbf(await readFile(path));}export async function loadMplayerEdl(path) {- return parseMplayerEdl(await fs.readFile(path, 'utf-8'));+ return parseMplayerEdl(await readFile(path, 'utf-8'));}export async function loadCue(path) {@@ -45,41 +45,40 @@ export async function loadCue(path) {}export async function loadSrt(path) {- return parseSrt(await fs.readFile(path, 'utf-8'));+ return parseSrt(await readFile(path, 'utf-8'));}export async function saveCsv(path, cutSegments) {- await fs.writeFile(path, await formatCsvSeconds(cutSegments));+ await writeFile(path, await formatCsvSeconds(cutSegments));}export async function saveCsvHuman(path, cutSegments) {- await fs.writeFile(path, await formatCsvHuman(cutSegments));+ await writeFile(path, await formatCsvHuman(cutSegments));}export async function saveCsvFrames({ path, cutSegments, getFrameCount }) {- await fs.writeFile(path, await formatCsvFrames({ cutSegments, getFrameCount }));+ await writeFile(path, await formatCsvFrames({ cutSegments, getFrameCount }));}export async function saveTsv(path, cutSegments) {- await fs.writeFile(path, await formatTsv(cutSegments));+ await writeFile(path, await formatTsv(cutSegments));}export async function saveSrt(path, cutSegments) {- await fs.writeFile(path, await formatSrt(cutSegments));+ await writeFile(path, await formatSrt(cutSegments));}-export async function saveLlcProject({ savePath, filePath, cutSegments }) {const projectData = {version: 1,mediaFileName: basename(filePath),cutSegments: cutSegments.map(({ start, end, name, tags }) => ({ start, end, name, tags })),};- await fs.writeFile(savePath, JSON5.stringify(projectData, null, 2));+ await writeFile(savePath, JSON5.stringify(projectData, null, 2));}export async function loadLlcProject(path) {- return JSON5.parse(await fs.readFile(path));+ return JSON5.parse(await readFile(path));}diff --git a/src/hooks/useFfmpegOperations.js b/src/hooks/useFfmpegOperations.jsindex 51495231..27bf1762 100644--- a/src/hooks/useFfmpegOperations.js+++ b/src/hooks/useFfmpegOperations.js@@ -3,7 +3,7 @@ import flatMap from 'lodash/flatMap';import sum from 'lodash/sum';import pMap from 'p-map';-import { getSuffixedOutPath, transferTimestamps, getOutFileExtension, getOutDir, deleteDispositionValue, getHtml5ifiedPath } from '../util';+import { getSuffixedOutPath, transferTimestamps, getOutFileExtension, getOutDir, deleteDispositionValue, getHtml5ifiedPath, unlinkWithRetry } from '../util';import { isCuttingStart, isCuttingEnd, runFfmpegWithProgress, getFfCommandLine, getDuration, createChaptersFromSegments, readFileMeta, cutEncodeSmartPart, getExperimentalArgs, html5ify as ffmpegHtml5ify, getVideoTimescaleArgs, logStdoutStderr, runFfmpegConcat } from '../ffmpeg';import { getMapStreamsArgs, getStreamIdsToCopy } from '../util/streams';import { getSmartCutParams } from '../smartcut';@@ -11,7 +11,7 @@ import { isDurationValid } from '../segments';const { join, resolve, dirname } = window.require('path');const { pathExists } = window.require('fs-extra');-const { writeFile, unlink, mkdir } = window.require('fs/promises');+const { writeFile, mkdir } = window.require('fs/promises');async function writeChaptersFfmetadata(outDir, chapters) {if (!chapters || chapters.length === 0) return undefined;@@ -52,9 +52,9 @@ function getMatroskaFlags() {const getChaptersInputArgs = (ffmetadataPath) => (ffmetadataPath ? ['-f', 'ffmetadata', '-i', ffmetadataPath] : []);-const tryDeleteFiles = async (paths) => pMap(paths, (path) => {- unlink(path).catch((err) => console.error('Failed to delete', path, err));-}, { concurrency: 5 });+async function tryDeleteFiles(paths) {+ return pMap(paths, (path) => unlinkWithRetry(path).catch((err) => console.error('Failed to delete', path, err)), { concurrency: 5 });+}function useFfmpegOperations({ filePath, treatInputFileModifiedTimeAsStart, treatOutputFileModifiedTimeAsStart, needSmartCut, enableOverwriteOutput, outputPlaybackRate }) {const shouldSkipExistingFile = useCallback(async (path) => {diff --git a/src/hooks/useFrameCapture.js b/src/hooks/useFrameCapture.jsindex 2ed81288..6dde5d89 100644--- a/src/hooks/useFrameCapture.js+++ b/src/hooks/useFrameCapture.js@@ -1,14 +1,13 @@import dataUriToBuffer from 'data-uri-to-buffer';import pMap from 'p-map';-import { getSuffixedOutPath, getOutDir, transferTimestamps, getSuffixedFileName, getOutPath, escapeRegExp } from '../util';+import { getSuffixedOutPath, getOutDir, transferTimestamps, getSuffixedFileName, getOutPath, escapeRegExp, fsOperationWithRetry } from '../util';import { getNumDigits } from '../segments';import { captureFrame as ffmpegCaptureFrame, captureFrames as ffmpegCaptureFrames } from '../ffmpeg';-const fs = window.require('fs-extra');const mime = window.require('mime-types');-const { rename, readdir } = window.require('fs/promises');+const { rename, readdir, writeFile } = window.require('fs/promises');function getFrameFromVideo(video, format, quality) {@@ -63,7 +62,7 @@ export default ({ formatTimecode, treatOutputFileModifiedTimeAsStart }) => {const duration = formatTimecode({ seconds: fromTime + (frameNum / fps), fileNameFriendly: true });const renameFromPath = getOutPath({ customOutDir, filePath, fileName });const renameToPath = getOutPath({ customOutDir, filePath, fileName: getSuffixedFileName(filePath, getSuffix(duration, captureFormat)) });- await rename(renameFromPath, renameToPath);+ await fsOperationWithRetry(async () => rename(renameFromPath, renameToPath));return renameToPath;}, { concurrency: 1 });@@ -87,7 +86,7 @@ export default ({ formatTimecode, treatOutputFileModifiedTimeAsStart }) => {const time = formatTimecode({ seconds: currentTime, fileNameFriendly: true });const outPath = getSuffixedOutPath({ customOutDir, filePath, nameSuffix: `${time}.${ext}` });- await fs.writeFile(outPath, buf);+ await writeFile(outPath, buf);await transferTimestamps({ inPath: filePath, outPath, cutFrom: currentTime, treatOutputFileModifiedTimeAsStart });return outPath;diff --git a/src/util.js b/src/util.jsindex 076643d9..d45f04fd 100644--- a/src/util.js+++ b/src/util.js@@ -11,13 +11,11 @@ import { ffmpegExtractWindow } from './util/constants';const { dirname, parse: parsePath, join, extname, isAbsolute, resolve, basename } = window.require('path');const fsExtra = window.require('fs-extra');-const { stat, readdir } = window.require('fs/promises');+const { stat, readdir, utimes, unlink } = window.require('fs/promises');const os = window.require('os');const { ipcRenderer } = window.require('electron');const remote = window.require('@electron/remote');-const { unlink } = fsExtra;-const trashFile = async (path) => ipcRenderer.invoke('tryTrashItem', path);@@ -95,6 +93,30 @@ export async function dirExists(dirPath) {return (await pathExists(dirPath)) && (await fsExtra.lstat(dirPath)).isDirectory();}+// const testFailFsOperation = isDev;+const testFailFsOperation = false;++// Retry because sometimes write operations fail on windows due to the file being locked for various reasons (often anti-virus) #272 #1797 #1704+export async function fsOperationWithRetry(operation, { signal, retries = 10, minTimeout = 100, maxTimeout = 2000, ...opts }) {+ return pRetry(async () => {+ if (testFailFsOperation && Math.random() > 0.3) throw Object.assign(new Error('test delete failure'), { code: 'EPERM' });+ await operation();+ }, {+ retries,+ signal,+ minTimeout,+ maxTimeout,+ // mimic fs.rm `maxRetries` https://nodejs.org/api/fs.html#fspromisesrmpath-options+ shouldRetry: (err) => err instanceof Error && 'code' in err && ['EBUSY', 'EMFILE', 'ENFILE', 'EPERM'].includes(err.code),+ ...opts,+ });+}++// example error: index-18074aaf.js:166 Failed to delete C:\Users\USERNAME\Desktop\RC\New folder\2023-12-27 21-45-22 (GMT p5)-merged-1703933052361-00.01.04.915-00.01.07.424-seg1.mp4 Error: EPERM: operation not permitted, unlink 'C:\Users\USERNAME\Desktop\RC\New folder\2023-12-27 21-45-22 (GMT p5)-merged-1703933052361-00.01.04.915-00.01.07.424-seg1.mp4'+export const unlinkWithRetry = async (path, options) => fsOperationWithRetry(async () => unlink(path), { ...options, onFailedAttempt: (error) => console.warn('Retrying delete', path, error.attemptNumber) });+// example error: index-18074aaf.js:160 Error: EPERM: operation not permitted, utime 'C:\Users\USERNAME\Desktop\RC\New folder\2023-12-27 21-45-22 (GMT p5)-merged-1703933052361-cut-merged-1703933070237.mp4'+export const utimesWithRetry = async (path, atime, mtime, options) => fsOperationWithRetry(async () => utimes(path, atime, mtime), { ...options, onFailedAttempt: (error) => console.warn('Retrying utimes', path, error.attemptNumber) });+export async function transferTimestamps({ inPath, outPath, cutFrom = 0, cutTo = 0, duration = 0, treatInputFileModifiedTimeAsStart = true, treatOutputFileModifiedTimeAsStart }) {if (treatOutputFileModifiedTimeAsStart == null) return; // null means disabled;@@ -115,7 +137,7 @@ export async function transferTimestamps({ inPath, outPath, cutFrom = 0, cutTo =try {const { atime, mtime } = await stat(inPath);- await fsExtra.utimes(outPath, calculateTime((atime.getTime() / 1000)), calculateTime((mtime.getTime() / 1000)));+ await utimesWithRetry(outPath, calculateTime((atime.getTime() / 1000)), calculateTime((mtime.getTime() / 1000)));} catch (err) {console.error('Failed to set output file modified time', err);}@@ -239,13 +261,10 @@ export function getHtml5ifiedPath(cod, fp, type) {export async function deleteFiles({ paths, deleteIfTrashFails, signal }) {const failedToTrashFiles = [];- // const testFail = isDev;- const testFail = false;-// eslint-disable-next-line no-restricted-syntaxfor (const path of paths) {try {- if (testFail) throw new Error('test trash failure');+ if (testFailFsOperation) throw new Error('test trash failure');// eslint-disable-next-line no-await-in-loopawait trashFile(path);signal.throwIfAborted();@@ -267,19 +286,7 @@ export async function deleteFiles({ paths, deleteIfTrashFails, signal }) {if (!value) return;}- // Retry because sometimes it fails on windows #272 #1797- await pMap(failedToTrashFiles, async (path) => {- await pRetry(async () => {- if (testFail) throw new Error('test delete failure');- await unlink(path);- }, {- retries: 3,- signal,- onFailedAttempt: async () => {- console.warn('Retrying delete', path);- },- });- }, { concurrency: 1 });+ await pMap(failedToTrashFiles, async (path) => unlinkWithRetry(path, { signal }), { concurrency: 5 });}export const deleteDispositionValue = 'llc_disposition_remove';