Skip to content

TagSpaces is an offline, open source, document manager with tagging support

chore: improving the io copy and move functionalities

Ilian Sapundshiev1c3a4dd6ff01
diff --git a/src/renderer/components/dialogs/FileUploadDialog.tsx b/src/renderer/components/dialogs/FileUploadDialog.tsx
index ee57c12d5..6e2f5fbad 100644
--- a/src/renderer/components/dialogs/FileUploadDialog.tsx
+++ b/src/renderer/components/dialogs/FileUploadDialog.tsx
@@ -21,6 +21,7 @@ import { CloseIcon, WarningIcon } from '-/components/CommonIcons';
import DraggablePaper from '-/components/DraggablePaper';
import InfoIcon from '-/components/InfoIcon';
import TsButton from '-/components/TsButton';
+import TsTooltip from '-/components/TsTooltip';
import TsDialogActions from '-/components/dialogs/components/TsDialogActions';
import TsDialogTitle from '-/components/dialogs/components/TsDialogTitle';
import { useCurrentLocationContext } from '-/hooks/useCurrentLocationContext';
@@ -31,9 +32,7 @@ import {
AppDispatch,
getProgress,
} from '-/reducers/app';
-import { uploadAbort } from '-/services/utils-io';
import { Grid, LinearProgress } from '@mui/material';
-import TsTooltip from '-/components/TsTooltip';
import Box from '@mui/material/Box';
import Dialog from '@mui/material/Dialog';
import DialogContent from '@mui/material/DialogContent';
@@ -67,20 +66,26 @@ function FileUploadDialog(props: Props) {
const { uploadMeta, transferMeta } = useFileUploadContext();
const progress = useSelector(getProgress);
- const targetPath = React.useRef<string>(getTargetPath()); // todo ContextProvider
const currentLocation = findLocation();
useEffect(() => {
if (AppConfig.isElectron) {
- window.electronIO.ipcRenderer.on('progress', (fileName, newProgress) => {
+ const handler = (fileName, newProgress) => {
dispatch(AppActions.onUploadProgress(newProgress, undefined));
- });
-
+ };
+ window.electronIO.ipcRenderer.on('progress', handler);
return () => {
- window.electronIO.ipcRenderer.removeAllListeners('progress');
+ // Best-effort cleanup: prefer removeListener so we don't kill listeners
+ // registered by other components on the same channel.
+ const ipc: any = window.electronIO.ipcRenderer;
+ if (typeof ipc.removeListener === 'function') {
+ ipc.removeListener('progress', handler);
+ } else {
+ ipc.removeAllListeners('progress');
+ }
};
}
- }, []);
+ }, [dispatch]);
function LinearProgressWithLabel(prop) {
return (
@@ -109,32 +114,21 @@ function FileUploadDialog(props: Props) {
);
}
- const stopAll = () => {
- if (progress) {
- if (currentLocation?.haveObjectStoreSupport()) {
- const progresses = progress.map((fileProgress) => {
- fileProgress.abort();
- return { ...fileProgress, state: 'finished' };
- });
- dispatch(AppActions.setProgresses(progresses));
- } else {
- return uploadAbort();
- }
- }
- };
-
- let haveProgress = false;
+ // Derived view-model — computed once per render, no ref mutation inside .map.
+ const progressArr = Array.isArray(progress) ? progress : [];
+ // Defensive copy: progress comes from Redux; never sort the original in place.
+ const sortedProgress = React.useMemo(
+ () => [...progressArr].sort((a, b) => ('' + a.path).localeCompare(b.path)),
+ [progressArr],
+ );
+ const firstProgressPath = progressArr[0]?.path
+ ? progressArr[0].path.split('?')[0]
+ : undefined;
+ const targetPath = props.targetPath ?? firstProgressPath;
- function getTargetPath() {
- if (props.targetPath) {
- return props.targetPath;
- }
- const pathProgress = progress.find((fileProgress) => fileProgress.path);
- if (pathProgress) {
- return pathProgress.path;
- }
- return undefined;
- }
+ const haveProgress = sortedProgress.some(
+ (p) => p.progress > -1 && p.progress < 100 && p.state !== 'finished',
+ );
function getTargetURL() {
if (props.targetPath) {
@@ -168,14 +162,31 @@ function FileUploadDialog(props: Props) {
);
}
}
- if (targetPath.current) {
- return targetPath.current;
+ if (targetPath) {
+ return targetPath;
} else if (currentDirectoryPath) {
return currentDirectoryPath;
}
return '/';
}
- const completed = progress?.filter((item) => item.progress === 100);
+
+ // Title counter: prefer the aggregate batch percentage for single-row
+ // batches (the common case after the single-row progress key change), fall
+ // back to the file count for multi-row batches (e.g. multi-dir copies).
+ const titleCounter = (() => {
+ if (!sortedProgress.length) {
+ return '';
+ }
+ if (sortedProgress.length === 1) {
+ const p = sortedProgress[0].progress;
+ if (p < 0) {
+ return '';
+ }
+ return ' (' + p + '%)';
+ }
+ const done = sortedProgress.filter((p) => p.progress === 100).length;
+ return ' (' + done + '/' + sortedProgress.length + ')';
+ })();
return (
<Dialog
@@ -194,10 +205,7 @@ function FileUploadDialog(props: Props) {
dialogTitle={
t(
'core:' + (title && title.length > 0 ? title : 'importDialogTitle'),
- ) +
- (progress
- ? ' (' + completed.length + '/' + progress.length + ')'
- : '')
+ ) + titleCounter
}
closeButtonTestId="closeFileUploadTID"
onClose={onClose}
@@ -211,82 +219,64 @@ function FileUploadDialog(props: Props) {
}}
>
<p>{t('core:moveCopyToPath') + ': ' + getTargetURL()}</p>
- {progress &&
- progress
- .sort((a, b) => ('' + a.path).localeCompare(b.path))
- .map((fileProgress) => {
- const percentage = fileProgress.progress;
- const { path, filePath } = fileProgress;
- targetPath.current = path.split('?')[0];
- let { abort } = fileProgress;
- if (
- percentage > -1 &&
- percentage < 100 &&
- fileProgress.state !== 'finished'
- ) {
- haveProgress = true;
- } /*else {
- abort = undefined;
- }*/
+ {sortedProgress.map((fileProgress) => {
+ const percentage = fileProgress.progress;
+ const { path, filePath, abort } = fileProgress;
+ const rowName = filePath
+ ? filePath
+ : extractFileName(
+ path.split('?')[0],
+ currentLocation?.getDirSeparator(),
+ );
- return (
- <Grid
- key={path}
- container
- sx={{
- justifyContent: 'center',
- alignItems: 'center',
- }}
- >
- <Grid
- size={{ xs: 10 }}
- sx={{
- display: 'flex',
- alignItems: 'center',
- wordBreak: 'break-word',
- }}
+ return (
+ <Grid
+ key={path}
+ container
+ sx={{
+ justifyContent: 'center',
+ alignItems: 'center',
+ }}
+ >
+ <Grid
+ size={{ xs: 10 }}
+ sx={{
+ display: 'flex',
+ alignItems: 'center',
+ wordBreak: 'break-word',
+ }}
+ >
+ {rowName}
+ {percentage === -1 && (
+ <TsTooltip
+ title={
+ abort && typeof abort === 'string'
+ ? abort
+ : t('core:fileExist')
+ }
>
- {filePath
- ? filePath
- : extractFileName(
- targetPath.current,
- currentLocation?.getDirSeparator(),
- )}
- {percentage === -1 && (
- <TsTooltip
- title={
- abort && typeof abort === 'string'
- ? abort
- : t('core:fileExist')
- }
- >
- <WarningIcon color="warning" />
- </TsTooltip>
- )}
- </Grid>
- <Grid size={{ xs: 2 }}>
- {abort &&
- typeof abort === 'function' &&
- percentage !== 100 && (
- <TsButton
- tooltip={t('core:abort')}
- onClick={() => abort()}
- >
- <CloseIcon />
- </TsButton>
- )}
- </Grid>
- <Grid size={{ xs: 12 }}>
- {percentage > -1 && (
- <LinearProgressWithLabel value={percentage} />
- )}
- </Grid>
- </Grid>
- );
- })}
+ <WarningIcon color="warning" />
+ </TsTooltip>
+ )}
+ </Grid>
+ <Grid size={{ xs: 2 }}>
+ {abort && typeof abort === 'function' && percentage !== 100 && (
+ <TsButton tooltip={t('core:abort')} onClick={() => abort()}>
+ <CloseIcon />
+ </TsButton>
+ )}
+ </Grid>
+ <Grid size={{ xs: 12 }}>
+ {percentage > -1 && (
+ <LinearProgressWithLabel value={percentage} />
+ )}
+ </Grid>
+ </Grid>
+ );
+ })}
</DialogContent>
<TsDialogActions>
- {progress && progress.length > 0 && !haveProgress && (
+ {sortedProgress.length > 0 && !haveProgress && (
<>
{transferMeta && AppConfig.isElectron && (
<TsButton
@@ -314,11 +304,6 @@ function FileUploadDialog(props: Props) {
<TsButton data-tid="uploadMinimizeDialogTID" onClick={onClose}>
{t('core:minimize')}
</TsButton>
- {haveProgress && (
- <TsButton data-tid="uploadStopAllTID" onClick={stopAll}>
- {t('core:stopAll')}
- </TsButton>
- )}
</TsDialogActions>
</Dialog>
);
diff --git a/src/renderer/components/dialogs/MoveCopyFilesDialog.tsx b/src/renderer/components/dialogs/MoveCopyFilesDialog.tsx
index 7bbb76601..466c57699 100644
--- a/src/renderer/components/dialogs/MoveCopyFilesDialog.tsx
+++ b/src/renderer/components/dialogs/MoveCopyFilesDialog.tsx
@@ -292,6 +292,13 @@ function MoveCopyFilesDialog(props: Props) {
};
function handleMove() {
+ if (
+ !skipTargetPicker &&
+ (selectedFiles.length > 0 || selectedDirs.length > 0)
+ ) {
+ dispatch(AppActions.resetProgress());
+ openFileUploadDialog(targetDir, 'moveEntriesTitle');
+ }
if (selectedFiles.length > 0) {
const filePaths = selectedFiles;
const movePromise = moveFiles(
@@ -314,10 +321,6 @@ function MoveCopyFilesDialog(props: Props) {
setTargetPath('');
}
if (selectedDirs.length > 0) {
- if (!skipTargetPicker) {
- dispatch(AppActions.resetProgress());
- openFileUploadDialog(undefined, 'moveEntriesTitle');
- }
moveDirs(
getEntriesCount(selectedDirs),
targetPath,
diff --git a/src/renderer/components/dialogs/hooks/FileUploadDialogContextProvider.tsx b/src/renderer/components/dialogs/hooks/FileUploadDialogContextProvider.tsx
index 61c8a4507..720c25ce2 100644
--- a/src/renderer/components/dialogs/hooks/FileUploadDialogContextProvider.tsx
+++ b/src/renderer/components/dialogs/hooks/FileUploadDialogContextProvider.tsx
@@ -51,6 +51,13 @@ export const FileUploadDialogContextProvider = ({
children,
}: FileUploadDialogContextProviderProps) => {
const open = useRef<boolean>(false);
+ // True minimize: the user explicitly hid the dialog mid-batch. We keep the
+ // progress data so the small indicator near the search bar stays visible,
+ // and we do NOT re-open the dialog on subsequent progress dispatches.
+ // Reset on (a) a fresh batch (empty → non-empty transition) or
+ // (b) explicit reopen via the indicator / openFileUploadDialog call.
+ const userMinimized = useRef<boolean>(false);
+ const prevProgressLength = useRef<number>(0);
const targetPath = useRef<string>(undefined);
//const transferMeta = useRef<boolean>(undefined);
const dialogTitle = useRef<string>('importDialogTitle');
@@ -60,31 +67,57 @@ export const FileUploadDialogContextProvider = ({
const [ignored, forceUpdate] = useReducer((x) => x + 1, 0, undefined);
useEffect(() => {
- if (
- !firstRender &&
- !open.current &&
- Array.isArray(progress) &&
- progress.length > 0
- ) {
- openDialog();
+ if (firstRender) {
+ prevProgressLength.current = Array.isArray(progress)
+ ? progress.length
+ : 0;
+ return;
}
+ const len = Array.isArray(progress) ? progress.length : 0;
+ const wasEmpty = prevProgressLength.current === 0;
+ prevProgressLength.current = len;
+
+ if (wasEmpty && len > 0) {
+ // Fresh batch — surface the dialog regardless of previous minimize.
+ userMinimized.current = false;
+ if (!open.current) {
+ open.current = true;
+ forceUpdate();
+ }
+ } else if (len === 0) {
+ // Everything cleared (e.g. user clicked "Close and Clear") — reset.
+ userMinimized.current = false;
+ if (open.current) {
+ open.current = false;
+ forceUpdate();
+ }
+ }
+ // Else: ongoing batch. Don't re-open if the user minimized — the
+ // indicator near the search bar continues to surface progress.
}, [progress]);
function openDialog(tPath?: string, dTitle?: string) {
- //, tMeta?: boolean) {
open.current = true;
+ userMinimized.current = false;
targetPath.current = tPath;
- //transferMeta.current = tMeta;
if (dTitle) {
dialogTitle.current = dTitle;
}
forceUpdate();
}
+ // "Minimize" — hide the dialog but keep progress in the store so the
+ // indicator near the search bar stays visible.
+ function minimizeDialog() {
+ open.current = false;
+ userMinimized.current = true;
+ forceUpdate();
+ }
+
function closeDialog() {
open.current = false;
+ userMinimized.current = false;
targetPath.current = undefined;
- //transferMeta.current = undefined;
dialogTitle.current = 'importDialogTitle';
forceUpdate();
}
@@ -100,7 +133,7 @@ export const FileUploadDialogContextProvider = ({
{children}
<FileUploadDialog
open={open.current}
- onClose={() => closeDialog()}
+ onClose={() => minimizeDialog()}
title={dialogTitle.current}
targetPath={targetPath.current}
/>
diff --git a/src/renderer/hooks/IOActionsContextProvider.tsx b/src/renderer/hooks/IOActionsContextProvider.tsx
index 5e5e6f655..82edc5dc1 100644
--- a/src/renderer/hooks/IOActionsContextProvider.tsx
+++ b/src/renderer/hooks/IOActionsContextProvider.tsx
@@ -43,10 +43,10 @@ import {
import {
cleanMetaData,
downloadFile,
- executePromisesInBatches,
mergeFsEntryMeta,
openDirectoryMessage,
openFileMessage,
+ runConcurrent,
} from '-/services/utils-io';
import { TS } from '-/tagspaces.namespace';
import { CommonLocation } from '-/utils/CommonLocation';
@@ -575,59 +575,120 @@ export const IOActionsContextProvider = ({
});
}
- function moveDirs(
+ // Conservative cap: each dir IO job is a recursive walk (fs.copy on local,
+ // per-key fanout on S3). 2 keeps disk/network from thrashing while still
+ // overlapping a small dir behind a large one. Renderer doesn't accumulate
+ // per-file promises here — only one PromiseSettledResult per top-level dir.
+ const DIR_IO_CONCURRENCY = 2;
+
+ /**
+ * Shared runner for copyDirs/moveDirs. Caps concurrency, threads an abort
+ * signal through runConcurrent (clicking abort on any in-flight row stops
+ * new dirs from being picked up AND triggers the inner per-dir abort), and
+ * keeps the post-batch reflect/progress wiring identical to the original.
+ */
+ function runDirIOJobs(
dirPaths: Array<any>,
targetPath: string,
locationID: string,
- onProgress = undefined,
+ onProgress: any,
+ mode: 'copy' | 'move',
): Promise<boolean> {
- const progress = dirPaths.length > 10 ? undefined : onProgress;
- const promises = dirPaths.map(({ path, count }) => {
- const dirName = extractDirectoryName(
- path,
- currentLocation?.getDirSeparator(),
- );
- return moveDirectoryPromise(
- { path: path, total: count, locationID },
- joinPaths(currentLocation?.getDirSeparator(), targetPath, dirName),
- progress,
- false,
- )
- .then((newDirPath) => {
- // console.log('Moving dir from ' + path + ' to ' + targetPath);
- const action: TS.EditAction = {
- action: 'move',
- entry: currentLocation.toFsEntry(newDirPath, false),
- oldEntryPath: path,
- };
- return action;
- })
- .catch((err) => {
- console.log('Moving dirs failed ', err);
- showNotification(t('core:copyingFoldersFailed'));
- return undefined;
- });
- });
- return executePromisesInBatches(promises).then((actions) => {
- if (!progress) {
- const progresses = actions.map((action) =>
- action
- ? {
- path: action.entry.path,
- progress: 100,
+ const controller = new AbortController();
+
+ // Wrap onProgress so the per-row abort fn calls both the inner per-dir
+ // abort (stops the current walk where supported — fs-extra filter; S3
+ // mutes further progress) and our controller.abort (prevents new dirs
+ // from starting). Same semantics as copyFilesWithProgress.
+ const wrappedProgress = onProgress
+ ? (progress: any, innerAbort: any, fileName: any) => {
+ const combinedAbort = () => {
+ if (typeof innerAbort === 'function') {
+ try {
+ innerAbort();
+ } catch (e) {
+ console.log('inner dir abort failed:', e);
}
- : {
- path: action.entry.path,
- progress: 0,
- },
+ }
+ controller.abort();
+ };
+ onProgress(progress, combinedAbort, fileName);
+ }
+ : undefined;
+
+ return runConcurrent(
+ dirPaths,
+ ({ path, count }) => {
+ const dirName = extractDirectoryName(
+ path,
+ currentLocation?.getDirSeparator(),
);
- dispatch(AppActions.setProgresses(progresses));
+ const newDirPath = joinPaths(
+ currentLocation?.getDirSeparator(),
+ targetPath,
+ dirName,
+ );
+ const param = { path, total: count, locationID };
+ const inner =
+ mode === 'move'
+ ? moveDirectoryPromise(param, newDirPath, wrappedProgress, false)
+ : copyDirectoryPromise(param, newDirPath, wrappedProgress, false);
+ return inner.then((resolved): Promise<TS.EditAction | undefined> => {
+ if (mode === 'move') {
+ return Promise.resolve({
+ action: 'move',
+ entry: currentLocation!.toFsEntry(resolved, false),
+ oldEntryPath: path,
+ });
+ }
+ return getAllPropertiesPromise(resolved).then(
+ (fsEntry: TS.FileSystemEntry) => ({
+ action: 'add',
+ entry: fsEntry,
+ }),
+ );
+ });
+ },
+ DIR_IO_CONCURRENCY,
+ undefined,
+ controller.signal,
+ ).then((settled) => {
+ const actions: TS.EditAction[] = [];
+ for (const r of settled) {
+ if (r && r.status === 'fulfilled' && r.value) {
+ actions.push(r.value as TS.EditAction);
+ } else if (r && r.status === 'rejected') {
+ console.log(mode + ' dirs failed:', r.reason);
+ }
+ }
+ if (actions.length !== settled.length) {
+ showNotification(t('core:copyingFoldersFailed'));
+ }
+ // If onProgress was wired, the dialog already shows in-flight rows.
+ // If not, mirror the previous behavior of emitting a final 100/0 list
+ // so consumers without progress callbacks still see completion state.
+ if (!onProgress) {
+ const progresses = actions
+ .map((a) => a && a.entry && { path: a.entry.path, progress: 100 })
+ .filter(Boolean) as { path: string; progress: number }[];
+ if (progresses.length > 0) {
+ dispatch(AppActions.setProgresses(progresses));
+ }
}
- setReflectActions(...actions.filter((value) => value !== undefined));
+ setReflectActions(...actions);
return true;
});
}
+ function moveDirs(
+ dirPaths: Array<any>,
+ targetPath: string,
+ locationID: string,
+ onProgress = undefined,
+ ): Promise<boolean> {
+ return runDirIOJobs(dirPaths, targetPath, locationID, onProgress, 'move');
+ }
+
function moveFiles(
paths: Array<string>,
targetPath: string,
@@ -643,14 +704,8 @@ export const IOActionsContextProvider = ({
location.getDirSeparator() +
extractFileName(path, location.getDirSeparator()),
]);
- return moveFilesPromise(
- moveJobs,
- location.uuid,
- paths.length > 10 ? undefined : onProgress,
- false,
- force,
- )
- .then((moveArray) => {
+ return moveFilesPromise(moveJobs, location!.uuid, onProgress, false, force)
+ .then(async (moveArray) => {
if (moveArray !== undefined && moveArray.length > 0) {
setSelectedEntries([]);
const moveError = moveArray.find((err) => err instanceof Error);
@@ -726,8 +781,25 @@ export const IOActionsContextProvider = ({
]);
return true;
});
+ // Most files have no sidecar/thumb/pdf-text. Filter to only
+ // existing ones so we don't fan out thousands of guaranteed-to-fail
+ // IPC invokes — see the symmetric prefilter in copyFiles.
+ const sourceMetaPaths = moveMetaJobs.map((j) => j[0]);
+ const existenceResults = await runConcurrent(
+ sourceMetaPaths,
+ (p) =>
+ location
+ .checkFileExist(p)
+ .then((exists) => Boolean(exists))
+ .catch(() => false),
+ 16,
+ );
+ const existingMoveMetaJobs = moveMetaJobs.filter((_, i) => {
+ const r = existenceResults[i];
+ return r && r.status === 'fulfilled' && r.value === true;
+ });
return moveFilesPromise(
- moveMetaJobs,
+ existingMoveMetaJobs,
location.uuid,
undefined,
false,
@@ -765,54 +837,7 @@ export const IOActionsContextProvider = ({
locationID: string,
onProgress = undefined,
): Promise<boolean> {
- const progress = dirPaths.length > 10 ? undefined : onProgress;
- const promises = dirPaths.map(({ path, count }) => {
- const dirName = extractDirectoryName(
- path,
- currentLocation?.getDirSeparator(),
- );
- return copyDirectoryPromise(
- { path: path, total: count, locationID },
- joinPaths(currentLocation?.getDirSeparator(), targetPath, dirName),
- progress,
- false,
- )
- .then((newDirPath) => {
- // console.log('Copy dir from ' + path + ' to ' + targetPath);
- return getAllPropertiesPromise(newDirPath).then(
- (fsEntry: TS.FileSystemEntry) => {
- const action: TS.EditAction = {
- action: 'add',
- entry: fsEntry, //toFsEntry(newDirPath, false),
- };
- return action;
- },
- );
- })
- .catch((err) => {
- console.log('Copy dirs failed ', err);
- showNotification(t('core:copyingFoldersFailed'));
- return undefined;
- });
- });
- return executePromisesInBatches(promises).then((actions) => {
- if (!progress) {
- const progresses = actions.map((action) =>
- action
- ? {
- path: action.entry.path,
- progress: 100,
- }
- : {
- path: action.entry.path,
- progress: 0,
- },
- );
- dispatch(AppActions.setProgresses(progresses));
- }
- setReflectActions(...actions.filter((value) => value !== undefined));
- return true;
- });
+ return runDirIOJobs(dirPaths, targetPath, locationID, onProgress, 'copy');
}
function copyFiles(
@@ -821,13 +846,8 @@ export const IOActionsContextProvider = ({
locationID: string,
onProgress,
): Promise<boolean> {
- return copyFilesWithProgress(
- paths,
- targetPath,
- locationID,
- paths.length > 10 ? undefined : onProgress,
- )
- .then((success) => {
+ return copyFilesWithProgress(paths, targetPath, locationID, onProgress)
+ .then(async (success) => {
if (success) {
showNotification(t('core:filesCopiedSuccessful'));
const metaPaths = paths.flatMap((path) =>
@@ -850,10 +870,31 @@ export const IOActionsContextProvider = ({
],
);
+ // Most files have no sidecar/thumb. Filter to only existing ones so
+ // we don't fan out thousands of guaranteed-to-fail IPC invokes —
+ // each failed call round-trips a full-path Error string back to the
+ // renderer, which previously OOM'd V8 on 3500+ file copies.
+ const sidecarLocation = findLocation(locationID);
+ const existenceResults = sidecarLocation
+ ? await runConcurrent(
+ metaPaths,
+ (p) =>
+ sidecarLocation
+ .checkFileExist(p)
+ .then((exists) => Boolean(exists))
+ .catch(() => false),
+ 16,
+ )
+ : [];
+ const existingMetaPaths = metaPaths.filter((_, i) => {
+ const r = existenceResults[i];
+ return r && r.status === 'fulfilled' && r.value === true;
+ });
+
return copyFilesWithProgress(
- metaPaths,
+ existingMetaPaths,
getMetaDirectoryPath(targetPath),
- locationID, //metaPaths.length > 10 ? undefined : onProgress,
+ locationID, //existingMetaPaths.length > 10 ? undefined : onProgress,
false,
)
.then(() => {
diff --git a/src/renderer/hooks/LocationIndexContextProvider.tsx b/src/renderer/hooks/LocationIndexContextProvider.tsx
index 8fce6228c..6eb187966 100644
--- a/src/renderer/hooks/LocationIndexContextProvider.tsx
+++ b/src/renderer/hooks/LocationIndexContextProvider.tsx
@@ -35,6 +35,8 @@ import Fuse from 'fuse.js';
import { TS } from '-/tagspaces.namespace';
import { CommonLocation } from '-/utils/CommonLocation';
import {
+ applyBulkCreateToIndex,
+ applyBulkUpdateToIndex,
applyCreateToIndex,
applyDeleteToIndex,
applyUpdateToIndex,
@@ -189,6 +191,33 @@ export const LocationIndexContextProvider = ({
useEffect(() => {
if (!firstRender && actions && actions.length > 0) {
+ // Bulk fast paths. Calling reflectCreateEntry / reflectUpdateEntry per
+ // action does an O(n) array allocation each iteration — for a 3500-file
+ // bulk copy/move reflected into a 50k-entry index that's ~1.5 GB of
+ // intermediate arrays in one synchronous tick (the OOM the user saw).
+ // Detect uniform-add and uniform-move/update batches and apply them in
+ // a single allocation.
+ if (actions.length > 1 && actions.every((a) => a.action === 'add')) {
+ const next = applyBulkCreateToIndex(
+ index.current,
+ actions.map((a) => a.entry).filter(Boolean),
+ );
+ if (next) setIndex(next, currentLocation);
+ return;
+ }
+ if (
+ actions.length > 1 &&
+ actions.every((a) => a.action === 'move' || a.action === 'update')
+ ) {
+ const next = applyBulkUpdateToIndex(
+ index.current,
+ actions
+ .filter((a) => a.oldEntryPath && a.entry)
+ .map((a) => ({ oldPath: a.oldEntryPath, entry: a.entry })),
+ );
+ if (next) setIndex(next, currentLocation);
+ return;
+ }
for (const action of actions) {
if (action.action === 'add') {
reflectCreateEntry(action.entry);
diff --git a/src/renderer/hooks/PlatformFacadeContextProvider.tsx b/src/renderer/hooks/PlatformFacadeContextProvider.tsx
index 665dfd24f..35cb0f789 100644
--- a/src/renderer/hooks/PlatformFacadeContextProvider.tsx
+++ b/src/renderer/hooks/PlatformFacadeContextProvider.tsx
@@ -31,7 +31,7 @@ import { getUuid } from '@tagspaces/tagspaces-common/utils-io';
import { useTranslation } from 'react-i18next';
import { useSelector } from 'react-redux';
import { useEditedEntryContext } from '-/hooks/useEditedEntryContext';
-import { executePromisesInBatches } from '-/services/utils-io';
+import { executePromisesInBatches, runConcurrent } from '-/services/utils-io';
import { getUseTrashCan } from '-/reducers/settings';
import { useCurrentLocationContext } from '-/hooks/useCurrentLocationContext';
import { useDirectoryContentContext } from '-/hooks/useDirectoryContentContext';
@@ -295,6 +295,10 @@ export const PlatformFacadeContextProvider = ({
});
}
+ // Concurrency cap for bulk IPC IO. Tuned to keep V8 heap flat on huge
+ // copy/move batches (3500+ files) — eager fanout used to OOM the renderer.
+ const COPY_CONCURRENCY = 16;
+
function copyFilesWithProgress(
paths: string[],
targetPath: string,
@@ -303,91 +307,70 @@ export const PlatformFacadeContextProvider = ({
reflect = true,
): Promise<boolean> {
const controller = new AbortController();
- const signal = controller.signal;
const location = findLocation(locationID);
- const ioJobPromises = paths.map((path) => {
- const targetFile =
- normalizePath(targetPath) +
- (location ? location.getDirSeparator() : AppConfig.dirSeparator) +
- extractFileName(path, location?.getDirSeparator());
- return {
- promise: copyFilePromiseOverwrite(path, targetFile, locationID, false),
- path: path,
- };
- });
- const progress = (completed, path) => {
- if (onProgress) {
- const progress = {
- loaded: completed, //processedSize,
- total: completed, //ioJobPromises.length,
- key: path, //targetPath,
- };
- onProgress(
- progress,
- () => {
- controller.abort();
- },
- path,
- );
+ const buildTargetPath = (path: string) =>
+ normalizePath(targetPath) +
+ (location ? location.getDirSeparator() : AppConfig.dirSeparator) +
+ extractFileName(path, location?.getDirSeparator());
+
+ // Single-row batch progress. We dispatch all per-file completions under
+ // one stable key (the target dir) so the dialog renders ONE bar that
+ // fills `completed / total`, not N rows. Per-file rows for thousands of
+ // files thrashed the reducer (O(n) array rebuild on each dispatch) and
+ // the dialog (O(n log n) sort + N LinearProgress reconciliations) on
+ // every file settle — total O(n² log n). That made big copies feel slow
+ // even when the disk had finished. The `fileName` arg still updates the
+ // row label so the user sees which file just landed.
+ const batchKey = normalizePath(targetPath);
+ let completed = 0;
+ const report = (path: string) => {
+ if (!onProgress) {
+ return;
}
+ completed++;
+ onProgress(
+ {
+ loaded: completed,
+ total: paths.length,
+ key: batchKey,
+ },
+ () => {
+ controller.abort();
+ },
+ path,
+ );
};
- return trackProgress(ioJobPromises, signal, progress).then(() => {
+
+ return runConcurrent(
+ paths,
+ (path) =>
+ copyFilePromiseOverwrite(
+ path,
+ buildTargetPath(path),
+ locationID,
+ false,
+ ),
+ COPY_CONCURRENCY,
+ (path, _idx, result) => {
+ if (result.status === 'rejected') {
+ console.log('Copy ' + path + ' error:', result.reason);
+ }
+ report(path);
+ },
+ controller.signal,
+ ).then(() => {
if (reflect) {
- const targetPaths = paths.map((path) =>
- getAllPropertiesPromise(
- normalizePath(targetPath) +
- (location ? location.getDirSeparator() : AppConfig.dirSeparator) +
- extractFileName(path, location?.getDirSeparator()),
- locationID,
+ return reflectAddEntryPath(
+ ...paths.map((path) =>
+ getAllPropertiesPromise(buildTargetPath(path), locationID),
),
);
- return reflectAddEntryPath(...targetPaths);
}
return true;
});
}
- function trackProgress(promises, abortSignal, progress) {
- // const total = promises.length;
- let completed = 0;
- let aborted = false;
-
- // Create an array of promises that resolve when the original promises resolve
- const progressPromises = promises.map(({ promise, path }) =>
- promise
- .then(() => {
- if (!aborted) {
- completed++;
- // console.log(`Progress: ${completed}/${total}`);
- if (progress) {
- progress(completed, path);
- }
- }
- })
- .catch((err) => {
- completed++;
- if (progress) {
- progress(completed, path);
- }
- console.log('Promise ' + path + ' error:', err);
- throw err; //return err;
- }),
- );
-
- // Use Promise.race() to wait for all progress promises to resolve
- return Promise.race(progressPromises)
- .then(() => Promise.all(promises))
- .catch((err) => {
- if (abortSignal.aborted) {
- aborted = true;
- console.log('Promise execution aborted');
- } else {
- throw err;
- }
- });
- }
-
function copyFilePromiseOverwrite(
sourceFilePath: string,
targetFilePath: string,
@@ -501,29 +484,65 @@ export const PlatformFacadeContextProvider = ({
function moveFilesPromise(
renameJobs: Array<Array<string>>,
locationID: string,
- onProgress = undefined,
+ onProgress: any = undefined,
reflect = true,
force = false,
): Promise<any[]> {
const flatArray = renameJobs.flat();
ignoreByWatcher(...flatArray);
- return executePromisesInBatches(
- renameJobs.map(async (renameJob) => {
- try {
- return await getLocation({ locationID }).renameFilePromise(
- renameJob[0],
- renameJob[1],
- onProgress,
- force,
- );
- } catch (err) {
- console.log('Error rename file:', err);
- return err; //undefined;
+
+ const controller = new AbortController();
+
+ // Single-row batch progress (same reasoning as copyFilesWithProgress).
+ // moveFilesPromise has no single targetPath argument — each job carries
+ // its own dest — so derive the batch key from the first job's parent
+ // dir. Falls back to a generated id when called with an empty list.
+ const moveLocation = findLocation(locationID);
+ const moveSep = moveLocation
+ ? moveLocation.getDirSeparator()
+ : AppConfig.dirSeparator;
+ const batchKey =
+ renameJobs.length > 0
+ ? extractContainingDirectoryPath(renameJobs[0][1], moveSep)
+ : 'move-batch-' + getUuid();
+ let completed = 0;
+ const reportMoveProgress = (sourcePath: string) => {
+ if (!onProgress) {
+ return;
+ }
+ completed++;
+ onProgress(
+ { loaded: completed, total: renameJobs.length, key: batchKey },
+ () => {
+ controller.abort();
+ },
+ sourcePath,
+ );
+ };
+
+ return runConcurrent(
+ renameJobs,
+ (job) =>
+ getLocation({ locationID })
+ .renameFilePromise(job[0], job[1], onProgress, force)
+ .catch((err) => {
+ console.log('Error rename file:', err);
+ return err;
+ }),
+ COPY_CONCURRENCY,
+ (job, _idx, result) => {
+ reportMoveProgress(job[0]);
+ if (result.status === 'rejected') {
+ console.log('Move ' + job[0] + ' error:', result.reason);
}
- }),
- ).then((ret) => {
+ },
+ controller.signal,
+ ).then((settled) => {
deignoreByWatcher(...flatArray);
- const r = ret.filter((r) => r !== undefined);
+ const ret = settled.map((r) =>
+ r.status === 'fulfilled' ? r.value : r.reason,
+ );
+ const r = ret.filter((v) => v !== undefined);
if (reflect && r.length > 0) {
reflectMoveFiles(renameJobs);
}
diff --git a/src/renderer/services/utils-io.ts b/src/renderer/services/utils-io.ts
index ed9da1312..b17048341 100644
--- a/src/renderer/services/utils-io.ts
+++ b/src/renderer/services/utils-io.ts
@@ -948,6 +948,53 @@ export async function executePromisesInBatches<T>(
return results;
}
+/**
+ * Worker-pool runner with a real concurrency cap. Unlike
+ * `executePromisesInBatches`, the items are *factories* that are invoked only
+ * when a worker slot becomes free — so the underlying work (IPC invokes, fs
+ * ops) is throttled, not just the awaits. Important for big IO fanouts where
+ * eager `.map(() => invoke(...))` would queue thousands of in-flight promises
+ * and OOM the renderer.
+ */
+export async function runConcurrent<I, R>(
+ items: I[],
+ worker: (item: I, index: number) => Promise<R>,
+ concurrency: number,
+ onSettled?: (item: I, index: number, result: PromiseSettledResult<R>) => void,
+ abortSignal?: AbortSignal,
+): Promise<PromiseSettledResult<R>[]> {
+ const results: PromiseSettledResult<R>[] = new Array(items.length);
+ let cursor = 0;
+ const slots = Math.max(1, Math.min(concurrency, items.length));
+
+ const runWorker = async () => {
+ while (true) {
+ if (abortSignal && abortSignal.aborted) {
+ return;
+ }
+ const idx = cursor++;
+ if (idx >= items.length) {
+ return;
+ }
+ const item = items[idx];
+ let result: PromiseSettledResult<R>;
+ try {
+ const value = await worker(item, idx);
+ result = { status: 'fulfilled', value };
+ } catch (reason) {
+ result = { status: 'rejected', reason };
+ }
+ results[idx] = result;
+ if (onSettled) {
+ onSettled(item, idx, result);
+ }
+ }
+ };
+
+ await Promise.all(Array.from({ length: slots }, runWorker));
+ return results;
+}
+
export function setZoomFactorElectron(zoomLevel) {
if (AppConfig.isElectron) {
window.electronIO.ipcRenderer.sendMessage('setZoomFactor', zoomLevel);
diff --git a/src/renderer/utils/indexReflect.ts b/src/renderer/utils/indexReflect.ts
index 0e92b9df6..cc9918899 100644
--- a/src/renderer/utils/indexReflect.ts
+++ b/src/renderer/utils/indexReflect.ts
@@ -59,6 +59,36 @@ export function applyCreateToIndex(
return [...current, newEntry];
}
+/**
+ * Bulk variant of applyCreateToIndex. Calling the single-entry helper N times
+ * in a loop is O(N*M) (each iteration re-scans + re-copies the growing
+ * index). For huge batches (e.g. a 3500-file copy reflected into a 50k-entry
+ * index) that's ~1.5 GB of intermediate allocations in a synchronous tick,
+ * which OOMs the renderer before GC gets a chance. This variant is O(N+M):
+ * one dedup Set, one combined copy.
+ */
+export function applyBulkCreateToIndex(
+ current: Index,
+ newEntries: TS.FileSystemEntry[],
+): TS.FileSystemEntry[] | null {
+ if (!current || current.length < 1) return null;
+ if (!newEntries || newEntries.length === 0) return null;
+ const existingPaths = new Set<string>();
+ for (let i = 0; i < current.length; i += 1) {
+ existingPaths.add(current[i].path);
+ }
+ const toAdd: TS.FileSystemEntry[] = [];
+ for (let i = 0; i < newEntries.length; i += 1) {
+ const e = newEntries[i];
+ if (e && !existingPaths.has(e.path)) {
+ existingPaths.add(e.path);
+ toAdd.push(e);
+ }
+ }
+ if (toAdd.length === 0) return null;
+ return [...current, ...toAdd];
+}
+
/**
* Replace the entry matching `oldPath` with `newEntry`. Used for both
* `update` (same path, new meta) and `move` (new path) actions. Returns
@@ -73,3 +103,38 @@ export function applyUpdateToIndex(
if (!current.some((entry) => entry.path === oldPath)) return null;
return current.map((entry) => (entry.path === oldPath ? newEntry : entry));
}
+
+/**
+ * Bulk variant of applyUpdateToIndex. Same motivation as
+ * applyBulkCreateToIndex — a 3500-file move would otherwise re-scan +
+ * re-map the index 3500 times. This builds one path→entry replacement map
+ * and walks the index once: O(N + M).
+ */
+export function applyBulkUpdateToIndex(
+ current: Index,
+ updates: Array<{ oldPath: string; entry: TS.FileSystemEntry }>,
+): TS.FileSystemEntry[] | null {
+ if (!current || current.length < 1) return null;
+ if (!updates || updates.length === 0) return null;
+ const byOldPath = new Map<string, TS.FileSystemEntry>();
+ for (let i = 0; i < updates.length; i += 1) {
+ const u = updates[i];
+ if (u && u.oldPath && u.entry) {
+ byOldPath.set(u.oldPath, u.entry);
+ }
+ }
+ if (byOldPath.size === 0) return null;
+ let changed = false;
+ const next: TS.FileSystemEntry[] = new Array(current.length);
+ for (let i = 0; i < current.length; i += 1) {
+ const e = current[i];
+ const repl = byOldPath.get(e.path);
+ if (repl) {
+ next[i] = repl;
+ changed = true;
+ } else {
+ next[i] = e;
+ }
+ }
+ return changed ? next : null;
+}