Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

refractor(locker):split-hooks

Aswin Asok61309a167adc
diff --git a/web/apps/locker/src/components/create-item/scan-dropped-files.ts b/web/apps/locker/src/components/create-item/scan-dropped-files.ts
new file mode 100644
index 0000000000..ff4188c57a
--- /dev/null
+++ b/web/apps/locker/src/components/create-item/scan-dropped-files.ts
@@ -0,0 +1,223 @@
+import type { LockerUploadCandidate } from "@/types";
+import {
+ LOCKER_MAX_FILE_SIZE_BYTES,
+ type LockerUploadPreflightFailure,
+} from "../../services/locker-limits";
+
+export type DragDataTransferItem = DataTransferItem & {
+ webkitGetAsEntry?: () => FileSystemEntry | null;
+};
+
+type FileSystemFileEntry = FileSystemEntry & {
+ file: (
+ successCallback: (file: File) => void,
+ errorCallback?: (error: DOMException) => void,
+ ) => void;
+};
+
+interface FileSystemDirectoryReader {
+ readEntries: (
+ successCallback: (entries: FileSystemEntry[]) => void,
+ errorCallback?: (error: DOMException) => void,
+ ) => void;
+}
+
+type FileSystemDirectoryEntry = FileSystemEntry & {
+ createReader: () => FileSystemDirectoryReader;
+};
+
+const fileFromEntry = (entry: FileSystemFileEntry) =>
+ new Promise<File>((resolve, reject) => {
+ entry.file(resolve, reject);
+ });
+
+const readDirectoryEntries = (entry: FileSystemDirectoryEntry) =>
+ new Promise<FileSystemEntry[]>((resolve, reject) => {
+ const reader = entry.createReader();
+ const entries: FileSystemEntry[] = [];
+ const readBatch = () => {
+ reader.readEntries((batch) => {
+ if (batch.length === 0) {
+ resolve(entries);
+ return;
+ }
+ entries.push(...batch);
+ readBatch();
+ }, reject);
+ };
+ readBatch();
+ });
+
+const collectionNamesFromRelativePath = (relativePath?: string) => [
+ ...new Set((relativePath?.split("/").slice(0, -1) ?? []).filter(Boolean)),
+];
+
+const uploadCandidateFromFile = (
+ file: File,
+ relativePath?: string,
+): LockerUploadCandidate => ({
+ file,
+ relativePath,
+ suggestedCollectionNames: collectionNamesFromRelativePath(relativePath),
+});
+
+const droppedUploadCandidateKey = (file: File, relativePath?: string) =>
+ `${relativePath ?? file.name}:${file.size}:${file.lastModified}`;
+
+interface DropUploadScanOptions {
+ remainingFileCount: number;
+ freeStorage: number;
+}
+
+interface DropUploadScanResult {
+ items: LockerUploadCandidate[];
+ preflightFailure?: LockerUploadPreflightFailure;
+ scannedNonEmptyFileCount: number;
+ scannedTotalSize: number;
+}
+
+type PendingDroppedNode =
+ | { type: "entry"; entry: FileSystemEntry; parentPath: string }
+ | { type: "file"; file: File; relativePath: string };
+
+export const collectUploadCandidatesFromDrop = async (
+ droppedItems: DragDataTransferItem[],
+ fallbackFiles: File[],
+ options: DropUploadScanOptions,
+): Promise<DropUploadScanResult> => {
+ const items: LockerUploadCandidate[] = [];
+ const pendingNodes: PendingDroppedNode[] = [];
+ const seenUploadCandidateKeys = new Set<string>();
+ let scannedNonEmptyFileCount = 0;
+ let scannedTotalSize = 0;
+
+ const addFile = (
+ file: File,
+ relativePath: string,
+ ): LockerUploadPreflightFailure | undefined => {
+ if (file.size === 0) {
+ return undefined;
+ }
+
+ const dedupeKey = droppedUploadCandidateKey(file, relativePath);
+ if (seenUploadCandidateKeys.has(dedupeKey)) {
+ return undefined;
+ }
+ seenUploadCandidateKeys.add(dedupeKey);
+
+ if (file.size > LOCKER_MAX_FILE_SIZE_BYTES) {
+ return { reason: "fileTooLarge", fileName: file.name };
+ }
+
+ scannedNonEmptyFileCount += 1;
+ if (scannedNonEmptyFileCount > options.remainingFileCount) {
+ return { reason: "fileCountLimit" };
+ }
+
+ scannedTotalSize += file.size;
+ if (scannedTotalSize > options.freeStorage) {
+ return { reason: "storageLimit" };
+ }
+
+ items.push(uploadCandidateFromFile(file, relativePath));
+ return undefined;
+ };
+
+ for (let index = droppedItems.length - 1; index >= 0; index -= 1) {
+ const item = droppedItems[index]!;
+ if (item.kind !== "file") {
+ continue;
+ }
+
+ const getAsEntry = item.webkitGetAsEntry;
+ const entry =
+ typeof getAsEntry === "function" ? getAsEntry.call(item) : null;
+ if (entry) {
+ pendingNodes.push({ type: "entry", entry, parentPath: "" });
+ continue;
+ }
+
+ const file = item.getAsFile();
+ if (file) {
+ pendingNodes.push({
+ type: "file",
+ file,
+ relativePath: file.webkitRelativePath || file.name,
+ });
+ }
+ }
+
+ if (pendingNodes.length === 0) {
+ for (const file of fallbackFiles) {
+ const preflightFailure = addFile(
+ file,
+ file.webkitRelativePath || file.name,
+ );
+ if (preflightFailure) {
+ return {
+ items,
+ preflightFailure,
+ scannedNonEmptyFileCount,
+ scannedTotalSize,
+ };
+ }
+ }
+ return { items, scannedNonEmptyFileCount, scannedTotalSize };
+ }
+
+ while (pendingNodes.length > 0) {
+ const node = pendingNodes.pop()!;
+
+ if (node.type === "file") {
+ const preflightFailure = addFile(node.file, node.relativePath);
+ if (preflightFailure) {
+ return {
+ items,
+ preflightFailure,
+ scannedNonEmptyFileCount,
+ scannedTotalSize,
+ };
+ }
+ continue;
+ }
+
+ if (node.entry.isFile) {
+ const file = await fileFromEntry(node.entry as FileSystemFileEntry);
+ const relativePath = node.parentPath
+ ? `${node.parentPath}/${file.name}`
+ : file.name;
+ const preflightFailure = addFile(file, relativePath);
+ if (preflightFailure) {
+ return {
+ items,
+ preflightFailure,
+ scannedNonEmptyFileCount,
+ scannedTotalSize,
+ };
+ }
+ continue;
+ }
+
+ if (node.entry.isDirectory) {
+ const directoryPath = node.parentPath
+ ? `${node.parentPath}/${node.entry.name}`
+ : node.entry.name;
+ const directoryEntries = await readDirectoryEntries(
+ node.entry as FileSystemDirectoryEntry,
+ );
+ for (
+ let index = directoryEntries.length - 1;
+ index >= 0;
+ index -= 1
+ ) {
+ pendingNodes.push({
+ type: "entry",
+ entry: directoryEntries[index]!,
+ parentPath: directoryPath,
+ });
+ }
+ }
+ }
+
+ return { items, scannedNonEmptyFileCount, scannedTotalSize };
+};
diff --git a/web/apps/locker/src/components/create-item/use-create-item-dialog-state.ts b/web/apps/locker/src/components/create-item/use-create-item-dialog-state.ts
index b5ab561b25..0b21a8afea 100644
--- a/web/apps/locker/src/components/create-item/use-create-item-dialog-state.ts
+++ b/web/apps/locker/src/components/create-item/use-create-item-dialog-state.ts
@@ -3,13 +3,7 @@ import {
lockerUpgradeCTAType,
type LockerUpgradeCTAType,
} from "@/services/locker-errors";
-import {
- exceedsPaidLockerHardLimit,
- validateLockerUploadBatch,
- type LockerUploadLimitState,
- type LockerUploadPreflightFailure,
-} from "@/services/locker-limits";
-import type { LockerUploadProgress } from "@/services/remote";
+import type { LockerUploadLimitState } from "@/services/locker-limits";
import type {
LockerCollection,
LockerItemType,
@@ -27,15 +21,13 @@ import type { ChangeEvent } from "react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import {
collectionNamesByUploadItem,
- dedupeCollectionNames,
filterNonEmptyUploadItems,
- normalizeCollectionName,
uploadQueueItemKey,
} from "./file-upload-helpers";
import { getRequiredFields } from "./item-form-fields-utils";
+import { useUploadQueue, type LockerUploadCallbacks } from "./use-upload-queue";
export type CreateOption = LockerItemType;
-const MAX_PARALLEL_UPLOADS = 4;
export interface CreateItemDialogEditItem {
id: number;
@@ -45,7 +37,7 @@ export interface CreateItemDialogEditItem {
collectionIDs: number[];
}
-interface UseCreateItemDialogStateProps {
+interface UseCreateItemDialogStateProps extends LockerUploadCallbacks {
open: boolean;
collections: LockerCollection[];
onClose: () => void;
@@ -54,41 +46,12 @@ interface UseCreateItemDialogStateProps {
data: Record<string, unknown>,
collectionIDs: number[],
) => Promise<void>;
- onUploadProgress?: (
- file: File,
- collectionIDs: number[],
- onProgress: (progress: LockerUploadProgress) => void,
- ) => Promise<void>;
- onUploadItemComplete?: () => void;
- onUploadsFinished?: (uploadedCount: number) => Promise<void>;
- onEnsureCollections?: (
- names: string[],
- ) => Promise<Map<string, number> | Record<string, number>>;
- onEnsureUploadLimitState?: () => Promise<
- { userDetails: LockerUploadLimitState } | undefined
- >;
defaultCollectionID?: number | null;
initialItems?: LockerUploadCandidate[];
editItem?: CreateItemDialogEditItem | null;
userDetails?: LockerUploadLimitState;
}
-interface UploadState {
- completedFileKeys: Set<string>;
- failedFileKeys: Set<string>;
- uploadingFileKeys: Set<string>;
- uploadProgressByFileKey: Record<string, LockerUploadProgress | null>;
- uploadCapByFileKey: Record<string, number>;
-}
-
-const emptyUploadState = (): UploadState => ({
- completedFileKeys: new Set(),
- failedFileKeys: new Set(),
- uploadingFileKeys: new Set(),
- uploadProgressByFileKey: {},
- uploadCapByFileKey: {},
-});
-
const initialEditCollectionIDs = (
editItem: CreateItemDialogEditItem | null | undefined,
) => {
@@ -167,7 +130,6 @@ export const useCreateItemDialogState = ({
const [selectedCollectionIDs, setSelectedCollectionIDs] =
useState<number[]>(editCollectionIDs);
const [saving, setSaving] = useState(false);
- const [uploading, setUploading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [upgradeCTAType, setUpgradeCTAType] =
useState<LockerUpgradeCTAType | null>(null);
@@ -185,21 +147,28 @@ export const useCreateItemDialogState = ({
const [customCollectionNames, setCustomCollectionNames] = useState<
string[]
>([]);
- const [completedFileKeys, setCompletedFileKeys] = useState<Set<string>>(
- () => new Set(),
- );
- const [failedFileKeys, setFailedFileKeys] = useState<Set<string>>(
- () => new Set(),
- );
- const [uploadingFileKeys, setUploadingFileKeys] = useState<Set<string>>(
- () => new Set(),
- );
- const [uploadProgressByFileKey, setUploadProgressByFileKey] = useState<
- Record<string, LockerUploadProgress | null>
- >({});
- const [uploadCapByFileKey, setUploadCapByFileKey] = useState<
- Record<string, number>
- >({});
+ const {
+ uploading,
+ completedFileKeys,
+ failedFileKeys,
+ uploadingFileKeys,
+ uploadProgressByFileKey,
+ uploadCapByFileKey,
+ resetUploadState,
+ upload,
+ } = useUploadQueue({
+ displayCollections,
+ selectedUploadItems,
+ selectedCollectionNamesByFileKey,
+ userDetails,
+ setError,
+ setUpgradeCTAType,
+ onUploadProgress,
+ onUploadItemComplete,
+ onUploadsFinished,
+ onEnsureCollections,
+ onEnsureUploadLimitState,
+ });
const isFileMode = selectedOption === "file";
const editCollectionID = editItem?.collectionID ?? null;
@@ -223,15 +192,6 @@ export const useCreateItemDialogState = ({
[],
);
- const resetUploadState = useCallback(() => {
- const nextState = emptyUploadState();
- setCompletedFileKeys(nextState.completedFileKeys);
- setFailedFileKeys(nextState.failedFileKeys);
- setUploadingFileKeys(nextState.uploadingFileKeys);
- setUploadProgressByFileKey(nextState.uploadProgressByFileKey);
- setUploadCapByFileKey(nextState.uploadCapByFileKey);
- }, []);
-
useEffect(() => {
if (!open) {
return;
@@ -461,249 +421,10 @@ export const useCreateItemDialogState = ({
}
}, [formData, formType, handleClose, onSave, selectedCollectionIDs]);
- const handleUpload = useCallback(async () => {
- if (selectedUploadItems.length === 0 || !onUploadProgress) {
- return;
- }
-
- const pendingUploadItems = filterNonEmptyUploadItems(
- selectedUploadItems.filter(
- (item) => !completedFileKeys.has(uploadQueueItemKey(item)),
- ),
- );
- if (pendingUploadItems.length === 0) {
- return;
- }
-
- let effectiveUserDetails = userDetails;
- if (!effectiveUserDetails) {
- const uploadLimitState = await onEnsureUploadLimitState?.();
- effectiveUserDetails = uploadLimitState?.userDetails;
- }
-
- if (!effectiveUserDetails) {
- setError(t("generic_error_retry"));
- setUpgradeCTAType(null);
- return;
- }
-
- const preflightFailure = validateLockerUploadBatch(
- pendingUploadItems.map(({ file }) => file),
- effectiveUserDetails,
- );
- if (preflightFailure) {
- const hitsPaidHardCap = exceedsPaidLockerHardLimit(
- pendingUploadItems.map(({ file }) => file),
- effectiveUserDetails,
- );
- const preflightFailureMessage = (
- failure: LockerUploadPreflightFailure,
- ) => {
- if (
- hitsPaidHardCap &&
- (failure.reason === "fileCountLimit" ||
- failure.reason === "storageLimit")
- ) {
- return t("uploadLockerHardCapErrorBody");
- }
- switch (failure.reason) {
- case "fileCountLimit":
- return t("uploadFileCountLimitErrorBody");
- case "fileTooLarge":
- return t("uploadFileTooLargeErrorBody");
- case "storageLimit":
- return t("uploadStorageLimitErrorBody");
- }
- };
-
- setError(preflightFailureMessage(preflightFailure));
- setUpgradeCTAType(
- hitsPaidHardCap
- ? null
- : preflightFailure.reason === "fileCountLimit"
- ? "fileCountLimit"
- : preflightFailure.reason === "storageLimit"
- ? "storageLimit"
- : null,
- );
- return;
- }
-
- setUploading(true);
- setError(null);
- setUpgradeCTAType(null);
- const pendingUploadFileKeys = new Set(
- pendingUploadItems.map((item) => uploadQueueItemKey(item)),
- );
- setFailedFileKeys((current) => {
- const next = new Set(current);
- pendingUploadFileKeys.forEach((fileKey) => next.delete(fileKey));
- return next;
- });
- setUploadingFileKeys((current) => {
- const next = new Set(current);
- pendingUploadFileKeys.forEach((fileKey) => next.delete(fileKey));
- return next;
- });
- setUploadProgressByFileKey((current) =>
- Object.fromEntries(
- Object.entries(current).filter(
- ([fileKey]) => !pendingUploadFileKeys.has(fileKey),
- ),
- ),
- );
- setUploadCapByFileKey((current) =>
- Object.fromEntries(
- Object.entries(current).filter(
- ([fileKey]) => !pendingUploadFileKeys.has(fileKey),
- ),
- ),
- );
- let uploadedCount = 0;
- try {
- const existingNormalizedNameToID = new Map(
- displayCollections.map((collection) => [
- normalizeCollectionName(collection.name),
- collection.id,
- ]),
- );
- const normalizedNameToIDResult = await onEnsureCollections?.(
- dedupeCollectionNames(
- Object.values(selectedCollectionNamesByFileKey).flat(),
- ),
- );
- const normalizedNameToID = new Map(existingNormalizedNameToID);
- if (normalizedNameToIDResult instanceof Map) {
- for (const [name, id] of normalizedNameToIDResult.entries()) {
- normalizedNameToID.set(normalizeCollectionName(name), id);
- }
- } else {
- for (const [name, id] of Object.entries(
- normalizedNameToIDResult ?? {},
- )) {
- normalizedNameToID.set(normalizeCollectionName(name), id);
- }
- }
- const uploadTargets = pendingUploadItems.map((item) => {
- const fileKey = uploadQueueItemKey(item);
- const collectionIDs = dedupeCollectionNames(
- selectedCollectionNamesByFileKey[fileKey] ?? [],
- )
- .map((name) =>
- normalizedNameToID.get(normalizeCollectionName(name)),
- )
- .filter((id): id is number => typeof id === "number");
- return { item, fileKey, collectionIDs };
- });
-
- let nextIndex = 0;
- const worker = async () => {
- while (true) {
- const target = uploadTargets[nextIndex];
- nextIndex += 1;
- if (!target) {
- return;
- }
-
- const { item, fileKey, collectionIDs } = target;
- setUploadingFileKeys((current) =>
- new Set(current).add(fileKey),
- );
- setUploadCapByFileKey((current) => ({
- ...current,
- [fileKey]: 90 + Math.floor(Math.random() * 10),
- }));
- setUploadProgressByFileKey((current) => ({
- ...current,
- [fileKey]: { phase: "preparing" },
- }));
-
- try {
- await onUploadProgress(
- item.file,
- collectionIDs,
- (progress) => {
- setUploadProgressByFileKey((current) => ({
- ...current,
- [fileKey]: progress,
- }));
- },
- );
- uploadedCount += 1;
- setCompletedFileKeys((current) =>
- new Set(current).add(fileKey),
- );
- setFailedFileKeys((current) => {
- const next = new Set(current);
- next.delete(fileKey);
- return next;
- });
- onUploadItemComplete?.();
- } catch (error) {
- log.error("Failed to upload Locker file", error);
- const formattedError = await formatLockerMutationError(
- error,
- "uploadFile",
- );
- const nextUpgradeCTAType =
- await lockerUpgradeCTAType(error);
- setError((current) => current ?? formattedError);
- if (nextUpgradeCTAType) {
- setUpgradeCTAType(
- (current) => current ?? nextUpgradeCTAType,
- );
- }
- setFailedFileKeys((current) =>
- new Set(current).add(fileKey),
- );
- } finally {
- setUploadingFileKeys((current) => {
- const next = new Set(current);
- next.delete(fileKey);
- return next;
- });
- }
- }
- };
-
- await Promise.all(
- Array.from({
- length: Math.min(
- MAX_PARALLEL_UPLOADS,
- pendingUploadItems.length,
- ),
- }).map(() => worker()),
- );
-
- if (uploadedCount > 0) {
- await onUploadsFinished?.(uploadedCount);
- }
- if (
- completedFileKeys.size + uploadedCount ===
- selectedUploadItems.length
- ) {
- handleClose();
- }
- } catch (error) {
- log.error("Failed to upload Locker files", error);
- setError(await formatLockerMutationError(error, "uploadFile"));
- setUpgradeCTAType(await lockerUpgradeCTAType(error));
- } finally {
- setUploading(false);
- }
- }, [
- completedFileKeys,
- displayCollections,
- handleClose,
- onEnsureCollections,
- onEnsureUploadLimitState,
- onUploadItemComplete,
- onUploadProgress,
- onUploadsFinished,
- selectedCollectionNamesByFileKey,
- selectedUploadItems,
- userDetails,
- ]);
+ const handleUpload = useCallback(
+ () => upload(handleClose),
+ [upload, handleClose],
+ );
const canSave =
formType !== null &&
diff --git a/web/apps/locker/src/components/create-item/use-upload-queue.ts b/web/apps/locker/src/components/create-item/use-upload-queue.ts
new file mode 100644
index 0000000000..dba9db8aa2
--- /dev/null
+++ b/web/apps/locker/src/components/create-item/use-upload-queue.ts
@@ -0,0 +1,385 @@
+import {
+ formatLockerMutationError,
+ lockerUpgradeCTAType,
+ type LockerUpgradeCTAType,
+} from "@/services/locker-errors";
+import {
+ exceedsPaidLockerHardLimit,
+ validateLockerUploadBatch,
+ type LockerUploadLimitState,
+ type LockerUploadPreflightFailure,
+} from "@/services/locker-limits";
+import type { LockerUploadProgress } from "@/services/remote";
+import type { LockerCollection, LockerUploadCandidate } from "@/types";
+import log from "ente-base/log";
+import { t } from "i18next";
+import {
+ useCallback,
+ useState,
+ type Dispatch,
+ type SetStateAction,
+} from "react";
+import {
+ dedupeCollectionNames,
+ filterNonEmptyUploadItems,
+ normalizeCollectionName,
+ uploadQueueItemKey,
+} from "./file-upload-helpers";
+
+const MAX_PARALLEL_UPLOADS = 4;
+
+export interface LockerUploadCallbacks {
+ onUploadProgress?: (
+ file: File,
+ collectionIDs: number[],
+ onProgress: (progress: LockerUploadProgress) => void,
+ ) => Promise<void>;
+ onUploadItemComplete?: () => void;
+ onUploadsFinished?: (uploadedCount: number) => Promise<void>;
+ onEnsureCollections?: (
+ names: string[],
+ ) => Promise<Map<string, number> | Record<string, number>>;
+ onEnsureUploadLimitState?: () => Promise<
+ { userDetails: LockerUploadLimitState } | undefined
+ >;
+}
+
+interface UseUploadQueueProps extends LockerUploadCallbacks {
+ displayCollections: LockerCollection[];
+ selectedUploadItems: LockerUploadCandidate[];
+ selectedCollectionNamesByFileKey: Record<string, string[]>;
+ userDetails?: LockerUploadLimitState;
+ setError: Dispatch<SetStateAction<string | null>>;
+ setUpgradeCTAType: Dispatch<SetStateAction<LockerUpgradeCTAType | null>>;
+}
+
+interface UploadState {
+ completedFileKeys: Set<string>;
+ failedFileKeys: Set<string>;
+ uploadingFileKeys: Set<string>;
+ uploadProgressByFileKey: Record<string, LockerUploadProgress | null>;
+ uploadCapByFileKey: Record<string, number>;
+}
+
+const emptyUploadState = (): UploadState => ({
+ completedFileKeys: new Set(),
+ failedFileKeys: new Set(),
+ uploadingFileKeys: new Set(),
+ uploadProgressByFileKey: {},
+ uploadCapByFileKey: {},
+});
+
+export function useUploadQueue({
+ displayCollections,
+ selectedUploadItems,
+ selectedCollectionNamesByFileKey,
+ userDetails,
+ setError,
+ setUpgradeCTAType,
+ onUploadProgress,
+ onUploadItemComplete,
+ onUploadsFinished,
+ onEnsureCollections,
+ onEnsureUploadLimitState,
+}: UseUploadQueueProps) {
+ const [uploading, setUploading] = useState(false);
+ const [completedFileKeys, setCompletedFileKeys] = useState<Set<string>>(
+ () => new Set(),
+ );
+ const [failedFileKeys, setFailedFileKeys] = useState<Set<string>>(
+ () => new Set(),
+ );
+ const [uploadingFileKeys, setUploadingFileKeys] = useState<Set<string>>(
+ () => new Set(),
+ );
+ const [uploadProgressByFileKey, setUploadProgressByFileKey] = useState<
+ Record<string, LockerUploadProgress | null>
+ >({});
+ const [uploadCapByFileKey, setUploadCapByFileKey] = useState<
+ Record<string, number>
+ >({});
+
+ const resetUploadState = useCallback(() => {
+ const nextState = emptyUploadState();
+ setCompletedFileKeys(nextState.completedFileKeys);
+ setFailedFileKeys(nextState.failedFileKeys);
+ setUploadingFileKeys(nextState.uploadingFileKeys);
+ setUploadProgressByFileKey(nextState.uploadProgressByFileKey);
+ setUploadCapByFileKey(nextState.uploadCapByFileKey);
+ }, []);
+
+ const upload = useCallback(
+ async (onComplete: () => void) => {
+ if (selectedUploadItems.length === 0 || !onUploadProgress) {
+ return;
+ }
+
+ const pendingUploadItems = filterNonEmptyUploadItems(
+ selectedUploadItems.filter(
+ (item) => !completedFileKeys.has(uploadQueueItemKey(item)),
+ ),
+ );
+ if (pendingUploadItems.length === 0) {
+ return;
+ }
+
+ let effectiveUserDetails = userDetails;
+ if (!effectiveUserDetails) {
+ const uploadLimitState = await onEnsureUploadLimitState?.();
+ effectiveUserDetails = uploadLimitState?.userDetails;
+ }
+
+ if (!effectiveUserDetails) {
+ setError(t("generic_error_retry"));
+ setUpgradeCTAType(null);
+ return;
+ }
+
+ const preflightFailure = validateLockerUploadBatch(
+ pendingUploadItems.map(({ file }) => file),
+ effectiveUserDetails,
+ );
+ if (preflightFailure) {
+ const hitsPaidHardCap = exceedsPaidLockerHardLimit(
+ pendingUploadItems.map(({ file }) => file),
+ effectiveUserDetails,
+ );
+ const preflightFailureMessage = (
+ failure: LockerUploadPreflightFailure,
+ ) => {
+ if (
+ hitsPaidHardCap &&
+ (failure.reason === "fileCountLimit" ||
+ failure.reason === "storageLimit")
+ ) {
+ return t("uploadLockerHardCapErrorBody");
+ }
+ switch (failure.reason) {
+ case "fileCountLimit":
+ return t("uploadFileCountLimitErrorBody");
+ case "fileTooLarge":
+ return t("uploadFileTooLargeErrorBody");
+ case "storageLimit":
+ return t("uploadStorageLimitErrorBody");
+ }
+ };
+
+ setError(preflightFailureMessage(preflightFailure));
+ setUpgradeCTAType(
+ hitsPaidHardCap
+ ? null
+ : preflightFailure.reason === "fileCountLimit"
+ ? "fileCountLimit"
+ : preflightFailure.reason === "storageLimit"
+ ? "storageLimit"
+ : null,
+ );
+ return;
+ }
+
+ setUploading(true);
+ setError(null);
+ setUpgradeCTAType(null);
+ const pendingUploadFileKeys = new Set(
+ pendingUploadItems.map((item) => uploadQueueItemKey(item)),
+ );
+ setFailedFileKeys((current) => {
+ const next = new Set(current);
+ pendingUploadFileKeys.forEach((fileKey) =>
+ next.delete(fileKey),
+ );
+ return next;
+ });
+ setUploadingFileKeys((current) => {
+ const next = new Set(current);
+ pendingUploadFileKeys.forEach((fileKey) =>
+ next.delete(fileKey),
+ );
+ return next;
+ });
+ setUploadProgressByFileKey((current) =>
+ Object.fromEntries(
+ Object.entries(current).filter(
+ ([fileKey]) => !pendingUploadFileKeys.has(fileKey),
+ ),
+ ),
+ );
+ setUploadCapByFileKey((current) =>
+ Object.fromEntries(
+ Object.entries(current).filter(
+ ([fileKey]) => !pendingUploadFileKeys.has(fileKey),
+ ),
+ ),
+ );
+ let uploadedCount = 0;
+ try {
+ const existingNormalizedNameToID = new Map(
+ displayCollections.map((collection) => [
+ normalizeCollectionName(collection.name),
+ collection.id,
+ ]),
+ );
+ const normalizedNameToIDResult = await onEnsureCollections?.(
+ dedupeCollectionNames(
+ Object.values(selectedCollectionNamesByFileKey).flat(),
+ ),
+ );
+ const normalizedNameToID = new Map(existingNormalizedNameToID);
+ if (normalizedNameToIDResult instanceof Map) {
+ for (const [
+ name,
+ id,
+ ] of normalizedNameToIDResult.entries()) {
+ normalizedNameToID.set(
+ normalizeCollectionName(name),
+ id,
+ );
+ }
+ } else {
+ for (const [name, id] of Object.entries(
+ normalizedNameToIDResult ?? {},
+ )) {
+ normalizedNameToID.set(
+ normalizeCollectionName(name),
+ id,
+ );
+ }
+ }
+ const uploadTargets = pendingUploadItems.map((item) => {
+ const fileKey = uploadQueueItemKey(item);
+ const collectionIDs = dedupeCollectionNames(
+ selectedCollectionNamesByFileKey[fileKey] ?? [],
+ )
+ .map((name) =>
+ normalizedNameToID.get(
+ normalizeCollectionName(name),
+ ),
+ )
+ .filter((id): id is number => typeof id === "number");
+ return { item, fileKey, collectionIDs };
+ });
+
+ let nextIndex = 0;
+ const worker = async () => {
+ while (true) {
+ const target = uploadTargets[nextIndex];
+ nextIndex += 1;
+ if (!target) {
+ return;
+ }
+
+ const { item, fileKey, collectionIDs } = target;
+ setUploadingFileKeys((current) =>
+ new Set(current).add(fileKey),
+ );
+ setUploadCapByFileKey((current) => ({
+ ...current,
+ [fileKey]: 90 + Math.floor(Math.random() * 10),
+ }));
+ setUploadProgressByFileKey((current) => ({
+ ...current,
+ [fileKey]: { phase: "preparing" },
+ }));
+
+ try {
+ await onUploadProgress(
+ item.file,
+ collectionIDs,
+ (progress) => {
+ setUploadProgressByFileKey((current) => ({
+ ...current,
+ [fileKey]: progress,
+ }));
+ },
+ );
+ uploadedCount += 1;
+ setCompletedFileKeys((current) =>
+ new Set(current).add(fileKey),
+ );
+ setFailedFileKeys((current) => {
+ const next = new Set(current);
+ next.delete(fileKey);
+ return next;
+ });
+ onUploadItemComplete?.();
+ } catch (error) {
+ log.error("Failed to upload Locker file", error);
+ const formattedError =
+ await formatLockerMutationError(
+ error,
+ "uploadFile",
+ );
+ const nextUpgradeCTAType =
+ await lockerUpgradeCTAType(error);
+ setError((current) => current ?? formattedError);
+ if (nextUpgradeCTAType) {
+ setUpgradeCTAType(
+ (current) => current ?? nextUpgradeCTAType,
+ );
+ }
+ setFailedFileKeys((current) =>
+ new Set(current).add(fileKey),
+ );
+ } finally {
+ setUploadingFileKeys((current) => {
+ const next = new Set(current);
+ next.delete(fileKey);
+ return next;
+ });
+ }
+ }
+ };
+
+ await Promise.all(
+ Array.from({
+ length: Math.min(
+ MAX_PARALLEL_UPLOADS,
+ pendingUploadItems.length,
+ ),
+ }).map(() => worker()),
+ );
+
+ if (uploadedCount > 0) {
+ await onUploadsFinished?.(uploadedCount);
+ }
+ if (
+ completedFileKeys.size + uploadedCount ===
+ selectedUploadItems.length
+ ) {
+ onComplete();
+ }
+ } catch (error) {
+ log.error("Failed to upload Locker files", error);
+ setError(await formatLockerMutationError(error, "uploadFile"));
+ setUpgradeCTAType(await lockerUpgradeCTAType(error));
+ } finally {
+ setUploading(false);
+ }
+ },
+ [
+ completedFileKeys,
+ displayCollections,
+ onEnsureCollections,
+ onEnsureUploadLimitState,
+ onUploadItemComplete,
+ onUploadProgress,
+ onUploadsFinished,
+ selectedCollectionNamesByFileKey,
+ selectedUploadItems,
+ userDetails,
+ setError,
+ setUpgradeCTAType,
+ ],
+ );
+
+ return {
+ uploading,
+ completedFileKeys,
+ failedFileKeys,
+ uploadingFileKeys,
+ uploadProgressByFileKey,
+ uploadCapByFileKey,
+ resetUploadState,
+ upload,
+ };
+}
diff --git a/web/apps/locker/src/components/locker/use-locker-actions.tsx b/web/apps/locker/src/components/locker/use-locker-actions.tsx
index 92b18682a6..32fdde6569 100644
--- a/web/apps/locker/src/components/locker/use-locker-actions.tsx
+++ b/web/apps/locker/src/components/locker/use-locker-actions.tsx
@@ -1,6 +1,5 @@
import {
LOCKER_FILE_LIMIT_PAID,
- LOCKER_MAX_FILE_SIZE_BYTES,
LOCKER_STORAGE_LIMIT_PAID_BYTES,
lockerUploadAllowance,
type LockerUploadLimitState,
@@ -40,31 +39,13 @@ import type { DragEvent } from "react";
import { useCallback, useEffect, useRef, useState } from "react";
import { Trans } from "react-i18next";
import { filterNonEmptyUploadItems } from "../create-item/file-upload-helpers";
+import {
+ collectUploadCandidatesFromDrop,
+ type DragDataTransferItem,
+} from "../create-item/scan-dropped-files";
import type { CreateItemDialogEditItem } from "../create-item/use-create-item-dialog-state";
import type { EmptyTrashDialogState } from "./EmptyTrashDialog";
-type DragDataTransferItem = DataTransferItem & {
- webkitGetAsEntry?: () => FileSystemEntry | null;
-};
-
-type FileSystemFileEntry = FileSystemEntry & {
- file: (
- successCallback: (file: File) => void,
- errorCallback?: (error: DOMException) => void,
- ) => void;
-};
-
-interface FileSystemDirectoryReader {
- readEntries: (
- successCallback: (entries: FileSystemEntry[]) => void,
- errorCallback?: (error: DOMException) => void,
- ) => void;
-}
-
-type FileSystemDirectoryEntry = FileSystemEntry & {
- createReader: () => FileSystemDirectoryReader;
-};
-
export interface ConfirmDialogState {
illustration: string;
title: string;
@@ -89,202 +70,6 @@ const COLLECTION_MUTATION_REFRESH_DELAY_MS = 750;
const UPLOAD_REFRESH_DEBOUNCE_MS = 250;
const UPLOAD_REFRESH_FOLLOW_UP_DELAYS_MS = [1200, 3000, 6000] as const;
-const fileFromEntry = (entry: FileSystemFileEntry) =>
- new Promise<File>((resolve, reject) => {
- entry.file(resolve, reject);
- });
-
-const readDirectoryEntries = (entry: FileSystemDirectoryEntry) =>
- new Promise<FileSystemEntry[]>((resolve, reject) => {
- const reader = entry.createReader();
- const entries: FileSystemEntry[] = [];
- const readBatch = () => {
- reader.readEntries((batch) => {
- if (batch.length === 0) {
- resolve(entries);
- return;
- }
- entries.push(...batch);
- readBatch();
- }, reject);
- };
- readBatch();
- });
-
-const collectionNamesFromRelativePath = (relativePath?: string) => [
- ...new Set((relativePath?.split("/").slice(0, -1) ?? []).filter(Boolean)),
-];
-
-const uploadCandidateFromFile = (
- file: File,
- relativePath?: string,
-): LockerUploadCandidate => ({
- file,
- relativePath,
- suggestedCollectionNames: collectionNamesFromRelativePath(relativePath),
-});
-
-const droppedUploadCandidateKey = (file: File, relativePath?: string) =>
- `${relativePath ?? file.name}:${file.size}:${file.lastModified}`;
-
-interface DropUploadScanOptions {
- remainingFileCount: number;
- freeStorage: number;
-}
-
-interface DropUploadScanResult {
- items: LockerUploadCandidate[];
- preflightFailure?: LockerUploadPreflightFailure;
- scannedNonEmptyFileCount: number;
- scannedTotalSize: number;
-}
-
-type PendingDroppedNode =
- | { type: "entry"; entry: FileSystemEntry; parentPath: string }
- | { type: "file"; file: File; relativePath: string };
-
-const collectUploadCandidatesFromDrop = async (
- droppedItems: DragDataTransferItem[],
- fallbackFiles: File[],
- options: DropUploadScanOptions,
-): Promise<DropUploadScanResult> => {
- const items: LockerUploadCandidate[] = [];
- const pendingNodes: PendingDroppedNode[] = [];
- const seenUploadCandidateKeys = new Set<string>();
- let scannedNonEmptyFileCount = 0;
- let scannedTotalSize = 0;
-
- const addFile = (
- file: File,
- relativePath: string,
- ): LockerUploadPreflightFailure | undefined => {
- if (file.size === 0) {
- return undefined;
- }
-
- const dedupeKey = droppedUploadCandidateKey(file, relativePath);
- if (seenUploadCandidateKeys.has(dedupeKey)) {
- return undefined;
- }
- seenUploadCandidateKeys.add(dedupeKey);
-
- if (file.size > LOCKER_MAX_FILE_SIZE_BYTES) {
- return { reason: "fileTooLarge", fileName: file.name };
- }
-
- scannedNonEmptyFileCount += 1;
- if (scannedNonEmptyFileCount > options.remainingFileCount) {
- return { reason: "fileCountLimit" };
- }
-
- scannedTotalSize += file.size;
- if (scannedTotalSize > options.freeStorage) {
- return { reason: "storageLimit" };
- }
-
- items.push(uploadCandidateFromFile(file, relativePath));
- return undefined;
- };
-
- for (let index = droppedItems.length - 1; index >= 0; index -= 1) {
- const item = droppedItems[index]!;
- if (item.kind !== "file") {
- continue;
- }
-
- const getAsEntry = item.webkitGetAsEntry;
- const entry =
- typeof getAsEntry === "function" ? getAsEntry.call(item) : null;
- if (entry) {
- pendingNodes.push({ type: "entry", entry, parentPath: "" });
- continue;
- }
-
- const file = item.getAsFile();
- if (file) {
- pendingNodes.push({
- type: "file",
- file,
- relativePath: file.webkitRelativePath || file.name,
- });
- }
- }
-
- if (pendingNodes.length === 0) {
- for (const file of fallbackFiles) {
- const preflightFailure = addFile(
- file,
- file.webkitRelativePath || file.name,
- );
- if (preflightFailure) {
- return {
- items,
- preflightFailure,
- scannedNonEmptyFileCount,
- scannedTotalSize,
- };
- }
- }
- return { items, scannedNonEmptyFileCount, scannedTotalSize };
- }
-
- while (pendingNodes.length > 0) {
- const node = pendingNodes.pop()!;
-
- if (node.type === "file") {
- const preflightFailure = addFile(node.file, node.relativePath);
- if (preflightFailure) {
- return {
- items,
- preflightFailure,
- scannedNonEmptyFileCount,
- scannedTotalSize,
- };
- }
- continue;
- }
-
- if (node.entry.isFile) {
- const file = await fileFromEntry(node.entry as FileSystemFileEntry);
- const relativePath = node.parentPath
- ? `${node.parentPath}/${file.name}`
- : file.name;
- const preflightFailure = addFile(file, relativePath);
- if (preflightFailure) {
- return {
- items,
- preflightFailure,
- scannedNonEmptyFileCount,
- scannedTotalSize,
- };
- }
- continue;
- }
-
- if (node.entry.isDirectory) {
- const directoryPath = node.parentPath
- ? `${node.parentPath}/${node.entry.name}`
- : node.entry.name;
- const directoryEntries = await readDirectoryEntries(
- node.entry as FileSystemDirectoryEntry,
- );
- for (
- let index = directoryEntries.length - 1;
- index >= 0;
- index -= 1
- ) {
- pendingNodes.push({
- type: "entry",
- entry: directoryEntries[index]!,
- parentPath: directoryPath,
- });
- }
- }
- }
-
- return { items, scannedNonEmptyFileCount, scannedTotalSize };
-};
-
const collectionIDsForItemMutation = (
item: LockerItem,
selectedCollectionID: number | null,
diff --git a/web/apps/locker/tests/dropped-files.test.ts b/web/apps/locker/tests/dropped-files.test.ts
new file mode 100644
index 0000000000..b0efbbe60f
--- /dev/null
+++ b/web/apps/locker/tests/dropped-files.test.ts
@@ -0,0 +1,163 @@
+import { expect, test, vi } from "vitest";
+import {
+ collectUploadCandidatesFromDrop,
+ type DragDataTransferItem,
+} from "../src/components/create-item/scan-dropped-files";
+import { LOCKER_MAX_FILE_SIZE_BYTES } from "../src/services/locker-limits";
+
+const allowance = { remainingFileCount: 10, freeStorage: 100 };
+const file = (name: string, size = 1) =>
+ new File(["x".repeat(size)], name, { lastModified: 1 });
+const fileEntry = (value: File) =>
+ ({
+ isFile: true,
+ isDirectory: false,
+ name: value.name,
+ file: (resolve: (file: File) => void) => resolve(value),
+ }) as unknown as FileSystemEntry;
+const directory = (name: string, batches: FileSystemEntry[][]) =>
+ ({
+ isFile: false,
+ isDirectory: true,
+ name,
+ createReader: () => {
+ let index = 0;
+ return {
+ readEntries: (resolve: (entries: FileSystemEntry[]) => void) =>
+ resolve(batches[index++] ?? []),
+ };
+ },
+ }) as unknown as FileSystemEntry;
+const droppedEntry = (entry: FileSystemEntry) =>
+ ({ kind: "file", webkitGetAsEntry: () => entry }) as DragDataTransferItem;
+
+test("reads every directory batch in order and preserves nested paths", async () => {
+ const root = directory("root", [
+ [
+ fileEntry(file("first")),
+ directory("child", [[fileEntry(file("second"))]]),
+ ],
+ [fileEntry(file("third"))],
+ ]);
+ const result = await collectUploadCandidatesFromDrop(
+ [droppedEntry(root)],
+ [],
+ allowance,
+ );
+ expect(
+ result.items.map(({ relativePath, suggestedCollectionNames }) => ({
+ relativePath,
+ suggestedCollectionNames,
+ })),
+ ).toEqual([
+ { relativePath: "root/first", suggestedCollectionNames: ["root"] },
+ {
+ relativePath: "root/child/second",
+ suggestedCollectionNames: ["root", "child"],
+ },
+ { relativePath: "root/third", suggestedCollectionNames: ["root"] },
+ ]);
+ expect(result.scannedNonEmptyFileCount).toBe(3);
+ expect(result.scannedTotalSize).toBe(3);
+});
+
+test("fallback skips empty files and duplicate path/size/timestamp entries", async () => {
+ const result = await collectUploadCandidatesFromDrop(
+ [],
+ [file("empty", 0), file("same"), file("same"), file("last")],
+ allowance,
+ );
+ expect(result.items.map(({ file }) => file.name)).toEqual(["same", "last"]);
+ expect(result.scannedNonEmptyFileCount).toBe(2);
+ expect(result.scannedTotalSize).toBe(2);
+});
+
+test("uses getAsFile without also ingesting fallback files", async () => {
+ const value = file("selected");
+ const item = {
+ kind: "file",
+ getAsFile: () => value,
+ } as DragDataTransferItem;
+ const result = await collectUploadCandidatesFromDrop(
+ [item],
+ [file("fallback")],
+ allowance,
+ );
+ expect(result.items.map(({ file }) => file.name)).toEqual(["selected"]);
+});
+
+test.each([
+ {
+ options: { remainingFileCount: 1, freeStorage: 100 },
+ reason: "fileCountLimit",
+ count: 2,
+ size: 1,
+ },
+ {
+ options: { remainingFileCount: 10, freeStorage: 1 },
+ reason: "storageLimit",
+ count: 2,
+ size: 2,
+ },
+])("stops scanning at $reason", async ({ options, reason, count, size }) => {
+ const unread = vi.fn();
+ const later = { isFile: true, file: unread } as unknown as FileSystemEntry;
+ const result = await collectUploadCandidatesFromDrop(
+ [
+ droppedEntry(fileEntry(file("first"))),
+ droppedEntry(fileEntry(file("second"))),
+ droppedEntry(later),
+ ],
+ [],
+ options,
+ );
+ expect(result.preflightFailure).toEqual({ reason });
+ expect(result.items).toHaveLength(1);
+ expect(result.scannedNonEmptyFileCount).toBe(count);
+ expect(result.scannedTotalSize).toBe(size);
+ expect(unread).not.toHaveBeenCalled();
+});
+
+test("rejects oversized files before counting them", async () => {
+ const oversized = file("large");
+ Object.defineProperty(oversized, "size", {
+ value: LOCKER_MAX_FILE_SIZE_BYTES + 1,
+ });
+ const result = await collectUploadCandidatesFromDrop(
+ [],
+ [oversized],
+ allowance,
+ );
+ expect(result).toEqual({
+ items: [],
+ preflightFailure: { reason: "fileTooLarge", fileName: "large" },
+ scannedNonEmptyFileCount: 0,
+ scannedTotalSize: 0,
+ });
+});
+
+test("accepts exact count and storage boundaries", async () => {
+ const result = await collectUploadCandidatesFromDrop([], [file("one")], {
+ remainingFileCount: 1,
+ freeStorage: 1,
+ });
+ expect(result.preflightFailure).toBeUndefined();
+ expect(result.items).toHaveLength(1);
+});
+
+test("propagates directory read failures", async () => {
+ const failure = new DOMException("Cannot read directory");
+ const entry = {
+ isDirectory: true,
+ name: "blocked",
+ createReader: () => ({
+ readEntries: (
+ _resolve: unknown,
+ reject: (error: DOMException) => void,
+ ) => reject(failure),
+ }),
+ } as unknown as FileSystemEntry;
+ await expect(
+ collectUploadCandidatesFromDrop([droppedEntry(entry)], [], allowance),
+ ).rejects.toBe(failure);
+});