π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
refractor(locker):split-hooks
Aswin Asok61309a167adc
diff --git a/web/apps/locker/src/components/create-item/scan-dropped-files.ts b/web/apps/locker/src/components/create-item/scan-dropped-files.tsnew file mode 100644index 0000000000..ff4188c57a--- /dev/null+++ b/web/apps/locker/src/components/create-item/scan-dropped-files.ts@@ -0,0 +1,223 @@+import type { LockerUploadCandidate } from "@/types";+import {+ LOCKER_MAX_FILE_SIZE_BYTES,+ type LockerUploadPreflightFailure,+} from "../../services/locker-limits";++export type DragDataTransferItem = DataTransferItem & {+ webkitGetAsEntry?: () => FileSystemEntry | null;+};++type FileSystemFileEntry = FileSystemEntry & {+ file: (+ successCallback: (file: File) => void,+ errorCallback?: (error: DOMException) => void,+ ) => void;+};++interface FileSystemDirectoryReader {+ readEntries: (+ successCallback: (entries: FileSystemEntry[]) => void,+ errorCallback?: (error: DOMException) => void,+ ) => void;+}++type FileSystemDirectoryEntry = FileSystemEntry & {+ createReader: () => FileSystemDirectoryReader;+};++const fileFromEntry = (entry: FileSystemFileEntry) =>+ new Promise<File>((resolve, reject) => {+ entry.file(resolve, reject);+ });++const readDirectoryEntries = (entry: FileSystemDirectoryEntry) =>+ new Promise<FileSystemEntry[]>((resolve, reject) => {+ const reader = entry.createReader();+ const entries: FileSystemEntry[] = [];+ const readBatch = () => {+ reader.readEntries((batch) => {+ if (batch.length === 0) {+ resolve(entries);+ return;+ }+ entries.push(...batch);+ readBatch();+ }, reject);+ };+ readBatch();+ });++const collectionNamesFromRelativePath = (relativePath?: string) => [+ ...new Set((relativePath?.split("/").slice(0, -1) ?? []).filter(Boolean)),+];++const uploadCandidateFromFile = (+ file: File,+ relativePath?: string,+): LockerUploadCandidate => ({+ file,+ relativePath,+ suggestedCollectionNames: collectionNamesFromRelativePath(relativePath),+});++const droppedUploadCandidateKey = (file: File, relativePath?: string) =>+ `${relativePath ?? file.name}:${file.size}:${file.lastModified}`;++interface DropUploadScanOptions {+ remainingFileCount: number;+ freeStorage: number;+}++interface DropUploadScanResult {+ items: LockerUploadCandidate[];+ preflightFailure?: LockerUploadPreflightFailure;+ scannedNonEmptyFileCount: number;+ scannedTotalSize: number;+}++type PendingDroppedNode =+ | { type: "entry"; entry: FileSystemEntry; parentPath: string }+ | { type: "file"; file: File; relativePath: string };++export const collectUploadCandidatesFromDrop = async (+ droppedItems: DragDataTransferItem[],+ fallbackFiles: File[],+ options: DropUploadScanOptions,+): Promise<DropUploadScanResult> => {+ const items: LockerUploadCandidate[] = [];+ const pendingNodes: PendingDroppedNode[] = [];+ const seenUploadCandidateKeys = new Set<string>();+ let scannedNonEmptyFileCount = 0;+ let scannedTotalSize = 0;++ const addFile = (+ file: File,+ relativePath: string,+ ): LockerUploadPreflightFailure | undefined => {+ if (file.size === 0) {+ return undefined;+ }++ const dedupeKey = droppedUploadCandidateKey(file, relativePath);+ if (seenUploadCandidateKeys.has(dedupeKey)) {+ return undefined;+ }+ seenUploadCandidateKeys.add(dedupeKey);++ if (file.size > LOCKER_MAX_FILE_SIZE_BYTES) {+ return { reason: "fileTooLarge", fileName: file.name };+ }++ scannedNonEmptyFileCount += 1;+ if (scannedNonEmptyFileCount > options.remainingFileCount) {+ return { reason: "fileCountLimit" };+ }++ scannedTotalSize += file.size;+ if (scannedTotalSize > options.freeStorage) {+ return { reason: "storageLimit" };+ }++ items.push(uploadCandidateFromFile(file, relativePath));+ return undefined;+ };++ for (let index = droppedItems.length - 1; index >= 0; index -= 1) {+ const item = droppedItems[index]!;+ if (item.kind !== "file") {+ continue;+ }++ const getAsEntry = item.webkitGetAsEntry;+ const entry =+ typeof getAsEntry === "function" ? getAsEntry.call(item) : null;+ if (entry) {+ pendingNodes.push({ type: "entry", entry, parentPath: "" });+ continue;+ }++ const file = item.getAsFile();+ if (file) {+ pendingNodes.push({+ type: "file",+ file,+ relativePath: file.webkitRelativePath || file.name,+ });+ }+ }++ if (pendingNodes.length === 0) {+ for (const file of fallbackFiles) {+ const preflightFailure = addFile(+ file,+ file.webkitRelativePath || file.name,+ );+ if (preflightFailure) {+ return {+ items,+ preflightFailure,+ scannedNonEmptyFileCount,+ scannedTotalSize,+ };+ }+ }+ return { items, scannedNonEmptyFileCount, scannedTotalSize };+ }++ while (pendingNodes.length > 0) {+ const node = pendingNodes.pop()!;++ if (node.type === "file") {+ const preflightFailure = addFile(node.file, node.relativePath);+ if (preflightFailure) {+ return {+ items,+ preflightFailure,+ scannedNonEmptyFileCount,+ scannedTotalSize,+ };+ }+ continue;+ }++ if (node.entry.isFile) {+ const file = await fileFromEntry(node.entry as FileSystemFileEntry);+ const relativePath = node.parentPath+ ? `${node.parentPath}/${file.name}`+ : file.name;+ const preflightFailure = addFile(file, relativePath);+ if (preflightFailure) {+ return {+ items,+ preflightFailure,+ scannedNonEmptyFileCount,+ scannedTotalSize,+ };+ }+ continue;+ }++ if (node.entry.isDirectory) {+ const directoryPath = node.parentPath+ ? `${node.parentPath}/${node.entry.name}`+ : node.entry.name;+ const directoryEntries = await readDirectoryEntries(+ node.entry as FileSystemDirectoryEntry,+ );+ for (+ let index = directoryEntries.length - 1;+ index >= 0;+ index -= 1+ ) {+ pendingNodes.push({+ type: "entry",+ entry: directoryEntries[index]!,+ parentPath: directoryPath,+ });+ }+ }+ }++ return { items, scannedNonEmptyFileCount, scannedTotalSize };+};diff --git a/web/apps/locker/src/components/create-item/use-create-item-dialog-state.ts b/web/apps/locker/src/components/create-item/use-create-item-dialog-state.tsindex b5ab561b25..0b21a8afea 100644--- a/web/apps/locker/src/components/create-item/use-create-item-dialog-state.ts+++ b/web/apps/locker/src/components/create-item/use-create-item-dialog-state.ts@@ -3,13 +3,7 @@ import {lockerUpgradeCTAType,type LockerUpgradeCTAType,} from "@/services/locker-errors";-import {- exceedsPaidLockerHardLimit,- validateLockerUploadBatch,- type LockerUploadLimitState,- type LockerUploadPreflightFailure,-} from "@/services/locker-limits";-import type { LockerUploadProgress } from "@/services/remote";+import type { LockerUploadLimitState } from "@/services/locker-limits";import type {LockerCollection,LockerItemType,@@ -27,15 +21,13 @@ import type { ChangeEvent } from "react";import { useCallback, useEffect, useMemo, useRef, useState } from "react";import {collectionNamesByUploadItem,- dedupeCollectionNames,filterNonEmptyUploadItems,- normalizeCollectionName,uploadQueueItemKey,} from "./file-upload-helpers";import { getRequiredFields } from "./item-form-fields-utils";+import { useUploadQueue, type LockerUploadCallbacks } from "./use-upload-queue";export type CreateOption = LockerItemType;-const MAX_PARALLEL_UPLOADS = 4;export interface CreateItemDialogEditItem {id: number;@@ -45,7 +37,7 @@ export interface CreateItemDialogEditItem {collectionIDs: number[];}-interface UseCreateItemDialogStateProps {+interface UseCreateItemDialogStateProps extends LockerUploadCallbacks {open: boolean;collections: LockerCollection[];onClose: () => void;@@ -54,41 +46,12 @@ interface UseCreateItemDialogStateProps {data: Record<string, unknown>,collectionIDs: number[],) => Promise<void>;- onUploadProgress?: (- file: File,- collectionIDs: number[],- onProgress: (progress: LockerUploadProgress) => void,- ) => Promise<void>;- onUploadItemComplete?: () => void;- onUploadsFinished?: (uploadedCount: number) => Promise<void>;- onEnsureCollections?: (- names: string[],- ) => Promise<Map<string, number> | Record<string, number>>;- onEnsureUploadLimitState?: () => Promise<- { userDetails: LockerUploadLimitState } | undefined- >;defaultCollectionID?: number | null;initialItems?: LockerUploadCandidate[];editItem?: CreateItemDialogEditItem | null;userDetails?: LockerUploadLimitState;}-interface UploadState {- completedFileKeys: Set<string>;- failedFileKeys: Set<string>;- uploadingFileKeys: Set<string>;- uploadProgressByFileKey: Record<string, LockerUploadProgress | null>;- uploadCapByFileKey: Record<string, number>;-}--const emptyUploadState = (): UploadState => ({- completedFileKeys: new Set(),- failedFileKeys: new Set(),- uploadingFileKeys: new Set(),- uploadProgressByFileKey: {},- uploadCapByFileKey: {},-});-const initialEditCollectionIDs = (editItem: CreateItemDialogEditItem | null | undefined,) => {@@ -167,7 +130,6 @@ export const useCreateItemDialogState = ({const [selectedCollectionIDs, setSelectedCollectionIDs] =useState<number[]>(editCollectionIDs);const [saving, setSaving] = useState(false);- const [uploading, setUploading] = useState(false);const [error, setError] = useState<string | null>(null);const [upgradeCTAType, setUpgradeCTAType] =useState<LockerUpgradeCTAType | null>(null);@@ -185,21 +147,28 @@ export const useCreateItemDialogState = ({const [customCollectionNames, setCustomCollectionNames] = useState<string[]>([]);- const [completedFileKeys, setCompletedFileKeys] = useState<Set<string>>(- () => new Set(),- );- const [failedFileKeys, setFailedFileKeys] = useState<Set<string>>(- () => new Set(),- );- const [uploadingFileKeys, setUploadingFileKeys] = useState<Set<string>>(- () => new Set(),- );- const [uploadProgressByFileKey, setUploadProgressByFileKey] = useState<- Record<string, LockerUploadProgress | null>- >({});- const [uploadCapByFileKey, setUploadCapByFileKey] = useState<- Record<string, number>- >({});+ const {+ uploading,+ completedFileKeys,+ failedFileKeys,+ uploadingFileKeys,+ uploadProgressByFileKey,+ uploadCapByFileKey,+ resetUploadState,+ upload,+ } = useUploadQueue({+ displayCollections,+ selectedUploadItems,+ selectedCollectionNamesByFileKey,+ userDetails,+ setError,+ setUpgradeCTAType,+ onUploadProgress,+ onUploadItemComplete,+ onUploadsFinished,+ onEnsureCollections,+ onEnsureUploadLimitState,+ });const isFileMode = selectedOption === "file";const editCollectionID = editItem?.collectionID ?? null;@@ -223,15 +192,6 @@ export const useCreateItemDialogState = ({[],);- const resetUploadState = useCallback(() => {- const nextState = emptyUploadState();- setCompletedFileKeys(nextState.completedFileKeys);- setFailedFileKeys(nextState.failedFileKeys);- setUploadingFileKeys(nextState.uploadingFileKeys);- setUploadProgressByFileKey(nextState.uploadProgressByFileKey);- setUploadCapByFileKey(nextState.uploadCapByFileKey);- }, []);-useEffect(() => {if (!open) {return;@@ -461,249 +421,10 @@ export const useCreateItemDialogState = ({}}, [formData, formType, handleClose, onSave, selectedCollectionIDs]);- const handleUpload = useCallback(async () => {- if (selectedUploadItems.length === 0 || !onUploadProgress) {- return;- }-- const pendingUploadItems = filterNonEmptyUploadItems(- selectedUploadItems.filter(- (item) => !completedFileKeys.has(uploadQueueItemKey(item)),- ),- );- if (pendingUploadItems.length === 0) {- return;- }-- let effectiveUserDetails = userDetails;- if (!effectiveUserDetails) {- const uploadLimitState = await onEnsureUploadLimitState?.();- effectiveUserDetails = uploadLimitState?.userDetails;- }-- if (!effectiveUserDetails) {- setError(t("generic_error_retry"));- setUpgradeCTAType(null);- return;- }-- const preflightFailure = validateLockerUploadBatch(- pendingUploadItems.map(({ file }) => file),- effectiveUserDetails,- );- if (preflightFailure) {- const hitsPaidHardCap = exceedsPaidLockerHardLimit(- pendingUploadItems.map(({ file }) => file),- effectiveUserDetails,- );- const preflightFailureMessage = (- failure: LockerUploadPreflightFailure,- ) => {- if (- hitsPaidHardCap &&- (failure.reason === "fileCountLimit" ||- failure.reason === "storageLimit")- ) {- return t("uploadLockerHardCapErrorBody");- }- switch (failure.reason) {- case "fileCountLimit":- return t("uploadFileCountLimitErrorBody");- case "fileTooLarge":- return t("uploadFileTooLargeErrorBody");- case "storageLimit":- return t("uploadStorageLimitErrorBody");- }- };-- setError(preflightFailureMessage(preflightFailure));- setUpgradeCTAType(- hitsPaidHardCap- ? null- : preflightFailure.reason === "fileCountLimit"- ? "fileCountLimit"- : preflightFailure.reason === "storageLimit"- ? "storageLimit"- : null,- );- return;- }-- setUploading(true);- setError(null);- setUpgradeCTAType(null);- const pendingUploadFileKeys = new Set(- pendingUploadItems.map((item) => uploadQueueItemKey(item)),- );- setFailedFileKeys((current) => {- const next = new Set(current);- pendingUploadFileKeys.forEach((fileKey) => next.delete(fileKey));- return next;- });- setUploadingFileKeys((current) => {- const next = new Set(current);- pendingUploadFileKeys.forEach((fileKey) => next.delete(fileKey));- return next;- });- setUploadProgressByFileKey((current) =>- Object.fromEntries(- Object.entries(current).filter(- ([fileKey]) => !pendingUploadFileKeys.has(fileKey),- ),- ),- );- setUploadCapByFileKey((current) =>- Object.fromEntries(- Object.entries(current).filter(- ([fileKey]) => !pendingUploadFileKeys.has(fileKey),- ),- ),- );- let uploadedCount = 0;- try {- const existingNormalizedNameToID = new Map(- displayCollections.map((collection) => [- normalizeCollectionName(collection.name),- collection.id,- ]),- );- const normalizedNameToIDResult = await onEnsureCollections?.(- dedupeCollectionNames(- Object.values(selectedCollectionNamesByFileKey).flat(),- ),- );- const normalizedNameToID = new Map(existingNormalizedNameToID);- if (normalizedNameToIDResult instanceof Map) {- for (const [name, id] of normalizedNameToIDResult.entries()) {- normalizedNameToID.set(normalizeCollectionName(name), id);- }- } else {- for (const [name, id] of Object.entries(- normalizedNameToIDResult ?? {},- )) {- normalizedNameToID.set(normalizeCollectionName(name), id);- }- }- const uploadTargets = pendingUploadItems.map((item) => {- const fileKey = uploadQueueItemKey(item);- const collectionIDs = dedupeCollectionNames(- selectedCollectionNamesByFileKey[fileKey] ?? [],- )- .map((name) =>- normalizedNameToID.get(normalizeCollectionName(name)),- )- .filter((id): id is number => typeof id === "number");- return { item, fileKey, collectionIDs };- });-- let nextIndex = 0;- const worker = async () => {- while (true) {- const target = uploadTargets[nextIndex];- nextIndex += 1;- if (!target) {- return;- }-- const { item, fileKey, collectionIDs } = target;- setUploadingFileKeys((current) =>- new Set(current).add(fileKey),- );- setUploadCapByFileKey((current) => ({- ...current,- [fileKey]: 90 + Math.floor(Math.random() * 10),- }));- setUploadProgressByFileKey((current) => ({- ...current,- [fileKey]: { phase: "preparing" },- }));-- try {- await onUploadProgress(- item.file,- collectionIDs,- (progress) => {- setUploadProgressByFileKey((current) => ({- ...current,- [fileKey]: progress,- }));- },- );- uploadedCount += 1;- setCompletedFileKeys((current) =>- new Set(current).add(fileKey),- );- setFailedFileKeys((current) => {- const next = new Set(current);- next.delete(fileKey);- return next;- });- onUploadItemComplete?.();- } catch (error) {- log.error("Failed to upload Locker file", error);- const formattedError = await formatLockerMutationError(- error,- "uploadFile",- );- const nextUpgradeCTAType =- await lockerUpgradeCTAType(error);- setError((current) => current ?? formattedError);- if (nextUpgradeCTAType) {- setUpgradeCTAType(- (current) => current ?? nextUpgradeCTAType,- );- }- setFailedFileKeys((current) =>- new Set(current).add(fileKey),- );- } finally {- setUploadingFileKeys((current) => {- const next = new Set(current);- next.delete(fileKey);- return next;- });- }- }- };-- await Promise.all(- Array.from({- length: Math.min(- MAX_PARALLEL_UPLOADS,- pendingUploadItems.length,- ),- }).map(() => worker()),- );-- if (uploadedCount > 0) {- await onUploadsFinished?.(uploadedCount);- }- if (- completedFileKeys.size + uploadedCount ===- selectedUploadItems.length- ) {- handleClose();- }- } catch (error) {- log.error("Failed to upload Locker files", error);- setError(await formatLockerMutationError(error, "uploadFile"));- setUpgradeCTAType(await lockerUpgradeCTAType(error));- } finally {- setUploading(false);- }- }, [- completedFileKeys,- displayCollections,- handleClose,- onEnsureCollections,- onEnsureUploadLimitState,- onUploadItemComplete,- onUploadProgress,- onUploadsFinished,- selectedCollectionNamesByFileKey,- selectedUploadItems,- userDetails,- ]);+ const handleUpload = useCallback(+ () => upload(handleClose),+ [upload, handleClose],+ );const canSave =formType !== null &&diff --git a/web/apps/locker/src/components/create-item/use-upload-queue.ts b/web/apps/locker/src/components/create-item/use-upload-queue.tsnew file mode 100644index 0000000000..dba9db8aa2--- /dev/null+++ b/web/apps/locker/src/components/create-item/use-upload-queue.ts@@ -0,0 +1,385 @@+import {+ formatLockerMutationError,+ lockerUpgradeCTAType,+ type LockerUpgradeCTAType,+} from "@/services/locker-errors";+import {+ exceedsPaidLockerHardLimit,+ validateLockerUploadBatch,+ type LockerUploadLimitState,+ type LockerUploadPreflightFailure,+} from "@/services/locker-limits";+import type { LockerUploadProgress } from "@/services/remote";+import type { LockerCollection, LockerUploadCandidate } from "@/types";+import log from "ente-base/log";+import { t } from "i18next";+import {+ useCallback,+ useState,+ type Dispatch,+ type SetStateAction,+} from "react";+import {+ dedupeCollectionNames,+ filterNonEmptyUploadItems,+ normalizeCollectionName,+ uploadQueueItemKey,+} from "./file-upload-helpers";++const MAX_PARALLEL_UPLOADS = 4;++export interface LockerUploadCallbacks {+ onUploadProgress?: (+ file: File,+ collectionIDs: number[],+ onProgress: (progress: LockerUploadProgress) => void,+ ) => Promise<void>;+ onUploadItemComplete?: () => void;+ onUploadsFinished?: (uploadedCount: number) => Promise<void>;+ onEnsureCollections?: (+ names: string[],+ ) => Promise<Map<string, number> | Record<string, number>>;+ onEnsureUploadLimitState?: () => Promise<+ { userDetails: LockerUploadLimitState } | undefined+ >;+}++interface UseUploadQueueProps extends LockerUploadCallbacks {+ displayCollections: LockerCollection[];+ selectedUploadItems: LockerUploadCandidate[];+ selectedCollectionNamesByFileKey: Record<string, string[]>;+ userDetails?: LockerUploadLimitState;+ setError: Dispatch<SetStateAction<string | null>>;+ setUpgradeCTAType: Dispatch<SetStateAction<LockerUpgradeCTAType | null>>;+}++interface UploadState {+ completedFileKeys: Set<string>;+ failedFileKeys: Set<string>;+ uploadingFileKeys: Set<string>;+ uploadProgressByFileKey: Record<string, LockerUploadProgress | null>;+ uploadCapByFileKey: Record<string, number>;+}++const emptyUploadState = (): UploadState => ({+ completedFileKeys: new Set(),+ failedFileKeys: new Set(),+ uploadingFileKeys: new Set(),+ uploadProgressByFileKey: {},+ uploadCapByFileKey: {},+});++export function useUploadQueue({+ displayCollections,+ selectedUploadItems,+ selectedCollectionNamesByFileKey,+ userDetails,+ setError,+ setUpgradeCTAType,+ onUploadProgress,+ onUploadItemComplete,+ onUploadsFinished,+ onEnsureCollections,+ onEnsureUploadLimitState,+}: UseUploadQueueProps) {+ const [uploading, setUploading] = useState(false);+ const [completedFileKeys, setCompletedFileKeys] = useState<Set<string>>(+ () => new Set(),+ );+ const [failedFileKeys, setFailedFileKeys] = useState<Set<string>>(+ () => new Set(),+ );+ const [uploadingFileKeys, setUploadingFileKeys] = useState<Set<string>>(+ () => new Set(),+ );+ const [uploadProgressByFileKey, setUploadProgressByFileKey] = useState<+ Record<string, LockerUploadProgress | null>+ >({});+ const [uploadCapByFileKey, setUploadCapByFileKey] = useState<+ Record<string, number>+ >({});++ const resetUploadState = useCallback(() => {+ const nextState = emptyUploadState();+ setCompletedFileKeys(nextState.completedFileKeys);+ setFailedFileKeys(nextState.failedFileKeys);+ setUploadingFileKeys(nextState.uploadingFileKeys);+ setUploadProgressByFileKey(nextState.uploadProgressByFileKey);+ setUploadCapByFileKey(nextState.uploadCapByFileKey);+ }, []);++ const upload = useCallback(+ async (onComplete: () => void) => {+ if (selectedUploadItems.length === 0 || !onUploadProgress) {+ return;+ }++ const pendingUploadItems = filterNonEmptyUploadItems(+ selectedUploadItems.filter(+ (item) => !completedFileKeys.has(uploadQueueItemKey(item)),+ ),+ );+ if (pendingUploadItems.length === 0) {+ return;+ }++ let effectiveUserDetails = userDetails;+ if (!effectiveUserDetails) {+ const uploadLimitState = await onEnsureUploadLimitState?.();+ effectiveUserDetails = uploadLimitState?.userDetails;+ }++ if (!effectiveUserDetails) {+ setError(t("generic_error_retry"));+ setUpgradeCTAType(null);+ return;+ }++ const preflightFailure = validateLockerUploadBatch(+ pendingUploadItems.map(({ file }) => file),+ effectiveUserDetails,+ );+ if (preflightFailure) {+ const hitsPaidHardCap = exceedsPaidLockerHardLimit(+ pendingUploadItems.map(({ file }) => file),+ effectiveUserDetails,+ );+ const preflightFailureMessage = (+ failure: LockerUploadPreflightFailure,+ ) => {+ if (+ hitsPaidHardCap &&+ (failure.reason === "fileCountLimit" ||+ failure.reason === "storageLimit")+ ) {+ return t("uploadLockerHardCapErrorBody");+ }+ switch (failure.reason) {+ case "fileCountLimit":+ return t("uploadFileCountLimitErrorBody");+ case "fileTooLarge":+ return t("uploadFileTooLargeErrorBody");+ case "storageLimit":+ return t("uploadStorageLimitErrorBody");+ }+ };++ setError(preflightFailureMessage(preflightFailure));+ setUpgradeCTAType(+ hitsPaidHardCap+ ? null+ : preflightFailure.reason === "fileCountLimit"+ ? "fileCountLimit"+ : preflightFailure.reason === "storageLimit"+ ? "storageLimit"+ : null,+ );+ return;+ }++ setUploading(true);+ setError(null);+ setUpgradeCTAType(null);+ const pendingUploadFileKeys = new Set(+ pendingUploadItems.map((item) => uploadQueueItemKey(item)),+ );+ setFailedFileKeys((current) => {+ const next = new Set(current);+ pendingUploadFileKeys.forEach((fileKey) =>+ next.delete(fileKey),+ );+ return next;+ });+ setUploadingFileKeys((current) => {+ const next = new Set(current);+ pendingUploadFileKeys.forEach((fileKey) =>+ next.delete(fileKey),+ );+ return next;+ });+ setUploadProgressByFileKey((current) =>+ Object.fromEntries(+ Object.entries(current).filter(+ ([fileKey]) => !pendingUploadFileKeys.has(fileKey),+ ),+ ),+ );+ setUploadCapByFileKey((current) =>+ Object.fromEntries(+ Object.entries(current).filter(+ ([fileKey]) => !pendingUploadFileKeys.has(fileKey),+ ),+ ),+ );+ let uploadedCount = 0;+ try {+ const existingNormalizedNameToID = new Map(+ displayCollections.map((collection) => [+ normalizeCollectionName(collection.name),+ collection.id,+ ]),+ );+ const normalizedNameToIDResult = await onEnsureCollections?.(+ dedupeCollectionNames(+ Object.values(selectedCollectionNamesByFileKey).flat(),+ ),+ );+ const normalizedNameToID = new Map(existingNormalizedNameToID);+ if (normalizedNameToIDResult instanceof Map) {+ for (const [+ name,+ id,+ ] of normalizedNameToIDResult.entries()) {+ normalizedNameToID.set(+ normalizeCollectionName(name),+ id,+ );+ }+ } else {+ for (const [name, id] of Object.entries(+ normalizedNameToIDResult ?? {},+ )) {+ normalizedNameToID.set(+ normalizeCollectionName(name),+ id,+ );+ }+ }+ const uploadTargets = pendingUploadItems.map((item) => {+ const fileKey = uploadQueueItemKey(item);+ const collectionIDs = dedupeCollectionNames(+ selectedCollectionNamesByFileKey[fileKey] ?? [],+ )+ .map((name) =>+ normalizedNameToID.get(+ normalizeCollectionName(name),+ ),+ )+ .filter((id): id is number => typeof id === "number");+ return { item, fileKey, collectionIDs };+ });++ let nextIndex = 0;+ const worker = async () => {+ while (true) {+ const target = uploadTargets[nextIndex];+ nextIndex += 1;+ if (!target) {+ return;+ }++ const { item, fileKey, collectionIDs } = target;+ setUploadingFileKeys((current) =>+ new Set(current).add(fileKey),+ );+ setUploadCapByFileKey((current) => ({+ ...current,+ [fileKey]: 90 + Math.floor(Math.random() * 10),+ }));+ setUploadProgressByFileKey((current) => ({+ ...current,+ [fileKey]: { phase: "preparing" },+ }));++ try {+ await onUploadProgress(+ item.file,+ collectionIDs,+ (progress) => {+ setUploadProgressByFileKey((current) => ({+ ...current,+ [fileKey]: progress,+ }));+ },+ );+ uploadedCount += 1;+ setCompletedFileKeys((current) =>+ new Set(current).add(fileKey),+ );+ setFailedFileKeys((current) => {+ const next = new Set(current);+ next.delete(fileKey);+ return next;+ });+ onUploadItemComplete?.();+ } catch (error) {+ log.error("Failed to upload Locker file", error);+ const formattedError =+ await formatLockerMutationError(+ error,+ "uploadFile",+ );+ const nextUpgradeCTAType =+ await lockerUpgradeCTAType(error);+ setError((current) => current ?? formattedError);+ if (nextUpgradeCTAType) {+ setUpgradeCTAType(+ (current) => current ?? nextUpgradeCTAType,+ );+ }+ setFailedFileKeys((current) =>+ new Set(current).add(fileKey),+ );+ } finally {+ setUploadingFileKeys((current) => {+ const next = new Set(current);+ next.delete(fileKey);+ return next;+ });+ }+ }+ };++ await Promise.all(+ Array.from({+ length: Math.min(+ MAX_PARALLEL_UPLOADS,+ pendingUploadItems.length,+ ),+ }).map(() => worker()),+ );++ if (uploadedCount > 0) {+ await onUploadsFinished?.(uploadedCount);+ }+ if (+ completedFileKeys.size + uploadedCount ===+ selectedUploadItems.length+ ) {+ onComplete();+ }+ } catch (error) {+ log.error("Failed to upload Locker files", error);+ setError(await formatLockerMutationError(error, "uploadFile"));+ setUpgradeCTAType(await lockerUpgradeCTAType(error));+ } finally {+ setUploading(false);+ }+ },+ [+ completedFileKeys,+ displayCollections,+ onEnsureCollections,+ onEnsureUploadLimitState,+ onUploadItemComplete,+ onUploadProgress,+ onUploadsFinished,+ selectedCollectionNamesByFileKey,+ selectedUploadItems,+ userDetails,+ setError,+ setUpgradeCTAType,+ ],+ );++ return {+ uploading,+ completedFileKeys,+ failedFileKeys,+ uploadingFileKeys,+ uploadProgressByFileKey,+ uploadCapByFileKey,+ resetUploadState,+ upload,+ };+}diff --git a/web/apps/locker/src/components/locker/use-locker-actions.tsx b/web/apps/locker/src/components/locker/use-locker-actions.tsxindex 92b18682a6..32fdde6569 100644--- a/web/apps/locker/src/components/locker/use-locker-actions.tsx+++ b/web/apps/locker/src/components/locker/use-locker-actions.tsx@@ -1,6 +1,5 @@import {LOCKER_FILE_LIMIT_PAID,- LOCKER_MAX_FILE_SIZE_BYTES,LOCKER_STORAGE_LIMIT_PAID_BYTES,lockerUploadAllowance,type LockerUploadLimitState,@@ -40,31 +39,13 @@ import type { DragEvent } from "react";import { useCallback, useEffect, useRef, useState } from "react";import { Trans } from "react-i18next";import { filterNonEmptyUploadItems } from "../create-item/file-upload-helpers";+import {+ collectUploadCandidatesFromDrop,+ type DragDataTransferItem,+} from "../create-item/scan-dropped-files";import type { CreateItemDialogEditItem } from "../create-item/use-create-item-dialog-state";import type { EmptyTrashDialogState } from "./EmptyTrashDialog";-type DragDataTransferItem = DataTransferItem & {- webkitGetAsEntry?: () => FileSystemEntry | null;-};--type FileSystemFileEntry = FileSystemEntry & {- file: (- successCallback: (file: File) => void,- errorCallback?: (error: DOMException) => void,- ) => void;-};--interface FileSystemDirectoryReader {- readEntries: (- successCallback: (entries: FileSystemEntry[]) => void,- errorCallback?: (error: DOMException) => void,- ) => void;-}--type FileSystemDirectoryEntry = FileSystemEntry & {- createReader: () => FileSystemDirectoryReader;-};-export interface ConfirmDialogState {illustration: string;title: string;@@ -89,202 +70,6 @@ const COLLECTION_MUTATION_REFRESH_DELAY_MS = 750;const UPLOAD_REFRESH_DEBOUNCE_MS = 250;const UPLOAD_REFRESH_FOLLOW_UP_DELAYS_MS = [1200, 3000, 6000] as const;-const fileFromEntry = (entry: FileSystemFileEntry) =>- new Promise<File>((resolve, reject) => {- entry.file(resolve, reject);- });--const readDirectoryEntries = (entry: FileSystemDirectoryEntry) =>- new Promise<FileSystemEntry[]>((resolve, reject) => {- const reader = entry.createReader();- const entries: FileSystemEntry[] = [];- const readBatch = () => {- reader.readEntries((batch) => {- if (batch.length === 0) {- resolve(entries);- return;- }- entries.push(...batch);- readBatch();- }, reject);- };- readBatch();- });--const collectionNamesFromRelativePath = (relativePath?: string) => [- ...new Set((relativePath?.split("/").slice(0, -1) ?? []).filter(Boolean)),-];--const uploadCandidateFromFile = (- file: File,- relativePath?: string,-): LockerUploadCandidate => ({- file,- relativePath,- suggestedCollectionNames: collectionNamesFromRelativePath(relativePath),-});--const droppedUploadCandidateKey = (file: File, relativePath?: string) =>- `${relativePath ?? file.name}:${file.size}:${file.lastModified}`;--interface DropUploadScanOptions {- remainingFileCount: number;- freeStorage: number;-}--interface DropUploadScanResult {- items: LockerUploadCandidate[];- preflightFailure?: LockerUploadPreflightFailure;- scannedNonEmptyFileCount: number;- scannedTotalSize: number;-}--type PendingDroppedNode =- | { type: "entry"; entry: FileSystemEntry; parentPath: string }- | { type: "file"; file: File; relativePath: string };--const collectUploadCandidatesFromDrop = async (- droppedItems: DragDataTransferItem[],- fallbackFiles: File[],- options: DropUploadScanOptions,-): Promise<DropUploadScanResult> => {- const items: LockerUploadCandidate[] = [];- const pendingNodes: PendingDroppedNode[] = [];- const seenUploadCandidateKeys = new Set<string>();- let scannedNonEmptyFileCount = 0;- let scannedTotalSize = 0;-- const addFile = (- file: File,- relativePath: string,- ): LockerUploadPreflightFailure | undefined => {- if (file.size === 0) {- return undefined;- }-- const dedupeKey = droppedUploadCandidateKey(file, relativePath);- if (seenUploadCandidateKeys.has(dedupeKey)) {- return undefined;- }- seenUploadCandidateKeys.add(dedupeKey);-- if (file.size > LOCKER_MAX_FILE_SIZE_BYTES) {- return { reason: "fileTooLarge", fileName: file.name };- }-- scannedNonEmptyFileCount += 1;- if (scannedNonEmptyFileCount > options.remainingFileCount) {- return { reason: "fileCountLimit" };- }-- scannedTotalSize += file.size;- if (scannedTotalSize > options.freeStorage) {- return { reason: "storageLimit" };- }-- items.push(uploadCandidateFromFile(file, relativePath));- return undefined;- };-- for (let index = droppedItems.length - 1; index >= 0; index -= 1) {- const item = droppedItems[index]!;- if (item.kind !== "file") {- continue;- }-- const getAsEntry = item.webkitGetAsEntry;- const entry =- typeof getAsEntry === "function" ? getAsEntry.call(item) : null;- if (entry) {- pendingNodes.push({ type: "entry", entry, parentPath: "" });- continue;- }-- const file = item.getAsFile();- if (file) {- pendingNodes.push({- type: "file",- file,- relativePath: file.webkitRelativePath || file.name,- });- }- }-- if (pendingNodes.length === 0) {- for (const file of fallbackFiles) {- const preflightFailure = addFile(- file,- file.webkitRelativePath || file.name,- );- if (preflightFailure) {- return {- items,- preflightFailure,- scannedNonEmptyFileCount,- scannedTotalSize,- };- }- }- return { items, scannedNonEmptyFileCount, scannedTotalSize };- }-- while (pendingNodes.length > 0) {- const node = pendingNodes.pop()!;-- if (node.type === "file") {- const preflightFailure = addFile(node.file, node.relativePath);- if (preflightFailure) {- return {- items,- preflightFailure,- scannedNonEmptyFileCount,- scannedTotalSize,- };- }- continue;- }-- if (node.entry.isFile) {- const file = await fileFromEntry(node.entry as FileSystemFileEntry);- const relativePath = node.parentPath- ? `${node.parentPath}/${file.name}`- : file.name;- const preflightFailure = addFile(file, relativePath);- if (preflightFailure) {- return {- items,- preflightFailure,- scannedNonEmptyFileCount,- scannedTotalSize,- };- }- continue;- }-- if (node.entry.isDirectory) {- const directoryPath = node.parentPath- ? `${node.parentPath}/${node.entry.name}`- : node.entry.name;- const directoryEntries = await readDirectoryEntries(- node.entry as FileSystemDirectoryEntry,- );- for (- let index = directoryEntries.length - 1;- index >= 0;- index -= 1- ) {- pendingNodes.push({- type: "entry",- entry: directoryEntries[index]!,- parentPath: directoryPath,- });- }- }- }-- return { items, scannedNonEmptyFileCount, scannedTotalSize };-};-const collectionIDsForItemMutation = (item: LockerItem,selectedCollectionID: number | null,diff --git a/web/apps/locker/tests/dropped-files.test.ts b/web/apps/locker/tests/dropped-files.test.tsnew file mode 100644index 0000000000..b0efbbe60f--- /dev/null+++ b/web/apps/locker/tests/dropped-files.test.ts@@ -0,0 +1,163 @@+import { expect, test, vi } from "vitest";+import {+ collectUploadCandidatesFromDrop,+ type DragDataTransferItem,+} from "../src/components/create-item/scan-dropped-files";+import { LOCKER_MAX_FILE_SIZE_BYTES } from "../src/services/locker-limits";++const allowance = { remainingFileCount: 10, freeStorage: 100 };+const file = (name: string, size = 1) =>+ new File(["x".repeat(size)], name, { lastModified: 1 });+const fileEntry = (value: File) =>+ ({+ isFile: true,+ isDirectory: false,+ name: value.name,+ file: (resolve: (file: File) => void) => resolve(value),+ }) as unknown as FileSystemEntry;+const directory = (name: string, batches: FileSystemEntry[][]) =>+ ({+ isFile: false,+ isDirectory: true,+ name,+ createReader: () => {+ let index = 0;+ return {+ readEntries: (resolve: (entries: FileSystemEntry[]) => void) =>+ resolve(batches[index++] ?? []),+ };+ },+ }) as unknown as FileSystemEntry;+const droppedEntry = (entry: FileSystemEntry) =>+ ({ kind: "file", webkitGetAsEntry: () => entry }) as DragDataTransferItem;++test("reads every directory batch in order and preserves nested paths", async () => {+ const root = directory("root", [+ [+ fileEntry(file("first")),+ directory("child", [[fileEntry(file("second"))]]),+ ],+ [fileEntry(file("third"))],+ ]);+ const result = await collectUploadCandidatesFromDrop(+ [droppedEntry(root)],+ [],+ allowance,+ );+ expect(+ result.items.map(({ relativePath, suggestedCollectionNames }) => ({+ relativePath,+ suggestedCollectionNames,+ })),+ ).toEqual([+ { relativePath: "root/first", suggestedCollectionNames: ["root"] },+ {+ relativePath: "root/child/second",+ suggestedCollectionNames: ["root", "child"],+ },+ { relativePath: "root/third", suggestedCollectionNames: ["root"] },+ ]);+ expect(result.scannedNonEmptyFileCount).toBe(3);+ expect(result.scannedTotalSize).toBe(3);+});++test("fallback skips empty files and duplicate path/size/timestamp entries", async () => {+ const result = await collectUploadCandidatesFromDrop(+ [],+ [file("empty", 0), file("same"), file("same"), file("last")],+ allowance,+ );+ expect(result.items.map(({ file }) => file.name)).toEqual(["same", "last"]);+ expect(result.scannedNonEmptyFileCount).toBe(2);+ expect(result.scannedTotalSize).toBe(2);+});++test("uses getAsFile without also ingesting fallback files", async () => {+ const value = file("selected");+ const item = {+ kind: "file",+ getAsFile: () => value,+ } as DragDataTransferItem;+ const result = await collectUploadCandidatesFromDrop(+ [item],+ [file("fallback")],+ allowance,+ );+ expect(result.items.map(({ file }) => file.name)).toEqual(["selected"]);+});++test.each([+ {+ options: { remainingFileCount: 1, freeStorage: 100 },+ reason: "fileCountLimit",+ count: 2,+ size: 1,+ },+ {+ options: { remainingFileCount: 10, freeStorage: 1 },+ reason: "storageLimit",+ count: 2,+ size: 2,+ },+])("stops scanning at $reason", async ({ options, reason, count, size }) => {+ const unread = vi.fn();+ const later = { isFile: true, file: unread } as unknown as FileSystemEntry;+ const result = await collectUploadCandidatesFromDrop(+ [+ droppedEntry(fileEntry(file("first"))),+ droppedEntry(fileEntry(file("second"))),+ droppedEntry(later),+ ],+ [],+ options,+ );+ expect(result.preflightFailure).toEqual({ reason });+ expect(result.items).toHaveLength(1);+ expect(result.scannedNonEmptyFileCount).toBe(count);+ expect(result.scannedTotalSize).toBe(size);+ expect(unread).not.toHaveBeenCalled();+});++test("rejects oversized files before counting them", async () => {+ const oversized = file("large");+ Object.defineProperty(oversized, "size", {+ value: LOCKER_MAX_FILE_SIZE_BYTES + 1,+ });+ const result = await collectUploadCandidatesFromDrop(+ [],+ [oversized],+ allowance,+ );+ expect(result).toEqual({+ items: [],+ preflightFailure: { reason: "fileTooLarge", fileName: "large" },+ scannedNonEmptyFileCount: 0,+ scannedTotalSize: 0,+ });+});++test("accepts exact count and storage boundaries", async () => {+ const result = await collectUploadCandidatesFromDrop([], [file("one")], {+ remainingFileCount: 1,+ freeStorage: 1,+ });+ expect(result.preflightFailure).toBeUndefined();+ expect(result.items).toHaveLength(1);+});++test("propagates directory read failures", async () => {+ const failure = new DOMException("Cannot read directory");+ const entry = {+ isDirectory: true,+ name: "blocked",+ createReader: () => ({+ readEntries: (+ _resolve: unknown,+ reject: (error: DOMException) => void,+ ) => reject(failure),+ }),+ } as unknown as FileSystemEntry;+ await expect(+ collectUploadCandidatesFromDrop([droppedEntry(entry)], [], allowance),+ ).rejects.toBe(failure);+});