Papermark is the open-source DocSend alternative and secure data rooms with built-in analytics and custom domains.
Implement multipart S3 upload with parallel processing for large files
Cursor Agentffa0b1b4a05c
diff --git a/lib/files/put-file.ts b/lib/files/put-file.tsindex 78ea3b4c..7842f8be 100644--- a/lib/files/put-file.ts+++ b/lib/files/put-file.ts@@ -10,6 +10,20 @@ import {import { SUPPORTED_DOCUMENT_MIME_TYPES } from "../constants";+/**+ * Uploads a file to the configured storage backend (S3 or Vercel).+ *+ * For S3 uploads:+ * - Files larger than 10MB automatically use multipart upload with pre-signed URLs+ * - Files are uploaded in 10MB chunks with parallel processing (batches of 3)+ * - Provides better performance and reliability for large files+ * - Falls back to single upload for smaller files or on multipart failure+ *+ * @param file - The file to upload+ * @param teamId - The team ID for storage configuration+ * @param docId - Optional document ID (generated if not provided)+ * @returns Upload result with storage type, key/URL, page count, and file size+ */export const putFile = async ({file,teamId,@@ -44,25 +58,30 @@ export const putFile = async ({};const putFileInVercel = async (file: File) => {- const newBlob = await upload(file.name, file, {+ const url = await upload(file.name, file, {access: "public",- handleUploadUrl: "/api/file/browser-upload",+ handleUploadUrl: "/api/file/vercel",});let numPages: number = 1;+ // get page count for pdf filesif (file.type === "application/pdf") {- const contents = await file.arrayBuffer();- numPages = await getPagesCount(contents);+ const body = await file.arrayBuffer();+ numPages = await getPagesCount(body);}return {type: DocumentStorageType.VERCEL_BLOB,- data: newBlob.url,+ data: url.url,numPages: numPages,fileSize: file.size,};};+// Multipart upload threshold: 10MB+const MULTIPART_THRESHOLD = 10 * 1024 * 1024;+const PART_SIZE = 10 * 1024 * 1024; // 10MB chunks+const putFileInS3 = async ({file,teamId,@@ -87,6 +106,24 @@ const putFileInS3 = async ({);}+ // Check if file should use multipart upload+ if (file.size > MULTIPART_THRESHOLD) {+ return await putFileMultipart({ file, teamId, docId });+ } else {+ return await putFileSingle({ file, teamId, docId });+ }+};++// Single file upload (existing logic)+const putFileSingle = async ({+ file,+ teamId,+ docId,+}: {+ file: File;+ teamId: string;+ docId: string;+}) => {const presignedResponse = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL}/api/file/s3/get-presigned-post-url`,{@@ -136,14 +173,6 @@ const putFileInS3 = async ({const body = await file.arrayBuffer();numPages = await getPagesCount(body);}- // get sheet count for excel files- // else if (- // SUPPORTED_DOCUMENT_MIME_TYPES.includes(file.type) &&- // file.type !== "application/pdf"- // ) {- // const body = await file.arrayBuffer();- // numPages = getSheetsCount(body);- // }return {type: DocumentStorageType.S3_PATH,@@ -152,3 +181,153 @@ const putFileInS3 = async ({fileSize: file.size,};};++// Multipart file upload for large files+const putFileMultipart = async ({+ file,+ teamId,+ docId,+}: {+ file: File;+ teamId: string;+ docId: string;+}) => {+ try {+ // Step 1: Initiate multipart upload+ const initiateResponse = await fetch(+ `${process.env.NEXT_PUBLIC_BASE_URL}/api/file/s3/multipart`,+ {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ },+ body: JSON.stringify({+ action: "initiate",+ fileName: file.name,+ contentType: file.type,+ teamId: teamId,+ docId: docId,+ }),+ },+ );++ if (!initiateResponse.ok) {+ throw new Error(+ `Failed to initiate multipart upload, status: ${initiateResponse.status}`,+ );+ }++ const { uploadId, key, fileName } = await initiateResponse.json();++ // Step 2: Get pre-signed URLs for parts+ const partUrlsResponse = await fetch(+ `${process.env.NEXT_PUBLIC_BASE_URL}/api/file/s3/multipart`,+ {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ },+ body: JSON.stringify({+ action: "get-part-urls",+ fileName: file.name,+ contentType: file.type,+ teamId: teamId,+ docId: docId,+ uploadId: uploadId,+ fileSize: file.size,+ partSize: PART_SIZE,+ }),+ },+ );++ if (!partUrlsResponse.ok) {+ throw new Error(+ `Failed to get part URLs, status: ${partUrlsResponse.status}`,+ );+ }++ const { urls } = await partUrlsResponse.json();++ // Step 3: Upload parts in parallel (batches of 3)+ const uploadPart = async ({ partNumber, url }: { partNumber: number; url: string }) => {+ const start = (partNumber - 1) * PART_SIZE;+ const end = Math.min(start + PART_SIZE, file.size);+ const chunk = file.slice(start, end);++ const response = await fetch(url, {+ method: "PUT",+ body: chunk,+ });++ if (!response.ok) {+ throw new Error(+ `Failed to upload part ${partNumber}, status: ${response.status}`,+ );+ }++ const etag = response.headers.get("ETag");+ if (!etag) {+ throw new Error(`Missing ETag in response for part ${partNumber}`);+ }++ return { PartNumber: partNumber, ETag: etag };+ };++ // Upload parts in batches to avoid overwhelming the connection+ const batchSize = 3;+ const parts: Array<{ PartNumber: number; ETag: string }> = [];++ for (let i = 0; i < urls.length; i += batchSize) {+ const batch = urls.slice(i, i + batchSize);+ const batchResults = await Promise.all(+ batch.map(uploadPart)+ );+ parts.push(...batchResults);+ }++ // Step 4: Complete multipart upload+ const completeResponse = await fetch(+ `${process.env.NEXT_PUBLIC_BASE_URL}/api/file/s3/multipart`,+ {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ },+ body: JSON.stringify({+ action: "complete",+ fileName: file.name,+ contentType: file.type,+ teamId: teamId,+ docId: docId,+ uploadId: uploadId,+ parts: parts,+ }),+ },+ );++ if (!completeResponse.ok) {+ throw new Error(+ `Failed to complete multipart upload, status: ${completeResponse.status}`,+ );+ }++ let numPages: number = 1;+ // get page count for pdf files+ if (file.type === "application/pdf") {+ const body = await file.arrayBuffer();+ numPages = await getPagesCount(body);+ }++ return {+ type: DocumentStorageType.S3_PATH,+ data: key,+ numPages: numPages,+ fileSize: file.size,+ };+ } catch (error) {+ console.error("Multipart upload failed:", error);+ // Fallback to single upload on error+ console.log("Falling back to single upload...");+ return await putFileSingle({ file, teamId, docId });+ }+};diff --git a/pages/api/file/s3/multipart.ts b/pages/api/file/s3/multipart.tsnew file mode 100644index 00000000..7bcda62d--- /dev/null+++ b/pages/api/file/s3/multipart.ts@@ -0,0 +1,162 @@+import { NextApiRequest, NextApiResponse } from "next";++import {+ CompleteMultipartUploadCommand,+ CreateMultipartUploadCommand,+ UploadPartCommand,+} from "@aws-sdk/client-s3";+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";+import slugify from "@sindresorhus/slugify";+import { getServerSession } from "next-auth";+import path from "node:path";++import { ONE_HOUR, ONE_SECOND } from "@/lib/constants";+import { getTeamS3ClientAndConfig } from "@/lib/files/aws-client";+import prisma from "@/lib/prisma";+import { CustomUser } from "@/lib/types";++import { authOptions } from "../../auth/[...nextauth]";++export default async function handler(+ req: NextApiRequest,+ res: NextApiResponse,+) {+ if (req.method !== "POST") {+ return res.status(405).end("Method Not Allowed");+ }++ const session = await getServerSession(req, res, authOptions);+ if (!session) {+ return res.status(401).end("Unauthorized");+ }++ try {+ const {+ action,+ fileName,+ contentType,+ teamId,+ docId,+ uploadId,+ fileSize,+ partSize = 10 * 1024 * 1024, // Default 10MB chunks+ parts,+ } = req.body as {+ action: "initiate" | "get-part-urls" | "complete";+ fileName: string;+ contentType: string;+ teamId: string;+ docId: string;+ uploadId?: string;+ fileSize?: number;+ partSize?: number;+ parts?: Array<{ ETag: string; PartNumber: number }>;+ };++ // Verify team access+ const team = await prisma.team.findUnique({+ where: {+ id: teamId,+ users: {+ some: {+ userId: (session.user as CustomUser).id,+ },+ },+ },+ select: { id: true },+ });++ if (!team) {+ return res.status(403).end("Unauthorized to access this team");+ }++ // Get the basename and extension for the file+ const { name, ext } = path.parse(fileName);+ const slugifiedName = slugify(name) + ext;+ const key = `${team.id}/${docId}/${slugifiedName}`;++ const { client, config } = await getTeamS3ClientAndConfig(team.id);++ switch (action) {+ case "initiate": {+ // Step 1: Start multipart upload+ const createCommand = new CreateMultipartUploadCommand({+ Bucket: config.bucket,+ Key: key,+ ContentType: contentType,+ ContentDisposition: `attachment; filename="${slugifiedName}"`,+ });++ const createResponse = await client.send(createCommand);++ return res.status(200).json({+ uploadId: createResponse.UploadId,+ key,+ fileName: slugifiedName,+ });+ }++ case "get-part-urls": {+ // Step 2: Generate pre-signed URLs for each part+ if (!uploadId || !fileSize) {+ return res.status(400).json({+ error: "uploadId and fileSize are required for get-part-urls action",+ });+ }++ const numParts = Math.ceil(fileSize / partSize);+ const urls = await Promise.all(+ Array.from({ length: numParts }, async (_, index) => {+ const partNumber = index + 1;+ const command = new UploadPartCommand({+ Bucket: config.bucket,+ Key: key,+ PartNumber: partNumber,+ UploadId: uploadId,+ });++ const url = await getSignedUrl(client, command, {+ expiresIn: ONE_HOUR / ONE_SECOND,+ });++ return { partNumber, url };+ }),+ );++ return res.status(200).json({ urls });+ }++ case "complete": {+ // Step 3: Complete multipart upload+ if (!uploadId || !parts) {+ return res.status(400).json({+ error: "uploadId and parts are required for complete action",+ });+ }++ const completeCommand = new CompleteMultipartUploadCommand({+ Bucket: config.bucket,+ Key: key,+ UploadId: uploadId,+ MultipartUpload: {+ Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber),+ },+ });++ await client.send(completeCommand);++ return res.status(200).json({+ success: true,+ key,+ fileName: slugifiedName,+ });+ }++ default:+ return res.status(400).json({ error: "Invalid action" });+ }+ } catch (error) {+ console.error("Multipart upload error:", error);+ return res.status(500).json({ error: "Internal server error" });+ }+}\ No newline at end of file