Skip to content

Papermark is the open-source DocSend alternative and secure data rooms with built-in analytics and custom domains.

Implement multipart S3 upload with parallel processing for large files

Cursor Agentffa0b1b4a05c
diff --git a/lib/files/put-file.ts b/lib/files/put-file.ts
index 78ea3b4c..7842f8be 100644
--- a/lib/files/put-file.ts
+++ b/lib/files/put-file.ts
@@ -10,6 +10,20 @@ import {
import { SUPPORTED_DOCUMENT_MIME_TYPES } from "../constants";
+/**
+ * Uploads a file to the configured storage backend (S3 or Vercel).
+ *
+ * For S3 uploads:
+ * - Files larger than 10MB automatically use multipart upload with pre-signed URLs
+ * - Files are uploaded in 10MB chunks with parallel processing (batches of 3)
+ * - Provides better performance and reliability for large files
+ * - Falls back to single upload for smaller files or on multipart failure
+ *
+ * @param file - The file to upload
+ * @param teamId - The team ID for storage configuration
+ * @param docId - Optional document ID (generated if not provided)
+ * @returns Upload result with storage type, key/URL, page count, and file size
+ */
export const putFile = async ({
file,
teamId,
@@ -44,25 +58,30 @@ export const putFile = async ({
};
const putFileInVercel = async (file: File) => {
- const newBlob = await upload(file.name, file, {
+ const url = await upload(file.name, file, {
access: "public",
- handleUploadUrl: "/api/file/browser-upload",
+ handleUploadUrl: "/api/file/vercel",
});
let numPages: number = 1;
+ // get page count for pdf files
if (file.type === "application/pdf") {
- const contents = await file.arrayBuffer();
- numPages = await getPagesCount(contents);
+ const body = await file.arrayBuffer();
+ numPages = await getPagesCount(body);
}
return {
type: DocumentStorageType.VERCEL_BLOB,
- data: newBlob.url,
+ data: url.url,
numPages: numPages,
fileSize: file.size,
};
};
+// Multipart upload threshold: 10MB
+const MULTIPART_THRESHOLD = 10 * 1024 * 1024;
+const PART_SIZE = 10 * 1024 * 1024; // 10MB chunks
+
const putFileInS3 = async ({
file,
teamId,
@@ -87,6 +106,24 @@ const putFileInS3 = async ({
);
}
+ // Check if file should use multipart upload
+ if (file.size > MULTIPART_THRESHOLD) {
+ return await putFileMultipart({ file, teamId, docId });
+ } else {
+ return await putFileSingle({ file, teamId, docId });
+ }
+};
+
+// Single file upload (existing logic)
+const putFileSingle = async ({
+ file,
+ teamId,
+ docId,
+}: {
+ file: File;
+ teamId: string;
+ docId: string;
+}) => {
const presignedResponse = await fetch(
`${process.env.NEXT_PUBLIC_BASE_URL}/api/file/s3/get-presigned-post-url`,
{
@@ -136,14 +173,6 @@ const putFileInS3 = async ({
const body = await file.arrayBuffer();
numPages = await getPagesCount(body);
}
- // get sheet count for excel files
- // else if (
- // SUPPORTED_DOCUMENT_MIME_TYPES.includes(file.type) &&
- // file.type !== "application/pdf"
- // ) {
- // const body = await file.arrayBuffer();
- // numPages = getSheetsCount(body);
- // }
return {
type: DocumentStorageType.S3_PATH,
@@ -152,3 +181,153 @@ const putFileInS3 = async ({
fileSize: file.size,
};
};
+
+// Multipart file upload for large files
+const putFileMultipart = async ({
+ file,
+ teamId,
+ docId,
+}: {
+ file: File;
+ teamId: string;
+ docId: string;
+}) => {
+ try {
+ // Step 1: Initiate multipart upload
+ const initiateResponse = await fetch(
+ `${process.env.NEXT_PUBLIC_BASE_URL}/api/file/s3/multipart`,
+ {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ action: "initiate",
+ fileName: file.name,
+ contentType: file.type,
+ teamId: teamId,
+ docId: docId,
+ }),
+ },
+ );
+
+ if (!initiateResponse.ok) {
+ throw new Error(
+ `Failed to initiate multipart upload, status: ${initiateResponse.status}`,
+ );
+ }
+
+ const { uploadId, key, fileName } = await initiateResponse.json();
+
+ // Step 2: Get pre-signed URLs for parts
+ const partUrlsResponse = await fetch(
+ `${process.env.NEXT_PUBLIC_BASE_URL}/api/file/s3/multipart`,
+ {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ action: "get-part-urls",
+ fileName: file.name,
+ contentType: file.type,
+ teamId: teamId,
+ docId: docId,
+ uploadId: uploadId,
+ fileSize: file.size,
+ partSize: PART_SIZE,
+ }),
+ },
+ );
+
+ if (!partUrlsResponse.ok) {
+ throw new Error(
+ `Failed to get part URLs, status: ${partUrlsResponse.status}`,
+ );
+ }
+
+ const { urls } = await partUrlsResponse.json();
+
+ // Step 3: Upload parts in parallel (batches of 3)
+ const uploadPart = async ({ partNumber, url }: { partNumber: number; url: string }) => {
+ const start = (partNumber - 1) * PART_SIZE;
+ const end = Math.min(start + PART_SIZE, file.size);
+ const chunk = file.slice(start, end);
+
+ const response = await fetch(url, {
+ method: "PUT",
+ body: chunk,
+ });
+
+ if (!response.ok) {
+ throw new Error(
+ `Failed to upload part ${partNumber}, status: ${response.status}`,
+ );
+ }
+
+ const etag = response.headers.get("ETag");
+ if (!etag) {
+ throw new Error(`Missing ETag in response for part ${partNumber}`);
+ }
+
+ return { PartNumber: partNumber, ETag: etag };
+ };
+
+ // Upload parts in batches to avoid overwhelming the connection
+ const batchSize = 3;
+ const parts: Array<{ PartNumber: number; ETag: string }> = [];
+
+ for (let i = 0; i < urls.length; i += batchSize) {
+ const batch = urls.slice(i, i + batchSize);
+ const batchResults = await Promise.all(
+ batch.map(uploadPart)
+ );
+ parts.push(...batchResults);
+ }
+
+ // Step 4: Complete multipart upload
+ const completeResponse = await fetch(
+ `${process.env.NEXT_PUBLIC_BASE_URL}/api/file/s3/multipart`,
+ {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ action: "complete",
+ fileName: file.name,
+ contentType: file.type,
+ teamId: teamId,
+ docId: docId,
+ uploadId: uploadId,
+ parts: parts,
+ }),
+ },
+ );
+
+ if (!completeResponse.ok) {
+ throw new Error(
+ `Failed to complete multipart upload, status: ${completeResponse.status}`,
+ );
+ }
+
+ let numPages: number = 1;
+ // get page count for pdf files
+ if (file.type === "application/pdf") {
+ const body = await file.arrayBuffer();
+ numPages = await getPagesCount(body);
+ }
+
+ return {
+ type: DocumentStorageType.S3_PATH,
+ data: key,
+ numPages: numPages,
+ fileSize: file.size,
+ };
+ } catch (error) {
+ console.error("Multipart upload failed:", error);
+ // Fallback to single upload on error
+ console.log("Falling back to single upload...");
+ return await putFileSingle({ file, teamId, docId });
+ }
+};
diff --git a/pages/api/file/s3/multipart.ts b/pages/api/file/s3/multipart.ts
new file mode 100644
index 00000000..7bcda62d
--- /dev/null
+++ b/pages/api/file/s3/multipart.ts
@@ -0,0 +1,162 @@
+import { NextApiRequest, NextApiResponse } from "next";
+
+import {
+ CompleteMultipartUploadCommand,
+ CreateMultipartUploadCommand,
+ UploadPartCommand,
+} from "@aws-sdk/client-s3";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import slugify from "@sindresorhus/slugify";
+import { getServerSession } from "next-auth";
+import path from "node:path";
+
+import { ONE_HOUR, ONE_SECOND } from "@/lib/constants";
+import { getTeamS3ClientAndConfig } from "@/lib/files/aws-client";
+import prisma from "@/lib/prisma";
+import { CustomUser } from "@/lib/types";
+
+import { authOptions } from "../../auth/[...nextauth]";
+
+export default async function handler(
+ req: NextApiRequest,
+ res: NextApiResponse,
+) {
+ if (req.method !== "POST") {
+ return res.status(405).end("Method Not Allowed");
+ }
+
+ const session = await getServerSession(req, res, authOptions);
+ if (!session) {
+ return res.status(401).end("Unauthorized");
+ }
+
+ try {
+ const {
+ action,
+ fileName,
+ contentType,
+ teamId,
+ docId,
+ uploadId,
+ fileSize,
+ partSize = 10 * 1024 * 1024, // Default 10MB chunks
+ parts,
+ } = req.body as {
+ action: "initiate" | "get-part-urls" | "complete";
+ fileName: string;
+ contentType: string;
+ teamId: string;
+ docId: string;
+ uploadId?: string;
+ fileSize?: number;
+ partSize?: number;
+ parts?: Array<{ ETag: string; PartNumber: number }>;
+ };
+
+ // Verify team access
+ const team = await prisma.team.findUnique({
+ where: {
+ id: teamId,
+ users: {
+ some: {
+ userId: (session.user as CustomUser).id,
+ },
+ },
+ },
+ select: { id: true },
+ });
+
+ if (!team) {
+ return res.status(403).end("Unauthorized to access this team");
+ }
+
+ // Get the basename and extension for the file
+ const { name, ext } = path.parse(fileName);
+ const slugifiedName = slugify(name) + ext;
+ const key = `${team.id}/${docId}/${slugifiedName}`;
+
+ const { client, config } = await getTeamS3ClientAndConfig(team.id);
+
+ switch (action) {
+ case "initiate": {
+ // Step 1: Start multipart upload
+ const createCommand = new CreateMultipartUploadCommand({
+ Bucket: config.bucket,
+ Key: key,
+ ContentType: contentType,
+ ContentDisposition: `attachment; filename="${slugifiedName}"`,
+ });
+
+ const createResponse = await client.send(createCommand);
+
+ return res.status(200).json({
+ uploadId: createResponse.UploadId,
+ key,
+ fileName: slugifiedName,
+ });
+ }
+
+ case "get-part-urls": {
+ // Step 2: Generate pre-signed URLs for each part
+ if (!uploadId || !fileSize) {
+ return res.status(400).json({
+ error: "uploadId and fileSize are required for get-part-urls action",
+ });
+ }
+
+ const numParts = Math.ceil(fileSize / partSize);
+ const urls = await Promise.all(
+ Array.from({ length: numParts }, async (_, index) => {
+ const partNumber = index + 1;
+ const command = new UploadPartCommand({
+ Bucket: config.bucket,
+ Key: key,
+ PartNumber: partNumber,
+ UploadId: uploadId,
+ });
+
+ const url = await getSignedUrl(client, command, {
+ expiresIn: ONE_HOUR / ONE_SECOND,
+ });
+
+ return { partNumber, url };
+ }),
+ );
+
+ return res.status(200).json({ urls });
+ }
+
+ case "complete": {
+ // Step 3: Complete multipart upload
+ if (!uploadId || !parts) {
+ return res.status(400).json({
+ error: "uploadId and parts are required for complete action",
+ });
+ }
+
+ const completeCommand = new CompleteMultipartUploadCommand({
+ Bucket: config.bucket,
+ Key: key,
+ UploadId: uploadId,
+ MultipartUpload: {
+ Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber),
+ },
+ });
+
+ await client.send(completeCommand);
+
+ return res.status(200).json({
+ success: true,
+ key,
+ fileName: slugifiedName,
+ });
+ }
+
+ default:
+ return res.status(400).json({ error: "Invalid action" });
+ }
+ } catch (error) {
+ console.error("Multipart upload error:", error);
+ return res.status(500).json({ error: "Internal server error" });
+ }
+}
\ No newline at end of file