Skip to content

Papermark is the open-source DocSend alternative and secure data rooms with built-in analytics and custom domains.

refactor: stream the buffer to s3

Marc Seitzc32b571e3f69
diff --git a/lib/files/put-file-server.ts b/lib/files/put-file-server.ts
index a26048f9..95e5e2ce 100644
--- a/lib/files/put-file-server.ts
+++ b/lib/files/put-file-server.ts
@@ -7,12 +7,14 @@ import { PutObjectCommand } from "@aws-sdk/client-s3";
import { getS3Client } from "./aws-client";
import slugify from "@sindresorhus/slugify";
import path from "node:path";
+import { Readable } from "stream";
// `File` is a web API type and not available server-side, so we need to define our own type
type File = {
name: string;
type: string;
- buffer: Buffer;
+ buffer?: Buffer;
+ stream?: Readable;
};
export const putFileServer = async ({
@@ -41,7 +43,7 @@ export const putFileServer = async ({
};
const putFileInVercelServer = async (file: File) => {
- const contents = file.buffer;
+ const contents = file.buffer || file.stream?.read();
const blob = await put(file.name, contents, {
access: "public",
@@ -73,7 +75,7 @@ const putFileInS3Server = async ({
const key = `${teamId}/${docId}/${slugify(name)}${ext}`;
- const buffer = file.buffer;
+ const buffer = file.buffer || file.stream;
const params = {
Bucket: process.env.NEXT_PRIVATE_UPLOAD_BUCKET,
diff --git a/pages/api/mupdf/convert-page.ts b/pages/api/mupdf/convert-page.ts
index 5fe06bd7..0e755c06 100644
--- a/pages/api/mupdf/convert-page.ts
+++ b/pages/api/mupdf/convert-page.ts
@@ -4,6 +4,7 @@ import mupdf from "mupdf";
import prisma from "@/lib/prisma";
import { putFileServer } from "@/lib/files/put-file-server";
import { log } from "@/lib/utils";
+import { Readable } from "stream";
// This function can run for a maximum of 60 seconds
export const config = {
@@ -69,7 +70,10 @@ export default async (req: NextApiRequest, res: NextApiResponse) => {
var pngBuffer = pixmap.asPNG(); // as PNG
- const buffer = Buffer.from(pngBuffer);
+ // let buffer = Buffer.from(pngBuffer);
+
+ // Assuming `pngBuffer` is your buffer from the MuPDF conversion
+ const pngStream = Readable.from(pngBuffer);
// get docId from url with starts with "doc_" with regex
const match = url.match(/(doc_[^\/]+)\//);
@@ -79,12 +83,15 @@ export default async (req: NextApiRequest, res: NextApiResponse) => {
file: {
name: `page-${pageNumber}.png`,
type: "image/png",
- buffer: buffer,
+ // buffer: buffer,
+ stream: pngStream,
},
teamId: teamId,
docId: docId,
});
+ // buffer = Buffer.alloc(0);
+
if (!data || !type) {
throw new Error(`Failed to upload document page ${pageNumber}`);
}