Skip to content

Papermark is the open-source DocSend alternative and secure data rooms with built-in analytics and custom domains.

feat: add trigger for converting docs to pdf

Marc Seitz0bf6a70713e7
diff --git a/jobs/convert-pdf-to-image.ts b/jobs/convert-pdf-to-image.ts
index c53d909c..09b3c22c 100644
--- a/jobs/convert-pdf-to-image.ts
+++ b/jobs/convert-pdf-to-image.ts
@@ -40,7 +40,7 @@ client.defineJob({
);
// 1. get file url from document version
- const documentUrl = await io.runTask("get-document-url", async () => {
+ const documentVersion = await io.runTask("get-document-url", async () => {
return prisma.documentVersion.findUnique({
where: {
id: documentVersionId,
@@ -53,8 +53,8 @@ client.defineJob({
});
});
- // if documentUrl is null, log error and return
- if (!documentUrl) {
+ // if documentVersion is null, log error and return
+ if (!documentVersion) {
await io.logger.error("File not found", { payload });
await processingDocumentStatus.update("error", {
//set data, this overrides the previous value
@@ -72,8 +72,8 @@ client.defineJob({
// 2. get signed url from file
const signedUrl = await io.runTask("get-signed-url", async () => {
return await getFile({
- type: documentUrl.storageType,
- data: documentUrl.file,
+ type: documentVersion.storageType,
+ data: documentVersion.file,
});
});
@@ -92,17 +92,17 @@ client.defineJob({
return;
}
- let numPages = documentUrl.numPages;
+ let numPages = documentVersion.numPages;
// skip if the numPages are already defined
- if (!numPages) {
+ if (!numPages || numPages === 1) {
// 3. send file to api/convert endpoint in a task and get back number of pages
const muDocument = await io.runTask("get-number-of-pages", async () => {
const response = await fetch(
`${process.env.NEXT_PUBLIC_BASE_URL}/api/mupdf/get-pages`,
{
method: "POST",
- body: JSON.stringify({ url: documentUrl.file }),
+ body: JSON.stringify({ url: signedUrl }),
headers: {
"Content-Type": "application/json",
},
@@ -110,6 +110,14 @@ client.defineJob({
);
await io.logger.info("log response", { response });
+ if (!response.ok) {
+ await io.logger.error("Failed to get number of pages", {
+ signedUrl,
+ response,
+ });
+ throw new Error("Failed to get number of pages");
+ }
+
const { numPages } = (await response.json()) as { numPages: number };
return { numPages };
});
@@ -221,6 +229,7 @@ client.defineJob({
id: documentVersionId,
},
data: {
+ numPages: numPages,
hasPages: true,
isPrimary: true,
},
diff --git a/lib/files/put-file-server.ts b/lib/files/put-file-server.ts
index ba805e30..bd1030e4 100644
--- a/lib/files/put-file-server.ts
+++ b/lib/files/put-file-server.ts
@@ -67,8 +67,12 @@ const putFileInS3Server = async ({
docId = newId("doc");
}
- if (file.type !== "image/png" && file.type !== "image/jpeg") {
- throw new Error("Only PNG or JPEG files are supported");
+ if (
+ file.type !== "image/png" &&
+ file.type !== "image/jpeg" &&
+ file.type !== "application/pdf"
+ ) {
+ throw new Error("Only PNG, JPEG or PDF files are supported");
}
const client = getS3Client();
diff --git a/lib/trigger/convert-files.ts b/lib/trigger/convert-files.ts
new file mode 100644
index 00000000..08d16e0e
--- /dev/null
+++ b/lib/trigger/convert-files.ts
@@ -0,0 +1,178 @@
+import { client } from "@/trigger";
+import { envvars, logger, retry, task } from "@trigger.dev/sdk/v3";
+
+import { getFile } from "@/lib/files/get-file";
+import { putFileServer } from "@/lib/files/put-file-server";
+import prisma from "@/lib/prisma";
+
+type ConvertPayload = {
+ documentId: string;
+ documentVersionId: string;
+ teamId: string;
+};
+
+export const convertFilesToPdfTask = task({
+ id: "convert-files-to-pdf",
+ retry: { maxAttempts: 3 },
+ queue: {
+ concurrencyLimit: 10,
+ },
+ run: async (payload: ConvertPayload) => {
+ const team = await prisma.team.findUnique({
+ where: {
+ id: payload.teamId,
+ },
+ });
+
+ if (!team) {
+ logger.error("Team not found", { teamId: payload.teamId });
+ return;
+ }
+
+ const document = await prisma.document.findUnique({
+ where: {
+ id: payload.documentId,
+ },
+ select: {
+ name: true,
+ versions: {
+ where: {
+ id: payload.documentVersionId,
+ },
+ select: {
+ file: true,
+ originalFile: true,
+ contentType: true,
+ storageType: true,
+ },
+ },
+ },
+ });
+
+ if (
+ !document ||
+ !document.versions[0] ||
+ !document.versions[0].originalFile ||
+ !document.versions[0].contentType
+ ) {
+ logger.error("Document not found", {
+ documentId: payload.documentId,
+ documentVersionId: payload.documentVersionId,
+ teamId: payload.teamId,
+ });
+ return;
+ }
+
+ const fileUrl = await getFile({
+ data: document.versions[0].originalFile,
+ type: document.versions[0].storageType,
+ });
+ const fileResponse = await fetch(fileUrl);
+ const fileBuffer = Buffer.from(await fileResponse.arrayBuffer());
+
+ // Prepare form data
+ const formData = new FormData();
+ formData.append(
+ "files",
+ new Blob([fileBuffer], {
+ type: document.versions[0].contentType,
+ }),
+ document.name,
+ );
+ formData.append("quality", "50");
+ console.log("formData", formData);
+
+ // Make the conversion request
+ const conversionResponse = await retry.fetch(
+ `${process.env.NEXT_PRIVATE_CONVERSION_BASE_URL}/forms/libreoffice/convert`,
+ {
+ method: "POST",
+ body: formData,
+ headers: {
+ Authorization: `Basic ${process.env.NEXT_PRIVATE_INTERNAL_AUTH_TOKEN}`,
+ },
+ retry: {
+ byStatus: {
+ "500-599": {
+ strategy: "backoff",
+ maxAttempts: 3,
+ factor: 2,
+ minTimeoutInMs: 1_000,
+ maxTimeoutInMs: 30_000,
+ randomize: false,
+ },
+ },
+ },
+ },
+ );
+
+ if (!conversionResponse.ok) {
+ const body = await conversionResponse.json();
+ throw new Error(
+ `Conversion failed: ${body.message} ${conversionResponse.status}`,
+ );
+ }
+
+ const conversionBuffer = Buffer.from(
+ await conversionResponse.arrayBuffer(),
+ );
+
+ console.log("conversionBuffer", conversionBuffer);
+
+ // get docId from url with starts with "doc_" with regex
+ const match = document.versions[0].originalFile.match(/(doc_[^\/]+)\//);
+ const docId = match ? match[1] : undefined;
+
+ // Save the converted file to the database
+ const { type: storageType, data } = await putFileServer({
+ file: {
+ name: `${document.name}.pdf`,
+ type: "application/pdf",
+ buffer: conversionBuffer,
+ },
+ teamId: payload.teamId,
+ docId: docId,
+ });
+
+ if (!data || !storageType) {
+ logger.error("Failed to save converted file to database", {
+ documentId: payload.documentId,
+ documentVersionId: payload.documentVersionId,
+ teamId: payload.teamId,
+ docId: docId,
+ });
+ return;
+ }
+
+ console.log("data from conversion", data);
+ console.log("storageType from conversion", storageType);
+
+ await prisma.documentVersion.update({
+ where: { id: payload.documentVersionId },
+ data: {
+ file: data,
+ type: "pdf",
+ storageType: storageType,
+ },
+ });
+
+ // v2: trigger document uploaded event to trigger convert-pdf-to-image job
+ await client.sendEvent({
+ id: payload.documentVersionId, // unique eventId for the run
+ name: "document.uploaded",
+ payload: {
+ documentVersionId: payload.documentVersionId,
+ teamId: payload.teamId,
+ documentId: payload.documentId,
+ },
+ });
+
+ logger.info("Document converted", {
+ documentId: payload.documentId,
+ documentVersionId: payload.documentVersionId,
+ teamId: payload.teamId,
+ docId: docId,
+ });
+ return;
+ },
+});
diff --git a/pages/api/teams/[teamId]/documents/[id]/versions/index.ts b/pages/api/teams/[teamId]/documents/[id]/versions/index.ts
index ad88ffff..df0d345e 100644
--- a/pages/api/teams/[teamId]/documents/[id]/versions/index.ts
+++ b/pages/api/teams/[teamId]/documents/[id]/versions/index.ts
@@ -97,6 +97,21 @@ export default async function handle(
},
});
+ if (type === "docs" || type === "slides") {
+ console.log("converting docx or pptx to pdf");
+ // Trigger convert-files-to-pdf task
+ await convertFilesToPdfTask.trigger(
+ {
+ documentVersionId: version.id,
+ teamId,
+ documentId,
+ },
+ {
+ idempotencyKey: `${teamId}-${version.id}`,
+ tags: [`team_${teamId}`, `document_${documentId}`],
+ },
+ );
+ }
// trigger document uploaded event to trigger convert-pdf-to-image job
if (type === "pdf") {
await client.sendEvent({
diff --git a/pages/api/teams/[teamId]/documents/index.ts b/pages/api/teams/[teamId]/documents/index.ts
index da9427af..c2901bfb 100644
--- a/pages/api/teams/[teamId]/documents/index.ts
+++ b/pages/api/teams/[teamId]/documents/index.ts
@@ -158,6 +158,22 @@ export default async function handle(
},
});
+ if (type === "docs" || type === "slides") {
+ console.log("converting docx or pptx to pdf");
+ // Trigger convert-files-to-pdf task
+ await convertFilesToPdfTask.trigger(
+ {
+ documentId: document.id,
+ documentVersionId: document.versions[0].id,
+ teamId,
+ },
+ {
+ idempotencyKey: `${teamId}-${document.versions[0].id}`,
+ tags: [`team_${teamId}`, `document_${document.id}`],
+ },
+ );
+ }
+
// skip triggering convert-pdf-to-image job for "notion" / "excel" documents
if (type === "pdf") {
// trigger document uploaded event to trigger convert-pdf-to-image job
diff --git a/trigger.config.ts b/trigger.config.ts
index 9af39bb7..c8aef3ee 100644
--- a/trigger.config.ts
+++ b/trigger.config.ts
@@ -15,7 +15,7 @@ export const config: TriggerConfig = {
randomize: true,
},
},
- dependenciesToBundle: ["nanoid"],
+ dependenciesToBundle: ["nanoid", /@sindresorhus/, "escape-string-regexp"],
additionalFiles: ["./prisma/schema.prisma"],
additionalPackages: ["[email protected]"],
postInstall: "npm exec --package prisma -- prisma generate",