Papermark is the open-source DocSend alternative and secure data rooms with built-in analytics and custom domains.
feat: add trigger for converting docs to pdf
Marc Seitz0bf6a70713e7
diff --git a/jobs/convert-pdf-to-image.ts b/jobs/convert-pdf-to-image.tsindex c53d909c..09b3c22c 100644--- a/jobs/convert-pdf-to-image.ts+++ b/jobs/convert-pdf-to-image.ts@@ -40,7 +40,7 @@ client.defineJob({);// 1. get file url from document version- const documentUrl = await io.runTask("get-document-url", async () => {+ const documentVersion = await io.runTask("get-document-url", async () => {return prisma.documentVersion.findUnique({where: {id: documentVersionId,@@ -53,8 +53,8 @@ client.defineJob({});});- // if documentUrl is null, log error and return- if (!documentUrl) {+ // if documentVersion is null, log error and return+ if (!documentVersion) {await io.logger.error("File not found", { payload });await processingDocumentStatus.update("error", {//set data, this overrides the previous value@@ -72,8 +72,8 @@ client.defineJob({// 2. get signed url from fileconst signedUrl = await io.runTask("get-signed-url", async () => {return await getFile({- type: documentUrl.storageType,- data: documentUrl.file,+ type: documentVersion.storageType,+ data: documentVersion.file,});});@@ -92,17 +92,17 @@ client.defineJob({return;}- let numPages = documentUrl.numPages;+ let numPages = documentVersion.numPages;// skip if the numPages are already defined- if (!numPages) {+ if (!numPages || numPages === 1) {// 3. send file to api/convert endpoint in a task and get back number of pagesconst muDocument = await io.runTask("get-number-of-pages", async () => {const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL}/api/mupdf/get-pages`,{method: "POST",- body: JSON.stringify({ url: documentUrl.file }),+ body: JSON.stringify({ url: signedUrl }),headers: {"Content-Type": "application/json",},@@ -110,6 +110,14 @@ client.defineJob({);await io.logger.info("log response", { response });+ if (!response.ok) {+ await io.logger.error("Failed to get number of pages", {+ signedUrl,+ response,+ });+ throw new Error("Failed to get number of pages");+ }+const { numPages } = (await response.json()) as { numPages: number };return { numPages };});@@ -221,6 +229,7 @@ client.defineJob({id: documentVersionId,},data: {+ numPages: numPages,hasPages: true,isPrimary: true,},diff --git a/lib/files/put-file-server.ts b/lib/files/put-file-server.tsindex ba805e30..bd1030e4 100644--- a/lib/files/put-file-server.ts+++ b/lib/files/put-file-server.ts@@ -67,8 +67,12 @@ const putFileInS3Server = async ({docId = newId("doc");}- if (file.type !== "image/png" && file.type !== "image/jpeg") {- throw new Error("Only PNG or JPEG files are supported");+ if (+ file.type !== "image/png" &&+ file.type !== "image/jpeg" &&+ file.type !== "application/pdf"+ ) {+ throw new Error("Only PNG, JPEG or PDF files are supported");}const client = getS3Client();diff --git a/lib/trigger/convert-files.ts b/lib/trigger/convert-files.tsnew file mode 100644index 00000000..08d16e0e--- /dev/null+++ b/lib/trigger/convert-files.ts@@ -0,0 +1,178 @@+import { client } from "@/trigger";+import { envvars, logger, retry, task } from "@trigger.dev/sdk/v3";++import { getFile } from "@/lib/files/get-file";+import { putFileServer } from "@/lib/files/put-file-server";+import prisma from "@/lib/prisma";++type ConvertPayload = {+ documentId: string;+ documentVersionId: string;+ teamId: string;+};++export const convertFilesToPdfTask = task({+ id: "convert-files-to-pdf",+ retry: { maxAttempts: 3 },+ queue: {+ concurrencyLimit: 10,+ },+ run: async (payload: ConvertPayload) => {+ const team = await prisma.team.findUnique({+ where: {+ id: payload.teamId,+ },+ });++ if (!team) {+ logger.error("Team not found", { teamId: payload.teamId });+ return;+ }++ const document = await prisma.document.findUnique({+ where: {+ id: payload.documentId,+ },+ select: {+ name: true,+ versions: {+ where: {+ id: payload.documentVersionId,+ },+ select: {+ file: true,+ originalFile: true,+ contentType: true,+ storageType: true,+ },+ },+ },+ });++ if (+ !document ||+ !document.versions[0] ||+ !document.versions[0].originalFile ||+ !document.versions[0].contentType+ ) {+ logger.error("Document not found", {+ documentId: payload.documentId,+ documentVersionId: payload.documentVersionId,+ teamId: payload.teamId,+ });+ return;+ }++ const fileUrl = await getFile({+ data: document.versions[0].originalFile,+ type: document.versions[0].storageType,+ });+ const fileResponse = await fetch(fileUrl);+ const fileBuffer = Buffer.from(await fileResponse.arrayBuffer());++ // Prepare form data+ const formData = new FormData();+ formData.append(+ "files",+ new Blob([fileBuffer], {+ type: document.versions[0].contentType,+ }),+ document.name,+ );+ formData.append("quality", "50");+ console.log("formData", formData);++ // Make the conversion request+ const conversionResponse = await retry.fetch(+ `${process.env.NEXT_PRIVATE_CONVERSION_BASE_URL}/forms/libreoffice/convert`,+ {+ method: "POST",+ body: formData,+ headers: {+ Authorization: `Basic ${process.env.NEXT_PRIVATE_INTERNAL_AUTH_TOKEN}`,+ },+ retry: {+ byStatus: {+ "500-599": {+ strategy: "backoff",+ maxAttempts: 3,+ factor: 2,+ minTimeoutInMs: 1_000,+ maxTimeoutInMs: 30_000,+ randomize: false,+ },+ },+ },+ },+ );++ if (!conversionResponse.ok) {+ const body = await conversionResponse.json();+ throw new Error(+ `Conversion failed: ${body.message} ${conversionResponse.status}`,+ );+ }++ const conversionBuffer = Buffer.from(+ await conversionResponse.arrayBuffer(),+ );++ console.log("conversionBuffer", conversionBuffer);++ // get docId from url with starts with "doc_" with regex+ const match = document.versions[0].originalFile.match(/(doc_[^\/]+)\//);+ const docId = match ? match[1] : undefined;++ // Save the converted file to the database+ const { type: storageType, data } = await putFileServer({+ file: {+ name: `${document.name}.pdf`,+ type: "application/pdf",+ buffer: conversionBuffer,+ },+ teamId: payload.teamId,+ docId: docId,+ });++ if (!data || !storageType) {+ logger.error("Failed to save converted file to database", {+ documentId: payload.documentId,+ documentVersionId: payload.documentVersionId,+ teamId: payload.teamId,+ docId: docId,+ });+ return;+ }++ console.log("data from conversion", data);+ console.log("storageType from conversion", storageType);++ await prisma.documentVersion.update({+ where: { id: payload.documentVersionId },+ data: {+ file: data,+ type: "pdf",+ storageType: storageType,+ },+ });++ // v2: trigger document uploaded event to trigger convert-pdf-to-image job+ await client.sendEvent({+ id: payload.documentVersionId, // unique eventId for the run+ name: "document.uploaded",+ payload: {+ documentVersionId: payload.documentVersionId,+ teamId: payload.teamId,+ documentId: payload.documentId,+ },+ });++ logger.info("Document converted", {+ documentId: payload.documentId,+ documentVersionId: payload.documentVersionId,+ teamId: payload.teamId,+ docId: docId,+ });+ return;+ },+});diff --git a/pages/api/teams/[teamId]/documents/[id]/versions/index.ts b/pages/api/teams/[teamId]/documents/[id]/versions/index.tsindex ad88ffff..df0d345e 100644--- a/pages/api/teams/[teamId]/documents/[id]/versions/index.ts+++ b/pages/api/teams/[teamId]/documents/[id]/versions/index.ts@@ -97,6 +97,21 @@ export default async function handle(},});+ if (type === "docs" || type === "slides") {+ console.log("converting docx or pptx to pdf");+ // Trigger convert-files-to-pdf task+ await convertFilesToPdfTask.trigger(+ {+ documentVersionId: version.id,+ teamId,+ documentId,+ },+ {+ idempotencyKey: `${teamId}-${version.id}`,+ tags: [`team_${teamId}`, `document_${documentId}`],+ },+ );+ }// trigger document uploaded event to trigger convert-pdf-to-image jobif (type === "pdf") {await client.sendEvent({diff --git a/pages/api/teams/[teamId]/documents/index.ts b/pages/api/teams/[teamId]/documents/index.tsindex da9427af..c2901bfb 100644--- a/pages/api/teams/[teamId]/documents/index.ts+++ b/pages/api/teams/[teamId]/documents/index.ts@@ -158,6 +158,22 @@ export default async function handle(},});+ if (type === "docs" || type === "slides") {+ console.log("converting docx or pptx to pdf");+ // Trigger convert-files-to-pdf task+ await convertFilesToPdfTask.trigger(+ {+ documentId: document.id,+ documentVersionId: document.versions[0].id,+ teamId,+ },+ {+ idempotencyKey: `${teamId}-${document.versions[0].id}`,+ tags: [`team_${teamId}`, `document_${document.id}`],+ },+ );+ }+// skip triggering convert-pdf-to-image job for "notion" / "excel" documentsif (type === "pdf") {// trigger document uploaded event to trigger convert-pdf-to-image jobdiff --git a/trigger.config.ts b/trigger.config.tsindex 9af39bb7..c8aef3ee 100644--- a/trigger.config.ts+++ b/trigger.config.ts@@ -15,7 +15,7 @@ export const config: TriggerConfig = {randomize: true,},},- dependenciesToBundle: ["nanoid"],+ dependenciesToBundle: ["nanoid", /@sindresorhus/, "escape-string-regexp"],additionalFiles: ["./prisma/schema.prisma"],additionalPackages: ["[email protected]"],postInstall: "npm exec --package prisma -- prisma generate",