Skip to content

Papermark is the open-source DocSend alternative and secure data rooms with built-in analytics and custom domains.

feat: add new mupdf conversion route

Marc Seitz517aa511f85c
diff --git a/lib/trigger/pdf-to-image-route.ts b/lib/trigger/pdf-to-image-route.ts
new file mode 100644
index 00000000..f68dac74
--- /dev/null
+++ b/lib/trigger/pdf-to-image-route.ts
@@ -0,0 +1,217 @@
+import { logger, task } from "@trigger.dev/sdk/v3";
+
+import { getFile } from "@/lib/files/get-file";
+import prisma from "@/lib/prisma";
+import { updateStatus } from "@/lib/utils/generate-trigger-status";
+
+type ConvertPdfToImagePayload = {
+ documentId: string;
+ documentVersionId: string;
+ teamId: string;
+ versionNumber?: number;
+};
+
+export const convertPdfToImageRoute = task({
+ id: "convert-pdf-to-image-route",
+ run: async (payload: ConvertPdfToImagePayload) => {
+ const { documentVersionId, teamId, documentId, versionNumber } = payload;
+
+ updateStatus({ progress: 0, text: "Initializing..." });
+
+ // 1. get file url from document version
+ const documentVersion = await prisma.documentVersion.findUnique({
+ where: {
+ id: documentVersionId,
+ },
+ select: {
+ file: true,
+ storageType: true,
+ numPages: true,
+ },
+ });
+
+ // if documentVersion is null, log error and return
+ if (!documentVersion) {
+ logger.error("File not found", { payload });
+ updateStatus({ progress: 0, text: "Document not found" });
+ return;
+ }
+
+ updateStatus({ progress: 10, text: "Retrieving file..." });
+
+ // 2. get signed url from file
+ const signedUrl = await getFile({
+ type: documentVersion.storageType,
+ data: documentVersion.file,
+ });
+
+ if (!signedUrl) {
+ logger.error("Failed to get signed url", { payload });
+ updateStatus({ progress: 0, text: "Failed to retrieve document" });
+ return;
+ }
+
+ let numPages = documentVersion.numPages;
+
+ // skip if the numPages are already defined
+ if (!numPages || numPages === 1) {
+ // 3. send file to api/convert endpoint in a task and get back number of pages
+
+ const response = await fetch(
+ `${process.env.NEXT_PUBLIC_BASE_URL}/api/mupdf/get-pages`,
+ {
+ method: "POST",
+ body: JSON.stringify({ url: signedUrl }),
+ headers: {
+ "Content-Type": "application/json",
+ },
+ },
+ );
+
+ if (!response.ok) {
+ logger.error("Failed to get number of pages", {
+ signedUrl,
+ response,
+ });
+ throw new Error("Failed to get number of pages");
+ }
+
+ const { numPagesResult } = (await response.json()) as {
+ numPagesResult: number;
+ };
+
+ if (numPagesResult < 1) {
+ logger.error("Failed to get number of pages", { payload });
+ updateStatus({ progress: 0, text: "Failed to get number of pages" });
+ return;
+ }
+
+ numPages = numPagesResult;
+ }
+
+ updateStatus({ progress: 20, text: "Converting document..." });
+
+ // 4. iterate through pages and upload to blob in a task
+ let currentPage = 0;
+ let conversionWithoutError = true;
+ for (var i = 0; i < numPages; ++i) {
+ if (!conversionWithoutError) {
+ break;
+ }
+
+ // increment currentPage
+ currentPage = i + 1;
+ try {
+ // send page number to api/convert-page endpoint in a task and get back page img url
+ const response = await fetch(
+ `${process.env.NEXT_PUBLIC_BASE_URL}/api/mupdf/convert-page`,
+ {
+ method: "POST",
+ body: JSON.stringify({
+ documentVersionId: documentVersionId,
+ pageNumber: currentPage,
+ url: signedUrl,
+ teamId: teamId,
+ }),
+ headers: {
+ "Content-Type": "application/json",
+ Authorization: `Bearer ${process.env.INTERNAL_API_KEY}`,
+ },
+ },
+ );
+
+ if (!response.ok) {
+ throw new Error("Failed to convert page");
+ }
+
+ const { documentPageId } = (await response.json()) as {
+ documentPageId: string;
+ };
+
+ logger.info(`Created document page for page ${currentPage}:`, {
+ documentPageId,
+ payload,
+ });
+ } catch (error: unknown) {
+ conversionWithoutError = false;
+ if (error instanceof Error) {
+ logger.error("Failed to convert page", {
+ error: error.message,
+ });
+ }
+ }
+
+ updateStatus({
+ progress: (currentPage / numPages) * 100,
+ text: `${currentPage} / ${numPages} pages processed`,
+ });
+ }
+
+ if (!conversionWithoutError) {
+ logger.error("Failed to process pages", { payload });
+ updateStatus({
+ progress: (currentPage / numPages) * 100,
+ text: `Error processing page ${currentPage} of ${numPages}`,
+ });
+ return;
+ }
+
+ // 5. after all pages are uploaded, update document version to hasPages = true
+ await prisma.documentVersion.update({
+ where: {
+ id: documentVersionId,
+ },
+ data: {
+ numPages: numPages,
+ hasPages: true,
+ isPrimary: true,
+ },
+ select: {
+ id: true,
+ hasPages: true,
+ isPrimary: true,
+ },
+ });
+
+ updateStatus({
+ progress: 90,
+ text: "Enabling pages...",
+ });
+
+ if (versionNumber) {
+ // after all pages are uploaded, update all other versions to be not primary
+ await prisma.documentVersion.updateMany({
+ where: {
+ documentId: documentId,
+ versionNumber: {
+ not: versionNumber,
+ },
+ },
+ data: {
+ isPrimary: false,
+ },
+ });
+ }
+
+ updateStatus({
+ progress: 95,
+ text: "Revalidating link...",
+ });
+
+ // initialize link revalidation for all the document's links
+ await fetch(
+ `${process.env.NEXTAUTH_URL}/api/revalidate?secret=${process.env.REVALIDATE_TOKEN}&documentId=${documentId}`,
+ );
+
+ updateStatus({
+ progress: 100,
+ text: "Processing complete",
+ });
+
+ return {
+ success: true,
+ message: "Successfully converted PDF to images",
+ totalPages: numPages,
+ };
+ },
+});
diff --git a/pages/api/teams/[teamId]/documents/[id]/versions/index.ts b/pages/api/teams/[teamId]/documents/[id]/versions/index.ts
index aa0badb5..ecc1cab2 100644
--- a/pages/api/teams/[teamId]/documents/[id]/versions/index.ts
+++ b/pages/api/teams/[teamId]/documents/[id]/versions/index.ts
@@ -10,6 +10,7 @@ import prisma from "@/lib/prisma";
import { getTeamWithUsersAndDocument } from "@/lib/team/helper";
import { convertFilesToPdfTask } from "@/lib/trigger/convert-files";
import { convertPdfToImage } from "@/lib/trigger/pdf-to-image";
+import { convertPdfToImageRoute } from "@/lib/trigger/pdf-to-image-route";
import { CustomUser } from "@/lib/types";
import { log } from "@/lib/utils";
import { conversionQueue } from "@/lib/utils/trigger-utils";
@@ -123,12 +124,12 @@ export default async function handle(
}
// trigger document uploaded event to trigger convert-pdf-to-image job
if (type === "pdf") {
- await convertPdfToImage.trigger(
+ await convertPdfToImageRoute.trigger(
{
documentId: documentId,
documentVersionId: version.id,
teamId,
- docId: version.file.split("/")[1], // Extract doc_xxxx from teamId/doc_xxxx/filename
+ // docId: version.file.split("/")[1], // Extract doc_xxxx from teamId/doc_xxxx/filename
versionNumber: version.versionNumber,
},
{
diff --git a/pages/api/teams/[teamId]/documents/agreement.ts b/pages/api/teams/[teamId]/documents/agreement.ts
index 50c1450b..eef61587 100644
--- a/pages/api/teams/[teamId]/documents/agreement.ts
+++ b/pages/api/teams/[teamId]/documents/agreement.ts
@@ -11,6 +11,7 @@ import prisma from "@/lib/prisma";
import { getTeamWithUsersAndDocument } from "@/lib/team/helper";
import { convertFilesToPdfTask } from "@/lib/trigger/convert-files";
import { convertPdfToImage } from "@/lib/trigger/pdf-to-image";
+import { convertPdfToImageRoute } from "@/lib/trigger/pdf-to-image-route";
import { CustomUser } from "@/lib/types";
import { getExtension, log } from "@/lib/utils";
import { conversionQueue } from "@/lib/utils/trigger-utils";
@@ -139,12 +140,12 @@ export default async function handle(
}
if (type === "pdf") {
- await convertPdfToImage.trigger(
+ await convertPdfToImageRoute.trigger(
{
documentId: document.id,
documentVersionId: document.versions[0].id,
teamId,
- docId: fileUrl.split("/")[1],
+ // docId: fileUrl.split("/")[1],
},
{
idempotencyKey: `${teamId}-${document.versions[0].id}`,
diff --git a/pages/api/teams/[teamId]/documents/index.ts b/pages/api/teams/[teamId]/documents/index.ts
index bb98f81d..cdd5cc54 100644
--- a/pages/api/teams/[teamId]/documents/index.ts
+++ b/pages/api/teams/[teamId]/documents/index.ts
@@ -18,6 +18,7 @@ import {
} from "@/lib/trigger/convert-files";
import { processVideo } from "@/lib/trigger/optimize-video-files";
import { convertPdfToImage } from "@/lib/trigger/pdf-to-image";
+import { convertPdfToImageRoute } from "@/lib/trigger/pdf-to-image-route";
import { CustomUser } from "@/lib/types";
import { getExtension, log } from "@/lib/utils";
import { conversionQueue } from "@/lib/utils/trigger-utils";
@@ -331,12 +332,11 @@ export default async function handle(
// skip triggering convert-pdf-to-image job for "notion" / "excel" documents
if (type === "pdf") {
- await convertPdfToImage.trigger(
+ await convertPdfToImageRoute.trigger(
{
documentId: document.id,
documentVersionId: document.versions[0].id,
teamId,
- docId: fileUrl.split("/")[1],
},
{
idempotencyKey: `${teamId}-${document.versions[0].id}`,