Papermark is the open-source DocSend alternative and secure data rooms with built-in analytics and custom domains.
feat: add trigger realtime for conversion progress / status
Marc Seitz85c647749e04
diff --git a/components/documents/file-process-status-bar.tsx b/components/documents/file-process-status-bar.tsxnew file mode 100644index 00000000..33694254--- /dev/null+++ b/components/documents/file-process-status-bar.tsx@@ -0,0 +1,73 @@+import useSWRImmutable from "swr/immutable";++import { Progress } from "@/components/ui/progress";++import { cn, fetcher } from "@/lib/utils";+import { useDocumentProgressStatus } from "@/lib/utils/use-progress-status";++export default function FileProcessStatusBar({+ documentVersionId,+ className,+ mutateDocument,+}: {+ documentVersionId: string;+ className?: string;+ mutateDocument: () => void;+}) {+ const { data } = useSWRImmutable<{ publicAccessToken: string }>(+ `/api/progress-token?documentVersionId=${documentVersionId}`,+ fetcher,+ );++ const { status: progressStatus, error: progressError } =+ useDocumentProgressStatus(documentVersionId, data?.publicAccessToken);++ if (progressStatus.state === "QUEUED") {+ return (+ <Progress+ value={0}+ text="Processing document..."+ className={cn(+ "w-full rounded-none text-[8px] font-semibold",+ className,+ )}+ />+ );+ }++ if (+ progressError ||+ ["FAILED", "CRASHED", "CANCELED", "SYSTEM_FAILURE"].includes(+ progressStatus.state,+ )+ ) {+ return (+ <Progress+ value={0}+ text={+ progressError?.message ||+ progressStatus.text ||+ "Error processing document"+ }+ error={true}+ className={cn(+ "w-full rounded-none text-[8px] font-semibold",+ className,+ )}+ />+ );+ }++ if (progressStatus.state === "COMPLETED") {+ mutateDocument();+ return null;+ }++ return (+ <Progress+ value={progressStatus.progress || 0}+ text={progressStatus.text || "Processing document..."}+ className={cn("w-full rounded-none text-[8px] font-semibold", className)}+ />+ );+}diff --git a/components/documents/process-status-bar.tsx b/components/documents/process-status-bar.tsxindex 582fa041..c3160691 100644--- a/components/documents/process-status-bar.tsx+++ b/components/documents/process-status-bar.tsx@@ -4,7 +4,7 @@ import { Progress } from "@/components/ui/progress";import { cn } from "@/lib/utils";-export default function ProcessStatusBar({+export default function FileProcessStatusBar({documentVersionId,className,}: {@@ -18,7 +18,7 @@ export default function ProcessStatusBar({return (<Progressvalue={0}- text="Processing document..."+ text="Processing file..."className={cn("w-full rounded-none text-[8px] font-semibold",className,@@ -31,7 +31,8 @@ export default function ProcessStatusBar({return (<Progressvalue={0}- text={error.message}+ text={error?.message || "Error processing file"}+ error={true}className={cn("w-full rounded-none text-[8px] font-semibold",className,@@ -45,13 +46,13 @@ export default function ProcessStatusBar({}const progress = Number(statuses[0]?.data?.progress) * 100 || 0;- const text = String(statuses[0]?.data?.text) || "";+ const text = String(statuses[0]?.data?.text) || "Processing file...";if (run.status === "FAILURE") {return (<Progressvalue={progress}- text={`Error processing document page ${Number(statuses[0]?.data?.currentPage)}`}+ text="Error processing file"error={true}className={cn("w-full rounded-none text-[8px] font-semibold",diff --git a/components/links/links-table.tsx b/components/links/links-table.tsxindex 27bd137a..c2f5f6c8 100644--- a/components/links/links-table.tsx+++ b/components/links/links-table.tsx@@ -47,6 +47,7 @@ import useLimits from "@/lib/swr/use-limits";import { LinkWithViews, WatermarkConfig } from "@/lib/types";import { cn, copyToClipboard, fetcher, nFormatter, timeAgo } from "@/lib/utils";+import FileProcessStatusBar from "../documents/file-process-status-bar";import ProcessStatusBar from "../documents/process-status-bar";import BarChart from "../shared/icons/bar-chart";import ChevronDown from "../shared/icons/chevron-down";@@ -63,10 +64,12 @@ export default function LinksTable({targetType,links,primaryVersion,+ mutateDocument,}: {targetType: "DOCUMENT" | "DATAROOM";links?: LinkWithViews[];primaryVersion?: DocumentVersion;+ mutateDocument: () => void;}) {const router = useRouter();const { plan } = usePlan();@@ -315,14 +318,31 @@ export default function LinksTable({)}>{/* Progress bar */}- {primaryVersion &&+ {/* {primaryVersion &&primaryVersion.type === "pdf" &&!primaryVersion.hasPages ? (<ProcessStatusBardocumentVersionId={primaryVersion.id}className="absolute bottom-0 left-0 right-0 top-0 z-20 flex h-full items-center gap-x-8"/>- ) : null}+ ) : null} */}++ {primaryVersion && !primaryVersion.hasPages && (+ <>+ {primaryVersion.type === "pdf" ? (+ <ProcessStatusBar+ documentVersionId={primaryVersion.id}+ className="absolute bottom-0 left-0 right-0 top-0 z-20 flex h-full items-center gap-x-8"+ />+ ) : (+ <FileProcessStatusBar+ documentVersionId={primaryVersion.id}+ className="absolute bottom-0 left-0 right-0 top-0 z-20 flex h-full items-center gap-x-8"+ mutateDocument={mutateDocument}+ />+ )}+ </>+ )}<div className="flex w-full whitespace-nowrap text-sm group-hover/cell:opacity-0">{link.domainIddiff --git a/lib/swr/use-document.ts b/lib/swr/use-document.tsindex 676100e2..5a601557 100644--- a/lib/swr/use-document.ts+++ b/lib/swr/use-document.ts@@ -17,7 +17,11 @@ export function useDocument() {id: string;};- const { data: document, error } = useSWR<DocumentWithVersion>(+ const {+ data: document,+ error,+ mutate,+ } = useSWR<DocumentWithVersion>(teamInfo?.currentTeam?.id &&id &&`/api/teams/${teamInfo?.currentTeam?.id}/documents/${encodeURIComponent(@@ -34,6 +38,7 @@ export function useDocument() {primaryVersion: document?.versions[0],loading: !error && !document,error,+ mutate,};}diff --git a/lib/trigger/convert-files.ts b/lib/trigger/convert-files.tsindex b7776940..198e90e4 100644--- a/lib/trigger/convert-files.ts+++ b/lib/trigger/convert-files.ts@@ -5,9 +5,10 @@ import { getFile } from "@/lib/files/get-file";import { putFileServer } from "@/lib/files/put-file-server";import prisma from "@/lib/prisma";+import { updateStatus } from "../utils/generate-trigger-status";import { getExtensionFromContentType } from "../utils/get-content-type";-type ConvertPayload = {+export type ConvertPayload = {documentId: string;documentVersionId: string;teamId: string;@@ -20,6 +21,8 @@ export const convertFilesToPdfTask = task({concurrencyLimit: 10,},run: async (payload: ConvertPayload) => {+ updateStatus({ progress: 0, text: "Initializing..." });+const team = await prisma.team.findUnique({where: {id: payload.teamId,@@ -57,6 +60,8 @@ export const convertFilesToPdfTask = task({!document.versions[0].originalFile ||!document.versions[0].contentType) {+ updateStatus({ progress: 0, text: "Document not found" });+logger.error("Document not found", {documentId: payload.documentId,documentVersionId: payload.documentVersionId,@@ -65,6 +70,8 @@ export const convertFilesToPdfTask = task({return;}+ updateStatus({ progress: 10, text: "Retrieving file..." });+const fileUrl = await getFile({data: document.versions[0].originalFile,type: document.versions[0].storageType,@@ -80,7 +87,9 @@ export const convertFilesToPdfTask = task({},]),);- formData.append("quality", "50");+ formData.append("quality", "75");++ updateStatus({ progress: 20, text: "Converting document..." });// Make the conversion requestconst conversionResponse = await retry.fetch(@@ -107,6 +116,7 @@ export const convertFilesToPdfTask = task({);if (!conversionResponse.ok) {+ updateStatus({ progress: 0, text: "Conversion failed" });const body = await conversionResponse.json();throw new Error(`Conversion failed: ${body.message} ${conversionResponse.status}`,@@ -123,6 +133,8 @@ export const convertFilesToPdfTask = task({const match = document.versions[0].originalFile.match(/(doc_[^\/]+)\//);const docId = match ? match[1] : undefined;+ updateStatus({ progress: 30, text: "Saving converted file..." });+// Save the converted file to the databaseconst { type: storageType, data } = await putFileServer({file: {@@ -135,6 +147,8 @@ export const convertFilesToPdfTask = task({});if (!data || !storageType) {+ updateStatus({ progress: 0, text: "Failed to save converted file" });+logger.error("Failed to save converted file to database", {documentId: payload.documentId,documentVersionId: payload.documentVersionId,@@ -156,6 +170,8 @@ export const convertFilesToPdfTask = task({},});+ updateStatus({ progress: 40, text: "Initiating document processing..." });+// v2: trigger document uploaded event to trigger convert-pdf-to-image jobawait client.sendEvent({id: payload.documentVersionId, // unique eventId for the rundiff --git a/lib/utils/generate-trigger-auth-token.ts b/lib/utils/generate-trigger-auth-token.tsnew file mode 100644index 00000000..2d4d95bd--- /dev/null+++ b/lib/utils/generate-trigger-auth-token.ts@@ -0,0 +1,12 @@+import { auth } from "@trigger.dev/sdk/v3";++export async function generateTriggerPublicAccessToken(tag: string) {+ return auth.createPublicToken({+ scopes: {+ read: {+ tags: [tag],+ },+ },+ expirationTime: "15m",+ });+}diff --git a/lib/utils/generate-trigger-status.ts b/lib/utils/generate-trigger-status.tsnew file mode 100644index 00000000..7581d686--- /dev/null+++ b/lib/utils/generate-trigger-status.ts@@ -0,0 +1,31 @@+import { metadata } from "@trigger.dev/sdk/v3";+import { z } from "zod";++const ZDocumentProgressStatus = z.object({+ progress: z.number(),+ text: z.string(),+});++type TDocumentProgressStatus = z.infer<typeof ZDocumentProgressStatus>;++const ZDocumentProgressMetadata = z.object({+ status: ZDocumentProgressStatus,+});++type TDocumentProgressMetadata = z.infer<typeof ZDocumentProgressMetadata>;++/**+ * Update the status of the convert document task. Wraps the `metadata.set` method.+ */+export function updateStatus(status: TDocumentProgressStatus) {+ // `metadata.set` can be used to update the status of the task+ // as long as `updateStatus` is called within the task's `run` function.+ metadata.set("status", status);+}++/**+ * Parse the status from the metadata.+ */+export function parseStatus(data: unknown): TDocumentProgressStatus {+ return ZDocumentProgressMetadata.parse(data).status;+}diff --git a/lib/utils/use-progress-status.ts b/lib/utils/use-progress-status.tsnew file mode 100644index 00000000..89675b85--- /dev/null+++ b/lib/utils/use-progress-status.ts@@ -0,0 +1,45 @@+"use client";++import { type RunStatus } from "@trigger.dev/core/v3";+import { useRealtimeRunsWithTag } from "@trigger.dev/react-hooks";++import { parseStatus } from "@/lib/utils/generate-trigger-status";++interface IDocumentProgressStatus {+ state: RunStatus;+ progress: number;+ text: string;+}++export function useDocumentProgressStatus(+ documentVersionId: string,+ publicAccessToken: string | undefined,+) {+ const { runs, error } = useRealtimeRunsWithTag(+ `version:${documentVersionId}`,+ {+ enabled: !!publicAccessToken,+ accessToken: publicAccessToken,+ },+ );++ const run = runs[0];+ const status: IDocumentProgressStatus = {+ state: run?.status ?? "QUEUED",+ progress: 0,+ text: "Initializing...",+ };++ // Parse metadata if available+ if (run?.metadata) {+ const { progress, text } = parseStatus(run.metadata);+ status.progress = progress;+ status.text = text;+ }++ return {+ status,+ error,+ run,+ };+}diff --git a/package-lock.json b/package-lock.jsonindex e4d6f9bf..2ee46c44 100644--- a/package-lock.json+++ b/package-lock.json@@ -50,9 +50,10 @@"@tanstack/react-table": "^8.20.5","@teamhanko/passkeys-next-auth-provider": "^0.2.7","@tremor/react": "^3.18.4",- "@trigger.dev/nextjs": "3.2.1",- "@trigger.dev/react": "3.2.1",- "@trigger.dev/sdk": "3.2.1",+ "@trigger.dev/nextjs": "3.3.1",+ "@trigger.dev/react": "3.3.1",+ "@trigger.dev/react-hooks": "^3.3.1",+ "@trigger.dev/sdk": "3.3.1","@tus/s3-store": "^1.6.0","@tus/server": "^1.9.0","@tus/utils": "^0.4.0",@@ -125,7 +126,7 @@},"devDependencies": {"@tailwindcss/forms": "^0.5.9",- "@trigger.dev/build": "3.2.1",+ "@trigger.dev/build": "3.3.1","@trivago/prettier-plugin-sort-imports": "^4.3.0","@types/bcryptjs": "^2.4.6","@types/js-cookie": "^3.0.6",@@ -6230,12 +6231,12 @@}},"node_modules/@trigger.dev/build": {- "version": "3.2.1",- "resolved": "https://registry.npmjs.org/@trigger.dev/build/-/build-3.2.1.tgz",- "integrity": "sha512-XK/ixe1MbaOQKDKYLxg4pict6kl3H9JFgqzK5UkbalbWH5T7RREZYDE/rctw6z3YSUK+tehQtl79dg7xfUkhfw==",+ "version": "3.3.1",+ "resolved": "https://registry.npmjs.org/@trigger.dev/build/-/build-3.3.1.tgz",+ "integrity": "sha512-lO7DgvzfGmZRlhT/fJG5meStKXbn5JpHA9eVybMagVa+hBYtLSeMZf+PV7v49kShOBYz4iZcgldMvU00epfzFg==","dev": true,"dependencies": {- "@trigger.dev/core": "3.2.1",+ "@trigger.dev/core": "3.3.1","pkg-types": "^1.1.3","tinyglobby": "^0.2.2","tsconfck": "3.1.3"@@ -6245,9 +6246,9 @@}},"node_modules/@trigger.dev/core": {- "version": "3.2.1",- "resolved": "https://registry.npmjs.org/@trigger.dev/core/-/core-3.2.1.tgz",- "integrity": "sha512-7M4QILckaCXVwLmbLiTLWjy+y812ITwbSAF/Xj/WdMBIQwMipVuPvznq9VyQt5dQzvPrOpNOiQxE94hgdCReIg==",+ "version": "3.3.1",+ "resolved": "https://registry.npmjs.org/@trigger.dev/core/-/core-3.3.1.tgz",+ "integrity": "sha512-F3jgGtjPDRwvycI9yNR+iLAsMMxranaF/0FOUsUu9r7HER2odBPAmRmh0j8F6Yhsta9aVfwyFm9Hg1xDgX9C5A==","dependencies": {"@electric-sql/client": "0.7.1","@google-cloud/precise-date": "^4.0.0",@@ -6336,9 +6337,9 @@}},"node_modules/@trigger.dev/nextjs": {- "version": "3.2.1",- "resolved": "https://registry.npmjs.org/@trigger.dev/nextjs/-/nextjs-3.2.1.tgz",- "integrity": "sha512-OVwh0IiDv/ziiN5X4CxCU5QJgLg3wYMZlwLi5ZdvpqALpQX3wuPSGoSJ3qZJ9usKZ7nrc+/6EgyzEPczH59yvw==",+ "version": "3.3.1",+ "resolved": "https://registry.npmjs.org/@trigger.dev/nextjs/-/nextjs-3.3.1.tgz",+ "integrity": "sha512-TEhdvI/KpyNVVvmtiLfDXpfTLiau52rOG6BthZ8/f+igcW5QDQoIvuygEXSpSV0dNxv4HvM8Dsw5l8e4Hk+qtA==","dependencies": {"debug": "^4.3.4"},@@ -6351,9 +6352,9 @@}},"node_modules/@trigger.dev/react": {- "version": "3.2.1",- "resolved": "https://registry.npmjs.org/@trigger.dev/react/-/react-3.2.1.tgz",- "integrity": "sha512-MDtqnGt0WDYQcW5nkundv1TuDuY+lpr6esigz0gqYzS0melIy6tU2c9mSWvQArPWwH3SQI/yWI0BTlPVJLaa5A==",+ "version": "3.3.1",+ "resolved": "https://registry.npmjs.org/@trigger.dev/react/-/react-3.3.1.tgz",+ "integrity": "sha512-SYvQT4eurMCkTHdg7MyYaTJoZQT5wKbeX2PyhcFbBemrjvNQDs/SlNomTmC4xXfHP5Pd43YyDAsOt8W70PAkYQ==","dependencies": {"@tanstack/react-query": "5.0.0-beta.2","@trigger.dev/core": "~2.3.0 || ^3.0.0",@@ -6364,6 +6365,22 @@"react": "^16.8.0 || ^17.0.0 || ^18"}},+ "node_modules/@trigger.dev/react-hooks": {+ "version": "3.3.1",+ "resolved": "https://registry.npmjs.org/@trigger.dev/react-hooks/-/react-hooks-3.3.1.tgz",+ "integrity": "sha512-M2BlFA9n62sjtwebylnFaHnB29oMpthVLwb78WH/nR0U7lXp4dYQ6dhcUkJ8AMLWDVLzO3Vp3fh7LwJqRv0/Yw==",+ "dependencies": {+ "@trigger.dev/core": "^3.3.1",+ "swr": "^2.2.5"+ },+ "engines": {+ "node": ">=18.20.0"+ },+ "peerDependencies": {+ "react": ">=18 || >=19.0.0-beta",+ "react-dom": ">=18 || >=19.0.0-beta"+ }+ },"node_modules/@trigger.dev/react/node_modules/zod": {"version": "3.22.3","resolved": "https://registry.npmjs.org/zod/-/zod-3.22.3.tgz",@@ -6373,14 +6390,14 @@}},"node_modules/@trigger.dev/sdk": {- "version": "3.2.1",- "resolved": "https://registry.npmjs.org/@trigger.dev/sdk/-/sdk-3.2.1.tgz",- "integrity": "sha512-KOHkp6M0zxWAgeRtN2/LPS64HtRTuP219ADeqkoNPRY9CgQjnXGeXT+5sEbSOU8w5+AkbRq+wTjcA9tqMTQsBQ==",+ "version": "3.3.1",+ "resolved": "https://registry.npmjs.org/@trigger.dev/sdk/-/sdk-3.3.1.tgz",+ "integrity": "sha512-v8zBuroUGkI1emQvJIFo22K8vkAq5FMuVKipJFRWZff5qSplj3rI2O3LzOARDDnw/lCiAkZFTB2qNsgMEfVXrA==","dependencies": {"@opentelemetry/api": "1.9.0","@opentelemetry/api-logs": "0.52.1","@opentelemetry/semantic-conventions": "1.25.1",- "@trigger.dev/core": "3.2.1",+ "@trigger.dev/core": "3.3.1","chalk": "^5.2.0","cronstrue": "^2.21.0","debug": "^4.3.4",diff --git a/package.json b/package.jsonindex 5074775b..454e0342 100644--- a/package.json+++ b/package.json@@ -61,9 +61,10 @@"@tanstack/react-table": "^8.20.5","@teamhanko/passkeys-next-auth-provider": "^0.2.7","@tremor/react": "^3.18.4",- "@trigger.dev/nextjs": "3.2.1",- "@trigger.dev/react": "3.2.1",- "@trigger.dev/sdk": "3.2.1",+ "@trigger.dev/nextjs": "3.3.1",+ "@trigger.dev/react": "3.3.1",+ "@trigger.dev/react-hooks": "^3.3.1",+ "@trigger.dev/sdk": "3.3.1","@tus/s3-store": "^1.6.0","@tus/server": "^1.9.0","@tus/utils": "^0.4.0",@@ -136,7 +137,7 @@},"devDependencies": {"@tailwindcss/forms": "^0.5.9",- "@trigger.dev/build": "3.2.1",+ "@trigger.dev/build": "3.3.1","@trivago/prettier-plugin-sort-imports": "^4.3.0","@types/bcryptjs": "^2.4.6","@types/js-cookie": "^3.0.6",@@ -160,4 +161,4 @@"react-pdf": "8.0.2"}}-}+}\ No newline at end of filediff --git a/pages/api/progress-token.ts b/pages/api/progress-token.tsnew file mode 100644index 00000000..6ae71bf5--- /dev/null+++ b/pages/api/progress-token.ts@@ -0,0 +1,28 @@+import { NextApiRequest, NextApiResponse } from "next";++import { generateTriggerPublicAccessToken } from "@/lib/utils/generate-trigger-auth-token";++export default async function handle(+ req: NextApiRequest,+ res: NextApiResponse,+) {+ if (req.method !== "GET") {+ return res.status(405).json({ error: "Method not allowed" });+ }++ const { documentVersionId } = req.query;++ if (!documentVersionId || typeof documentVersionId !== "string") {+ return res.status(400).json({ error: "Document version ID is required" });+ }++ try {+ const publicAccessToken = await generateTriggerPublicAccessToken(+ `version:${documentVersionId}`,+ );+ return res.status(200).json({ publicAccessToken });+ } catch (error) {+ console.error("Error generating token:", error);+ return res.status(500).json({ error: "Failed to generate token" });+ }+}diff --git a/pages/api/teams/[teamId]/documents/index.ts b/pages/api/teams/[teamId]/documents/index.tsindex 702f4f57..ec9a7306 100644--- a/pages/api/teams/[teamId]/documents/index.ts+++ b/pages/api/teams/[teamId]/documents/index.ts@@ -3,6 +3,7 @@ import { NextApiRequest, NextApiResponse } from "next";import { authOptions } from "@/pages/api/auth/[...nextauth]";import { client } from "@/trigger";import { DocumentStorageType, Prisma } from "@prisma/client";+import { RunHandle } from "@trigger.dev/sdk/v3";import { getServerSession } from "next-auth/next";import { parsePageId } from "notion-utils";@@ -12,6 +13,7 @@ import notion from "@/lib/notion";import prisma from "@/lib/prisma";import { getTeamWithUsersAndDocument } from "@/lib/team/helper";import {+ ConvertPayload,convertCadToPdfTask,convertFilesToPdfTask,} from "@/lib/trigger/convert-files";@@ -274,7 +276,11 @@ export default async function handle(},{idempotencyKey: `${teamId}-${document.versions[0].id}`,- tags: [`team_${teamId}`, `document_${document.id}`],+ tags: [+ `team_${teamId}`,+ `document_${document.id}`,+ `version:${document.versions[0].id}`,+ ],},);}@@ -290,7 +296,11 @@ export default async function handle(},{idempotencyKey: `${teamId}-${document.versions[0].id}`,- tags: [`team_${teamId}`, `document_${document.id}`],+ tags: [+ `team_${teamId}`,+ `document_${document.id}`,+ `version:${document.versions[0].id}`,+ ],},);}diff --git a/pages/documents/[id]/index.tsx b/pages/documents/[id]/index.tsxindex 7732b6bd..dbe46018 100644--- a/pages/documents/[id]/index.tsx+++ b/pages/documents/[id]/index.tsx@@ -19,7 +19,12 @@ import { useDocument, useDocumentLinks } from "@/lib/swr/use-document";import useLimits from "@/lib/swr/use-limits";export default function DocumentPage() {- const { document: prismaDocument, primaryVersion, error } = useDocument();+ const {+ document: prismaDocument,+ primaryVersion,+ error,+ mutate: mutateDocument,+ } = useDocument();const { links } = useDocumentLinks();const teamInfo = useTeam();@@ -95,6 +100,7 @@ export default function DocumentPage() {links={links}targetType={"DOCUMENT"}primaryVersion={primaryVersion}+ mutateDocument={mutateDocument}/>{/* Visitors */}