Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix: process large Loom files with multipart uploads
Richie McIlroy617dcf6ee8bb
diff --git a/apps/web/__tests__/unit/loom-file-processing.test.ts b/apps/web/__tests__/unit/loom-file-processing.test.tsnew file mode 100644index 000000000..ca5c3d8e5--- /dev/null+++ b/apps/web/__tests__/unit/loom-file-processing.test.ts@@ -0,0 +1,129 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ transition: vi.fn(),+ start: vi.fn(),+ setError: vi.fn(),+}));++vi.mock("@/lib/video-processing", () => ({+ transitionVideoToProcessing: mocks.transition,+ setVideoProcessingError: mocks.setError,+}));++vi.mock("@/lib/loom-import-start", () => ({+ startLoomImportWorkflow: mocks.start,+ LoomImportStartError: class extends Error {+ constructor(+ readonly canRetry: boolean,+ cause: unknown,+ ) {+ super("Loom startup failed", { cause });+ }+ },+}));++describe("large Loom file processing", () => {+ const input = {+ videoId: "existing-video" as never,+ userId: "owner-id",+ loomVideoId: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ rawFileKey: "owner-id/existing-video/raw-upload.mp4",+ bucketId: null,+ processingMessage: "Processing Loom video...",+ startFailureMessage: "Loom file processing could not start.",+ };++ beforeEach(() => {+ vi.resetAllMocks();+ mocks.transition.mockResolvedValue("started");+ mocks.start.mockResolvedValue(undefined);+ mocks.setError.mockResolvedValue(undefined);+ });++ it("claims the existing upload and reuses its raw source through the multipart Loom workflow", async () => {+ const { startLoomFileProcessingWorkflow } = await import(+ "@/lib/loom-file-processing"+ );+ await expect(startLoomFileProcessingWorkflow(input)).resolves.toBe(+ "started",+ );+ expect(mocks.transition).toHaveBeenCalledExactlyOnceWith({+ videoId: input.videoId,+ rawFileKey: input.rawFileKey,+ processingMessage: input.processingMessage,+ });+ expect(mocks.start).toHaveBeenCalledExactlyOnceWith({+ videoId: input.videoId,+ userId: input.userId,+ rawFileKey: input.rawFileKey,+ bucketId: null,+ loomVideoId: input.loomVideoId,+ reuseExistingRawUpload: true,+ });+ expect(mocks.setError).not.toHaveBeenCalled();+ });++ it.each(["already-processing", "already-complete"])(+ "does not dispatch another run when the upload is %s",+ async (status) => {+ mocks.transition.mockResolvedValue(status);+ const { startLoomFileProcessingWorkflow } = await import(+ "@/lib/loom-file-processing"+ );+ await expect(startLoomFileProcessingWorkflow(input)).resolves.toBe(+ status,+ );+ expect(mocks.start).not.toHaveBeenCalled();+ },+ );++ it("rejects an invalid source before claiming the upload", async () => {+ const { startLoomFileProcessingWorkflow } = await import(+ "@/lib/loom-file-processing"+ );+ await expect(+ startLoomFileProcessingWorkflow({ ...input, loomVideoId: "invalid" }),+ ).rejects.toThrow("no valid Loom source mapping");+ expect(mocks.transition).not.toHaveBeenCalled();+ expect(mocks.start).not.toHaveBeenCalled();+ });++ it("does not dispatch when claiming the upload fails", async () => {+ mocks.transition.mockRejectedValue(new Error("Claim changed"));+ const { startLoomFileProcessingWorkflow } = await import(+ "@/lib/loom-file-processing"+ );+ await expect(startLoomFileProcessingWorkflow(input)).rejects.toThrow(+ "Claim changed",+ );+ expect(mocks.start).not.toHaveBeenCalled();+ expect(mocks.setError).not.toHaveBeenCalled();+ });++ it("restores a retryable error only after a confirmed queue rejection", async () => {+ const { LoomImportStartError } = await import("@/lib/loom-import-start");+ const error = new LoomImportStartError(true, new Error("Rejected"));+ mocks.start.mockRejectedValue(error);+ const { startLoomFileProcessingWorkflow } = await import(+ "@/lib/loom-file-processing"+ );+ await expect(startLoomFileProcessingWorkflow(input)).rejects.toBe(error);+ expect(mocks.setError).toHaveBeenCalledExactlyOnceWith(+ input.videoId,+ input.startFailureMessage,+ error,+ );+ });++ it("keeps an uncertain dispatch claimed so it cannot be started twice", async () => {+ const { LoomImportStartError } = await import("@/lib/loom-import-start");+ const error = new LoomImportStartError(false, new Error("Response lost"));+ mocks.start.mockRejectedValue(error);+ const { startLoomFileProcessingWorkflow } = await import(+ "@/lib/loom-file-processing"+ );+ await expect(startLoomFileProcessingWorkflow(input)).rejects.toBe(error);+ expect(mocks.setError).not.toHaveBeenCalled();+ });+});diff --git a/apps/web/__tests__/unit/loom-import.test.ts b/apps/web/__tests__/unit/loom-import.test.tsindex 523425255..213676322 100644--- a/apps/web/__tests__/unit/loom-import.test.ts+++ b/apps/web/__tests__/unit/loom-import.test.ts@@ -10,6 +10,7 @@ const revalidatePathMock = vi.fn();const storageGetWritableAccessForUserMock = vi.hoisted(() => vi.fn());const storageGetAccessForVideoMock = vi.hoisted(() => vi.fn());const startVideoProcessingWorkflowMock = vi.hoisted(() => vi.fn());+const startLoomFileProcessingWorkflowMock = vi.hoisted(() => vi.fn());const uploadedFileHeadMock = vi.hoisted(() => vi.fn());const checkRateLimitMock = vi.hoisted(() => vi.fn());const headersMock = vi.hoisted(() => vi.fn());@@ -176,6 +177,11 @@ vi.mock("@/lib/video-processing", () => ({startVideoProcessingWorkflow: startVideoProcessingWorkflowMock,}));+vi.mock("@/lib/loom-file-processing", () => ({+ LOOM_FILE_SINGLE_UPLOAD_MAX_BYTES: 5 * 1024 ** 3,+ startLoomFileProcessingWorkflow: startLoomFileProcessingWorkflowMock,+}));+vi.mock("@/lib/video-storage", () => ({decodeStorageVideo: (video: unknown) => video,}));@@ -1634,6 +1640,7 @@ describe("resuming an uploaded concierge Loom file", () => {metadata: { loomImportRun: { runId: "old-run", dispatch: "accepted" } },};const uploaded = {+ loomVideoId: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",video: {id: "existing-video",ownerId: "owner-id",@@ -1658,12 +1665,13 @@ describe("resuming an uploaded concierge Loom file", () => {conciergeMocks.reserveConciergeImport.mockResolvedValue("lease");conciergeMocks.releaseConciergeImport.mockResolvedValue(undefined);storageGetAccessForVideoMock.mockReturnValue(- Effect.succeed([{ headObject: uploadedFileHeadMock }]),+ Effect.succeed([{ provider: "s3", headObject: uploadedFileHeadMock }]),);uploadedFileHeadMock.mockReturnValue(Effect.succeed({ ContentLength: 6 * 1024 ** 3 }),);startVideoProcessingWorkflowMock.mockResolvedValue("started");+ startLoomFileProcessingWorkflowMock.mockResolvedValue("started");getRunMock.mockReturnValue({ status: Promise.resolve("failed") });});@@ -1718,19 +1726,60 @@ describe("resuming an uploaded concierge Loom file", () => {ok: true,value: { status: "started" },});- expect(startVideoProcessingWorkflowMock).toHaveBeenCalledExactlyOnceWith({- videoId: "existing-video",- userId: "owner-id",- rawFileKey: uploaded.rawFileKey,- bucketId: null,- processingMessage: "Processing Loom video...",- startFailureMessage: "Loom file processing could not start.",- });+ expect(startLoomFileProcessingWorkflowMock).toHaveBeenCalledExactlyOnceWith(+ {+ videoId: "existing-video",+ userId: "owner-id",+ rawFileKey: uploaded.rawFileKey,+ bucketId: null,+ processingMessage: "Processing Loom video...",+ startFailureMessage: "Loom file processing could not start.",+ loomVideoId: uploaded.loomVideoId,+ },+ );+ expect(startVideoProcessingWorkflowMock).not.toHaveBeenCalled();expect(mockDb.insert).not.toHaveBeenCalled();expect(mockDb.update).not.toHaveBeenCalled();expect(storageGetWritableAccessForUserMock).not.toHaveBeenCalled();});+ it.each([1024, 5 * 1024 ** 3])(+ "keeps the existing file workflow for a %i-byte upload",+ async (bytes) => {+ whereMock+ .mockReturnValueOnce(withLimit([recording]))+ .mockReturnValueOnce(withLimit([uploaded]));+ uploadedFileHeadMock.mockReturnValue(+ Effect.succeed({ ContentLength: bytes }),+ );+ const { resumeConciergeLoomFileUpload } = await import("@/actions/loom");+ await expect(resumeConciergeLoomFileUpload(input)).resolves.toMatchObject(+ {+ ok: true,+ },+ );+ expect(startVideoProcessingWorkflowMock).toHaveBeenCalledOnce();+ expect(startLoomFileProcessingWorkflowMock).not.toHaveBeenCalled();+ },+ );++ it("keeps non-S3 storage on its existing processing path", async () => {+ whereMock+ .mockReturnValueOnce(withLimit([recording]))+ .mockReturnValueOnce(withLimit([uploaded]));+ storageGetAccessForVideoMock.mockReturnValue(+ Effect.succeed([+ { provider: "google-drive", headObject: uploadedFileHeadMock },+ ]),+ );+ const { resumeConciergeLoomFileUpload } = await import("@/actions/loom");+ await expect(resumeConciergeLoomFileUpload(input)).resolves.toMatchObject({+ ok: true,+ });+ expect(startVideoProcessingWorkflowMock).toHaveBeenCalledOnce();+ expect(startLoomFileProcessingWorkflowMock).not.toHaveBeenCalled();+ });+it.each([{ records: [] },{ records: [{ ...recording, ownerEmail: "[email protected]" }] },diff --git a/apps/web/actions/loom.ts b/apps/web/actions/loom.tsindex 06d722bf4..2af2a6018 100644--- a/apps/web/actions/loom.ts+++ b/apps/web/actions/loom.ts@@ -51,6 +51,10 @@ import {getLoomDownloadUrl,LoomDownloadTemporaryError,} from "@/lib/loom-download-url";+import {+ LOOM_FILE_SINGLE_UPLOAD_MAX_BYTES,+ startLoomFileProcessingWorkflow,+} from "@/lib/loom-file-processing";import type { LoomImportDestination } from "@/lib/loom-import-destination";import { LOOM_IMPORT_MAX_BYTES } from "@/lib/loom-import-limits";import {@@ -1643,6 +1647,7 @@ async function startConciergeLoomFileProcessing({const [record] = await db().select({video: videos,+ loomVideoId: importedVideos.sourceId,rawFileKey: videoUploads.rawFileKey,phase: videoUploads.phase,processingMessage: videoUploads.processingMessage,@@ -1709,14 +1714,22 @@ async function startConciergeLoomFileProcessing({if (maxBytes !== undefined && (head.ContentLength ?? 0) > maxBytes) {throw new LoomMigrationError("The uploaded Loom file exceeds 10 GB.");}- const status = await startVideoProcessingWorkflow({+ const processingInput = {videoId,userId: record.video.ownerId,rawFileKey: record.rawFileKey,bucketId: record.video.bucket,processingMessage: "Processing Loom video...",startFailureMessage: "Loom file processing could not start.",- });+ };+ const status =+ bucket.provider === "s3" &&+ (head.ContentLength ?? 0) > LOOM_FILE_SINGLE_UPLOAD_MAX_BYTES+ ? await startLoomFileProcessingWorkflow({+ ...processingInput,+ loomVideoId: record.loomVideoId,+ })+ : await startVideoProcessingWorkflow(processingInput);revalidatePath("/dashboard/import/loom");revalidatePath("/dashboard/admin/loom-migrations");return { status };diff --git a/apps/web/lib/loom-file-processing.ts b/apps/web/lib/loom-file-processing.tsnew file mode 100644index 000000000..177b9b490--- /dev/null+++ b/apps/web/lib/loom-file-processing.ts@@ -0,0 +1,58 @@+import {+ LoomImportStartError,+ startLoomImportWorkflow,+} from "@/lib/loom-import-start";+import {+ setVideoProcessingError,+ type startVideoProcessingWorkflow,+ transitionVideoToProcessing,+ type VideoProcessingStartStatus,+} from "@/lib/video-processing";++export const LOOM_FILE_SINGLE_UPLOAD_MAX_BYTES = 5 * 1024 ** 3;++export async function startLoomFileProcessingWorkflow({+ loomVideoId,+ ...input+}: Pick<+ Parameters<typeof startVideoProcessingWorkflow>[0],+ | "videoId"+ | "userId"+ | "rawFileKey"+ | "bucketId"+ | "processingMessage"+ | "startFailureMessage"+> & {+ loomVideoId: string;+}): Promise<VideoProcessingStartStatus> {+ if (!/^[a-f0-9]{32}$/i.test(loomVideoId)) {+ throw new Error("The uploaded file has no valid Loom source mapping.");+ }+ const status = await transitionVideoToProcessing({+ videoId: input.videoId,+ rawFileKey: input.rawFileKey,+ processingMessage: input.processingMessage,+ });+ if (status !== "started") return status;++ try {+ await startLoomImportWorkflow({+ videoId: input.videoId,+ userId: input.userId,+ rawFileKey: input.rawFileKey,+ bucketId: input.bucketId,+ loomVideoId,+ reuseExistingRawUpload: true,+ });+ return "started";+ } catch (error) {+ if (!(error instanceof LoomImportStartError) || error.canRetry) {+ await setVideoProcessingError(+ input.videoId,+ input.startFailureMessage,+ error,+ );+ }+ throw error;+ }+}