Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix: process large Loom files with multipart uploads

Richie McIlroy617dcf6ee8bb
diff --git a/apps/web/__tests__/unit/loom-file-processing.test.ts b/apps/web/__tests__/unit/loom-file-processing.test.ts
new file mode 100644
index 000000000..ca5c3d8e5
--- /dev/null
+++ b/apps/web/__tests__/unit/loom-file-processing.test.ts
@@ -0,0 +1,129 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ transition: vi.fn(),
+ start: vi.fn(),
+ setError: vi.fn(),
+}));
+
+vi.mock("@/lib/video-processing", () => ({
+ transitionVideoToProcessing: mocks.transition,
+ setVideoProcessingError: mocks.setError,
+}));
+
+vi.mock("@/lib/loom-import-start", () => ({
+ startLoomImportWorkflow: mocks.start,
+ LoomImportStartError: class extends Error {
+ constructor(
+ readonly canRetry: boolean,
+ cause: unknown,
+ ) {
+ super("Loom startup failed", { cause });
+ }
+ },
+}));
+
+describe("large Loom file processing", () => {
+ const input = {
+ videoId: "existing-video" as never,
+ userId: "owner-id",
+ loomVideoId: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ rawFileKey: "owner-id/existing-video/raw-upload.mp4",
+ bucketId: null,
+ processingMessage: "Processing Loom video...",
+ startFailureMessage: "Loom file processing could not start.",
+ };
+
+ beforeEach(() => {
+ vi.resetAllMocks();
+ mocks.transition.mockResolvedValue("started");
+ mocks.start.mockResolvedValue(undefined);
+ mocks.setError.mockResolvedValue(undefined);
+ });
+
+ it("claims the existing upload and reuses its raw source through the multipart Loom workflow", async () => {
+ const { startLoomFileProcessingWorkflow } = await import(
+ "@/lib/loom-file-processing"
+ );
+ await expect(startLoomFileProcessingWorkflow(input)).resolves.toBe(
+ "started",
+ );
+ expect(mocks.transition).toHaveBeenCalledExactlyOnceWith({
+ videoId: input.videoId,
+ rawFileKey: input.rawFileKey,
+ processingMessage: input.processingMessage,
+ });
+ expect(mocks.start).toHaveBeenCalledExactlyOnceWith({
+ videoId: input.videoId,
+ userId: input.userId,
+ rawFileKey: input.rawFileKey,
+ bucketId: null,
+ loomVideoId: input.loomVideoId,
+ reuseExistingRawUpload: true,
+ });
+ expect(mocks.setError).not.toHaveBeenCalled();
+ });
+
+ it.each(["already-processing", "already-complete"])(
+ "does not dispatch another run when the upload is %s",
+ async (status) => {
+ mocks.transition.mockResolvedValue(status);
+ const { startLoomFileProcessingWorkflow } = await import(
+ "@/lib/loom-file-processing"
+ );
+ await expect(startLoomFileProcessingWorkflow(input)).resolves.toBe(
+ status,
+ );
+ expect(mocks.start).not.toHaveBeenCalled();
+ },
+ );
+
+ it("rejects an invalid source before claiming the upload", async () => {
+ const { startLoomFileProcessingWorkflow } = await import(
+ "@/lib/loom-file-processing"
+ );
+ await expect(
+ startLoomFileProcessingWorkflow({ ...input, loomVideoId: "invalid" }),
+ ).rejects.toThrow("no valid Loom source mapping");
+ expect(mocks.transition).not.toHaveBeenCalled();
+ expect(mocks.start).not.toHaveBeenCalled();
+ });
+
+ it("does not dispatch when claiming the upload fails", async () => {
+ mocks.transition.mockRejectedValue(new Error("Claim changed"));
+ const { startLoomFileProcessingWorkflow } = await import(
+ "@/lib/loom-file-processing"
+ );
+ await expect(startLoomFileProcessingWorkflow(input)).rejects.toThrow(
+ "Claim changed",
+ );
+ expect(mocks.start).not.toHaveBeenCalled();
+ expect(mocks.setError).not.toHaveBeenCalled();
+ });
+
+ it("restores a retryable error only after a confirmed queue rejection", async () => {
+ const { LoomImportStartError } = await import("@/lib/loom-import-start");
+ const error = new LoomImportStartError(true, new Error("Rejected"));
+ mocks.start.mockRejectedValue(error);
+ const { startLoomFileProcessingWorkflow } = await import(
+ "@/lib/loom-file-processing"
+ );
+ await expect(startLoomFileProcessingWorkflow(input)).rejects.toBe(error);
+ expect(mocks.setError).toHaveBeenCalledExactlyOnceWith(
+ input.videoId,
+ input.startFailureMessage,
+ error,
+ );
+ });
+
+ it("keeps an uncertain dispatch claimed so it cannot be started twice", async () => {
+ const { LoomImportStartError } = await import("@/lib/loom-import-start");
+ const error = new LoomImportStartError(false, new Error("Response lost"));
+ mocks.start.mockRejectedValue(error);
+ const { startLoomFileProcessingWorkflow } = await import(
+ "@/lib/loom-file-processing"
+ );
+ await expect(startLoomFileProcessingWorkflow(input)).rejects.toBe(error);
+ expect(mocks.setError).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/web/__tests__/unit/loom-import.test.ts b/apps/web/__tests__/unit/loom-import.test.ts
index 523425255..213676322 100644
--- a/apps/web/__tests__/unit/loom-import.test.ts
+++ b/apps/web/__tests__/unit/loom-import.test.ts
@@ -10,6 +10,7 @@ const revalidatePathMock = vi.fn();
const storageGetWritableAccessForUserMock = vi.hoisted(() => vi.fn());
const storageGetAccessForVideoMock = vi.hoisted(() => vi.fn());
const startVideoProcessingWorkflowMock = vi.hoisted(() => vi.fn());
+const startLoomFileProcessingWorkflowMock = vi.hoisted(() => vi.fn());
const uploadedFileHeadMock = vi.hoisted(() => vi.fn());
const checkRateLimitMock = vi.hoisted(() => vi.fn());
const headersMock = vi.hoisted(() => vi.fn());
@@ -176,6 +177,11 @@ vi.mock("@/lib/video-processing", () => ({
startVideoProcessingWorkflow: startVideoProcessingWorkflowMock,
}));
+vi.mock("@/lib/loom-file-processing", () => ({
+ LOOM_FILE_SINGLE_UPLOAD_MAX_BYTES: 5 * 1024 ** 3,
+ startLoomFileProcessingWorkflow: startLoomFileProcessingWorkflowMock,
+}));
+
vi.mock("@/lib/video-storage", () => ({
decodeStorageVideo: (video: unknown) => video,
}));
@@ -1634,6 +1640,7 @@ describe("resuming an uploaded concierge Loom file", () => {
metadata: { loomImportRun: { runId: "old-run", dispatch: "accepted" } },
};
const uploaded = {
+ loomVideoId: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
video: {
id: "existing-video",
ownerId: "owner-id",
@@ -1658,12 +1665,13 @@ describe("resuming an uploaded concierge Loom file", () => {
conciergeMocks.reserveConciergeImport.mockResolvedValue("lease");
conciergeMocks.releaseConciergeImport.mockResolvedValue(undefined);
storageGetAccessForVideoMock.mockReturnValue(
- Effect.succeed([{ headObject: uploadedFileHeadMock }]),
+ Effect.succeed([{ provider: "s3", headObject: uploadedFileHeadMock }]),
);
uploadedFileHeadMock.mockReturnValue(
Effect.succeed({ ContentLength: 6 * 1024 ** 3 }),
);
startVideoProcessingWorkflowMock.mockResolvedValue("started");
+ startLoomFileProcessingWorkflowMock.mockResolvedValue("started");
getRunMock.mockReturnValue({ status: Promise.resolve("failed") });
});
@@ -1718,19 +1726,60 @@ describe("resuming an uploaded concierge Loom file", () => {
ok: true,
value: { status: "started" },
});
- expect(startVideoProcessingWorkflowMock).toHaveBeenCalledExactlyOnceWith({
- videoId: "existing-video",
- userId: "owner-id",
- rawFileKey: uploaded.rawFileKey,
- bucketId: null,
- processingMessage: "Processing Loom video...",
- startFailureMessage: "Loom file processing could not start.",
- });
+ expect(startLoomFileProcessingWorkflowMock).toHaveBeenCalledExactlyOnceWith(
+ {
+ videoId: "existing-video",
+ userId: "owner-id",
+ rawFileKey: uploaded.rawFileKey,
+ bucketId: null,
+ processingMessage: "Processing Loom video...",
+ startFailureMessage: "Loom file processing could not start.",
+ loomVideoId: uploaded.loomVideoId,
+ },
+ );
+ expect(startVideoProcessingWorkflowMock).not.toHaveBeenCalled();
expect(mockDb.insert).not.toHaveBeenCalled();
expect(mockDb.update).not.toHaveBeenCalled();
expect(storageGetWritableAccessForUserMock).not.toHaveBeenCalled();
});
+ it.each([1024, 5 * 1024 ** 3])(
+ "keeps the existing file workflow for a %i-byte upload",
+ async (bytes) => {
+ whereMock
+ .mockReturnValueOnce(withLimit([recording]))
+ .mockReturnValueOnce(withLimit([uploaded]));
+ uploadedFileHeadMock.mockReturnValue(
+ Effect.succeed({ ContentLength: bytes }),
+ );
+ const { resumeConciergeLoomFileUpload } = await import("@/actions/loom");
+ await expect(resumeConciergeLoomFileUpload(input)).resolves.toMatchObject(
+ {
+ ok: true,
+ },
+ );
+ expect(startVideoProcessingWorkflowMock).toHaveBeenCalledOnce();
+ expect(startLoomFileProcessingWorkflowMock).not.toHaveBeenCalled();
+ },
+ );
+
+ it("keeps non-S3 storage on its existing processing path", async () => {
+ whereMock
+ .mockReturnValueOnce(withLimit([recording]))
+ .mockReturnValueOnce(withLimit([uploaded]));
+ storageGetAccessForVideoMock.mockReturnValue(
+ Effect.succeed([
+ { provider: "google-drive", headObject: uploadedFileHeadMock },
+ ]),
+ );
+ const { resumeConciergeLoomFileUpload } = await import("@/actions/loom");
+ await expect(resumeConciergeLoomFileUpload(input)).resolves.toMatchObject({
+ ok: true,
+ });
+ expect(startVideoProcessingWorkflowMock).toHaveBeenCalledOnce();
+ expect(startLoomFileProcessingWorkflowMock).not.toHaveBeenCalled();
+ });
+
it.each([
{ records: [] },
{ records: [{ ...recording, ownerEmail: "[email protected]" }] },
diff --git a/apps/web/actions/loom.ts b/apps/web/actions/loom.ts
index 06d722bf4..2af2a6018 100644
--- a/apps/web/actions/loom.ts
+++ b/apps/web/actions/loom.ts
@@ -51,6 +51,10 @@ import {
getLoomDownloadUrl,
LoomDownloadTemporaryError,
} from "@/lib/loom-download-url";
+import {
+ LOOM_FILE_SINGLE_UPLOAD_MAX_BYTES,
+ startLoomFileProcessingWorkflow,
+} from "@/lib/loom-file-processing";
import type { LoomImportDestination } from "@/lib/loom-import-destination";
import { LOOM_IMPORT_MAX_BYTES } from "@/lib/loom-import-limits";
import {
@@ -1643,6 +1647,7 @@ async function startConciergeLoomFileProcessing({
const [record] = await db()
.select({
video: videos,
+ loomVideoId: importedVideos.sourceId,
rawFileKey: videoUploads.rawFileKey,
phase: videoUploads.phase,
processingMessage: videoUploads.processingMessage,
@@ -1709,14 +1714,22 @@ async function startConciergeLoomFileProcessing({
if (maxBytes !== undefined && (head.ContentLength ?? 0) > maxBytes) {
throw new LoomMigrationError("The uploaded Loom file exceeds 10 GB.");
}
- const status = await startVideoProcessingWorkflow({
+ const processingInput = {
videoId,
userId: record.video.ownerId,
rawFileKey: record.rawFileKey,
bucketId: record.video.bucket,
processingMessage: "Processing Loom video...",
startFailureMessage: "Loom file processing could not start.",
- });
+ };
+ const status =
+ bucket.provider === "s3" &&
+ (head.ContentLength ?? 0) > LOOM_FILE_SINGLE_UPLOAD_MAX_BYTES
+ ? await startLoomFileProcessingWorkflow({
+ ...processingInput,
+ loomVideoId: record.loomVideoId,
+ })
+ : await startVideoProcessingWorkflow(processingInput);
revalidatePath("/dashboard/import/loom");
revalidatePath("/dashboard/admin/loom-migrations");
return { status };
diff --git a/apps/web/lib/loom-file-processing.ts b/apps/web/lib/loom-file-processing.ts
new file mode 100644
index 000000000..177b9b490
--- /dev/null
+++ b/apps/web/lib/loom-file-processing.ts
@@ -0,0 +1,58 @@
+import {
+ LoomImportStartError,
+ startLoomImportWorkflow,
+} from "@/lib/loom-import-start";
+import {
+ setVideoProcessingError,
+ type startVideoProcessingWorkflow,
+ transitionVideoToProcessing,
+ type VideoProcessingStartStatus,
+} from "@/lib/video-processing";
+
+export const LOOM_FILE_SINGLE_UPLOAD_MAX_BYTES = 5 * 1024 ** 3;
+
+export async function startLoomFileProcessingWorkflow({
+ loomVideoId,
+ ...input
+}: Pick<
+ Parameters<typeof startVideoProcessingWorkflow>[0],
+ | "videoId"
+ | "userId"
+ | "rawFileKey"
+ | "bucketId"
+ | "processingMessage"
+ | "startFailureMessage"
+> & {
+ loomVideoId: string;
+}): Promise<VideoProcessingStartStatus> {
+ if (!/^[a-f0-9]{32}$/i.test(loomVideoId)) {
+ throw new Error("The uploaded file has no valid Loom source mapping.");
+ }
+ const status = await transitionVideoToProcessing({
+ videoId: input.videoId,
+ rawFileKey: input.rawFileKey,
+ processingMessage: input.processingMessage,
+ });
+ if (status !== "started") return status;
+
+ try {
+ await startLoomImportWorkflow({
+ videoId: input.videoId,
+ userId: input.userId,
+ rawFileKey: input.rawFileKey,
+ bucketId: input.bucketId,
+ loomVideoId,
+ reuseExistingRawUpload: true,
+ });
+ return "started";
+ } catch (error) {
+ if (!(error instanceof LoomImportStartError) || error.canRetry) {
+ await setVideoProcessingError(
+ input.videoId,
+ input.startFailureMessage,
+ error,
+ );
+ }
+ throw error;
+ }
+}