Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
feat: send Loom downloads to the dedicated importer when configured
Richie McIlroy83aa19c87393
diff --git a/apps/web/__tests__/unit/media-processing-workflows.test.ts b/apps/web/__tests__/unit/media-processing-workflows.test.tsindex 5e8fb252b..8a7bd4942 100644--- a/apps/web/__tests__/unit/media-processing-workflows.test.ts+++ b/apps/web/__tests__/unit/media-processing-workflows.test.ts@@ -15,6 +15,8 @@ const mocks = vi.hoisted(() => ({dispatch: vi.fn(),mediaJobFailed: vi.fn(),sleep: vi.fn(),+ env: {} as Record<string, string>,+ loomSourceUrl: "https://cdn.loom.com/original.mp4",}));vi.mock("@cap/database", () => ({@@ -40,6 +42,7 @@ vi.mock("@cap/env", () => ({MEDIA_SERVER_URL: "https://worker.example.com",WEB_URL: "https://cap.example.com",MEDIA_SERVER_WEBHOOK_SECRET: "test-secret",+ ...mocks.env,}),}));vi.mock("@cap/web-domain", () => ({@@ -108,6 +111,8 @@ function s3Error(name: string, httpStatusCode: number) {describe("media processing workflows", () => {beforeEach(() => {mocks.rows = [];+ mocks.env = {};+ mocks.loomSourceUrl = "https://cdn.loom.com/original.mp4";mocks.where.mockReset().mockImplementation(async () => mocks.rows.shift() ?? []);@@ -148,9 +153,12 @@ describe("media processing workflows", () => {);mocks.fetch.mockReset().mockImplementation(async (url: string) => {if (url.startsWith("https://www.loom.com/")) {- return Response.json({ url: "https://cdn.loom.com/original.mp4" });+ return Response.json({ url: mocks.loomSourceUrl });}- if (url.startsWith("https://worker.example.com/")) {+ if (+ url.startsWith("https://worker.example.com/") ||+ url.startsWith("https://media.example.com/loom-importer/")+ ) {return Response.json({ jobId: "job-1", status: "queued" });}throw new Error("Video bytes must not pass through Vercel");@@ -234,6 +242,111 @@ describe("media processing workflows", () => {expect(mocks.remove).not.toHaveBeenCalled();});+ describe("with the dedicated Loom importer", () => {+ const importerUrl = "https://media.example.com/loom-importer";+ const dispatchedBody = () => {+ const call = mocks.fetch.mock.calls.find(([url]) =>+ String(url).startsWith(importerUrl),+ );+ return call ? JSON.parse(call[1].body) : undefined;+ };++ beforeEach(() => {+ mocks.env = { LOOM_IMPORTER_URL: importerUrl };+ });++ it("sends HLS downloads to the importer and defers transcription", async () => {+ mocks.loomSourceUrl = "https://luna.loom.com/id/x/playlist.m3u8";+ mocks.rows = [[video], [pending], [], [metadata]];+ await expect(importLoomVideoWorkflow(payload)).resolves.toMatchObject({+ success: true,+ });+ expect(mocks.statusFetch.mock.calls.map(([url]) => url)).toEqual([+ "https://worker.example.com/video/status",+ `${importerUrl}/video/status`,+ ]);+ expect(+ mocks.fetch.mock.calls.some(([url]) =>+ String(url).startsWith("https://worker.example.com/"),+ ),+ ).toBe(false);+ const body = dispatchedBody();+ expect(mocks.fetch.mock.calls.at(-1)?.[0]).toBe(+ `${importerUrl}/video/import`,+ );+ expect(body).toMatchObject({+ videoUrl: "https://luna.loom.com/id/x/playlist.m3u8",+ webhookUrl:+ "https://cap.example.com/api/webhooks/media-server/progress?retryable=true&transcription=on-view",+ });+ expect(body.sourcePresignedUrl).toBeUndefined();+ expect(body.sourceUpload).toBeUndefined();+ expect(body.originalUpload).toBeUndefined();+ expect(mocks.remove).not.toHaveBeenCalled();+ });++ it("gives DASH downloads a WebM original destination", async () => {+ mocks.loomSourceUrl = "https://luna.loom.com/id/x/playlist.mpd";+ mocks.rows = [[video], [pending], [], [metadata]];+ await importLoomVideoWorkflow(payload);+ expect(dispatchedBody()?.originalUpload).toEqual({+ key: "owner/video/raw-upload.webm",+ upload: {+ type: "put",+ url: "https://storage.example.com/owner/video/raw-upload.webm?upload=1",+ },+ });+ expect(mocks.put).toHaveBeenCalledWith("owner/video/raw-upload.webm", {+ ContentType: "video/webm",+ });+ expect(mocks.remove).toHaveBeenCalledWith("owner/video/raw-upload.webm");+ });++ it("does not keep a second copy of progressive MP4 downloads", async () => {+ mocks.rows = [[video], [pending], [], [metadata]];+ await importLoomVideoWorkflow(payload);+ const body = dispatchedBody();+ expect(body.videoUrl).toBe("https://cdn.loom.com/original.mp4");+ expect(body.sourcePresignedUrl).toBeUndefined();+ expect(body.originalUpload).toBeUndefined();+ expect(mocks.put).not.toHaveBeenCalledWith(+ payload.rawFileKey,+ expect.anything(),+ expect.anything(),+ );+ });++ it("keeps processing a preserved original on the media server", async () => {+ mocks.rows = [[video], [video], [], [metadata]];+ await importLoomVideoWorkflow({+ ...payload,+ reuseExistingRawUpload: true,+ });+ expect(mocks.fetch).toHaveBeenCalledOnce();+ expect(mocks.fetch.mock.calls[0]?.[0]).toBe(+ "https://worker.example.com/video/process",+ );+ expect(+ JSON.parse(mocks.fetch.mock.calls[0]?.[1].body).webhookUrl,+ ).not.toContain("transcription=on-view");+ });++ it("waits for the importer when it cannot confirm Loom support", async () => {+ mocks.statusFetch.mockImplementation(async (url: string) =>+ url.startsWith(importerUrl)+ ? new Response("unavailable", { status: 503 })+ : Response.json({+ loomMultipartUploads: true,+ loomImportMaxBytes: 10 * 1024 ** 3,+ }),+ );+ await expect(importLoomVideoWorkflow(payload)).rejects.toThrow(+ "Could not verify Loom import worker capabilities",+ );+ expect(dispatchedBody()).toBeUndefined();+ });+ });+it("reuses a preserved Loom upload when retrying", async () => {mocks.rows = [[video], [video], [], [metadata]];await importLoomVideoWorkflow({ ...payload, reuseExistingRawUpload: true });@@ -396,7 +509,11 @@ describe("media processing workflows", () => {await expect(importLoomVideoWorkflow(payload)).rejects.toThrow("must be reconciled before retrying",);- expect(mocks.mediaJobFailed).toHaveBeenCalledWith("job-1", "video");+ expect(mocks.mediaJobFailed).toHaveBeenCalledWith(+ "job-1",+ "video",+ "https://worker.example.com",+ );expect(mocks.fetch.mock.calls.filter(([url]) =>url.startsWith("https://worker.example.com/"),diff --git a/apps/web/lib/loom-import-upload.ts b/apps/web/lib/loom-import-upload.tsindex ba779b31b..8d800826f 100644--- a/apps/web/lib/loom-import-upload.ts+++ b/apps/web/lib/loom-import-upload.ts@@ -110,7 +110,9 @@ export async function prepareLoomMultipartUploads({const created: Array<{ key: string; uploadId: string }> = [];const create = async (key: string) => {const { UploadId: uploadId } = await bucket.multipart- .create(key, { ContentType: "video/mp4" })+ .create(key, {+ ContentType: key.endsWith(".webm") ? "video/webm" : "video/mp4",+ }).pipe(runWorkflowPromise);if (!uploadId)throw new Error("Storage did not return a multipart upload id");diff --git a/apps/web/lib/loom-media-dispatch.ts b/apps/web/lib/loom-media-dispatch.tsindex a9b73e8a6..de30c3fa7 100644--- a/apps/web/lib/loom-media-dispatch.ts+++ b/apps/web/lib/loom-media-dispatch.ts@@ -46,12 +46,32 @@ export async function recordLoomMediaDispatch({throw new Error("Loom media dispatch ownership changed");}-export async function isLoomMediaJobFailed(jobId: string, videoId: string) {+export async function isLoomMediaJobFailed(+ jobId: string,+ videoId: string,+ workerUrl?: string,+) {const env = serverEnv();- if (!env.MEDIA_SERVER_URL || !env.MEDIA_SERVER_WEBHOOK_SECRET) return false;+ const workerUrls = workerUrl+ ? [workerUrl]+ : [env.MEDIA_SERVER_URL, env.LOOM_IMPORTER_URL].filter(+ (url): url is string => Boolean(url),+ );+ for (const url of workerUrls)+ if (await isJobFailedOnWorker(url, jobId, videoId)) return true;+ return false;+}++async function isJobFailedOnWorker(+ workerUrl: string,+ jobId: string,+ videoId: string,+) {+ const env = serverEnv();+ if (!env.MEDIA_SERVER_WEBHOOK_SECRET) return false;try {const response = await fetch(- `${env.MEDIA_SERVER_URL}/video/process/${encodeURIComponent(jobId)}/status`,+ `${workerUrl}/video/process/${encodeURIComponent(jobId)}/status`,{headers: { "x-media-server-secret": env.MEDIA_SERVER_WEBHOOK_SECRET },signal: AbortSignal.timeout(15_000),diff --git a/apps/web/workflows/import-loom-video.ts b/apps/web/workflows/import-loom-video.tsindex 2177e5f00..de3a5d344 100644--- a/apps/web/workflows/import-loom-video.ts+++ b/apps/web/workflows/import-loom-video.ts@@ -65,6 +65,17 @@ function isStreamingUrl(url: string): boolean {return path.endsWith(".m3u8") || path.endsWith(".mpd");}+function isDashUrl(url: string): boolean {+ return (url.split("?")[0] ?? "").toLowerCase().endsWith(".mpd");+}++type LoomWorker = "importer" | "media";++function getLoomWorkerUrl(worker: LoomWorker): string | undefined {+ const env = serverEnv();+ return worker === "importer" ? env.LOOM_IMPORTER_URL : env.MEDIA_SERVER_URL;+}+async function fetchFreshLoomDownloadUrl(loomVideoId: string,existingUrl?: string,@@ -112,7 +123,16 @@ async function requireLoomMediaServerSupport() {const env = serverEnv();if (!env.MEDIA_SERVER_URL)throw new FatalError("MEDIA_SERVER_URL is not configured");- const response = await fetch(`${env.MEDIA_SERVER_URL}/video/status`, {+ await requireLoomWorkerCapabilities(env.MEDIA_SERVER_URL);+ if (env.LOOM_IMPORTER_URL)+ await requireLoomWorkerCapabilities(env.LOOM_IMPORTER_URL);+}++requireLoomMediaServerSupport.maxRetries = 60;++async function requireLoomWorkerCapabilities(workerUrl: string) {+ const env = serverEnv();+ const response = await fetch(`${workerUrl}/video/status`, {headers: env.MEDIA_SERVER_WEBHOOK_SECRET? { "x-media-server-secret": env.MEDIA_SERVER_WEBHOOK_SECRET }: undefined,@@ -139,8 +159,6 @@ async function requireLoomMediaServerSupport() {}}-requireLoomMediaServerSupport.maxRetries = 60;-async function claimAgentImport(operationId: string | undefined) {"use step";@@ -225,7 +243,7 @@ export async function importLoomVideoWorkflow(let metadata: ProcessedVideoMetadata;for (let processingAttempt = 0; ; processingAttempt++) {- let mediaJobId: string;+ let mediaJob: { jobId: string; worker: LoomWorker };let capacityRetryCount = 0;let previousSourceUrl =processingAttempt === 0 ? payload.loomDownloadUrl : undefined;@@ -238,7 +256,7 @@ export async function importLoomVideoWorkflow(): undefined;previousSourceUrl = loomSourceUrl;- mediaJobId = await processVideoOnMediaServer(+ mediaJob = await processVideoOnMediaServer(payload,loomSourceUrl,`${workflowRunId}:${processingAttempt}`,@@ -262,7 +280,13 @@ export async function importLoomVideoWorkflow() {throw error;}- if (!(await confirmLoomMediaJobFailure(mediaJobId, payload.videoId))) {+ if (+ !(await confirmLoomMediaJobFailure(+ mediaJob.jobId,+ payload.videoId,+ mediaJob.worker,+ ))+ ) {throw new FatalError("The previous Loom worker must be reconciled before retrying",);@@ -307,10 +331,14 @@ function getInputExtension(url: string): string | undefined {return undefined;}-async function confirmLoomMediaJobFailure(jobId: string, videoId: string) {+async function confirmLoomMediaJobFailure(+ jobId: string,+ videoId: string,+ worker: LoomWorker,+) {"use step";- return isLoomMediaJobFailed(jobId, videoId);+ return isLoomMediaJobFailed(jobId, videoId, getLoomWorkerUrl(worker));}async function hasExistingLoomUpload(@@ -371,6 +399,10 @@ async function startMediaServerProcessJob(sourcePresignedUrl?: string;sourceUpload?: LoomMultipartUploadTarget;outputUpload?: LoomMultipartUploadTarget;+ originalUpload?: {+ key: string;+ upload: LoomMultipartUploadTarget | { type: "put"; url: string };+ };existingOutput?: { url: string; objectIdentity: string; fileSize: number };maxImportBytes: number;outputPresignedUrl: string;@@ -385,6 +417,7 @@ async function startMediaServerProcessJob(state: "accepted" | "rejected" | "uncertain",jobId?: string,) => Promise<void>,+ importOnly = false,): Promise<string> {for (let attempt = 0; attempt < MEDIA_SERVER_START_MAX_ATTEMPTS; attempt++) {const headers: Record<string, string> = {@@ -395,7 +428,9 @@ async function startMediaServerProcessJob(}const endpoint =- body.sourcePresignedUrl || body.sourceUpload ? "import" : "process";+ importOnly || body.sourcePresignedUrl || body.sourceUpload+ ? "import"+ : "process";await recordDispatch("uncertain");const response = await fetch(`${mediaServerUrl}/video/${endpoint}`, {method: "POST",@@ -461,21 +496,28 @@ async function processVideoOnMediaServer(loomSourceUrl: string | undefined,attemptId: string,runId: string,-): Promise<string> {+): Promise<{ jobId: string; worker: LoomWorker }> {"use step";const { videoId, userId, rawFileKey } = payload;- const mediaServerUrl = serverEnv().MEDIA_SERVER_URL;+ const worker: LoomWorker =+ loomSourceUrl && serverEnv().LOOM_IMPORTER_URL ? "importer" : "media";+ const mediaServerUrl = getLoomWorkerUrl(worker);if (!mediaServerUrl) {throw new FatalError("MEDIA_SERVER_URL is not configured");}+ const originalKey =+ worker === "importer" && loomSourceUrl && isDashUrl(loomSourceUrl)+ ? `${userId}/${videoId}/raw-upload.webm`+ : undefined;const webhookBaseUrl =serverEnv().MEDIA_SERVER_WEBHOOK_URL || serverEnv().WEB_URL;const {rawVideoUrl,+ originalUpload,sourcePresignedUrl,outputPresignedUrl,thumbnailPresignedUrl,@@ -516,7 +558,7 @@ async function processVideoOnMediaServer(expiresIn: MEDIA_SERVER_PRESIGNED_GET_EXPIRES_SECONDS,});const sourcePresignedUrl =- loomSourceUrl && !isStreamingUrl(loomSourceUrl)+ worker === "media" && loomSourceUrl && !isStreamingUrl(loomSourceUrl)? yield* bucket.getInternalPresignedPutUrl(rawFileKey,{ ContentType: "video/mp4" },@@ -546,22 +588,46 @@ async function processVideoOnMediaServer(},{ expiresIn: MEDIA_SERVER_PRESIGNED_PUT_EXPIRES_SECONDS },);+ if (originalKey)+ yield* bucket+ .deleteObject(originalKey)+ .pipe(Effect.catchAll(() => Effect.void));+ const originalPutUrl =+ originalKey && bucket.provider !== "s3"+ ? yield* bucket.getInternalPresignedPutUrl(originalKey, {+ ContentType: "video/webm",+ })+ : undefined;const multipart = yield* Effect.promise(() =>prepareLoomMultipartUploads({video,bucket,attemptId,sourceKey:- loomSourceUrl && !isStreamingUrl(loomSourceUrl)- ? rawFileKey- : undefined,+ worker === "importer"+ ? originalKey+ : loomSourceUrl && !isStreamingUrl(loomSourceUrl)+ ? rawFileKey+ : undefined,webhookBaseUrl,webhookSecret: serverEnv().MEDIA_SERVER_WEBHOOK_SECRET || undefined,}),);+ const originalUpload = originalKey+ ? multipart.sourceUpload+ ? { key: originalKey, upload: multipart.sourceUpload }+ : originalPutUrl+ ? {+ key: originalKey,+ upload: { type: "put" as const, url: originalPutUrl },+ }+ : undefined+ : undefined;+return {rawVideoUrl: multipart.existingOutput?.url ?? rawVideoUrl,+ originalUpload,sourcePresignedUrl:multipart.sourceUpload || multipart.existingOutput? undefined@@ -588,7 +654,7 @@ async function processVideoOnMediaServer(};}).pipe(runWorkflowPromise);- const webhookUrl = `${webhookBaseUrl}/api/webhooks/media-server/progress?retryable=true`;+ const webhookUrl = `${webhookBaseUrl}/api/webhooks/media-server/progress?retryable=true${worker === "importer" ? "&transcription=on-view" : ""}`;const webhookSecret = serverEnv().MEDIA_SERVER_WEBHOOK_SECRET;try {@@ -608,14 +674,15 @@ async function processVideoOnMediaServer(}try {- return await startMediaServerProcessJob(+ const jobId = await startMediaServerProcessJob(mediaServerUrl,{videoId,userId,videoUrl: rawVideoUrl,sourcePresignedUrl,- sourceUpload,+ sourceUpload: worker === "importer" ? undefined : sourceUpload,+ originalUpload,outputUpload,existingOutput,maxImportBytes: LOOM_IMPORT_MAX_BYTES,@@ -636,7 +703,9 @@ async function processVideoOnMediaServer(state,jobId,}),+ worker === "importer",);+ return { jobId, worker };} catch (error) {if (error instanceof LoomMediaRequestRejectedError) await abortUploads();throw error;diff --git a/packages/env/server.ts b/packages/env/server.tsindex 6e882f61c..db3a9ed96 100644--- a/packages/env/server.ts+++ b/packages/env/server.ts@@ -228,6 +228,12 @@ function createServerEnv() {.describe("Base URL for media server webhooks (use host.docker.internal for Docker setups)",),+ LOOM_IMPORTER_URL: z+ .string()+ .optional()+ .describe(+ "URL of the dedicated Loom importer; Loom downloads use the media server when unset",+ ),/// IgnoreNODE_ENV: z.string(),