Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

feat: send Loom downloads to the dedicated importer when configured

Richie McIlroy83aa19c87393
diff --git a/apps/web/__tests__/unit/media-processing-workflows.test.ts b/apps/web/__tests__/unit/media-processing-workflows.test.ts
index 5e8fb252b..8a7bd4942 100644
--- a/apps/web/__tests__/unit/media-processing-workflows.test.ts
+++ b/apps/web/__tests__/unit/media-processing-workflows.test.ts
@@ -15,6 +15,8 @@ const mocks = vi.hoisted(() => ({
dispatch: vi.fn(),
mediaJobFailed: vi.fn(),
sleep: vi.fn(),
+ env: {} as Record<string, string>,
+ loomSourceUrl: "https://cdn.loom.com/original.mp4",
}));
vi.mock("@cap/database", () => ({
@@ -40,6 +42,7 @@ vi.mock("@cap/env", () => ({
MEDIA_SERVER_URL: "https://worker.example.com",
WEB_URL: "https://cap.example.com",
MEDIA_SERVER_WEBHOOK_SECRET: "test-secret",
+ ...mocks.env,
}),
}));
vi.mock("@cap/web-domain", () => ({
@@ -108,6 +111,8 @@ function s3Error(name: string, httpStatusCode: number) {
describe("media processing workflows", () => {
beforeEach(() => {
mocks.rows = [];
+ mocks.env = {};
+ mocks.loomSourceUrl = "https://cdn.loom.com/original.mp4";
mocks.where
.mockReset()
.mockImplementation(async () => mocks.rows.shift() ?? []);
@@ -148,9 +153,12 @@ describe("media processing workflows", () => {
);
mocks.fetch.mockReset().mockImplementation(async (url: string) => {
if (url.startsWith("https://www.loom.com/")) {
- return Response.json({ url: "https://cdn.loom.com/original.mp4" });
+ return Response.json({ url: mocks.loomSourceUrl });
}
- if (url.startsWith("https://worker.example.com/")) {
+ if (
+ url.startsWith("https://worker.example.com/") ||
+ url.startsWith("https://media.example.com/loom-importer/")
+ ) {
return Response.json({ jobId: "job-1", status: "queued" });
}
throw new Error("Video bytes must not pass through Vercel");
@@ -234,6 +242,111 @@ describe("media processing workflows", () => {
expect(mocks.remove).not.toHaveBeenCalled();
});
+ describe("with the dedicated Loom importer", () => {
+ const importerUrl = "https://media.example.com/loom-importer";
+ const dispatchedBody = () => {
+ const call = mocks.fetch.mock.calls.find(([url]) =>
+ String(url).startsWith(importerUrl),
+ );
+ return call ? JSON.parse(call[1].body) : undefined;
+ };
+
+ beforeEach(() => {
+ mocks.env = { LOOM_IMPORTER_URL: importerUrl };
+ });
+
+ it("sends HLS downloads to the importer and defers transcription", async () => {
+ mocks.loomSourceUrl = "https://luna.loom.com/id/x/playlist.m3u8";
+ mocks.rows = [[video], [pending], [], [metadata]];
+ await expect(importLoomVideoWorkflow(payload)).resolves.toMatchObject({
+ success: true,
+ });
+ expect(mocks.statusFetch.mock.calls.map(([url]) => url)).toEqual([
+ "https://worker.example.com/video/status",
+ `${importerUrl}/video/status`,
+ ]);
+ expect(
+ mocks.fetch.mock.calls.some(([url]) =>
+ String(url).startsWith("https://worker.example.com/"),
+ ),
+ ).toBe(false);
+ const body = dispatchedBody();
+ expect(mocks.fetch.mock.calls.at(-1)?.[0]).toBe(
+ `${importerUrl}/video/import`,
+ );
+ expect(body).toMatchObject({
+ videoUrl: "https://luna.loom.com/id/x/playlist.m3u8",
+ webhookUrl:
+ "https://cap.example.com/api/webhooks/media-server/progress?retryable=true&transcription=on-view",
+ });
+ expect(body.sourcePresignedUrl).toBeUndefined();
+ expect(body.sourceUpload).toBeUndefined();
+ expect(body.originalUpload).toBeUndefined();
+ expect(mocks.remove).not.toHaveBeenCalled();
+ });
+
+ it("gives DASH downloads a WebM original destination", async () => {
+ mocks.loomSourceUrl = "https://luna.loom.com/id/x/playlist.mpd";
+ mocks.rows = [[video], [pending], [], [metadata]];
+ await importLoomVideoWorkflow(payload);
+ expect(dispatchedBody()?.originalUpload).toEqual({
+ key: "owner/video/raw-upload.webm",
+ upload: {
+ type: "put",
+ url: "https://storage.example.com/owner/video/raw-upload.webm?upload=1",
+ },
+ });
+ expect(mocks.put).toHaveBeenCalledWith("owner/video/raw-upload.webm", {
+ ContentType: "video/webm",
+ });
+ expect(mocks.remove).toHaveBeenCalledWith("owner/video/raw-upload.webm");
+ });
+
+ it("does not keep a second copy of progressive MP4 downloads", async () => {
+ mocks.rows = [[video], [pending], [], [metadata]];
+ await importLoomVideoWorkflow(payload);
+ const body = dispatchedBody();
+ expect(body.videoUrl).toBe("https://cdn.loom.com/original.mp4");
+ expect(body.sourcePresignedUrl).toBeUndefined();
+ expect(body.originalUpload).toBeUndefined();
+ expect(mocks.put).not.toHaveBeenCalledWith(
+ payload.rawFileKey,
+ expect.anything(),
+ expect.anything(),
+ );
+ });
+
+ it("keeps processing a preserved original on the media server", async () => {
+ mocks.rows = [[video], [video], [], [metadata]];
+ await importLoomVideoWorkflow({
+ ...payload,
+ reuseExistingRawUpload: true,
+ });
+ expect(mocks.fetch).toHaveBeenCalledOnce();
+ expect(mocks.fetch.mock.calls[0]?.[0]).toBe(
+ "https://worker.example.com/video/process",
+ );
+ expect(
+ JSON.parse(mocks.fetch.mock.calls[0]?.[1].body).webhookUrl,
+ ).not.toContain("transcription=on-view");
+ });
+
+ it("waits for the importer when it cannot confirm Loom support", async () => {
+ mocks.statusFetch.mockImplementation(async (url: string) =>
+ url.startsWith(importerUrl)
+ ? new Response("unavailable", { status: 503 })
+ : Response.json({
+ loomMultipartUploads: true,
+ loomImportMaxBytes: 10 * 1024 ** 3,
+ }),
+ );
+ await expect(importLoomVideoWorkflow(payload)).rejects.toThrow(
+ "Could not verify Loom import worker capabilities",
+ );
+ expect(dispatchedBody()).toBeUndefined();
+ });
+ });
+
it("reuses a preserved Loom upload when retrying", async () => {
mocks.rows = [[video], [video], [], [metadata]];
await importLoomVideoWorkflow({ ...payload, reuseExistingRawUpload: true });
@@ -396,7 +509,11 @@ describe("media processing workflows", () => {
await expect(importLoomVideoWorkflow(payload)).rejects.toThrow(
"must be reconciled before retrying",
);
- expect(mocks.mediaJobFailed).toHaveBeenCalledWith("job-1", "video");
+ expect(mocks.mediaJobFailed).toHaveBeenCalledWith(
+ "job-1",
+ "video",
+ "https://worker.example.com",
+ );
expect(
mocks.fetch.mock.calls.filter(([url]) =>
url.startsWith("https://worker.example.com/"),
diff --git a/apps/web/lib/loom-import-upload.ts b/apps/web/lib/loom-import-upload.ts
index ba779b31b..8d800826f 100644
--- a/apps/web/lib/loom-import-upload.ts
+++ b/apps/web/lib/loom-import-upload.ts
@@ -110,7 +110,9 @@ export async function prepareLoomMultipartUploads({
const created: Array<{ key: string; uploadId: string }> = [];
const create = async (key: string) => {
const { UploadId: uploadId } = await bucket.multipart
- .create(key, { ContentType: "video/mp4" })
+ .create(key, {
+ ContentType: key.endsWith(".webm") ? "video/webm" : "video/mp4",
+ })
.pipe(runWorkflowPromise);
if (!uploadId)
throw new Error("Storage did not return a multipart upload id");
diff --git a/apps/web/lib/loom-media-dispatch.ts b/apps/web/lib/loom-media-dispatch.ts
index a9b73e8a6..de30c3fa7 100644
--- a/apps/web/lib/loom-media-dispatch.ts
+++ b/apps/web/lib/loom-media-dispatch.ts
@@ -46,12 +46,32 @@ export async function recordLoomMediaDispatch({
throw new Error("Loom media dispatch ownership changed");
}
-export async function isLoomMediaJobFailed(jobId: string, videoId: string) {
+export async function isLoomMediaJobFailed(
+ jobId: string,
+ videoId: string,
+ workerUrl?: string,
+) {
const env = serverEnv();
- if (!env.MEDIA_SERVER_URL || !env.MEDIA_SERVER_WEBHOOK_SECRET) return false;
+ const workerUrls = workerUrl
+ ? [workerUrl]
+ : [env.MEDIA_SERVER_URL, env.LOOM_IMPORTER_URL].filter(
+ (url): url is string => Boolean(url),
+ );
+ for (const url of workerUrls)
+ if (await isJobFailedOnWorker(url, jobId, videoId)) return true;
+ return false;
+}
+
+async function isJobFailedOnWorker(
+ workerUrl: string,
+ jobId: string,
+ videoId: string,
+) {
+ const env = serverEnv();
+ if (!env.MEDIA_SERVER_WEBHOOK_SECRET) return false;
try {
const response = await fetch(
- `${env.MEDIA_SERVER_URL}/video/process/${encodeURIComponent(jobId)}/status`,
+ `${workerUrl}/video/process/${encodeURIComponent(jobId)}/status`,
{
headers: { "x-media-server-secret": env.MEDIA_SERVER_WEBHOOK_SECRET },
signal: AbortSignal.timeout(15_000),
diff --git a/apps/web/workflows/import-loom-video.ts b/apps/web/workflows/import-loom-video.ts
index 2177e5f00..de3a5d344 100644
--- a/apps/web/workflows/import-loom-video.ts
+++ b/apps/web/workflows/import-loom-video.ts
@@ -65,6 +65,17 @@ function isStreamingUrl(url: string): boolean {
return path.endsWith(".m3u8") || path.endsWith(".mpd");
}
+function isDashUrl(url: string): boolean {
+ return (url.split("?")[0] ?? "").toLowerCase().endsWith(".mpd");
+}
+
+type LoomWorker = "importer" | "media";
+
+function getLoomWorkerUrl(worker: LoomWorker): string | undefined {
+ const env = serverEnv();
+ return worker === "importer" ? env.LOOM_IMPORTER_URL : env.MEDIA_SERVER_URL;
+}
+
async function fetchFreshLoomDownloadUrl(
loomVideoId: string,
existingUrl?: string,
@@ -112,7 +123,16 @@ async function requireLoomMediaServerSupport() {
const env = serverEnv();
if (!env.MEDIA_SERVER_URL)
throw new FatalError("MEDIA_SERVER_URL is not configured");
- const response = await fetch(`${env.MEDIA_SERVER_URL}/video/status`, {
+ await requireLoomWorkerCapabilities(env.MEDIA_SERVER_URL);
+ if (env.LOOM_IMPORTER_URL)
+ await requireLoomWorkerCapabilities(env.LOOM_IMPORTER_URL);
+}
+
+requireLoomMediaServerSupport.maxRetries = 60;
+
+async function requireLoomWorkerCapabilities(workerUrl: string) {
+ const env = serverEnv();
+ const response = await fetch(`${workerUrl}/video/status`, {
headers: env.MEDIA_SERVER_WEBHOOK_SECRET
? { "x-media-server-secret": env.MEDIA_SERVER_WEBHOOK_SECRET }
: undefined,
@@ -139,8 +159,6 @@ async function requireLoomMediaServerSupport() {
}
}
-requireLoomMediaServerSupport.maxRetries = 60;
-
async function claimAgentImport(operationId: string | undefined) {
"use step";
@@ -225,7 +243,7 @@ export async function importLoomVideoWorkflow(
let metadata: ProcessedVideoMetadata;
for (let processingAttempt = 0; ; processingAttempt++) {
- let mediaJobId: string;
+ let mediaJob: { jobId: string; worker: LoomWorker };
let capacityRetryCount = 0;
let previousSourceUrl =
processingAttempt === 0 ? payload.loomDownloadUrl : undefined;
@@ -238,7 +256,7 @@ export async function importLoomVideoWorkflow(
)
: undefined;
previousSourceUrl = loomSourceUrl;
- mediaJobId = await processVideoOnMediaServer(
+ mediaJob = await processVideoOnMediaServer(
payload,
loomSourceUrl,
`${workflowRunId}:${processingAttempt}`,
@@ -262,7 +280,13 @@ export async function importLoomVideoWorkflow(
) {
throw error;
}
- if (!(await confirmLoomMediaJobFailure(mediaJobId, payload.videoId))) {
+ if (
+ !(await confirmLoomMediaJobFailure(
+ mediaJob.jobId,
+ payload.videoId,
+ mediaJob.worker,
+ ))
+ ) {
throw new FatalError(
"The previous Loom worker must be reconciled before retrying",
);
@@ -307,10 +331,14 @@ function getInputExtension(url: string): string | undefined {
return undefined;
}
-async function confirmLoomMediaJobFailure(jobId: string, videoId: string) {
+async function confirmLoomMediaJobFailure(
+ jobId: string,
+ videoId: string,
+ worker: LoomWorker,
+) {
"use step";
- return isLoomMediaJobFailed(jobId, videoId);
+ return isLoomMediaJobFailed(jobId, videoId, getLoomWorkerUrl(worker));
}
async function hasExistingLoomUpload(
@@ -371,6 +399,10 @@ async function startMediaServerProcessJob(
sourcePresignedUrl?: string;
sourceUpload?: LoomMultipartUploadTarget;
outputUpload?: LoomMultipartUploadTarget;
+ originalUpload?: {
+ key: string;
+ upload: LoomMultipartUploadTarget | { type: "put"; url: string };
+ };
existingOutput?: { url: string; objectIdentity: string; fileSize: number };
maxImportBytes: number;
outputPresignedUrl: string;
@@ -385,6 +417,7 @@ async function startMediaServerProcessJob(
state: "accepted" | "rejected" | "uncertain",
jobId?: string,
) => Promise<void>,
+ importOnly = false,
): Promise<string> {
for (let attempt = 0; attempt < MEDIA_SERVER_START_MAX_ATTEMPTS; attempt++) {
const headers: Record<string, string> = {
@@ -395,7 +428,9 @@ async function startMediaServerProcessJob(
}
const endpoint =
- body.sourcePresignedUrl || body.sourceUpload ? "import" : "process";
+ importOnly || body.sourcePresignedUrl || body.sourceUpload
+ ? "import"
+ : "process";
await recordDispatch("uncertain");
const response = await fetch(`${mediaServerUrl}/video/${endpoint}`, {
method: "POST",
@@ -461,21 +496,28 @@ async function processVideoOnMediaServer(
loomSourceUrl: string | undefined,
attemptId: string,
runId: string,
-): Promise<string> {
+): Promise<{ jobId: string; worker: LoomWorker }> {
"use step";
const { videoId, userId, rawFileKey } = payload;
- const mediaServerUrl = serverEnv().MEDIA_SERVER_URL;
+ const worker: LoomWorker =
+ loomSourceUrl && serverEnv().LOOM_IMPORTER_URL ? "importer" : "media";
+ const mediaServerUrl = getLoomWorkerUrl(worker);
if (!mediaServerUrl) {
throw new FatalError("MEDIA_SERVER_URL is not configured");
}
+ const originalKey =
+ worker === "importer" && loomSourceUrl && isDashUrl(loomSourceUrl)
+ ? `${userId}/${videoId}/raw-upload.webm`
+ : undefined;
const webhookBaseUrl =
serverEnv().MEDIA_SERVER_WEBHOOK_URL || serverEnv().WEB_URL;
const {
rawVideoUrl,
+ originalUpload,
sourcePresignedUrl,
outputPresignedUrl,
thumbnailPresignedUrl,
@@ -516,7 +558,7 @@ async function processVideoOnMediaServer(
expiresIn: MEDIA_SERVER_PRESIGNED_GET_EXPIRES_SECONDS,
});
const sourcePresignedUrl =
- loomSourceUrl && !isStreamingUrl(loomSourceUrl)
+ worker === "media" && loomSourceUrl && !isStreamingUrl(loomSourceUrl)
? yield* bucket.getInternalPresignedPutUrl(
rawFileKey,
{ ContentType: "video/mp4" },
@@ -546,22 +588,46 @@ async function processVideoOnMediaServer(
},
{ expiresIn: MEDIA_SERVER_PRESIGNED_PUT_EXPIRES_SECONDS },
);
+ if (originalKey)
+ yield* bucket
+ .deleteObject(originalKey)
+ .pipe(Effect.catchAll(() => Effect.void));
+ const originalPutUrl =
+ originalKey && bucket.provider !== "s3"
+ ? yield* bucket.getInternalPresignedPutUrl(originalKey, {
+ ContentType: "video/webm",
+ })
+ : undefined;
const multipart = yield* Effect.promise(() =>
prepareLoomMultipartUploads({
video,
bucket,
attemptId,
sourceKey:
- loomSourceUrl && !isStreamingUrl(loomSourceUrl)
- ? rawFileKey
- : undefined,
+ worker === "importer"
+ ? originalKey
+ : loomSourceUrl && !isStreamingUrl(loomSourceUrl)
+ ? rawFileKey
+ : undefined,
webhookBaseUrl,
webhookSecret: serverEnv().MEDIA_SERVER_WEBHOOK_SECRET || undefined,
}),
);
+ const originalUpload = originalKey
+ ? multipart.sourceUpload
+ ? { key: originalKey, upload: multipart.sourceUpload }
+ : originalPutUrl
+ ? {
+ key: originalKey,
+ upload: { type: "put" as const, url: originalPutUrl },
+ }
+ : undefined
+ : undefined;
+
return {
rawVideoUrl: multipart.existingOutput?.url ?? rawVideoUrl,
+ originalUpload,
sourcePresignedUrl:
multipart.sourceUpload || multipart.existingOutput
? undefined
@@ -588,7 +654,7 @@ async function processVideoOnMediaServer(
};
}).pipe(runWorkflowPromise);
- const webhookUrl = `${webhookBaseUrl}/api/webhooks/media-server/progress?retryable=true`;
+ const webhookUrl = `${webhookBaseUrl}/api/webhooks/media-server/progress?retryable=true${worker === "importer" ? "&transcription=on-view" : ""}`;
const webhookSecret = serverEnv().MEDIA_SERVER_WEBHOOK_SECRET;
try {
@@ -608,14 +674,15 @@ async function processVideoOnMediaServer(
}
try {
- return await startMediaServerProcessJob(
+ const jobId = await startMediaServerProcessJob(
mediaServerUrl,
{
videoId,
userId,
videoUrl: rawVideoUrl,
sourcePresignedUrl,
- sourceUpload,
+ sourceUpload: worker === "importer" ? undefined : sourceUpload,
+ originalUpload,
outputUpload,
existingOutput,
maxImportBytes: LOOM_IMPORT_MAX_BYTES,
@@ -636,7 +703,9 @@ async function processVideoOnMediaServer(
state,
jobId,
}),
+ worker === "importer",
);
+ return { jobId, worker };
} catch (error) {
if (error instanceof LoomMediaRequestRejectedError) await abortUploads();
throw error;
diff --git a/packages/env/server.ts b/packages/env/server.ts
index 6e882f61c..db3a9ed96 100644
--- a/packages/env/server.ts
+++ b/packages/env/server.ts
@@ -228,6 +228,12 @@ function createServerEnv() {
.describe(
"Base URL for media server webhooks (use host.docker.internal for Docker setups)",
),
+ LOOM_IMPORTER_URL: z
+ .string()
+ .optional()
+ .describe(
+ "URL of the dedicated Loom importer; Loom downloads use the media server when unset",
+ ),
/// Ignore
NODE_ENV: z.string(),