Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf: wake Loom import workflows from the finished webhook instead of polling

Richie McIlroyead7b39eb553
diff --git a/apps/web/__tests__/unit/media-processing-workflows.test.ts b/apps/web/__tests__/unit/media-processing-workflows.test.ts
index 540bd7b26..5f1546cd8 100644
--- a/apps/web/__tests__/unit/media-processing-workflows.test.ts
+++ b/apps/web/__tests__/unit/media-processing-workflows.test.ts
@@ -2,6 +2,8 @@ import { Effect } from "effect";
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
+ createHook: vi.fn(),
+ hookDispose: vi.fn(),
rows: [] as unknown[][],
where: vi.fn(),
write: vi.fn(),
@@ -66,6 +68,7 @@ vi.mock("@/lib/ai-generation-entitlement", () => ({
isAiGenerationEnabledForUser: () => false,
}));
vi.mock("workflow", () => ({
+ createHook: mocks.createHook,
sleep: mocks.sleep,
FatalError: class FatalError extends Error {},
getWorkflowMetadata: () => ({ workflowRunId: "run-import" }),
@@ -110,6 +113,12 @@ function s3Error(name: string, httpStatusCode: number) {
describe("media processing workflows", () => {
beforeEach(() => {
+ mocks.hookDispose.mockReset();
+ mocks.createHook.mockReset().mockImplementation(() => ({
+ then: (resolve: (value: unknown) => void) =>
+ resolve({ phase: "complete" }),
+ dispose: mocks.hookDispose,
+ }));
mocks.rows = [];
mocks.env = {};
mocks.loomSourceUrl = "https://cdn.loom.com/original.mp4";
@@ -257,7 +266,8 @@ describe("media processing workflows", () => {
it("sends HLS downloads to the importer and defers transcription", async () => {
mocks.loomSourceUrl = "https://luna.loom.com/id/x/playlist.m3u8";
- mocks.rows = [[video], [pending], [], [metadata]];
+ mocks.rows = [[video], [], [metadata]];
+ mocks.sleep.mockImplementation(() => new Promise(() => undefined));
await expect(importLoomVideoWorkflow(payload)).resolves.toMatchObject({
success: true,
});
@@ -277,8 +287,12 @@ describe("media processing workflows", () => {
expect(body).toMatchObject({
videoUrl: "https://luna.loom.com/id/x/playlist.m3u8",
webhookUrl:
- "https://cap.example.com/api/webhooks/media-server/progress?retryable=true&transcription=on-view&preview=on-view",
+ "https://cap.example.com/api/webhooks/media-server/progress?retryable=true&signal=loom-import%3Avideo%3Arun-import%3A0&transcription=on-view&preview=on-view",
+ });
+ expect(mocks.createHook).toHaveBeenCalledWith({
+ token: "loom-import:video:run-import:0",
});
+ expect(mocks.hookDispose).toHaveBeenCalledTimes(1);
expect(body.previewGifPresignedUrl).toBeUndefined();
expect(body.thumbnailPresignedUrl).toBeDefined();
expect(body.sourcePresignedUrl).toBeUndefined();
diff --git a/apps/web/__tests__/unit/media-server-progress.test.ts b/apps/web/__tests__/unit/media-server-progress.test.ts
index 6ff04f4f7..f11129b9c 100644
--- a/apps/web/__tests__/unit/media-server-progress.test.ts
+++ b/apps/web/__tests__/unit/media-server-progress.test.ts
@@ -17,6 +17,7 @@ const mocks = vi.hoisted(() => ({
transcribe: vi.fn(),
invalidateQuota: vi.fn(),
audio: vi.fn(),
+ signal: vi.fn(),
tables: {
videos: {
id: "videos.id",
@@ -75,6 +76,10 @@ vi.mock("@/lib/queue-video-transcription", () => ({
shouldQueueTranscriptionAfterMediaComplete: () => true,
}));
+vi.mock("@/lib/loom-import-signal", () => ({
+ signalLoomImportFinished: mocks.signal,
+}));
+
import { POST } from "@/app/api/webhooks/media-server/progress/route";
import {
parseDesktopRecordingJob,
@@ -946,6 +951,56 @@ describe("Loom importer completions", () => {
expect(write.sql).not.toContain("loomPreviewGif");
});
+ it("wakes the waiting import once the finished state is saved", async () => {
+ const database = loomVideo();
+ mocks.signal.mockReset().mockImplementation(async () => {
+ expect(database.mutations).toContainEqual({
+ operation: "delete",
+ table: mocks.tables.uploads,
+ });
+ });
+ const token = `loom-import:${videoId}:run-1:0`;
+ const response = await request(
+ complete(),
+ "media-secret",
+ `?retryable=true&signal=${encodeURIComponent(token)}`,
+ );
+ expect(response.status).toBe(200);
+ expect(mocks.signal).toHaveBeenCalledWith(token, videoId, "complete");
+ });
+
+ it("wakes the waiting import when the worker fails", async () => {
+ loomVideo();
+ mocks.signal.mockReset().mockResolvedValue(undefined);
+ await request(
+ {
+ jobId: "importer-1",
+ videoId,
+ phase: "error",
+ progress: 40,
+ error: "Loom 404",
+ },
+ "media-secret",
+ "?retryable=true&signal=loom-import%3Aowned-video%3Arun-1%3A0",
+ );
+ expect(mocks.signal).toHaveBeenCalledWith(
+ "loom-import:owned-video:run-1:0",
+ videoId,
+ "error",
+ );
+ });
+
+ it("does not wake the import for progress updates", async () => {
+ loomVideo();
+ mocks.signal.mockReset().mockResolvedValue(undefined);
+ await request(
+ { jobId: "importer-1", videoId, phase: "downloading", progress: 40 },
+ "media-secret",
+ "?retryable=true&signal=loom-import%3Aowned-video%3Arun-1%3A0",
+ );
+ expect(mocks.signal).not.toHaveBeenCalled();
+ });
+
it("keeps queueing transcription for media server completions", async () => {
const database = loomVideo();
await request(
diff --git a/apps/web/__tests__/unit/video-processing-status.test.ts b/apps/web/__tests__/unit/video-processing-status.test.ts
index 979821b2a..17acef0c4 100644
--- a/apps/web/__tests__/unit/video-processing-status.test.ts
+++ b/apps/web/__tests__/unit/video-processing-status.test.ts
@@ -26,6 +26,7 @@ vi.mock("workflow", () => ({
import {
readVideoProcessingStatus,
+ waitForSignalledVideoProcessing,
waitForVideoProcessing,
} from "@/workflows/video-processing-status";
@@ -108,3 +109,75 @@ describe("durable video processing completion", () => {
);
});
});
+
+describe("signalled video processing completion", () => {
+ beforeEach(() => {
+ mocks.rows = [];
+ mocks.now = 0;
+ vi.spyOn(Date, "now").mockImplementation(() => mocks.now);
+ mocks.where.mockReset().mockImplementation(async () => mocks.rows.shift());
+ mocks.sleep.mockReset().mockImplementation(async (delay: number) => {
+ mocks.now += delay;
+ });
+ });
+
+ afterEach(() => vi.restoreAllMocks());
+
+ it("reads the status once when the worker signals", async () => {
+ mocks.sleep.mockImplementation(() => new Promise(() => undefined));
+ mocks.rows = [[], [metadata]];
+ await expect(
+ waitForSignalledVideoProcessing("video", Promise.resolve()),
+ ).resolves.toEqual(metadata);
+ expect(mocks.where).toHaveBeenCalledTimes(2);
+ });
+
+ it("checks every five minutes when no signal arrives", async () => {
+ mocks.rows = [[pending], [pending], [], [metadata]];
+ await expect(
+ waitForSignalledVideoProcessing("video", new Promise(() => undefined)),
+ ).resolves.toEqual(metadata);
+ expect(mocks.sleep.mock.calls).toEqual([[300_000], [300_000], [300_000]]);
+ });
+
+ it("falls back to short polling if the signal beats the saved state", async () => {
+ const sleeps: number[] = [];
+ let signalled = false;
+ mocks.sleep.mockImplementation((delay: number) => {
+ sleeps.push(delay);
+ mocks.now += delay;
+ return delay === 300_000
+ ? new Promise(() => undefined)
+ : Promise.resolve();
+ });
+ mocks.rows = [[pending], [pending], [], [metadata]];
+ await expect(
+ waitForSignalledVideoProcessing(
+ "video",
+ Promise.resolve().then(() => {
+ signalled = true;
+ }),
+ ),
+ ).resolves.toEqual(metadata);
+ expect(signalled).toBe(true);
+ expect(sleeps).toEqual([300_000, 5_000, 10_000]);
+ });
+
+ it("propagates worker failures after the signal", async () => {
+ mocks.sleep.mockImplementation(() => new Promise(() => undefined));
+ mocks.rows = [[{ ...pending, processingError: "Loom 404" }]];
+ await expect(
+ waitForSignalledVideoProcessing("video", Promise.resolve()),
+ ).rejects.toThrow("Loom 404");
+ });
+
+ it("subscribes to the signal only once", async () => {
+ const then = vi.fn();
+ mocks.where.mockResolvedValue([pending]);
+ await expect(
+ waitForSignalledVideoProcessing("video", { then }),
+ ).rejects.toThrow("Video processing timed out");
+ expect(then).toHaveBeenCalledTimes(1);
+ expect(mocks.now).toBeLessThanOrEqual(60 * 60 * 1000 + 300_000);
+ });
+});
diff --git a/apps/web/app/api/webhooks/media-server/progress/route.ts b/apps/web/app/api/webhooks/media-server/progress/route.ts
index 9abb8f3a2..4147a4605 100644
--- a/apps/web/app/api/webhooks/media-server/progress/route.ts
+++ b/apps/web/app/api/webhooks/media-server/progress/route.ts
@@ -15,6 +15,7 @@ import {
} from "@/lib/desktop-recording-verification";
import { queueDesktopSegmentsFinalization } from "@/lib/desktop-segments-finalization";
import { invalidateGoogleDriveStorageQuotaCache } from "@/lib/google-drive-storage-quota";
+import { signalLoomImportFinished } from "@/lib/loom-import-signal";
import {
queueVideoTranscription,
shouldQueueTranscriptionAfterMediaComplete,
@@ -381,6 +382,13 @@ export async function POST(request: NextRequest) {
.where(eq(videoUploads.videoId, payload.videoId as Video.VideoId));
}
+ if (dbPhase === "complete" || dbPhase === "error")
+ await signalLoomImportFinished(
+ request.nextUrl.searchParams.get("signal"),
+ payload.videoId,
+ payload.phase,
+ );
+
return NextResponse.json({ success: true });
} catch (error) {
console.error("[media-server-webhook] Error processing webhook:", error);
diff --git a/apps/web/lib/loom-import-signal-token.ts b/apps/web/lib/loom-import-signal-token.ts
new file mode 100644
index 000000000..e29118933
--- /dev/null
+++ b/apps/web/lib/loom-import-signal-token.ts
@@ -0,0 +1,9 @@
+export const LOOM_IMPORT_SIGNAL_PREFIX = "loom-import:";
+
+export function getLoomImportSignalToken(videoId: string, attemptId: string) {
+ return `${LOOM_IMPORT_SIGNAL_PREFIX}${videoId}:${attemptId}`;
+}
+
+export function isLoomImportSignalToken(token: string, videoId: string) {
+ return token.startsWith(`${LOOM_IMPORT_SIGNAL_PREFIX}${videoId}:`);
+}
diff --git a/apps/web/lib/loom-import-signal.ts b/apps/web/lib/loom-import-signal.ts
new file mode 100644
index 000000000..ed9b95752
--- /dev/null
+++ b/apps/web/lib/loom-import-signal.ts
@@ -0,0 +1,22 @@
+import "server-only";
+
+import { resumeHook } from "workflow/api";
+import { HookNotFoundError } from "workflow/errors";
+import { isLoomImportSignalToken } from "@/lib/loom-import-signal-token";
+
+export async function signalLoomImportFinished(
+ token: string | null,
+ videoId: string,
+ phase: string,
+) {
+ if (!token || !isLoomImportSignalToken(token, videoId)) return;
+ try {
+ await resumeHook(token, { phase });
+ } catch (error) {
+ if (HookNotFoundError.is(error)) return;
+ console.warn("[loom-import-signal] Could not wake the import", {
+ videoId,
+ error,
+ });
+ }
+}
diff --git a/apps/web/workflows/import-loom-video.ts b/apps/web/workflows/import-loom-video.ts
index b0644a108..f0323cb7d 100644
--- a/apps/web/workflows/import-loom-video.ts
+++ b/apps/web/workflows/import-loom-video.ts
@@ -6,6 +6,7 @@ import { Video } from "@cap/web-domain";
import { and, eq, sql } from "drizzle-orm";
import { Effect } from "effect";
import {
+ createHook,
FatalError,
getWorkflowMetadata,
RetryableError,
@@ -17,6 +18,7 @@ import {
LoomDownloadTemporaryError,
} from "@/lib/loom-download-url";
import { cleanupLoomFileSource } from "@/lib/loom-file-cleanup";
+import { getLoomImportSignalToken } from "@/lib/loom-import-signal-token";
import {
LOOM_IMPORT_MAX_BYTES,
type LoomMultipartUploadTarget,
@@ -35,7 +37,7 @@ import { runWorkflowPromise } from "@/lib/workflow-runtime";
import {
type ProcessedVideoMetadata,
VideoProcessingFailedError,
- waitForVideoProcessing,
+ waitForSignalledVideoProcessing,
} from "./video-processing-status";
interface ImportLoomPayload {
@@ -243,57 +245,70 @@ export async function importLoomVideoWorkflow(
let metadata: ProcessedVideoMetadata;
for (let processingAttempt = 0; ; processingAttempt++) {
- let mediaJob: { jobId: string; worker: LoomWorker };
- let capacityRetryCount = 0;
- let previousSourceUrl =
- processingAttempt === 0 ? payload.loomDownloadUrl : undefined;
- while (true) {
+ const finished = createHook<{ phase: string }>({
+ token: getLoomImportSignalToken(
+ payload.videoId,
+ `${workflowRunId}:${processingAttempt}`,
+ ),
+ });
+ try {
+ let mediaJob: { jobId: string; worker: LoomWorker };
+ let capacityRetryCount = 0;
+ let previousSourceUrl =
+ processingAttempt === 0 ? payload.loomDownloadUrl : undefined;
+ while (true) {
+ try {
+ const loomSourceUrl = processingInput.importFromLoom
+ ? await fetchFreshLoomDownloadUrl(
+ payload.loomVideoId,
+ previousSourceUrl,
+ )
+ : undefined;
+ previousSourceUrl = loomSourceUrl;
+ mediaJob = await processVideoOnMediaServer(
+ payload,
+ loomSourceUrl,
+ `${workflowRunId}:${processingAttempt}`,
+ workflowRunId,
+ );
+ break;
+ } catch (error) {
+ if (!isMediaServerCapacityError(error)) throw error;
+ await markLoomImportWaitingForCapacity(payload.videoId);
+ await sleep(`${Math.min(180, 30 + capacityRetryCount * 15)}s`);
+ capacityRetryCount++;
+ }
+ }
try {
- const loomSourceUrl = processingInput.importFromLoom
- ? await fetchFreshLoomDownloadUrl(
- payload.loomVideoId,
- previousSourceUrl,
- )
- : undefined;
- previousSourceUrl = loomSourceUrl;
- mediaJob = await processVideoOnMediaServer(
- payload,
- loomSourceUrl,
- `${workflowRunId}:${processingAttempt}`,
- workflowRunId,
+ metadata = await waitForSignalledVideoProcessing(
+ payload.videoId,
+ finished,
);
break;
} catch (error) {
- if (!isMediaServerCapacityError(error)) throw error;
+ if (
+ !(error instanceof VideoProcessingFailedError) ||
+ processingAttempt >= 2
+ ) {
+ throw error;
+ }
+ if (
+ !(await confirmLoomMediaJobFailure(
+ mediaJob.jobId,
+ payload.videoId,
+ mediaJob.worker,
+ ))
+ ) {
+ throw new FatalError(
+ "The previous Loom worker must be reconciled before retrying",
+ );
+ }
await markLoomImportWaitingForCapacity(payload.videoId);
- await sleep(`${Math.min(180, 30 + capacityRetryCount * 15)}s`);
- capacityRetryCount++;
- }
- }
- try {
- metadata = await waitForVideoProcessing(payload.videoId);
- break;
- } catch (error) {
- if (
- !(error instanceof VideoProcessingFailedError) ||
- processingAttempt >= 2
- ) {
- throw error;
- }
- if (
- !(await confirmLoomMediaJobFailure(
- mediaJob.jobId,
- payload.videoId,
- mediaJob.worker,
- ))
- ) {
- throw new FatalError(
- "The previous Loom worker must be reconciled before retrying",
- );
+ if (await hasExistingLoomUpload(payload)) processingInput = {};
+ await sleep(15_000 * (processingAttempt + 1));
}
- await markLoomImportWaitingForCapacity(payload.videoId);
- if (await hasExistingLoomUpload(payload)) processingInput = {};
- await sleep(15_000 * (processingAttempt + 1));
+ } finally {
+ finished.dispose();
}
}
await saveMetadataAndComplete(payload.videoId, metadata);
@@ -654,7 +669,7 @@ async function processVideoOnMediaServer(
};
}).pipe(runWorkflowPromise);
- const webhookUrl = `${webhookBaseUrl}/api/webhooks/media-server/progress?retryable=true${worker === "importer" ? "&transcription=on-view&preview=on-view" : ""}`;
+ const webhookUrl = `${webhookBaseUrl}/api/webhooks/media-server/progress?retryable=true&signal=${encodeURIComponent(getLoomImportSignalToken(videoId, attemptId))}${worker === "importer" ? "&transcription=on-view&preview=on-view" : ""}`;
const webhookSecret = serverEnv().MEDIA_SERVER_WEBHOOK_SECRET;
try {
diff --git a/apps/web/workflows/video-processing-status.ts b/apps/web/workflows/video-processing-status.ts
index 8ff993a2e..34790eecf 100644
--- a/apps/web/workflows/video-processing-status.ts
+++ b/apps/web/workflows/video-processing-status.ts
@@ -109,3 +109,33 @@ export async function waitForVideoProcessing(
}
throw new FatalError(`Video processing timed out while ${lastStatus}`);
}
+
+const SIGNALLED_WAIT_FALLBACK_MS = 5 * 60 * 1000;
+
+export async function waitForSignalledVideoProcessing(
+ videoId: string,
+ signal: PromiseLike<unknown>,
+): Promise<ProcessedVideoMetadata> {
+ const deadline = Date.now() + 60 * 60 * 1000;
+ const signalReceived = Promise.resolve(signal).then(() => true);
+ let signalled = false;
+ let lastStatus = "processing";
+ for (let attempt = 0; Date.now() < deadline; attempt++) {
+ if (signalled) {
+ await sleep(Math.min(5_000 * attempt, 30_000));
+ } else {
+ signalled = await Promise.race([
+ signalReceived,
+ sleep(SIGNALLED_WAIT_FALLBACK_MS).then(() => false),
+ ]);
+ }
+ const result = await readVideoProcessingStatus(videoId);
+ if (result.status === "complete") return result.metadata;
+ if (result.status === "failed") {
+ throw new VideoProcessingFailedError(result.message);
+ }
+ if (result.status === "error") throw new FatalError(result.message);
+ lastStatus = result.message;
+ }
+ throw new FatalError(`Video processing timed out while ${lastStatus}`);
+}