Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

feat: add an authenticated restart for failed Loom imports

Richie McIlroy44580ea17cf3
diff --git a/apps/web/__tests__/unit/loom-importer-redrive.test.ts b/apps/web/__tests__/unit/loom-importer-redrive.test.ts
new file mode 100644
index 000000000..c7748480b
--- /dev/null
+++ b/apps/web/__tests__/unit/loom-importer-redrive.test.ts
@@ -0,0 +1,292 @@
+import { NextRequest } from "next/server";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ rows: [] as unknown[],
+ affectedRows: 1,
+ updates: [] as Array<Record<string, unknown>>,
+ inserts: [] as Array<Record<string, unknown>>,
+ runState: vi.fn(),
+ start: vi.fn(),
+ restore: vi.fn(),
+ startProcessing: vi.fn(),
+}));
+
+vi.mock("@cap/database", () => {
+ const selection = {
+ from: () => selection,
+ innerJoin: () => selection,
+ where: () => selection,
+ limit: async () => mocks.rows,
+ };
+ return {
+ db: () => ({
+ select: () => selection,
+ update: () => ({
+ set: (values: Record<string, unknown>) => ({
+ where: async () => {
+ mocks.updates.push(values);
+ return [{ affectedRows: mocks.affectedRows }];
+ },
+ }),
+ }),
+ insert: () => ({
+ values: (values: Record<string, unknown>) => ({
+ onDuplicateKeyUpdate: async () => {
+ mocks.inserts.push(values);
+ },
+ }),
+ }),
+ }),
+ };
+});
+vi.mock("@cap/database/schema", () => ({
+ importedVideos: { id: "i.id", orgId: "i.orgId", source: "i.source" },
+ videos: { id: "v.id", orgId: "v.orgId", metadata: "v.metadata" },
+ videoUploads: { videoId: "u.videoId", phase: "u.phase", updatedAt: "u.at" },
+}));
+vi.mock("@cap/env", () => ({
+ serverEnv: () => ({ MEDIA_SERVER_WEBHOOK_SECRET: "media-secret" }),
+}));
+vi.mock("@cap/web-domain", () => ({
+ Video: { VideoId: { make: (id: string) => id } },
+}));
+vi.mock("@/lib/loom-import-start", () => ({
+ getLoomImportRunState: mocks.runState,
+ startLoomImportWorkflow: mocks.start,
+ restoreLoomImportStartError: mocks.restore,
+ LoomImportStartError: class LoomImportStartError extends Error {
+ constructor(readonly canRetry: boolean) {
+ super("Loom import could not start. Please try again.");
+ }
+ },
+}));
+vi.mock("@/lib/video-processing", () => ({
+ startVideoProcessingWorkflow: mocks.startProcessing,
+}));
+
+import { POST } from "@/app/api/internal/loom-imports/redrive/route";
+import {
+ getPendingLoomProcessing,
+ startDeferredLoomProcessing,
+} from "@/lib/loom-deferred-processing";
+import { redriveLoomImport } from "@/lib/loom-import-redrive";
+import { LoomImportStartError } from "@/lib/loom-import-start";
+
+const now = new Date("2026-10-24T12:00:00Z");
+const hoursAgo = (hours: number) =>
+ new Date(now.getTime() - hours * 60 * 60 * 1000);
+const failedImport = (overrides: Record<string, unknown> = {}) => ({
+ ownerId: "owner",
+ bucketId: "bucket",
+ metadata: { loomImportRun: { runId: "run", dispatch: "accepted" } },
+ phase: "error",
+ rawFileKey: null,
+ uploadUpdatedAt: hoursAgo(48),
+ loomVideoId: "a".repeat(32),
+ ...overrides,
+});
+
+beforeEach(() => {
+ mocks.rows = [failedImport()];
+ mocks.affectedRows = 1;
+ mocks.updates = [];
+ mocks.inserts = [];
+ mocks.runState.mockReset().mockResolvedValue("inactive");
+ mocks.start.mockReset().mockResolvedValue(undefined);
+ mocks.restore.mockReset().mockResolvedValue(undefined);
+ mocks.startProcessing.mockReset().mockResolvedValue("started");
+});
+
+describe("redriveLoomImport", () => {
+ it("restarts a failed import in place for its current owner", async () => {
+ await expect(redriveLoomImport("video", now)).resolves.toEqual({
+ videoId: "video",
+ status: "restarted",
+ });
+ expect(mocks.updates[0]).toMatchObject({
+ phase: "processing",
+ processingError: null,
+ rawFileKey: "owner/video/raw-upload.mp4",
+ });
+ expect(mocks.start).toHaveBeenCalledWith({
+ videoId: "video",
+ userId: "owner",
+ rawFileKey: "owner/video/raw-upload.mp4",
+ bucketId: "bucket",
+ loomVideoId: "a".repeat(32),
+ reuseExistingRawUpload: true,
+ });
+ });
+
+ it.each([
+ [[], "not an unfinished Loom import"],
+ [[failedImport({ phase: "uploading" })], "phase uploading"],
+ [
+ [failedImport({ phase: "processing", uploadUpdatedAt: hoursAgo(30) })],
+ "phase processing",
+ ],
+ ])("skips imports that are not failed: %#", async (rows, reason) => {
+ mocks.rows = rows;
+ await expect(redriveLoomImport("video", now)).resolves.toMatchObject({
+ status: "skipped",
+ reason,
+ });
+ expect(mocks.start).not.toHaveBeenCalled();
+ expect(mocks.updates).toEqual([]);
+ });
+
+ it("restarts imports stuck processing for over a week", async () => {
+ mocks.rows = [
+ failedImport({
+ phase: "processing",
+ rawFileKey: "owner/video/raw-upload.mp4",
+ uploadUpdatedAt: hoursAgo(24 * 8),
+ }),
+ ];
+ await expect(redriveLoomImport("video", now)).resolves.toMatchObject({
+ status: "restarted",
+ });
+ });
+
+ it.each(["pending", "active"])("leaves a %s run alone", async (state) => {
+ mocks.runState.mockResolvedValue(state);
+ await expect(redriveLoomImport("video", now)).resolves.toMatchObject({
+ status: "skipped",
+ });
+ expect(mocks.updates).toEqual([]);
+ });
+
+ it("only restarts an unconfirmed worker once it can no longer be running", async () => {
+ mocks.runState.mockResolvedValue("unconfirmed");
+ mocks.rows = [failedImport({ uploadUpdatedAt: hoursAgo(2) })];
+ await expect(redriveLoomImport("video", now)).resolves.toMatchObject({
+ status: "skipped",
+ reason: "run unconfirmed",
+ });
+ mocks.rows = [failedImport({ uploadUpdatedAt: hoursAgo(7) })];
+ await expect(redriveLoomImport("video", now)).resolves.toMatchObject({
+ status: "restarted",
+ });
+ });
+
+ it("does not start a workflow when another restart claimed the row", async () => {
+ mocks.affectedRows = 0;
+ await expect(redriveLoomImport("video", now)).resolves.toMatchObject({
+ status: "skipped",
+ reason: "already restarting",
+ });
+ expect(mocks.start).not.toHaveBeenCalled();
+ });
+
+ it("puts the import back into its error state when the start is rejected", async () => {
+ mocks.start.mockRejectedValue(new LoomImportStartError(true));
+ await expect(redriveLoomImport("video", now)).resolves.toMatchObject({
+ status: "failed",
+ });
+ expect(mocks.restore).toHaveBeenCalledWith(
+ "video",
+ "processing",
+ expect.any(Date),
+ "Loom import could not start. Please try again.",
+ );
+ });
+});
+
+describe("deferred Loom processing", () => {
+ it.each([
+ [
+ { rawFileKey: "owner/video/raw-upload.webm", status: "pending" },
+ "owner/video/raw-upload.webm",
+ ],
+ [{ rawFileKey: "owner/video/raw-upload.webm", status: "started" }, null],
+ [{ rawFileKey: "other/video/raw-upload.webm", status: "pending" }, null],
+ [undefined, null],
+ ] as const)("finds pending work in %o", (deferred, expected) => {
+ expect(
+ getPendingLoomProcessing(
+ deferred ? { loomDeferredProcessing: { ...deferred } } : {},
+ "owner",
+ "video",
+ ),
+ ).toBe(expected);
+ });
+
+ it("starts processing the original once", async () => {
+ const input = {
+ videoId: "video" as never,
+ ownerId: "owner",
+ bucketId: "bucket",
+ rawFileKey: "owner/video/raw-upload.webm",
+ };
+ await expect(startDeferredLoomProcessing(input)).resolves.toBe("started");
+ expect(mocks.inserts[0]).toMatchObject({
+ videoId: "video",
+ phase: "uploading",
+ rawFileKey: "owner/video/raw-upload.webm",
+ });
+ expect(mocks.startProcessing).toHaveBeenCalledWith(
+ expect.objectContaining({
+ videoId: "video",
+ userId: "owner",
+ rawFileKey: "owner/video/raw-upload.webm",
+ bucketId: "bucket",
+ }),
+ );
+ mocks.affectedRows = 0;
+ await expect(startDeferredLoomProcessing(input)).resolves.toBe(
+ "already-started",
+ );
+ expect(mocks.startProcessing).toHaveBeenCalledOnce();
+ });
+});
+
+describe("POST /api/internal/loom-imports/redrive", () => {
+ const call = (body: unknown, secret: string | null = "media-secret") =>
+ POST(
+ new NextRequest("https://cap.so/api/internal/loom-imports/redrive", {
+ method: "POST",
+ headers: secret ? { "x-media-server-secret": secret } : {},
+ body: JSON.stringify(body),
+ }),
+ );
+
+ it("requires the media server secret", async () => {
+ expect((await call({ videoIds: ["abcdefghjkmnpqr"] }, null)).status).toBe(
+ 401,
+ );
+ expect(
+ (await call({ videoIds: ["abcdefghjkmnpqr"] }, "wrong")).status,
+ ).toBe(401);
+ expect(mocks.start).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ {},
+ { videoIds: [] },
+ { videoIds: ["../../etc"] },
+ {
+ videoIds: Array.from(
+ { length: 26 },
+ (_, i) => `abcdefghjkmn${String(i).padStart(3, "0")}`,
+ ),
+ },
+ ])("rejects invalid batches %#", async (body) => {
+ expect((await call(body)).status).toBe(400);
+ });
+
+ it("restarts each distinct video once", async () => {
+ const response = await call({
+ videoIds: ["abcdefghjkmnpqr", "abcdefghjkmnpqr", "bbcdefghjkmnpqr"],
+ });
+ expect(response.status).toBe(200);
+ const { results } = (await response.json()) as {
+ results: Array<{ videoId: string; status: string }>;
+ };
+ expect(results.map(({ videoId }) => videoId).sort()).toEqual([
+ "abcdefghjkmnpqr",
+ "bbcdefghjkmnpqr",
+ ]);
+ expect(mocks.start).toHaveBeenCalledTimes(2);
+ });
+});
diff --git a/apps/web/app/api/internal/loom-imports/redrive/route.ts b/apps/web/app/api/internal/loom-imports/redrive/route.ts
new file mode 100644
index 000000000..f38f0422b
--- /dev/null
+++ b/apps/web/app/api/internal/loom-imports/redrive/route.ts
@@ -0,0 +1,57 @@
+import { createHash, timingSafeEqual } from "node:crypto";
+import { serverEnv } from "@cap/env";
+import { type NextRequest, NextResponse } from "next/server";
+import { z } from "zod";
+import {
+ type LoomRedriveOutcome,
+ redriveLoomImport,
+} from "@/lib/loom-import-redrive";
+
+const MAX_VIDEOS_PER_REQUEST = 25;
+const CONCURRENCY = 5;
+
+const bodySchema = z.object({
+ videoIds: z
+ .array(z.string().regex(/^[a-z0-9]{15}$/))
+ .min(1)
+ .max(MAX_VIDEOS_PER_REQUEST),
+});
+
+function isAuthorized(request: NextRequest) {
+ const secret = serverEnv().MEDIA_SERVER_WEBHOOK_SECRET;
+ const header = request.headers.get("x-media-server-secret");
+ if (!secret || !header) return false;
+ const digest = (value: string) =>
+ createHash("sha256").update(value, "utf8").digest();
+ return timingSafeEqual(digest(header), digest(secret));
+}
+
+export async function POST(request: NextRequest) {
+ if (!isAuthorized(request))
+ return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
+ const parsed = bodySchema.safeParse(await request.json().catch(() => null));
+ if (!parsed.success)
+ return NextResponse.json(
+ { error: "Invalid request", details: parsed.error.issues },
+ { status: 400 },
+ );
+ const videoIds = [...new Set(parsed.data.videoIds)];
+ const results: LoomRedriveOutcome[] = [];
+ let next = 0;
+ await Promise.all(
+ Array.from({ length: Math.min(CONCURRENCY, videoIds.length) }, async () => {
+ while (next < videoIds.length) {
+ const videoId = videoIds[next++];
+ if (!videoId) continue;
+ results.push(
+ await redriveLoomImport(videoId).catch((error) => ({
+ videoId,
+ status: "failed" as const,
+ reason: error instanceof Error ? error.message : String(error),
+ })),
+ );
+ }
+ }),
+ );
+ return NextResponse.json({ results });
+}
diff --git a/apps/web/lib/loom-import-redrive.ts b/apps/web/lib/loom-import-redrive.ts
new file mode 100644
index 000000000..b436e2d22
--- /dev/null
+++ b/apps/web/lib/loom-import-redrive.ts
@@ -0,0 +1,107 @@
+import { db } from "@cap/database";
+import { importedVideos, videos, videoUploads } from "@cap/database/schema";
+import { Video } from "@cap/web-domain";
+import { and, eq } from "drizzle-orm";
+import {
+ getLoomImportRunState,
+ LoomImportStartError,
+ restoreLoomImportStartError,
+ startLoomImportWorkflow,
+} from "@/lib/loom-import-start";
+
+const FINISHED_WORKER_AGE_MS = 6 * 60 * 60 * 1000;
+const STALE_PROCESSING_AGE_MS = 7 * 24 * 60 * 60 * 1000;
+
+export type LoomRedriveOutcome =
+ | { videoId: string; status: "restarted" }
+ | { videoId: string; status: "skipped" | "failed"; reason: string };
+
+function getAffectedRows(result: unknown) {
+ const header = Array.isArray(result) ? result[0] : result;
+ return (header as { affectedRows?: number } | undefined)?.affectedRows ?? 0;
+}
+
+export async function redriveLoomImport(
+ videoId: string,
+ now = new Date(),
+): Promise<LoomRedriveOutcome> {
+ const id = Video.VideoId.make(videoId);
+ const [row] = await db()
+ .select({
+ ownerId: videos.ownerId,
+ bucketId: videos.bucket,
+ metadata: videos.metadata,
+ phase: videoUploads.phase,
+ rawFileKey: videoUploads.rawFileKey,
+ uploadUpdatedAt: videoUploads.updatedAt,
+ loomVideoId: importedVideos.sourceId,
+ })
+ .from(videos)
+ .innerJoin(
+ importedVideos,
+ and(
+ eq(importedVideos.id, videos.id),
+ eq(importedVideos.orgId, videos.orgId),
+ eq(importedVideos.source, "loom"),
+ ),
+ )
+ .innerJoin(videoUploads, eq(videoUploads.videoId, videos.id))
+ .where(eq(videos.id, id))
+ .limit(1);
+ if (!row)
+ return {
+ videoId,
+ status: "skipped",
+ reason: "not an unfinished Loom import",
+ };
+ const age = now.getTime() - row.uploadUpdatedAt.getTime();
+ const staleProcessing =
+ row.phase === "processing" && age > STALE_PROCESSING_AGE_MS;
+ if (row.phase !== "error" && !staleProcessing)
+ return { videoId, status: "skipped", reason: `phase ${row.phase}` };
+ const runState = await getLoomImportRunState(row.metadata?.loomImportRun);
+ if (runState === "pending" || runState === "active")
+ return { videoId, status: "skipped", reason: `run ${runState}` };
+ if (runState === "unconfirmed" && age < FINISHED_WORKER_AGE_MS)
+ return { videoId, status: "skipped", reason: "run unconfirmed" };
+
+ const rawFileKey = row.rawFileKey ?? `${row.ownerId}/${id}/raw-upload.mp4`;
+ const claimedAt = new Date(Math.floor(now.getTime() / 1000) * 1000);
+ const claim = await db()
+ .update(videoUploads)
+ .set({
+ phase: "processing",
+ processingProgress: 0,
+ processingMessage: "Restarting Loom import...",
+ processingError: null,
+ rawFileKey,
+ updatedAt: claimedAt,
+ })
+ .where(
+ and(
+ eq(videoUploads.videoId, id),
+ eq(videoUploads.phase, row.phase),
+ eq(videoUploads.updatedAt, row.uploadUpdatedAt),
+ ),
+ );
+ if (getAffectedRows(claim) !== 1)
+ return { videoId, status: "skipped", reason: "already restarting" };
+
+ try {
+ await startLoomImportWorkflow({
+ videoId: id,
+ userId: row.ownerId,
+ rawFileKey,
+ bucketId: row.bucketId,
+ loomVideoId: row.loomVideoId,
+ reuseExistingRawUpload: true,
+ });
+ } catch (error) {
+ const message =
+ error instanceof Error ? error.message : "Loom import could not restart.";
+ if (error instanceof LoomImportStartError && error.canRetry)
+ await restoreLoomImportStartError(id, "processing", claimedAt, message);
+ return { videoId, status: "failed", reason: message };
+ }
+ return { videoId, status: "restarted" };
+}