Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix: make deferred Loom processing retryable and idempotent
Richie McIlroyee6afbef1cb6
diff --git a/apps/web/__tests__/unit/media-server-progress.test.ts b/apps/web/__tests__/unit/media-server-progress.test.tsindex 1da5dc8c6..ed15fdca8 100644--- a/apps/web/__tests__/unit/media-server-progress.test.ts+++ b/apps/web/__tests__/unit/media-server-progress.test.ts@@ -873,7 +873,9 @@ describe("Loom importer completions", () => {"?retryable=true&transcription=on-view",);const [write] = metadataWrites(database.mutations);- expect(write?.sql).toContain("'$.loomDeferredProcessing', JSON_OBJECT(");+ expect(write?.sql).toContain(+ "JSON_OBJECT('rawFileKey', ?, 'status', 'pending')",+ );expect(write?.params).toContain(key);});@@ -892,6 +894,34 @@ describe("Loom importer completions", () => {expect(write.sql).not.toContain("loomDeferredProcessing");});+ it("keeps an existing deferred claim when a completion is repeated", async () => {+ const database = loomVideo();+ await request(+ complete({ deferredSourceKey: `${ownerId}/${videoId}/raw-upload.webm` }),+ "media-secret",+ "?retryable=true&transcription=on-view",+ );+ const [write] = metadataWrites(database.mutations);+ expect(write?.sql).toContain(+ "IF(JSON_EXTRACT(?, '$.loomDeferredProcessing') IS NULL",+ );+ });++ it("marks a deferred original complete once it has been processed", async () => {+ const database = loomVideo();+ const key = `${ownerId}/${videoId}/raw-upload.webm`;+ database.video.metadata = {+ loomDeferredProcessing: { rawFileKey: key, status: "started" },+ };+ database.setRawFileKey(key);+ await request(complete(), "media-secret", "?retryable=true");+ const [write] = metadataWrites(database.mutations);+ expect(write?.sql).toContain(+ "'$.loomDeferredProcessing.status', 'complete'",+ );+ expect(mocks.transcribe).toHaveBeenCalledWith(videoId);+ });+it("keeps queueing transcription for media server completions", async () => {const database = loomVideo();await request(diff --git a/apps/web/app/api/webhooks/media-server/progress/route.ts b/apps/web/app/api/webhooks/media-server/progress/route.tsindex 4c455ad68..bde5992d2 100644--- a/apps/web/app/api/webhooks/media-server/progress/route.ts+++ b/apps/web/app/api/webhooks/media-server/progress/route.ts@@ -3,7 +3,7 @@ import { db } from "@cap/database";import { videos, videoUploads } from "@cap/database/schema";import { serverEnv } from "@cap/env";import type { Video } from "@cap/web-domain";-import { eq, sql } from "drizzle-orm";+import { and, eq, isNull, ne, or, sql } from "drizzle-orm";import { type NextRequest, NextResponse } from "next/server";import { handleAudioLevelPublication } from "@/lib/audio-level-publication";import { SourceCommitPendingError } from "@/lib/desktop-recording-jobs";@@ -248,9 +248,14 @@ export async function POST(request: NextRequest) {...(transcribeOnView? [sql`'$.transcribeOnView', JSON_EXTRACT('true', '$')`]: []),+ ...(currentVideo?.metadata?.loomDeferredProcessing &&+ currentUpload?.rawFileKey ===+ currentVideo.metadata.loomDeferredProcessing.rawFileKey+ ? [sql`'$.loomDeferredProcessing.status', 'complete'`]+ : []),...(deferredSourceKey? [- sql`'$.loomDeferredProcessing', JSON_OBJECT('rawFileKey', ${deferredSourceKey}, 'status', 'pending')`,+ sql`'$.loomDeferredProcessing', IF(JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing') IS NULL, JSON_OBJECT('rawFileKey', ${deferredSourceKey}, 'status', 'pending'), JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing'))`,]: []),];@@ -291,7 +296,17 @@ export async function POST(request: NextRequest) {} else {await db().delete(videoUploads)- .where(eq(videoUploads.videoId, payload.videoId as Video.VideoId));+ .where(+ and(+ eq(videoUploads.videoId, payload.videoId as Video.VideoId),+ deferredSourceKey+ ? or(+ isNull(videoUploads.rawFileKey),+ ne(videoUploads.rawFileKey, deferredSourceKey),+ )+ : undefined,+ ),+ );}await invalidateGoogleDriveStorageQuotaCache(currentVideo?.storageIntegrationId,diff --git a/apps/web/app/embed/[videoId]/page.tsx b/apps/web/app/embed/[videoId]/page.tsxindex 73bb3cd29..8ff97bc13 100644--- a/apps/web/app/embed/[videoId]/page.tsx+++ b/apps/web/app/embed/[videoId]/page.tsx@@ -27,6 +27,7 @@ import { Effect, Option } from "effect";import type { Metadata } from "next";import Link from "next/link";import { notFound } from "next/navigation";+import { after } from "next/server";import {getPendingLoomProcessing,startDeferredLoomProcessing,@@ -278,17 +279,20 @@ async function EmbedContent({? null: getPendingLoomProcessing(currentMetadata, video.ownerId, video.id);if (deferredLoomRawFileKey) {- startDeferredLoomProcessing({- videoId: video.id,- ownerId: video.ownerId,- bucketId: video.bucket,- rawFileKey: deferredLoomRawFileKey,- }).catch((error) => {- console.error(- `[EmbedVideoPage] Error starting deferred processing for ${video.id}:`,- error,- );- });+ const rawFileKey = deferredLoomRawFileKey;+ after(() =>+ startDeferredLoomProcessing({+ videoId: video.id,+ ownerId: video.ownerId,+ bucketId: video.bucket,+ rawFileKey,+ }).catch((error) => {+ console.error(+ `[EmbedVideoPage] Error starting deferred processing for ${video.id}:`,+ error,+ );+ }),+ );}let initialAiData = null;diff --git a/apps/web/app/s/[videoId]/page.tsx b/apps/web/app/s/[videoId]/page.tsxindex 878eb1853..e8dbf48de 100644--- a/apps/web/app/s/[videoId]/page.tsx+++ b/apps/web/app/s/[videoId]/page.tsx@@ -39,6 +39,7 @@ import type { Metadata } from "next";import { headers } from "next/headers";import Link from "next/link";import { notFound } from "next/navigation";+import { after } from "next/server";import { getVideoAnalytics } from "@/actions/videos/get-analytics";import {getDashboardSpacesData,@@ -881,17 +882,20 @@ async function AuthorizedContent({? null: getPendingLoomProcessing(metadata, video.owner.id, videoId);if (deferredLoomRawFileKey) {- startDeferredLoomProcessing({- videoId,- ownerId: video.owner.id,- bucketId: video.bucket,- rawFileKey: deferredLoomRawFileKey,- }).catch((error) => {- console.error(- `[ShareVideoPage] Error starting deferred processing for ${videoId}:`,- error,- );- });+ const rawFileKey = deferredLoomRawFileKey;+ after(() =>+ startDeferredLoomProcessing({+ videoId,+ ownerId: video.owner.id,+ bucketId: video.bucket,+ rawFileKey,+ }).catch((error) => {+ console.error(+ `[ShareVideoPage] Error starting deferred processing for ${videoId}:`,+ error,+ );+ }),+ );}const aiGenerationStatus = metadata.aiGenerationStatus || null;diff --git a/apps/web/lib/loom-deferred-processing.ts b/apps/web/lib/loom-deferred-processing.tsindex e0dc3deed..63437dbef 100644--- a/apps/web/lib/loom-deferred-processing.ts+++ b/apps/web/lib/loom-deferred-processing.ts@@ -5,6 +5,8 @@ import type { Video } from "@cap/web-domain";import { and, eq, sql } from "drizzle-orm";import { startVideoProcessingWorkflow } from "@/lib/video-processing";+export const DEFERRED_PROCESSING_RECLAIM_MS = 30 * 60 * 1000;+function getAffectedRows(result: unknown) {const header = Array.isArray(result) ? result[0] : result;return (header as { affectedRows?: number } | undefined)?.affectedRows ?? 0;@@ -14,59 +16,95 @@ export function getPendingLoomProcessing(metadata: VideoMetadata | null | undefined,ownerId: string,videoId: string,+ now = Date.now(),) {const deferred = metadata?.loomDeferredProcessing;if (- deferred?.status !== "pending" ||+ !deferred ||typeof deferred.rawFileKey !== "string" ||!deferred.rawFileKey.startsWith(`${ownerId}/${videoId}/`))return null;- return deferred.rawFileKey;+ if (deferred.status === "pending") return deferred.rawFileKey;+ const startedAt = Date.parse(deferred.startedAt ?? "");+ return deferred.status === "started" &&+ Number.isFinite(startedAt) &&+ now - startedAt > DEFERRED_PROCESSING_RECLAIM_MS+ ? deferred.rawFileKey+ : null;}-export async function startDeferredLoomProcessing({- videoId,- ownerId,- bucketId,- rawFileKey,-}: {- videoId: Video.VideoId;- ownerId: string;- bucketId: string | null;- rawFileKey: string;-}) {+export async function startDeferredLoomProcessing(+ {+ videoId,+ ownerId,+ bucketId,+ rawFileKey,+ }: {+ videoId: Video.VideoId;+ ownerId: string;+ bucketId: string | null;+ rawFileKey: string;+ },+ now = new Date(),+) {+ const startedAt = now.toISOString();+ const reclaimBefore = new Date(+ now.getTime() - DEFERRED_PROCESSING_RECLAIM_MS,+ ).toISOString();const claim = await db().update(videos).set({- metadata: sql`JSON_SET(${videos.metadata}, '$.loomDeferredProcessing.status', 'started', '$.loomDeferredProcessing.startedAt', ${new Date().toISOString()})`,+ metadata: sql`JSON_SET(${videos.metadata}, '$.loomDeferredProcessing.status', 'started', '$.loomDeferredProcessing.startedAt', ${startedAt})`,}).where(and(eq(videos.id, videoId),- sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing.status')) = 'pending'`,sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing.rawFileKey')) = ${rawFileKey}`,+ sql`(JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing.status')) = 'pending' OR (JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing.status')) = 'started' AND JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing.startedAt')) < ${reclaimBefore}))`,),);if (getAffectedRows(claim) !== 1) return "already-started" as const;- await db()- .insert(videoUploads)- .values({+ try {+ await db()+ .insert(videoUploads)+ .values({+ videoId,+ mode: "singlepart",+ phase: "uploading",+ rawFileKey,+ processingMessage: "Optimizing imported video...",+ })+ .onDuplicateKeyUpdate({ set: { rawFileKey, phase: "uploading" } });+ return await startVideoProcessingWorkflow({videoId,- mode: "singlepart",- phase: "uploading",+ userId: ownerId,rawFileKey,+ bucketId,processingMessage: "Optimizing imported video...",- })- .onDuplicateKeyUpdate({ set: { rawFileKey } });-- return startVideoProcessingWorkflow({- videoId,- userId: ownerId,- rawFileKey,- bucketId,- processingMessage: "Optimizing imported video...",- startFailureMessage: "Imported video could not be optimized",- });+ startFailureMessage: "Imported video could not be optimized",+ });+ } catch (error) {+ await db()+ .delete(videoUploads)+ .where(+ and(+ eq(videoUploads.videoId, videoId),+ eq(videoUploads.rawFileKey, rawFileKey),+ ),+ );+ await db()+ .update(videos)+ .set({+ metadata: sql`JSON_SET(${videos.metadata}, '$.loomDeferredProcessing.status', 'pending')`,+ })+ .where(+ and(+ eq(videos.id, videoId),+ sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing.startedAt')) = ${startedAt}`,+ ),+ );+ throw error;+ }}diff --git a/packages/database/types/metadata.ts b/packages/database/types/metadata.tsindex c0794bb9f..921c149a4 100644--- a/packages/database/types/metadata.ts+++ b/packages/database/types/metadata.ts@@ -25,7 +25,7 @@ export interface VideoMetadata {};loomDeferredProcessing?: {rawFileKey: string;- status: "pending" | "started";+ status: "pending" | "started" | "complete";startedAt?: string;};transcribeOnView?: boolean;