Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix: make deferred Loom processing retryable and idempotent

Richie McIlroyee6afbef1cb6
diff --git a/apps/web/__tests__/unit/media-server-progress.test.ts b/apps/web/__tests__/unit/media-server-progress.test.ts
index 1da5dc8c6..ed15fdca8 100644
--- a/apps/web/__tests__/unit/media-server-progress.test.ts
+++ b/apps/web/__tests__/unit/media-server-progress.test.ts
@@ -873,7 +873,9 @@ describe("Loom importer completions", () => {
"?retryable=true&transcription=on-view",
);
const [write] = metadataWrites(database.mutations);
- expect(write?.sql).toContain("'$.loomDeferredProcessing', JSON_OBJECT(");
+ expect(write?.sql).toContain(
+ "JSON_OBJECT('rawFileKey', ?, 'status', 'pending')",
+ );
expect(write?.params).toContain(key);
});
@@ -892,6 +894,34 @@ describe("Loom importer completions", () => {
expect(write.sql).not.toContain("loomDeferredProcessing");
});
+ it("keeps an existing deferred claim when a completion is repeated", async () => {
+ const database = loomVideo();
+ await request(
+ complete({ deferredSourceKey: `${ownerId}/${videoId}/raw-upload.webm` }),
+ "media-secret",
+ "?retryable=true&transcription=on-view",
+ );
+ const [write] = metadataWrites(database.mutations);
+ expect(write?.sql).toContain(
+ "IF(JSON_EXTRACT(?, '$.loomDeferredProcessing') IS NULL",
+ );
+ });
+
+ it("marks a deferred original complete once it has been processed", async () => {
+ const database = loomVideo();
+ const key = `${ownerId}/${videoId}/raw-upload.webm`;
+ database.video.metadata = {
+ loomDeferredProcessing: { rawFileKey: key, status: "started" },
+ };
+ database.setRawFileKey(key);
+ await request(complete(), "media-secret", "?retryable=true");
+ const [write] = metadataWrites(database.mutations);
+ expect(write?.sql).toContain(
+ "'$.loomDeferredProcessing.status', 'complete'",
+ );
+ expect(mocks.transcribe).toHaveBeenCalledWith(videoId);
+ });
+
it("keeps queueing transcription for media server completions", async () => {
const database = loomVideo();
await request(
diff --git a/apps/web/app/api/webhooks/media-server/progress/route.ts b/apps/web/app/api/webhooks/media-server/progress/route.ts
index 4c455ad68..bde5992d2 100644
--- a/apps/web/app/api/webhooks/media-server/progress/route.ts
+++ b/apps/web/app/api/webhooks/media-server/progress/route.ts
@@ -3,7 +3,7 @@ import { db } from "@cap/database";
import { videos, videoUploads } from "@cap/database/schema";
import { serverEnv } from "@cap/env";
import type { Video } from "@cap/web-domain";
-import { eq, sql } from "drizzle-orm";
+import { and, eq, isNull, ne, or, sql } from "drizzle-orm";
import { type NextRequest, NextResponse } from "next/server";
import { handleAudioLevelPublication } from "@/lib/audio-level-publication";
import { SourceCommitPendingError } from "@/lib/desktop-recording-jobs";
@@ -248,9 +248,14 @@ export async function POST(request: NextRequest) {
...(transcribeOnView
? [sql`'$.transcribeOnView', JSON_EXTRACT('true', '$')`]
: []),
+ ...(currentVideo?.metadata?.loomDeferredProcessing &&
+ currentUpload?.rawFileKey ===
+ currentVideo.metadata.loomDeferredProcessing.rawFileKey
+ ? [sql`'$.loomDeferredProcessing.status', 'complete'`]
+ : []),
...(deferredSourceKey
? [
- sql`'$.loomDeferredProcessing', JSON_OBJECT('rawFileKey', ${deferredSourceKey}, 'status', 'pending')`,
+ sql`'$.loomDeferredProcessing', IF(JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing') IS NULL, JSON_OBJECT('rawFileKey', ${deferredSourceKey}, 'status', 'pending'), JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing'))`,
]
: []),
];
@@ -291,7 +296,17 @@ export async function POST(request: NextRequest) {
} else {
await db()
.delete(videoUploads)
- .where(eq(videoUploads.videoId, payload.videoId as Video.VideoId));
+ .where(
+ and(
+ eq(videoUploads.videoId, payload.videoId as Video.VideoId),
+ deferredSourceKey
+ ? or(
+ isNull(videoUploads.rawFileKey),
+ ne(videoUploads.rawFileKey, deferredSourceKey),
+ )
+ : undefined,
+ ),
+ );
}
await invalidateGoogleDriveStorageQuotaCache(
currentVideo?.storageIntegrationId,
diff --git a/apps/web/app/embed/[videoId]/page.tsx b/apps/web/app/embed/[videoId]/page.tsx
index 73bb3cd29..8ff97bc13 100644
--- a/apps/web/app/embed/[videoId]/page.tsx
+++ b/apps/web/app/embed/[videoId]/page.tsx
@@ -27,6 +27,7 @@ import { Effect, Option } from "effect";
import type { Metadata } from "next";
import Link from "next/link";
import { notFound } from "next/navigation";
+import { after } from "next/server";
import {
getPendingLoomProcessing,
startDeferredLoomProcessing,
@@ -278,17 +279,20 @@ async function EmbedContent({
? null
: getPendingLoomProcessing(currentMetadata, video.ownerId, video.id);
if (deferredLoomRawFileKey) {
- startDeferredLoomProcessing({
- videoId: video.id,
- ownerId: video.ownerId,
- bucketId: video.bucket,
- rawFileKey: deferredLoomRawFileKey,
- }).catch((error) => {
- console.error(
- `[EmbedVideoPage] Error starting deferred processing for ${video.id}:`,
- error,
- );
- });
+ const rawFileKey = deferredLoomRawFileKey;
+ after(() =>
+ startDeferredLoomProcessing({
+ videoId: video.id,
+ ownerId: video.ownerId,
+ bucketId: video.bucket,
+ rawFileKey,
+ }).catch((error) => {
+ console.error(
+ `[EmbedVideoPage] Error starting deferred processing for ${video.id}:`,
+ error,
+ );
+ }),
+ );
}
let initialAiData = null;
diff --git a/apps/web/app/s/[videoId]/page.tsx b/apps/web/app/s/[videoId]/page.tsx
index 878eb1853..e8dbf48de 100644
--- a/apps/web/app/s/[videoId]/page.tsx
+++ b/apps/web/app/s/[videoId]/page.tsx
@@ -39,6 +39,7 @@ import type { Metadata } from "next";
import { headers } from "next/headers";
import Link from "next/link";
import { notFound } from "next/navigation";
+import { after } from "next/server";
import { getVideoAnalytics } from "@/actions/videos/get-analytics";
import {
getDashboardSpacesData,
@@ -881,17 +882,20 @@ async function AuthorizedContent({
? null
: getPendingLoomProcessing(metadata, video.owner.id, videoId);
if (deferredLoomRawFileKey) {
- startDeferredLoomProcessing({
- videoId,
- ownerId: video.owner.id,
- bucketId: video.bucket,
- rawFileKey: deferredLoomRawFileKey,
- }).catch((error) => {
- console.error(
- `[ShareVideoPage] Error starting deferred processing for ${videoId}:`,
- error,
- );
- });
+ const rawFileKey = deferredLoomRawFileKey;
+ after(() =>
+ startDeferredLoomProcessing({
+ videoId,
+ ownerId: video.owner.id,
+ bucketId: video.bucket,
+ rawFileKey,
+ }).catch((error) => {
+ console.error(
+ `[ShareVideoPage] Error starting deferred processing for ${videoId}:`,
+ error,
+ );
+ }),
+ );
}
const aiGenerationStatus = metadata.aiGenerationStatus || null;
diff --git a/apps/web/lib/loom-deferred-processing.ts b/apps/web/lib/loom-deferred-processing.ts
index e0dc3deed..63437dbef 100644
--- a/apps/web/lib/loom-deferred-processing.ts
+++ b/apps/web/lib/loom-deferred-processing.ts
@@ -5,6 +5,8 @@ import type { Video } from "@cap/web-domain";
import { and, eq, sql } from "drizzle-orm";
import { startVideoProcessingWorkflow } from "@/lib/video-processing";
+export const DEFERRED_PROCESSING_RECLAIM_MS = 30 * 60 * 1000;
+
function getAffectedRows(result: unknown) {
const header = Array.isArray(result) ? result[0] : result;
return (header as { affectedRows?: number } | undefined)?.affectedRows ?? 0;
@@ -14,59 +16,95 @@ export function getPendingLoomProcessing(
metadata: VideoMetadata | null | undefined,
ownerId: string,
videoId: string,
+ now = Date.now(),
) {
const deferred = metadata?.loomDeferredProcessing;
if (
- deferred?.status !== "pending" ||
+ !deferred ||
typeof deferred.rawFileKey !== "string" ||
!deferred.rawFileKey.startsWith(`${ownerId}/${videoId}/`)
)
return null;
- return deferred.rawFileKey;
+ if (deferred.status === "pending") return deferred.rawFileKey;
+ const startedAt = Date.parse(deferred.startedAt ?? "");
+ return deferred.status === "started" &&
+ Number.isFinite(startedAt) &&
+ now - startedAt > DEFERRED_PROCESSING_RECLAIM_MS
+ ? deferred.rawFileKey
+ : null;
}
-export async function startDeferredLoomProcessing({
- videoId,
- ownerId,
- bucketId,
- rawFileKey,
-}: {
- videoId: Video.VideoId;
- ownerId: string;
- bucketId: string | null;
- rawFileKey: string;
-}) {
+export async function startDeferredLoomProcessing(
+ {
+ videoId,
+ ownerId,
+ bucketId,
+ rawFileKey,
+ }: {
+ videoId: Video.VideoId;
+ ownerId: string;
+ bucketId: string | null;
+ rawFileKey: string;
+ },
+ now = new Date(),
+) {
+ const startedAt = now.toISOString();
+ const reclaimBefore = new Date(
+ now.getTime() - DEFERRED_PROCESSING_RECLAIM_MS,
+ ).toISOString();
const claim = await db()
.update(videos)
.set({
- metadata: sql`JSON_SET(${videos.metadata}, '$.loomDeferredProcessing.status', 'started', '$.loomDeferredProcessing.startedAt', ${new Date().toISOString()})`,
+ metadata: sql`JSON_SET(${videos.metadata}, '$.loomDeferredProcessing.status', 'started', '$.loomDeferredProcessing.startedAt', ${startedAt})`,
})
.where(
and(
eq(videos.id, videoId),
- sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing.status')) = 'pending'`,
sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing.rawFileKey')) = ${rawFileKey}`,
+ sql`(JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing.status')) = 'pending' OR (JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing.status')) = 'started' AND JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing.startedAt')) < ${reclaimBefore}))`,
),
);
if (getAffectedRows(claim) !== 1) return "already-started" as const;
- await db()
- .insert(videoUploads)
- .values({
+ try {
+ await db()
+ .insert(videoUploads)
+ .values({
+ videoId,
+ mode: "singlepart",
+ phase: "uploading",
+ rawFileKey,
+ processingMessage: "Optimizing imported video...",
+ })
+ .onDuplicateKeyUpdate({ set: { rawFileKey, phase: "uploading" } });
+ return await startVideoProcessingWorkflow({
videoId,
- mode: "singlepart",
- phase: "uploading",
+ userId: ownerId,
rawFileKey,
+ bucketId,
processingMessage: "Optimizing imported video...",
- })
- .onDuplicateKeyUpdate({ set: { rawFileKey } });
-
- return startVideoProcessingWorkflow({
- videoId,
- userId: ownerId,
- rawFileKey,
- bucketId,
- processingMessage: "Optimizing imported video...",
- startFailureMessage: "Imported video could not be optimized",
- });
+ startFailureMessage: "Imported video could not be optimized",
+ });
+ } catch (error) {
+ await db()
+ .delete(videoUploads)
+ .where(
+ and(
+ eq(videoUploads.videoId, videoId),
+ eq(videoUploads.rawFileKey, rawFileKey),
+ ),
+ );
+ await db()
+ .update(videos)
+ .set({
+ metadata: sql`JSON_SET(${videos.metadata}, '$.loomDeferredProcessing.status', 'pending')`,
+ })
+ .where(
+ and(
+ eq(videos.id, videoId),
+ sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.loomDeferredProcessing.startedAt')) = ${startedAt}`,
+ ),
+ );
+ throw error;
+ }
}
diff --git a/packages/database/types/metadata.ts b/packages/database/types/metadata.ts
index c0794bb9f..921c149a4 100644
--- a/packages/database/types/metadata.ts
+++ b/packages/database/types/metadata.ts
@@ -25,7 +25,7 @@ export interface VideoMetadata {
};
loomDeferredProcessing?: {
rawFileKey: string;
- status: "pending" | "started";
+ status: "pending" | "started" | "complete";
startedAt?: string;
};
transcribeOnView?: boolean;