Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(web): upload small desktop recording outputs with one PUT
Richie McIlroy929e1cf1721e
diff --git a/apps/web/__tests__/unit/finalize-desktop-recording.test.ts b/apps/web/__tests__/unit/finalize-desktop-recording.test.tsindex f5461d3d9..59b8178df 100644--- a/apps/web/__tests__/unit/finalize-desktop-recording.test.ts+++ b/apps/web/__tests__/unit/finalize-desktop-recording.test.ts@@ -1,5 +1,5 @@import type { User, Video } from "@cap/web-domain";-import { Effect } from "effect";+import { Effect, Option } from "effect";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import type {DesktopRecordingAttempt,@@ -33,6 +33,8 @@ const mocks = vi.hoisted(() => ({put: vi.fn(),databaseRows: vi.fn(),reserveBudget: vi.fn(),+ storage: vi.fn(),+ multipartCreate: vi.fn(),}));vi.mock("@/lib/media-processing-budget", () => ({reserveMediaProcessingBudget: mocks.reserveBudget,@@ -97,15 +99,7 @@ vi.mock("@cap/web-backend/src/Storage/index", async () => {const { Effect } = await import("effect");return {Storage: {- getAccessForVideo: () =>- Effect.succeed([- {- provider: "test",- getInternalSignedObjectUrl: (key: string) =>- Effect.succeed(`https://storage.test/${key}`),- getInternalPresignedPutUrl: mocks.put,- },- ]),+ getAccessForVideo: () => Effect.sync(mocks.storage),},};});@@ -150,6 +144,19 @@ const fixture: DesktopRecordingAttempt = {};let current: DesktopRecordingJob | null;+function storageAccess(provider: string, customBucket = false) {+ return [+ {+ provider,+ getInternalSignedObjectUrl: (key: string) =>+ Effect.succeed(`https://storage.test/${key}`),+ getInternalPresignedPutUrl: mocks.put,+ multipart: { create: mocks.multipartCreate },+ },+ customBucket ? Option.some({ id: "bucket" }) : Option.none(),+ ];+}+function withCurrent(update: Partial<DesktopRecordingJob>) {if (!current) throw new Error("Fixture job disappeared");current = { ...current, ...update };@@ -157,6 +164,10 @@ function withCurrent(update: Partial<DesktopRecordingJob>) {beforeEach(() => {mocks.reserveBudget.mockResolvedValue(100_000_000);+ mocks.storage.mockImplementation(() => storageAccess("test"));+ mocks.multipartCreate.mockImplementation(() =>+ Effect.succeed({ UploadId: "upload" }),+ );mocks.databaseRows.mockResolvedValue([{ id: "video", ownerId: "user", source: { type: "desktopSegments" } },]);@@ -891,6 +902,57 @@ describe("source commitment and media request compatibility", () => {);});+ function withSourceBytes(size: number) {+ mocks.sourceUrls.mockResolvedValue({+ videoInitUrl: "https://source.test/init.mp4",+ videoSegmentUrls: ["https://source.test/segment.m4s"],+ sourceObjects: [+ {+ url: "https://source.test/segment.m4s",+ objectIdentity: '"segment"',+ size,+ },+ ],+ });+ }++ it("uploads a small Cap storage output with one PUT instead of multipart", async () => {+ mocks.storage.mockImplementation(() => storageAccess("s3"));+ withSourceBytes(64 * 1024 * 1024);+ await startDesktopRecordingJob(fixture);+ const body = JSON.parse(mocks.fetch.mock.calls[0]?.[1]?.body ?? "{}");+ expect(body.outputUpload).toEqual({+ type: "put",+ url: "https://storage.test/user/video/.recording/outputs/generation/attempt.mp4",+ ifNoneMatch: "*",+ });+ expect(mocks.multipartCreate).not.toHaveBeenCalled();+ });++ it("keeps multipart output uploads for recordings larger than one part", async () => {+ mocks.storage.mockImplementation(() => storageAccess("s3"));+ withSourceBytes(64 * 1024 * 1024 + 1);+ await startDesktopRecordingJob(fixture);+ const body = JSON.parse(mocks.fetch.mock.calls[0]?.[1]?.body ?? "{}");+ expect(body.outputUpload).toMatchObject({+ type: "multipart",+ uploadId: "upload",+ partSize: 64 * 1024 * 1024,+ signPartUrl:+ "https://cap.test/api/webhooks/media-server/multipart/sign-part",+ });+ expect(mocks.multipartCreate).toHaveBeenCalledOnce();+ });++ it("keeps multipart output uploads for small recordings in a custom bucket", async () => {+ mocks.storage.mockImplementation(() => storageAccess("s3", true));+ withSourceBytes(1024);+ await startDesktopRecordingJob(fixture);+ const body = JSON.parse(mocks.fetch.mock.calls[0]?.[1]?.body ?? "{}");+ expect(body.outputUpload).toMatchObject({ type: "multipart" });+ expect(mocks.multipartCreate).toHaveBeenCalledOnce();+ });+it("reuses a recorded remote job after a step replay instead of submitting another mux", async () => {withCurrent({ remoteJobId: "existing-worker" });expect(await startDesktopRecordingJob(fixture)).toBe("existing-worker");diff --git a/apps/web/workflows/finalize-desktop-recording.ts b/apps/web/workflows/finalize-desktop-recording.tsindex 2caeb0482..1c2309883 100644--- a/apps/web/workflows/finalize-desktop-recording.ts+++ b/apps/web/workflows/finalize-desktop-recording.ts@@ -4,6 +4,7 @@ import { serverEnv } from "@cap/env";import { Storage } from "@cap/web-backend/src/Storage/index";import { type User, Video } from "@cap/web-domain";import { and, eq } from "drizzle-orm";+import { Option } from "effect";import { sleep } from "workflow";import { z } from "zod";import { isAiGenerationEnabledForUser } from "@/lib/ai-generation-entitlement";@@ -83,6 +84,9 @@ const COMPLETION_POLL_INTERVAL_MS = 15_000;const ATTEMPT_MAX_DURATION_MS = 6 * 60 * 60 * 1_000;const PRESIGNED_EXPIRES_SECONDS = 6 * 60 * 60;const MULTIPART_OUTPUT_PART_SIZE_BYTES = 64 * 1024 * 1024;+// A remux is about as large as its source, so a source that fits in one part is+// uploaded with one PUT instead of sign-part + PUT + complete round trips.+const SINGLE_PUT_OUTPUT_MAX_SOURCE_BYTES = MULTIPART_OUTPUT_PART_SIZE_BYTES;const SOURCE_COMMIT_BATCH_MAX_CHECKPOINTS = 32;const SOURCE_COMMIT_BATCH_DURATION_MS = 15_000;const TRANSCRIPTION_QUEUE_MAX_ATTEMPTS = 12;@@ -359,11 +363,13 @@ async function completeWithoutMediaServerIfUnavailable(async function buildDesktopSegmentsOutput({video,attempt,+ sourceBytes,}: {video: typeof videos.$inferSelect;attempt: DesktopRecordingAttemptFence;+ sourceBytes: number;}) {- const [bucket] = await Storage.getAccessForVideo(+ const [bucket, customBucket] = await Storage.getAccessForVideo(decodeStorageVideo(video),).pipe(runWorkflowPromise);const outputKey = getDesktopRecordingOutputKey(@@ -393,7 +399,11 @@ async function buildDesktopSegmentsOutput({url: outputPresignedUrl,ifNoneMatch: "*",};- if (bucket.provider === "s3" && webhookSecret) {+ const singlePut =+ bucket.provider === "s3" &&+ Option.isNone(customBucket) &&+ sourceBytes <= SINGLE_PUT_OUTPUT_MAX_SOURCE_BYTES;+ if (bucket.provider === "s3" && webhookSecret && !singlePut) {const multipart = await bucket.multipart.create(outputKey, { ContentType: "video/mp4" }).pipe(runWorkflowPromise);@@ -500,16 +510,17 @@ export async function startDesktopRecordingJob(}throw error;});+ const sourceBytes = urls.sourceObjects.reduce(+ (total, object) => total + object.size,+ 0,+ );let downloadBudgetBytes: number;try {downloadBudgetBytes = await reserveMediaProcessingBudget({videoId: current.videoId,generation: current.generation,attemptId: attempt.attemptId,- sourceBytes: urls.sourceObjects.reduce(- (total, object) => total + object.size,- 0,- ),+ sourceBytes,});} catch (error) {if (error instanceof MediaProcessingBudgetError) {@@ -577,7 +588,7 @@ export async function startDesktopRecordingJob(audioLevels: true,...context,...urls,- ...(await buildDesktopSegmentsOutput({ video, attempt })),+ ...(await buildDesktopSegmentsOutput({ video, attempt, sourceBytes })),manifestSha256: current.source.manifestSha256,};}