Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(web): upload small desktop recording outputs with one PUT

Richie McIlroy929e1cf1721e
diff --git a/apps/web/__tests__/unit/finalize-desktop-recording.test.ts b/apps/web/__tests__/unit/finalize-desktop-recording.test.ts
index f5461d3d9..59b8178df 100644
--- a/apps/web/__tests__/unit/finalize-desktop-recording.test.ts
+++ b/apps/web/__tests__/unit/finalize-desktop-recording.test.ts
@@ -1,5 +1,5 @@
import type { User, Video } from "@cap/web-domain";
-import { Effect } from "effect";
+import { Effect, Option } from "effect";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type {
DesktopRecordingAttempt,
@@ -33,6 +33,8 @@ const mocks = vi.hoisted(() => ({
put: vi.fn(),
databaseRows: vi.fn(),
reserveBudget: vi.fn(),
+ storage: vi.fn(),
+ multipartCreate: vi.fn(),
}));
vi.mock("@/lib/media-processing-budget", () => ({
reserveMediaProcessingBudget: mocks.reserveBudget,
@@ -97,15 +99,7 @@ vi.mock("@cap/web-backend/src/Storage/index", async () => {
const { Effect } = await import("effect");
return {
Storage: {
- getAccessForVideo: () =>
- Effect.succeed([
- {
- provider: "test",
- getInternalSignedObjectUrl: (key: string) =>
- Effect.succeed(`https://storage.test/${key}`),
- getInternalPresignedPutUrl: mocks.put,
- },
- ]),
+ getAccessForVideo: () => Effect.sync(mocks.storage),
},
};
});
@@ -150,6 +144,19 @@ const fixture: DesktopRecordingAttempt = {
};
let current: DesktopRecordingJob | null;
+function storageAccess(provider: string, customBucket = false) {
+ return [
+ {
+ provider,
+ getInternalSignedObjectUrl: (key: string) =>
+ Effect.succeed(`https://storage.test/${key}`),
+ getInternalPresignedPutUrl: mocks.put,
+ multipart: { create: mocks.multipartCreate },
+ },
+ customBucket ? Option.some({ id: "bucket" }) : Option.none(),
+ ];
+}
+
function withCurrent(update: Partial<DesktopRecordingJob>) {
if (!current) throw new Error("Fixture job disappeared");
current = { ...current, ...update };
@@ -157,6 +164,10 @@ function withCurrent(update: Partial<DesktopRecordingJob>) {
beforeEach(() => {
mocks.reserveBudget.mockResolvedValue(100_000_000);
+ mocks.storage.mockImplementation(() => storageAccess("test"));
+ mocks.multipartCreate.mockImplementation(() =>
+ Effect.succeed({ UploadId: "upload" }),
+ );
mocks.databaseRows.mockResolvedValue([
{ id: "video", ownerId: "user", source: { type: "desktopSegments" } },
]);
@@ -891,6 +902,57 @@ describe("source commitment and media request compatibility", () => {
);
});
+ function withSourceBytes(size: number) {
+ mocks.sourceUrls.mockResolvedValue({
+ videoInitUrl: "https://source.test/init.mp4",
+ videoSegmentUrls: ["https://source.test/segment.m4s"],
+ sourceObjects: [
+ {
+ url: "https://source.test/segment.m4s",
+ objectIdentity: '"segment"',
+ size,
+ },
+ ],
+ });
+ }
+
+ it("uploads a small Cap storage output with one PUT instead of multipart", async () => {
+ mocks.storage.mockImplementation(() => storageAccess("s3"));
+ withSourceBytes(64 * 1024 * 1024);
+ await startDesktopRecordingJob(fixture);
+ const body = JSON.parse(mocks.fetch.mock.calls[0]?.[1]?.body ?? "{}");
+ expect(body.outputUpload).toEqual({
+ type: "put",
+ url: "https://storage.test/user/video/.recording/outputs/generation/attempt.mp4",
+ ifNoneMatch: "*",
+ });
+ expect(mocks.multipartCreate).not.toHaveBeenCalled();
+ });
+
+ it("keeps multipart output uploads for recordings larger than one part", async () => {
+ mocks.storage.mockImplementation(() => storageAccess("s3"));
+ withSourceBytes(64 * 1024 * 1024 + 1);
+ await startDesktopRecordingJob(fixture);
+ const body = JSON.parse(mocks.fetch.mock.calls[0]?.[1]?.body ?? "{}");
+ expect(body.outputUpload).toMatchObject({
+ type: "multipart",
+ uploadId: "upload",
+ partSize: 64 * 1024 * 1024,
+ signPartUrl:
+ "https://cap.test/api/webhooks/media-server/multipart/sign-part",
+ });
+ expect(mocks.multipartCreate).toHaveBeenCalledOnce();
+ });
+
+ it("keeps multipart output uploads for small recordings in a custom bucket", async () => {
+ mocks.storage.mockImplementation(() => storageAccess("s3", true));
+ withSourceBytes(1024);
+ await startDesktopRecordingJob(fixture);
+ const body = JSON.parse(mocks.fetch.mock.calls[0]?.[1]?.body ?? "{}");
+ expect(body.outputUpload).toMatchObject({ type: "multipart" });
+ expect(mocks.multipartCreate).toHaveBeenCalledOnce();
+ });
+
it("reuses a recorded remote job after a step replay instead of submitting another mux", async () => {
withCurrent({ remoteJobId: "existing-worker" });
expect(await startDesktopRecordingJob(fixture)).toBe("existing-worker");
diff --git a/apps/web/workflows/finalize-desktop-recording.ts b/apps/web/workflows/finalize-desktop-recording.ts
index 2caeb0482..1c2309883 100644
--- a/apps/web/workflows/finalize-desktop-recording.ts
+++ b/apps/web/workflows/finalize-desktop-recording.ts
@@ -4,6 +4,7 @@ import { serverEnv } from "@cap/env";
import { Storage } from "@cap/web-backend/src/Storage/index";
import { type User, Video } from "@cap/web-domain";
import { and, eq } from "drizzle-orm";
+import { Option } from "effect";
import { sleep } from "workflow";
import { z } from "zod";
import { isAiGenerationEnabledForUser } from "@/lib/ai-generation-entitlement";
@@ -83,6 +84,9 @@ const COMPLETION_POLL_INTERVAL_MS = 15_000;
const ATTEMPT_MAX_DURATION_MS = 6 * 60 * 60 * 1_000;
const PRESIGNED_EXPIRES_SECONDS = 6 * 60 * 60;
const MULTIPART_OUTPUT_PART_SIZE_BYTES = 64 * 1024 * 1024;
+// A remux is about as large as its source, so a source that fits in one part is
+// uploaded with one PUT instead of sign-part + PUT + complete round trips.
+const SINGLE_PUT_OUTPUT_MAX_SOURCE_BYTES = MULTIPART_OUTPUT_PART_SIZE_BYTES;
const SOURCE_COMMIT_BATCH_MAX_CHECKPOINTS = 32;
const SOURCE_COMMIT_BATCH_DURATION_MS = 15_000;
const TRANSCRIPTION_QUEUE_MAX_ATTEMPTS = 12;
@@ -359,11 +363,13 @@ async function completeWithoutMediaServerIfUnavailable(
async function buildDesktopSegmentsOutput({
video,
attempt,
+ sourceBytes,
}: {
video: typeof videos.$inferSelect;
attempt: DesktopRecordingAttemptFence;
+ sourceBytes: number;
}) {
- const [bucket] = await Storage.getAccessForVideo(
+ const [bucket, customBucket] = await Storage.getAccessForVideo(
decodeStorageVideo(video),
).pipe(runWorkflowPromise);
const outputKey = getDesktopRecordingOutputKey(
@@ -393,7 +399,11 @@ async function buildDesktopSegmentsOutput({
url: outputPresignedUrl,
ifNoneMatch: "*",
};
- if (bucket.provider === "s3" && webhookSecret) {
+ const singlePut =
+ bucket.provider === "s3" &&
+ Option.isNone(customBucket) &&
+ sourceBytes <= SINGLE_PUT_OUTPUT_MAX_SOURCE_BYTES;
+ if (bucket.provider === "s3" && webhookSecret && !singlePut) {
const multipart = await bucket.multipart
.create(outputKey, { ContentType: "video/mp4" })
.pipe(runWorkflowPromise);
@@ -500,16 +510,17 @@ export async function startDesktopRecordingJob(
}
throw error;
});
+ const sourceBytes = urls.sourceObjects.reduce(
+ (total, object) => total + object.size,
+ 0,
+ );
let downloadBudgetBytes: number;
try {
downloadBudgetBytes = await reserveMediaProcessingBudget({
videoId: current.videoId,
generation: current.generation,
attemptId: attempt.attemptId,
- sourceBytes: urls.sourceObjects.reduce(
- (total, object) => total + object.size,
- 0,
- ),
+ sourceBytes,
});
} catch (error) {
if (error instanceof MediaProcessingBudgetError) {
@@ -577,7 +588,7 @@ export async function startDesktopRecordingJob(
audioLevels: true,
...context,
...urls,
- ...(await buildDesktopSegmentsOutput({ video, attempt })),
+ ...(await buildDesktopSegmentsOutput({ video, attempt, sourceBytes })),
manifestSha256: current.source.manifestSha256,
};
}