Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(web): have the media server decode prepared recording fragments while recording
Richie McIlroy9bce8a068ee6
diff --git a/apps/web/__tests__/unit/desktop-recording-source.test.ts b/apps/web/__tests__/unit/desktop-recording-source.test.tsindex aed466c33..ba8b59918 100644--- a/apps/web/__tests__/unit/desktop-recording-source.test.ts+++ b/apps/web/__tests__/unit/desktop-recording-source.test.ts@@ -8,6 +8,7 @@ import type { DesktopRecordingSourceCheckpoint } from "@/lib/desktop-recording-simport type { RecordingVerification } from "@/lib/desktop-recording-verification";const mocks = vi.hoisted(() => ({+ env: vi.fn((): Record<string, string | undefined> => ({})),storage: vi.fn(),pageSize: undefined as number | undefined,treePageSize: undefined as number | undefined,@@ -16,6 +17,7 @@ const mocks = vi.hoisted(() => ({vi.mock("@cap/web-backend/src/Storage/index", () => ({Storage: { getAccessForVideo: mocks.storage },}));+vi.mock("@cap/env", () => ({ serverEnv: mocks.env }));vi.mock("@/lib/desktop-recording-source-checkpoint", async (importOriginal) => {const actual =await importOriginal<@@ -45,6 +47,7 @@ vi.mock("@/lib/video-storage", () => ({import {advanceDesktopRecordingSourceCommit,buildDesktopRecordingSourceUrls,+ buildPreverifiedRunUrls,commitDesktopRecordingSource,DesktopRecordingSourceError,getDesktopRecordingOutputKey,@@ -266,6 +269,9 @@ function storageFixture(getInternalSignedObjectUrl: vi.fn((key: string) =>checked(() => `https://storage.example/${key}?signed=1`),),+ getInternalPresignedPutUrl: vi.fn((key: string) =>+ checked(() => `https://storage.example/${key}?put=1`),+ ),multipart: {create: vi.fn((key: string, options?: { Metadata?: Record<string, string> }) =>@@ -486,6 +492,8 @@ function seedSegments(beforeEach(() => {vi.clearAllMocks();+ vi.unstubAllGlobals();+ mocks.env.mockReturnValue({});mocks.pageSize = undefined;mocks.treePageSize = undefined;});@@ -2847,3 +2855,143 @@ describe("listed source commit on Cap's bucket", () => {).toHaveLength(3);});});++describe("decoding prepared fragments during the recording", () => {+ const mediaEnv = {+ MEDIA_SERVER_URL: "https://media.test/",+ MEDIA_SERVER_WEBHOOK_SECRET: "media-secret",+ };++ function stubMediaServer() {+ const fetch = vi.fn(+ async (_url: string | URL, _init?: RequestInit) =>+ new Response(null, { status: 202 }),+ );+ vi.stubGlobal("fetch", fetch);+ return fetch;+ }++ function requests(fetch: ReturnType<typeof stubMediaServer>) {+ return fetch.mock.calls.map(([url, init]) => ({+ url: String(url),+ secret: (init as RequestInit).headers as Record<string, string>,+ body: JSON.parse(String((init as RequestInit).body)) as {+ videoId: string;+ init: { url: string; objectIdentity: string; size: number };+ segments: { url: string; objectIdentity: string; size: number }[];+ resultUploadUrl: string;+ },+ }));+ }++ it("asks the media server to decode each contiguous run of prepared video", async () => {+ mocks.env.mockReturnValue(mediaEnv);+ const fetch = stubMediaServer();+ const storage = listedStorage();+ seedRecording(storage, 8, 3);+ const prepared = await prepareDesktopRecordingSegments(+ recording(),+ [1, 2, 3, 5, 6].map((index) => ({ track: "video" as const, index })),+ async () => true,+ );+ expect(prepared).toHaveLength(5);+ const sent = requests(fetch);+ expect(sent).toHaveLength(2);+ expect(+ sent.every(+ (request) =>+ request.url === "https://media.test/video/preverify-segments",+ ),+ ).toBe(true);+ expect(+ sent.every(+ (request) => request.secret["x-media-server-secret"] === "media-secret",+ ),+ ).toBe(true);+ const [first, second] = sent+ .map((request) => request.body)+ .sort((left, right) => left.segments.length - right.segments.length)+ .reverse();+ expect(first?.segments).toHaveLength(3);+ expect(second?.segments).toHaveLength(2);+ expect(first?.init).toEqual({+ url: expect.stringContaining("/segments/video/init.mp4"),+ objectIdentity: storage.object(videoInitKey).identity,+ size: storage.object(videoInitKey).size,+ });+ for (const segment of first?.segments ?? []) {+ expect(segment.url).toContain("/.recording/sources/prepared/");+ }+ expect(first?.segments.map((segment) => segment.objectIdentity)).toEqual(+ [1, 2, 3].map(+ (index) =>+ storage.object(+ `${prefix}/segments/video/segment_${String(index).padStart(3, "0")}.m4s`,+ ).identity,+ ),+ );+ expect(first?.resultUploadUrl).toMatch(+ /\/\.recording\/sources\/preverified\/000001-000003-[a-f0-9]{16}\.json\?put=1$/,+ );+ });++ it.each([+ [+ "a custom bucket",+ () =>+ storageFixture("s3", true, { listing: true, copiesKeepIdentity: true }),+ mediaEnv,+ ],+ ["Google Drive", () => storageFixture("googleDrive"), mediaEnv],+ ["no media server", () => listedStorage(), {}],+ ] as const)("does not ask for %s", async (_, makeStorage, env) => {+ mocks.env.mockReturnValue(env);+ const fetch = stubMediaServer();+ const storage = makeStorage();+ seedRecording(storage, 4, 0);+ const prepared = await prepareDesktopRecordingSegments(+ recording(),+ [1, 2].map((index) => ({ track: "video" as const, index })),+ async () => true,+ );+ expect(prepared).toHaveLength(2);+ expect(fetch).not.toHaveBeenCalled();+ });++ it("still reports prepared fragments when the media server cannot be reached", async () => {+ mocks.env.mockReturnValue(mediaEnv);+ vi.stubGlobal(+ "fetch",+ vi.fn(async () => {+ throw new Error("offline");+ }),+ );+ const storage = listedStorage();+ seedRecording(storage, 4, 0);+ const prepared = await prepareDesktopRecordingSegments(+ recording(),+ [1, 2].map((index) => ({ track: "video" as const, index })),+ async () => true,+ );+ expect(prepared).toHaveLength(2);+ });++ it("signs every saved run for the final verification on Cap's bucket only", async () => {+ const storage = listedStorage();+ storage.seed(+ `${prefix}/.recording/sources/preverified/000001-000003-a.json`,+ "{}",+ );+ storage.seed(+ `${prefix}/.recording/sources/preverified/000004-000006-b.json`,+ "{}",+ );+ storage.seed(`${prefix}/.recording/sources/preverified/partial.tmp`, "x");+ expect(await buildPreverifiedRunUrls(recording())).toEqual([+ `https://storage.example/${prefix}/.recording/sources/preverified/000001-000003-a.json?signed=1`,+ `https://storage.example/${prefix}/.recording/sources/preverified/000004-000006-b.json?signed=1`,+ ]);+ storageFixture("s3", true, { listing: true, copiesKeepIdentity: true });+ expect(await buildPreverifiedRunUrls(recording())).toEqual([]);+ });+});diff --git a/apps/web/__tests__/unit/finalize-desktop-recording.test.ts b/apps/web/__tests__/unit/finalize-desktop-recording.test.tsindex 4b203feb0..6fbcaae03 100644--- a/apps/web/__tests__/unit/finalize-desktop-recording.test.ts+++ b/apps/web/__tests__/unit/finalize-desktop-recording.test.ts@@ -25,6 +25,7 @@ const mocks = vi.hoisted(() => ({checkpoint: vi.fn(),saveCheckpoint: vi.fn(),sourceUrls: vi.fn(),+ preverifiedRunUrls: vi.fn(async (): Promise<string[]> => []),observe: vi.fn(),sleep: vi.fn(),env: vi.fn(),@@ -79,6 +80,7 @@ vi.mock("@/lib/desktop-recording-jobs", () => ({vi.mock("@/lib/desktop-recording-source", () => ({advanceDesktopRecordingSourceCommit: mocks.commitSource,buildDesktopRecordingSourceUrls: mocks.sourceUrls,+ buildPreverifiedRunUrls: mocks.preverifiedRunUrls,getDesktopRecordingOutputKey: (owner: string,video: string,@@ -171,6 +173,7 @@ function withCurrent(update: Partial<DesktopRecordingJob>) {beforeEach(() => {mocks.stepMetadata.mockReturnValue({ stepId: "step_test", attempt: 4 });+ mocks.preverifiedRunUrls.mockResolvedValue([]);mocks.reserveBudget.mockResolvedValue(100_000_000);mocks.storage.mockImplementation(() => storageAccess("test"));mocks.multipartCreate.mockImplementation(() =>@@ -1064,6 +1067,22 @@ describe("source commitment and media request compatibility", () => {});}+ it("passes runs decoded during the recording to the media server only when there are any", async () => {+ await startDesktopRecordingJob(fixture);+ const plain = JSON.parse(String(mocks.fetch.mock.calls[0]?.[1]?.body));+ expect(plain).not.toHaveProperty("preverifiedRunUrls");+ mocks.fetch.mockClear();+ withCurrent({ remoteJobId: null });+ mocks.preverifiedRunUrls.mockResolvedValue([+ "https://storage.test/run-1.json",+ ]);+ await startDesktopRecordingJob(fixture);+ expect(+ JSON.parse(String(mocks.fetch.mock.calls[0]?.[1]?.body))+ .preverifiedRunUrls,+ ).toEqual(["https://storage.test/run-1.json"]);+ });+it("uploads a small Cap storage output with one PUT instead of multipart", async () => {mocks.storage.mockImplementation(() => storageAccess("s3"));withSourceBytes(64 * 1024 * 1024);diff --git a/apps/web/lib/desktop-recording-source.ts b/apps/web/lib/desktop-recording-source.tsindex 41f425168..dbb70c3e0 100644--- a/apps/web/lib/desktop-recording-source.ts+++ b/apps/web/lib/desktop-recording-source.ts@@ -1,5 +1,6 @@import { createHash, randomUUID } from "node:crypto";import type { videos } from "@cap/database/schema";+import { serverEnv } from "@cap/env";import { Storage } from "@cap/web-backend/src/Storage/index";import { getRecordingObjectIdentity } from "@cap/web-backend/src/Storage/recording-object-identity";import { getPublishedRecordingOutputKey } from "@cap/web-backend/src/Storage/recording-output";@@ -1209,7 +1210,7 @@ export async function prepareDesktopRecordingSegments(operation.pipe(Effect.timeout(Math.min(5_000, remaining))),);};- const [bucket] = await run(+ const [bucket, customBucket] = await run(Storage.getAccessForVideo(decodeStorageVideo(video), {resolvePublishedOutput: false,}),@@ -1222,6 +1223,7 @@ export async function prepareDesktopRecordingSegments(concurrency: SOURCE_COMMIT_CONCURRENCY,pageSize: SOURCE_COMMIT_PAGE_SIZE,};+ const videoCopies: SourceObject[] = [];const unique = [...new Map(requested.map((entry) => [`${entry.track}/${entry.index}`, entry]),@@ -1243,11 +1245,12 @@ export async function prepareDesktopRecordingSegments(...segment,});if (original.size > 64 * 1024 * 1024) return null;- await copySmallObject(+ const copy = await copySmallObject({ ...context, prefix: preparedSourcePrefix(video, original) },original,original.index,);+ if (segment.track === "video") videoCopies.push(copy);return segment;} catch {return null;@@ -1266,9 +1269,152 @@ export async function prepareDesktopRecordingSegments(markerSaved = true;}}+ if (+ bucket.provider === "s3" &&+ Option.isNone(customBucket) &&+ videoCopies.length > 0+ )+ await requestSegmentPreverification(context, videoCopies).catch(+ (error: unknown) => {+ console.warn(+ "[prepareDesktopRecordingSegments] Preverification skipped",+ {+ videoId: video.id,+ error: error instanceof Error ? error.message : String(error),+ },+ );+ },+ );return prepared;}+function preverifiedRunPrefix(video: DbVideo) {+ return `${sourcePrefix(video)}preverified/`;+}++// Asks the media server to decode the video fragments just secured, so the final verification only decodes+// the tail. Each contiguous run becomes one signed result; nothing depends on the request succeeding.+async function requestSegmentPreverification(+ context: SourceContext,+ copies: SourceObject[],+) {+ const env = serverEnv();+ if (!env.MEDIA_SERVER_URL || !env.MEDIA_SERVER_WEBHOOK_SECRET) return;+ const secret = env.MEDIA_SERVER_WEBHOOK_SECRET;+ const { video, bucket } = context;+ const initKey = `${video.ownerId}/${video.id}/segments/video/init.mp4`;+ const init = await captureOriginal(context, {+ originalKey: initKey,+ track: "video",+ index: 0,+ });+ const sign = (key: string) =>+ context.run(bucket.getInternalSignedObjectUrl(key, { expiresIn: 3_600 }));+ const initUrl = await sign(initKey);+ const sorted = [...copies].sort((left, right) => left.index - right.index);+ const runs: SourceObject[][] = [];+ for (const copy of sorted) {+ const current = runs.at(-1);+ if (current && current.at(-1)?.index === copy.index - 1) current.push(copy);+ else runs.push([copy]);+ }+ const mediaServerUrl = env.MEDIA_SERVER_URL.replace(/\/$/, "");+ await Promise.all(+ runs.map(async (run) => {+ const first = run[0];+ const last = run.at(-1);+ if (!first || !last) return;+ const digest = hash(+ JSON.stringify([+ init.originalIdentity,+ ...run.map((copy) => copy.objectIdentity),+ ]),+ ).slice(0, 16);+ const resultKey = `${preverifiedRunPrefix(video)}${String(first.index).padStart(6, "0")}-${String(last.index).padStart(6, "0")}-${digest}.json`;+ const [segments, resultUploadUrl] = await Promise.all([+ Promise.all(+ run.map(async (copy) => ({+ url: await sign(copy.key),+ objectIdentity: copy.objectIdentity,+ size: copy.size,+ })),+ ),+ context.run(+ bucket.getInternalPresignedPutUrl(+ resultKey,+ { ContentType: "application/json" },+ { expiresIn: 3_600 },+ ),+ ),+ ]);+ const response = await fetch(+ `${mediaServerUrl}/video/preverify-segments`,+ {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ "x-media-server-secret": secret,+ },+ body: JSON.stringify({+ videoId: video.id,+ init: {+ url: initUrl,+ objectIdentity: init.originalIdentity,+ size: init.size,+ },+ segments,+ resultUploadUrl,+ }),+ signal: AbortSignal.timeout(3_000),+ },+ );+ await response.body?.cancel().catch(() => {});+ }),+ );+}++// Signed URLs for every run decoded during the recording; the media server checks each run's signature+// and fragment identities itself, so an unusable run only costs a decode it would have done anyway.+export async function buildPreverifiedRunUrls(+ video: DbVideo,+): Promise<string[]> {+ try {+ const access = await runWorkflowPromise(+ Storage.getAccessForVideo(decodeStorageVideo(video), {+ resolvePublishedOutput: false,+ }),+ );+ const [bucket, customBucket] = access;+ if (bucket.provider !== "s3" || Option.isSome(customBucket)) return [];+ const listed = await listObjectsUnder(+ {+ video,+ bucket,+ run: (operation) =>+ runWorkflowPromise(operation.pipe(Effect.timeout(10_000))),+ prefix: sourcePrefix(video).slice(0, -1),+ ...sourceLimits(access),+ },+ preverifiedRunPrefix(video),+ );+ if (!listed) return [];+ const keys = [...listed.keys()]+ .filter((key) => key.endsWith(".json"))+ .sort()+ .slice(0, 4_096);+ return await mapBounded(+ keys,+ (key) =>+ runWorkflowPromise(+ bucket.getInternalSignedObjectUrl(key, { expiresIn: 6 * 60 * 60 }),+ ),+ 64,+ );+ } catch {+ return [];+ }+}+async function advanceMultipartCopy(context: SourceContext,checkpoint: DesktopRecordingSourceCheckpoint,diff --git a/apps/web/workflows/finalize-desktop-recording.ts b/apps/web/workflows/finalize-desktop-recording.tsindex f67a6ca84..a9d575a8e 100644--- a/apps/web/workflows/finalize-desktop-recording.ts+++ b/apps/web/workflows/finalize-desktop-recording.ts@@ -29,6 +29,7 @@ import {import {advanceDesktopRecordingSourceCommit,buildDesktopRecordingSourceUrls,+ buildPreverifiedRunUrls,getDesktopRecordingOutputKey,} from "@/lib/desktop-recording-source";import type { RecordingVerification } from "@/lib/desktop-recording-verification";@@ -742,12 +743,17 @@ async function dispatchDesktopRecordingJob();}path = "/video/mux-segments";+ const [output, preverifiedRunUrls] = await Promise.all([+ buildDesktopSegmentsOutput({ video, attempt, sourceBytes }),+ buildPreverifiedRunUrls(video),+ ]);body = {audioLevels: true,...context,...urls,- ...(await buildDesktopSegmentsOutput({ video, attempt, sourceBytes })),+ ...output,manifestSha256: current.source.manifestSha256,+ ...(preverifiedRunUrls.length > 0 ? { preverifiedRunUrls } : {}),};}try {