Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(web): commit Google Drive recording sources 32 at a time in 64-entry pages without reading receipts back
Richie McIlroy044d7bf19c29
diff --git a/apps/web/__tests__/unit/desktop-recording-source.test.ts b/apps/web/__tests__/unit/desktop-recording-source.test.tsindex 89b49ea3b..774b5d7b7 100644--- a/apps/web/__tests__/unit/desktop-recording-source.test.ts+++ b/apps/web/__tests__/unit/desktop-recording-source.test.ts@@ -1287,15 +1287,20 @@ describe("durable desktop recording source commit", () => {).toBe(false);});- it("still reads Drive receipts back after writing them", async () => {+ it("writes Drive receipts without reading them back", async () => {const storage = storageFixture("googleDrive");seedSegments(storage);await commitDesktopRecordingSource(recording(), generation);expect(- storage.bucket.getObject.mock.calls.some(([key]) =>+ storage.bucket.putObject.mock.calls.some(([key]) =>key.endsWith("/receipt.json"),),).toBe(true);+ expect(+ storage.bucket.getObject.mock.calls.some(([key]) =>+ key.endsWith("/receipt.json"),+ ),+ ).toBe(false);});});@@ -1427,7 +1432,7 @@ describe("large desktop recording snapshots", () => {it.each([["s3", 256],- ["googleDrive", 16],+ ["googleDrive", 64],] as const)("resumes a completed %s copy when the database checkpoint write was lost",async (provider, page) => {@@ -1872,11 +1877,11 @@ describe("source commit page size", () => {);it.each([- ["a custom S3-compatible bucket", "s3", true],- ["Google Drive", "googleDrive", false],+ ["a custom S3-compatible bucket", "s3", true, 16],+ ["Google Drive", "googleDrive", false, 64],] as const)(- "keeps 16-entry checkpoints for %s",- async (_, provider, customBucket) => {+ "uses %s's checkpoint page size",+ async (_, provider, customBucket, pageSize) => {pagedFixture(provider, customBucket);mocks.pageSize = undefined;let checkpoint = snapshotCheckpoint();@@ -1890,7 +1895,9 @@ describe("source commit page size", () => {if ("source" in result) break;checkpoint = result.checkpoint;}- expect(checkpoints).toBe(2 + 3 * Math.ceil((segmentCount + 3) / 16));+ expect(checkpoints).toBe(+ 2 + 3 * Math.ceil((segmentCount + 3) / pageSize),+ );},);@@ -1953,7 +1960,7 @@ describe("source commit page size", () => {it.each([["Cap's AWS bucket", "s3", false, 256],["a custom S3-compatible bucket", "s3", true, 8],- ["Google Drive", "googleDrive", false, 8],+ ["Google Drive", "googleDrive", false, 32],] as const)("bounds parallel storage requests for %s",async (_, provider, customBucket, expected) => {diff --git a/apps/web/lib/desktop-recording-source-checkpoint.ts b/apps/web/lib/desktop-recording-source-checkpoint.tsindex a91369213..67425864b 100644--- a/apps/web/lib/desktop-recording-source-checkpoint.ts+++ b/apps/web/lib/desktop-recording-source-checkpoint.ts@@ -3,6 +3,7 @@ import { z } from "zod";export const SOURCE_COMMIT_PAGE_SIZE = 16;// Cap's own AWS bucket answers fast enough to finish 16x larger pages well inside a step's deadline.export const SOURCE_COMMIT_AWS_PAGE_SIZE = 256;+export const SOURCE_COMMIT_GOOGLE_DRIVE_PAGE_SIZE = 64;export const SOURCE_COMMIT_PART_BATCH_SIZE = 4;export const SOURCE_COMMIT_PART_PAGE_SIZE = 64;// Readers accept larger tree pages than writers produce, so a deployment can be rolled back while commits written by a newer one are in flight.diff --git a/apps/web/lib/desktop-recording-source.ts b/apps/web/lib/desktop-recording-source.tsindex b9bebc3bc..7535f387b 100644--- a/apps/web/lib/desktop-recording-source.ts+++ b/apps/web/lib/desktop-recording-source.ts@@ -10,6 +10,7 @@ import {type DesktopRecordingSourceCheckpoint,desktopRecordingSourceCheckpointSchema,SOURCE_COMMIT_AWS_PAGE_SIZE,+ SOURCE_COMMIT_GOOGLE_DRIVE_PAGE_SIZE,SOURCE_COMMIT_MAX_OBJECTS,SOURCE_COMMIT_MAX_TREE_PAGE_ENTRIES,SOURCE_COMMIT_PAGE_SIZE,@@ -267,17 +268,24 @@ const SOURCE_COMMIT_STEP_TIMEOUT_MS = 180_000;const SOURCE_COMMIT_IO_TIMEOUT_MS = 60_000;const SOURCE_COMMIT_CONCURRENCY = 8;const SOURCE_COMMIT_AWS_CONCURRENCY = 256;+// About 100 Drive requests a second, half of Drive's 12,000-a-minute per-user quota; driveFetch backs off on rate limits.+const SOURCE_COMMIT_GOOGLE_DRIVE_CONCURRENCY = 32;function sourceLimits([bucket, customBucket]: RecordingStorageAccess) {- return bucket.provider === "s3" && Option.isNone(customBucket)- ? {- concurrency: SOURCE_COMMIT_AWS_CONCURRENCY,- pageSize: SOURCE_COMMIT_AWS_PAGE_SIZE,- }- : {- concurrency: SOURCE_COMMIT_CONCURRENCY,- pageSize: SOURCE_COMMIT_PAGE_SIZE,- };+ if (bucket.provider === "s3" && Option.isNone(customBucket))+ return {+ concurrency: SOURCE_COMMIT_AWS_CONCURRENCY,+ pageSize: SOURCE_COMMIT_AWS_PAGE_SIZE,+ };+ if (bucket.provider === "googleDrive")+ return {+ concurrency: SOURCE_COMMIT_GOOGLE_DRIVE_CONCURRENCY,+ pageSize: SOURCE_COMMIT_GOOGLE_DRIVE_PAGE_SIZE,+ };+ return {+ concurrency: SOURCE_COMMIT_CONCURRENCY,+ pageSize: SOURCE_COMMIT_PAGE_SIZE,+ };}function invalidSource(message: string): never {@@ -791,9 +799,7 @@ async function saveObjectReceipt(context: SourceContext, object: SourceObject) {const key = `${objectDirectory(object.key)}/receipt.json`;const content = JSON.stringify(object);// A receipt only lets a retry find this copy, and a retry re-checks the receipt and the copy- // before reusing it, so an S3 receipt skips the readback that inventory pages need.- if (context.bucket.provider !== "s3")- return writeSourceText(context, key, content).then(() => object);+ // before reusing it, so a receipt skips the readback that inventory pages need.assertContextKey(context, key);await context.run(context.bucket.putObject(key, content, { contentType: "application/json" }),