Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(web): commit Google Drive recording sources 32 at a time in 64-entry pages without reading receipts back

Richie McIlroy044d7bf19c29
diff --git a/apps/web/__tests__/unit/desktop-recording-source.test.ts b/apps/web/__tests__/unit/desktop-recording-source.test.ts
index 89b49ea3b..774b5d7b7 100644
--- a/apps/web/__tests__/unit/desktop-recording-source.test.ts
+++ b/apps/web/__tests__/unit/desktop-recording-source.test.ts
@@ -1287,15 +1287,20 @@ describe("durable desktop recording source commit", () => {
).toBe(false);
});
- it("still reads Drive receipts back after writing them", async () => {
+ it("writes Drive receipts without reading them back", async () => {
const storage = storageFixture("googleDrive");
seedSegments(storage);
await commitDesktopRecordingSource(recording(), generation);
expect(
- storage.bucket.getObject.mock.calls.some(([key]) =>
+ storage.bucket.putObject.mock.calls.some(([key]) =>
key.endsWith("/receipt.json"),
),
).toBe(true);
+ expect(
+ storage.bucket.getObject.mock.calls.some(([key]) =>
+ key.endsWith("/receipt.json"),
+ ),
+ ).toBe(false);
});
});
@@ -1427,7 +1432,7 @@ describe("large desktop recording snapshots", () => {
it.each([
["s3", 256],
- ["googleDrive", 16],
+ ["googleDrive", 64],
] as const)(
"resumes a completed %s copy when the database checkpoint write was lost",
async (provider, page) => {
@@ -1872,11 +1877,11 @@ describe("source commit page size", () => {
);
it.each([
- ["a custom S3-compatible bucket", "s3", true],
- ["Google Drive", "googleDrive", false],
+ ["a custom S3-compatible bucket", "s3", true, 16],
+ ["Google Drive", "googleDrive", false, 64],
] as const)(
- "keeps 16-entry checkpoints for %s",
- async (_, provider, customBucket) => {
+ "uses %s's checkpoint page size",
+ async (_, provider, customBucket, pageSize) => {
pagedFixture(provider, customBucket);
mocks.pageSize = undefined;
let checkpoint = snapshotCheckpoint();
@@ -1890,7 +1895,9 @@ describe("source commit page size", () => {
if ("source" in result) break;
checkpoint = result.checkpoint;
}
- expect(checkpoints).toBe(2 + 3 * Math.ceil((segmentCount + 3) / 16));
+ expect(checkpoints).toBe(
+ 2 + 3 * Math.ceil((segmentCount + 3) / pageSize),
+ );
},
);
@@ -1953,7 +1960,7 @@ describe("source commit page size", () => {
it.each([
["Cap's AWS bucket", "s3", false, 256],
["a custom S3-compatible bucket", "s3", true, 8],
- ["Google Drive", "googleDrive", false, 8],
+ ["Google Drive", "googleDrive", false, 32],
] as const)(
"bounds parallel storage requests for %s",
async (_, provider, customBucket, expected) => {
diff --git a/apps/web/lib/desktop-recording-source-checkpoint.ts b/apps/web/lib/desktop-recording-source-checkpoint.ts
index a91369213..67425864b 100644
--- a/apps/web/lib/desktop-recording-source-checkpoint.ts
+++ b/apps/web/lib/desktop-recording-source-checkpoint.ts
@@ -3,6 +3,7 @@ import { z } from "zod";
export const SOURCE_COMMIT_PAGE_SIZE = 16;
// Cap's own AWS bucket answers fast enough to finish 16x larger pages well inside a step's deadline.
export const SOURCE_COMMIT_AWS_PAGE_SIZE = 256;
+export const SOURCE_COMMIT_GOOGLE_DRIVE_PAGE_SIZE = 64;
export const SOURCE_COMMIT_PART_BATCH_SIZE = 4;
export const SOURCE_COMMIT_PART_PAGE_SIZE = 64;
// Readers accept larger tree pages than writers produce, so a deployment can be rolled back while commits written by a newer one are in flight.
diff --git a/apps/web/lib/desktop-recording-source.ts b/apps/web/lib/desktop-recording-source.ts
index b9bebc3bc..7535f387b 100644
--- a/apps/web/lib/desktop-recording-source.ts
+++ b/apps/web/lib/desktop-recording-source.ts
@@ -10,6 +10,7 @@ import {
type DesktopRecordingSourceCheckpoint,
desktopRecordingSourceCheckpointSchema,
SOURCE_COMMIT_AWS_PAGE_SIZE,
+ SOURCE_COMMIT_GOOGLE_DRIVE_PAGE_SIZE,
SOURCE_COMMIT_MAX_OBJECTS,
SOURCE_COMMIT_MAX_TREE_PAGE_ENTRIES,
SOURCE_COMMIT_PAGE_SIZE,
@@ -267,17 +268,24 @@ const SOURCE_COMMIT_STEP_TIMEOUT_MS = 180_000;
const SOURCE_COMMIT_IO_TIMEOUT_MS = 60_000;
const SOURCE_COMMIT_CONCURRENCY = 8;
const SOURCE_COMMIT_AWS_CONCURRENCY = 256;
+// About 100 Drive requests a second, half of Drive's 12,000-a-minute per-user quota; driveFetch backs off on rate limits.
+const SOURCE_COMMIT_GOOGLE_DRIVE_CONCURRENCY = 32;
function sourceLimits([bucket, customBucket]: RecordingStorageAccess) {
- return bucket.provider === "s3" && Option.isNone(customBucket)
- ? {
- concurrency: SOURCE_COMMIT_AWS_CONCURRENCY,
- pageSize: SOURCE_COMMIT_AWS_PAGE_SIZE,
- }
- : {
- concurrency: SOURCE_COMMIT_CONCURRENCY,
- pageSize: SOURCE_COMMIT_PAGE_SIZE,
- };
+ if (bucket.provider === "s3" && Option.isNone(customBucket))
+ return {
+ concurrency: SOURCE_COMMIT_AWS_CONCURRENCY,
+ pageSize: SOURCE_COMMIT_AWS_PAGE_SIZE,
+ };
+ if (bucket.provider === "googleDrive")
+ return {
+ concurrency: SOURCE_COMMIT_GOOGLE_DRIVE_CONCURRENCY,
+ pageSize: SOURCE_COMMIT_GOOGLE_DRIVE_PAGE_SIZE,
+ };
+ return {
+ concurrency: SOURCE_COMMIT_CONCURRENCY,
+ pageSize: SOURCE_COMMIT_PAGE_SIZE,
+ };
}
function invalidSource(message: string): never {
@@ -791,9 +799,7 @@ async function saveObjectReceipt(context: SourceContext, object: SourceObject) {
const key = `${objectDirectory(object.key)}/receipt.json`;
const content = JSON.stringify(object);
// A receipt only lets a retry find this copy, and a retry re-checks the receipt and the copy
- // before reusing it, so an S3 receipt skips the readback that inventory pages need.
- if (context.bucket.provider !== "s3")
- return writeSourceText(context, key, content).then(() => object);
+ // before reusing it, so a receipt skips the readback that inventory pages need.
assertContextKey(context, key);
await context.run(
context.bucket.putObject(key, content, { contentType: "application/json" }),