Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(web): commit desktop sources from Cap's bucket in 128-object pages
Richie McIlroycda64297662f
diff --git a/apps/web/__tests__/unit/desktop-recording-source.test.ts b/apps/web/__tests__/unit/desktop-recording-source.test.tsindex fcfef5a14..2172d261b 100644--- a/apps/web/__tests__/unit/desktop-recording-source.test.ts+++ b/apps/web/__tests__/unit/desktop-recording-source.test.ts@@ -29,8 +29,8 @@ vi.mock("@/lib/desktop-recording-source-checkpoint", async (importOriginal) => {get SOURCE_COMMIT_AWS_PAGE_SIZE() {return mocks.pageSize ?? actual.SOURCE_COMMIT_AWS_PAGE_SIZE;},- get SOURCE_COMMIT_PART_PAGE_SIZE() {- return mocks.treePageSize ?? actual.SOURCE_COMMIT_PART_PAGE_SIZE;+ get SOURCE_COMMIT_TREE_PAGE_SIZE() {+ return mocks.treePageSize ?? actual.SOURCE_COMMIT_TREE_PAGE_SIZE;},};});@@ -1399,7 +1399,7 @@ describe("large desktop recording snapshots", () => {});it.each([- ["s3", 64],+ ["s3", 128],["googleDrive", 16],] as const)("resumes a completed %s copy when the database checkpoint write was lost",@@ -1771,30 +1771,33 @@ describe("source commit page size", () => {};}- it("uses 64-entry pages for Cap's bucket and keeps 16 elsewhere, both within the leaf schema", async () => {+ it("uses 128-entry pages for Cap's bucket and keeps 16 elsewhere, both within the leaf schema", async () => {const actual = await vi.importActual<typeof import("@/lib/desktop-recording-source-checkpoint")>("@/lib/desktop-recording-source-checkpoint");- expect(actual.SOURCE_COMMIT_AWS_PAGE_SIZE).toBe(64);+ expect(actual.SOURCE_COMMIT_AWS_PAGE_SIZE).toBe(128);expect(actual.SOURCE_COMMIT_PAGE_SIZE).toBe(16);expect(actual.SOURCE_COMMIT_AWS_PAGE_SIZE).toBeLessThanOrEqual(- actual.SOURCE_COMMIT_PART_PAGE_SIZE,+ actual.SOURCE_COMMIT_TREE_PAGE_SIZE,+ );+ expect(actual.SOURCE_COMMIT_TREE_PAGE_SIZE).toBeLessThanOrEqual(+ actual.SOURCE_COMMIT_MAX_TREE_PAGE_ENTRIES,);});- it("resumes a commit saved with larger tree pages using today's 64-entry writer, as after a rollback", async () => {- const reference = await referenceOutput(64);+ it("resumes a commit saved with larger tree pages using today's 128-entry writer, as after a rollback", async () => {+ const reference = await referenceOutput(128);const storage = pagedFixture();- mocks.treePageSize = 128;+ mocks.treePageSize = 256;const partial = await advanceWithPageSize(- 128,+ 256,snapshotCheckpoint(),(checkpoint) => checkpoint.phase === "verify",);expect(partial.source).toBeUndefined();expect(partial.checkpoint.phase).toBe("verify");mocks.treePageSize = undefined;- const resumed = await advanceWithPageSize(64, partial.checkpoint);+ const resumed = await advanceWithPageSize(128, partial.checkpoint);if (!resumed.source) throw new Error("Resumed commit did not complete");expect(await snapshotOutput(storage, resumed.source)).toEqual(reference.output,@@ -1906,7 +1909,7 @@ describe("source commit page size", () => {}it.each([- ["Cap's AWS bucket", "s3", false, 32],+ ["Cap's AWS bucket", "s3", false, 128],["a custom S3-compatible bucket", "s3", true, 8],["Google Drive", "googleDrive", false, 8],] as const)(diff --git a/apps/web/__tests__/unit/s3-bucket-connections.test.ts b/apps/web/__tests__/unit/s3-bucket-connections.test.tsindex 501751e4b..df73c43bc 100644--- a/apps/web/__tests__/unit/s3-bucket-connections.test.ts+++ b/apps/web/__tests__/unit/s3-bucket-connections.test.ts@@ -118,7 +118,7 @@ describe("S3 connection reuse", () => {}),);expect(fixture.authorizations).toHaveLength(192);- expect(fixture.connections()).toBeLessThanOrEqual(50);+ expect(fixture.connections()).toBeLessThanOrEqual(128);} finally {await fixture.close();}@@ -227,7 +227,7 @@ describe("S3 connection reuse", () => {expect(pools.map((pool) => pool.httpsAgent.maxFreeSockets)).toEqual([8, 32,]);- expect(pools.map((pool) => pool.httpsAgent.maxSockets)).toEqual([50, 50]);+ expect(pools.map((pool) => pool.httpsAgent.maxSockets)).toEqual([50, 128]);});it("reuses custom bucket connections while using current credentials", async () => {diff --git a/apps/web/lib/desktop-recording-source-checkpoint.ts b/apps/web/lib/desktop-recording-source-checkpoint.tsindex bf6275007..ec741b9b2 100644--- a/apps/web/lib/desktop-recording-source-checkpoint.ts+++ b/apps/web/lib/desktop-recording-source-checkpoint.ts@@ -1,12 +1,13 @@import { z } from "zod";export const SOURCE_COMMIT_PAGE_SIZE = 16;-// Cap's own AWS bucket answers fast enough to finish 4x larger pages well inside a step's deadline.-export const SOURCE_COMMIT_AWS_PAGE_SIZE = 64;+// Cap's own AWS bucket answers fast enough to finish 8x larger pages well inside a step's deadline.+export const SOURCE_COMMIT_AWS_PAGE_SIZE = 128;export const SOURCE_COMMIT_PART_BATCH_SIZE = 4;export const SOURCE_COMMIT_PART_PAGE_SIZE = 64;// Readers accept larger tree pages than writers produce, so a deployment can be rolled back while commits written by a newer one are in flight.export const SOURCE_COMMIT_MAX_TREE_PAGE_ENTRIES = 256;+export const SOURCE_COMMIT_TREE_PAGE_SIZE = 128;export const SOURCE_COMMIT_MAX_OBJECTS = 100_002;const sha256 = z.string().regex(/^[a-f0-9]{64}$/);diff --git a/apps/web/lib/desktop-recording-source.ts b/apps/web/lib/desktop-recording-source.tsindex 37353de85..3492c7247 100644--- a/apps/web/lib/desktop-recording-source.ts+++ b/apps/web/lib/desktop-recording-source.ts@@ -15,6 +15,7 @@ import {SOURCE_COMMIT_PAGE_SIZE,SOURCE_COMMIT_PART_BATCH_SIZE,SOURCE_COMMIT_PART_PAGE_SIZE,+ SOURCE_COMMIT_TREE_PAGE_SIZE,type SourceCommitPageReference,type SourceCommitReference,sourceCommitPageReferenceSchema,@@ -265,7 +266,7 @@ const committedInventorySchema = z.union([const SOURCE_COMMIT_STEP_TIMEOUT_MS = 180_000;const SOURCE_COMMIT_IO_TIMEOUT_MS = 60_000;const SOURCE_COMMIT_CONCURRENCY = 8;-const SOURCE_COMMIT_AWS_CONCURRENCY = 32;+const SOURCE_COMMIT_AWS_CONCURRENCY = 128;function sourceLimits([bucket, customBucket]: RecordingStorageAccess) {return bucket.provider === "s3" && Option.isNone(customBucket)@@ -390,7 +391,7 @@ async function appendTree(scope: string,entries: unknown[],) {- if (entries.length === 0 || entries.length > SOURCE_COMMIT_PART_PAGE_SIZE)+ if (entries.length === 0 || entries.length > SOURCE_COMMIT_TREE_PAGE_SIZE)invalidSource("Invalid source page size");const start = treeCount(roots);const leaf = await writeReference(context, {diff --git a/packages/web-backend/src/S3Buckets/S3ConnectionPool.ts b/packages/web-backend/src/S3Buckets/S3ConnectionPool.tsindex af3ff28eb..d63722a9b 100644--- a/packages/web-backend/src/S3Buckets/S3ConnectionPool.ts+++ b/packages/web-backend/src/S3Buckets/S3ConnectionPool.ts@@ -20,13 +20,13 @@ function boundIdleConnections<T extends HttpAgent>(agent: T) {return agent;}-const makeS3ConnectionPool = (maxFreeSockets: number) =>+const makeS3ConnectionPool = (maxFreeSockets: number, maxSockets = 50) =>Effect.acquireRelease(Effect.sync(() => {const options = {keepAlive: true,- maxSockets: 50,- maxTotalSockets: 128,+ maxSockets,+ maxTotalSockets: Math.max(128, maxSockets * 2),maxFreeSockets,};return {@@ -42,4 +42,5 @@ const makeS3ConnectionPool = (maxFreeSockets: number) =>);export const s3ConnectionPool = makeS3ConnectionPool(8);-export const defaultBucketS3ConnectionPool = makeS3ConnectionPool(32);+// Desktop source commits copy 128 objects at once in Cap's own bucket.+export const defaultBucketS3ConnectionPool = makeS3ConnectionPool(32, 128);