Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(web): keep 16-entry source pages for custom buckets and Drive
Richie McIlroy841aae72ecd5
diff --git a/apps/web/__tests__/unit/desktop-recording-source.test.ts b/apps/web/__tests__/unit/desktop-recording-source.test.tsindex fba603ce6..5db42c507 100644--- a/apps/web/__tests__/unit/desktop-recording-source.test.ts+++ b/apps/web/__tests__/unit/desktop-recording-source.test.ts@@ -25,6 +25,9 @@ vi.mock("@/lib/desktop-recording-source-checkpoint", async (importOriginal) => {get SOURCE_COMMIT_PAGE_SIZE() {return mocks.pageSize ?? actual.SOURCE_COMMIT_PAGE_SIZE;},+ get SOURCE_COMMIT_AWS_PAGE_SIZE() {+ return mocks.pageSize ?? actual.SOURCE_COMMIT_AWS_PAGE_SIZE;+ },};});vi.mock("@/lib/workflow-runtime", async () => {@@ -1390,9 +1393,12 @@ describe("large desktop recording snapshots", () => {).toBe(8 * 1024 ** 3);});- it.each(["s3", "googleDrive"])(+ it.each([+ ["s3", 64],+ ["googleDrive", 16],+ ] as const)("resumes a completed %s copy when the database checkpoint write was lost",- async (provider) => {+ async (provider, page) => {const storage = storageFixture(provider);seedSegments(storage,@@ -1410,14 +1416,14 @@ describe("large desktop recording snapshots", () => {checkpoint,);if (!("checkpoint" in copied)) throw new Error("Unexpected completion");- expect(copied.checkpoint.cursor).toBe(64);- expect(storage.bucket.copyObject).toHaveBeenCalledTimes(64);+ expect(copied.checkpoint.cursor).toBe(page);+ expect(storage.bucket.copyObject).toHaveBeenCalledTimes(page);const replay = await advanceDesktopRecordingSourceCommit(video,checkpoint,);if (!("checkpoint" in replay)) throw new Error("Unexpected completion");- expect(storage.bucket.copyObject).toHaveBeenCalledTimes(64);+ expect(storage.bucket.copyObject).toHaveBeenCalledTimes(page);expect(replay.checkpoint.snapshotId).toBe(checkpoint.snapshotId);const source = await finishSnapshot(video, replay.checkpoint);const urls = await buildDesktopRecordingSourceUrls(video, source);@@ -1760,16 +1766,40 @@ describe("source commit page size", () => {};}- it("uses 64-entry pages, which the leaf schema already accepts", async () => {+ it("uses 64-entry pages for Cap's bucket and keeps 16 elsewhere, both within the leaf schema", async () => {const actual = await vi.importActual<typeof import("@/lib/desktop-recording-source-checkpoint")>("@/lib/desktop-recording-source-checkpoint");- expect(actual.SOURCE_COMMIT_PAGE_SIZE).toBe(64);- expect(actual.SOURCE_COMMIT_PAGE_SIZE).toBeLessThanOrEqual(+ expect(actual.SOURCE_COMMIT_AWS_PAGE_SIZE).toBe(64);+ expect(actual.SOURCE_COMMIT_PAGE_SIZE).toBe(16);+ expect(actual.SOURCE_COMMIT_AWS_PAGE_SIZE).toBeLessThanOrEqual(actual.SOURCE_COMMIT_PART_PAGE_SIZE,);});+ it.each([+ ["a custom S3-compatible bucket", "s3", true],+ ["Google Drive", "googleDrive", false],+ ] as const)(+ "keeps 16-entry checkpoints for %s",+ async (_, provider, customBucket) => {+ pagedFixture(provider, customBucket);+ mocks.pageSize = undefined;+ let checkpoint = snapshotCheckpoint();+ let checkpoints = 0;+ while (true) {+ const result = await advanceDesktopRecordingSourceCommit(+ recording(),+ checkpoint,+ );+ checkpoints++;+ if ("source" in result) break;+ checkpoint = result.checkpoint;+ }+ expect(checkpoints).toBe(2 + 3 * Math.ceil((segmentCount + 3) / 16));+ },+ );+it("commits more than 64 objects with the same inventory and receipts as 16-entry pages", async () => {const legacy = await referenceOutput(16);vi.clearAllMocks();@@ -1874,6 +1904,9 @@ describe("source commit page size", () => {expect(peak.head).toBe(expected);const source = await finishSnapshot(recording(), planned.checkpoint);expect(peak.copy).toBe(expected);+ // Each copy/verify worker checks the copy and its original together, so the+ // bound is on objects in flight; requests peak at two per object.+ expect(peak.head).toBeLessThanOrEqual(expected * 2);expect(readInventory(storage, source.inventoryKey).objects).toHaveLength(segmentCount + 3,);diff --git a/apps/web/lib/desktop-recording-source-checkpoint.ts b/apps/web/lib/desktop-recording-source-checkpoint.tsindex a30539304..08add3f07 100644--- a/apps/web/lib/desktop-recording-source-checkpoint.ts+++ b/apps/web/lib/desktop-recording-source-checkpoint.ts@@ -1,6 +1,8 @@import { z } from "zod";-export const SOURCE_COMMIT_PAGE_SIZE = 64;+export const SOURCE_COMMIT_PAGE_SIZE = 16;+// Cap's own AWS bucket answers fast enough to finish 4x larger pages well inside a step's deadline.+export const SOURCE_COMMIT_AWS_PAGE_SIZE = 64;export const SOURCE_COMMIT_PART_BATCH_SIZE = 4;export const SOURCE_COMMIT_PART_PAGE_SIZE = 64;export const SOURCE_COMMIT_MAX_OBJECTS = 100_002;diff --git a/apps/web/lib/desktop-recording-source.ts b/apps/web/lib/desktop-recording-source.tsindex ef748f26f..4d4825589 100644--- a/apps/web/lib/desktop-recording-source.ts+++ b/apps/web/lib/desktop-recording-source.ts@@ -9,6 +9,7 @@ import { z } from "zod";import {type DesktopRecordingSourceCheckpoint,desktopRecordingSourceCheckpointSchema,+ SOURCE_COMMIT_AWS_PAGE_SIZE,SOURCE_COMMIT_MAX_OBJECTS,SOURCE_COMMIT_PAGE_SIZE,SOURCE_COMMIT_PART_BATCH_SIZE,@@ -201,6 +202,7 @@ type SourceContext = {prefix: string;run: SourceRun;concurrency: number;+ pageSize: number;};type TreeKind = "plan" | "objects" | "parts";@@ -261,10 +263,16 @@ const SOURCE_COMMIT_IO_TIMEOUT_MS = 60_000;const SOURCE_COMMIT_CONCURRENCY = 8;const SOURCE_COMMIT_AWS_CONCURRENCY = 32;-function sourceConcurrency([bucket, customBucket]: RecordingStorageAccess) {+function sourceLimits([bucket, customBucket]: RecordingStorageAccess) {return bucket.provider === "s3" && Option.isNone(customBucket)- ? SOURCE_COMMIT_AWS_CONCURRENCY- : SOURCE_COMMIT_CONCURRENCY;+ ? {+ concurrency: SOURCE_COMMIT_AWS_CONCURRENCY,+ pageSize: SOURCE_COMMIT_AWS_PAGE_SIZE,+ }+ : {+ concurrency: SOURCE_COMMIT_CONCURRENCY,+ pageSize: SOURCE_COMMIT_PAGE_SIZE,+ };}function invalidSource(message: string): never {@@ -940,6 +948,7 @@ export async function prepareDesktopRecordingSegments(run,prefix: sourcePrefix(video).slice(0, -1),concurrency: SOURCE_COMMIT_CONCURRENCY,+ pageSize: SOURCE_COMMIT_PAGE_SIZE,};const unique = [...new Map(@@ -1171,7 +1180,7 @@ async function advanceSourceSnapshot(invalidSource("Recording source plan cursor is incomplete");const end = Math.min(plan.objectCount,- checkpoint.cursor + SOURCE_COMMIT_PAGE_SIZE,+ checkpoint.cursor + context.pageSize,);const originals = await mapBounded(Array.from(@@ -1217,7 +1226,7 @@ async function advanceSourceSnapshot(scope,originalObjectSchema,checkpoint.cursor,- Math.min(plan.objectCount, checkpoint.cursor + SOURCE_COMMIT_PAGE_SIZE),+ Math.min(plan.objectCount, checkpoint.cursor + context.pageSize),);const large =context.bucket.provider === "s3"@@ -1281,7 +1290,7 @@ async function advanceSourceSnapshot(if (checkpoint.phase === "verify") {const end = Math.min(plan.objectCount,- checkpoint.cursor + SOURCE_COMMIT_PAGE_SIZE,+ checkpoint.cursor + context.pageSize,);const [originals, objects] = await Promise.all([readTree(@@ -1451,7 +1460,7 @@ export async function advanceDesktopRecordingSourceCommit(bucket: access[0],run,prefix: `${sourcePrefix(video)}${parsed.generation}/${parsed.snapshotId}`,- concurrency: sourceConcurrency(access),+ ...sourceLimits(access),},parsed,verification,@@ -1549,7 +1558,7 @@ export async function buildDesktopRecordingSourceUrls(bucket,run,prefix: sourcePrefix(video).slice(0, -1),- concurrency: sourceConcurrency(access),+ ...sourceLimits(access),},inventory.roots,"objects",