Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(web): keep 16-entry source pages for custom buckets and Drive

Richie McIlroy841aae72ecd5
diff --git a/apps/web/__tests__/unit/desktop-recording-source.test.ts b/apps/web/__tests__/unit/desktop-recording-source.test.ts
index fba603ce6..5db42c507 100644
--- a/apps/web/__tests__/unit/desktop-recording-source.test.ts
+++ b/apps/web/__tests__/unit/desktop-recording-source.test.ts
@@ -25,6 +25,9 @@ vi.mock("@/lib/desktop-recording-source-checkpoint", async (importOriginal) => {
get SOURCE_COMMIT_PAGE_SIZE() {
return mocks.pageSize ?? actual.SOURCE_COMMIT_PAGE_SIZE;
},
+ get SOURCE_COMMIT_AWS_PAGE_SIZE() {
+ return mocks.pageSize ?? actual.SOURCE_COMMIT_AWS_PAGE_SIZE;
+ },
};
});
vi.mock("@/lib/workflow-runtime", async () => {
@@ -1390,9 +1393,12 @@ describe("large desktop recording snapshots", () => {
).toBe(8 * 1024 ** 3);
});
- it.each(["s3", "googleDrive"])(
+ it.each([
+ ["s3", 64],
+ ["googleDrive", 16],
+ ] as const)(
"resumes a completed %s copy when the database checkpoint write was lost",
- async (provider) => {
+ async (provider, page) => {
const storage = storageFixture(provider);
seedSegments(
storage,
@@ -1410,14 +1416,14 @@ describe("large desktop recording snapshots", () => {
checkpoint,
);
if (!("checkpoint" in copied)) throw new Error("Unexpected completion");
- expect(copied.checkpoint.cursor).toBe(64);
- expect(storage.bucket.copyObject).toHaveBeenCalledTimes(64);
+ expect(copied.checkpoint.cursor).toBe(page);
+ expect(storage.bucket.copyObject).toHaveBeenCalledTimes(page);
const replay = await advanceDesktopRecordingSourceCommit(
video,
checkpoint,
);
if (!("checkpoint" in replay)) throw new Error("Unexpected completion");
- expect(storage.bucket.copyObject).toHaveBeenCalledTimes(64);
+ expect(storage.bucket.copyObject).toHaveBeenCalledTimes(page);
expect(replay.checkpoint.snapshotId).toBe(checkpoint.snapshotId);
const source = await finishSnapshot(video, replay.checkpoint);
const urls = await buildDesktopRecordingSourceUrls(video, source);
@@ -1760,16 +1766,40 @@ describe("source commit page size", () => {
};
}
- it("uses 64-entry pages, which the leaf schema already accepts", async () => {
+ it("uses 64-entry pages for Cap's bucket and keeps 16 elsewhere, both within the leaf schema", async () => {
const actual = await vi.importActual<
typeof import("@/lib/desktop-recording-source-checkpoint")
>("@/lib/desktop-recording-source-checkpoint");
- expect(actual.SOURCE_COMMIT_PAGE_SIZE).toBe(64);
- expect(actual.SOURCE_COMMIT_PAGE_SIZE).toBeLessThanOrEqual(
+ expect(actual.SOURCE_COMMIT_AWS_PAGE_SIZE).toBe(64);
+ expect(actual.SOURCE_COMMIT_PAGE_SIZE).toBe(16);
+ expect(actual.SOURCE_COMMIT_AWS_PAGE_SIZE).toBeLessThanOrEqual(
actual.SOURCE_COMMIT_PART_PAGE_SIZE,
);
});
+ it.each([
+ ["a custom S3-compatible bucket", "s3", true],
+ ["Google Drive", "googleDrive", false],
+ ] as const)(
+ "keeps 16-entry checkpoints for %s",
+ async (_, provider, customBucket) => {
+ pagedFixture(provider, customBucket);
+ mocks.pageSize = undefined;
+ let checkpoint = snapshotCheckpoint();
+ let checkpoints = 0;
+ while (true) {
+ const result = await advanceDesktopRecordingSourceCommit(
+ recording(),
+ checkpoint,
+ );
+ checkpoints++;
+ if ("source" in result) break;
+ checkpoint = result.checkpoint;
+ }
+ expect(checkpoints).toBe(2 + 3 * Math.ceil((segmentCount + 3) / 16));
+ },
+ );
+
it("commits more than 64 objects with the same inventory and receipts as 16-entry pages", async () => {
const legacy = await referenceOutput(16);
vi.clearAllMocks();
@@ -1874,6 +1904,9 @@ describe("source commit page size", () => {
expect(peak.head).toBe(expected);
const source = await finishSnapshot(recording(), planned.checkpoint);
expect(peak.copy).toBe(expected);
+ // Each copy/verify worker checks the copy and its original together, so the
+ // bound is on objects in flight; requests peak at two per object.
+ expect(peak.head).toBeLessThanOrEqual(expected * 2);
expect(readInventory(storage, source.inventoryKey).objects).toHaveLength(
segmentCount + 3,
);
diff --git a/apps/web/lib/desktop-recording-source-checkpoint.ts b/apps/web/lib/desktop-recording-source-checkpoint.ts
index a30539304..08add3f07 100644
--- a/apps/web/lib/desktop-recording-source-checkpoint.ts
+++ b/apps/web/lib/desktop-recording-source-checkpoint.ts
@@ -1,6 +1,8 @@
import { z } from "zod";
-export const SOURCE_COMMIT_PAGE_SIZE = 64;
+export const SOURCE_COMMIT_PAGE_SIZE = 16;
+// Cap's own AWS bucket answers fast enough to finish 4x larger pages well inside a step's deadline.
+export const SOURCE_COMMIT_AWS_PAGE_SIZE = 64;
export const SOURCE_COMMIT_PART_BATCH_SIZE = 4;
export const SOURCE_COMMIT_PART_PAGE_SIZE = 64;
export const SOURCE_COMMIT_MAX_OBJECTS = 100_002;
diff --git a/apps/web/lib/desktop-recording-source.ts b/apps/web/lib/desktop-recording-source.ts
index ef748f26f..4d4825589 100644
--- a/apps/web/lib/desktop-recording-source.ts
+++ b/apps/web/lib/desktop-recording-source.ts
@@ -9,6 +9,7 @@ import { z } from "zod";
import {
type DesktopRecordingSourceCheckpoint,
desktopRecordingSourceCheckpointSchema,
+ SOURCE_COMMIT_AWS_PAGE_SIZE,
SOURCE_COMMIT_MAX_OBJECTS,
SOURCE_COMMIT_PAGE_SIZE,
SOURCE_COMMIT_PART_BATCH_SIZE,
@@ -201,6 +202,7 @@ type SourceContext = {
prefix: string;
run: SourceRun;
concurrency: number;
+ pageSize: number;
};
type TreeKind = "plan" | "objects" | "parts";
@@ -261,10 +263,16 @@ const SOURCE_COMMIT_IO_TIMEOUT_MS = 60_000;
const SOURCE_COMMIT_CONCURRENCY = 8;
const SOURCE_COMMIT_AWS_CONCURRENCY = 32;
-function sourceConcurrency([bucket, customBucket]: RecordingStorageAccess) {
+function sourceLimits([bucket, customBucket]: RecordingStorageAccess) {
return bucket.provider === "s3" && Option.isNone(customBucket)
- ? SOURCE_COMMIT_AWS_CONCURRENCY
- : SOURCE_COMMIT_CONCURRENCY;
+ ? {
+ concurrency: SOURCE_COMMIT_AWS_CONCURRENCY,
+ pageSize: SOURCE_COMMIT_AWS_PAGE_SIZE,
+ }
+ : {
+ concurrency: SOURCE_COMMIT_CONCURRENCY,
+ pageSize: SOURCE_COMMIT_PAGE_SIZE,
+ };
}
function invalidSource(message: string): never {
@@ -940,6 +948,7 @@ export async function prepareDesktopRecordingSegments(
run,
prefix: sourcePrefix(video).slice(0, -1),
concurrency: SOURCE_COMMIT_CONCURRENCY,
+ pageSize: SOURCE_COMMIT_PAGE_SIZE,
};
const unique = [
...new Map(
@@ -1171,7 +1180,7 @@ async function advanceSourceSnapshot(
invalidSource("Recording source plan cursor is incomplete");
const end = Math.min(
plan.objectCount,
- checkpoint.cursor + SOURCE_COMMIT_PAGE_SIZE,
+ checkpoint.cursor + context.pageSize,
);
const originals = await mapBounded(
Array.from(
@@ -1217,7 +1226,7 @@ async function advanceSourceSnapshot(
scope,
originalObjectSchema,
checkpoint.cursor,
- Math.min(plan.objectCount, checkpoint.cursor + SOURCE_COMMIT_PAGE_SIZE),
+ Math.min(plan.objectCount, checkpoint.cursor + context.pageSize),
);
const large =
context.bucket.provider === "s3"
@@ -1281,7 +1290,7 @@ async function advanceSourceSnapshot(
if (checkpoint.phase === "verify") {
const end = Math.min(
plan.objectCount,
- checkpoint.cursor + SOURCE_COMMIT_PAGE_SIZE,
+ checkpoint.cursor + context.pageSize,
);
const [originals, objects] = await Promise.all([
readTree(
@@ -1451,7 +1460,7 @@ export async function advanceDesktopRecordingSourceCommit(
bucket: access[0],
run,
prefix: `${sourcePrefix(video)}${parsed.generation}/${parsed.snapshotId}`,
- concurrency: sourceConcurrency(access),
+ ...sourceLimits(access),
},
parsed,
verification,
@@ -1549,7 +1558,7 @@ export async function buildDesktopRecordingSourceUrls(
bucket,
run,
prefix: sourcePrefix(video).slice(0, -1),
- concurrency: sourceConcurrency(access),
+ ...sourceLimits(access),
},
inventory.roots,
"objects",