Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(web): reuse checkpoint pages a recording source commit already verified instead of re-reading them

Richie McIlroy3c6bad8235cf
diff --git a/apps/web/__tests__/unit/desktop-recording-source.test.ts b/apps/web/__tests__/unit/desktop-recording-source.test.ts
index 774b5d7b7..d173d6ce9 100644
--- a/apps/web/__tests__/unit/desktop-recording-source.test.ts
+++ b/apps/web/__tests__/unit/desktop-recording-source.test.ts
@@ -1302,6 +1302,36 @@ describe("durable desktop recording source commit", () => {
),
).toBe(false);
});
+ it.each(["s3", "googleDrive"] as const)(
+ "reads each %s checkpoint page back once and reuses it for the rest of the commit",
+ async (provider) => {
+ const storage = storageFixture(provider);
+ seedSegments(
+ storage,
+ Array.from({ length: 150 }, (_, index) => index + 1),
+ );
+ await commitDesktopRecordingSource(recording(), generation);
+ const pageReads = new Map<string, number>();
+ for (const [key] of storage.bucket.getObject.mock.calls)
+ if (key.includes("/pages/"))
+ pageReads.set(key, (pageReads.get(key) ?? 0) + 1);
+ expect(pageReads.size).toBeGreaterThan(1);
+ expect(Math.max(...pageReads.values())).toBe(1);
+ },
+ );
+
+ it("still reads committed pages from storage when building source URLs", async () => {
+ const storage = storageFixture("googleDrive");
+ seedSegments(storage);
+ const source = await commitDesktopRecordingSource(recording(), generation);
+ const before = storage.bucket.getObject.mock.calls.length;
+ await buildDesktopRecordingSourceUrls(recording(), source);
+ expect(
+ storage.bucket.getObject.mock.calls
+ .slice(before)
+ .some(([key]) => key.includes("/pages/")),
+ ).toBe(true);
+ });
});
describe("large desktop recording snapshots", () => {
diff --git a/apps/web/lib/desktop-recording-source.ts b/apps/web/lib/desktop-recording-source.ts
index 7535f387b..c11f10881 100644
--- a/apps/web/lib/desktop-recording-source.ts
+++ b/apps/web/lib/desktop-recording-source.ts
@@ -206,6 +206,8 @@ type SourceContext = {
run: SourceRun;
concurrency: number;
pageSize: number;
+ // Only a commit in progress may reuse pages it verified; committed sources must prove their pages still exist.
+ reuseVerifiedPages?: boolean;
};
type TreeKind = "plan" | "objects" | "parts";
@@ -342,17 +344,41 @@ async function readSourceText(context: SourceContext, key: string) {
return result.value;
}
+// Checkpoint pages are addressed by their SHA-256, so a page this process already wrote and read back, or
+// read and verified, can be served from memory: storage could only return these same bytes or fail the hash.
+const VERIFIED_PAGE_CACHE_MAX_BYTES = 32 * 1024 * 1024;
+const verifiedPages = new Map<string, string>();
+let verifiedPageBytes = 0;
+
+function rememberVerifiedPage(key: string, sha256: string, content: string) {
+ const id = `${key}\n${sha256}`;
+ if (verifiedPages.has(id) || content.length > VERIFIED_PAGE_CACHE_MAX_BYTES)
+ return;
+ verifiedPages.set(id, content);
+ verifiedPageBytes += content.length;
+ for (const [oldest, value] of verifiedPages) {
+ if (verifiedPageBytes <= VERIFIED_PAGE_CACHE_MAX_BYTES) break;
+ verifiedPages.delete(oldest);
+ verifiedPageBytes -= value.length;
+ }
+}
+
async function readReference(
context: SourceContext,
reference: SourceCommitReference,
) {
assertContextKey(context, reference.key);
+ const cached = context.reuseVerifiedPages
+ ? verifiedPages.get(`${reference.key}\n${reference.sha256}`)
+ : undefined;
+ if (cached !== undefined) return parseSourceJson(cached);
const content = await readSourceText(context, reference.key);
if (hash(content) !== reference.sha256)
throw new DesktopRecordingSourceError(
"source-changed",
"A recording source checkpoint changed after it was saved",
);
+ rememberVerifiedPage(reference.key, reference.sha256, content);
return parseSourceJson(content);
}
@@ -367,7 +393,9 @@ async function writeSourceText(
);
if ((await readSourceText(context, key)) !== content)
throw new Error("Durable recording source readback did not match");
- return { key, sha256: hash(content) };
+ const sha256 = hash(content);
+ rememberVerifiedPage(key, sha256, content);
+ return { key, sha256 };
}
async function writeReference(context: SourceContext, value: unknown) {
@@ -1486,6 +1514,7 @@ export async function advanceDesktopRecordingSourceCommit(
run,
prefix: `${sourcePrefix(video)}${parsed.generation}/${parsed.snapshotId}`,
...sourceLimits(access),
+ reuseVerifiedPages: true,
},
parsed,
verification,