Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(web): verify Google Drive checkpoint pages with the SHA-256 Drive returns instead of reading them back

Richie McIlroy696c0339439b
diff --git a/apps/web/__tests__/unit/desktop-recording-source.test.ts b/apps/web/__tests__/unit/desktop-recording-source.test.ts
index d173d6ce9..15e7cbf0d 100644
--- a/apps/web/__tests__/unit/desktop-recording-source.test.ts
+++ b/apps/web/__tests__/unit/desktop-recording-source.test.ts
@@ -180,6 +180,18 @@ function storageFixture(provider = "s3", customBucket = false) {
seed(key, body);
}),
),
+ ...(provider === "googleDrive"
+ ? {
+ putObjectWithChecksum: vi.fn((key: string, body: string) =>
+ checked(() => {
+ seed(key, body);
+ return {
+ sha256Checksum: hash(objects.get(key)?.body ?? ""),
+ } as { sha256Checksum: string | null };
+ }),
+ ),
+ }
+ : {}),
copyObject: vi.fn((source: string, target: string, options: CopyOptions) =>
checked(() => {
const { value } = copiedSource(source, options);
@@ -1315,8 +1327,12 @@ describe("durable desktop recording source commit", () => {
for (const [key] of storage.bucket.getObject.mock.calls)
if (key.includes("/pages/"))
pageReads.set(key, (pageReads.get(key) ?? 0) + 1);
- expect(pageReads.size).toBeGreaterThan(1);
- expect(Math.max(...pageReads.values())).toBe(1);
+ const pagesWritten = [
+ ...storage.bucket.putObject.mock.calls,
+ ...(storage.bucket.putObjectWithChecksum?.mock.calls ?? []),
+ ].filter(([key]) => key.includes("/pages/")).length;
+ expect(pagesWritten).toBeGreaterThan(1);
+ expect(Math.max(0, ...pageReads.values())).toBeLessThanOrEqual(1);
},
);
@@ -1332,6 +1348,67 @@ describe("durable desktop recording source commit", () => {
.some(([key]) => key.includes("/pages/")),
).toBe(true);
});
+ it("trusts Drive's stored SHA-256 for checkpoint pages instead of reading them back", async () => {
+ const pageReads = async (checksum: boolean) => {
+ const storage = storageFixture("googleDrive");
+ seedSegments(
+ storage,
+ Array.from({ length: 150 }, (_, index) => index + 1),
+ );
+ const put = storage.bucket.putObjectWithChecksum;
+ const store = put?.getMockImplementation();
+ if (!put || !store) throw new Error("Missing fixture operation");
+ if (!checksum)
+ put.mockImplementation((key, body) =>
+ store(key, body).pipe(Effect.as({ sha256Checksum: null })),
+ );
+ await commitDesktopRecordingSource(recording(), generation);
+ return {
+ written: put.mock.calls.length,
+ reads: storage.bucket.getObject.mock.calls.filter(([key]) =>
+ key.includes("/pages/"),
+ ).length,
+ };
+ };
+ const trusted = await pageReads(true);
+ const fallback = await pageReads(false);
+ expect(trusted.written).toBeGreaterThan(3);
+ expect(trusted.written).toBe(fallback.written);
+ expect(trusted.reads).toBe(fallback.reads - fallback.written);
+ });
+
+ it("reads a Drive checkpoint page back when Drive returns no checksum", async () => {
+ const storage = storageFixture("googleDrive");
+ seedSegments(storage);
+ const put = storage.bucket.putObjectWithChecksum;
+ const store = put?.getMockImplementation();
+ if (!put || !store) throw new Error("Missing fixture operation");
+ put.mockImplementation((key, body) =>
+ store(key, body).pipe(Effect.as({ sha256Checksum: null })),
+ );
+ await commitDesktopRecordingSource(recording(), generation);
+ const reads = new Set(
+ storage.bucket.getObject.mock.calls.map(([key]) => key),
+ );
+ expect(put.mock.calls.every(([key]) => reads.has(key))).toBe(true);
+ });
+
+ it("does not commit a Drive checkpoint page stored with different bytes", async () => {
+ const storage = storageFixture("googleDrive");
+ seedSegments(storage);
+ const put = storage.bucket.putObjectWithChecksum;
+ const store = put?.getMockImplementation();
+ if (!put || !store) throw new Error("Missing fixture operation");
+ put.mockImplementation((key, body) => store(key, `${body}\ncorrupt`));
+ await expect(
+ commitDesktopRecordingSource(recording(), generation),
+ ).rejects.toThrow("Durable recording source readback did not match");
+ expect(
+ storage.bucket.putObject.mock.calls.some(([key]) =>
+ key.endsWith("/inventory.json"),
+ ),
+ ).toBe(false);
+ });
});
describe("large desktop recording snapshots", () => {
diff --git a/apps/web/__tests__/unit/google-drive-storage.test.ts b/apps/web/__tests__/unit/google-drive-storage.test.ts
index 2367371e0..6b5b5cd3b 100644
--- a/apps/web/__tests__/unit/google-drive-storage.test.ts
+++ b/apps/web/__tests__/unit/google-drive-storage.test.ts
@@ -2369,3 +2369,86 @@ describe("Google Drive rate limits", () => {
expect(fetch).toHaveBeenCalledTimes(2);
});
});
+
+describe("Google Drive uploads with a stored checksum", () => {
+ const uploadHarness = () => {
+ const harness = makeRepoHarness([
+ {
+ id: videoId,
+ name: "Checksummed",
+ ownerId: videoOwnerId,
+ storageIntegrationId: integrationId,
+ },
+ ]);
+ harness.putObject(
+ storedObjectInput(".cap-folders/video-1", "folder-id", {
+ contentType: GOOGLE_DRIVE_FOLDER_MIME_TYPE,
+ }),
+ );
+ harness.putObject(
+ storedObjectInput(
+ ".cap-warnings/video-1/DO_NOT_EDIT_OR_DELETE.txt",
+ "warning-id",
+ { contentType: "text/plain" },
+ ),
+ );
+ return harness;
+ };
+ const stubUpload = (stored: Record<string, unknown> | string) =>
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async (input: string | URL | Request, init?: RequestInit) => {
+ const url = String(input);
+ if (url.includes("/files/generateIds"))
+ return jsonResponse({ ids: ["page-id"] });
+ if (url.includes("uploadType=resumable") && init?.method === "POST")
+ return new Response(null, {
+ status: 200,
+ headers: { Location: "https://upload.test/session" },
+ });
+ if (url === "https://upload.test/session" && init?.method === "PUT")
+ return typeof stored === "string"
+ ? new Response(stored, { status: 200 })
+ : jsonResponse(stored);
+ throw new Error(`Unexpected Drive request: ${init?.method} ${url}`);
+ }),
+ );
+
+ it("returns the SHA-256 Drive computed for the stored file", async () => {
+ const harness = uploadHarness();
+ stubUpload({
+ id: "page-id",
+ size: "7",
+ sha256Checksum: "ABCDEF0123",
+ });
+ const access = await getStorageAccess(harness);
+ if (!("putObjectWithChecksum" in access))
+ throw new Error("Drive access has no checksum upload");
+ await expect(
+ Effect.runPromise(
+ access.putObjectWithChecksum(resultKey, '{"a":1}', {
+ contentType: "application/json",
+ }),
+ ),
+ ).resolves.toEqual({ sha256Checksum: "abcdef0123" });
+ expect(
+ [...harness.objects.values()].find(
+ (object) => object.objectKey === resultKey,
+ )?.uploadStatus,
+ ).toBe("complete");
+ });
+
+ it.each([
+ ["no checksum field", { id: "page-id", size: "7" }],
+ ["an unreadable body", "not json"],
+ ])("returns no checksum when Drive's response has %s", async (_, stored) => {
+ const harness = uploadHarness();
+ stubUpload(stored);
+ const access = await getStorageAccess(harness);
+ if (!("putObjectWithChecksum" in access))
+ throw new Error("Drive access has no checksum upload");
+ await expect(
+ Effect.runPromise(access.putObjectWithChecksum(resultKey, '{"a":1}')),
+ ).resolves.toEqual({ sha256Checksum: null });
+ });
+});
diff --git a/apps/web/lib/desktop-recording-source.ts b/apps/web/lib/desktop-recording-source.ts
index c11f10881..423176f39 100644
--- a/apps/web/lib/desktop-recording-source.ts
+++ b/apps/web/lib/desktop-recording-source.ts
@@ -399,11 +399,29 @@ async function writeSourceText(
}
async function writeReference(context: SourceContext, value: unknown) {
- return writeSourceText(
- context,
- `${context.prefix}/pages/${randomUUID()}.json`,
- JSON.stringify(value),
- );
+ const key = `${context.prefix}/pages/${randomUUID()}.json`;
+ const content = JSON.stringify(value);
+ const bucket = context.bucket;
+ // Pages get a fresh random key, so no other writer can replace them; a SHA-256 Drive computed from the
+ // stored bytes proves the write as well as reading it back, which costs another Drive round trip.
+ if (bucket.provider === "googleDrive" && "putObjectWithChecksum" in bucket) {
+ assertContextKey(context, key);
+ const sha256 = hash(content);
+ const stored = await context.run(
+ bucket.putObjectWithChecksum(key, content, {
+ contentType: "application/json",
+ }),
+ );
+ if (stored.sha256Checksum !== sha256) {
+ if (stored.sha256Checksum !== null)
+ throw new Error("Durable recording source readback did not match");
+ if ((await readSourceText(context, key)) !== content)
+ throw new Error("Durable recording source readback did not match");
+ }
+ rememberVerifiedPage(key, sha256, content);
+ return { key, sha256 };
+ }
+ return writeSourceText(context, key, content);
}
function treeCount(roots: SourceCommitPageReference[]) {
diff --git a/packages/web-backend/src/Storage/index.ts b/packages/web-backend/src/Storage/index.ts
index 10455252b..8de42a248 100644
--- a/packages/web-backend/src/Storage/index.ts
+++ b/packages/web-backend/src/Storage/index.ts
@@ -809,6 +809,68 @@ const makeGoogleDriveAccess = ({
),
};
+ const putObjectWithChecksum = (
+ key: string,
+ body: string | Uint8Array | ArrayBuffer,
+ fields?: { contentType?: string; contentLength?: number },
+ ) =>
+ Effect.gen(function* () {
+ const contentType = fields?.contentType ?? "application/octet-stream";
+ const contentLength =
+ fields?.contentLength ??
+ (typeof body === "string"
+ ? new TextEncoder().encode(body).byteLength
+ : body.byteLength);
+ const uploadUrl = yield* createGoogleDriveResumableUpload(
+ repo,
+ config,
+ {
+ integrationId,
+ ownerId,
+ videoId: parseVideoIdFromObjectKey(key).pipe(
+ Option.map((id) => id as Video.VideoId),
+ Option.getOrNull,
+ ),
+ key,
+ contentType,
+ contentLength,
+ },
+ tokenStore,
+ ).pipe(mapStorageError);
+ const uploadBody =
+ body instanceof Uint8Array ? new Uint8Array(body).buffer : body;
+ const response = yield* Effect.tryPromise({
+ try: () =>
+ fetch(uploadUrl, {
+ method: "PUT",
+ headers: getGoogleDriveUploadHeaders(contentType, contentLength),
+ body: uploadBody,
+ }),
+ catch: (cause) => new StorageDomain.StorageError({ cause }),
+ });
+ if (!response.ok) {
+ return yield* Effect.fail(
+ new StorageDomain.StorageError({
+ cause: new Error(`Google Drive upload failed: ${response.status}`),
+ }),
+ );
+ }
+ const stored: unknown = yield* Effect.promise(() =>
+ response.json().catch(() => null),
+ );
+ yield* mapStorageError(
+ repo.markObjectComplete(integrationId, key, contentLength),
+ );
+ const checksum =
+ typeof stored === "object" &&
+ stored !== null &&
+ "sha256Checksum" in stored &&
+ typeof stored.sha256Checksum === "string"
+ ? stored.sha256Checksum.toLowerCase()
+ : null;
+ return { sha256Checksum: checksum };
+ });
+
return {
provider: "googleDrive" as const,
bucketName: "google-drive",
@@ -904,54 +966,8 @@ const makeGoogleDriveAccess = ({
key: string,
body: string | Uint8Array | ArrayBuffer,
fields?: { contentType?: string; contentLength?: number },
- ) =>
- Effect.gen(function* () {
- const contentType = fields?.contentType ?? "application/octet-stream";
- const contentLength =
- fields?.contentLength ??
- (typeof body === "string"
- ? new TextEncoder().encode(body).byteLength
- : body.byteLength);
- const uploadUrl = yield* createGoogleDriveResumableUpload(
- repo,
- config,
- {
- integrationId,
- ownerId,
- videoId: parseVideoIdFromObjectKey(key).pipe(
- Option.map((id) => id as Video.VideoId),
- Option.getOrNull,
- ),
- key,
- contentType,
- contentLength,
- },
- tokenStore,
- ).pipe(mapStorageError);
- const uploadBody =
- body instanceof Uint8Array ? new Uint8Array(body).buffer : body;
- const response = yield* Effect.tryPromise({
- try: () =>
- fetch(uploadUrl, {
- method: "PUT",
- headers: getGoogleDriveUploadHeaders(contentType, contentLength),
- body: uploadBody,
- }),
- catch: (cause) => new StorageDomain.StorageError({ cause }),
- });
- if (!response.ok) {
- return yield* Effect.fail(
- new StorageDomain.StorageError({
- cause: new Error(
- `Google Drive upload failed: ${response.status}`,
- ),
- }),
- );
- }
- yield* mapStorageError(
- repo.markObjectComplete(integrationId, key, contentLength),
- );
- }),
+ ) => putObjectWithChecksum(key, body, fields).pipe(Effect.asVoid),
+ putObjectWithChecksum,
copyObject,
copyObjectForRecording,
deleteObject,