Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(web): verify Google Drive checkpoint pages with the SHA-256 Drive returns instead of reading them back
Richie McIlroy696c0339439b
diff --git a/apps/web/__tests__/unit/desktop-recording-source.test.ts b/apps/web/__tests__/unit/desktop-recording-source.test.tsindex d173d6ce9..15e7cbf0d 100644--- a/apps/web/__tests__/unit/desktop-recording-source.test.ts+++ b/apps/web/__tests__/unit/desktop-recording-source.test.ts@@ -180,6 +180,18 @@ function storageFixture(provider = "s3", customBucket = false) {seed(key, body);}),),+ ...(provider === "googleDrive"+ ? {+ putObjectWithChecksum: vi.fn((key: string, body: string) =>+ checked(() => {+ seed(key, body);+ return {+ sha256Checksum: hash(objects.get(key)?.body ?? ""),+ } as { sha256Checksum: string | null };+ }),+ ),+ }+ : {}),copyObject: vi.fn((source: string, target: string, options: CopyOptions) =>checked(() => {const { value } = copiedSource(source, options);@@ -1315,8 +1327,12 @@ describe("durable desktop recording source commit", () => {for (const [key] of storage.bucket.getObject.mock.calls)if (key.includes("/pages/"))pageReads.set(key, (pageReads.get(key) ?? 0) + 1);- expect(pageReads.size).toBeGreaterThan(1);- expect(Math.max(...pageReads.values())).toBe(1);+ const pagesWritten = [+ ...storage.bucket.putObject.mock.calls,+ ...(storage.bucket.putObjectWithChecksum?.mock.calls ?? []),+ ].filter(([key]) => key.includes("/pages/")).length;+ expect(pagesWritten).toBeGreaterThan(1);+ expect(Math.max(0, ...pageReads.values())).toBeLessThanOrEqual(1);},);@@ -1332,6 +1348,67 @@ describe("durable desktop recording source commit", () => {.some(([key]) => key.includes("/pages/")),).toBe(true);});+ it("trusts Drive's stored SHA-256 for checkpoint pages instead of reading them back", async () => {+ const pageReads = async (checksum: boolean) => {+ const storage = storageFixture("googleDrive");+ seedSegments(+ storage,+ Array.from({ length: 150 }, (_, index) => index + 1),+ );+ const put = storage.bucket.putObjectWithChecksum;+ const store = put?.getMockImplementation();+ if (!put || !store) throw new Error("Missing fixture operation");+ if (!checksum)+ put.mockImplementation((key, body) =>+ store(key, body).pipe(Effect.as({ sha256Checksum: null })),+ );+ await commitDesktopRecordingSource(recording(), generation);+ return {+ written: put.mock.calls.length,+ reads: storage.bucket.getObject.mock.calls.filter(([key]) =>+ key.includes("/pages/"),+ ).length,+ };+ };+ const trusted = await pageReads(true);+ const fallback = await pageReads(false);+ expect(trusted.written).toBeGreaterThan(3);+ expect(trusted.written).toBe(fallback.written);+ expect(trusted.reads).toBe(fallback.reads - fallback.written);+ });++ it("reads a Drive checkpoint page back when Drive returns no checksum", async () => {+ const storage = storageFixture("googleDrive");+ seedSegments(storage);+ const put = storage.bucket.putObjectWithChecksum;+ const store = put?.getMockImplementation();+ if (!put || !store) throw new Error("Missing fixture operation");+ put.mockImplementation((key, body) =>+ store(key, body).pipe(Effect.as({ sha256Checksum: null })),+ );+ await commitDesktopRecordingSource(recording(), generation);+ const reads = new Set(+ storage.bucket.getObject.mock.calls.map(([key]) => key),+ );+ expect(put.mock.calls.every(([key]) => reads.has(key))).toBe(true);+ });++ it("does not commit a Drive checkpoint page stored with different bytes", async () => {+ const storage = storageFixture("googleDrive");+ seedSegments(storage);+ const put = storage.bucket.putObjectWithChecksum;+ const store = put?.getMockImplementation();+ if (!put || !store) throw new Error("Missing fixture operation");+ put.mockImplementation((key, body) => store(key, `${body}\ncorrupt`));+ await expect(+ commitDesktopRecordingSource(recording(), generation),+ ).rejects.toThrow("Durable recording source readback did not match");+ expect(+ storage.bucket.putObject.mock.calls.some(([key]) =>+ key.endsWith("/inventory.json"),+ ),+ ).toBe(false);+ });});describe("large desktop recording snapshots", () => {diff --git a/apps/web/__tests__/unit/google-drive-storage.test.ts b/apps/web/__tests__/unit/google-drive-storage.test.tsindex 2367371e0..6b5b5cd3b 100644--- a/apps/web/__tests__/unit/google-drive-storage.test.ts+++ b/apps/web/__tests__/unit/google-drive-storage.test.ts@@ -2369,3 +2369,86 @@ describe("Google Drive rate limits", () => {expect(fetch).toHaveBeenCalledTimes(2);});});++describe("Google Drive uploads with a stored checksum", () => {+ const uploadHarness = () => {+ const harness = makeRepoHarness([+ {+ id: videoId,+ name: "Checksummed",+ ownerId: videoOwnerId,+ storageIntegrationId: integrationId,+ },+ ]);+ harness.putObject(+ storedObjectInput(".cap-folders/video-1", "folder-id", {+ contentType: GOOGLE_DRIVE_FOLDER_MIME_TYPE,+ }),+ );+ harness.putObject(+ storedObjectInput(+ ".cap-warnings/video-1/DO_NOT_EDIT_OR_DELETE.txt",+ "warning-id",+ { contentType: "text/plain" },+ ),+ );+ return harness;+ };+ const stubUpload = (stored: Record<string, unknown> | string) =>+ vi.stubGlobal(+ "fetch",+ vi.fn(async (input: string | URL | Request, init?: RequestInit) => {+ const url = String(input);+ if (url.includes("/files/generateIds"))+ return jsonResponse({ ids: ["page-id"] });+ if (url.includes("uploadType=resumable") && init?.method === "POST")+ return new Response(null, {+ status: 200,+ headers: { Location: "https://upload.test/session" },+ });+ if (url === "https://upload.test/session" && init?.method === "PUT")+ return typeof stored === "string"+ ? new Response(stored, { status: 200 })+ : jsonResponse(stored);+ throw new Error(`Unexpected Drive request: ${init?.method} ${url}`);+ }),+ );++ it("returns the SHA-256 Drive computed for the stored file", async () => {+ const harness = uploadHarness();+ stubUpload({+ id: "page-id",+ size: "7",+ sha256Checksum: "ABCDEF0123",+ });+ const access = await getStorageAccess(harness);+ if (!("putObjectWithChecksum" in access))+ throw new Error("Drive access has no checksum upload");+ await expect(+ Effect.runPromise(+ access.putObjectWithChecksum(resultKey, '{"a":1}', {+ contentType: "application/json",+ }),+ ),+ ).resolves.toEqual({ sha256Checksum: "abcdef0123" });+ expect(+ [...harness.objects.values()].find(+ (object) => object.objectKey === resultKey,+ )?.uploadStatus,+ ).toBe("complete");+ });++ it.each([+ ["no checksum field", { id: "page-id", size: "7" }],+ ["an unreadable body", "not json"],+ ])("returns no checksum when Drive's response has %s", async (_, stored) => {+ const harness = uploadHarness();+ stubUpload(stored);+ const access = await getStorageAccess(harness);+ if (!("putObjectWithChecksum" in access))+ throw new Error("Drive access has no checksum upload");+ await expect(+ Effect.runPromise(access.putObjectWithChecksum(resultKey, '{"a":1}')),+ ).resolves.toEqual({ sha256Checksum: null });+ });+});diff --git a/apps/web/lib/desktop-recording-source.ts b/apps/web/lib/desktop-recording-source.tsindex c11f10881..423176f39 100644--- a/apps/web/lib/desktop-recording-source.ts+++ b/apps/web/lib/desktop-recording-source.ts@@ -399,11 +399,29 @@ async function writeSourceText(}async function writeReference(context: SourceContext, value: unknown) {- return writeSourceText(- context,- `${context.prefix}/pages/${randomUUID()}.json`,- JSON.stringify(value),- );+ const key = `${context.prefix}/pages/${randomUUID()}.json`;+ const content = JSON.stringify(value);+ const bucket = context.bucket;+ // Pages get a fresh random key, so no other writer can replace them; a SHA-256 Drive computed from the+ // stored bytes proves the write as well as reading it back, which costs another Drive round trip.+ if (bucket.provider === "googleDrive" && "putObjectWithChecksum" in bucket) {+ assertContextKey(context, key);+ const sha256 = hash(content);+ const stored = await context.run(+ bucket.putObjectWithChecksum(key, content, {+ contentType: "application/json",+ }),+ );+ if (stored.sha256Checksum !== sha256) {+ if (stored.sha256Checksum !== null)+ throw new Error("Durable recording source readback did not match");+ if ((await readSourceText(context, key)) !== content)+ throw new Error("Durable recording source readback did not match");+ }+ rememberVerifiedPage(key, sha256, content);+ return { key, sha256 };+ }+ return writeSourceText(context, key, content);}function treeCount(roots: SourceCommitPageReference[]) {diff --git a/packages/web-backend/src/Storage/index.ts b/packages/web-backend/src/Storage/index.tsindex 10455252b..8de42a248 100644--- a/packages/web-backend/src/Storage/index.ts+++ b/packages/web-backend/src/Storage/index.ts@@ -809,6 +809,68 @@ const makeGoogleDriveAccess = ({),};+ const putObjectWithChecksum = (+ key: string,+ body: string | Uint8Array | ArrayBuffer,+ fields?: { contentType?: string; contentLength?: number },+ ) =>+ Effect.gen(function* () {+ const contentType = fields?.contentType ?? "application/octet-stream";+ const contentLength =+ fields?.contentLength ??+ (typeof body === "string"+ ? new TextEncoder().encode(body).byteLength+ : body.byteLength);+ const uploadUrl = yield* createGoogleDriveResumableUpload(+ repo,+ config,+ {+ integrationId,+ ownerId,+ videoId: parseVideoIdFromObjectKey(key).pipe(+ Option.map((id) => id as Video.VideoId),+ Option.getOrNull,+ ),+ key,+ contentType,+ contentLength,+ },+ tokenStore,+ ).pipe(mapStorageError);+ const uploadBody =+ body instanceof Uint8Array ? new Uint8Array(body).buffer : body;+ const response = yield* Effect.tryPromise({+ try: () =>+ fetch(uploadUrl, {+ method: "PUT",+ headers: getGoogleDriveUploadHeaders(contentType, contentLength),+ body: uploadBody,+ }),+ catch: (cause) => new StorageDomain.StorageError({ cause }),+ });+ if (!response.ok) {+ return yield* Effect.fail(+ new StorageDomain.StorageError({+ cause: new Error(`Google Drive upload failed: ${response.status}`),+ }),+ );+ }+ const stored: unknown = yield* Effect.promise(() =>+ response.json().catch(() => null),+ );+ yield* mapStorageError(+ repo.markObjectComplete(integrationId, key, contentLength),+ );+ const checksum =+ typeof stored === "object" &&+ stored !== null &&+ "sha256Checksum" in stored &&+ typeof stored.sha256Checksum === "string"+ ? stored.sha256Checksum.toLowerCase()+ : null;+ return { sha256Checksum: checksum };+ });+return {provider: "googleDrive" as const,bucketName: "google-drive",@@ -904,54 +966,8 @@ const makeGoogleDriveAccess = ({key: string,body: string | Uint8Array | ArrayBuffer,fields?: { contentType?: string; contentLength?: number },- ) =>- Effect.gen(function* () {- const contentType = fields?.contentType ?? "application/octet-stream";- const contentLength =- fields?.contentLength ??- (typeof body === "string"- ? new TextEncoder().encode(body).byteLength- : body.byteLength);- const uploadUrl = yield* createGoogleDriveResumableUpload(- repo,- config,- {- integrationId,- ownerId,- videoId: parseVideoIdFromObjectKey(key).pipe(- Option.map((id) => id as Video.VideoId),- Option.getOrNull,- ),- key,- contentType,- contentLength,- },- tokenStore,- ).pipe(mapStorageError);- const uploadBody =- body instanceof Uint8Array ? new Uint8Array(body).buffer : body;- const response = yield* Effect.tryPromise({- try: () =>- fetch(uploadUrl, {- method: "PUT",- headers: getGoogleDriveUploadHeaders(contentType, contentLength),- body: uploadBody,- }),- catch: (cause) => new StorageDomain.StorageError({ cause }),- });- if (!response.ok) {- return yield* Effect.fail(- new StorageDomain.StorageError({- cause: new Error(- `Google Drive upload failed: ${response.status}`,- ),- }),- );- }- yield* mapStorageError(- repo.markObjectComplete(integrationId, key, contentLength),- );- }),+ ) => putObjectWithChecksum(key, body, fields).pipe(Effect.asVoid),+ putObjectWithChecksum,copyObject,copyObjectForRecording,deleteObject,