Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(web): check Cap bucket recording sources with listings in one checkpoint

Richie McIlroy8653d754f798
diff --git a/apps/web/__tests__/unit/desktop-recording-source.test.ts b/apps/web/__tests__/unit/desktop-recording-source.test.ts
index e7e3b6a0a..aed466c33 100644
--- a/apps/web/__tests__/unit/desktop-recording-source.test.ts
+++ b/apps/web/__tests__/unit/desktop-recording-source.test.ts
@@ -121,7 +121,12 @@ function recording(source: VideoRow["source"] = { type: "desktopSegments" }) {
return { id: "owned-video", ownerId: "owned-user", source } as VideoRow;
}
-function storageFixture(provider = "s3", customBucket = false) {
+function storageFixture(
+ provider = "s3",
+ customBucket = false,
+ // Mirrors Cap's S3 bucket: listings carry sizes and content ETags, and a copy keeps its source's ETag.
+ fixture = { listing: false, copiesKeepIdentity: false },
+) {
const objects = new Map<string, StoredObject>();
const uploads = new Map<
string,
@@ -153,7 +158,9 @@ function storageFixture(provider = "s3", customBucket = false) {
objects.set(key, {
body,
size: Buffer.byteLength(body),
- identity: `"original-${++identity}"`,
+ identity: fixture.listing
+ ? `"${createHash("md5").update(body).digest("hex")}"`
+ : `"original-${++identity}"`,
});
};
const bucket = {
@@ -180,6 +187,16 @@ function storageFixture(provider = "s3", customBucket = false) {
seed(key, body);
}),
),
+ ...(fixture.listing
+ ? {
+ putObjectWithETag: vi.fn((key: string, body: string) =>
+ checked(() => {
+ seed(key, body);
+ return { ETag: objects.get(key)?.identity };
+ }),
+ ),
+ }
+ : {}),
...(provider === "googleDrive"
? {
putObjectWithChecksum: vi.fn((key: string, body: string) =>
@@ -204,7 +221,9 @@ function storageFixture(provider = "s3", customBucket = false) {
}
const copied = {
...value,
- identity: `"snapshot-${++identity}"`,
+ identity: fixture.copiesKeepIdentity
+ ? value.identity
+ : `"snapshot-${++identity}"`,
metadata: provider === "s3" ? options.Metadata : undefined,
};
objects.set(target, copied);
@@ -227,9 +246,15 @@ function storageFixture(provider = "s3", customBucket = false) {
.sort();
const start = continuationToken ? Number(continuationToken) : 0;
return {
- Contents: keys
- .slice(start, start + maxKeys)
- .map((Key) => ({ Key })),
+ Contents: keys.slice(start, start + maxKeys).map((Key) =>
+ fixture.listing
+ ? {
+ Key,
+ Size: objects.get(Key)?.size,
+ ETag: objects.get(Key)?.identity,
+ }
+ : { Key },
+ ),
IsTruncated: start + maxKeys < keys.length,
NextContinuationToken:
start + maxKeys < keys.length
@@ -351,7 +376,7 @@ function storageFixture(provider = "s3", customBucket = false) {
customBucket ? Option.some({ id: "custom-bucket" }) : Option.none(),
]),
);
- return { bucket, objects, uploads, seed, object };
+ return { bucket, objects, uploads, seed, object, fixture };
}
function snapshotCheckpoint(): DesktopRecordingSourceCheckpoint {
@@ -2377,3 +2402,448 @@ describe("committed source access", () => {
}
});
});
+
+function listedStorage() {
+ return storageFixture("s3", false, {
+ listing: true,
+ copiesKeepIdentity: true,
+ });
+}
+
+function seedRecording(
+ storage: ReturnType<typeof storageFixture>,
+ videoSegments: number,
+ audioSegments: number,
+) {
+ const manifest = {
+ version: 2,
+ video_init_uploaded: true,
+ audio_init_uploaded: audioSegments > 0,
+ video_segments: Array.from({ length: videoSegments }, (_, i) => i + 1),
+ audio_segments: Array.from({ length: audioSegments }, (_, i) => i + 1),
+ is_complete: true,
+ };
+ const json = `${JSON.stringify(manifest, null, 2)}\n`;
+ storage.seed(manifestKey, json);
+ storage.seed(videoInitKey, "video-init");
+ if (audioSegments > 0) storage.seed(audioInitKey, "audio-init");
+ const segmentKey = (track: "video" | "audio", index: number) =>
+ `${prefix}/segments/${track}/segment_${String(index).padStart(3, "0")}.m4s`;
+ for (let index = 1; index <= videoSegments; index++)
+ storage.seed(segmentKey("video", index), `video-fragment-${index}`);
+ for (let index = 1; index <= audioSegments; index++)
+ storage.seed(segmentKey("audio", index), `audio-fragment-${index}`);
+ return { json, manifest, segmentKey };
+}
+
+async function prepareAll(track: "video" | "audio", indices: number[]) {
+ for (let offset = 0; offset < indices.length; offset += 32)
+ await prepareDesktopRecordingSegments(
+ recording(),
+ indices.slice(offset, offset + 32).map((index) => ({ track, index })),
+ async () => true,
+ );
+}
+
+function inventoryShape(
+ storage: ReturnType<typeof storageFixture>,
+ inventoryKey: string,
+) {
+ return readInventory(storage, inventoryKey)
+ .objects.map((entry) => ({
+ originalKey: entry.originalKey,
+ size: entry.size,
+ body: storage.object(entry.key).body,
+ prepared: entry.key.includes("/prepared/"),
+ identityMatchesCopy:
+ storage.object(entry.key).identity === entry.objectIdentity,
+ originalIdentityMatches:
+ storage.object(entry.originalKey).identity === entry.originalIdentity,
+ }))
+ .sort((left, right) => left.originalKey.localeCompare(right.originalKey));
+}
+
+// Jumps the clock past the listed advance budget after the first checkpoint page is written.
+function expireBudgetAfterFirstPage(
+ storage: ReturnType<typeof storageFixture>,
+) {
+ const realNow = Date.now.bind(Date);
+ let offset = 0;
+ const clock = vi
+ .spyOn(Date, "now")
+ .mockImplementation(() => realNow() + offset);
+ const writer = storage.bucket.putObjectWithETag ?? storage.bucket.putObject;
+ const write = writer.getMockImplementation();
+ if (!write) throw new Error("Missing put implementation");
+ writer.mockImplementation((key: string, body: string) => {
+ const result = write(key, body);
+ if (offset === 0 && key.includes("/pages/")) offset = 25_000;
+ return result;
+ });
+ return () => {
+ clock.mockRestore();
+ writer.mockImplementation(write);
+ };
+}
+
+describe("listed source commit on Cap's bucket", () => {
+ it("commits the same inventory as per-object checks in one checkpoint, without per-fragment HEADs", async () => {
+ const legacy = storageFixture();
+ seedRecording(legacy, 300, 150);
+ await prepareAll(
+ "video",
+ Array.from({ length: 280 }, (_, i) => i + 1),
+ );
+ await prepareAll(
+ "audio",
+ Array.from({ length: 140 }, (_, i) => i + 1),
+ );
+ const legacySource = await commitDesktopRecordingSource(
+ recording(),
+ generation,
+ );
+ const expected = inventoryShape(legacy, legacySource.inventoryKey);
+
+ vi.clearAllMocks();
+ const listed = listedStorage();
+ seedRecording(listed, 300, 150);
+ await prepareAll(
+ "video",
+ Array.from({ length: 280 }, (_, i) => i + 1),
+ );
+ await prepareAll(
+ "audio",
+ Array.from({ length: 140 }, (_, i) => i + 1),
+ );
+ listed.bucket.headObject.mockClear();
+ listed.bucket.copyObject.mockClear();
+ const first = await advanceDesktopRecordingSourceCommit(
+ recording(),
+ snapshotCheckpoint(),
+ );
+ if (!("source" in first))
+ throw new Error("Listed commit should finish in one checkpoint");
+ expect(inventoryShape(listed, first.source.inventoryKey)).toEqual(expected);
+ expect(
+ inventoryShape(listed, first.source.inventoryKey).filter(
+ (entry) => entry.prepared,
+ ),
+ ).toHaveLength(420);
+ expect(listed.bucket.headObject).not.toHaveBeenCalled();
+ // Only the unprepared tail and init fragments are copied.
+ expect(listed.bucket.copyObject).toHaveBeenCalledTimes(32);
+ const urls = await buildDesktopRecordingSourceUrls(
+ recording(),
+ first.source,
+ );
+ expect(urls.videoSegmentUrls).toHaveLength(300);
+ expect(urls.audioSegmentUrls).toHaveLength(150);
+ });
+
+ it("advances one revision per checkpoint and resumes with the per-object path", async () => {
+ mocks.pageSize = 8;
+ const storage = listedStorage();
+ seedRecording(storage, 40, 0);
+ await prepareAll(
+ "video",
+ Array.from({ length: 30 }, (_, i) => i + 1),
+ );
+ const restore = expireBudgetAfterFirstPage(storage);
+ const first = await advanceDesktopRecordingSourceCommit(
+ recording(),
+ snapshotCheckpoint(),
+ );
+ restore();
+ if (!("checkpoint" in first)) throw new Error("Expected partial progress");
+ expect(first.checkpoint.revision).toBe(1);
+ expect(first.checkpoint).toMatchObject({ phase: "enumerate", cursor: 8 });
+ storage.fixture.listing = false;
+ const source = await finishSnapshot(recording(), first.checkpoint);
+ const shape = inventoryShape(storage, source.inventoryKey);
+ expect(shape).toHaveLength(41);
+ expect(shape.every((entry) => entry.identityMatchesCopy)).toBe(true);
+ expect(
+ shape.every((entry) =>
+ entry.body?.startsWith(
+ entry.originalKey.endsWith("init.mp4")
+ ? "video-init"
+ : "video-fragment",
+ ),
+ ),
+ ).toBe(true);
+ });
+
+ it("finishes a commit that per-object checks started", async () => {
+ mocks.pageSize = 8;
+ const storage = listedStorage();
+ seedRecording(storage, 40, 3);
+ await prepareAll(
+ "video",
+ Array.from({ length: 40 }, (_, i) => i + 1),
+ );
+ storage.fixture.listing = false;
+ let checkpoint = await advanceToPhase(
+ recording(),
+ snapshotCheckpoint(),
+ "copy",
+ );
+ const step = await advanceDesktopRecordingSourceCommit(
+ recording(),
+ checkpoint,
+ );
+ if (!("checkpoint" in step)) throw new Error("Expected copy progress");
+ checkpoint = step.checkpoint;
+ storage.fixture.listing = true;
+ const done = await advanceDesktopRecordingSourceCommit(
+ recording(),
+ checkpoint,
+ );
+ if (!("source" in done)) throw new Error("Listed advance should finish");
+ const shape = inventoryShape(storage, done.source.inventoryKey);
+ expect(shape).toHaveLength(45);
+ expect(shape.every((entry) => entry.originalIdentityMatches)).toBe(true);
+ });
+
+ it.each(["copy", "verify"] as const)(
+ "rejects a fragment replaced after its identity was planned, during %s",
+ async (phase) => {
+ mocks.pageSize = 8;
+ const storage = listedStorage();
+ const { segmentKey } = seedRecording(storage, 20, 0);
+ await prepareAll(
+ "video",
+ Array.from({ length: 20 }, (_, i) => i + 1),
+ );
+ storage.fixture.listing = false;
+ const checkpoint = await advanceToPhase(
+ recording(),
+ snapshotCheckpoint(),
+ phase,
+ );
+ storage.fixture.listing = true;
+ storage.seed(segmentKey("video", 7), "video-fragment-replaced");
+ await expect(
+ advanceDesktopRecordingSourceCommit(recording(), checkpoint),
+ ).rejects.toMatchObject({ code: "source-changed" });
+ expect(
+ storage.bucket.putObject.mock.calls.some(([key]) =>
+ key.endsWith("/inventory.json"),
+ ),
+ ).toBe(false);
+ },
+ );
+
+ it("rejects a snapshot copy deleted before verification", async () => {
+ mocks.pageSize = 8;
+ const storage = listedStorage();
+ seedRecording(storage, 12, 0);
+ storage.fixture.listing = false;
+ const checkpoint = await advanceToPhase(
+ recording(),
+ snapshotCheckpoint(),
+ "verify",
+ );
+ const copied = storage.bucket.copyObject.mock.calls.at(-1)?.[1];
+ if (!copied) throw new Error("Missing snapshot copy");
+ storage.objects.delete(copied);
+ storage.fixture.listing = true;
+ await expect(
+ advanceDesktopRecordingSourceCommit(recording(), checkpoint),
+ ).rejects.toMatchObject({ code: "source-missing" });
+ });
+
+ it("rejects a recording whose fragment never finished uploading", async () => {
+ const storage = listedStorage();
+ const { segmentKey } = seedRecording(storage, 10, 2);
+ storage.objects.delete(segmentKey("video", 4));
+ await expect(
+ commitDesktopRecordingSource(recording(), generation),
+ ).rejects.toMatchObject({ code: "source-missing" });
+ });
+
+ it("does not reuse a prepared copy holding different bytes of the same size", async () => {
+ const storage = listedStorage();
+ seedRecording(storage, 3, 0);
+ await prepareAll("video", [1, 2, 3]);
+ const preparedCopy = storage.bucket.copyObject.mock.calls.find(([, key]) =>
+ key.endsWith("/video/2.m4s"),
+ )?.[1];
+ if (!preparedCopy) throw new Error("Missing prepared copy");
+ storage.seed(preparedCopy, "video-fragment-X");
+ const source = await commitDesktopRecordingSource(recording(), generation);
+ const fragment = inventoryShape(storage, source.inventoryKey).find(
+ (entry) => entry.originalKey.endsWith("segment_002.m4s"),
+ );
+ expect(fragment).toMatchObject({
+ body: "video-fragment-2",
+ prepared: false,
+ identityMatchesCopy: true,
+ });
+ });
+
+ it("checks copies whose ETag differs from the original one by one", async () => {
+ const storage = listedStorage();
+ storage.fixture.copiesKeepIdentity = false;
+ seedRecording(storage, 6, 2);
+ await prepareAll("video", [1, 2, 3, 4, 5, 6]);
+ storage.bucket.headObject.mockClear();
+ const source = await commitDesktopRecordingSource(recording(), generation);
+ const shape = inventoryShape(storage, source.inventoryKey);
+ expect(shape.filter((entry) => entry.prepared)).toHaveLength(6);
+ expect(shape.every((entry) => entry.identityMatchesCopy)).toBe(true);
+ expect(
+ storage.bucket.headObject.mock.calls.filter(([key]) =>
+ key.endsWith("segment_003.m4s"),
+ ).length,
+ ).toBeGreaterThan(0);
+ });
+
+ it("proves checkpoint writes by their ETag instead of reading them back", async () => {
+ const storage = listedStorage();
+ seedRecording(storage, 20, 10);
+ await commitDesktopRecordingSource(recording(), generation);
+ const written = new Set(
+ storage.bucket.putObjectWithETag?.mock.calls.map(([key]) => key),
+ );
+ expect(written.size).toBeGreaterThan(0);
+ // Finalize still compares the saved manifest copy with the original; that is a content check, not a readback.
+ expect(
+ storage.bucket.getObject.mock.calls.filter(
+ ([key]) => written.has(key) && !key.endsWith("/manifest.json"),
+ ),
+ ).toHaveLength(0);
+ expect(storage.bucket.putObject).not.toHaveBeenCalled();
+ });
+
+ it("reads a checkpoint write back when its ETag is not the MD5 of its bytes", async () => {
+ const storage = listedStorage();
+ seedRecording(storage, 4, 1);
+ storage.bucket.putObjectWithETag?.mockImplementation(
+ (key: string, body: string) =>
+ checked(() => {
+ storage.seed(key, body);
+ return { ETag: '"not-an-md5"' };
+ }),
+ );
+ const source = await commitDesktopRecordingSource(recording(), generation);
+ expect(
+ storage.bucket.getObject.mock.calls.some(
+ ([key]) => key === source.inventoryKey,
+ ),
+ ).toBe(true);
+ storage.bucket.putObjectWithETag?.mockImplementation(
+ (key: string, body: string) =>
+ checked(() => {
+ storage.seed(key, `${body} `);
+ return { ETag: '"not-an-md5"' };
+ }),
+ );
+ await expect(
+ commitDesktopRecordingSource(recording(), generation),
+ ).rejects.toThrow("Durable recording source readback did not match");
+ });
+
+ it("checks a new copy whose response does not carry the original's ETag", async () => {
+ const storage = listedStorage();
+ storage.fixture.copiesKeepIdentity = false;
+ seedRecording(storage, 4, 1);
+ const source = await commitDesktopRecordingSource(recording(), generation);
+ const shape = inventoryShape(storage, source.inventoryKey);
+ expect(shape).toHaveLength(7);
+ expect(shape.every((entry) => entry.identityMatchesCopy)).toBe(true);
+ expect(
+ storage.bucket.putObject.mock.calls.filter(([key]) =>
+ key.endsWith("/receipt.json"),
+ ),
+ ).toHaveLength(7);
+ });
+
+ it("lets the per-object path verify and reuse copies made without receipts", async () => {
+ mocks.pageSize = 4;
+ const storage = listedStorage();
+ seedRecording(storage, 10, 0);
+ storage.fixture.listing = false;
+ let checkpoint = await advanceToPhase(
+ recording(),
+ snapshotCheckpoint(),
+ "copy",
+ );
+ storage.fixture.listing = true;
+ const restore = expireBudgetAfterFirstPage(storage);
+ const step = await advanceDesktopRecordingSourceCommit(
+ recording(),
+ checkpoint,
+ );
+ restore();
+ if (!("checkpoint" in step)) throw new Error("Expected copy progress");
+ checkpoint = step.checkpoint;
+ expect(checkpoint).toMatchObject({ phase: "copy", cursor: 4 });
+ expect(
+ storage.bucket.putObject.mock.calls.some(([key]) =>
+ key.endsWith("/receipt.json"),
+ ),
+ ).toBe(false);
+ storage.fixture.listing = false;
+ const source = await finishSnapshot(recording(), checkpoint);
+ const shape = inventoryShape(storage, source.inventoryKey);
+ expect(shape).toHaveLength(11);
+ expect(shape.every((entry) => entry.identityMatchesCopy)).toBe(true);
+ expect(storage.bucket.copyObject).toHaveBeenCalledTimes(11);
+ });
+
+ it.each(["missing ETag", "truncated without a token"] as const)(
+ "falls back to per-object checks when a listing has a %s",
+ async (defect) => {
+ const storage = listedStorage();
+ seedRecording(storage, 5, 1);
+ const list = storage.bucket.listObjects.getMockImplementation();
+ if (!list) throw new Error("Missing listObjects implementation");
+ storage.bucket.listObjects.mockImplementation((input) =>
+ list(input).pipe(
+ Effect.map((listing) =>
+ defect === "missing ETag"
+ ? {
+ ...listing,
+ Contents: listing.Contents.map(({ Key, Size }) => ({
+ Key,
+ Size,
+ ETag: undefined,
+ })),
+ }
+ : {
+ ...listing,
+ IsTruncated: true,
+ NextContinuationToken: undefined,
+ },
+ ),
+ ),
+ );
+ storage.bucket.headObject.mockClear();
+ const source = await commitDesktopRecordingSource(
+ recording(),
+ generation,
+ );
+ expect(inventoryShape(storage, source.inventoryKey)).toHaveLength(8);
+ expect(
+ storage.bucket.headObject.mock.calls.filter(
+ ([key]) => key === `${prefix}/segments/video/segment_001.m4s`,
+ ),
+ ).toHaveLength(3);
+ },
+ );
+
+ it("leaves custom S3-compatible buckets on per-object checks", async () => {
+ const storage = storageFixture("s3", true, {
+ listing: true,
+ copiesKeepIdentity: true,
+ });
+ seedRecording(storage, 4, 1);
+ storage.bucket.headObject.mockClear();
+ await commitDesktopRecordingSource(recording(), generation);
+ expect(
+ storage.bucket.headObject.mock.calls.filter(
+ ([key]) => key === `${prefix}/segments/video/segment_001.m4s`,
+ ),
+ ).toHaveLength(3);
+ });
+});
diff --git a/apps/web/lib/desktop-recording-source.ts b/apps/web/lib/desktop-recording-source.ts
index f3b4d3d86..41f425168 100644
--- a/apps/web/lib/desktop-recording-source.ts
+++ b/apps/web/lib/desktop-recording-source.ts
@@ -208,6 +208,8 @@ type SourceContext = {
pageSize: number;
// Only a commit in progress may reuse pages it verified; committed sources must prove their pages still exist.
reuseVerifiedPages?: boolean;
+ // Cap's own bucket lists sizes and ETags, so a commit can check every fragment with a few listings.
+ listed?: boolean;
};
type TreeKind = "plan" | "objects" | "parts";
@@ -278,6 +280,7 @@ function sourceLimits([bucket, customBucket]: RecordingStorageAccess) {
return {
concurrency: SOURCE_COMMIT_AWS_CONCURRENCY,
pageSize: SOURCE_COMMIT_AWS_PAGE_SIZE,
+ listed: true,
};
if (bucket.provider === "googleDrive")
return {
@@ -388,11 +391,27 @@ async function writeSourceText(
content: string,
) {
assertContextKey(context, key);
- await context.run(
- context.bucket.putObject(key, content, { contentType: "application/json" }),
- );
- if ((await readSourceText(context, key)) !== content)
- throw new Error("Durable recording source readback did not match");
+ const bucket = context.bucket;
+ // Cap's bucket answers a single-part PUT with the MD5 of the bytes it stored, which proves the write as a
+ // readback does; any other answer still reads the object back.
+ if (context.listed && "putObjectWithETag" in bucket) {
+ const stored = await context.run(
+ bucket.putObjectWithETag(key, content, {
+ contentType: "application/json",
+ }),
+ );
+ if (
+ stored.ETag !== `"${createHash("md5").update(content).digest("hex")}"` &&
+ (await readSourceText(context, key)) !== content
+ )
+ throw new Error("Durable recording source readback did not match");
+ } else {
+ await context.run(
+ bucket.putObject(key, content, { contentType: "application/json" }),
+ );
+ if ((await readSourceText(context, key)) !== content)
+ throw new Error("Durable recording source readback did not match");
+ }
const sha256 = hash(content);
rememberVerifiedPage(key, sha256, content);
return { key, sha256 };
@@ -794,12 +813,7 @@ async function checkCopy(
key: string,
expectedIdentity?: string,
): Promise<SourceObject> {
- assertContextKey(
- key.startsWith(`${preparedSourcePrefix(context.video, original)}/`)
- ? { ...context, prefix: preparedSourcePrefix(context.video, original) }
- : context,
- key,
- );
+ assertCopyKey(context, original, key);
const [head, originalHead] = await Promise.all([
context.run(context.bucket.headObject(key)),
checkOriginal(context, original),
@@ -975,6 +989,48 @@ async function copySmallObject(
return saveObjectReceipt(context, await checkCopy(context, original, key));
}
+// A listing already showed there is no reusable copy, and S3 copies the exact original version or fails,
+// so a copy whose response carries the original's ETag needs no HEAD or receipt; the listed verify
+// phase still lists it, and a retry on the per-object path reuses it through its copy metadata.
+async function copyListedObject(
+ context: SourceContext,
+ original: OriginalObject,
+ position: number,
+): Promise<SourceObject> {
+ const key = newCopyKey(context, original, position);
+ let copied: unknown;
+ try {
+ copied = await context.run(
+ context.bucket.copyObject(
+ `${context.bucket.bucketName}/${original.originalKey}`,
+ key,
+ {
+ CopySourceIfMatch: original.originalIdentity,
+ IfNoneMatch: "*",
+ MetadataDirective: "REPLACE",
+ Metadata: copyMetadata(original),
+ ContentType: original.track === "audio" ? "audio/mp4" : "video/mp4",
+ },
+ ),
+ );
+ } catch (error) {
+ await checkOriginal(context, original);
+ throw error;
+ }
+ const identity =
+ copied &&
+ typeof copied === "object" &&
+ "CopyObjectResult" in copied &&
+ copied.CopyObjectResult &&
+ typeof copied.CopyObjectResult === "object" &&
+ "ETag" in copied.CopyObjectResult
+ ? copied.CopyObjectResult.ETag
+ : undefined;
+ if (identity === original.originalIdentity)
+ return { ...original, key, objectIdentity: identity };
+ return saveObjectReceipt(context, await checkCopy(context, original, key));
+}
+
function preparedSourcePrefix(video: DbVideo, original: OriginalObject) {
return `${sourcePrefix(video)}prepared/${hash(
JSON.stringify([
@@ -989,6 +1045,139 @@ function preparationMarkerKey(video: DbVideo) {
return `${sourcePrefix(video)}preparation.json`;
}
+type ListedObject = { size: number; identity: string };
+type SnapshotListing = {
+ originals: Map<string, ListedObject>;
+ copies: Map<string, ListedObject>;
+ copyDirectories: Map<string, string[]>;
+};
+
+const LISTED_ADVANCE_BUDGET_MS = 20_000;
+const LISTING_MAX_PAGES = 64;
+const HEX_DIGITS = [..."0123456789abcdef"];
+
+// Returns null when the listing is incomplete or lacks a size or ETag, so callers fall back to per-object checks.
+async function listObjectsUnder(
+ context: SourceContext,
+ prefix: string,
+): Promise<Map<string, ListedObject> | null> {
+ const listed = new Map<string, ListedObject>();
+ let continuationToken: string | undefined;
+ for (let page = 0; page < LISTING_MAX_PAGES; page++) {
+ const listing = await context.run(
+ context.bucket.listObjects({ prefix, maxKeys: 1000, continuationToken }),
+ );
+ for (const entry of listing.Contents ?? []) {
+ if (!entry.Key?.startsWith(prefix)) return null;
+ const identity = "ETag" in entry ? entry.ETag : undefined;
+ if (
+ typeof entry.Size !== "number" ||
+ !Number.isSafeInteger(entry.Size) ||
+ typeof identity !== "string" ||
+ !recordingObjectIdentitySchema.safeParse(identity).success
+ )
+ return null;
+ listed.set(entry.Key, { size: entry.Size, identity });
+ }
+ if (!listing.IsTruncated) return listed;
+ if (!listing.NextContinuationToken) return null;
+ continuationToken = listing.NextContinuationToken;
+ }
+ return null;
+}
+
+async function listSnapshot(
+ context: SourceContext,
+ includeCopies: boolean,
+): Promise<SnapshotListing | null> {
+ const segments = `${context.video.ownerId}/${context.video.id}/segments/`;
+ const prepared = `${sourcePrefix(context.video)}prepared/`;
+ const listings = await Promise.all([
+ listObjectsUnder(context, `${segments}video/`),
+ listObjectsUnder(context, `${segments}audio/`),
+ ...(includeCopies
+ ? [
+ listObjectsUnder(context, `${context.prefix}/copies/`),
+ ...HEX_DIGITS.map((digit) =>
+ listObjectsUnder(context, `${prepared}${digit}`),
+ ),
+ ]
+ : []),
+ ]);
+ if (listings.some((listing) => listing === null)) return null;
+ const [video, audio, ...copyListings] = listings as Map<
+ string,
+ ListedObject
+ >[];
+ const copies = new Map(copyListings.flatMap((listing) => [...listing]));
+ const copyDirectories = new Map<string, string[]>();
+ for (const key of copies.keys()) {
+ const directory =
+ /^(.+\/copies\/\d+\/)[^/]+\/(?:video|audio)\/\d+\.(?:mp4|m4s)$/.exec(
+ key,
+ )?.[1];
+ if (!directory) continue;
+ const keys = copyDirectories.get(directory);
+ if (keys) keys.push(key);
+ else copyDirectories.set(directory, [key]);
+ }
+ return {
+ originals: new Map([...(video ?? []), ...(audio ?? [])]),
+ copies,
+ copyDirectories,
+ };
+}
+
+function listedOriginalMatches(
+ listing: SnapshotListing,
+ original: OriginalObject,
+) {
+ const listed = listing.originals.get(original.originalKey);
+ return (
+ listed?.identity === original.originalIdentity &&
+ listed.size === original.size
+ );
+}
+
+// A copy holds the original's bytes when it has the original's size and S3 content ETag. "unlisted" means no
+// copy exists to reuse; any other copy is left to the per-object checks, which compare its identity metadata.
+function findListedCopy(
+ context: SourceContext,
+ listing: SnapshotListing,
+ original: OriginalObject,
+ position: number,
+ prepared: boolean,
+): SourceObject | "unlisted" | null {
+ if (!listedOriginalMatches(listing, original)) return null;
+ const directories = [
+ ...(prepared && original.index > 0
+ ? [
+ `${preparedSourcePrefix(context.video, original)}/copies/${original.index}/`,
+ ]
+ : []),
+ `${context.prefix}/copies/${position}/`,
+ ];
+ let candidates = 0;
+ for (const directory of directories) {
+ for (const key of listing.copyDirectories.get(directory) ?? []) {
+ const copy = listing.copies.get(key);
+ candidates++;
+ if (
+ !copy ||
+ !key.endsWith(
+ `/${original.track}/${original.index}${original.index === 0 ? ".mp4" : ".m4s"}`,
+ ) ||
+ copy.size !== original.size ||
+ copy.identity !== original.originalIdentity
+ )
+ continue;
+ assertCopyKey(context, original, key);
+ return { ...original, key, objectIdentity: copy.identity };
+ }
+ }
+ return candidates === 0 ? "unlisted" : null;
+}
+
export const recordingPreparationSegmentSchema = z.object({
track: z.enum(["video", "audio"]),
index: z.number().int().min(1).max(50_000),
@@ -1235,19 +1424,84 @@ async function copyMultipartPartBatch(
};
}
+// Listed commits run phase after phase inside one checkpoint while their time budget lasts, saving the
+// checkpoint round trips; the saved checkpoint still advances by exactly one revision.
async function advanceSourceSnapshot(
context: SourceContext,
checkpoint: DesktopRecordingSourceCheckpoint,
verification?: RecordingVerification,
+): Promise<
+ | { checkpoint: DesktopRecordingSourceCheckpoint }
+ | { source: DesktopRecordingSource }
+> {
+ if (!context.listed)
+ return advanceSourceSnapshotOnce(context, checkpoint, verification);
+ const deadline = Date.now() + LISTED_ADVANCE_BUDGET_MS;
+ let current = checkpoint;
+ for (;;) {
+ const result = await advanceSourceSnapshotOnce(
+ context,
+ current,
+ verification,
+ deadline,
+ );
+ if ("source" in result) return { source: result.source };
+ current = result.checkpoint;
+ if (!result.listed || Date.now() >= deadline)
+ return { checkpoint: { ...current, revision: checkpoint.revision + 1 } };
+ }
+}
+
+function assertCopyKey(
+ context: SourceContext,
+ original: OriginalObject,
+ key: string,
) {
+ const prepared = preparedSourcePrefix(context.video, original);
+ assertContextKey(
+ key.startsWith(`${prepared}/`) ? { ...context, prefix: prepared } : context,
+ key,
+ );
+}
+
+function assertReceiptMatchesPlan(
+ object: SourceObject,
+ original: OriginalObject | undefined,
+): asserts original is OriginalObject {
+ if (
+ !original ||
+ object.originalKey !== original.originalKey ||
+ object.originalIdentity !== original.originalIdentity ||
+ object.size !== original.size ||
+ object.index !== original.index ||
+ object.track !== original.track
+ )
+ invalidSource("Recording copy receipt does not match its source plan");
+}
+
+async function advanceSourceSnapshotOnce(
+ context: SourceContext,
+ checkpoint: DesktopRecordingSourceCheckpoint,
+ verification?: RecordingVerification,
+ deadline?: number,
+): Promise<
+ | { checkpoint: DesktopRecordingSourceCheckpoint; listed?: boolean }
+ | { source: DesktopRecordingSource }
+> {
const next = { ...checkpoint, revision: checkpoint.revision + 1 };
if (checkpoint.phase === "plan") {
+ const segmented =
+ context.video.source.type === "desktopSegments" ||
+ verification?.artifact.kind === "segments";
+ const [plan, listing] = await Promise.all([
+ createSourcePlan(context, verification),
+ context.listed && segmented && deadline !== undefined
+ ? listSnapshot(context, false)
+ : null,
+ ]);
return {
- checkpoint: {
- ...next,
- phase: "enumerate" as const,
- plan: await createSourcePlan(context, verification),
- },
+ checkpoint: { ...next, phase: "enumerate" as const, plan },
+ listed: listing !== null,
};
}
if (!checkpoint.plan)
@@ -1258,9 +1512,56 @@ async function advanceSourceSnapshot(
if (!parsedPlan.success) invalidSource("Recording source plan is invalid");
const plan = parsedPlan.data;
const scope = checkpoint.plan.sha256;
+ const listed =
+ context.listed && plan.kind === "segments" && deadline !== undefined;
+ const pageEnd = (cursor: number) =>
+ Math.min(plan.objectCount, cursor + context.pageSize);
+ const positions = (start: number, end: number) =>
+ Array.from({ length: end - start }, (_, index) => start + index);
if (checkpoint.phase === "enumerate") {
if (treeCount(checkpoint.planRoots) !== checkpoint.cursor)
invalidSource("Recording source plan cursor is incomplete");
+ const listing = listed ? await listSnapshot(context, false) : null;
+ if (listing && deadline !== undefined) {
+ let cursor = checkpoint.cursor;
+ let planRoots = checkpoint.planRoots;
+ do {
+ const end = pageEnd(cursor);
+ const originals = await mapBounded(
+ positions(cursor, end),
+ (position) => {
+ const original = originalAt(context.video, plan, position);
+ const found = listing.originals.get(original.originalKey);
+ return found && found.size > 0
+ ? Promise.resolve({
+ ...original,
+ originalIdentity: found.identity,
+ size: found.size,
+ })
+ : captureOriginal(context, original, verification);
+ },
+ context.concurrency,
+ );
+ planRoots = await appendTree(
+ context,
+ planRoots,
+ "plan",
+ scope,
+ originals,
+ );
+ cursor = end;
+ } while (cursor < plan.objectCount && Date.now() < deadline);
+ const done = cursor === plan.objectCount;
+ return {
+ checkpoint: {
+ ...next,
+ planRoots,
+ cursor: done ? 0 : cursor,
+ phase: done ? ("copy" as const) : ("enumerate" as const),
+ },
+ listed: true,
+ };
+ }
const end = Math.min(
plan.objectCount,
checkpoint.cursor + context.pageSize,
@@ -1302,6 +1603,71 @@ async function advanceSourceSnapshot(
invalidSource(
"Recording copy receipts do not match their checkpoint cursor",
);
+ const listing =
+ listed && !checkpoint.multipart
+ ? await listSnapshot(context, true)
+ : null;
+ if (listing && deadline !== undefined) {
+ let cursor = checkpoint.cursor;
+ let receiptRoots = checkpoint.receiptRoots;
+ do {
+ const end = pageEnd(cursor);
+ const originals = await readTree(
+ context,
+ checkpoint.planRoots,
+ "plan",
+ scope,
+ originalObjectSchema,
+ cursor,
+ end,
+ );
+ // Multipart copies keep their own per-object checkpoints, so they stay on the per-page path.
+ if (originals.some((original) => original.size > 5 * 1024 ** 3)) break;
+ const objects = await mapBounded(
+ originals.map((original, index) => ({
+ original,
+ position: cursor + index,
+ })),
+ ({ original, position }) => {
+ const prepared = plan.sourcePreparation === true;
+ const found = findListedCopy(
+ context,
+ listing,
+ original,
+ position,
+ prepared,
+ );
+ if (found === "unlisted")
+ return copyListedObject(context, original, position);
+ return found
+ ? Promise.resolve(found)
+ : copySmallObject(context, original, position, prepared);
+ },
+ context.concurrency,
+ );
+ receiptRoots = await appendTree(
+ context,
+ receiptRoots,
+ "objects",
+ scope,
+ objects,
+ );
+ cursor = end;
+ } while (cursor < plan.objectCount && Date.now() < deadline);
+ if (cursor !== checkpoint.cursor) {
+ const done = cursor === plan.objectCount;
+ return {
+ checkpoint: {
+ ...next,
+ receiptRoots,
+ multipart: undefined,
+ cursor: done ? 0 : cursor,
+ phase: done ? ("verify" as const) : ("copy" as const),
+ },
+ listed: true,
+ };
+ }
+ }
let originals = await readTree(
context,
checkpoint.planRoots,
@@ -1371,6 +1737,71 @@ async function advanceSourceSnapshot(
if (treeCount(checkpoint.receiptRoots) !== plan.objectCount)
invalidSource("Recording snapshot is missing required copy receipts");
if (checkpoint.phase === "verify") {
+ const listing = listed ? await listSnapshot(context, true) : null;
+ if (listing && deadline !== undefined) {
+ let cursor = checkpoint.cursor;
+ do {
+ const end = pageEnd(cursor);
+ const [originals, objects] = await Promise.all([
+ readTree(
+ context,
+ checkpoint.planRoots,
+ "plan",
+ scope,
+ originalObjectSchema,
+ cursor,
+ end,
+ ),
+ readTree(
+ context,
+ checkpoint.receiptRoots,
+ "objects",
+ scope,
+ sourceObjectSchema,
+ cursor,
+ end,
+ ),
+ ]);
+ await mapBounded(
+ objects.map((object, index) => ({
+ object,
+ original: originals[index],
+ })),
+ async ({ object, original }) => {
+ assertReceiptMatchesPlan(object, original);
+ const copy = listing.copies.get(object.key);
+ if (
+ copy?.size === original.size &&
+ copy.identity === object.objectIdentity &&
+ object.objectIdentity === original.originalIdentity &&
+ listedOriginalMatches(listing, original)
+ ) {
+ assertCopyKey(context, original, object.key);
+ return;
+ }
+ await checkCopy(
+ context,
+ original,
+ object.key,
+ object.objectIdentity,
+ );
+ },
+ context.concurrency,
+ );
+ cursor = end;
+ } while (cursor < plan.objectCount && Date.now() < deadline);
+ return {
+ checkpoint: {
+ ...next,
+ cursor,
+ phase:
+ cursor === plan.objectCount
+ ? ("finalize" as const)
+ : ("verify" as const),
+ },
+ listed: true,
+ };
+ }
const end = Math.min(
plan.objectCount,
checkpoint.cursor + context.pageSize,
@@ -1398,17 +1829,7 @@ async function advanceSourceSnapshot(
await mapBounded(
objects.map((object, index) => ({ object, original: originals[index] })),
async ({ object, original }) => {
- if (
- !original ||
- object.originalKey !== original.originalKey ||
- object.originalIdentity !== original.originalIdentity ||
- object.size !== original.size ||
- object.index !== original.index ||
- object.track !== original.track
- )
- invalidSource(
- "Recording copy receipt does not match its source plan",
- );
+ assertReceiptMatchesPlan(object, original);
await checkCopy(context, original, object.key, object.objectIdentity);
},
context.concurrency,
diff --git a/packages/web-backend/src/Storage/index.ts b/packages/web-backend/src/Storage/index.ts
index 8de42a248..934fb3f63 100644
--- a/packages/web-backend/src/Storage/index.ts
+++ b/packages/web-backend/src/Storage/index.ts
@@ -439,6 +439,7 @@ const makeS3Access = (s3: S3BucketAccess) => ({
Contents: result.Contents?.map((object) => ({
Key: object.Key,
Size: object.Size,
+ ETag: object.ETag,
LastModified: object.LastModified,
})),
KeyCount: result.KeyCount,
@@ -462,6 +463,14 @@ const makeS3Access = (s3: S3BucketAccess) => ({
body: Parameters<S3BucketAccess["putObject"]>[1],
fields?: Parameters<S3BucketAccess["putObject"]>[2],
) => mapStorageError(s3.putObject(key, body, fields)).pipe(Effect.asVoid),
+ putObjectWithETag: (
+ key: string,
+ body: string,
+ fields?: Parameters<S3BucketAccess["putObject"]>[2],
+ ) =>
+ mapStorageError(s3.putObject(key, body, fields)).pipe(
+ Effect.map((result) => ({ ETag: result.ETag })),
+ ),
copyObject: (
source: string,
key: string,
@@ -924,6 +933,7 @@ const makeGoogleDriveAccess = ({
.map((object) => ({
Key: object.objectKey,
Size: object.contentLength ?? undefined,
+ ETag: undefined,
LastModified: object.updatedAt,
}));
return {