Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(web): commit recording source snapshots in larger parallel pages

Richie McIlroyb71df390c94e
diff --git a/apps/web/__tests__/integration/desktop-recording-production-replay.test.ts b/apps/web/__tests__/integration/desktop-recording-production-replay.test.ts
index 4f810487c..0e223ed7c 100644
--- a/apps/web/__tests__/integration/desktop-recording-production-replay.test.ts
+++ b/apps/web/__tests__/integration/desktop-recording-production-replay.test.ts
@@ -165,7 +165,7 @@ function replay(sample: Sample) {
getInternalSignedObjectUrl: (name: string) =>
checked(() => `https://replay.invalid/${name}`),
};
- mocks.storage.mockReturnValue(Effect.succeed([bucket]));
+ mocks.storage.mockReturnValue(Effect.succeed([bucket, Option.none()]));
const manifest = (complete: boolean) => {
const parsed = readCompletedRecordingManifest(
JSON.stringify(sample.manifest),
diff --git a/apps/web/__tests__/unit/desktop-recording-source.test.ts b/apps/web/__tests__/unit/desktop-recording-source.test.ts
index 023f65f50..fba603ce6 100644
--- a/apps/web/__tests__/unit/desktop-recording-source.test.ts
+++ b/apps/web/__tests__/unit/desktop-recording-source.test.ts
@@ -7,11 +7,26 @@ import { z } from "zod";
import type { DesktopRecordingSourceCheckpoint } from "@/lib/desktop-recording-source-checkpoint";
import type { RecordingVerification } from "@/lib/desktop-recording-verification";
-const mocks = vi.hoisted(() => ({ storage: vi.fn() }));
+const mocks = vi.hoisted(() => ({
+ storage: vi.fn(),
+ pageSize: undefined as number | undefined,
+}));
vi.mock("@cap/web-backend/src/Storage/index", () => ({
Storage: { getAccessForVideo: mocks.storage },
}));
+vi.mock("@/lib/desktop-recording-source-checkpoint", async (importOriginal) => {
+ const actual =
+ await importOriginal<
+ typeof import("@/lib/desktop-recording-source-checkpoint")
+ >();
+ return {
+ ...actual,
+ get SOURCE_COMMIT_PAGE_SIZE() {
+ return mocks.pageSize ?? actual.SOURCE_COMMIT_PAGE_SIZE;
+ },
+ };
+});
vi.mock("@/lib/workflow-runtime", async () => {
const { Effect } = await import("effect");
return { runWorkflowPromise: Effect.runPromise };
@@ -99,7 +114,7 @@ function recording(source: VideoRow["source"] = { type: "desktopSegments" }) {
return { id: "owned-video", ownerId: "owned-user", source } as VideoRow;
}
-function storageFixture(provider = "s3") {
+function storageFixture(provider = "s3", customBucket = false) {
const objects = new Map<string, StoredObject>();
const uploads = new Map<
string,
@@ -311,7 +326,12 @@ function storageFixture(provider = "s3") {
),
},
};
- mocks.storage.mockReturnValue(Effect.succeed([bucket]));
+ mocks.storage.mockReturnValue(
+ Effect.succeed([
+ bucket,
+ customBucket ? Option.some({ id: "custom-bucket" }) : Option.none(),
+ ]),
+ );
return { bucket, objects, uploads, seed, object };
}
@@ -422,6 +442,7 @@ function seedSegments(
beforeEach(() => {
vi.clearAllMocks();
+ mocks.pageSize = undefined;
});
describe("recording source preparation during upload", () => {
@@ -1375,7 +1396,7 @@ describe("large desktop recording snapshots", () => {
const storage = storageFixture(provider);
seedSegments(
storage,
- Array.from({ length: 33 }, (_, index) => index + 1),
+ Array.from({ length: 129 }, (_, index) => index + 1),
false,
);
const video = recording();
@@ -1389,19 +1410,19 @@ describe("large desktop recording snapshots", () => {
checkpoint,
);
if (!("checkpoint" in copied)) throw new Error("Unexpected completion");
- expect(copied.checkpoint.cursor).toBe(16);
- expect(storage.bucket.copyObject).toHaveBeenCalledTimes(16);
+ expect(copied.checkpoint.cursor).toBe(64);
+ expect(storage.bucket.copyObject).toHaveBeenCalledTimes(64);
const replay = await advanceDesktopRecordingSourceCommit(
video,
checkpoint,
);
if (!("checkpoint" in replay)) throw new Error("Unexpected completion");
- expect(storage.bucket.copyObject).toHaveBeenCalledTimes(16);
+ expect(storage.bucket.copyObject).toHaveBeenCalledTimes(64);
expect(replay.checkpoint.snapshotId).toBe(checkpoint.snapshotId);
const source = await finishSnapshot(video, replay.checkpoint);
const urls = await buildDesktopRecordingSourceUrls(video, source);
- expect(urls.sourceObjects).toHaveLength(34);
- expect(storage.bucket.copyObject).toHaveBeenCalledTimes(34);
+ expect(urls.sourceObjects).toHaveLength(130);
+ expect(storage.bucket.copyObject).toHaveBeenCalledTimes(130);
expect(JSON.stringify(replay.checkpoint).length).toBeLessThan(5_000);
},
);
@@ -1627,6 +1648,239 @@ describe("large desktop recording snapshots", () => {
});
});
+describe("source commit page size", () => {
+ const segmentCount = 130;
+
+ function pagedFixture(provider = "s3", customBucket = false) {
+ const storage = storageFixture(provider, customBucket);
+ seedSegments(
+ storage,
+ Array.from({ length: segmentCount }, (_, index) => index + 1),
+ );
+ return storage;
+ }
+
+ async function advanceWithPageSize(
+ pageSize: number,
+ checkpoint: DesktopRecordingSourceCheckpoint,
+ done: (checkpoint: DesktopRecordingSourceCheckpoint) => boolean = () =>
+ false,
+ ) {
+ mocks.pageSize = pageSize;
+ let checkpoints = 0;
+ while (!done(checkpoint)) {
+ const result = await advanceDesktopRecordingSourceCommit(
+ recording(),
+ checkpoint,
+ );
+ checkpoints++;
+ if ("source" in result)
+ return { source: result.source, checkpoint, checkpoints };
+ checkpoint = result.checkpoint;
+ }
+ return { source: undefined, checkpoint, checkpoints };
+ }
+
+ function leafSizes(
+ storage: ReturnType<typeof storageFixture>,
+ inventoryKey: string,
+ ) {
+ const page = z.discriminatedUnion("type", [
+ z.object({ type: z.literal("leaf"), entries: z.array(z.unknown()) }),
+ z.object({
+ type: z.literal("branch"),
+ children: z.array(z.object({ key: z.string() })),
+ }),
+ ]);
+ const sizes = (key: string): number[] => {
+ const node = page.parse(JSON.parse(storage.object(key).body ?? ""));
+ return node.type === "leaf"
+ ? [node.entries.length]
+ : node.children.flatMap((child) => sizes(child.key));
+ };
+ return z
+ .object({ roots: z.array(z.object({ key: z.string() })) })
+ .parse(JSON.parse(storage.object(inventoryKey).body ?? ""))
+ .roots.flatMap((root) => sizes(root.key));
+ }
+
+ async function snapshotOutput(
+ storage: ReturnType<typeof storageFixture>,
+ source: Awaited<ReturnType<typeof commitDesktopRecordingSource>>,
+ ) {
+ const objects = readInventory(storage, source.inventoryKey).objects.map(
+ (object) => {
+ const copy = storage.object(object.key);
+ expect(copy.identity).toBe(object.objectIdentity);
+ const receiptKey = object.key.replace(
+ /\/(video|audio)\/\d+\.(mp4|m4s)$/,
+ "/receipt.json",
+ );
+ expect(JSON.parse(storage.object(receiptKey).body ?? "")).toEqual(
+ object,
+ );
+ return {
+ key: object.key.replace(/\/copies\/(\d+)\/[0-9a-f-]{36}\//, "/$1/"),
+ originalKey: object.originalKey,
+ originalIdentity: object.originalIdentity,
+ size: object.size,
+ track: object.track,
+ index: object.index,
+ body: copy.body,
+ };
+ },
+ );
+ const urls = await buildDesktopRecordingSourceUrls(recording(), source);
+ return {
+ source: {
+ kind: source.kind,
+ manifestSha256: source.manifestSha256,
+ requiredAudio: source.requiredAudio,
+ },
+ objects,
+ urls: {
+ sourceObjects: urls.sourceObjects.map(({ size }) => size),
+ videoSegments: urls.videoSegmentUrls.length,
+ audioSegments: urls.audioSegmentUrls.length,
+ videoInit: urls.videoInitUrl !== undefined,
+ audioInit: urls.audioInitUrl !== undefined,
+ },
+ copies: storage.bucket.copyObject.mock.calls.length,
+ };
+ }
+
+ async function referenceOutput(pageSize: number) {
+ const storage = pagedFixture();
+ const result = await advanceWithPageSize(pageSize, snapshotCheckpoint());
+ if (!result.source) throw new Error("Snapshot did not complete");
+ return {
+ output: await snapshotOutput(storage, result.source),
+ checkpoints: result.checkpoints,
+ leaves: leafSizes(storage, result.source.inventoryKey),
+ };
+ }
+
+ it("uses 64-entry pages, which the leaf schema already accepts", async () => {
+ const actual = await vi.importActual<
+ typeof import("@/lib/desktop-recording-source-checkpoint")
+ >("@/lib/desktop-recording-source-checkpoint");
+ expect(actual.SOURCE_COMMIT_PAGE_SIZE).toBe(64);
+ expect(actual.SOURCE_COMMIT_PAGE_SIZE).toBeLessThanOrEqual(
+ actual.SOURCE_COMMIT_PART_PAGE_SIZE,
+ );
+ });
+
+ it("commits more than 64 objects with the same inventory and receipts as 16-entry pages", async () => {
+ const legacy = await referenceOutput(16);
+ vi.clearAllMocks();
+ const current = await referenceOutput(64);
+ const objectCount = segmentCount + 3;
+ expect(current.output.objects).toHaveLength(objectCount);
+ expect(current.output).toEqual(legacy.output);
+ expect(current.output.copies).toBe(objectCount);
+ expect(legacy.checkpoints).toBe(2 + 3 * Math.ceil(objectCount / 16));
+ expect(current.checkpoints).toBe(2 + 3 * Math.ceil(objectCount / 64));
+ expect(legacy.leaves.every((size) => size <= 16)).toBe(true);
+ expect(current.leaves).toEqual([64, 64, objectCount - 128]);
+ });
+
+ for (const [from, to] of [
+ [16, 64],
+ [64, 16],
+ ] as const) {
+ it.each(["enumerate", "copy", "verify"] as const)(
+ `resumes a %s checkpoint saved with ${from}-entry pages after switching to ${to}`,
+ async (phase) => {
+ const reference = await referenceOutput(64);
+ vi.clearAllMocks();
+ const storage = pagedFixture();
+ const saved = await advanceWithPageSize(
+ from,
+ snapshotCheckpoint(),
+ (checkpoint) =>
+ checkpoint.phase === phase && checkpoint.cursor === 64,
+ );
+ expect(saved.source).toBeUndefined();
+ const next = await advanceDesktopRecordingSourceCommit(
+ recording(),
+ saved.checkpoint,
+ );
+ if (!("checkpoint" in next)) throw new Error("Unexpected completion");
+ expect(next.checkpoint).toMatchObject({
+ phase,
+ cursor: 64 + from,
+ plan: saved.checkpoint.plan,
+ snapshotId: saved.checkpoint.snapshotId,
+ });
+ const resumed = await advanceWithPageSize(to, next.checkpoint);
+ if (!resumed.source) throw new Error("Snapshot did not complete");
+ expect(await snapshotOutput(storage, resumed.source)).toEqual(
+ reference.output,
+ );
+ const leaves = leafSizes(storage, resumed.source.inventoryKey);
+ expect(leaves.reduce((total, size) => total + size, 0)).toBe(
+ segmentCount + 3,
+ );
+ if (phase !== "verify") expect(new Set(leaves).size).toBeGreaterThan(1);
+ },
+ );
+ }
+
+ it.each([
+ ["Cap's AWS bucket", "s3", false, 32],
+ ["a custom S3-compatible bucket", "s3", true, 8],
+ ["Google Drive", "googleDrive", false, 8],
+ ] as const)(
+ "bounds parallel storage requests for %s",
+ async (_, provider, customBucket, expected) => {
+ const storage = pagedFixture(provider, customBucket);
+ const active = { head: 0, copy: 0 };
+ const peak = { head: 0, copy: 0 };
+ const track = <A, E>(
+ name: "head" | "copy",
+ effect: Effect.Effect<A, E>,
+ ) =>
+ Effect.suspend(() => {
+ active[name]++;
+ peak[name] = Math.max(peak[name], active[name]);
+ return Effect.sleep("1 millis").pipe(
+ Effect.zipRight(effect),
+ Effect.ensuring(
+ Effect.sync(() => {
+ active[name]--;
+ }),
+ ),
+ );
+ });
+ const head = storage.bucket.headObject.getMockImplementation();
+ const copy = storage.bucket.copyObject.getMockImplementation();
+ if (!head || !copy) throw new Error("Missing storage fixture");
+ const enumerate = await advanceToPhase(
+ recording(),
+ snapshotCheckpoint(),
+ "enumerate",
+ );
+ storage.bucket.headObject.mockImplementation((key) =>
+ track("head", head(key)),
+ );
+ storage.bucket.copyObject.mockImplementation((source, target, options) =>
+ track("copy", copy(source, target, options)),
+ );
+ const planned = await advanceDesktopRecordingSourceCommit(
+ recording(),
+ enumerate,
+ );
+ if (!("checkpoint" in planned)) throw new Error("Unexpected completion");
+ expect(peak.head).toBe(expected);
+ const source = await finishSnapshot(recording(), planned.checkpoint);
+ expect(peak.copy).toBe(expected);
+ expect(readInventory(storage, source.inventoryKey).objects).toHaveLength(
+ segmentCount + 3,
+ );
+ },
+ );
+});
+
describe("committed source access", () => {
it("classifies malformed committed inventory JSON as a permanent source error", async () => {
const storage = storageFixture();
diff --git a/apps/web/__tests__/unit/s3-bucket-connections.test.ts b/apps/web/__tests__/unit/s3-bucket-connections.test.ts
index 07df81095..501751e4b 100644
--- a/apps/web/__tests__/unit/s3-bucket-connections.test.ts
+++ b/apps/web/__tests__/unit/s3-bucket-connections.test.ts
@@ -40,7 +40,10 @@ vi.mock("@cap/web-backend/src/S3Buckets/S3BucketsRepo.ts", async () => {
});
import { S3Buckets } from "@cap/web-backend/src/S3Buckets";
-import { s3ConnectionPool } from "@cap/web-backend/src/S3Buckets/S3ConnectionPool";
+import {
+ defaultBucketS3ConnectionPool,
+ s3ConnectionPool,
+} from "@cap/web-backend/src/S3Buckets/S3ConnectionPool";
async function storageFixture() {
let connections = 0;
@@ -193,6 +196,40 @@ describe("S3 connection reuse", () => {
}
});
+ it("keeps default bucket connections warm across 32-wide snapshot pages", async () => {
+ const fixture = await storageFixture();
+ try {
+ const [access] = await fixture.runtime.runPromise(
+ fixture.service.getBucketAccess(Option.none()),
+ );
+ for (let checkpoint = 0; checkpoint < 10; checkpoint++) {
+ await Promise.all(
+ Array.from({ length: 32 }, (_, index) =>
+ fixture.runtime.runPromise(
+ access.headObject(`recording/${checkpoint}/${index}`),
+ ),
+ ),
+ );
+ }
+ expect(fixture.authorizations).toHaveLength(320);
+ expect(fixture.connections()).toBeLessThanOrEqual(40);
+ } finally {
+ await fixture.close();
+ }
+ });
+
+ it("keeps the custom bucket idle connection bound", async () => {
+ const pools = await Effect.runPromise(
+ Effect.scoped(
+ Effect.all([s3ConnectionPool, defaultBucketS3ConnectionPool]),
+ ),
+ );
+ expect(pools.map((pool) => pool.httpsAgent.maxFreeSockets)).toEqual([
+ 8, 32,
+ ]);
+ expect(pools.map((pool) => pool.httpsAgent.maxSockets)).toEqual([50, 50]);
+ });
+
it("reuses custom bucket connections while using current credentials", async () => {
const fixture = await storageFixture();
try {
diff --git a/apps/web/lib/desktop-recording-source-checkpoint.ts b/apps/web/lib/desktop-recording-source-checkpoint.ts
index 328a6d253..a30539304 100644
--- a/apps/web/lib/desktop-recording-source-checkpoint.ts
+++ b/apps/web/lib/desktop-recording-source-checkpoint.ts
@@ -1,6 +1,6 @@
import { z } from "zod";
-export const SOURCE_COMMIT_PAGE_SIZE = 16;
+export const SOURCE_COMMIT_PAGE_SIZE = 64;
export const SOURCE_COMMIT_PART_BATCH_SIZE = 4;
export const SOURCE_COMMIT_PART_PAGE_SIZE = 64;
export const SOURCE_COMMIT_MAX_OBJECTS = 100_002;
diff --git a/apps/web/lib/desktop-recording-source.ts b/apps/web/lib/desktop-recording-source.ts
index 514421bb0..ef748f26f 100644
--- a/apps/web/lib/desktop-recording-source.ts
+++ b/apps/web/lib/desktop-recording-source.ts
@@ -60,9 +60,10 @@ export type DesktopRecordingSource = z.infer<
>;
type SourceObject = z.infer<typeof sourceObjectSchema>;
type DbVideo = typeof videos.$inferSelect;
-type RecordingStorage = Effect.Effect.Success<
+type RecordingStorageAccess = Effect.Effect.Success<
ReturnType<typeof Storage.getAccessForVideo>
->[0];
+>;
+type RecordingStorage = RecordingStorageAccess[0];
const sourceInventorySchema = z.object({
version: z.literal(1),
@@ -199,6 +200,7 @@ type SourceContext = {
bucket: RecordingStorage;
prefix: string;
run: SourceRun;
+ concurrency: number;
};
type TreeKind = "plan" | "objects" | "parts";
@@ -256,6 +258,14 @@ const committedInventorySchema = z.union([
]);
const SOURCE_COMMIT_STEP_TIMEOUT_MS = 180_000;
const SOURCE_COMMIT_IO_TIMEOUT_MS = 60_000;
+const SOURCE_COMMIT_CONCURRENCY = 8;
+const SOURCE_COMMIT_AWS_CONCURRENCY = 32;
+
+function sourceConcurrency([bucket, customBucket]: RecordingStorageAccess) {
+ return bucket.provider === "s3" && Option.isNone(customBucket)
+ ? SOURCE_COMMIT_AWS_CONCURRENCY
+ : SOURCE_COMMIT_CONCURRENCY;
+}
function invalidSource(message: string): never {
throw new DesktopRecordingSourceError("source-invalid", message);
@@ -272,12 +282,13 @@ function parseSourceJson(content: string): unknown {
async function mapBounded<T, R>(
items: T[],
operation: (item: T) => Promise<R>,
+ concurrency = SOURCE_COMMIT_CONCURRENCY,
) {
const output = new Array<R>(items.length);
const entries = items.entries();
let failure: unknown;
await Promise.all(
- Array.from({ length: Math.min(8, items.length) }, async () => {
+ Array.from({ length: Math.min(concurrency, items.length) }, async () => {
while (failure === undefined) {
const next = entries.next();
if (next.done) return;
@@ -424,51 +435,57 @@ async function readTree<T>(
let pending = roots.filter(overlaps);
const leaves: { start: number; entries: T[] }[] = [];
while (pending.length > 0) {
- const pages = await mapBounded(pending, async (reference) => {
- const parsed = treePageSchema.safeParse(
- await readReference(context, reference),
- );
- if (!parsed.success) invalidSource("Recording source page is invalid");
- const page = parsed.data;
- if (page.kind !== kind || page.scope !== scope)
- invalidSource("Recording source page belongs to a different snapshot");
- if (page.type === "leaf") {
+ const pages = await mapBounded(
+ pending,
+ async (reference) => {
+ const parsed = treePageSchema.safeParse(
+ await readReference(context, reference),
+ );
+ if (!parsed.success) invalidSource("Recording source page is invalid");
+ const page = parsed.data;
+ if (page.kind !== kind || page.scope !== scope)
+ invalidSource(
+ "Recording source page belongs to a different snapshot",
+ );
+ if (page.type === "leaf") {
+ if (
+ reference.level !== 0 ||
+ page.start !== reference.start ||
+ page.entries.length !== reference.count
+ )
+ invalidSource(
+ "Recording source leaf does not match its committed range",
+ );
+ return {
+ children: [],
+ leaf: {
+ start: page.start,
+ entries: page.entries.map((entry) => {
+ const parsed = schema.safeParse(entry);
+ if (!parsed.success)
+ invalidSource(
+ "Recording source page contains an invalid receipt",
+ );
+ return parsed.data;
+ }),
+ },
+ };
+ }
+ const [left, right] = page.children;
if (
- reference.level !== 0 ||
- page.start !== reference.start ||
- page.entries.length !== reference.count
+ left.start !== reference.start ||
+ left.start + left.count !== right.start ||
+ left.count + right.count !== reference.count ||
+ left.level + 1 !== reference.level ||
+ right.level !== left.level
)
invalidSource(
- "Recording source leaf does not match its committed range",
+ "Recording source branch does not match its committed range",
);
- return {
- children: [],
- leaf: {
- start: page.start,
- entries: page.entries.map((entry) => {
- const parsed = schema.safeParse(entry);
- if (!parsed.success)
- invalidSource(
- "Recording source page contains an invalid receipt",
- );
- return parsed.data;
- }),
- },
- };
- }
- const [left, right] = page.children;
- if (
- left.start !== reference.start ||
- left.start + left.count !== right.start ||
- left.count + right.count !== reference.count ||
- left.level + 1 !== reference.level ||
- right.level !== left.level
- )
- invalidSource(
- "Recording source branch does not match its committed range",
- );
- return { children: page.children.filter(overlaps), leaf: null };
- });
+ return { children: page.children.filter(overlaps), leaf: null };
+ },
+ context.concurrency,
+ );
pending = [];
for (const page of pages) {
pending.push(...page.children);
@@ -922,6 +939,7 @@ export async function prepareDesktopRecordingSegments(
bucket,
run,
prefix: sourcePrefix(video).slice(0, -1),
+ concurrency: SOURCE_COMMIT_CONCURRENCY,
};
const unique = [
...new Map(
@@ -1166,6 +1184,7 @@ async function advanceSourceSnapshot(
originalAt(context.video, plan, position),
verification,
),
+ context.concurrency,
);
const planRoots = await appendTree(
context,
@@ -1235,6 +1254,7 @@ async function advanceSourceSnapshot(
position,
plan.sourcePreparation === true,
),
+ context.concurrency,
);
}
const receiptRoots = await appendTree(
@@ -1299,6 +1319,7 @@ async function advanceSourceSnapshot(
);
await checkCopy(context, original, object.key, object.objectIdentity);
},
+ context.concurrency,
);
return {
checkpoint: {
@@ -1419,7 +1440,7 @@ export async function advanceDesktopRecordingSourceCommit(
);
};
try {
- const [bucket] = await run(
+ const access = await run(
Storage.getAccessForVideo(decodeStorageVideo(video), {
resolvePublishedOutput: false,
}),
@@ -1427,9 +1448,10 @@ export async function advanceDesktopRecordingSourceCommit(
const result = await advanceSourceSnapshot(
{
video,
- bucket,
+ bucket: access[0],
run,
prefix: `${sourcePrefix(video)}${parsed.generation}/${parsed.snapshotId}`,
+ concurrency: sourceConcurrency(access),
},
parsed,
verification,
@@ -1480,9 +1502,10 @@ export async function buildDesktopRecordingSourceUrls(
source: DesktopRecordingSource,
) {
assertSourceKey(video, source.inventoryKey);
- const [bucket] = await runWorkflowPromise(
+ const access = await runWorkflowPromise(
Storage.getAccessForVideo(decodeStorageVideo(video)),
);
+ const [bucket] = access;
const content = await readRequiredObject(bucket, source.inventoryKey);
if (hash(content) !== source.inventorySha256) {
throw new DesktopRecordingSourceError(
@@ -1526,6 +1549,7 @@ export async function buildDesktopRecordingSourceUrls(
bucket,
run,
prefix: sourcePrefix(video).slice(0, -1),
+ concurrency: sourceConcurrency(access),
},
inventory.roots,
"objects",
diff --git a/packages/web-backend/src/S3Buckets/S3ConnectionPool.ts b/packages/web-backend/src/S3Buckets/S3ConnectionPool.ts
index be0a15c8a..af3ff28eb 100644
--- a/packages/web-backend/src/S3Buckets/S3ConnectionPool.ts
+++ b/packages/web-backend/src/S3Buckets/S3ConnectionPool.ts
@@ -20,22 +20,26 @@ function boundIdleConnections<T extends HttpAgent>(agent: T) {
return agent;
}
-export const s3ConnectionPool = Effect.acquireRelease(
- Effect.sync(() => {
- const options = {
- keepAlive: true,
- maxSockets: 50,
- maxTotalSockets: 128,
- maxFreeSockets: 8,
- };
- return {
- httpAgent: boundIdleConnections(new HttpAgent(options)),
- httpsAgent: boundIdleConnections(new HttpsAgent(options)),
- };
- }),
- (pool) =>
+const makeS3ConnectionPool = (maxFreeSockets: number) =>
+ Effect.acquireRelease(
Effect.sync(() => {
- pool.httpAgent.destroy();
- pool.httpsAgent.destroy();
+ const options = {
+ keepAlive: true,
+ maxSockets: 50,
+ maxTotalSockets: 128,
+ maxFreeSockets,
+ };
+ return {
+ httpAgent: boundIdleConnections(new HttpAgent(options)),
+ httpsAgent: boundIdleConnections(new HttpsAgent(options)),
+ };
}),
-);
+ (pool) =>
+ Effect.sync(() => {
+ pool.httpAgent.destroy();
+ pool.httpsAgent.destroy();
+ }),
+ );
+
+export const s3ConnectionPool = makeS3ConnectionPool(8);
+export const defaultBucketS3ConnectionPool = makeS3ConnectionPool(32);
diff --git a/packages/web-backend/src/S3Buckets/index.ts b/packages/web-backend/src/S3Buckets/index.ts
index 49ecb912e..08c24be02 100644
--- a/packages/web-backend/src/S3Buckets/index.ts
+++ b/packages/web-backend/src/S3Buckets/index.ts
@@ -10,13 +10,17 @@ import { Database } from "../Database.ts";
import { createS3BucketAccess } from "./S3BucketAccess.ts";
import { S3BucketClientProvider } from "./S3BucketClientProvider.ts";
import { S3BucketsRepo } from "./S3BucketsRepo.ts";
-import { s3ConnectionPool } from "./S3ConnectionPool.ts";
+import {
+ defaultBucketS3ConnectionPool,
+ s3ConnectionPool,
+} from "./S3ConnectionPool.ts";
export class S3Buckets extends Effect.Service<S3Buckets>()("S3Buckets", {
scoped: Effect.gen(function* () {
const repo = yield* S3BucketsRepo;
const { credentials } = yield* AwsCredentials;
const requestHandler = yield* s3ConnectionPool;
+ const defaultBucketRequestHandler = yield* defaultBucketS3ConnectionPool;
const defaultConfigs = {
publicEndpoint: yield* Config.string("S3_PUBLIC_ENDPOINT").pipe(
@@ -45,7 +49,7 @@ export class S3Buckets extends Effect.Service<S3Buckets>()("S3Buckets", {
credentials: defaultConfigs.credentials,
forcePathStyle: defaultConfigs.forcePathStyle,
requestStreamBufferSize: 16 * 1024,
- requestHandler,
+ requestHandler: defaultBucketRequestHandler,
});
const defaultInternalClient = createDefaultClient(true);
const defaultPublicClient = createDefaultClient(false);