Skip to content

Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.

perf: prepare content collections concurrently (#6412)

Oleg Isonenade7a3056056
diff --git a/packages/asset-uploader/src/asset-repository.test.ts b/packages/asset-uploader/src/asset-repository.test.ts
index 22e948fc39..7b671e866f 100644
--- a/packages/asset-uploader/src/asset-repository.test.ts
+++ b/packages/asset-uploader/src/asset-repository.test.ts
@@ -204,6 +204,75 @@ describe("PostgresAssetRepository", () => {
);
});
+ test("validates independent collection folders concurrently", async () => {
+ const configSource = createDefaultCollectionConfig();
+ const templateSource = "---\ndraft: true\n---\n\nStart writing.\n";
+ const assets: Asset[] = [];
+ const sources = new Map<string, string>();
+ for (let index = 0; index < 9; index += 1) {
+ const folderId = `collection-${index}`;
+ const configName = `config-${index}.json`;
+ const templateName = `template-${index}.mdx`;
+ assets.push(
+ {
+ id: configName,
+ projectId: "project-1",
+ name: configName,
+ filename: "collection",
+ folderId,
+ type: "file",
+ format: "json",
+ size: configSource.length,
+ description: null,
+ createdAt: "2026-09-03T00:00:00.000Z",
+ meta: {},
+ },
+ {
+ id: templateName,
+ projectId: "project-1",
+ name: templateName,
+ filename: "template",
+ folderId,
+ type: "file",
+ format: "mdx",
+ size: templateSource.length,
+ description: null,
+ createdAt: "2026-09-03T00:00:00.000Z",
+ meta: {},
+ }
+ );
+ sources.set(configName, configSource);
+ sources.set(templateName, templateSource);
+ }
+ let activeReads = 0;
+ let maximumActiveReads = 0;
+ const repository = new PostgresAssetRepository({
+ projectId: "project-1",
+ context,
+ assetStore: {
+ readFile: async (name) => {
+ const source = sources.get(name);
+ if (source === undefined) {
+ throw new Error(`Unexpected read: ${name}`);
+ }
+ activeReads += 1;
+ maximumActiveReads = Math.max(maximumActiveReads, activeReads);
+ await new Promise((resolve) => setTimeout(resolve, 1));
+ activeReads -= 1;
+ return {
+ data: new Blob([source]).stream(),
+ contentLength: source.length,
+ };
+ },
+ },
+ dependencies: createDependencies(),
+ });
+
+ await repository.validateCollections(assets);
+
+ expect(maximumActiveReads).toBeGreaterThan(1);
+ });
+
test.each([
{ name: "notes.txt", extension: "mdx", collection: true },
{ name: "template.txt", extension: "mdx", collection: true },
@@ -3429,6 +3498,77 @@ describe("PostgresAssetRepository", () => {
);
});
+ test("inspects independent collection folders concurrently", async () => {
+ const dependencies = createDependencies();
+ const configSource = createDefaultCollectionConfig();
+ const templateSource = "---\ndraft: true\n---\nTemplate\n";
+ const entries: CanonicalAssetFileEntry[] = [];
+ const sources = new Map<string, string>();
+ for (let index = 0; index < 9; index += 1) {
+ const folderId = `collection-${index}`;
+ for (const [name, mimeType, source] of [
+ ["collection.json", "application/json", configSource],
+ ["template.mdx", "text/mdx", templateSource],
+ ] as const) {
+ const id = `${folderId}-${name}`;
+ entries.push({
+ projectId: "project-1",
+ assetId: id,
+ revision: `revision-${id}`,
+ document: {
+ _id: id,
+ _type: "asset.file",
+ name,
+ path: `${folderId}/${name}`,
+ key: name.slice(0, name.lastIndexOf(".")),
+ extension: name.slice(name.lastIndexOf(".") + 1),
+ folderId,
+ mimeType,
+ size: source.length,
+ revision: `revision-${id}`,
+ contentRef: id,
+ properties: {},
+ },
+ });
+ sources.set(id, source);
+ }
+ }
+ dependencies.loadCanonicalAssetBaseEntries.mockResolvedValue(entries);
+ dependencies.createAssetIndex.mockResolvedValue({} as never);
+ let activeReads = 0;
+ let maximumActiveReads = 0;
+ const repository = new PostgresAssetRepository({
+ projectId: "project-1",
+ context,
+ assetStore: {
+ readFile: async (name) => {
+ const source = sources.get(name);
+ if (source === undefined) {
+ throw new Error(`Unexpected read: ${name}`);
+ }
+ activeReads += 1;
+ maximumActiveReads = Math.max(maximumActiveReads, activeReads);
+ await new Promise((resolve) => setTimeout(resolve, 1));
+ activeReads -= 1;
+ return {
+ data: new Blob([source]).stream(),
+ contentLength: source.length,
+ };
+ },
+ },
+ dependencies,
+ });
+
+ await repository.prepareIndex(
+ createCompilationPlan({
+ where: { all: [] },
+ output: { mode: "base", includeMetadata: true },
+ })
+ );
+
+ expect(maximumActiveReads).toBeGreaterThan(1);
+ });
+
test("rejects a truncated collection config while preparing an index", async () => {
const dependencies = createDependencies();
const configSource = createDefaultCollectionConfig();
diff --git a/packages/asset-uploader/src/asset-repository.ts b/packages/asset-uploader/src/asset-repository.ts
index f2abc670cc..a3bf2d12e0 100644
--- a/packages/asset-uploader/src/asset-repository.ts
+++ b/packages/asset-uploader/src/asset-repository.ts
@@ -50,6 +50,7 @@ import {
fullCanonicalAssetMetadataRequirements,
materializeContentSource,
materializeContentSnapshot,
+ mapBounded,
ContentSourceChangedError,
DocumentSourceDiagnosticsError,
readBoundedBytes,
@@ -2107,14 +2108,17 @@ export class PostgresAssetRepository implements AssetRepository {
async validateCollections(assets: readonly Asset[]) {
await this.assertCanBuild();
- for (const folderId of getCollectionFolderIds(assets)) {
- await validateCollectionFolder({
- assets,
- folderId,
- assetStore: this.assetStore,
- validateEntries: false,
- });
- }
+ await mapBounded(
+ [...getCollectionFolderIds(assets)],
+ contentEngineLimits.concurrentContentReads,
+ (folderId) =>
+ validateCollectionFolder({
+ assets,
+ folderId,
+ assetStore: this.assetStore,
+ validateEntries: false,
+ })
+ );
}
private async synchronizeTrusted(
@@ -2380,50 +2384,59 @@ export class PostgresAssetRepository implements AssetRepository {
: []
)
);
- for (const folderId of folderIds) {
- const siblings = entries.filter(
- (candidate) => candidate.document.folderId === folderId
- );
- try {
- const inspected = await inspectContentCollection({
- files: siblings.map((entry) => ({
- file: entry,
- id: entry.assetId,
- filename: entry.document.name,
- basename: entry.document.key,
- isMdx: entry.document.extension.toLowerCase() === "mdx",
- })),
- readSource: async ({ file: entry }) => {
- if (entry.document.size > contentEngineLimits.hydratedFileBytes) {
- throw new ContentCollectionError(
- `Collection file "${entry.document.name}" exceeds the content size limit`
- );
- }
- const response = await readFile(entry.document.contentRef);
- const bytes = await readBoundedBytes(
- response.data,
- contentEngineLimits.hydratedFileBytes
- );
- if (bytes.byteLength !== entry.document.size) {
- throw new ContentCollectionError(
- `Collection file "${entry.document.name}" content length does not match its metadata`
+ const inspectedCollections = await mapBounded(
+ [...folderIds],
+ contentEngineLimits.concurrentContentReads,
+ async (folderId) => {
+ const siblings = entries.filter(
+ (candidate) => candidate.document.folderId === folderId
+ );
+ try {
+ const inspected = await inspectContentCollection({
+ files: siblings.map((entry) => ({
+ file: entry,
+ id: entry.assetId,
+ filename: entry.document.name,
+ basename: entry.document.key,
+ isMdx: entry.document.extension.toLowerCase() === "mdx",
+ })),
+ readSource: async ({ file: entry }) => {
+ if (
+ entry.document.size > contentEngineLimits.hydratedFileBytes
+ ) {
+ throw new ContentCollectionError(
+ `Collection file "${entry.document.name}" exceeds the content size limit`
+ );
+ }
+ const response = await readFile(entry.document.contentRef);
+ const bytes = await readBoundedBytes(
+ response.data,
+ contentEngineLimits.hydratedFileBytes
);
- }
- return decodeUtf8(bytes);
- },
- validateTemplate: false,
- validateEntries: false,
- });
- reservedAssetIds.add(inspected.configFile.id);
- reservedAssetIds.add(inspected.templateFile.id);
- } catch (error) {
- if (error instanceof ContentCollectionError) {
- throw error;
+ if (bytes.byteLength !== entry.document.size) {
+ throw new ContentCollectionError(
+ `Collection file "${entry.document.name}" content length does not match its metadata`
+ );
+ }
+ return decodeUtf8(bytes);
+ },
+ validateTemplate: false,
+ validateEntries: false,
+ });
+ return [inspected.configFile.id, inspected.templateFile.id];
+ } catch (error) {
+ if (error instanceof ContentCollectionError) {
+ throw error;
+ }
+ throw new ContentCollectionError(
+ "Collection configuration could not be read"
+ );
}
- throw new ContentCollectionError(
- "Collection configuration could not be read"
- );
}
+ );
+ for (const [configId, templateId] of inspectedCollections) {
+ reservedAssetIds.add(configId);
+ reservedAssetIds.add(templateId);
}
return entries.filter(
(entry) => reservedAssetIds.has(entry.assetId) === false
diff --git a/packages/asset-uploader/src/collection-persistence.test.ts b/packages/asset-uploader/src/collection-persistence.test.ts
index 57644e96a0..e0a514fa74 100644
--- a/packages/asset-uploader/src/collection-persistence.test.ts
+++ b/packages/asset-uploader/src/collection-persistence.test.ts
@@ -1,9 +1,75 @@
import { describe, expect, test } from "vitest";
import { createDefaultCollectionConfig } from "@webstudio-is/content-engine";
import type { Asset } from "@webstudio-is/sdk";
-import { validateCollectionFolder } from "./collection-persistence";
+import {
+ getCollectionReservedAssetIds,
+ validateCollectionFolder,
+} from "./collection-persistence";
describe("collection persistence", () => {
+ test("reads independent collection folders concurrently", async () => {
+ const configSource = createDefaultCollectionConfig();
+ const templateSource = "---\ndraft: true\n---\n\nStart writing.\n";
+ const assets: Asset[] = [];
+ const sources = new Map<string, string>();
+ for (let index = 0; index < 9; index += 1) {
+ const folderId = `collection-${index}`;
+ const configName = `config-${index}.json`;
+ const templateName = `template-${index}.mdx`;
+ assets.push(
+ {
+ id: configName,
+ projectId: "project-1",
+ name: configName,
+ filename: "collection",
+ folderId,
+ type: "file",
+ format: "json",
+ size: configSource.length,
+ description: null,
+ createdAt: "2026-09-03T00:00:00.000Z",
+ meta: {},
+ },
+ {
+ id: templateName,
+ projectId: "project-1",
+ name: templateName,
+ filename: "template",
+ folderId,
+ type: "file",
+ format: "mdx",
+ size: templateSource.length,
+ description: null,
+ createdAt: "2026-09-03T00:00:00.000Z",
+ meta: {},
+ }
+ );
+ sources.set(configName, configSource);
+ sources.set(templateName, templateSource);
+ }
+ let activeReads = 0;
+ let maximumActiveReads = 0;
+ const assetStore = {
+ readFile: async (name: string) => {
+ const source = sources.get(name);
+ if (source === undefined) {
+ throw new Error(`Unexpected read: ${name}`);
+ }
+ activeReads += 1;
+ maximumActiveReads = Math.max(maximumActiveReads, activeReads);
+ await new Promise((resolve) => setTimeout(resolve, 1));
+ activeReads -= 1;
+ return {
+ data: new Blob([source]).stream(),
+ contentLength: source.length,
+ };
+ },
+ };
+
+ await getCollectionReservedAssetIds({ assets, assetStore });
+ expect(maximumActiveReads).toBeGreaterThan(1);
+ });
+
test("rejects a truncated collection config read", async () => {
const source = createDefaultCollectionConfig();
const configAsset: Asset = {
diff --git a/packages/asset-uploader/src/collection-persistence.ts b/packages/asset-uploader/src/collection-persistence.ts
index 01b99c7da0..21a6c4f9e5 100644
--- a/packages/asset-uploader/src/collection-persistence.ts
+++ b/packages/asset-uploader/src/collection-persistence.ts
@@ -8,6 +8,7 @@ import {
} from "@webstudio-is/content-engine";
import {
decodeUtf8,
+ mapBounded,
readBoundedBytes,
} from "@webstudio-is/content-engine/compiler";
import {
@@ -142,39 +143,43 @@ export const getCollectionReservedAssetIds = async ({
assetStore: AssetObjectReader;
folderIds?: ReadonlySet<string>;
}) => {
- const reservedIds = new Set<string>();
- for (const folderId of getCollectionFolderIds(assets)) {
- if (folderIds !== undefined && folderIds.has(folderId) === false) {
- continue;
- }
- const siblings = getFolderAssets(assets, folderId);
- const configAssets = siblings.filter(
- (asset) => formatAssetName(asset) === collectionConfigFilename
- );
- for (const configAsset of configAssets) {
- reservedIds.add(configAsset.id);
- }
- if (configAssets.length !== 1) {
- for (const asset of siblings.filter(isMdxFileAsset)) {
- reservedIds.add(asset.id);
+ const folderReservedIds = await mapBounded(
+ [...getCollectionFolderIds(assets)].filter(
+ (folderId) => folderIds === undefined || folderIds.has(folderId)
+ ),
+ contentEngineLimits.concurrentContentReads,
+ async (folderId) => {
+ const reservedIds = new Set<string>();
+ const siblings = getFolderAssets(assets, folderId);
+ const configAssets = siblings.filter(
+ (asset) => formatAssetName(asset) === collectionConfigFilename
+ );
+ for (const configAsset of configAssets) {
+ reservedIds.add(configAsset.id);
}
- continue;
- }
- const templateName = await readConfiguredTemplateName(
- configAssets[0],
- assetStore
- );
- if (templateName === undefined) {
- for (const asset of siblings.filter(isMdxFileAsset)) {
- reservedIds.add(asset.id);
+ if (configAssets.length !== 1) {
+ for (const asset of siblings.filter(isMdxFileAsset)) {
+ reservedIds.add(asset.id);
+ }
+ return reservedIds;
}
- continue;
- }
- for (const asset of siblings) {
- if (formatAssetName(asset) === templateName && isMdxFileAsset(asset)) {
- reservedIds.add(asset.id);
+ const templateName = await readConfiguredTemplateName(
+ configAssets[0],
+ assetStore
+ );
+ if (templateName === undefined) {
+ for (const asset of siblings.filter(isMdxFileAsset)) {
+ reservedIds.add(asset.id);
+ }
+ return reservedIds;
+ }
+ for (const asset of siblings) {
+ if (formatAssetName(asset) === templateName && isMdxFileAsset(asset)) {
+ reservedIds.add(asset.id);
+ }
}
+ return reservedIds;
}
- }
- return reservedIds;
+ );
+ return new Set(folderReservedIds.flatMap((ids) => [...ids]));
};