Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.
TypeScript accessibilityalternativecloudflarecloudflare-workersdesignfreefrontendopen-sourceradix-uireactremixweb-developmentwebflowwebsite-builderwebsite-developmentwebsiteswebstudio
fix: Batch asset queries during project import (#6368)
Oleg Isonen649ee3669d54
diff --git a/apps/builder/app/services/project-import.server.test.ts b/apps/builder/app/services/project-import.server.test.tsindex a036183324..78cc11a706 100644--- a/apps/builder/app/services/project-import.server.test.ts+++ b/apps/builder/app/services/project-import.server.test.ts@@ -111,7 +111,9 @@ const createPostgrestClient = (existingFileNames?: string[];existingFiles?: ReturnType<typeof createFileRow>[];fileFilters?: [string, string][];+ selectedFileNameBatches?: string[][];restoredFileFilters?: [string, string][];+ restoredFileNameBatches?: string[][];insertedFolders?: Array<{ id: string; projectId: string }>;buildUpdateCount?: number;} = {}@@ -123,19 +125,22 @@ const createPostgrestClient = (options.fileFilters?.push([column, value]);return selectFiles;},- in: async () => {+ in: async (_column: string, names: string[]) => {calls.push("files-select");+ options.selectedFileNameBatches?.push(names);+ const requestedNames = new Set(names);+ const existingFiles =+ options.existingFiles ??+ (options.existingFileNames ?? []).map((name) =>+ createFileRow({+ name,+ format: name.split(".").at(-1) ?? "",+ size: 1,+ meta: name.endsWith(".png") ? { width: 1, height: 1 } : {},+ })+ );return {- data:- options.existingFiles ??- (options.existingFileNames ?? []).map((name) =>- createFileRow({- name,- format: name.split(".").at(-1) ?? "",- size: 1,- meta: name.endsWith(".png") ? { width: 1, height: 1 } : {},- })- ),+ data: existingFiles.filter((file) => requestedNames.has(file.name)),error: undefined,};},@@ -148,8 +153,9 @@ const createPostgrestClient = (options.restoredFileFilters?.push([column, value]);return restoreFiles;},- in: async () => {+ in: async (_column: string, names: string[]) => {calls.push("files-restore");+ options.restoredFileNameBatches?.push(names);return { error: undefined };},};@@ -712,6 +718,45 @@ describe("build import helpers", () => {]);});+ test("chunks imported file lookups and restores by bounded filter size", async () => {+ const fileNames = Array.from(+ { length: 12 },+ (_, index) => `${String(index).padStart(2, "0")}-${"a".repeat(490)}.png`+ );+ const assets = fileNames.map((name, index) => ({+ ...createData().assets[0],+ id: `asset-${index}`,+ name,+ }));+ const selectedFileNameBatches: string[][] = [];+ const restoredFileNameBatches: string[][] = [];++ await importPublishedProjectBundle(+ {+ ctx: {+ postgrest: {+ client: createPostgrestClient([], {+ existingFileNames: fileNames,+ selectedFileNameBatches,+ restoredFileNameBatches,+ }),+ },+ } as never,+ data: createData({ assets }),+ projectId: "target-project",+ },+ {+ hasProjectPermit,+ loadDevBuildByProjectId,+ }+ );++ expect(selectedFileNameBatches.length).toBeGreaterThan(1);+ expect(restoredFileNameBatches.length).toBeGreaterThan(1);+ expect(new Set(selectedFileNameBatches.flat())).toEqual(new Set(fileNames));+ expect(new Set(restoredFileNameBatches.flat())).toEqual(new Set(fileNames));+ });+test("rejects imported asset names with path separators", () => {expect(() =>assertImportedAssetNames([diff --git a/apps/builder/app/services/project-import.server.ts b/apps/builder/app/services/project-import.server.tsindex 0b4c38822e..7fe488f27c 100644--- a/apps/builder/app/services/project-import.server.ts+++ b/apps/builder/app/services/project-import.server.ts@@ -14,6 +14,7 @@ import {createAssetRows,formatAsset,getCollectionFolderIds,+ chunkPostgrestFileNames,validateCollectionFolder,type AssetObjectReader,} from "@webstudio-is/asset-uploader/server";@@ -183,23 +184,26 @@ const loadImportedAssetFiles = async ({return { assets, fileNames: new Set<string>() };}- const files = await ctx.postgrest.client- .from("File")- .select("name, format, description, size, createdAt, updatedAt, meta")- .eq("status", "UPLOADED")- .eq("uploaderProjectId", projectId)- .in(- "name",- assets.map((asset) => asset.name)- );-- if (files.error) {- throw files.error;+ const filesByName = new Map<+ string,+ Parameters<typeof formatAsset>[0]["file"]+ >();+ for (const names of chunkPostgrestFileNames(+ assets.map((asset) => asset.name)+ )) {+ const files = await ctx.postgrest.client+ .from("File")+ .select("name, format, description, size, createdAt, updatedAt, meta")+ .eq("status", "UPLOADED")+ .eq("uploaderProjectId", projectId)+ .in("name", names);+ if (files.error) {+ throw files.error;+ }+ for (const file of files.data ?? []) {+ filesByName.set(file.name, file);+ }}-- const filesByName = new Map(- (files.data ?? []).map((file) => [file.name, file])- );const missingAssets = assets.filter((asset) => filesByName.has(asset.name) === false);@@ -242,13 +246,15 @@ const restoreImportedAssetFiles = async ({if (fileNames.size === 0) {return;}- const visibleFiles = await ctx.postgrest.client- .from("File")- .update({ isDeleted: false })- .eq("uploaderProjectId", projectId)- .in("name", Array.from(fileNames));- if (visibleFiles.error) {- throw visibleFiles.error;+ for (const names of chunkPostgrestFileNames(fileNames)) {+ const visibleFiles = await ctx.postgrest.client+ .from("File")+ .update({ isDeleted: false })+ .eq("uploaderProjectId", projectId)+ .in("name", names);+ if (visibleFiles.error) {+ throw visibleFiles.error;+ }}};diff --git a/packages/asset-uploader/src/asset-patch-core.ts b/packages/asset-uploader/src/asset-patch-core.tsindex b6578280a2..172249129b 100644--- a/packages/asset-uploader/src/asset-patch-core.ts+++ b/packages/asset-uploader/src/asset-patch-core.ts@@ -104,7 +104,7 @@ const maxConcurrentAssetMetadataPostgrestRequests = Math.ceil(// longer than Asset ids, so a fixed item count cannot provide the same bound.const maxPostgrestInFilterCharacters = 4 * 1024;-const chunkPostgrestInValues = (values: Iterable<string>) => {+export const chunkPostgrestFileNames = (values: Iterable<string>) => {const chunks: string[][] = [];let chunk: string[] = [];for (const value of new Set(values)) {@@ -139,7 +139,7 @@ const loadUploadedFilesByNames = async ({names: Iterable<string>;client: Client;}) => {- const chunks = chunkPostgrestInValues(names);+ const chunks = chunkPostgrestFileNames(names);return (await mapBounded(chunks,@@ -165,7 +165,7 @@ const restoreUploadedFilesByNames = async ({client: Client;}) => {await mapBounded(- chunkPostgrestInValues(names),+ chunkPostgrestFileNames(names),maxConcurrentAssetMetadataPostgrestRequests,async (chunk) => {const restoredFiles = await client