Skip to content

Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.

Split s3 and fs logic from asset uploader into "clients" (#1386)

Bogdan Chadkin304e82f97984
diff --git a/apps/builder/app/env/env.server.ts b/apps/builder/app/env/env.server.ts
index b916401216..53304f5101 100644
--- a/apps/builder/app/env/env.server.ts
+++ b/apps/builder/app/env/env.server.ts
@@ -14,7 +14,7 @@ const env = {
PUBLISHER_ENDPOINT: process.env.PUBLISHER_ENDPOINT,
PUBLISHER_TOKEN: process.env.PUBLISHER_TOKEN,
- // DEPLOYMENT_ENVIRONMENT developement | preview | production
+ // DEPLOYMENT_ENVIRONMENT development | preview | production
DEPLOYMENT_ENVIRONMENT: process.env.DEPLOYMENT_ENVIRONMENT,
DEPLOYMENT_URL: process.env.DEPLOYMENT_URL,
diff --git a/apps/builder/app/routes/rest.assets.$projectId.tsx b/apps/builder/app/routes/rest.assets.$projectId.tsx
index 5a1475258b..30f4f5dea1 100644
--- a/apps/builder/app/routes/rest.assets.$projectId.tsx
+++ b/apps/builder/app/routes/rest.assets.$projectId.tsx
@@ -10,6 +10,7 @@ import { toast } from "@webstudio-is/design-system";
import type { ActionData } from "~/builder/shared/assets";
import { sentryException } from "~/shared/sentry";
import { createContext } from "~/shared/context.server";
+import { createAssetClient } from "~/shared/asset-client";
export const loader = async ({
params,
@@ -40,7 +41,8 @@ export const action = async (
request,
projectId: params.projectId,
},
- context
+ context,
+ createAssetClient()
);
return {
uploadedAssets: assets.map((asset) => ({
diff --git a/apps/builder/app/routes/rest.patch.ts b/apps/builder/app/routes/rest.patch.ts
index 5c6dd60b89..bcec92722e 100644
--- a/apps/builder/app/routes/rest.patch.ts
+++ b/apps/builder/app/routes/rest.patch.ts
@@ -13,6 +13,7 @@ import {
import { patchAssets } from "@webstudio-is/asset-uploader/server";
import type { Project } from "@webstudio-is/project";
import { createContext } from "~/shared/context.server";
+import { createAssetClient } from "~/shared/asset-client";
type PatchData = {
transactions: Array<SyncItem>;
@@ -57,7 +58,7 @@ export const action = async ({ request }: ActionArgs) => {
} else if (namespace === "breakpoints") {
await patchBreakpoints({ buildId, projectId }, patches, context);
} else if (namespace === "assets") {
- await patchAssets({ projectId }, patches, context);
+ await patchAssets({ projectId }, patches, context, createAssetClient());
} else {
return { errors: `Unknown namespace "${namespace}"` };
}
diff --git a/apps/builder/app/shared/asset-client.ts b/apps/builder/app/shared/asset-client.ts
new file mode 100644
index 0000000000..bee0d4af5c
--- /dev/null
+++ b/apps/builder/app/shared/asset-client.ts
@@ -0,0 +1,19 @@
+import { S3Env, FsEnv } from "@webstudio-is/asset-uploader";
+import {
+ createFSClient,
+ createS3Client,
+} from "@webstudio-is/asset-uploader/server";
+
+export const createAssetClient = () => {
+ if (process.env.NODE_ENV === "development") {
+ const env = FsEnv.parse(process.env);
+ return createFSClient({
+ maxUploadSize: env.MAX_UPLOAD_SIZE,
+ });
+ } else {
+ const env = S3Env.parse(process.env);
+ return createS3Client({
+ maxUploadSize: env.MAX_UPLOAD_SIZE,
+ });
+ }
+};
diff --git a/packages/asset-uploader/src/client.ts b/packages/asset-uploader/src/client.ts
new file mode 100644
index 0000000000..d045a50486
--- /dev/null
+++ b/packages/asset-uploader/src/client.ts
@@ -0,0 +1,6 @@
+import type { AssetData } from "./utils/get-asset-data";
+
+export type AssetClient = {
+ uploadFile: (request: Request) => Promise<AssetData>;
+ deleteFile: (name: string) => Promise<void>;
+};
diff --git a/packages/asset-uploader/src/targets/fs/delete.ts b/packages/asset-uploader/src/clients/fs/delete.ts
similarity index 100%
rename from packages/asset-uploader/src/targets/fs/delete.ts
rename to packages/asset-uploader/src/clients/fs/delete.ts
diff --git a/packages/asset-uploader/src/targets/fs/file-path.ts b/packages/asset-uploader/src/clients/fs/file-path.ts
similarity index 100%
rename from packages/asset-uploader/src/targets/fs/file-path.ts
rename to packages/asset-uploader/src/clients/fs/file-path.ts
diff --git a/packages/asset-uploader/src/clients/fs/fs.ts b/packages/asset-uploader/src/clients/fs/fs.ts
new file mode 100644
index 0000000000..4d0a18cd25
--- /dev/null
+++ b/packages/asset-uploader/src/clients/fs/fs.ts
@@ -0,0 +1,15 @@
+import type { AssetClient } from "../../client";
+import { deleteFromFs } from "./delete";
+import { uploadToFs } from "./upload";
+
+type FSClientOptions = {
+ maxUploadSize: number;
+};
+
+export const createFSClient = (clientOptions: FSClientOptions): AssetClient => {
+ const { maxUploadSize } = clientOptions;
+ return {
+ uploadFile: (request) => uploadToFs({ request, maxSize: maxUploadSize }),
+ deleteFile: deleteFromFs,
+ };
+};
diff --git a/packages/asset-uploader/src/clients/fs/upload.ts b/packages/asset-uploader/src/clients/fs/upload.ts
new file mode 100644
index 0000000000..1b4271eabc
--- /dev/null
+++ b/packages/asset-uploader/src/clients/fs/upload.ts
@@ -0,0 +1,64 @@
+import { z } from "zod";
+import {
+ unstable_parseMultipartFormData as unstableParseMultipartFormData,
+ unstable_createFileUploadHandler as unstableCreateFileUploadHandler,
+ unstable_composeUploadHandlers as unstableComposeUploadHandlers,
+ NodeOnDiskFile,
+} from "@remix-run/node";
+import { Location } from "@webstudio-is/prisma-client";
+import { AssetData, getAssetData } from "../../utils/get-asset-data";
+import { FILE_DIRECTORY } from "./file-path";
+import { idsFormDataFieldName } from "../../schema";
+import { getUniqueFilename } from "../../utils/get-unique-filename";
+import { sanitizeS3Key } from "../../utils/sanitize-s3-key";
+import { uuidHandler } from "../../utils/uuid-handler";
+
+const AssetsFromFs = z.array(z.instanceof(NodeOnDiskFile));
+const Ids = z.array(z.string().uuid());
+
+/**
+ * Do not change. Upload code assumes its 1.
+ */
+const MAX_FILES_PER_REQUEST = 1;
+
+export const uploadToFs = async ({
+ request,
+ maxSize,
+}: {
+ request: Request;
+ maxSize: number;
+}): Promise<AssetData> => {
+ const uploadHandler = unstableCreateFileUploadHandler({
+ maxPartSize: maxSize,
+ directory: FILE_DIRECTORY,
+ file: ({ filename }) => getUniqueFilename(sanitizeS3Key(filename)),
+ });
+
+ const formData = await unstableParseMultipartFormData(
+ request,
+ unstableComposeUploadHandlers(uploadHandler, uuidHandler)
+ );
+
+ const formDataImages = AssetsFromFs.parse(formData.getAll("image"));
+ const formDataFonts = AssetsFromFs.parse(formData.getAll("font"));
+ const formDataAll = [...formDataImages, ...formDataFonts].slice(
+ 0,
+ MAX_FILES_PER_REQUEST
+ );
+ const ids = Ids.parse(formData.getAll(idsFormDataFieldName));
+
+ const assets = formDataAll.map(async (asset, index) =>
+ getAssetData({
+ id: ids[index],
+ type: formDataFonts.includes(asset) ? "font" : "image",
+ name: asset.name,
+ size: asset.size,
+ data: new Uint8Array(await asset.arrayBuffer()),
+ location: Location.FS,
+ })
+ );
+
+ const assetsData = await Promise.all(assets);
+
+ return assetsData[0];
+};
diff --git a/packages/asset-uploader/src/targets/s3/client.ts b/packages/asset-uploader/src/clients/s3/client.ts
similarity index 100%
rename from packages/asset-uploader/src/targets/s3/client.ts
rename to packages/asset-uploader/src/clients/s3/client.ts
diff --git a/packages/asset-uploader/src/targets/s3/delete.ts b/packages/asset-uploader/src/clients/s3/delete.ts
similarity index 85%
rename from packages/asset-uploader/src/targets/s3/delete.ts
rename to packages/asset-uploader/src/clients/s3/delete.ts
index 79988370eb..c8a7db6ddc 100644
--- a/packages/asset-uploader/src/targets/s3/delete.ts
+++ b/packages/asset-uploader/src/clients/s3/delete.ts
@@ -1,6 +1,6 @@
import { DeleteObjectCommand } from "@aws-sdk/client-s3";
import { S3Env } from "../../schema";
-import { getS3Client } from "../../targets/s3/client";
+import { getS3Client } from "./client";
export const deleteFromS3 = async (name: string) => {
const s3Env = S3Env.parse(process.env);
diff --git a/packages/asset-uploader/src/clients/s3/s3.ts b/packages/asset-uploader/src/clients/s3/s3.ts
new file mode 100644
index 0000000000..6982742be2
--- /dev/null
+++ b/packages/asset-uploader/src/clients/s3/s3.ts
@@ -0,0 +1,15 @@
+import type { AssetClient } from "../../client";
+import { deleteFromS3 } from "./delete";
+import { uploadToS3 } from "./upload";
+
+type S3ClientOptions = {
+ maxUploadSize: number;
+};
+
+export const createS3Client = (clientOptions: S3ClientOptions): AssetClient => {
+ const { maxUploadSize } = clientOptions;
+ return {
+ uploadFile: (request) => uploadToS3({ request, maxSize: maxUploadSize }),
+ deleteFile: deleteFromS3,
+ };
+};
diff --git a/packages/asset-uploader/src/targets/s3/upload.ts b/packages/asset-uploader/src/clients/s3/upload.ts
similarity index 73%
rename from packages/asset-uploader/src/targets/s3/upload.ts
rename to packages/asset-uploader/src/clients/s3/upload.ts
index 76e591a235..0e8304a7f5 100644
--- a/packages/asset-uploader/src/targets/s3/upload.ts
+++ b/packages/asset-uploader/src/clients/s3/upload.ts
@@ -11,13 +11,11 @@ import { Location } from "@webstudio-is/prisma-client";
import { S3Env } from "../../schema";
import { toUint8Array } from "../../utils/to-uint8-array";
import { getAssetData, AssetData } from "../../utils/get-asset-data";
-import { createAssetWithLimit } from "../../db";
-import { idsFormDataFieldName, type Asset } from "../../schema";
+import { idsFormDataFieldName } from "../../schema";
import { getUniqueFilename } from "../../utils/get-unique-filename";
import { getS3Client } from "./client";
import { sanitizeS3Key } from "../../utils/sanitize-s3-key";
import { uuidHandler } from "../../utils/uuid-handler";
-import type { AppContext } from "@webstudio-is/trpc-interface/server";
const AssetsUploadedSuccess = z.object({
Location: z.string(),
@@ -30,51 +28,38 @@ const Ids = z.array(z.string().uuid());
*/
const MAX_FILES_PER_REQUEST = 1;
-export const uploadToS3 = async (
- {
+export const uploadToS3 = async ({
+ request,
+ maxSize,
+}: {
+ request: Request;
+ maxSize: number;
+}): Promise<AssetData> => {
+ const uploadHandler = createUploadHandler(MAX_FILES_PER_REQUEST);
+
+ const formData = await unstableCreateFileUploadHandler(
request,
- projectId,
- maxSize,
- }: {
- request: Request;
- projectId: string;
- maxSize: number;
- },
- context: AppContext
-): Promise<Array<Asset>> => {
- const asset = await createAssetWithLimit(
- projectId,
- async () => {
- const uploadHandler = createUploadHandler(MAX_FILES_PER_REQUEST);
-
- const formData = await unstableCreateFileUploadHandler(
- request,
- unstableComposeUploadHandlers(
- (file: UploadHandlerPart) =>
- uploadHandler({
- file,
- maxSize,
- }),
- uuidHandler
- )
- );
-
- const imagesFormData = formData.getAll("image") as Array<string>;
- const fontsFormData = formData.getAll("font") as Array<string>;
- const ids = Ids.parse(formData.getAll(idsFormDataFieldName));
-
- const assetsData = [...imagesFormData, ...fontsFormData]
- .slice(0, MAX_FILES_PER_REQUEST)
- .map((dataString, index) => {
- return AssetData.parse({ ...JSON.parse(dataString), id: ids[index] });
- });
-
- return assetsData[0];
- },
- context
+ unstableComposeUploadHandlers(
+ (file: UploadHandlerPart) =>
+ uploadHandler({
+ file,
+ maxSize,
+ }),
+ uuidHandler
+ )
);
- return [asset];
+ const imagesFormData = formData.getAll("image") as Array<string>;
+ const fontsFormData = formData.getAll("font") as Array<string>;
+ const ids = Ids.parse(formData.getAll(idsFormDataFieldName));
+
+ const assetsData = [...imagesFormData, ...fontsFormData]
+ .slice(0, MAX_FILES_PER_REQUEST)
+ .map((dataString, index) => {
+ return AssetData.parse({ ...JSON.parse(dataString), id: ids[index] });
+ });
+
+ return assetsData[0];
};
const createUploadHandler = (maxFiles: number) => {
diff --git a/packages/asset-uploader/src/delete.ts b/packages/asset-uploader/src/delete.ts
index fff370acd6..afb9eea8dc 100644
--- a/packages/asset-uploader/src/delete.ts
+++ b/packages/asset-uploader/src/delete.ts
@@ -3,10 +3,9 @@ import {
authorizeProject,
type AppContext,
} from "@webstudio-is/trpc-interface/server";
+import type { AssetClient } from "./client";
import { deleteFromDb } from "./db";
import type { Asset } from "./schema";
-import { deleteFromFs } from "./targets/fs/delete";
-import { deleteFromS3 } from "./targets/s3/delete";
import { formatAsset } from "./utils/format-asset";
export const deleteAssets = async (
@@ -14,7 +13,8 @@ export const deleteAssets = async (
ids: Array<Asset["id"]>;
projectId: Project["id"];
},
- context: AppContext
+ context: AppContext,
+ client: AssetClient
): Promise<Array<Asset>> => {
const canDelete = await authorizeProject.hasProjectPermit(
{ projectId: props.projectId, permit: "edit" },
@@ -45,11 +45,7 @@ export const deleteAssets = async (
continue;
}
- if (asset.location === "REMOTE") {
- await deleteFromS3(asset.name);
- } else {
- await deleteFromFs(asset.name);
- }
+ await client.deleteFile(asset.name);
}
return assets.map(formatAsset);
diff --git a/packages/asset-uploader/src/index.server.ts b/packages/asset-uploader/src/index.server.ts
index 08d8563d71..36b3f709e5 100644
--- a/packages/asset-uploader/src/index.server.ts
+++ b/packages/asset-uploader/src/index.server.ts
@@ -3,3 +3,5 @@ export * from "./upload";
export * from "./delete";
export * from "./patch";
export * from "./utils/format-asset";
+export * from "./clients/fs/fs";
+export * from "./clients/s3/s3";
diff --git a/packages/asset-uploader/src/patch.ts b/packages/asset-uploader/src/patch.ts
index 26f3476834..856c163f2b 100644
--- a/packages/asset-uploader/src/patch.ts
+++ b/packages/asset-uploader/src/patch.ts
@@ -7,11 +7,13 @@ import {
import { type Asset, Assets } from "./schema";
import { deleteAssets } from "./delete";
import { loadByProject } from "./db/load";
+import type { AssetClient } from "./client";
export const patchAssets = async (
{ projectId }: { projectId: Project["id"] },
patches: Array<Patch>,
- context: AppContext
+ context: AppContext,
+ assetClient: AssetClient
) => {
const canEdit = await authorizeProject.hasProjectPermit(
{ projectId, permit: "edit" },
@@ -36,6 +38,6 @@ export const patchAssets = async (
}
}
if (deletedAssetIds.length !== 0) {
- deleteAssets({ projectId, ids: deletedAssetIds }, context);
+ deleteAssets({ projectId, ids: deletedAssetIds }, context, assetClient);
}
};
diff --git a/packages/asset-uploader/src/targets/fs/upload.ts b/packages/asset-uploader/src/targets/fs/upload.ts
deleted file mode 100644
index dc9a0c80f4..0000000000
--- a/packages/asset-uploader/src/targets/fs/upload.ts
+++ /dev/null
@@ -1,79 +0,0 @@
-import { z } from "zod";
-import {
- unstable_parseMultipartFormData as unstableParseMultipartFormData,
- unstable_createFileUploadHandler as unstableCreateFileUploadHandler,
- unstable_composeUploadHandlers as unstableComposeUploadHandlers,
- NodeOnDiskFile,
-} from "@remix-run/node";
-import { Location } from "@webstudio-is/prisma-client";
-import { getAssetData } from "../../utils/get-asset-data";
-import { createAssetWithLimit } from "../../db";
-import { FILE_DIRECTORY } from "./file-path";
-import { Asset, idsFormDataFieldName } from "../../schema";
-import { getUniqueFilename } from "../../utils/get-unique-filename";
-import { sanitizeS3Key } from "../../utils/sanitize-s3-key";
-import { uuidHandler } from "../../utils/uuid-handler";
-import type { AppContext } from "@webstudio-is/trpc-interface/server";
-
-const AssetsFromFs = z.array(z.instanceof(NodeOnDiskFile));
-const Ids = z.array(z.string().uuid());
-
-/**
- * Do not change. Upload code assumes its 1.
- */
-const MAX_FILES_PER_REQUEST = 1;
-
-export const uploadToFs = async (
- {
- request,
- projectId,
- maxSize,
- }: {
- request: Request;
- projectId: string;
- maxSize: number;
- },
- context: AppContext
-): Promise<Array<Asset>> => {
- const asset = await createAssetWithLimit(
- projectId,
- async () => {
- const uploadHandler = await unstableCreateFileUploadHandler({
- maxPartSize: maxSize,
- directory: FILE_DIRECTORY,
- file: ({ filename }) => getUniqueFilename(sanitizeS3Key(filename)),
- });
-
- const formData = await unstableParseMultipartFormData(
- request,
- unstableComposeUploadHandlers(uploadHandler, uuidHandler)
- );
-
- const formDataImages = AssetsFromFs.parse(formData.getAll("image"));
- const formDataFonts = AssetsFromFs.parse(formData.getAll("font"));
- const formDataAll = [...formDataImages, ...formDataFonts].slice(
- 0,
- MAX_FILES_PER_REQUEST
- );
- const ids = Ids.parse(formData.getAll(idsFormDataFieldName));
-
- const assets = formDataAll.map(async (asset, index) =>
- getAssetData({
- id: ids[index],
- type: formDataFonts.includes(asset) ? "font" : "image",
- name: asset.name,
- size: asset.size,
- data: new Uint8Array(await asset.arrayBuffer()),
- location: Location.FS,
- })
- );
-
- const assetsData = await Promise.all(assets);
-
- return assetsData[0];
- },
- context
- );
-
- return [asset];
-};
diff --git a/packages/asset-uploader/src/upload.ts b/packages/asset-uploader/src/upload.ts
index e2963e191b..0e0e68f72a 100644
--- a/packages/asset-uploader/src/upload.ts
+++ b/packages/asset-uploader/src/upload.ts
@@ -1,10 +1,6 @@
-import { S3Env, FsEnv } from "./schema";
-import { uploadToFs } from "./targets/fs/upload";
-import { uploadToS3 } from "./targets/s3/upload";
import type { AppContext } from "@webstudio-is/trpc-interface/server";
-
-const isS3Upload = S3Env.safeParse(process.env).success;
-const fsEnv = FsEnv.parse(process.env);
+import type { AssetClient } from "./client";
+import { createAssetWithLimit } from "./db/create";
export const uploadAssets = async (
{
@@ -14,24 +10,16 @@ export const uploadAssets = async (
request: Request;
projectId: string;
},
- context: AppContext
+ context: AppContext,
+ client: AssetClient
) => {
- if (isS3Upload) {
- return await uploadToS3(
- {
- request,
- projectId,
- maxSize: fsEnv.MAX_UPLOAD_SIZE,
- },
- context
- );
- }
- return await uploadToFs(
- {
- request,
- projectId,
- maxSize: fsEnv.MAX_UPLOAD_SIZE,
+ const asset = await createAssetWithLimit(
+ projectId,
+ () => {
+ return client.uploadFile(request);
},
context
);
+
+ return [asset];
};