Skip to content

Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.

Upload file instead of form data (#1620)

Bogdan Chadkindd92436fe67a
diff --git a/apps/builder/app/builder/shared/assets/use-assets.tsx b/apps/builder/app/builder/shared/assets/use-assets.tsx
index 8c4716b270..a1cd644206 100644
--- a/apps/builder/app/builder/shared/assets/use-assets.tsx
+++ b/apps/builder/app/builder/shared/assets/use-assets.tsx
@@ -139,13 +139,11 @@ const uploadAsset = async ({
throw Error(metaData.errors);
}
- const uploadFormData = new FormData();
- uploadFormData.append("file", file, metaData.name);
const uploadResponse = await fetch(
restAssetsUploadPath({ name: metaData.name }),
{
method: "POST",
- body: uploadFormData,
+ body: file,
}
);
const uploadData: UploadData = await uploadResponse.json();
diff --git a/apps/builder/app/routes/rest.assets.$name.tsx b/apps/builder/app/routes/rest.assets.$name.tsx
index 5c362ce170..2baa6e838d 100644
--- a/apps/builder/app/routes/rest.assets.$name.tsx
+++ b/apps/builder/app/routes/rest.assets.$name.tsx
@@ -15,12 +15,10 @@ export const action = async (
}
try {
- if (request.method === "POST") {
+ if (request.method === "POST" && request.body !== null) {
const asset = await uploadFile(
- {
- name: params.name,
- request,
- },
+ params.name,
+ request.body,
createAssetClient()
);
return {
diff --git a/packages/asset-uploader/package.json b/packages/asset-uploader/package.json
index 9aa081d3db..bc3877d5ad 100644
--- a/packages/asset-uploader/package.json
+++ b/packages/asset-uploader/package.json
@@ -17,7 +17,6 @@
"dependencies": {
"@aws-sdk/client-s3": "^3.131.0",
"@aws-sdk/lib-storage": "^3.131.0",
- "@remix-run/node": "1.15.0",
"@webstudio-is/fonts": "workspace:^",
"@webstudio-is/prisma-client": "workspace:^",
"@webstudio-is/trpc-interface": "workspace:^",
diff --git a/packages/asset-uploader/src/client.ts b/packages/asset-uploader/src/client.ts
index ebd9acec12..67fa5a4dd3 100644
--- a/packages/asset-uploader/src/client.ts
+++ b/packages/asset-uploader/src/client.ts
@@ -4,7 +4,7 @@ export type AssetClient = {
uploadFile: (
name: string,
type: string,
- request: Request
+ data: AsyncIterable<Uint8Array>
) => Promise<AssetData>;
deleteFile: (name: string) => Promise<void>;
};
diff --git a/packages/asset-uploader/src/clients/fs/fs.ts b/packages/asset-uploader/src/clients/fs/fs.ts
index d02b37b7f4..e663b3864e 100644
--- a/packages/asset-uploader/src/clients/fs/fs.ts
+++ b/packages/asset-uploader/src/clients/fs/fs.ts
@@ -9,11 +9,11 @@ type FsClientOptions = {
export const createFsClient = (options: FsClientOptions): AssetClient => {
return {
- uploadFile: (name, type, request) =>
+ uploadFile: (name, type, data) =>
uploadToFs({
name,
type,
- request,
+ data,
maxSize: options.maxUploadSize,
fileDirectory: options.fileDirectory,
}),
diff --git a/packages/asset-uploader/src/clients/fs/upload.ts b/packages/asset-uploader/src/clients/fs/upload.ts
index a8d1e41f95..5de9c36354 100644
--- a/packages/asset-uploader/src/clients/fs/upload.ts
+++ b/packages/asset-uploader/src/clients/fs/upload.ts
@@ -1,41 +1,36 @@
-import { z } from "zod";
-import {
- unstable_parseMultipartFormData as parseMultipartFormData,
- unstable_createFileUploadHandler as createFileUploadHandler,
- NodeOnDiskFile,
-} from "@remix-run/node";
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
import { Location } from "@webstudio-is/prisma-client";
-import { AssetData, getAssetData } from "../../utils/get-asset-data";
-
-const AssetFromFs = z.instanceof(NodeOnDiskFile);
+import { getAssetData } from "../../utils/get-asset-data";
+import { toUint8Array } from "../../utils/to-uint8-array";
+import { createSizeLimiter } from "../../utils/size-limiter";
export const uploadToFs = async ({
name,
type,
- request,
+ data: dataStream,
maxSize,
fileDirectory,
}: {
name: string;
type: string;
- request: Request;
+ data: AsyncIterable<Uint8Array>;
maxSize: number;
fileDirectory: string;
-}): Promise<AssetData> => {
- const uploadHandler = createFileUploadHandler({
- maxPartSize: maxSize,
- directory: fileDirectory,
- file: ({ filename }) => filename,
- });
+}) => {
+ const filepath = resolve(fileDirectory, name);
- const formData = await parseMultipartFormData(request, uploadHandler);
+ // eslint-disable-next-line @typescript-eslint/no-empty-function
+ await mkdir(dirname(filepath), { recursive: true }).catch(() => {});
+ const limitSize = createSizeLimiter(maxSize, name);
- const file = AssetFromFs.parse(formData.get("file"));
+ const data = await toUint8Array(limitSize(dataStream));
+ await writeFile(filepath, data);
const assetData = await getAssetData({
type: type.startsWith("image") ? "image" : "font",
- size: file.size,
- data: new Uint8Array(await file.arrayBuffer()),
+ size: data.byteLength,
+ data,
location: Location.FS,
});
diff --git a/packages/asset-uploader/src/clients/s3/s3.ts b/packages/asset-uploader/src/clients/s3/s3.ts
index cf27c4a4b8..6790045850 100644
--- a/packages/asset-uploader/src/clients/s3/s3.ts
+++ b/packages/asset-uploader/src/clients/s3/s3.ts
@@ -23,12 +23,12 @@ export const createS3Client = (options: S3ClientOptions): AssetClient => {
},
});
- const uploadFile: AssetClient["uploadFile"] = async (name, type, request) => {
- return await uploadToS3({
+ const uploadFile: AssetClient["uploadFile"] = async (name, type, data) => {
+ return uploadToS3({
client,
name,
type,
- request,
+ data,
maxSize: options.maxUploadSize,
bucket: options.bucket,
acl: options.acl,
diff --git a/packages/asset-uploader/src/clients/s3/upload.ts b/packages/asset-uploader/src/clients/s3/upload.ts
index bf70f62ec2..0a757bde57 100644
--- a/packages/asset-uploader/src/clients/s3/upload.ts
+++ b/packages/asset-uploader/src/clients/s3/upload.ts
@@ -1,28 +1,20 @@
import { z } from "zod";
-import {
- unstable_parseMultipartFormData as parseMultipartFormData,
- MaxPartSizeExceededError,
-} from "@remix-run/node";
import type { PutObjectCommandInput, S3Client } from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
import { Location } from "@webstudio-is/prisma-client";
import { toUint8Array } from "../../utils/to-uint8-array";
-import { getAssetData, AssetData } from "../../utils/get-asset-data";
+import { getAssetData } from "../../utils/get-asset-data";
+import { createSizeLimiter } from "../../utils/size-limiter";
const AssetsUploadedSuccess = z.object({
Location: z.string(),
});
-/**
- * Do not change. Upload code assumes its 1.
- */
-const MAX_FILES_PER_REQUEST = 1;
-
export const uploadToS3 = async ({
client,
name,
type,
- request,
+ data: dataStream,
maxSize,
bucket,
acl,
@@ -30,98 +22,45 @@ export const uploadToS3 = async ({
client: S3Client;
name: string;
type: string;
- request: Request;
+ data: AsyncIterable<Uint8Array>;
maxSize: number;
bucket: string;
acl?: string;
-}): Promise<AssetData> => {
- const uploadHandler = createUploadHandler(MAX_FILES_PER_REQUEST, client);
+}) => {
+ const limitSize = createSizeLimiter(maxSize, name);
+
+ // @todo this is going to put the entire file in memory
+ // this has to be a stream that goes directly to s3
+ // Size check has to happen as you stream and interrupted when size is too big
+ // Also check if S3 client has an option to check the size limit
+ const data = await toUint8Array(limitSize(dataStream));
+
+ // if there is no ACL passed we do not default since some providers do not support it
+ const ACL = acl ? { ACL: acl } : {};
+
+ const params: PutObjectCommandInput = {
+ ...ACL,
+ Bucket: bucket,
+ Key: name,
+ Body: data,
+ ContentType: type,
+ CacheControl: "public, max-age=31536004,immutable",
+ Metadata: {
+ // encodeURIComponent is needed to support special characters like Cyrillic
+ filename: encodeURIComponent(name) || "unnamed",
+ },
+ };
- const formData = await parseMultipartFormData(request, async (file) => {
- // Do not parse if it's not a file
- if (file.filename === undefined) {
- return;
- }
- return uploadHandler({
- name,
- type,
- data: file.data,
- maxSize,
- bucket,
- acl,
- });
- });
+ const upload = new Upload({ client, params });
- const file = formData.get("file") as string;
+ AssetsUploadedSuccess.parse(await upload.done());
- const assetData = AssetData.parse(JSON.parse(file));
+ const assetData = await getAssetData({
+ type: type.startsWith("image") ? "image" : "font",
+ size: data.byteLength,
+ data,
+ location: Location.REMOTE,
+ });
return assetData;
};
-
-const createUploadHandler = (maxFiles: number, client: S3Client) => {
- let count = 0;
-
- return async ({
- name,
- type,
- data: dataStream,
- maxSize,
- bucket,
- acl,
- }: {
- name: string;
- type: string;
- data: AsyncIterable<Uint8Array>;
- maxSize: number;
- bucket: string;
- acl?: string;
- }): Promise<string | undefined> => {
- if (count >= maxFiles) {
- // Do not throw, just ignore the file
- // In case of throw we need to delete previously uploaded files
- return;
- }
-
- count++;
-
- // @todo this is going to put the entire file in memory
- // this has to be a stream that goes directly to s3
- // Size check has to happen as you stream and interrupted when size is too big
- // Also check if S3 client has an option to check the size limit
- const data = await toUint8Array(dataStream);
-
- if (data.byteLength > maxSize) {
- throw new MaxPartSizeExceededError(name, maxSize);
- }
-
- // if there is no ACL passed we do not default since some providers do not support it
- const ACL = acl ? { ACL: acl } : {};
-
- const params: PutObjectCommandInput = {
- ...ACL,
- Bucket: bucket,
- Key: name,
- Body: data,
- ContentType: type,
- CacheControl: "public, max-age=31536004,immutable",
- Metadata: {
- // encodeURIComponent is needed to support special characters like Cyrillic
- filename: encodeURIComponent(name) || "unnamed",
- },
- };
-
- const upload = new Upload({ client, params });
-
- AssetsUploadedSuccess.parse(await upload.done());
-
- const assetData = await getAssetData({
- type: type.startsWith("image") ? "image" : "font",
- size: data.byteLength,
- data,
- location: Location.REMOTE,
- });
-
- return JSON.stringify(assetData);
- };
-};
diff --git a/packages/asset-uploader/src/upload.ts b/packages/asset-uploader/src/upload.ts
index 8309954aa8..74ef79fa77 100644
--- a/packages/asset-uploader/src/upload.ts
+++ b/packages/asset-uploader/src/upload.ts
@@ -101,7 +101,8 @@ export const createUploadName = async (
};
export const uploadFile = async (
- { name, request }: { name: string; request: Request },
+ name: string,
+ data: ReadableStream<Uint8Array>,
client: AssetClient
) => {
const asset = await prisma.asset.findFirst({
@@ -123,7 +124,12 @@ export const uploadFile = async (
}
try {
- const assetData = await client.uploadFile(name, asset.file.format, request);
+ const assetData = await client.uploadFile(
+ name,
+ asset.file.format,
+ // global web streams types do not define ReadableStream as async iterable
+ data as unknown as AsyncIterable<Uint8Array>
+ );
const { meta, format, location, size } = assetData;
const dbAsset = await prisma.asset.update({
select: {
diff --git a/packages/asset-uploader/src/utils/size-limiter.ts b/packages/asset-uploader/src/utils/size-limiter.ts
new file mode 100644
index 0000000000..a7707e6628
--- /dev/null
+++ b/packages/asset-uploader/src/utils/size-limiter.ts
@@ -0,0 +1,12 @@
+export const createSizeLimiter = (maxSize: number, name: string) => {
+ return async function* (data: AsyncIterable<ArrayBuffer>) {
+ let size = 0;
+ for await (const chunk of data) {
+ size += chunk.byteLength;
+ if (size > maxSize) {
+ throw Error(`File "${name}" exceeded upload size of ${maxSize} bytes`);
+ }
+ yield chunk;
+ }
+ };
+};
diff --git a/packages/asset-uploader/src/utils/to-uint8-array.ts b/packages/asset-uploader/src/utils/to-uint8-array.ts
index 39ca572d9c..426bece0a4 100644
--- a/packages/asset-uploader/src/utils/to-uint8-array.ts
+++ b/packages/asset-uploader/src/utils/to-uint8-array.ts
@@ -1,4 +1,4 @@
-export const toUint8Array = async (data: AsyncIterable<Uint8Array>) => {
+export const toUint8Array = async (data: AsyncIterable<ArrayBuffer>) => {
const result = [];
for await (const chunk of data) {
result.push(Buffer.from(chunk));
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index a339418607..1b6797677a 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -366,9 +366,6 @@ importers:
'@aws-sdk/lib-storage':
specifier: ^3.131.0
version: 3.131.0(@aws-sdk/[email protected])(@aws-sdk/[email protected])
- '@remix-run/node':
- specifier: 1.15.0
- version: 1.15.0
'@webstudio-is/fonts':
specifier: workspace:^
version: link:../fonts