Skip to content

Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.

Upload files by url (#1547)

Bogdan Chadkin7f0e1bb2f360
diff --git a/apps/builder/app/builder/shared/assets/use-assets.tsx b/apps/builder/app/builder/shared/assets/use-assets.tsx
index 485932eda0..93141618d7 100644
--- a/apps/builder/app/builder/shared/assets/use-assets.tsx
+++ b/apps/builder/app/builder/shared/assets/use-assets.tsx
@@ -9,7 +9,7 @@ import {
} from "@webstudio-is/asset-uploader";
import { toast } from "@webstudio-is/design-system";
import { sanitizeS3Key } from "@webstudio-is/asset-uploader";
-import { restAssetsPath } from "~/shared/router-utils";
+import { restAssetsUploadPath, restAssetsPath } from "~/shared/router-utils";
import type { AssetContainer, PreviewAsset } from "./types";
import { usePersistentFetcher } from "~/shared/fetcher";
import type { ActionData } from "~/builder/shared/assets";
@@ -113,7 +113,8 @@ const assetContainersStore = computed(
export type UploadData = ActionData;
export const useUploadAsset = () => {
- const submit = usePersistentFetcher();
+ const submitAsset = usePersistentFetcher();
+ const submitUpload = usePersistentFetcher();
const handleAfterSubmit = (assetId: string, data: UploadData) => {
// remove uploaded or failed asset
@@ -160,21 +161,37 @@ export const useUploadAsset = () => {
for (const fileData of filesData) {
const { id, type, file } = fileData;
- // sanitizeS3Key here is just because of https://github.com/remix-run/remix/issues/4443
// should be removed after fix
const formData = new FormData();
- formData.append(type, file, sanitizeS3Key(file.name));
- formData.append(idsFormDataFieldName, id);
- submit<UploadData>(
+ formData.append("projectId", projectId);
+ formData.append("assetId", id);
+ // sanitizeS3Key here is just because of https://github.com/remix-run/remix/issues/4443
+ // should be removed after fix
+ formData.append("filename", sanitizeS3Key(file.name));
+ submitAsset<{ name: string }>(
formData,
{
method: "post",
- action: restAssetsPath({ projectId, authToken }),
+ action: restAssetsPath({ authToken }),
encType: "multipart/form-data",
},
(data) => {
- URL.revokeObjectURL(fileData.objectURL);
- handleAfterSubmit(fileData.id, data);
+ const formData = new FormData();
+ formData.append(type, file, data.name);
+ formData.append("projectId", projectId);
+ formData.append(idsFormDataFieldName, id);
+ submitUpload<UploadData>(
+ formData,
+ {
+ method: "post",
+ action: restAssetsUploadPath({ name: data.name }),
+ encType: "multipart/form-data",
+ },
+ (data) => {
+ URL.revokeObjectURL(fileData.objectURL);
+ handleAfterSubmit(fileData.id, data);
+ }
+ );
}
);
}
diff --git a/apps/builder/app/routes/rest.assets.$name.tsx b/apps/builder/app/routes/rest.assets.$name.tsx
new file mode 100644
index 0000000000..5c362ce170
--- /dev/null
+++ b/apps/builder/app/routes/rest.assets.$name.tsx
@@ -0,0 +1,38 @@
+import type { ActionArgs } from "@remix-run/node";
+import type { Asset } from "@webstudio-is/asset-uploader";
+import { uploadFile } from "@webstudio-is/asset-uploader/index.server";
+import type { ActionData } from "~/builder/shared/assets";
+import { sentryException } from "~/shared/sentry";
+import { createAssetClient } from "~/shared/asset-client";
+
+export const action = async (
+ props: ActionArgs
+): Promise<ActionData | Array<Asset> | undefined> => {
+ const { request, params } = props;
+
+ if (params.name === undefined) {
+ throw new Error("Name is undefined");
+ }
+
+ try {
+ if (request.method === "POST") {
+ const asset = await uploadFile(
+ {
+ name: params.name,
+ request,
+ },
+ createAssetClient()
+ );
+ return {
+ uploadedAssets: [asset],
+ };
+ }
+ } catch (error) {
+ if (error instanceof Error) {
+ sentryException({ error });
+ return {
+ errors: error.message,
+ };
+ }
+ }
+};
diff --git a/apps/builder/app/routes/rest.assets.$projectId.tsx b/apps/builder/app/routes/rest.assets.$projectId.tsx
deleted file mode 100644
index ec5d133615..0000000000
--- a/apps/builder/app/routes/rest.assets.$projectId.tsx
+++ /dev/null
@@ -1,74 +0,0 @@
-import { useEffect } from "react";
-import type { ActionArgs, LoaderArgs } from "@remix-run/node";
-import { useActionData } from "@remix-run/react";
-import { MaxAssets, type Asset } from "@webstudio-is/asset-uploader";
-import {
- uploadAssets,
- loadByProject,
-} from "@webstudio-is/asset-uploader/index.server";
-import { toast } from "@webstudio-is/design-system";
-import type { ActionData } from "~/builder/shared/assets";
-import { sentryException } from "~/shared/sentry";
-import { createContext } from "~/shared/context.server";
-import { createAssetClient } from "~/shared/asset-client";
-import env from "~/env/env.server";
-
-export const loader = async ({
- params,
- request,
-}: LoaderArgs): Promise<Array<Asset>> => {
- if (params.projectId === undefined) {
- throw new Error("Project id undefined");
- }
- const context = await createContext(request);
- return await loadByProject(params.projectId, context);
-};
-
-export const action = async (
- props: ActionArgs
-): Promise<ActionData | Array<Asset> | undefined> => {
- const { request, params } = props;
-
- if (params.projectId === undefined) {
- throw new Error("Project id undefined");
- }
-
- const context = await createContext(request);
-
- try {
- if (request.method === "POST") {
- const assets = await uploadAssets(
- {
- request,
- projectId: params.projectId,
- maxAssetsPerProject: MaxAssets.parse(env.MAX_ASSETS_PER_PROJECT),
- },
- context,
- createAssetClient()
- );
- return {
- uploadedAssets: assets.map((asset) => ({
- ...asset,
- status: "uploaded",
- })),
- };
- }
- } catch (error) {
- if (error instanceof Error) {
- sentryException({ error });
- return {
- errors: error.message,
- };
- }
- }
-};
-
-export const useAction = () => {
- const actionData: ActionData | undefined = useActionData();
-
- useEffect(() => {
- if (actionData?.errors) {
- toast.error(actionData.errors);
- }
- }, [actionData]);
-};
diff --git a/apps/builder/app/routes/rest.assets.tsx b/apps/builder/app/routes/rest.assets.tsx
new file mode 100644
index 0000000000..5473ffa355
--- /dev/null
+++ b/apps/builder/app/routes/rest.assets.tsx
@@ -0,0 +1,55 @@
+import type { ActionArgs, LoaderArgs } from "@remix-run/node";
+import { MaxAssets, type Asset } from "@webstudio-is/asset-uploader";
+import {
+ loadByProject,
+ createUploadName,
+} from "@webstudio-is/asset-uploader/index.server";
+import { sentryException } from "~/shared/sentry";
+import { createContext } from "~/shared/context.server";
+import env from "~/env/env.server";
+
+export const loader = async ({
+ params,
+ request,
+}: LoaderArgs): Promise<Array<Asset>> => {
+ if (params.projectId === undefined) {
+ throw new Error("Project id undefined");
+ }
+ const context = await createContext(request);
+ return await loadByProject(params.projectId, context);
+};
+
+export const action = async (props: ActionArgs) => {
+ const { request } = props;
+
+ const context = await createContext(request);
+
+ try {
+ if (request.method === "POST") {
+ const formData = await request.formData();
+ const projectId = formData.get("projectId") as string;
+ const assetId = formData.get("assetId") as string;
+ const filename = formData.get("filename") as string;
+ if (projectId === null || assetId === null || filename === null) {
+ throw Error("Project id, asset id or filename are missing");
+ }
+ const name = await createUploadName(
+ {
+ projectId,
+ assetId,
+ filename,
+ maxAssetsPerProject: MaxAssets.parse(env.MAX_ASSETS_PER_PROJECT),
+ },
+ context
+ );
+ return {
+ name,
+ };
+ }
+ } catch (error) {
+ if (error instanceof Error) {
+ sentryException({ error });
+ throw error;
+ }
+ }
+};
diff --git a/apps/builder/app/shared/fetcher/persistent-fetcher.tsx b/apps/builder/app/shared/fetcher/persistent-fetcher.tsx
index 7778747044..7f94b429cf 100644
--- a/apps/builder/app/shared/fetcher/persistent-fetcher.tsx
+++ b/apps/builder/app/shared/fetcher/persistent-fetcher.tsx
@@ -54,8 +54,10 @@ export const PersistentFetcherProvider = ({
isIdle.current = true;
if (currentRequestCallback.current) {
- currentRequestCallback.current(data);
+ const callback = currentRequestCallback.current;
+ // reset before calling to enable sequential submits
currentRequestCallback.current = undefined;
+ callback(data);
}
const nextTask = queue.current.shift();
diff --git a/apps/builder/app/shared/router-utils/path-utils.ts b/apps/builder/app/shared/router-utils/path-utils.ts
index 66262b138c..8296b9e6d3 100644
--- a/apps/builder/app/shared/router-utils/path-utils.ts
+++ b/apps/builder/app/shared/router-utils/path-utils.ts
@@ -76,24 +76,22 @@ export const authPath = ({
provider: "google" | "github" | "dev";
}) => `/auth/${provider}`;
-export const restAssetsPath = ({
- projectId,
- authToken,
-}: {
- projectId: string;
- authToken?: string;
-}) => {
+export const restAssetsPath = ({ authToken }: { authToken?: string }) => {
const urlSearchParams = new URLSearchParams();
if (authToken !== undefined) {
urlSearchParams.set("authToken", authToken);
}
const urlSearchParamsString = urlSearchParams.toString();
- return `/rest/assets/${projectId}${
+ return `/rest/assets${
urlSearchParamsString ? `?${urlSearchParamsString}` : ""
}`;
};
+export const restAssetsUploadPath = ({ name }: { name: string }) => {
+ return `/rest/assets/${name}`;
+};
+
export const restThemePath = ({ setting }: { setting: ThemeSetting }) =>
`/rest/theme/${setting}`;
diff --git a/packages/asset-uploader/src/clients/fs/upload.ts b/packages/asset-uploader/src/clients/fs/upload.ts
index ac1e026aa5..2b25e87183 100644
--- a/packages/asset-uploader/src/clients/fs/upload.ts
+++ b/packages/asset-uploader/src/clients/fs/upload.ts
@@ -9,8 +9,6 @@ import {
import { Location } from "@webstudio-is/prisma-client";
import { AssetData, getAssetData } from "../../utils/get-asset-data";
import { idsFormDataFieldName } from "../../schema";
-import { getUniqueFilename } from "../../utils/get-unique-filename";
-import { sanitizeS3Key } from "../../utils/sanitize-s3-key";
const AssetsFromFs = z.array(z.instanceof(NodeOnDiskFile));
const Ids = z.array(z.string().uuid());
@@ -32,7 +30,7 @@ export const uploadToFs = async ({
const uploadHandler = createFileUploadHandler({
maxPartSize: maxSize,
directory: fileDirectory,
- file: ({ filename }) => getUniqueFilename(sanitizeS3Key(filename)),
+ file: ({ filename }) => filename,
});
const formData = await parseMultipartFormData(
diff --git a/packages/asset-uploader/src/clients/s3/upload.ts b/packages/asset-uploader/src/clients/s3/upload.ts
index 0129aca6df..c25cc09284 100644
--- a/packages/asset-uploader/src/clients/s3/upload.ts
+++ b/packages/asset-uploader/src/clients/s3/upload.ts
@@ -12,8 +12,6 @@ import { Location } from "@webstudio-is/prisma-client";
import { toUint8Array } from "../../utils/to-uint8-array";
import { getAssetData, AssetData } from "../../utils/get-asset-data";
import { idsFormDataFieldName } from "../../schema";
-import { getUniqueFilename } from "../../utils/get-unique-filename";
-import { sanitizeS3Key } from "../../utils/sanitize-s3-key";
const AssetsUploadedSuccess = z.object({
Location: z.string(),
@@ -111,9 +109,9 @@ const createUploadHandler = (maxFiles: number, client: S3Client) => {
throw new MaxPartSizeExceededError(file.name, maxSize);
}
- const fileName = sanitizeS3Key(file.filename);
+ const fileName = file.filename;
- const uniqueFilename = getUniqueFilename(fileName);
+ const uniqueFilename = fileName;
// if there is no ACL passed we do not default since some providers do not support it
const ACL = acl ? { ACL: acl } : {};
diff --git a/packages/asset-uploader/src/db/create.ts b/packages/asset-uploader/src/db/create.ts
deleted file mode 100644
index 420781717d..0000000000
--- a/packages/asset-uploader/src/db/create.ts
+++ /dev/null
@@ -1,137 +0,0 @@
-import type { FontMeta } from "@webstudio-is/fonts";
-import {
- prisma,
- type Location,
- type Project,
- type Asset,
-} from "@webstudio-is/prisma-client";
-import {
- authorizeProject,
- type AppContext,
-} from "@webstudio-is/trpc-interface/index.server";
-import type { ImageMeta } from "../schema";
-import { formatAsset } from "../utils/format-asset";
-
-type BaseOptions = {
- id: string;
- name: string;
- size: number;
- location: Location;
- format: string;
- status?: Asset["status"];
-};
-
-type Options =
- | ({
- type: "image";
- meta: ImageMeta;
- } & BaseOptions)
- | ({ type: "font"; meta: FontMeta } & BaseOptions);
-
-export const createAssetWithLimit = async (
- projectId: Project["id"],
- uploadAsset: () => Promise<Options>,
- context: AppContext,
- options: {
- maxAssetsPerProject: number;
- }
-) => {
- let updated: { id: string } | undefined;
-
- const canEdit = await authorizeProject.hasProjectPermit(
- { projectId, permit: "edit" },
- context
- );
-
- if (canEdit === false) {
- throw new Error("You don't have access to create this project assets");
- }
-
- try {
- /**
- * sometimes for example on request timeout we don't know what happened to the "UPLOADING" asset,
- * so we don't take into account assets with the "UPLOADING" status that were created more
- * than UPLOADING_STALE_TIMEOUT milliseconds ago
- **/
- const UPLOADING_STALE_TIMEOUT = 1000 * 60 * 30; // 30 minutes
-
- const count = await prisma.asset.count({
- where: {
- OR: [
- { projectId, status: "UPLOADED" },
- {
- projectId,
- status: "UPLOADING",
- createdAt: { gt: new Date(Date.now() - UPLOADING_STALE_TIMEOUT) },
- },
- ],
- },
- });
-
- if (count >= options.maxAssetsPerProject) {
- /**
- * Here is right to write `Max ${MAX_ASSETS_PER_PROJECT}` but see the comment below,
- * it's probable that the user can exceed the limit a little bit.
- * So it can be a little bit strange that the limit is 5 but the user already has 7.
- **/
- throw new Error(`The maximum number of assets per project is ${count}.`);
- }
-
- /**
- * Create a temporary "UPLOADING" asset, so it can be counted in the next query
- * Assumptions:
- * - it's possible to create more assets than MAX_ASSETS_PER_PROJECT,
- * but for now we assume that the time since the `count` query above and the `create` query below is negligible,
- * and some kind of rate limiting exists on API.
- * Also no locking exists in Prisma, and no raw query locking like
- * "SELECT id FROM "Project" where id=? FOR UPDATE;" is shareable between sqlite and postgres.
- **/
-
- updated = await prisma.asset.create({
- data: {
- projectId,
- status: "UPLOADING",
- format: "unknown",
- location: "REMOTE",
- name: "unknown",
- size: 0,
- },
- select: {
- id: true,
- },
- });
-
- const asset = await uploadAsset();
-
- const size = asset.size ?? 0;
- const { id, meta, format, name, location } = asset;
-
- const dbAsset = await prisma.asset.update({
- where: {
- id_projectId: { id: updated.id, projectId },
- },
- data: {
- id,
- location,
- name,
- size,
- format,
- projectId,
- meta: JSON.stringify(meta),
- status: "UPLOADED",
- },
- });
-
- return formatAsset(dbAsset);
- } catch (error) {
- if (updated) {
- await prisma.asset.delete({
- where: {
- id_projectId: { id: updated.id, projectId },
- },
- });
- }
-
- throw error;
- }
-};
diff --git a/packages/asset-uploader/src/db/index.ts b/packages/asset-uploader/src/db/index.ts
index 3595c75e02..0119e4f719 100644
--- a/packages/asset-uploader/src/db/index.ts
+++ b/packages/asset-uploader/src/db/index.ts
@@ -1,4 +1,3 @@
-export * from "./create";
export * from "./load";
export * from "./delete";
export * from "./clone";
diff --git a/packages/asset-uploader/src/upload.ts b/packages/asset-uploader/src/upload.ts
index f164dfcc42..72619743f5 100644
--- a/packages/asset-uploader/src/upload.ts
+++ b/packages/asset-uploader/src/upload.ts
@@ -1,30 +1,125 @@
-import type { AppContext } from "@webstudio-is/trpc-interface/index.server";
+import { prisma } from "@webstudio-is/prisma-client";
+import {
+ type AppContext,
+ authorizeProject,
+} from "@webstudio-is/trpc-interface/index.server";
import type { AssetClient } from "./client";
-import { createAssetWithLimit } from "./db/create";
-
-export const uploadAssets = async (
- {
- request,
- projectId,
- maxAssetsPerProject,
- }: {
- request: Request;
- projectId: string;
- maxAssetsPerProject: number;
- },
- context: AppContext,
+import { getUniqueFilename } from "./utils/get-unique-filename";
+import { sanitizeS3Key } from "./utils/sanitize-s3-key";
+import { formatAsset } from "./utils/format-asset";
+
+type UploadData = {
+ projectId: string;
+ assetId: string;
+ filename: string;
+ maxAssetsPerProject: number;
+};
+
+const UPLOADING_STALE_TIMEOUT = 1000 * 60 * 30; // 30 minutes
+
+export const createUploadName = async (
+ data: UploadData,
+ context: AppContext
+) => {
+ const { projectId, maxAssetsPerProject, assetId, filename } = data;
+ const canEdit = await authorizeProject.hasProjectPermit(
+ { projectId, permit: "edit" },
+ context
+ );
+ if (canEdit === false) {
+ throw new Error("You don't have access to create this project assets");
+ }
+
+ /**
+ * sometimes for example on request timeout we don't know what happened to the "UPLOADING" asset,
+ * so we don't take into account assets with the "UPLOADING" status that were created more
+ * than UPLOADING_STALE_TIMEOUT milliseconds ago
+ **/
+
+ const count = await prisma.asset.count({
+ where: {
+ OR: [
+ { projectId, status: "UPLOADED" },
+ {
+ projectId,
+ status: "UPLOADING",
+ createdAt: { gt: new Date(Date.now() - UPLOADING_STALE_TIMEOUT) },
+ },
+ ],
+ },
+ });
+
+ if (count >= maxAssetsPerProject) {
+ /**
+ * Here is right to write `Max ${MAX_ASSETS_PER_PROJECT}` but see the comment below,
+ * it's probable that the user can exceed the limit a little bit.
+ * So it can be a little bit strange that the limit is 5 but the user already has 7.
+ **/
+ throw new Error(`The maximum number of assets per project is ${count}.`);
+ }
+
+ /**
+ * Create a temporary "UPLOADING" asset, so it can be counted in the next query
+ * Assumptions:
+ * - it's possible to create more assets than MAX_ASSETS_PER_PROJECT,
+ * but for now we assume that the time since the `count` query above and the `create` query below is negligible,
+ * and some kind of rate limiting exists on API.
+ * Also no locking exists in Prisma, and no raw query locking like
+ * "SELECT id FROM "Project" where id=? FOR UPDATE;" is shareable between sqlite and postgres.
+ **/
+ const name = getUniqueFilename(sanitizeS3Key(filename));
+ await prisma.asset.create({
+ data: {
+ id: assetId,
+ name,
+ projectId,
+ status: "UPLOADING",
+ format: "unknown",
+ location: "REMOTE",
+ size: 0,
+ },
+ });
+ return name;
+};
+
+export const uploadFile = async (
+ { name, request }: { name: string; request: Request },
client: AssetClient
) => {
- const asset = await createAssetWithLimit(
- projectId,
- () => {
- return client.uploadFile(request);
+ const asset = await prisma.asset.findFirst({
+ where: {
+ name: { equals: name },
+ status: { equals: "UPLOADING" },
+ createdAt: { gt: new Date(Date.now() - UPLOADING_STALE_TIMEOUT) },
},
- context,
- {
- maxAssetsPerProject,
- }
- );
+ });
+ if (asset === null) {
+ throw Error("File already uploaded or url is expired");
+ }
+
+ try {
+ const assetData = await client.uploadFile(request);
+ const { meta, format, location, size } = assetData;
+ const dbAsset = await prisma.asset.update({
+ where: {
+ id_projectId: { id: asset.id, projectId: asset.projectId },
+ },
+ data: {
+ location,
+ size,
+ format,
+ meta: JSON.stringify(meta),
+ status: "UPLOADED",
+ },
+ });
+ return formatAsset(dbAsset);
+ } catch (error) {
+ await prisma.asset.delete({
+ where: {
+ id_projectId: { id: asset.id, projectId: asset.projectId },
+ },
+ });
- return [asset];
+ throw error;
+ }
};