Skip to content

Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.

15.1 KB
Raw Blame
import {
  type AppContext,
  authorizeProject,
  AuthorizationError,
  getProjectPlanFeatures,
} from "@webstudio-is/trpc-interface/index.server";
import { createHash } from "node:crypto";
import {
  createId as createSdkId,
  getFileExtension,
  getFileNameParts,
  type Asset,
} from "@webstudio-is/sdk";
import type { Database } from "@webstudio-is/postgrest/index.server";
import type { AssetInfoFallback, AssetObjectWriter } from "./client";
import type { AssetDataOverride } from "./utils/get-asset-data";
import { createUniqueAssetFilename } from "./utils/get-unique-filename";
import { sanitizeS3Key } from "./utils/sanitize-s3-key";
import { formatAsset, formatAssetForRead } from "./utils/format-asset";
import { assertPostgrestSuccess } from "./patch-utils";
import type { UploadTicket } from "./types";

export type CreateUploadTicketInput = {
  projectId: string;
  type: string;
  filename: string;
  displayFilename?: string;
  description?: string;
  folderId?: string;
  contentHash?: string;
};

export class AssetUploadCountLimitError extends Error {
  constructor(maxAssetsPerProject: number) {
    super(
      `The maximum number of assets per project is ${maxAssetsPerProject}.`
    );
  }
}

const UPLOADING_STALE_TIMEOUT = 1000 * 60 * 30; // 30 minutes
const maxCreateUploadTicketAttempts = 3;
const contentHashUploadPollInterval = 100;
const contentHashUploadPollAttempts = 50;

type AssetIdentity = {
  filename: string;
  description: string | null;
  folderId: string | null;
};

const getDeduplicatedAssetId = (
  projectId: string,
  name: string,
  identity: AssetIdentity
) =>
  createHash("sha256")
    .update(
      JSON.stringify([
        projectId,
        name,
        identity.filename,
        identity.description,
        identity.folderId,
      ])
    )
    .digest("base64url")
    .slice(0, 21);

const assertAssetCapacity = async (projectId: string, context: AppContext) => {
  const { maxAssetsPerProject } = await getProjectPlanFeatures(
    projectId,
    context
  );
  const assetCount = await context.postgrest.client
    .from("Asset")
    .select("id, file:File!inner(status)", { count: "exact", head: true })
    .eq("projectId", projectId)
    .eq("file.status", "UPLOADED");
  const uploadingCount = await context.postgrest.client
    .from("File")
    .select("*", { count: "exact", head: true })
    .eq("isDeleted", false)
    .eq("uploaderProjectId", projectId)
    .eq("status", "UPLOADING")
    .gt(
      "createdAt",
      new Date(Date.now() - UPLOADING_STALE_TIMEOUT).toISOString()
    );
  const count = (assetCount.count ?? 0) + (uploadingCount.count ?? 0);
  if (count >= maxAssetsPerProject) {
    throw new AssetUploadCountLimitError(maxAssetsPerProject);
  }
};

const findAsset = async ({
  projectId,
  name,
  identity,
  context,
}: {
  projectId: string;
  name: string;
  identity?: AssetIdentity;
  context: AppContext;
}) => {
  let query = context.postgrest.client
    .from("Asset")
    .select("id, projectId, filename, description, folderId")
    .eq("projectId", projectId)
    .eq("name", name);
  if (identity !== undefined) {
    query = query.eq("filename", identity.filename);
    query =
      identity.description === null
        ? query.is("description", null)
        : query.eq("description", identity.description);
    query =
      identity.folderId === null
        ? query.is("folderId", null)
        : query.eq("folderId", identity.folderId);
  }
  const result = await query.order("id").limit(1).maybeSingle();
  if (result.error) {
    throw new Error(result.error.message);
  }
  return result.data ?? undefined;
};

const getContentDeduplicationExtension = (filename: string) =>
  getFileExtension(filename)?.toLowerCase() ?? "";

const findFileByContent = async ({
  projectId,
  contentHash,
  filename,
  context,
}: {
  projectId: string;
  contentHash: string;
  filename: string;
  context: AppContext;
}) => {
  const files = await context.postgrest.client
    .from("File")
    .select("*")
    .eq("uploaderProjectId", projectId)
    .eq("contentHash", contentHash);
  if (files.error) {
    throw new Error(files.error.message);
  }
  const extension = getContentDeduplicationExtension(filename);
  return files.data?.find(
    (file) => getContentDeduplicationExtension(file.name) === extension
  );
};

const findContentHashUploadTicket = async ({
  data,
  context,
}: {
  data: CreateUploadTicketInput & { contentHash: string };
  context: AppContext;
}): Promise<UploadTicket | undefined> => {
  const {
    projectId,
    contentHash,
    filename,
    displayFilename = getFileNameParts(filename).basename,
    description,
    folderId,
  } = data;
  const assetIdentity: AssetIdentity = {
    filename: displayFilename,
    description: description ?? null,
    folderId: folderId ?? null,
  };
  for (let attempt = 0; attempt < contentHashUploadPollAttempts; attempt += 1) {
    const file = await findFileByContent({
      projectId,
      contentHash,
      filename,
      context,
    });
    if (file === undefined) {
      return;
    }
    if (file.status === "UPLOADED") {
      const existingAsset = await findAsset({
        projectId,
        name: file.name,
        identity: assetIdentity,
        context,
      });
      let asset = existingAsset ?? {
        id: getDeduplicatedAssetId(projectId, file.name, assetIdentity),
        projectId,
        ...assetIdentity,
      };
      const { asset: formattedAsset, fontMetaIssues } = formatAssetForRead({
        assetId: asset.id,
        projectId: asset.projectId,
        filename: asset.filename,
        description: asset.description,
        folderId: asset.folderId,
        file,
      });
      if (fontMetaIssues !== undefined) {
        // Keep existing references, but free the unique content hash so a new
        // upload can parse valid metadata instead of reusing the broken file.
        const excludedFile = await context.postgrest.client
          .from("File")
          .update({ contentHash: null })
          .eq("name", file.name)
          .eq("uploaderProjectId", projectId)
          .eq("contentHash", contentHash)
          .eq("status", "UPLOADED")
          .eq("format", file.format)
          .eq("meta", file.meta)
          .select("name")
          .maybeSingle();
        assertPostgrestSuccess(excludedFile);
        if (excludedFile.data === null) {
          // Another request changed the file; reconsider its current metadata.
          continue;
        }
        return;
      }
      if (existingAsset === undefined) {
        await assertAssetCapacity(projectId, context);
        const insertedAsset = await context.postgrest.client
          .from("Asset")
          .insert({ ...asset, name: file.name });
        if (insertedAsset.error?.code === "23505") {
          const concurrentAsset = await findAsset({
            projectId,
            name: file.name,
            identity: assetIdentity,
            context,
          });
          if (concurrentAsset === undefined) {
            throw new Error("Concurrent deduplicated asset is missing.");
          }
          asset = concurrentAsset;
        } else {
          assertPostgrestSuccess(insertedAsset);
        }
      }
      if (file.isDeleted) {
        const restoredFile = await context.postgrest.client
          .from("File")
          .update({ isDeleted: false })
          .eq("name", file.name)
          .eq("uploaderProjectId", projectId);
        assertPostgrestSuccess(restoredFile);
      }
      return {
        assetId: asset.id,
        name: file.name,
        deduplicated: true,
        asset: { ...formattedAsset, id: asset.id },
      };
    }
    if (
      new Date(file.createdAt).getTime() <
      Date.now() - UPLOADING_STALE_TIMEOUT
    ) {
      const deletedAsset = await context.postgrest.client
        .from("Asset")
        .delete()
        .eq("projectId", projectId)
        .eq("name", file.name);
      assertPostgrestSuccess(deletedAsset);
      const deletedFile = await context.postgrest.client
        .from("File")
        .delete()
        .eq("name", file.name);
      assertPostgrestSuccess(deletedFile);
      return;
    }
    await new Promise((resolve) =>
      setTimeout(resolve, contentHashUploadPollInterval)
    );
  }
  throw new Error(
    "An identical asset upload is still in progress. Retry after it completes."
  );
};

export type UploadErrorCleanup = (
  name: string,
  context: AppContext
) => Promise<void>;

const cleanupUploadError = async (
  name: string,
  context: AppContext,
  onUploadError?: UploadErrorCleanup
) => {
  if (onUploadError) {
    await onUploadError(name, context);
    return;
  }
  await context.postgrest.client.from("Asset").delete().eq("name", name);
  await context.postgrest.client.from("File").delete().eq("name", name);
};

export const createUploadTicket = async (
  data: CreateUploadTicketInput,
  context: AppContext,
  createId: () => Asset["id"] = () => createSdkId("nano")
): Promise<UploadTicket> => {
  const {
    projectId,
    type,
    filename,
    displayFilename = getFileNameParts(filename).basename,
    description,
    folderId,
    contentHash,
  } = data;
  const sanitizedFilename = sanitizeS3Key(filename);
  const canEdit = await authorizeProject.hasProjectPermit(
    { projectId, permit: "edit" },
    context
  );
  if (canEdit === false) {
    throw new AuthorizationError(
      "You don't have access to create this project assets"
    );
  }

  if (contentHash !== undefined) {
    const existing = await findContentHashUploadTicket({
      data: { ...data, contentHash },
      context,
    });
    if (existing !== undefined) {
      return existing;
    }
  }

  await assertAssetCapacity(projectId, context);

  /**
   * Create a temporary "UPLOADING" asset, so it can be counted in the next query
   * Assumptions:
   * - it's possible to create more assets than maxAssetsPerProject,
   *   but for now we assume that the time since the `count` query above and the `create` query below is negligible,
   *   and some kind of rate limiting exists on API.
   * Also no locking exists in Prisma, and no raw query locking like
   * "SELECT id FROM "Project" where id=? FOR UPDATE;" is shareable between sqlite and postgres.
   **/
  for (
    let attempt = 1;
    attempt <= maxCreateUploadTicketAttempts;
    attempt += 1
  ) {
    const assetId = createId();
    const name = createUniqueAssetFilename(sanitizedFilename);

    const fileInsert = await context.postgrest.client.from("File").insert({
      name,
      status: "UPLOADING",
      // store content type in related field
      format: type,
      size: 0,
      uploaderProjectId: projectId,
      contentHash,
    });
    if (fileInsert.error) {
      if (fileInsert.error.code === "23505" && contentHash !== undefined) {
        const existing = await findContentHashUploadTicket({
          data: { ...data, contentHash },
          context,
        });
        if (existing !== undefined) {
          return existing;
        }
      }
      throw new Error(fileInsert.error.message);
    }

    const assetInsert = await context.postgrest.client.from("Asset").insert({
      id: assetId,
      projectId,
      name,
      filename: displayFilename,
      description,
      folderId,
    });
    if (assetInsert.error) {
      await context.postgrest.client.from("File").delete().eq("name", name);
      if (
        assetInsert.error.code === "23505" &&
        attempt < maxCreateUploadTicketAttempts
      ) {
        continue;
      }
      throw new Error(assetInsert.error.message);
    }

    return { assetId, name, deduplicated: false };
  }

  throw new Error("Unable to reserve asset upload.");
};

export const uploadFileData = async (
  name: string,
  data: ReadableStream<Uint8Array>,
  client: AssetObjectWriter,
  context: AppContext,
  assetInfoFallback: AssetInfoFallback | undefined,
  assetDataOverride?: AssetDataOverride,
  onUploadError?: UploadErrorCleanup
): Promise<Database["public"]["Tables"]["File"]["Row"]> => {
  let file = await context.postgrest.client
    .from("File")
    .select("*")
    .eq("name", name)
    .eq("status", "UPLOADING")
    .gt(
      "updatedAt",
      new Date(Date.now() - UPLOADING_STALE_TIMEOUT).toISOString()
    )
    .single();
  if (file.data === null) {
    throw Error("File already uploaded or url is expired");
  }

  try {
    const expectedContentHash = file.data.contentHash ?? undefined;
    const contentHasher =
      expectedContentHash === undefined ? undefined : createHash("sha256");
    const uploadData = async function* () {
      for await (const chunk of data as unknown as AsyncIterable<Uint8Array>) {
        contentHasher?.update(chunk);
        yield chunk;
      }
      if (
        expectedContentHash !== undefined &&
        contentHasher?.digest("hex") !== expectedContentHash
      ) {
        throw new Error(
          "Uploaded asset content does not match its upload ticket."
        );
      }
    };
    const assetData = await client.uploadFile(
      name,
      file.data.format,
      uploadData(),
      assetInfoFallback,
      assetDataOverride
    );
    const { meta, format, size } = assetData;
    file = await context.postgrest.client
      .from("File")
      .update({
        size,
        format,
        meta: JSON.stringify(meta),
        status: "UPLOADED",
      })
      .eq("name", name)
      .select()
      .single();
    if (file.data === null) {
      throw Error("File not found");
    }
    return file.data;
  } catch (error) {
    await cleanupUploadError(name, context, onUploadError);
    throw error;
  }
};

export const getUploadedAsset = async ({
  name,
  projectId,
  context,
  file: providedFile,
}: {
  name: string;
  projectId: string;
  context: AppContext;
  file?: Database["public"]["Tables"]["File"]["Row"];
}): Promise<Asset> => {
  let file = providedFile;
  if (file === undefined) {
    const result = await context.postgrest.client
      .from("File")
      .select("*")
      .eq("name", name)
      .eq("status", "UPLOADED")
      .single();
    if (result.error || result.data === null) {
      throw new Error(result.error?.message ?? "File not found");
    }
    file = result.data;
  }
  const asset = await findAsset({ projectId, name, context });
  if (asset === undefined) {
    throw new Error("Asset not found");
  }
  return formatAsset({
    assetId: asset.id,
    projectId: asset.projectId,
    filename: asset.filename,
    description: asset.description,
    folderId: asset.folderId,
    file,
  });
};

export const uploadFile = async (
  name: string,
  data: ReadableStream<Uint8Array>,
  client: AssetObjectWriter,
  context: AppContext,
  assetInfoFallback: AssetInfoFallback | undefined,
  assetDataOverride?: AssetDataOverride,
  onUploadError?: UploadErrorCleanup
): Promise<Asset> => {
  const file = await uploadFileData(
    name,
    data,
    client,
    context,
    assetInfoFallback,
    assetDataOverride,
    onUploadError
  );
  try {
    const projectId = file.uploaderProjectId;
    if (typeof projectId !== "string") {
      throw Error("File uploader project is missing");
    }
    const asset = await getUploadedAsset({
      name,
      projectId,
      context,
      file,
    });
    return asset;
  } catch (error) {
    await cleanupUploadError(name, context, onUploadError);
    throw error;
  }
};