Skip to content

Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.

24.3 KB
Raw Blame
import { describe, expect, test, vi } from "vitest";
import type { ContentArtifactV1 } from "@webstudio-is/content-engine";
import { parseMdxDocumentRecovering } from "@webstudio-is/content-engine/mdx";
import {
  createStructuredAssetQueryResourceBody,
  encodeDataVariableId,
  type DataSource,
  type Instance,
  type Pages,
  type Prop,
  type Resource,
} from "@webstudio-is/sdk";
import {
  createBuildContentCompilationPlan,
  createPublishedMdxDependencyClosureResolver,
  createPublishedBuildContentCompilationPlan,
  getDynamicPublishedMdxSourceBlockIds,
  getPublishedMdxContentDatabaseMaxBytes,
  resolvePublishedMdxAssetCandidates,
  resolvePublishedMdxDependencyClosure,
} from "./content-database";

const createBuild = ({
  component = "ws:block",
  sourceType = "asset",
}: {
  component?: string;
  sourceType?: "asset" | "expression" | "string";
}): {
  instances: Instance[];
  props: Prop[];
  dataSources: DataSource[];
  resources: Resource[];
  pages: Pages;
} => ({
  instances: [
    {
      type: "instance" as const,
      id: "block",
      component,
      children: [],
    },
  ],
  props: [
    {
      id: "source",
      instanceId: "block",
      name: "src",
      type: sourceType,
      value: sourceType === "expression" ? '"article.mdx"' : "article.mdx",
    },
  ],
  dataSources: [],
  resources: [],
  pages: {
    meta: {},
    homePageId: "home",
    rootFolderId: "root-folder",
    pages: new Map([
      [
        "home",
        {
          id: "home",
          path: "",
          name: "Home",
          title: "Home",
          rootInstanceId: "block",
          meta: {},
        },
      ],
    ]),
    folders: new Map(),
  },
});

describe("Content Block MDX compilation", () => {
  test("reuses parsed MDX while a publication plan converges", async () => {
    const source = "# Article\n\nBody";
    const artifact = {
      format: "webstudio-content-database",
      version: 1,
      documents: [
        {
          _id: "article.mdx",
          _type: "asset.file",
          name: "article.mdx",
          path: "article.mdx",
          key: "article",
          extension: "mdx",
          mimeType: "text/mdx",
          size: source.length,
          revision: "article-revision",
          contentRef: "article.mdx",
        },
      ],
      contents: { "article.mdx": source },
    } as unknown as ContentArtifactV1;
    const parseDocument = vi.fn(parseMdxDocumentRecovering);
    const resolve = createPublishedMdxDependencyClosureResolver({
      parseMdxDocumentRecovering: parseDocument,
    });
    const build = createBuild({});
    build.instances[0].children = [{ type: "id", value: "templates" }];
    build.instances.push({
      type: "instance",
      id: "templates",
      component: "ws:block-template",
      children: [],
    });

    await resolve({ build, artifact });
    await resolve({ build, artifact });

    expect(parseDocument).toHaveBeenCalledOnce();

    await resolve({
      build,
      artifact: {
        ...artifact,
        documents: artifact.documents.map((document) => ({
          ...document,
          revision: "article-revision-2",
        })),
      },
    });

    expect(parseDocument).toHaveBeenCalledTimes(2);
  });

  test("adds a separately bounded MDX body budget", () => {
    expect(
      getPublishedMdxContentDatabaseMaxBytes({
        baseBytes: 500,
        assets: [
          { type: "file", format: "mdx", size: 10_000_000 },
          { type: "file", format: "mdx", size: 30_000_000 },
          { type: "file", format: "md", size: 10_000_000 },
        ],
      })
    ).toBe(32 * 1024 * 1024 + 500);
  });
  test.each(["asset", "expression"] as const)(
    "retains the exact MDX dependency for a %s source",
    (sourceType) => {
      const plan = createPublishedBuildContentCompilationPlan(
        createBuild({ sourceType })
      );

      expect(plan?.queries).toContainEqual({
        id: "__content-block-mdx__:article.mdx",
        result: "many",
        where: {
          all: [
            {
              field: ["extension"],
              operator: "eq",
              value: { type: "literal", value: "mdx" },
            },
            {
              field: ["id"],
              operator: "eq",
              value: { type: "literal", value: "article.mdx" },
            },
          ],
        },
        sort: [],
        limit: { type: "dynamic" },
        offset: { type: "literal", value: 0 },
        output: {
          mode: "fields",
          includeMetadata: false,
          fields: [["url"], ["width"], ["height"]],
        },
        content: { mode: "full" },
      });
    }
  );

  test("does not retain MDX for unrelated src props", () => {
    expect(
      createPublishedBuildContentCompilationPlan(
        createBuild({ component: "ws:element" })
      )
    ).toBeUndefined();
  });

  test("does not retain MDX for an invalid Content Block source type", () => {
    expect(
      createPublishedBuildContentCompilationPlan(
        createBuild({ sourceType: "string" })
      )
    ).toBeUndefined();
  });

  test("does not retain conflicting duplicate Content Block sources", () => {
    const build = createBuild({});
    build.props.push({
      id: "duplicate-source",
      instanceId: "block",
      name: "src",
      type: "asset",
      value: "private.mdx",
    });

    expect(createPublishedBuildContentCompilationPlan(build)).toBeUndefined();
    expect(getDynamicPublishedMdxSourceBlockIds(build)).toEqual([]);
    expect(resolvePublishedMdxAssetCandidates({ build })).toEqual(new Map());
  });

  test("does not retain MDX used only by a draft page", () => {
    const build = createBuild({});
    build.instances.push({
      type: "instance",
      id: "draft-block",
      component: "ws:block",
      children: [],
    });
    build.props.push({
      id: "draft-source",
      instanceId: "draft-block",
      name: "src",
      type: "asset",
      value: "private-draft.mdx",
    });
    build.pages.pages.set("draft", {
      id: "draft",
      path: "draft",
      name: "Draft",
      title: "Draft",
      rootInstanceId: "draft-block",
      meta: {},
      isDraft: true,
    });

    const plan = createPublishedBuildContentCompilationPlan(build);

    expect(plan?.queries.map(({ id }) => id)).not.toContain(
      "__content-block-mdx__:private-draft.mdx"
    );
  });

  test("discovers nested sources when the MDX body must be loaded", async () => {
    const build = createBuild({});
    build.instances[0].children = [{ type: "id", value: "templates" }];
    build.instances.push(
      {
        type: "instance",
        id: "templates",
        component: "ws:block-template",
        children: [
          { type: "id", value: "hero" },
          { type: "id", value: "unused" },
        ],
      },
      {
        type: "instance",
        id: "hero",
        component: "ws:element",
        tag: "section",
        label: "Hero",
        children: [{ type: "id", value: "nested" }],
      },
      {
        type: "instance",
        id: "nested",
        component: "ws:block",
        children: [],
      },
      {
        type: "instance",
        id: "unused",
        component: "ws:block",
        label: "Unused",
        children: [],
      }
    );
    build.props.push(
      {
        id: "nested-source",
        instanceId: "nested",
        name: "src",
        type: "asset",
        value: "nested.mdx",
      },
      {
        id: "unused-source",
        instanceId: "unused",
        name: "src",
        type: "asset",
        value: "private.mdx",
      }
    );
    const source = '{unsafe}\n\n<ws.element ws:name="Hero" />\n\n<Missing />';
    const artifact = {
      format: "webstudio-content-database",
      version: 1,
      documents: [
        {
          _id: "article.mdx",
          _type: "asset.file",
          name: "article.mdx",
          path: "article.mdx",
          key: "article",
          extension: "mdx",
          mimeType: "text/mdx",
          size: source.length,
          revision: "article-revision",
          contentRef: "article.mdx",
        },
      ],
      contents: {},
    } as unknown as ContentArtifactV1;

    const omissions: { assetId: string; templateName: string }[] = [];
    const loadDocumentSource = vi.fn(async () => source);
    const plan = await resolvePublishedMdxDependencyClosure({
      build,
      artifact,
      onTemplateOmission: (issue) => omissions.push(issue),
      loadDocumentSource,
    });
    expect(loadDocumentSource).toHaveBeenCalledExactlyOnceWith({
      id: "article.mdx",
      revision: "article-revision",
      contentRef: "article.mdx",
    });
    expect(omissions).toEqual([
      expect.objectContaining({
        assetId: "article.mdx",
        templateName: "Missing",
      }),
    ]);
    const queryIds = plan?.queries.map(({ id }) => id);

    expect(queryIds).toContain("__content-block-mdx__:nested.mdx");
    expect(queryIds).not.toContain("__content-block-mdx__:private.mdx");
  });

  test("keeps dependency planning non-blocking when source loading fails", async () => {
    const build = createBuild({});
    const artifact = {
      format: "webstudio-content-database",
      version: 1,
      documents: [
        {
          _id: "article.mdx",
          _type: "asset.file",
          name: "article.mdx",
          path: "article.mdx",
          key: "article",
          extension: "mdx",
          mimeType: "text/mdx",
          size: 1,
          revision: "article-revision",
          contentRef: "article.mdx",
        },
      ],
      contents: {},
    } as unknown as ContentArtifactV1;

    const failures: unknown[] = [];
    const plan = await resolvePublishedMdxDependencyClosure({
      build,
      artifact,
      loadDocumentSource: async () => {
        throw new Error("asset store unavailable");
      },
      onDependencyReadFailure: (failure) => failures.push(failure),
    });

    expect(plan?.queries.map(({ id }) => id)).toContain(
      "__content-block-mdx__:article.mdx"
    );
    expect(failures).toEqual([
      {
        blockInstanceId: expect.any(String),
        assetId: "article.mdx",
        contentRef: "article.mdx",
        message: expect.stringContaining("asset store unavailable"),
      },
    ]);
  });

  test("loads MDX dependencies in the same publication depth concurrently", async () => {
    const build = createBuild({});
    build.instances[0].children = [
      { type: "id", value: "first-templates" },
      { type: "id", value: "second-block" },
    ];
    build.instances.push({
      type: "instance",
      id: "second-block",
      component: "ws:block",
      children: [{ type: "id", value: "second-templates" }],
    });
    build.instances.push(
      {
        type: "instance",
        id: "first-templates",
        component: "ws:block-template",
        children: [],
      },
      {
        type: "instance",
        id: "second-templates",
        component: "ws:block-template",
        children: [],
      }
    );
    build.props.push({
      id: "second-source",
      instanceId: "second-block",
      name: "src",
      type: "asset",
      value: "second.mdx",
    });
    const createDocument = (id: string) => ({
      _id: id,
      _type: "asset.file",
      name: id,
      path: id,
      key: id,
      extension: "mdx",
      mimeType: "text/mdx",
      size: 1,
      revision: `${id}-revision`,
      contentRef: id,
    });
    const artifact = {
      format: "webstudio-content-database",
      version: 1,
      documents: [createDocument("article.mdx"), createDocument("second.mdx")],
      contents: {},
    } as unknown as ContentArtifactV1;
    let resolveBothReads: () => void = () => {};
    const bothReadsStarted = new Promise<void>((resolve) => {
      resolveBothReads = resolve;
    });
    const startedIds: string[] = [];

    const plan = await resolvePublishedMdxDependencyClosure({
      build,
      artifact,
      loadDocumentSource: async ({ id }) => {
        startedIds.push(id);
        if (startedIds.length === 2) {
          resolveBothReads();
        }
        await bothReadsStarted;
        return "# Article";
      },
    });

    expect(startedIds).toEqual(["article.mdx", "second.mdx"]);
    expect(plan?.queries.map(({ id }) => id)).toEqual(
      expect.arrayContaining([
        "__content-block-mdx__:article.mdx",
        "__content-block-mdx__:second.mdx",
      ])
    );
  });

  test("retains nested sources through semantic component templates", async () => {
    const build = createBuild({});
    build.instances[0].children = [{ type: "id", value: "templates" }];
    build.instances.push(
      {
        type: "instance",
        id: "templates",
        component: "ws:block-template",
        children: [{ type: "id", value: "paragraph" }],
      },
      {
        type: "instance",
        id: "paragraph",
        component: "Paragraph",
        children: [{ type: "id", value: "nested" }],
      },
      {
        type: "instance",
        id: "nested",
        component: "ws:block",
        children: [],
      }
    );
    build.props.push({
      id: "nested-source",
      instanceId: "nested",
      name: "src",
      type: "asset",
      value: "nested.mdx",
    });
    const source = "Authored paragraph\n";
    const artifact = {
      format: "webstudio-content-database",
      version: 1,
      documents: [
        {
          _id: "article.mdx",
          _type: "asset.file",
          name: "article.mdx",
          path: "article.mdx",
          key: "article",
          extension: "mdx",
          mimeType: "text/mdx",
          size: source.length,
          revision: "article-revision",
          contentRef: "article.mdx",
        },
      ],
      contents: { "article.mdx": source },
    } as unknown as ContentArtifactV1;

    const plan = await resolvePublishedMdxDependencyClosure({
      build,
      artifact,
    });

    expect(plan?.queries.map(({ id }) => id)).toContain(
      "__content-block-mdx__:nested.mdx"
    );
  });

  test("keeps internal MDX retention out of Assets Resource plans", () => {
    expect(createBuildContentCompilationPlan(createBuild({}))).toBeUndefined();
  });

  test("omits an expression without a bounded publication dependency set", () => {
    const build = createBuild({ sourceType: "expression" });
    build.props[0].value = "post.body";

    expect(createPublishedBuildContentCompilationPlan(build)).toBeUndefined();
  });

  test.each(["detail", "collection"] as const)(
    "derives finite %s candidates while excluding unrelated MDX",
    (kind) => {
      const resourceVariable = encodeDataVariableId("posts-data");
      const itemVariable = encodeDataVariableId("collection-item");
      const build = createBuild({ sourceType: "expression" });
      build.instances.unshift({
        type: "instance",
        id: "collection",
        component: "ws:collection",
        children: [{ type: "id", value: "block" }],
      });
      build.props[0].value =
        kind === "detail"
          ? `${resourceVariable}.data.properties.mdx`
          : `${itemVariable}.properties.mdx`;
      if (kind === "collection") {
        build.props.push({
          id: "collection-data",
          instanceId: "collection",
          name: "data",
          type: "expression",
          value: `${resourceVariable}.data`,
        });
        build.props.push({
          id: "collection-item-prop",
          instanceId: "collection",
          name: "item",
          type: "parameter",
          value: "collection-item",
        });
        build.dataSources.push({
          type: "parameter",
          id: "collection-item",
          scopeInstanceId: "collection",
          name: "collectionItem",
        });
      }
      build.dataSources.push({
        type: "resource",
        id: "posts-data",
        scopeInstanceId: "collection",
        name: "posts",
        resourceId: "posts",
      });
      build.resources.push({
        id: "posts",
        name: "Posts",
        control: "system",
        method: "post",
        url: '"/$resources/assets"',
        headers: [],
        body: createStructuredAssetQueryResourceBody({
          where: {
            all: [
              {
                field: ["properties", "category"],
                operator: "eq",
                value: '"published"',
              },
            ],
          },
          sort: [],
          limit: "1",
          offset: "0",
          output: { mode: "all", includeMetadata: true },
          content: { mode: "none" },
        }),
      });
      const artifact = {
        documentGraph: {
          nodes: [
            { id: "post", revision: "r1", contentRef: "post" },
            { id: "author", revision: "r1", contentRef: "author" },
          ],
          edges: [
            {
              sourceId: "post",
              referenceId: "#frontmatter/author",
              reference: {
                documentId: "author",
                revision: "r1",
                representation: { type: "document" },
              },
            },
          ],
        },
        documents: [
          {
            _id: "post",
            name: "post.json",
            path: "post.json",
            key: "post",
            extension: "json",
            mimeType: "application/json",
            size: 1,
            properties: { category: "published", mdx: "article.mdx" },
          },
          {
            _id: "private-post",
            name: "private.json",
            path: "private.json",
            key: "private",
            extension: "json",
            mimeType: "application/json",
            size: 1,
            properties: { category: "private", mdx: "private.mdx" },
          },
          {
            _id: "second-post",
            name: "second.json",
            path: "second.json",
            key: "second",
            extension: "json",
            mimeType: "application/json",
            size: 1,
            properties: { category: "published", mdx: "second.mdx" },
          },
        ],
      } as unknown as ContentArtifactV1;

      const candidates = resolvePublishedMdxAssetCandidates({
        build,
        artifact,
      });
      const plan = createPublishedBuildContentCompilationPlan(
        build,
        candidates
      );

      expect(candidates.get("block")).toEqual(["article.mdx"]);
      expect(plan?.queries.map(({ id }) => id)).toContain(
        "__content-block-mdx__:article.mdx"
      );
      expect(plan?.queries.map(({ id }) => id)).not.toContain(
        "__content-block-mdx__:private.mdx"
      );
      expect(plan?.queries.map(({ id }) => id)).not.toContain(
        "__content-block-mdx__:second.mdx"
      );

      // Returning an unrelated author is safe; using that unresolved reference
      // as the MDX source is still rejected, including inside a Collection.
      build.props[0].value =
        kind === "detail"
          ? `${resourceVariable}.data.properties.author.mdx`
          : `${itemVariable}.properties.author.mdx`;
      expect(() =>
        resolvePublishedMdxAssetCandidates({ build, artifact })
      ).toThrow(
        "Dynamic MDX source candidates through resolved document references"
      );
      expect(
        resolvePublishedMdxAssetCandidates({
          build,
          artifact,
          allowUnresolved: true,
        }).has("block")
      ).toBe(false);
    }
  );

  test("excludes known non-MDX assets from dynamic source candidates", () => {
    const resourceVariable = encodeDataVariableId("posts-data");
    const build = createBuild({ sourceType: "expression" });
    build.props[0].value = `${resourceVariable}.data.id`;
    build.dataSources.push({
      type: "resource",
      id: "posts-data",
      scopeInstanceId: "block",
      name: "posts",
      resourceId: "posts",
    });
    build.resources.push({
      id: "posts",
      name: "Posts",
      control: "system",
      method: "post",
      url: '"/$resources/assets"',
      headers: [],
      body: createStructuredAssetQueryResourceBody({
        where: {
          all: [
            {
              field: ["properties", "slug"],
              operator: "eq",
              value: '"legal"',
            },
          ],
        },
        sort: [],
        limit: "10",
        offset: "0",
        output: { mode: "all", includeMetadata: false },
        content: { mode: "none" },
      }),
    });
    const artifact = {
      documents: [
        {
          _id: "article.mdx",
          name: "article.mdx",
          path: "MDX-Data/Guidelines/article.mdx",
          key: "article",
          extension: "mdx",
          mimeType: "text/mdx",
          size: 1,
          properties: { slug: "legal", mdx: "collection.json" },
        },
        {
          _id: "collection.json",
          name: "collection.json",
          path: "MDX-Data/Guidelines/collection.json",
          key: "collection",
          extension: "json",
          mimeType: "application/json",
          size: 1,
          properties: { slug: "legal" },
        },
      ],
    } as unknown as ContentArtifactV1;

    expect(
      resolvePublishedMdxAssetCandidates({
        build,
        artifact,
        allAssetIds: new Set(["article.mdx", "collection.json"]),
        allMdxAssetIds: new Set(["article.mdx"]),
      }).get("block")
    ).toEqual(["article.mdx"]);

    // An explicitly referenced file must survive candidate discovery so
    // materialization can report that it is not an MDX asset.
    build.props[0].value = `${resourceVariable}.data.properties.mdx`;
    expect(
      resolvePublishedMdxAssetCandidates({
        build,
        artifact,
        allAssetIds: new Set(["article.mdx", "collection.json"]),
        allMdxAssetIds: new Set(["article.mdx"]),
      }).get("block")
    ).toEqual(["collection.json"]);
  });

  test("allows a valid Content Block query with no matching documents", () => {
    const build = createBuild({ sourceType: "expression" });
    build.props[0].value = `${encodeDataVariableId("posts-data")}.data.id`;
    build.dataSources.push({
      type: "resource",
      id: "posts-data",
      scopeInstanceId: "block",
      name: "posts",
      resourceId: "posts",
    });
    build.resources.push({
      id: "posts",
      name: "Posts",
      control: "system",
      method: "post",
      url: '"/$resources/assets"',
      headers: [],
      body: createStructuredAssetQueryResourceBody({
        where: { all: [] },
        sort: [],
        limit: "10",
        offset: "0",
        output: { mode: "all", includeMetadata: true },
        content: { mode: "none" },
      }),
    });

    const candidates = resolvePublishedMdxAssetCandidates({
      build,
      artifact: { documents: [] } as unknown as ContentArtifactV1,
    });
    expect(candidates.get("block")).toEqual([]);
    expect(
      createPublishedBuildContentCompilationPlan(build, candidates)?.queries
    ).not.toContainEqual(
      expect.objectContaining({
        id: expect.stringContaining("__content-block-mdx__"),
      })
    );
    expect(() => resolvePublishedMdxAssetCandidates({ build })).toThrow(
      'Content Block "block" has no finite dynamic MDX Asset candidates'
    );

    // A matching document with no src field is not an empty query result.
    build.props[0].value = `${encodeDataVariableId("posts-data")}.data.properties.missing`;
    expect(() =>
      resolvePublishedMdxAssetCandidates({
        build,
        artifact: {
          documents: [
            {
              _id: "article.mdx",
              name: "article.mdx",
              path: "posts/article.mdx",
              key: "article",
              extension: "mdx",
              mimeType: "text/mdx",
              size: 1,
              properties: {},
            },
          ],
        } as unknown as ContentArtifactV1,
      })
    ).toThrow(
      'Content Block "block" has no finite dynamic MDX Asset candidates'
    );
  });

  test("bounds a mutable project variable to the project's MDX assets", () => {
    const build = createBuild({ sourceType: "expression" });
    build.props[0].value = encodeDataVariableId("article-source");
    build.dataSources.push({
      type: "variable",
      id: "article-source",
      scopeInstanceId: "block",
      name: "articleSource",
      value: { type: "string", value: "article.mdx" },
    });

    expect(
      resolvePublishedMdxAssetCandidates({
        build,
        allMdxAssetIds: new Set(["article.mdx", "another-article.mdx"]),
      }).get("block")
    ).toEqual(["another-article.mdx", "article.mdx"]);
    expect(() => resolvePublishedMdxAssetCandidates({ build })).toThrow(
      'Content Block "block" has no finite dynamic MDX Asset candidates'
    );
  });
});