Skip to content

Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.

13.9 KB
Raw Blame
import { mkdir, writeFile } from "node:fs/promises";
import hash from "@emotion/hash";
import {
  coreMetas,
  createScope,
  elementComponent,
  type Instance,
  type Instances,
  type Prop,
  type Props,
} from "@webstudio-is/sdk";
import { generateWebstudioComponent } from "@webstudio-is/react-sdk";
import {
  findByClasses,
  findByTags,
  getAttr,
  getTextContent,
  loadHtmlIndices,
  loadSvgSinglePage,
  parseHtml,
} from "./crawler";
import { possibleStandardNames } from "./possible-standard-names";
import { ignoredTags } from "./overrides";

const validHtmlAttributes = new Set<string>();

type Attribute = {
  name: string;
  description: string;
  required?: boolean;
  type: "string" | "boolean" | "number" | "select" | "url";
  options?: string[];
  defaultValue?: string;
};

const overrides: Record<string, Record<string, false | Partial<Attribute>>> = {
  "*": {
    // react has own opinions about it
    style: false,
    // specific to input in react types
    enterkeyhint: false,
    inert: false,
    popover: false,
    writingsuggestions: false,
    hidden: {
      // "until-found"; "hidden"; the empty string
      type: "boolean",
      options: undefined,
    },
  },
  a: {
    href: { type: "url", required: true },
    target: { required: true },
    download: { type: "boolean", required: true },
  },
  blockquote: {
    cite: { required: true },
  },
  form: {
    action: { required: true },
    method: { required: true },
    enctype: { required: true },
  },
  area: {
    ping: false,
  },
  button: {
    type: { required: true },
    command: false,
    commandfor: false,
    popovertarget: false,
    popovertargetaction: false,
  },
  label: {
    for: { required: true },
  },
  dialog: {
    closedby: false,
  },
  img: {
    src: { required: true },
    alt: { required: true },
    width: { required: true },
    height: { required: true },
    ismap: false,
  },
  input: {
    name: { required: true },
    value: { required: true },
    checked: { required: true },
    type: { required: true, defaultValue: "text" },
    placeholder: { required: true },
    required: { required: true },
    autofocus: { required: true },
    alpha: false,
    colorspace: false,
    // react types have it only in textarea
    dirname: false,
    popovertarget: false,
    popovertargetaction: false,
  },
  textarea: {
    name: { required: true },
    placeholder: { required: true },
    required: { required: true },
    autofocus: { required: true },
  },
  select: {
    name: { required: true },
    required: { required: true },
    autofocus: { required: true },
    // mutltiple mode is not considered accessible
    // and we cannot express it in builder so easier to remove
    multiple: false,
  },
  option: {
    label: { required: true },
    value: { required: true },
    disabled: { required: true },
    // enforce fake value attribute on select element
    selected: false,
  },
};

// Crawl WHATWG HTML.
const html = await loadHtmlIndices();
const document = parseHtml(html);
const table = findByTags(document, "table").find(
  (table) => getAttr(table, "id")?.value === "attributes-1"
);
const [tbody] = findByTags(table, "tbody");
const rows = findByTags(tbody, "tr");

const attributesByTag: Record<string, Attribute[]> = {};
// textarea does not have value attribute and text content is used as initial value
// introduce fake value attribute to manage initial state similar to input
attributesByTag.textarea = [
  {
    name: "value",
    description: "Value of the form control",
    type: "string",
    required: true,
  },
];
// select does not have value attribute and selected options are used as initial value
// introduce fake value attribute to manage initial state similar to input
attributesByTag.select = [
  {
    name: "value",
    description: "Value of the form control",
    type: "string",
    required: true,
  },
];

for (const row of rows) {
  const attribute = getTextContent(row.childNodes[0]).trim();
  const elements = getTextContent(row.childNodes[1]).trim();
  const description = getTextContent(row.childNodes[2]).trim();
  const tags = /HTML elements/.test(elements)
    ? ["*"]
    : elements.split(/;/g).map((d) => d.replace(/\([^)]+\)/g, "").trim());
  let value = getTextContent(row.childNodes[3]).trim().toLowerCase();
  if (value.endsWith(";")) {
    value = value.slice(0, -1);
  }
  let possibleOptions = value
    .split(/\s*;\s*/)
    .filter((item) => item.startsWith('"') && item.endsWith('"'))
    .map((item) => item.slice(1, -1));
  if (attribute === "target" || attribute === "formtarget") {
    possibleOptions = ["_blank", "_self", "_parent", "_top"];
  }
  if (value.includes("input type keyword")) {
    possibleOptions = [
      "hidden",
      "text",
      "search",
      "tel",
      "url",
      "email",
      "password",
      "date",
      "month",
      "week",
      "time",
      "datetime-local",
      "number",
      "range",
      "color",
      "checkbox",
      "radio",
      "file",
      "submit",
      "image",
      "reset",
      "button",
    ];
  }
  let type: "string" | "boolean" | "number" | "select" = "string";
  let options: undefined | string[];
  if (possibleOptions.length > 0) {
    type = "select";
    options = possibleOptions;
  } else if (value.includes("boolean attribute")) {
    type = "boolean";
  } else if (
    (value.includes("number") || value.includes("integer")) &&
    !value.includes("list") &&
    !value.includes("string")
  ) {
    type = "number";
  }
  validHtmlAttributes.add(attribute);
  for (let tag of tags) {
    tag = tag.toLowerCase().trim();
    if (/custom elements/i.test(tag)) {
      continue;
    }
    if (ignoredTags.includes(tag)) {
      continue;
    }
    if (!attributesByTag[tag]) {
      attributesByTag[tag] = [];
    }
    const attributes = attributesByTag[tag];
    if (!attributes.some((item) => item.name === attribute)) {
      const override = overrides[tag]?.[attribute];
      if (override !== false) {
        attributes.push({
          name: attribute,
          description,
          type,
          options,
          ...override,
        });
      }
    }
  }
}

{
  const svg = await loadSvgSinglePage();
  const document = parseHtml(svg);
  const attributeOptions = new Map<string, string[]>();
  // find all property definition and extract there keywords
  for (const propdef of findByClasses(document, "propdef")) {
    let options: undefined | string[];
    for (const row of findByTags(propdef, "tr")) {
      const [nameNode, valueNode] = row.childNodes;
      const name = getTextContent(nameNode);
      const list = getTextContent(valueNode)
        .trim()
        .split(/\s+\|\s+/);
      if (
        name.toLowerCase().includes("value") &&
        list.every((item) => item.match(/^[a-zA-Z-]+$/))
      ) {
        options = list;
      }
    }
    for (const propNameNode of findByClasses(propdef, "propdef-title")) {
      const propName = getTextContent(propNameNode).slice(1, -1);
      if (options) {
        attributeOptions.set(propName, options);
      }
    }
  }

  for (const summary of findByClasses(document, "element-summary")) {
    const [tag] = findByClasses(summary, "element-summary-name").map((item) =>
      getTextContent(item).slice(1, -1)
    );
    // ignore existing
    if (attributesByTag[tag] || ignoredTags.includes(tag)) {
      continue;
    }
    const attributes = new Set<string>();
    const [dl] = findByTags(summary, "dl");
    for (let index = 0; index < dl.childNodes.length; index += 1) {
      const child = dl.childNodes[index];
      if (getTextContent(child).toLowerCase().includes("attributes")) {
        const dd = dl.childNodes[index + 1];
        for (const attrNameNode of findByClasses(dd, "attr-name")) {
          const attrName = getTextContent(attrNameNode).slice(1, -1);
          // skip events
          if (attrName.startsWith("on") || attrName === "style") {
            continue;
          }
          validHtmlAttributes.add(attrName);
          attributes.add(attrName);
        }
      }
    }
    attributesByTag[tag] = Array.from(attributes)
      .sort()
      .map((name) => {
        let options = attributeOptions.get(name);
        if (name === "externalResourcesRequired") {
          options = ["true", "false"];
        }
        if (name === "accumulate") {
          options = ["none", "sum"];
        }
        if (name === "additive") {
          options = ["replace", "sum"];
        }
        if (name === "preserveAlpha") {
          options = ["true", "false"];
        }
        if (options) {
          return { name, description: "", type: "select", options };
        }
        return { name, description: "", type: "string" };
      });
  }
}

// sort tags and attributes
const tags = Object.keys(attributesByTag).sort();
const attributesByHash = new Map<string, Attribute>();
const reusableAttributesByHash = new Map<string, Attribute>();
for (const tag of tags) {
  const attributes = attributesByTag[tag];
  delete attributesByTag[tag];
  attributes.sort((left, right) => left.name.localeCompare(right.name));
  if (attributes.length > 0) {
    for (const attribute of attributes) {
      const attributeHash = hash(JSON.stringify(attribute));
      if (attributesByHash.has(attributeHash)) {
        reusableAttributesByHash.set(attributeHash, attribute);
      } else {
        attributesByHash.set(attributeHash, attribute);
      }
    }
    attributesByTag[tag] = attributes;
  }
}

let attributesContent = `type Attribute = {
  name: string,
  description: string,
  required?: boolean,
  type: 'string' | 'boolean' | 'number' | 'select' | 'url',
  options?: string[],
  defaultValue?: string
}

`;

const attributeVariableByHash = new Map<string, string>();
for (const [key, attribute] of reusableAttributesByHash) {
  const normalizedName = attribute.name
    .replaceAll("-", "_")
    .replaceAll(":", "_");
  const variableName = `attribute_${normalizedName}_${key}`;
  attributeVariableByHash.set(key, variableName);
  attributesContent += `const ${variableName}: Attribute = ${JSON.stringify(attribute, null, 2)};\n\n`;
}

const serializableAttributesByTag: Record<
  string,
  Array<string | Attribute>
> = {};
for (const tag of tags) {
  const attributes = attributesByTag[tag];
  serializableAttributesByTag[tag] = attributes.map((attribute) => {
    const key = hash(JSON.stringify(attribute));
    const variableName = attributeVariableByHash.get(key);
    if (variableName) {
      return variableName;
    }
    return attribute;
  });
}

attributesContent += `
export const attributesByTag: Record<string, undefined | Attribute[]> = ${JSON.stringify(serializableAttributesByTag, null, 2)};
`;
for (const variableName of attributeVariableByHash.values()) {
  attributesContent = attributesContent.replaceAll(
    `"${variableName}"`,
    variableName
  );
}

await mkdir("./src/__generated__", { recursive: true });
await writeFile("./src/__generated__/attributes.ts", attributesContent);

// generate jsx for testing react types

const instances: Instances = new Map();
const props: Props = new Map();

let id = 0;
const getId = () => {
  id += 1;
  return id.toString();
};

const body: Instance = {
  type: "instance",
  id: getId(),
  component: elementComponent,
  tag: "body",
  children: [],
};
instances.set(body.id, body);

for (const entry of Object.entries(attributesByTag)) {
  let [tag] = entry;
  const [_tag, attributes] = entry;
  if (tag === "*") {
    tag = "div";
  }
  const instance: Instance = {
    type: "instance",
    id: getId(),
    component: elementComponent,
    tag,
    children: [],
  };
  body.children.push({ type: "id", value: instance.id });
  instances.set(instance.id, instance);
  for (const { name, type, options } of attributes) {
    const id = getId();
    const instanceId = instance.id;
    if (type === "string" || type === "url") {
      const prop: Prop = { id, instanceId, type: "string", name, value: "" };
      props.set(prop.id, prop);
      continue;
    }
    if (type === "boolean") {
      const prop: Prop = { id, instanceId, type, name, value: true };
      props.set(prop.id, prop);
      continue;
    }
    if (type === "select") {
      const prop: Prop = {
        id,
        instanceId,
        type: "string",
        name,
        value: options?.[0] ?? "",
      };
      props.set(prop.id, prop);
      continue;
    }
    if (type === "number") {
      const prop: Prop = {
        id,
        instanceId,
        type: "number",
        name,
        value: 0,
      };
      props.set(prop.id, prop);
      continue;
    }
    (type) satisfies never;
    throw Error(`Unknown attribute ${name} with type ${type}`);
  }
}

await mkdir("./src/__generated__", { recursive: true });
await writeFile(
  "./src/__generated__/attributes-jsx-test.tsx",
  generateWebstudioComponent({
    name: "Page",
    scope: createScope(),
    metas: new Map(Object.entries(coreMetas)),
    instances,
    props,
    dataSources: new Map(),
    rootInstanceId: body.id,
    classesMap: new Map(),
    parameters: [],
  }) + "export { Page }"
);

// react does not have this one
possibleStandardNames["dirname"] = "dirName";
const standardAttributesToReactProps: Record<string, string> = {};
const reactPropsToStandardAttributes: Record<string, string> = {};
for (const [htmlAttribute, reactProperty] of Object.entries(
  possibleStandardNames
)) {
  if (
    validHtmlAttributes.has(htmlAttribute) &&
    htmlAttribute !== reactProperty
  ) {
    standardAttributesToReactProps[htmlAttribute] = reactProperty;
    reactPropsToStandardAttributes[reactProperty] = htmlAttribute;
  }
}

let standardAttributesContent = "";
standardAttributesContent += `export const standardAttributesToReactProps: Record<string, string> = ${JSON.stringify(standardAttributesToReactProps, null, 2)};\n\n`;
standardAttributesContent += `export const reactPropsToStandardAttributes: Record<string, string> = ${JSON.stringify(reactPropsToStandardAttributes, null, 2)};\n`;

await mkdir("../content-engine/src/__generated__", { recursive: true });
await writeFile(
  "../content-engine/src/__generated__/standard-attributes.ts",
  standardAttributesContent
);