Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.
TypeScript accessibilityalternativecloudflarecloudflare-workersdesignfreefrontendopen-sourceradix-uireactremixweb-developmentwebflowwebsite-builderwebsite-developmentwebsiteswebstudio
/**
* Materializes MDX Assets for published builds with the same template and
* Asset-reference behavior used by the Builder runtime.
*/
import {
parseMdxDocumentRecovering,
type MdxDocument,
} from "@webstudio-is/content-engine/mdx";
import type {
AssetRuntimeData,
AssetValueReference,
ContentArtifactV1,
} from "@webstudio-is/content-engine";
import {
contentEngineLimits,
createDocumentGraph,
DocumentGraphResolutionError,
getDocumentGraphClosure,
resolveAssetValueReferences,
resolveDocumentGraphProperties,
} from "@webstudio-is/content-engine";
import {
blockComponent,
decodeDataSourceVariable,
getAssetContentHash,
getContentBlockSources,
getStaticContentBlockSourceAssetId,
toAssetReferenceRuntimeData,
type DataSource,
type Prop,
type ContentBlockDiagnostic,
type ContentBlockExternalContentIdentity,
type ContentBlockSource,
type InvalidMdxDiagnosticReason,
type WebstudioData,
type WsComponentMeta,
} from "@webstudio-is/sdk";
import { parseStaticMemberPath } from "@webstudio-is/expression";
import { materializeMdxAuthoredContent } from "./mdx-authored-content";
import { createFindAvailableVariables } from "./data";
import {
materializeMdxTemplates,
type MdxTemplateDependency,
} from "./mdx-materialization";
import {
assertMdxTemplateStructure,
resolveMdxTemplates,
} from "./mdx-template-resolution";
import {
createMdxContentModelDiagnostics,
createMdxDiagnostics,
} from "./mdx-diagnostics";
export type PublishedMdxWarning = Readonly<{
route: string;
diagnostic: ContentBlockDiagnostic;
}>;
export type PublishedMdxRoot = Readonly<{
source: ContentBlockSource;
dynamic: boolean;
identity: ContentBlockExternalContentIdentity;
document: MdxDocument;
resolvedFrontmatter: Readonly<Record<string, unknown>>;
fragment: ReturnType<typeof materializeMdxAuthoredContent>["fragment"];
templateDependencies: readonly MdxTemplateDependency[];
templateNames: readonly string[];
dependencyRevision: `sha256:${string}`;
}>;
export const getUnsafeDynamicPublishedMdxDiagnostic = ({
root,
route,
dataSources,
props,
}: {
root: PublishedMdxRoot;
route: string;
dataSources: ReadonlyMap<DataSource["id"], DataSource>;
props: ReadonlyMap<Prop["id"], Prop>;
}): ContentBlockDiagnostic | undefined => {
if (root.dynamic === false) {
return;
}
const sourcePath =
root.source.type === "expression"
? parseStaticMemberPath(root.source.value)
: undefined;
const sourceDataSourceId =
sourcePath === undefined
? undefined
: decodeDataSourceVariable(sourcePath[0]);
if (
root.fragment.resources.length > 0 &&
sourceDataSourceId !== undefined &&
dataSources.get(sourceDataSourceId)?.type === "parameter"
) {
return {
code: "invalid-mdx",
severity: "error",
reason: "dynamic-resource",
blockInstanceId: root.identity.blockInstanceId,
assetId: root.identity.assetId,
renderScope: `route:${route}:block:${root.identity.blockInstanceId}`,
message:
"Dynamic MDX selected by a Collection item cannot contain Resources because a single Resource cannot be selected safely for the whole route.",
};
}
const actionResourceIds = new Set(
[...props.values(), ...root.fragment.props].flatMap((prop) =>
prop.type === "resource" ? [prop.value] : []
)
);
if (root.fragment.resources.some(({ id }) => actionResourceIds.has(id))) {
return {
code: "invalid-mdx",
severity: "error",
reason: "dynamic-resource",
blockInstanceId: root.identity.blockInstanceId,
assetId: root.identity.assetId,
renderScope: `route:${route}:block:${root.identity.blockInstanceId}`,
message:
"Dynamic MDX cannot contain action Resources because the submitted action cannot be selected safely at runtime.",
};
}
};
const publishedMdxMaterializerRevision = 2;
export const createPublishedMdxMaterializationCache = () => ({
documents: new Map<
string,
Promise<Awaited<ReturnType<typeof parseMdxDocumentRecovering>>>
>(),
frontmatter: new Map<string, Promise<Readonly<Record<string, unknown>>>>(),
});
type PublishedMdxMaterializationCache = ReturnType<
typeof createPublishedMdxMaterializationCache
>;
const getMdxDocuments = (artifact: ContentArtifactV1) =>
artifact.documents.filter(
(document) =>
document.extension?.toLowerCase() === "mdx" ||
document.mimeType === "text/mdx"
);
const getNestedDocumentDependencies = ({
graph,
assetId,
}: {
graph: ReturnType<typeof createDocumentGraph> | undefined;
assetId: string;
}) => {
if (
graph === undefined ||
graph.nodes.some(({ id }) => id === assetId) === false
) {
return [];
}
return getDocumentGraphClosure({ graph, rootIds: [assetId] })
.map(({ id, revision, contentRef }) => ({ id, revision, contentRef }))
.sort((left, right) => left.id.localeCompare(right.id));
};
const createDependencyRevision = async ({
identity,
nestedDocuments,
assetReferences,
assetDependencies,
templateNames,
templateDependencies,
}: {
identity: ContentBlockExternalContentIdentity;
nestedDocuments: ReturnType<typeof getNestedDocumentDependencies>;
assetReferences: readonly AssetValueReference[];
assetDependencies: readonly Readonly<{
id: string;
contentRef: string;
revision?: string;
}>[];
templateNames: readonly string[];
templateDependencies: readonly MdxTemplateDependency[];
}) =>
`sha256:${await getAssetContentHash(
new TextEncoder().encode(
JSON.stringify({
materializerRevision: publishedMdxMaterializerRevision,
identity,
nestedDocuments,
assetReferences: [...assetReferences]
.map(({ path, assetId, suffix, structured }) => ({
path,
assetId,
suffix,
structured,
}))
.sort((left, right) =>
JSON.stringify(left).localeCompare(JSON.stringify(right))
),
assetDependencies: [...assetDependencies].sort((left, right) =>
left.id.localeCompare(right.id)
),
templateNames: [...templateNames].sort(),
templateDependencies: [...templateDependencies].sort((left, right) =>
left.templateInstanceId.localeCompare(right.templateInstanceId)
),
})
)
)}` as const;
export const materializePublishedMdx = async ({
route,
data,
artifact,
metas,
projectId,
cache = createPublishedMdxMaterializationCache(),
dynamicAssetIdsByBlock = new Map(),
blockInstanceIds,
runtimeAssets: providedRuntimeAssets,
loadDocumentSource,
}: {
route: string;
data: Omit<WebstudioData, "pages">;
artifact: ContentArtifactV1;
metas: Map<string, WsComponentMeta>;
projectId: string;
cache?: PublishedMdxMaterializationCache;
dynamicAssetIdsByBlock?: ReadonlyMap<string, readonly string[]>;
blockInstanceIds?: ReadonlySet<string>;
runtimeAssets?: Readonly<Record<string, AssetRuntimeData>>;
loadDocumentSource?: (node: {
id: string;
revision: string;
contentRef: string;
}) => Promise<string>;
}): Promise<{
roots: readonly PublishedMdxRoot[];
warnings: readonly PublishedMdxWarning[];
}> => {
const documentsById = new Map(
artifact.documents.map((document) => [document._id, document])
);
const mdxDocumentsById = new Map(
getMdxDocuments(artifact).map((document) => [document._id, document])
);
const documentGraph =
artifact.documentGraph === undefined
? undefined
: createDocumentGraph(artifact.documentGraph);
const runtimeAssets =
providedRuntimeAssets ??
Object.fromEntries(
Array.from(data.assets.values(), (asset) => [
asset.id,
toAssetReferenceRuntimeData(asset, "https://webstudio.local"),
])
);
const resolveFrontmatter = (
assetId: string,
document: MdxDocument
): Promise<Readonly<Record<string, unknown>>> => {
const cached = cache.frontmatter.get(assetId);
if (cached !== undefined) {
return cached;
}
const resolving = (async () => {
if (
documentGraph !== undefined &&
documentGraph.nodes.some(({ id }) => id === assetId)
) {
return (
(await resolveDocumentGraphProperties({
graph: documentGraph,
rootId: assetId,
assetValueReferences: artifact.assetValueReferences,
runtimeAssets,
load: async (node) => {
const source =
artifact.contents?.[node.contentRef] ??
(await loadDocumentSource?.(node));
if (source === undefined || node.format === undefined) {
throw new Error(`Published document ${node.id} is unavailable`);
}
return {
format: node.format,
revision: node.revision,
source,
};
},
})) ?? document.frontmatter.properties
);
}
return resolveAssetValueReferences({
value: { properties: document.frontmatter.properties },
references: artifact.assetValueReferences?.[assetId],
runtimeAssets,
}).properties;
})();
cache.frontmatter.set(assetId, resolving);
return resolving;
};
const sourcesByBlockId = getContentBlockSources({
instances: data.instances.values(),
props: data.props.values(),
});
const roots: PublishedMdxRoot[] = [];
const warnings: PublishedMdxWarning[] = [];
let findAvailableVariables:
| ReturnType<typeof createFindAvailableVariables>
| undefined;
const warnUnavailableSource = ({
blockInstanceId,
assetId,
reason,
message,
}: {
blockInstanceId: string;
assetId?: string;
reason?: InvalidMdxDiagnosticReason;
message: string;
}) => {
warnings.push({
route,
diagnostic: {
code: "invalid-mdx",
severity: "error",
...(reason === undefined ? {} : { reason }),
blockInstanceId,
...(assetId === undefined ? {} : { assetId }),
renderScope: `route:${route}:block:${blockInstanceId}`,
message,
},
});
};
for (const block of data.instances.values()) {
if (
block.component !== blockComponent ||
(blockInstanceIds !== undefined &&
blockInstanceIds.has(block.id) === false)
) {
continue;
}
const source = sourcesByBlockId.get(block.id);
if (source === undefined) {
continue;
}
const staticAssetId = getStaticContentBlockSourceAssetId(source);
const isDynamicSource = typeof staticAssetId !== "string";
const resolvedAssetIds =
typeof staticAssetId === "string" && staticAssetId.length > 0
? [staticAssetId]
: (dynamicAssetIdsByBlock.get(block.id) ?? []);
if (resolvedAssetIds.length > contentEngineLimits.candidateDocuments) {
warnUnavailableSource({
blockInstanceId: block.id,
reason: "dynamic-source-unbounded",
message: `Published Content Block "${block.id}" exceeds the safe MDX candidate limit`,
});
continue;
}
if (resolvedAssetIds.length === 0) {
warnUnavailableSource({
blockInstanceId: block.id,
reason: "dynamic-source-unbounded",
message: `Published Content Block "${block.id}" has no bounded dynamic MDX dependency set`,
});
continue;
}
const candidates = Array.from(new Set(resolvedAssetIds))
.flatMap((assetId) => {
const document = mdxDocumentsById.get(assetId);
return document === undefined ? [] : [document];
})
.sort((left, right) => left._id.localeCompare(right._id));
for (const assetId of resolvedAssetIds) {
if (mdxDocumentsById.has(assetId)) {
continue;
}
warnUnavailableSource({
blockInstanceId: block.id,
assetId,
reason: "missing-asset",
message: `Published Content Block "${block.id}" requires unavailable MDX Asset "${assetId}"`,
});
}
for (const candidate of candidates) {
if (
candidate.revision === undefined ||
candidate.contentRef === undefined
) {
warnUnavailableSource({
blockInstanceId: block.id,
assetId: candidate._id,
reason: "missing-source",
message: `Published MDX Asset "${candidate._id}" has no revision identity`,
});
continue;
}
let sourceText = artifact.contents?.[candidate.contentRef];
if (sourceText === undefined && loadDocumentSource !== undefined) {
try {
sourceText = await loadDocumentSource({
id: candidate._id,
revision: candidate.revision,
contentRef: candidate.contentRef,
});
} catch (error) {
warnUnavailableSource({
blockInstanceId: block.id,
assetId: candidate._id,
reason: "source-read-failed",
message: `Published MDX Asset "${
candidate._id
}" could not be loaded: ${
error instanceof Error ? error.message : String(error)
}`,
});
continue;
}
}
if (sourceText === undefined) {
warnUnavailableSource({
blockInstanceId: block.id,
assetId: candidate._id,
reason: "missing-source",
message: `Published MDX Asset "${candidate._id}" content is unavailable`,
});
continue;
}
const identity: ContentBlockExternalContentIdentity = {
blockInstanceId: block.id,
assetId: candidate._id,
revision: candidate.revision,
contentRef: candidate.contentRef,
format: "mdx",
renderScope: `route:${route}:block:${block.id}${
isDynamicSource ? `:asset:${candidate._id}` : ""
}`,
};
const documentKey = `${candidate._id}:${candidate.revision}:${candidate.contentRef}`;
let parsedDocument = cache.documents.get(documentKey);
if (parsedDocument === undefined) {
if (cache.documents.size >= contentEngineLimits.hydratedFileCount) {
const oldestKey = cache.documents.keys().next().value;
if (oldestKey !== undefined) {
cache.documents.delete(oldestKey);
}
}
parsedDocument = parseMdxDocumentRecovering({ source: sourceText });
cache.documents.set(documentKey, parsedDocument);
}
const parsed = await parsedDocument;
const recoveryDiagnostics = createMdxDiagnostics({
identity,
diagnostics: parsed.diagnostics,
});
for (const diagnostic of recoveryDiagnostics) {
warnings.push({ route, diagnostic });
}
if (parsed.status === "unrecoverable") {
continue;
}
const document = parsed.document;
const assetReferences =
artifact.assetValueReferences?.[candidate._id] ?? [];
const resolution = resolveMdxTemplates({
document,
identity,
instances: data.instances,
props: data.props,
metas,
});
assertMdxTemplateStructure(resolution);
if (
findAvailableVariables === undefined &&
resolution.references.some(
(reference) => reference.type === "resolved-template"
)
) {
findAvailableVariables = createFindAvailableVariables({
instances: data.instances,
dataSources: data.dataSources,
});
}
const templates = await materializeMdxTemplates({
identity,
resolution,
data,
metas,
projectId,
assetReferences,
findAvailableVariables,
});
const referencedTemplateNames = Array.from(
new Set(
resolution.references.map((reference) => reference.templateName)
)
);
let resolvedFrontmatter = document.frontmatter.properties;
try {
resolvedFrontmatter = await resolveFrontmatter(candidate._id, document);
} catch (error) {
if (
error instanceof DocumentGraphResolutionError &&
error.code === "REQUEST_CANCELLED"
) {
throw error;
}
if (error instanceof DocumentGraphResolutionError) {
const linkedDocument =
error.documentId === undefined
? "Linked document"
: `Linked document "${error.documentId}"`;
const reference =
error.referenceId === undefined
? ""
: ` (reference "${error.referenceId}")`;
const failure =
error.code === "DOCUMENT_LOAD_FAILED"
? "could not be loaded"
: `could not be resolved: ${error.message}`;
warnUnavailableSource({
blockInstanceId: block.id,
assetId: candidate._id,
reason: "linked-document-unavailable",
message: `${linkedDocument} ${failure}${reference} while resolving frontmatter for published MDX Asset "${candidate._id}".`,
});
} else {
throw error;
}
}
const materialized = materializeMdxAuthoredContent({
identity,
document,
templateMaterialization: templates,
assetReferences,
metas,
});
if (
materialized.fragment.instances.length > contentEngineLimits.mdxNodes ||
materialized.fragment.props.length > contentEngineLimits.mdxProps
) {
warnUnavailableSource({
blockInstanceId: block.id,
assetId: candidate._id,
message: `Published MDX Asset "${candidate._id}" materialization exceeds safe limits`,
});
continue;
}
for (const diagnostic of templates.diagnostics) {
warnings.push({ route, diagnostic });
}
for (const diagnostic of materialized.diagnostics ?? []) {
warnings.push({ route, diagnostic });
}
for (const diagnostic of createMdxContentModelDiagnostics({
root: materialized,
metas,
})) {
warnings.push({ route, diagnostic });
}
roots.push({
source,
dynamic: isDynamicSource,
identity,
document,
resolvedFrontmatter,
fragment: materialized.fragment,
templateDependencies: templates.dependencies.templates,
templateNames: referencedTemplateNames,
dependencyRevision: await createDependencyRevision({
identity,
nestedDocuments: getNestedDocumentDependencies({
graph: documentGraph,
assetId: candidate._id,
}),
assetReferences,
assetDependencies: assetReferences.map(({ assetId }) => {
const asset = data.assets.get(assetId);
const documentDependency = documentsById.get(assetId);
return {
id: assetId,
contentRef:
asset?.name ?? documentDependency?.contentRef ?? "unavailable",
...(documentDependency?.revision === undefined
? {}
: { revision: documentDependency.revision }),
};
}),
templateNames: referencedTemplateNames,
templateDependencies: templates.dependencies.templates,
}),
});
}
}
const warningKeys = new Set<string>();
return {
roots,
warnings: warnings.filter(({ route: warningRoute, diagnostic }) => {
const key = JSON.stringify([warningRoute, diagnostic]);
if (warningKeys.has(key)) {
return false;
}
warningKeys.add(key);
return true;
}),
};
};