Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.
TypeScript accessibilityalternativecloudflarecloudflare-workersdesignfreefrontendopen-sourceradix-uireactremixweb-developmentwebflowwebsite-builderwebsite-developmentwebsiteswebstudio
fix: complete and speed up publish validation (#6478)
Oleg Isonen3d0f19ffe075
diff --git a/apps/builder/app/builder/features/publish/content-database-publish-warning.test.tsx b/apps/builder/app/builder/features/publish/content-database-publish-warning.test.tsxindex be61d0d034..27f0dbf4cc 100644--- a/apps/builder/app/builder/features/publish/content-database-publish-warning.test.tsx+++ b/apps/builder/app/builder/features/publish/content-database-publish-warning.test.tsx@@ -25,6 +25,7 @@ describe("content database publish findings", () => {diagnostic: {code: "invalid-mdx",severity: "error",+ reason: "html-content-model",blockInstanceId: "block-1",assetId: "asset-1",contentRef: "article-revision.mdx",@@ -41,6 +42,7 @@ describe("content database publish findings", () => {diagnostic: {code: "invalid-mdx",severity: "error",+ reason: "missing-asset",blockInstanceId: "block-2",assetId: "missing-asset",message:@@ -68,6 +70,36 @@ describe("content database publish findings", () => {);});+ test("suggests storage troubleshooting for MDX source read failures", () => {+ const [finding] = getContentDatabasePublishFindings(+ createDiagnostics({+ mdxErrors: [+ {+ filename: "article.mdx",+ diagnostic: {+ code: "invalid-mdx",+ severity: "error",+ reason: "source-read-failed",+ blockInstanceId: "block-1",+ assetId: "asset-1",+ contentRef: "article-revision.mdx",+ message: "Object storage returned an I/O error",+ },+ },+ ],+ })+ );++ expect(finding.reportText).toContain(+ "Check that the asset is available in Content Assets, then run Validate again."+ );+ expect(finding.reportText).toContain("https://wstd.us/mcp");+ expect(finding.reportText).toContain("Asset ID: asset-1");+ expect(finding.reportText).toContain(+ "Diagnostic reason: source-read-failed"+ );+ });+test("includes every missing template warning and its identifying IDs", () => {const findings = getContentDatabasePublishFindings(createDiagnostics({diff --git a/apps/builder/app/builder/features/publish/content-database-publish-warning.tsx b/apps/builder/app/builder/features/publish/content-database-publish-warning.tsxindex fee699377f..d0c9f4a670 100644--- a/apps/builder/app/builder/features/publish/content-database-publish-warning.tsx+++ b/apps/builder/app/builder/features/publish/content-database-publish-warning.tsx@@ -7,6 +7,24 @@ type ContentDatabasePublishDiagnostics = Awaited<ReturnType<typeof nativeClient.build.contentDatabasePublishDiagnostics.query>>;+const missingAssetFix =+ "Choose an available MDX asset in this Content Block's source settings, or restore the missing asset.";++const invalidMdxFixes = {+ "source-read-failed":+ "Check that the asset is available in Content Assets, then run Validate again. If it still fails, investigate the document with Webstudio MCP: https://wstd.us/mcp.",+ "missing-source": missingAssetFix,+ "missing-asset": missingAssetFix,+ "linked-document-unavailable":+ "Restore or reconnect the referenced document in Content Assets, then run validation again.",+ "html-content-model":+ "Edit the MDX element nesting to satisfy the HTML content model. For nested links, remove one of the links.",+ "dynamic-resource":+ "Use a static MDX source for this Content Block, or remove Resource-dependent content from the MDX file.",+ "dynamic-source-unbounded":+ "Use a static MDX source or narrow the Collection query so validation can identify its MDX assets.",+} as const;+export const getContentDatabasePublishFindings = (diagnostics: ContentDatabasePublishDiagnostics) => {@@ -19,27 +37,18 @@ export const getContentDatabasePublishFindings = (const message ="message" in diagnostic ? diagnostic.message : diagnostic.code;const fix =- message.includes("requires unavailable MDX Asset") ||- message.includes("content is unavailable")- ? "Choose an available MDX asset in this Content Block's source settings, or restore the missing asset."- : message.includes("Linked document")- ? "Restore or reconnect the referenced document in Content Assets, then run validation again."- : message.includes("violates HTML spec")- ? "Edit the MDX element nesting to satisfy the HTML content model. For nested links, remove one of the links."- : message.includes(- "Dynamic MDX selected by a Collection item cannot contain Resources"- ) ||- message.includes(- "Dynamic MDX cannot contain action Resources"- )- ? "Use a static MDX source for this Content Block, or remove Resource-dependent content from the MDX file."- : "Open this MDX source and fix the reported error.";+ diagnostic.code === "invalid-mdx" && diagnostic.reason !== undefined+ ? invalidMdxFixes[diagnostic.reason]+ : "Open this MDX source and fix the reported error.";const context = [`Diagnostic code: ${diagnostic.code}`,`Content Block instance ID: ${diagnostic.blockInstanceId}`,diagnostic.assetId === undefined? undefined: `Asset ID: ${diagnostic.assetId}`,+ diagnostic.code === "invalid-mdx" && diagnostic.reason !== undefined+ ? `Diagnostic reason: ${diagnostic.reason}`+ : undefined,diagnostic.contentRef === undefined? undefined: `Content reference: ${diagnostic.contentRef}`,diff --git a/apps/builder/app/builder/features/publish/publish.tsx b/apps/builder/app/builder/features/publish/publish.tsxindex c8300409da..08147cbe53 100644--- a/apps/builder/app/builder/features/publish/publish.tsx+++ b/apps/builder/app/builder/features/publish/publish.tsx@@ -239,12 +239,23 @@ const getPrePublishAuditFindings = (): PublishValidationFinding[] => {});};-const runPublishDiagnostics = async (projectId: Project["id"]) => {+const runPublishDiagnostics = async (+ projectId: Project["id"],+ domains?: string[]+) => {const auditFindings = getPrePublishAuditFindings();const diagnostics =- await nativeClient.build.contentDatabasePublishDiagnostics.query({- projectId,- });+ domains === undefined+ ? await nativeClient.build.contentDatabasePublishDiagnostics.query({+ projectId,+ })+ : (+ await nativeClient.api.publish.validate.query({+ projectId,+ target: "staging",+ domains,+ })+ ).diagnostics;const contentFindings = getContentDatabasePublishFindings(diagnostics).map((finding) => ({...finding,@@ -265,13 +276,10 @@ const runPublishDiagnostics = async (projectId: Project["id"]) => {const runPublishValidation = async (projectId: Project["id"],- checkPermission = false+ domains: string[]) => {- if (checkPermission) {- await nativeClient.build.checkProjectBuildPermission.query({ projectId });- }await flushExternalContentProject({ projectId });- return runPublishDiagnostics(projectId);+ return runPublishDiagnostics(projectId, domains);};const reportPublishValidationFailure = (@@ -790,7 +798,7 @@ const Publish = ({}try {- const checks = await runPublishValidation(project.id, true);+ const checks = await runPublishValidation(project.id, domains);setPublishFindings(checks.findings);if (checks.passed === false) {onValidationStateChange("idle");diff --git a/apps/builder/app/routes/trpc.$.ts b/apps/builder/app/routes/trpc.$.tsindex 9bcbd1f73b..58a132aec0 100644--- a/apps/builder/app/routes/trpc.$.ts+++ b/apps/builder/app/routes/trpc.$.ts@@ -6,6 +6,11 @@ import { preventCrossOriginCookie } from "~/services/no-cross-origin-cookie";import { getTrpcResponseMeta } from "~/services/trpc-response-meta.server";import { ensureApiCsrf } from "~/services/api-auth.server";+// Publish validation can read and compile a project's linked content assets.+// Give tRPC requests enough time to finish on Vercel instead of returning a+// function invocation timeout for larger projects.+export const config = { maxDuration: 60 };+const isServiceRequest = (request: Request) => {return isServiceAuthorization(request.headers.get("Authorization"));};diff --git a/apps/builder/app/services/api-router.server.test.ts b/apps/builder/app/services/api-router.server.test.tsindex 4d915043b6..abe9395f9e 100644--- a/apps/builder/app/services/api-router.server.test.ts+++ b/apps/builder/app/services/api-router.server.test.ts@@ -928,6 +928,7 @@ describe("api router permits", () => {"project-1",expect.anything(),expect.objectContaining({+ build: expect.objectContaining({ instances: [] }),onMdxTemplateOmissions: expect.any(Function),}));diff --git a/apps/builder/app/services/api-router.server.ts b/apps/builder/app/services/api-router.server.tsindex 20ad824f0a..4a8cfeb96d 100644--- a/apps/builder/app/services/api-router.server.ts+++ b/apps/builder/app/services/api-router.server.ts@@ -1097,7 +1097,8 @@ export const apiRouter = router({}const diagnostics = await loadContentDatabasePublishDiagnostics(input.projectId,- ctx+ ctx,+ { build });return {valid: true,diff --git a/apps/builder/app/services/auth-strategy/ws.server.test.ts b/apps/builder/app/services/auth-strategy/ws.server.test.tsnew file mode 100644index 0000000000..cd8d67176b--- /dev/null+++ b/apps/builder/app/services/auth-strategy/ws.server.test.ts@@ -0,0 +1,21 @@+import createDebug from "debug";+import { expect, test, vi } from "vitest";++test("loading multiple server bundles keeps OAuth debug output disabled", async () => {+ const originalDebug = process.env.DEBUG;+ createDebug.enable("*");+ try {+ for (let bundle = 0; bundle < 2; bundle += 1) {+ vi.resetModules();+ await import("./ws.server");+ expect(createDebug.enabled("OAuth2Strategy")).toBe(false);+ expect(createDebug.enabled("remix-auth:OAuth2Strategy:callback")).toBe(+ false+ );+ expect(createDebug.enabled("webstudio:request")).toBe(true);+ }+ } finally {+ createDebug.enable(originalDebug ?? "");+ vi.resetModules();+ }+});diff --git a/apps/builder/app/services/auth-strategy/ws.server.ts b/apps/builder/app/services/auth-strategy/ws.server.tsindex 2d4659b8d2..ec01c56a00 100644--- a/apps/builder/app/services/auth-strategy/ws.server.ts+++ b/apps/builder/app/services/auth-strategy/ws.server.ts@@ -20,7 +20,7 @@ const asyncLocalStorage = new AsyncLocalStorage<// remix-auth-oauth2 logs OAuth state, PKCE verifier, and full callback URLs.// Match both the exact namespace and any prefixed namespace enabled by DEBUG=*.createDebugRaw.enable(- `${createDebugRaw.disable()},-OAuth2Strategy,-*OAuth2Strategy*`+ `${process.env.DEBUG ?? ""},-OAuth2Strategy,-*OAuth2Strategy*`);/**diff --git a/apps/builder/app/services/content-database-publish-diagnostics.server.ts b/apps/builder/app/services/content-database-publish-diagnostics.server.tsindex 1ac27805a5..71f22bcdd8 100644--- a/apps/builder/app/services/content-database-publish-diagnostics.server.ts+++ b/apps/builder/app/services/content-database-publish-diagnostics.server.ts@@ -4,7 +4,9 @@ import {getContentDatabasePublishDiagnostics,} from "./content-database.server";import {+ getPublishedMdxAssetIds,resolvePublishedMdxAssetCandidates,+ type PublishedMdxDependencyReadFailure,type PublishedMdxTemplateOmission,} from "@webstudio-is/project-build";import {@@ -14,42 +16,74 @@ import {import { componentMetas } from "@webstudio-is/sdk-components-registry/metas";import { formatAssetName } from "@webstudio-is/sdk";import { migratePages } from "@webstudio-is/project-migrations/pages";-import { contentEngineLimits } from "@webstudio-is/content-engine";-import {- decodeUtf8,- readBoundedBytes,-} from "@webstudio-is/content-engine/compiler";+import { createPublishedMdxDependencySourceLoader } from "@webstudio-is/asset-uploader/server";import type { AppContext } from "@webstudio-is/trpc-interface/index.server";+import { loadDevBuildByProjectId } from "@webstudio-is/project-build/server";import { createAssetClient } from "~/shared/asset-client";+type Build = Awaited<ReturnType<typeof loadDevBuildByProjectId>>;+export const loadContentDatabasePublishDiagnostics = async (projectId: string,ctx: AppContext,dependencies: {- loadProjectBundleByProjectId: typeof loadProjectBundleByProjectId;- } = { loadProjectBundleByProjectId }+ loadProjectBundleByProjectId?: typeof loadProjectBundleByProjectId;+ build?: Build;+ } = {}) => {let mdxTemplateOmissions: readonly PublishedMdxTemplateOmission[] = [];+ let mdxDependencyReadFailures: readonly PublishedMdxDependencyReadFailure[] =+ [];+ const assetStore = createAssetClient();+ const loadDocumentSource =+ createPublishedMdxDependencySourceLoader(assetStore);const mdxBlockInstanceIds = new Set<string>();- const bundle = await dependencies.loadProjectBundleByProjectId(- projectId,- ctx,- {- onMdxTemplateOmissions: (issues) => {- mdxTemplateOmissions = issues;- },- onMdxBlockInstanceId: (blockInstanceId) =>- mdxBlockInstanceIds.add(blockInstanceId),- }- );+ const bundle = await (+ dependencies.loadProjectBundleByProjectId ?? loadProjectBundleByProjectId+ )(projectId, ctx, {+ ...(dependencies.build === undefined ? {} : { build: dependencies.build }),+ onMdxTemplateOmissions: (issues) => {+ mdxTemplateOmissions = issues;+ },+ onMdxDependencyReadFailures: (failures) => {+ mdxDependencyReadFailures = failures;+ },+ loadMdxDependencySource: loadDocumentSource,+ onMdxBlockInstanceId: (blockInstanceId) =>+ mdxBlockInstanceIds.add(blockInstanceId),+ });const mdxErrors: Array<{filename: string;diagnostic: Awaited<ReturnType<typeof materializePublishedMdx>>["warnings"][number]["diagnostic"];}> = [];+ const assets = new Map(bundle.assets.map((asset) => [asset.id, asset]));+ const getFilename = (assetId: string, fallback: string) => {+ const asset = assets.get(assetId);+ return asset === undefined ? fallback : formatAssetName(asset);+ };+ const dependencyReadFailureKeys = new Set(+ mdxDependencyReadFailures.map(({ blockInstanceId, assetId }) =>+ JSON.stringify([blockInstanceId, assetId])+ )+ );+ for (const failure of mdxDependencyReadFailures) {+ mdxErrors.push({+ filename: getFilename(failure.assetId, failure.contentRef),+ diagnostic: {+ code: "invalid-mdx",+ severity: "error",+ reason: "source-read-failed",+ blockInstanceId: failure.blockInstanceId,+ assetId: failure.assetId,+ contentRef: failure.contentRef,+ renderScope: `route:prepublish:block:${failure.blockInstanceId}`,+ message: failure.message,+ },+ });+ }if (bundle.assetIndex !== undefined) {- const assetStore = createAssetClient();const data = {instances: new Map(bundle.build.instances),props: new Map(bundle.build.props),@@ -72,6 +106,8 @@ export const loadContentDatabasePublishDiagnostics = async (build: { ...bundle.build, pages: migratePages(bundle.build.pages) },artifact: bundle.assetIndex,blockInstanceIds: mdxBlockInstanceIds,+ allAssetIds: new Set(bundle.assets.map(({ id }) => id)),+ allMdxAssetIds: getPublishedMdxAssetIds(bundle.assets),});const materialized = await materializePublishedMdx({route: "prepublish",@@ -81,18 +117,8 @@ export const loadContentDatabasePublishDiagnostics = async (projectId: bundle.build.projectId,blockInstanceIds: mdxBlockInstanceIds,dynamicAssetIdsByBlock,- loadDocumentSource: async ({ contentRef }) => {- const { data } = await assetStore.readFile(contentRef);- return decodeUtf8(- await readBoundedBytes(data, contentEngineLimits.hydratedFileBytes)- );- },+ loadDocumentSource,});- const assets = new Map(bundle.assets.map((asset) => [asset.id, asset]));- const getFilename = (assetId: string, fallback: string) => {- const asset = assets.get(assetId);- return asset === undefined ? fallback : formatAssetName(asset);- };for (const root of materialized.roots) {const diagnostic = getUnsafeDynamicPublishedMdxDiagnostic({root,@@ -114,6 +140,17 @@ export const loadContentDatabasePublishDiagnostics = async (if (diagnostic.severity !== "error") {continue;}+ if (+ diagnostic.code === "invalid-mdx" &&+ diagnostic.blockInstanceId !== undefined &&+ diagnostic.assetId !== undefined &&+ diagnostic.reason === "source-read-failed" &&+ dependencyReadFailureKeys.has(+ JSON.stringify([diagnostic.blockInstanceId, diagnostic.assetId])+ )+ ) {+ continue;+ }const assetId = diagnostic.assetId ?? "";mdxErrors.push({filename: getFilename(assetId, diagnostic.contentRef ?? "MDX content"),diff --git a/apps/builder/app/shared/db/canvas.server.ts b/apps/builder/app/shared/db/canvas.server.tsindex 9a145c22ce..ab9ac29fde 100644--- a/apps/builder/app/shared/db/canvas.server.ts+++ b/apps/builder/app/shared/db/canvas.server.ts@@ -15,6 +15,7 @@ import {getPublishedMdxContentDatabaseMaxBytes,resolvePublishedMdxAssetCandidates,type PublishedMdxTemplateOmission,+ type PublishedMdxDependencyReadFailure,} from "@webstudio-is/project-build";import { collectFontFamiliesFromStyleDecls } from "@webstudio-is/project-build/runtime";import {@@ -178,7 +179,15 @@ const addProjectMetadata = async (onMdxTemplateOmissions?: (issues: readonly PublishedMdxTemplateOmission[]) => void;+ onMdxDependencyReadFailures?: (+ failures: readonly PublishedMdxDependencyReadFailure[]+ ) => void;onMdxBlockInstanceId?: (blockInstanceId: string) => void;+ loadMdxDependencySource?: (input: {+ id: string;+ revision: string;+ contentRef: string;+ }) => Promise<string>;contentIndex?: "client";} = {}): Promise<PublishedProjectBundle> => {@@ -227,6 +236,7 @@ const addProjectMetadata = async (let publishedAssets = data.assets;let publishedAssetFolders = data.assetFolders;let mdxTemplateOmissions: PublishedMdxTemplateOmission[] = [];+ let mdxDependencyReadFailures: PublishedMdxDependencyReadFailure[] = [];if (assetRequirements !== undefined) {const resolveMdxDependencies =createPublishedMdxDependencyClosureResolver();@@ -247,16 +257,23 @@ const addProjectMetadata = async () || candidateDiscoveryPlan !== undefined? {resolvePlan: async (- artifact: NonNullable<PublishedProjectBundle["assetIndex"]>+ artifact: NonNullable<PublishedProjectBundle["assetIndex"]>,+ loadDocumentSource) => {const nextOmissions: PublishedMdxTemplateOmission[] = [];+ const nextReadFailures: PublishedMdxDependencyReadFailure[] = [];const plan = await resolveMdxDependencies({build: publicationBuild,artifact,+ loadDocumentSource:+ options.loadMdxDependencySource ?? loadDocumentSource,onTemplateOmission: (issue) => nextOmissions.push(issue),+ onDependencyReadFailure: (failure) =>+ nextReadFailures.push(failure),onMdxBlockInstanceId: options.onMdxBlockInstanceId,});mdxTemplateOmissions = nextOmissions;+ mdxDependencyReadFailures = nextReadFailures;return plan!;},}@@ -284,6 +301,7 @@ const addProjectMetadata = async (}options.onMdxTemplateOmissions?.(mdxTemplateOmissions);+ options.onMdxDependencyReadFailures?.(mdxDependencyReadFailures);return {...data,@@ -380,10 +398,19 @@ export const loadProjectBundleByProjectId = async (projectId: string,context: AppContext,options: {+ build?: Build;onMdxTemplateOmissions?: (issues: readonly PublishedMdxTemplateOmission[]) => void;+ onMdxDependencyReadFailures?: (+ failures: readonly PublishedMdxDependencyReadFailure[]+ ) => void;onMdxBlockInstanceId?: (blockInstanceId: string) => void;+ loadMdxDependencySource?: (input: {+ id: string;+ revision: string;+ contentRef: string;+ }) => Promise<string>;} = {}): Promise<PublishedProjectBundle> => {const project = await loadById(projectId, context);@@ -392,7 +419,7 @@ export const loadProjectBundleByProjectId = async (}return await addProjectMetadata(await createProjectBundle(- await loadDevBuildByProjectId(context, projectId),+ options.build ?? (await loadDevBuildByProjectId(context, projectId)),context),project,diff --git a/apps/builder/e2e/serve-built-remix.ts b/apps/builder/e2e/serve-built-remix.tsindex b239d6818c..50625e7edc 100644--- a/apps/builder/e2e/serve-built-remix.ts+++ b/apps/builder/e2e/serve-built-remix.ts@@ -2,6 +2,7 @@ import { installGlobals } from "@remix-run/node";import { createRequestHandler as createExpressRequestHandler } from "@remix-run/express";import type { ServerBuild } from "@remix-run/server-runtime";import express from "express";+import { matchRoutes, type RouteObject } from "react-router-dom";import { readdirSync, readFileSync } from "node:fs";import https from "node:https";import path from "node:path";@@ -9,23 +10,49 @@ import { pathToFileURL } from "node:url";installGlobals({ nativeFetch: true });-const resolveServerBuildPath = () => {+const start = async () => {const serverDirectory = path.resolve("build/server");- const serverBuild = readdirSync(serverDirectory, {- withFileTypes: true,- }).find((entry) => entry.isDirectory());-- if (serverBuild === undefined) {+ const builds = await Promise.all(+ readdirSync(serverDirectory, { withFileTypes: true })+ .filter((entry) => entry.isDirectory())+ .map(async (entry) => {+ const filename = path.join(serverDirectory, entry.name, "index.js");+ return (await import(pathToFileURL(filename).href)) as ServerBuild;+ })+ );+ const [build] = builds;+ if (build === undefined) {throw new Error(`Could not find server build in ${serverDirectory}`);}-- return path.join(serverDirectory, serverBuild.name, "index.js");-};--const start = async () => {- const build = (await import(- pathToFileURL(resolveServerBuildPath()).href- )) as ServerBuild;+ // Route-specific Vercel config produces multiple server bundles. Match+ // against the complete route tree before dispatching to its owning bundle.+ const routesById = new Map<string, RouteObject>();+ const handlers = new Map<string, express.RequestHandler>();+ for (const bundle of builds) {+ const handler = createExpressRequestHandler({+ build: bundle,+ mode: "production",+ });+ for (const route of Object.values(bundle.routes)) {+ routesById.set(route.id, {+ id: route.id,+ path: route.path,+ index: route.index,+ });+ handlers.set(route.id, handler);+ }+ }+ const routes: RouteObject[] = [];+ const manifest = Object.assign(+ {},+ ...builds.map((bundle) => bundle.routes)+ ) as ServerBuild["routes"];+ for (const route of Object.values(manifest)) {+ const parent =+ route.parentId === undefined ? undefined : routesById.get(route.parentId);+ const siblings = parent === undefined ? routes : (parent.children ??= []);+ siblings.push(routesById.get(route.id)!);+ }const port = Number(process.env.PORT ?? 3000);const host = process.env.HOST;const app = express();@@ -39,12 +66,11 @@ const start = async () => {}));app.use(express.static("public", { maxAge: "1h" }));- app.use(- createExpressRequestHandler({- build,- mode: "production",- })- );+ app.use((request, response, next) => {+ const matches = matchRoutes(routes, request.path, build.basename);+ const routeId = matches?.at(-1)?.route.id ?? "root";+ return handlers.get(routeId)!(request, response, next);+ });const server = https.createServer({diff --git a/packages/asset-uploader/src/publication.test.ts b/packages/asset-uploader/src/publication.test.tsindex 069214addc..677d3ca339 100644--- a/packages/asset-uploader/src/publication.test.ts+++ b/packages/asset-uploader/src/publication.test.ts@@ -1,5 +1,8 @@import { describe, expect, test, vi } from "vitest";-import { createDefaultCollectionConfig } from "@webstudio-is/content-engine";+import {+ createDefaultCollectionConfig,+ type ContentArtifactV1,+} from "@webstudio-is/content-engine";import type { Asset } from "@webstudio-is/sdk";import {preparePublishedAssetData,@@ -340,14 +343,38 @@ describe("published asset data", () => {.fn().mockResolvedValueOnce(preliminaryArtifact).mockResolvedValueOnce(finalArtifact);- const resolvePlan = vi.fn(() => resolvedPlan);+ const source = "# Article";+ const assetStore = {+ readFile: vi.fn().mockResolvedValue({+ data: new Blob([source]).stream(),+ }),+ };+ const resolvePlan = vi.fn(+ async (+ _artifact: ContentArtifactV1,+ loadDocumentSource: (input: {+ id: string;+ revision: string;+ contentRef: string;+ }) => Promise<string>+ ) => {+ expect(+ await loadDocumentSource({+ id: "article.mdx",+ revision: "revision-1",+ contentRef: "article.mdx",+ })+ ).toBe(source);+ return resolvedPlan;+ }+ );const assetData = { assets: [], assetFolders: [] };const result = await preparePublishedAssetData({projectId: "project-1",context: {} as never,- assetStore: {} as never,+ assetStore: assetStore as never,contentDatabaseMaxBytes: 512_000,plan: initialPlan,retainedAssetIds: [],@@ -366,10 +393,16 @@ describe("published asset data", () => {expect(result.artifact).toBe(finalArtifact);expect(prepareIndex.mock.calls).toEqual([[initialPlan], [resolvedPlan]]);- expect(resolvePlan.mock.calls).toEqual([+ expect(resolvePlan.mock.calls.map(([artifact]) => [artifact])).toEqual([[preliminaryArtifact],[finalArtifact],]);+ expect(+ resolvePlan.mock.calls.every(([, loadDocumentSource]) =>+ Boolean(loadDocumentSource)+ )+ ).toBe(true);+ expect(assetStore.readFile).toHaveBeenCalledExactlyOnceWith("article.mdx");});test("keeps the initial artifact when its dependency plan is already stable", async () => {diff --git a/packages/asset-uploader/src/publication.ts b/packages/asset-uploader/src/publication.tsindex 369be6d641..90613bd6c7 100644--- a/packages/asset-uploader/src/publication.ts+++ b/packages/asset-uploader/src/publication.ts@@ -1,10 +1,13 @@import {+ contentEngineLimits,getContentArtifactRuntimeAssetIds,type ContentCompilationPlan,type ContentArtifactV1,} from "@webstudio-is/content-engine";import {compileContentUntilPlanIsStable,+ decodeUtf8,+ readBoundedBytes,serializeJsonDeterministically,} from "@webstudio-is/content-engine/compiler";import type { Asset } from "@webstudio-is/sdk";@@ -120,6 +123,34 @@ export const validatePublishedAssetCollections = async (return assetData;};+export const createPublishedMdxDependencySourceLoader = (+ assetStore: Pick<AssetObjectStore, "readFile">+) => {+ const sources = new Map<string, Promise<string>>();+ return ({+ id,+ revision,+ contentRef,+ }: {+ id: string;+ revision: string;+ contentRef: string;+ }) => {+ const key = JSON.stringify([id, revision, contentRef]);+ let source = sources.get(key);+ if (source === undefined) {+ source = (async () => {+ const { data } = await assetStore.readFile(contentRef);+ return decodeUtf8(+ await readBoundedBytes(data, contentEngineLimits.hydratedFileBytes)+ );+ })();+ sources.set(key, source);+ }+ return source;+ };+};+export const preparePublishedAssetData = async ({projectId,@@ -137,7 +168,12 @@ export const preparePublishedAssetData = async (plan: ContentCompilationPlan;retainedAssetIds: Iterable<string>;resolvePlan?: (- artifact: ContentArtifactV1+ artifact: ContentArtifactV1,+ loadDocumentSource: (input: {+ id: string;+ revision: string;+ contentRef: string;+ }) => Promise<string>) => ContentCompilationPlan | Promise<ContentCompilationPlan>;},dependencies = defaultDependencies@@ -148,6 +184,8 @@ export const preparePublishedAssetData = async (assetStore,contentDatabaseMaxBytes,});+ const loadDocumentSource =+ createPublishedMdxDependencySourceLoader(assetStore);const { result: artifact, assetData } = await prepareStablePublishedAssetData({projectId,@@ -163,7 +201,8 @@ export const preparePublishedAssetData = async (: await compileContentUntilPlanIsStable({plan,compile: prepareIndex,- resolvePlan,+ resolvePlan: (artifact) =>+ resolvePlan(artifact, loadDocumentSource),});}),dependencies,diff --git a/packages/cli/src/prebuild.test.ts b/packages/cli/src/prebuild.test.tsindex 312e73c544..9c39eb050f 100644--- a/packages/cli/src/prebuild.test.ts+++ b/packages/cli/src/prebuild.test.ts@@ -963,7 +963,7 @@ describe("prebuild", () => {).rejects.toThrow(hasSource === undefined? "require a content database"- : 'Published MDX Asset "article" content is unavailable'+ : 'Published MDX Asset "article" could not be loaded');return;}diff --git a/packages/cli/src/prebuild.ts b/packages/cli/src/prebuild.tsindex 3b4f6faed3..1cc22addca 100644--- a/packages/cli/src/prebuild.ts+++ b/packages/cli/src/prebuild.ts@@ -981,6 +981,7 @@ export const prebuild = async (options: {const siteData = parsedSiteData.data;const pages = migratePages(siteData.build.pages);const publicationBuild = { ...siteData.build, pages };+ const allAssetIds = new Set(siteData.assets.map(({ id }) => id));const allMdxAssetIds = getPublishedMdxAssetIds(siteData.assets);if (siteData.assetIndex === undefined &&@@ -1068,6 +1069,7 @@ export const prebuild = async (options: {build: publicationBuild,artifact: verifiedAssetIndex,allowUnresolved: true,+ allAssetIds,allMdxAssetIds,});for (const blockInstanceId of getDynamicPublishedMdxSourceBlockIds(@@ -1336,6 +1338,7 @@ export const prebuild = async (options: {build: pageBuild,artifact: verifiedAssetIndex,blockInstanceIds: pendingBlockIds,+ allAssetIds,allMdxAssetIds,}): undefined;diff --git a/packages/http-client/src/index.test.ts b/packages/http-client/src/index.test.tsindex f183fe6bb1..13d7b300f1 100644--- a/packages/http-client/src/index.test.ts+++ b/packages/http-client/src/index.test.ts@@ -3,6 +3,7 @@ import { createServer, type IncomingMessage } from "node:http";import { afterEach, expect, test, vi } from "vitest";import {bundleVersion,+ maxProjectBundleSize,publicApiOperations,stagedUploadPath,} from "@webstudio-is/protocol";@@ -2247,7 +2248,8 @@ test("imports project bundle through staged upload", async () => {origin: `http://127.0.0.1:${address.port}`,projectId: "project-id",data: {- largeContent: "x".repeat(3 * 1024 * 1024 + 1),+ // Real customer builds can exceed the old 20 MiB import limit.+ largeContent: "x".repeat(28 * 1024 * 1024),assetIndex: { marker: "derived-index-marker" },} as unknown as PublishedProjectBundle,})@@ -2258,10 +2260,10 @@ test("imports project bundle through staged upload", async () => {});}- expect(uploadChunks).toHaveLength(2);- expect(Buffer.concat(uploadChunks).toString("utf8")).not.toContain(- "derived-index-marker"- );+ const importedData = Buffer.concat(uploadChunks);+ expect(importedData.byteLength).toBeGreaterThan(20 * 1024 * 1024);+ expect(importedData.byteLength).toBeLessThan(maxProjectBundleSize);+ expect(importedData.toString("utf8")).not.toContain("derived-index-marker");expect(JSON.stringify(trpcBody)).toContain('"uploadId":"upload-id"');expect(JSON.stringify(trpcBody)).not.toContain("largeContent");});@@ -2478,11 +2480,11 @@ test("rejects project bundles over the import size limit", async () => {origin: `http://127.0.0.1:${address.port}`,projectId: "project-id",data: {- largeContent: "x".repeat(20 * 1024 * 1024),+ largeContent: "x".repeat(maxProjectBundleSize),} as unknown as PublishedProjectBundle,})).rejects.toThrow(- "Project bundle is too large to import. Maximum size is 20 MiB."+ "Project bundle is too large to import. Maximum size is 32 MiB.");} finally {await new Promise<void>((resolve, reject) => {diff --git a/packages/project-build/src/content-database.test.ts b/packages/project-build/src/content-database.test.tsindex 515b0cd5f5..6af2664497 100644--- a/packages/project-build/src/content-database.test.ts+++ b/packages/project-build/src/content-database.test.ts@@ -239,7 +239,7 @@ describe("Content Block MDX compilation", () => {);});- test("retains nested sources only through referenced templates", async () => {+ test("discovers nested sources when the MDX body must be loaded", async () => {const build = createBuild({});build.instances[0].children = [{ type: "id", value: "templates" }];build.instances.push(@@ -308,14 +308,21 @@ describe("Content Block MDX compilation", () => {contentRef: "article.mdx",},],- contents: { "article.mdx": source },+ contents: {},} as unknown as ContentArtifactV1;const omissions: { assetId: string; templateName: string }[] = [];+ const loadDocumentSource = vi.fn(async () => source);const plan = await resolvePublishedMdxDependencyClosure({build,artifact,onTemplateOmission: (issue) => omissions.push(issue),+ loadDocumentSource,+ });+ expect(loadDocumentSource).toHaveBeenCalledExactlyOnceWith({+ id: "article.mdx",+ revision: "article-revision",+ contentRef: "article.mdx",});expect(omissions).toEqual([expect.objectContaining({@@ -329,6 +336,130 @@ describe("Content Block MDX compilation", () => {expect(queryIds).not.toContain("__content-block-mdx__:private.mdx");});+ test("keeps dependency planning non-blocking when source loading fails", async () => {+ const build = createBuild({});+ const artifact = {+ format: "webstudio-content-database",+ version: 1,+ documents: [+ {+ _id: "article.mdx",+ _type: "asset.file",+ name: "article.mdx",+ path: "article.mdx",+ key: "article",+ extension: "mdx",+ mimeType: "text/mdx",+ size: 1,+ revision: "article-revision",+ contentRef: "article.mdx",+ },+ ],+ contents: {},+ } as unknown as ContentArtifactV1;++ const failures: unknown[] = [];+ const plan = await resolvePublishedMdxDependencyClosure({+ build,+ artifact,+ loadDocumentSource: async () => {+ throw new Error("asset store unavailable");+ },+ onDependencyReadFailure: (failure) => failures.push(failure),+ });++ expect(plan?.queries.map(({ id }) => id)).toContain(+ "__content-block-mdx__:article.mdx"+ );+ expect(failures).toEqual([+ {+ blockInstanceId: expect.any(String),+ assetId: "article.mdx",+ contentRef: "article.mdx",+ message: expect.stringContaining("asset store unavailable"),+ },+ ]);+ });++ test("loads MDX dependencies in the same publication depth concurrently", async () => {+ const build = createBuild({});+ build.instances[0].children = [+ { type: "id", value: "first-templates" },+ { type: "id", value: "second-block" },+ ];+ build.instances.push({+ type: "instance",+ id: "second-block",+ component: "ws:block",+ children: [{ type: "id", value: "second-templates" }],+ });+ build.instances.push(+ {+ type: "instance",+ id: "first-templates",+ component: "ws:block-template",+ children: [],+ },+ {+ type: "instance",+ id: "second-templates",+ component: "ws:block-template",+ children: [],+ }+ );+ build.props.push({+ id: "second-source",+ instanceId: "second-block",+ name: "src",+ type: "asset",+ value: "second.mdx",+ });+ const createDocument = (id: string) => ({+ _id: id,+ _type: "asset.file",+ name: id,+ path: id,+ key: id,+ extension: "mdx",+ mimeType: "text/mdx",+ size: 1,+ revision: `${id}-revision`,+ contentRef: id,+ });+ const artifact = {+ format: "webstudio-content-database",+ version: 1,+ documents: [createDocument("article.mdx"), createDocument("second.mdx")],+ contents: {},+ } as unknown as ContentArtifactV1;+ let resolveBothReads: () => void = () => {};+ const bothReadsStarted = new Promise<void>((resolve) => {+ resolveBothReads = resolve;+ });+ const startedIds: string[] = [];++ const plan = await resolvePublishedMdxDependencyClosure({+ build,+ artifact,+ loadDocumentSource: async ({ id }) => {+ startedIds.push(id);+ if (startedIds.length === 2) {+ resolveBothReads();+ }+ await bothReadsStarted;+ return "# Article";+ },+ });++ expect(startedIds).toEqual(["article.mdx", "second.mdx"]);+ expect(plan?.queries.map(({ id }) => id)).toEqual(+ expect.arrayContaining([+ "__content-block-mdx__:article.mdx",+ "__content-block-mdx__:second.mdx",+ ])+ );+ });+test("retains nested sources through semantic component templates", async () => {const build = createBuild({});build.instances[0].children = [{ type: "id", value: "templates" }];@@ -556,6 +687,88 @@ describe("Content Block MDX compilation", () => {});+ test("excludes known non-MDX assets from dynamic source candidates", () => {+ const resourceVariable = encodeDataVariableId("posts-data");+ const build = createBuild({ sourceType: "expression" });+ build.props[0].value = `${resourceVariable}.data.id`;+ build.dataSources.push({+ type: "resource",+ id: "posts-data",+ scopeInstanceId: "block",+ name: "posts",+ resourceId: "posts",+ });+ build.resources.push({+ id: "posts",+ name: "Posts",+ control: "system",+ method: "post",+ url: '"/$resources/assets"',+ headers: [],+ body: createStructuredAssetQueryResourceBody({+ where: {+ all: [+ {+ field: ["properties", "slug"],+ operator: "eq",+ value: '"legal"',+ },+ ],+ },+ sort: [],+ limit: "10",+ offset: "0",+ output: { mode: "all", includeMetadata: false },+ content: { mode: "none" },+ }),+ });+ const artifact = {+ documents: [+ {+ _id: "article.mdx",+ name: "article.mdx",+ path: "MDX-Data/Guidelines/article.mdx",+ key: "article",+ extension: "mdx",+ mimeType: "text/mdx",+ size: 1,+ properties: { slug: "legal", mdx: "collection.json" },+ },+ {+ _id: "collection.json",+ name: "collection.json",+ path: "MDX-Data/Guidelines/collection.json",+ key: "collection",+ extension: "json",+ mimeType: "application/json",+ size: 1,+ properties: { slug: "legal" },+ },+ ],+ } as unknown as ContentArtifactV1;++ expect(+ resolvePublishedMdxAssetCandidates({+ build,+ artifact,+ allAssetIds: new Set(["article.mdx", "collection.json"]),+ allMdxAssetIds: new Set(["article.mdx"]),+ }).get("block")+ ).toEqual(["article.mdx"]);++ // An explicitly referenced file must survive candidate discovery so+ // materialization can report that it is not an MDX asset.+ build.props[0].value = `${resourceVariable}.data.properties.mdx`;+ expect(+ resolvePublishedMdxAssetCandidates({+ build,+ artifact,+ allAssetIds: new Set(["article.mdx", "collection.json"]),+ allMdxAssetIds: new Set(["article.mdx"]),+ }).get("block")+ ).toEqual(["collection.json"]);+ });+test("bounds a mutable project variable to the project's MDX assets", () => {const build = createBuild({ sourceType: "expression" });build.props[0].value = encodeDataVariableId("article-source");diff --git a/packages/project-build/src/content-database.ts b/packages/project-build/src/content-database.tsindex 7e2e845cde..b2df82d61f 100644--- a/packages/project-build/src/content-database.ts+++ b/packages/project-build/src/content-database.ts@@ -36,13 +36,18 @@ import {} from "@webstudio-is/sdk";import { parseStaticMemberPath } from "@webstudio-is/expression";import type { Pages } from "@webstudio-is/sdk";+import { mapBounded } from "@webstudio-is/content-engine/compiler";import { componentMetas } from "@webstudio-is/sdk-components-registry/metas";import { parseMdxDocumentRecovering } from "@webstudio-is/content-engine/mdx";+import {assertMdxTemplateStructure,resolveMdxTemplates,} from "./runtime/mdx-template-resolution";+const maxConcurrentMdxDependencyBlocks = 4;+const maxConcurrentMdxDependenciesPerBlock = 8;+type BuildValues<Value> =| readonly Value[]| readonly (readonly [string, Value])[];@@ -227,10 +232,25 @@ export type PublishedMdxTemplateOmission = {templateName: string;};+export type PublishedMdxDependencyReadFailure = {+ blockInstanceId: string;+ assetId: string;+ contentRef: string;+ message: string;+};+type ResolvePublishedMdxDependencyClosureOptions = {build: PublishedContentDatabaseBuild;artifact: ContentArtifactV1;+ loadDocumentSource?: (input: {+ id: string;+ revision: string;+ contentRef: string;+ }) => Promise<string>;onTemplateOmission?: (issue: PublishedMdxTemplateOmission) => void;+ onDependencyReadFailure?: (+ failure: PublishedMdxDependencyReadFailure+ ) => void;onMdxBlockInstanceId?: (blockInstanceId: string) => void;allMdxAssetIds?: ReadonlySet<string>;};@@ -238,7 +258,9 @@ type ResolvePublishedMdxDependencyClosureOptions = {const resolvePublishedMdxDependencyClosureWithParser = async ({build,artifact,+ loadDocumentSource,onTemplateOmission,+ onDependencyReadFailure,onMdxBlockInstanceId,allMdxAssetIds,parseDocument,@@ -286,73 +308,132 @@ const resolvePublishedMdxDependencyClosureWithParser = async ({};while (pending.length > 0) {- const blockId = pending.shift()!;- if (processed.has(blockId)) {- continue;- }- processed.add(blockId);- const source = sourcesByBlockId.get(blockId);- if (source === undefined) {- continue;- }- onMdxBlockInstanceId?.(blockId);- const staticValue = getStaticContentBlockSourceAssetId(source);- const assetIds =- typeof staticValue === "string"- ? [staticValue]- : (resolvePublishedMdxAssetCandidates({- build,- artifact,- blockInstanceIds: new Set([blockId]),- allMdxAssetIds,- }).get(blockId) ?? []);- for (const assetId of assetIds) {- const documentEntry = documentsById.get(assetId);- const sourceText =- documentEntry?.contentRef === undefined- ? undefined- : artifact.contents?.[documentEntry.contentRef];- if (- documentEntry?.revision === undefined ||- documentEntry.contentRef === undefined ||- sourceText === undefined- ) {- continue;- }- const parsed = await parseDocument({- assetId,- revision: documentEntry.revision,- contentRef: documentEntry.contentRef,- source: sourceText,- });- if (parsed.status === "unrecoverable") {- continue;+ // Resolve each dependency depth concurrently. A level can contain many+ // Content Blocks, and serial asset-store reads here would add their+ // latencies to publication preparation one by one.+ const blockIds = pending.splice(0);+ const parsedDocumentsByBlock = await mapBounded(+ blockIds,+ maxConcurrentMdxDependencyBlocks,+ async (blockId) => {+ if (processed.has(blockId)) {+ return { blockId, parsedDocuments: [] };+ }+ processed.add(blockId);+ const source = sourcesByBlockId.get(blockId);+ if (source === undefined) {+ return { blockId, parsedDocuments: [] };+ }+ onMdxBlockInstanceId?.(blockId);+ const staticValue = getStaticContentBlockSourceAssetId(source);+ const assetIds =+ typeof staticValue === "string"+ ? [staticValue]+ : (resolvePublishedMdxAssetCandidates({+ build,+ artifact,+ blockInstanceIds: new Set([blockId]),+ allMdxAssetIds,+ }).get(blockId) ?? []);+ const parsedDocuments = await mapBounded(+ assetIds,+ maxConcurrentMdxDependenciesPerBlock,+ async (assetId) => {+ const documentEntry = documentsById.get(assetId);+ if (+ documentEntry?.extension?.toLowerCase() !== "mdx" &&+ documentEntry?.mimeType !== "text/mdx"+ ) {+ return;+ }+ let sourceText =+ documentEntry.contentRef === undefined+ ? undefined+ : artifact.contents?.[documentEntry.contentRef];+ if (+ sourceText === undefined &&+ documentEntry.revision !== undefined &&+ documentEntry.contentRef !== undefined &&+ loadDocumentSource !== undefined+ ) {+ try {+ sourceText = await loadDocumentSource({+ id: assetId,+ revision: documentEntry.revision,+ contentRef: documentEntry.contentRef,+ });+ } catch (error) {+ onDependencyReadFailure?.({+ blockInstanceId: blockId,+ assetId,+ contentRef: documentEntry.contentRef,+ message: `Could not inspect this MDX source: ${+ error instanceof Error ? error.message : String(error)+ }`,+ });+ return;+ }+ }+ if (+ documentEntry.revision === undefined ||+ documentEntry.contentRef === undefined ||+ sourceText === undefined+ ) {+ return;+ }+ const parsed = await parseDocument({+ assetId,+ revision: documentEntry.revision,+ contentRef: documentEntry.contentRef,+ source: sourceText,+ });+ return parsed.status === "unrecoverable"+ ? undefined+ : { assetId, documentEntry, document: parsed.document };+ }+ );+ return {+ blockId,+ parsedDocuments: parsedDocuments.filter(+ (document) => document !== undefined+ ),+ };}- const document = parsed.document;- const resolution = resolveMdxTemplates({- document,- identity: {- blockInstanceId: blockId,- assetId,- revision: documentEntry.revision,- contentRef: documentEntry.contentRef,- format: "mdx",- renderScope: "publication-dependency-discovery",- },- instances,- props: propsById,- metas: componentMetas,- });- assertMdxTemplateStructure(resolution);- for (const reference of resolution.references) {- if (reference.type === "resolved-template") {- visitTemplateSubtree(reference.templateInstanceId);- } else {- onTemplateOmission?.({+ );++ for (const { blockId, parsedDocuments } of parsedDocumentsByBlock) {+ for (const { assetId, documentEntry, document } of parsedDocuments) {+ if (+ documentEntry.revision === undefined ||+ documentEntry.contentRef === undefined+ ) {+ continue;+ }+ const resolution = resolveMdxTemplates({+ document,+ identity: {blockInstanceId: blockId,assetId,- templateName: reference.templateName,- });+ revision: documentEntry.revision,+ contentRef: documentEntry.contentRef,+ format: "mdx",+ renderScope: "publication-dependency-discovery",+ },+ instances,+ props: propsById,+ metas: componentMetas,+ });+ assertMdxTemplateStructure(resolution);+ for (const reference of resolution.references) {+ if (reference.type === "resolved-template") {+ visitTemplateSubtree(reference.templateInstanceId);+ } else {+ onTemplateOmission?.({+ blockInstanceId: blockId,+ assetId,+ templateName: reference.templateName,+ });+ }}}}@@ -414,9 +495,20 @@ const isCandidateDocument = (value: unknown): value is CandidateDocument =>const getValueAtPath = (value: unknown | CandidateDocument,- path: readonly string[]+ path: readonly string[],+ knownNonMdxAssetIds?: ReadonlySet<string>): unknown => {if (isCandidateDocument(value)) {+ // Asset queries can include non-MDX records while route parameters are+ // unknown. Exclude their own IDs, but preserve explicit field references+ // so materialization can diagnose invalid sources.+ if (+ knownNonMdxAssetIds?.has(value[candidateDocumentKey]._id) &&+ (path.length === 0 ||+ (path.length === 1 && (path[0] === "id" || path[0] === "_id")))+ ) {+ return;+ }if (path[0] === "properties" &&value.graph !== undefined &&@@ -450,12 +542,14 @@ export const resolvePublishedMdxAssetCandidates = ({artifact,allowUnresolved = false,blockInstanceIds = getPublishedInstanceIds(build),+ allAssetIds,allMdxAssetIds,}: {build: PublishedContentDatabaseBuild;artifact?: ContentArtifactV1;allowUnresolved?: boolean;blockInstanceIds?: ReadonlySet<string>;+ allAssetIds?: ReadonlySet<string>;allMdxAssetIds?: ReadonlySet<string>;}) => {const instances = new Map(@@ -488,6 +582,12 @@ export const resolvePublishedMdxAssetCandidates = ({// Compiled documents are projected file records; metadata such as _type// can be omitted by the resource's output selection.const documents = artifact?.documents ?? [];+ const knownNonMdxAssetIds =+ allAssetIds === undefined || allMdxAssetIds === undefined+ ? undefined+ : new Set(+ [...allAssetIds].filter((assetId) => !allMdxAssetIds.has(assetId))+ );const evaluateDataSource = (dataSourceId: string,@@ -566,7 +666,8 @@ export const resolvePublishedMdxAssetCandidates = ({} satisfies CandidateDocument): getValueAtPath({ [candidateDocumentKey]: document, graph: documentGraph },- fieldPath+ fieldPath,+ knownNonMdxAssetIds));}@@ -622,7 +723,7 @@ export const resolvePublishedMdxAssetCandidates = ({}return [iterable];});- return items.map((item) => getValueAtPath(item, path));+ return items.map((item) => getValueAtPath(item, path, knownNonMdxAssetIds));};const candidatesByBlock = new Map<string, readonly string[]>();diff --git a/packages/project-build/src/contracts/__generated__/runtime-operation-contracts.ts b/packages/project-build/src/contracts/__generated__/runtime-operation-contracts.tsindex 76fc6c9378..5c3672ecbe 100644--- a/packages/project-build/src/contracts/__generated__/runtime-operation-contracts.ts+++ b/packages/project-build/src/contracts/__generated__/runtime-operation-contracts.ts@@ -162,6 +162,18 @@ export const runtimeOperationContractData = [type: "string",minLength: 1,},+ reason: {+ type: "string",+ enum: [+ "source-read-failed",+ "missing-source",+ "missing-asset",+ "linked-document-unavailable",+ "html-content-model",+ "dynamic-resource",+ "dynamic-source-unbounded",+ ],+ },},required: ["blockInstanceId", "code", "severity", "message"],additionalProperties: false,@@ -742,6 +754,18 @@ export const runtimeOperationContractData = [type: "string",minLength: 1,},+ reason: {+ type: "string",+ enum: [+ "source-read-failed",+ "missing-source",+ "missing-asset",+ "linked-document-unavailable",+ "html-content-model",+ "dynamic-resource",+ "dynamic-source-unbounded",+ ],+ },},required: ["blockInstanceId", "code", "severity", "message"],additionalProperties: false,@@ -1341,6 +1365,18 @@ export const runtimeOperationContractData = [type: "string",minLength: 1,},+ reason: {+ type: "string",+ enum: [+ "source-read-failed",+ "missing-source",+ "missing-asset",+ "linked-document-unavailable",+ "html-content-model",+ "dynamic-resource",+ "dynamic-source-unbounded",+ ],+ },},required: ["blockInstanceId", "code", "severity", "message"],additionalProperties: false,@@ -1906,6 +1942,18 @@ export const runtimeOperationContractData = [type: "string",minLength: 1,},+ reason: {+ type: "string",+ enum: [+ "source-read-failed",+ "missing-source",+ "missing-asset",+ "linked-document-unavailable",+ "html-content-model",+ "dynamic-resource",+ "dynamic-source-unbounded",+ ],+ },},required: ["blockInstanceId", "code", "severity", "message"],additionalProperties: false,@@ -2477,6 +2525,18 @@ export const runtimeOperationContractData = [type: "string",minLength: 1,},+ reason: {+ type: "string",+ enum: [+ "source-read-failed",+ "missing-source",+ "missing-asset",+ "linked-document-unavailable",+ "html-content-model",+ "dynamic-resource",+ "dynamic-source-unbounded",+ ],+ },},required: ["blockInstanceId", "code", "severity", "message"],additionalProperties: false,@@ -3030,6 +3090,18 @@ export const runtimeOperationContractData = [type: "string",minLength: 1,},+ reason: {+ type: "string",+ enum: [+ "source-read-failed",+ "missing-source",+ "missing-asset",+ "linked-document-unavailable",+ "html-content-model",+ "dynamic-resource",+ "dynamic-source-unbounded",+ ],+ },},required: ["blockInstanceId", "code", "severity", "message"],additionalProperties: false,@@ -3576,6 +3648,18 @@ export const runtimeOperationContractData = [type: "string",minLength: 1,},+ reason: {+ type: "string",+ enum: [+ "source-read-failed",+ "missing-source",+ "missing-asset",+ "linked-document-unavailable",+ "html-content-model",+ "dynamic-resource",+ "dynamic-source-unbounded",+ ],+ },},required: ["blockInstanceId", "code", "severity", "message"],additionalProperties: false,@@ -51482,6 +51566,18 @@ export const runtimeOperationContractData = [type: "string",minLength: 1,},+ reason: {+ type: "string",+ enum: [+ "source-read-failed",+ "missing-source",+ "missing-asset",+ "linked-document-unavailable",+ "html-content-model",+ "dynamic-resource",+ "dynamic-source-unbounded",+ ],+ },},required: ["blockInstanceId", "code", "severity", "message"],additionalProperties: false,diff --git a/packages/project-build/src/runtime/data.ts b/packages/project-build/src/runtime/data.tsindex 32ffd93867..2c2115242e 100644--- a/packages/project-build/src/runtime/data.ts+++ b/packages/project-build/src/runtime/data.ts@@ -669,6 +669,41 @@ const findVariablesByInstanceId = ({return { maskedIdByName, availableDataSourceIds };};+export const createFindAvailableVariables = ({+ instances,+ dataSources,+}: {+ instances: Instances;+ dataSources: DataSources;+}) => {+ // These indexes depend only on the project data, not on the starting+ // instance. Reuse them when resolving variables for many template roots.+ const parentInstanceById = getParentInstanceById(instances);+ const dataSourcesByScopeInstanceId =+ getDataSourcesByScopeInstanceId(dataSources);++ return (startingInstanceId: Instance["id"]) => {+ const { maskedIdByName } = findVariablesByInstanceId({+ startingInstanceId,+ parentInstanceById,+ instances,+ dataSources,+ dataSourcesByScopeInstanceId,+ });+ const availableVariables: DataSource[] = [];+ for (const dataSourceId of maskedIdByName.values()) {+ const dataSource = dataSources.get(dataSourceId);+ if (dataSource) {+ availableVariables.push(dataSource);+ }+ if (dataSourceId === SYSTEM_VARIABLE_ID) {+ availableVariables.push(systemParameter);+ }+ }+ return availableVariables;+ };+};+export const findAvailableVariables = ({startingInstanceId,instances,@@ -678,24 +713,9 @@ export const findAvailableVariables = ({instances: Instances;dataSources: DataSources;}) => {- const { maskedIdByName } = findVariablesByInstanceId({- startingInstanceId,- parentInstanceById: getParentInstanceById(instances),- instances,- dataSources,- dataSourcesByScopeInstanceId: getDataSourcesByScopeInstanceId(dataSources),- });- const availableVariables: DataSource[] = [];- for (const dataSourceId of maskedIdByName.values()) {- const dataSource = dataSources.get(dataSourceId);- if (dataSource) {- availableVariables.push(dataSource);- }- if (dataSourceId === SYSTEM_VARIABLE_ID) {- availableVariables.push(systemParameter);- }- }- return availableVariables;+ return createFindAvailableVariables({ instances, dataSources })(+ startingInstanceId+ );};export const bindExpressionToInstanceScope = ({diff --git a/packages/project-build/src/runtime/mdx-diagnostics.ts b/packages/project-build/src/runtime/mdx-diagnostics.tsindex 0d06db99b0..5661389d8d 100644--- a/packages/project-build/src/runtime/mdx-diagnostics.ts+++ b/packages/project-build/src/runtime/mdx-diagnostics.ts@@ -85,6 +85,7 @@ export const createMdxContentModelDiagnostics = ({{code: "invalid-mdx" as const,severity: "error" as const,+ reason: "html-content-model" as const,blockInstanceId: root.identity.blockInstanceId,assetId: root.identity.assetId,contentRef: root.identity.contentRef,diff --git a/packages/project-build/src/runtime/mdx-materialization.test.ts b/packages/project-build/src/runtime/mdx-materialization.test.tsindex e28c6dad22..2cc6beb4e9 100644--- a/packages/project-build/src/runtime/mdx-materialization.test.ts+++ b/packages/project-build/src/runtime/mdx-materialization.test.ts@@ -274,6 +274,46 @@ const getUnresolvedTemplate = (};describe("materializeMdxTemplates", () => {+ test("materializes repeated resolved references with consistent scope", async () => {+ const materialization = await materializeMdxTemplates({+ identity,+ resolution: {+ ...resolution,+ references: [+ resolution.references[0],+ { ...resolution.references[0], path: [1] },+ ],+ },+ data: createData(),+ metas,+ projectId: "target-project",+ });++ expect(materialization.templates).toHaveLength(2);+ expect(+ materialization.templates.every(+ (template) => template.type === "resolved-template"+ )+ ).toBe(true);+ const [first, second] = materialization.templates.map((template) =>+ getResolvedTemplate([template])+ );+ expect(+ first?.fragment.props.map(({ name, type, value }) => ({+ name,+ type,+ value,+ }))+ ).toEqual(+ second?.fragment.props.map(({ name, type, value }) => ({+ name,+ type,+ value,+ }))+ );+ expect(first?.fragment.instances).not.toEqual(second?.fragment.instances);+ });+test("copies resolved fragments and represents unresolved usages", async () => {const data = createData();const originalData = structuredClone(data);diff --git a/packages/project-build/src/runtime/mdx-materialization.ts b/packages/project-build/src/runtime/mdx-materialization.tsindex 504251d5ed..e09f69b432 100644--- a/packages/project-build/src/runtime/mdx-materialization.ts+++ b/packages/project-build/src/runtime/mdx-materialization.ts@@ -17,6 +17,7 @@ import {getHtmlTagFromInstance,type ContentBlockDiagnostic,type ContentBlockExternalContentIdentity,+ type DataSource,type Instance,type Prop,type WebstudioData,@@ -28,7 +29,7 @@ import {getContentModeEditableStaticPropNames,getContentModePropEligibility,} from "./content-mode-permissions";-import { findAvailableVariables } from "./data";+import { createFindAvailableVariables } from "./data";import {createWebstudioDataFromFragment,extractWebstudioFragment,@@ -244,6 +245,7 @@ export const materializeMdxTemplates = async ({metas,projectId,assetReferences = [],+ findAvailableVariables: sharedFindAvailableVariables,}: {identity: ContentBlockExternalContentIdentity;resolution: MdxTemplateResolution;@@ -251,6 +253,7 @@ export const materializeMdxTemplates = async ({metas: Map<string, WsComponentMeta>;projectId: string;assetReferences?: readonly AssetValueReference[];+ findAvailableVariables?: (startingInstanceId: Instance["id"]) => DataSource[];}): Promise<MdxTemplateMaterialization> => {const materializedTemplates: MaterializedMdxTemplate[] = [];const diagnostics: ContentBlockDiagnostic[] = [...resolution.diagnostics];@@ -267,6 +270,12 @@ export const materializeMdxTemplates = async ({reference,]));+ let findAvailableVariables = sharedFindAvailableVariables;+ const getFindAvailableVariables = () =>+ (findAvailableVariables ??= createFindAvailableVariables({+ instances: data.instances,+ dataSources: data.dataSources,+ }));for (const reference of resolution.references) {if (reference.type === "unresolved-template") {materializedTemplates.push({@@ -314,11 +323,9 @@ export const materializeMdxTemplates = async ({const { newInstanceIds } = insertWebstudioFragmentCopy({data: materializedData,fragment: sourceFragment,- availableVariables: findAvailableVariables({- startingInstanceId: reference.templateInstanceId,- instances: data.instances,- dataSources: data.dataSources,- }),+ availableVariables: getFindAvailableVariables()(+ reference.templateInstanceId+ ),projectId,createId,});diff --git a/packages/project-build/src/runtime/published-mdx.test.ts b/packages/project-build/src/runtime/published-mdx.test.tsindex 0cd0167e01..2368348b66 100644--- a/packages/project-build/src/runtime/published-mdx.test.ts+++ b/packages/project-build/src/runtime/published-mdx.test.ts@@ -127,6 +127,56 @@ describe("published MDX materialization", () => {expect(getFragmentText(result.roots[0].fragment)).toContain("Published");});+ test("loads an MDX source when it is not embedded in the artifact", async () => {+ const source = "# Loaded from asset storage";+ const artifact = createArtifact([{ id: "article", source }]);+ delete artifact.contents?.["article.mdx"];+ const loadDocumentSource = vi.fn(async () => source);++ const result = await materializePublishedMdx({+ route: "/blog/article",+ data: createData({}),+ artifact,+ metas: new Map(),+ projectId: "project",+ loadDocumentSource,+ });++ expect(result.roots).toHaveLength(1);+ expect(loadDocumentSource).toHaveBeenCalledExactlyOnceWith({+ id: "article",+ revision: revision("b"),+ contentRef: "article.mdx",+ });+ });++ test("reports an MDX source load failure without rejecting materialization", async () => {+ const artifact = createArtifact([{ id: "article", source: "# Published" }]);+ delete artifact.contents?.["article.mdx"];++ const result = await materializePublishedMdx({+ route: "/blog/article",+ data: createData({}),+ artifact,+ metas: new Map(),+ projectId: "project",+ loadDocumentSource: async () => {+ throw new Error("asset store unavailable");+ },+ });++ expect(result.roots).toHaveLength(0);+ expect(result.warnings).toEqual([+ expect.objectContaining({+ diagnostic: expect.objectContaining({+ severity: "error",+ assetId: "article",+ message: expect.stringContaining("asset store unavailable"),+ }),+ }),+ ]);+ });+test("rejects publication with multiple Templates containers", async () => {const data = createData({ withTemplate: true });const block = data.instances.get("block");diff --git a/packages/project-build/src/runtime/published-mdx.ts b/packages/project-build/src/runtime/published-mdx.tsindex ba7fe9e1c7..68289044b1 100644--- a/packages/project-build/src/runtime/published-mdx.ts+++ b/packages/project-build/src/runtime/published-mdx.ts@@ -31,11 +31,13 @@ import {type ContentBlockDiagnostic,type ContentBlockExternalContentIdentity,type ContentBlockSource,+ type InvalidMdxDiagnosticReason,type WebstudioData,type WsComponentMeta,} from "@webstudio-is/sdk";import { parseStaticMemberPath } from "@webstudio-is/expression";import { materializeMdxAuthoredContent } from "./mdx-authored-content";+import { createFindAvailableVariables } from "./data";import {materializeMdxTemplates,type MdxTemplateDependency,@@ -96,6 +98,7 @@ export const getUnsafeDynamicPublishedMdxDiagnostic = ({return {code: "invalid-mdx",severity: "error",+ reason: "dynamic-resource",blockInstanceId: root.identity.blockInstanceId,assetId: root.identity.assetId,renderScope: `route:${route}:block:${root.identity.blockInstanceId}`,@@ -112,6 +115,7 @@ export const getUnsafeDynamicPublishedMdxDiagnostic = ({return {code: "invalid-mdx",severity: "error",+ reason: "dynamic-resource",blockInstanceId: root.identity.blockInstanceId,assetId: root.identity.assetId,renderScope: `route:${route}:block:${root.identity.blockInstanceId}`,@@ -229,6 +233,7 @@ export const materializePublishedMdx = async ({runtimeAssets?: Readonly<Record<string, AssetRuntimeData>>;loadDocumentSource?: (node: {id: string;+ revision: string;contentRef: string;}) => Promise<string>;}): Promise<{@@ -303,13 +308,18 @@ export const materializePublishedMdx = async ({});const roots: PublishedMdxRoot[] = [];const warnings: PublishedMdxWarning[] = [];+ let findAvailableVariables:+ | ReturnType<typeof createFindAvailableVariables>+ | undefined;const warnUnavailableSource = ({blockInstanceId,assetId,+ reason,message,}: {blockInstanceId: string;assetId?: string;+ reason?: InvalidMdxDiagnosticReason;message: string;}) => {warnings.push({@@ -317,6 +327,7 @@ export const materializePublishedMdx = async ({diagnostic: {code: "invalid-mdx",severity: "error",+ ...(reason === undefined ? {} : { reason }),blockInstanceId,...(assetId === undefined ? {} : { assetId }),renderScope: `route:${route}:block:${blockInstanceId}`,@@ -345,6 +356,7 @@ export const materializePublishedMdx = async ({if (resolvedAssetIds.length > contentEngineLimits.candidateDocuments) {warnUnavailableSource({blockInstanceId: block.id,+ reason: "dynamic-source-unbounded",message: `Published Content Block "${block.id}" exceeds the safe MDX candidate limit`,});continue;@@ -352,6 +364,7 @@ export const materializePublishedMdx = async ({if (resolvedAssetIds.length === 0) {warnUnavailableSource({blockInstanceId: block.id,+ reason: "dynamic-source-unbounded",message: `Published Content Block "${block.id}" has no bounded dynamic MDX dependency set`,});continue;@@ -369,6 +382,7 @@ export const materializePublishedMdx = async ({warnUnavailableSource({blockInstanceId: block.id,assetId,+ reason: "missing-asset",message: `Published Content Block "${block.id}" requires unavailable MDX Asset "${assetId}"`,});}@@ -380,15 +394,38 @@ export const materializePublishedMdx = async ({warnUnavailableSource({blockInstanceId: block.id,assetId: candidate._id,+ reason: "missing-source",message: `Published MDX Asset "${candidate._id}" has no revision identity`,});continue;}- const sourceText = artifact.contents?.[candidate.contentRef];+ let sourceText = artifact.contents?.[candidate.contentRef];+ if (sourceText === undefined && loadDocumentSource !== undefined) {+ try {+ sourceText = await loadDocumentSource({+ id: candidate._id,+ revision: candidate.revision,+ contentRef: candidate.contentRef,+ });+ } catch (error) {+ warnUnavailableSource({+ blockInstanceId: block.id,+ assetId: candidate._id,+ reason: "source-read-failed",+ message: `Published MDX Asset "${+ candidate._id+ }" could not be loaded: ${+ error instanceof Error ? error.message : String(error)+ }`,+ });+ continue;+ }+ }if (sourceText === undefined) {warnUnavailableSource({blockInstanceId: block.id,assetId: candidate._id,+ reason: "missing-source",message: `Published MDX Asset "${candidate._id}" content is unavailable`,});continue;@@ -437,6 +474,17 @@ export const materializePublishedMdx = async ({metas,});assertMdxTemplateStructure(resolution);+ if (+ findAvailableVariables === undefined &&+ resolution.references.some(+ (reference) => reference.type === "resolved-template"+ )+ ) {+ findAvailableVariables = createFindAvailableVariables({+ instances: data.instances,+ dataSources: data.dataSources,+ });+ }const templates = await materializeMdxTemplates({identity,resolution,@@ -444,6 +492,7 @@ export const materializePublishedMdx = async ({metas,projectId,assetReferences,+ findAvailableVariables,});const referencedTemplateNames = Array.from(new Set(@@ -476,6 +525,7 @@ export const materializePublishedMdx = async ({warnUnavailableSource({blockInstanceId: block.id,assetId: candidate._id,+ reason: "linked-document-unavailable",message: `${linkedDocument} ${failure}${reference} while resolving frontmatter for published MDX Asset "${candidate._id}".`,});} else {diff --git a/packages/protocol/src/schema.ts b/packages/protocol/src/schema.tsindex fa8fc0befa..88d2547c70 100644--- a/packages/protocol/src/schema.ts+++ b/packages/protocol/src/schema.ts@@ -16,7 +16,7 @@ import { createContractVersion } from "./contract-version";// are imported through schema-only entrypoints to avoid duplicating model// definitions or pulling non-schema runtime code into protocol consumers.-export const maxProjectBundleSize = 20 * 1024 * 1024;+export const maxProjectBundleSize = 32 * 1024 * 1024;export const stagedUploadPath = "/rest/staged-upload";export const stagedUploadProjectIdHeader = "x-webstudio-project-id";export const projectSessionRestorePointPath = "build.restorePoint";diff --git a/packages/sdk/src/schema/content-block.ts b/packages/sdk/src/schema/content-block.tsindex 93077ce4fe..c6ca2eae41 100644--- a/packages/sdk/src/schema/content-block.ts+++ b/packages/sdk/src/schema/content-block.ts@@ -58,12 +58,34 @@ const diagnosticContext = {sourceRange: contentBlockSourceRange.optional(),};+export const invalidMdxDiagnosticReason = z.enum([+ // The MDX source was identified, but reading it from asset storage failed.+ "source-read-failed",+ // The MDX document exists, but its revision, content reference, or source text is unavailable.+ "missing-source",+ // No MDX document in the publication artifact matches the requested asset ID.+ "missing-asset",+ // A document referenced from MDX frontmatter could not be loaded or resolved.+ "linked-document-unavailable",+ // The rendered MDX violates HTML nesting rules, such as a link inside a link.+ "html-content-model",+ // Dynamic MDX contains Resources that cannot be selected safely for a Collection item or action.+ "dynamic-resource",+ // A dynamic source has no known candidates or exceeds the supported candidate limit.+ "dynamic-source-unbounded",+]);++export type InvalidMdxDiagnosticReason = z.infer<+ typeof invalidMdxDiagnosticReason+>;+export const contentBlockDiagnostic = z.discriminatedUnion("code", [z.strictObject({...diagnosticContext,code: z.literal("invalid-mdx"),severity: z.literal("error"),message: z.string().min(1),+ reason: invalidMdxDiagnosticReason.optional(),}),z.strictObject({...diagnosticContext,