Skip to content

Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.

19.0 KB
Raw Blame
// Declares high-impact agent scenarios, their allowed inputs, and structured
// observable outcomes without embedding the expected solution in prompts.
import {
  blockComponent,
  blockBodyComponent,
  blockTemplateComponent,
  type Asset,
  type AssetFolder,
  type Page,
} from "@webstudio-is/sdk";
import {
  fontAssetFixtureFiles,
  fontAssetFixtureUploadMeta,
} from "./font-assets-fixture";
import {
  markdownBlogFixtureArticles,
  markdownBlogFixtureDocuments,
} from "./markdown-blog-fixture";

export type EvaluationReasoningEffort = "low" | "medium" | "high";

type EvaluationAgentInputs = {
  assets: {
    directory: string;
    files: Array<{
      name: string;
      type: "file" | "font";
      format: string;
      meta: Record<string, unknown>;
    }>;
  };
};

type EvaluationAgentConfig = {
  reasoningEffort: EvaluationReasoningEffort;
  guidance: {
    workflow:
      | "general"
      | "authenticated-page"
      | "design-input"
      | "font-assets"
      | "markdown-blog"
      | "content-block-source";
  };
  inputs?: EvaluationAgentInputs;
};

export type EvaluationPage = {
  id: string;
  name: string;
  path: string;
  rootInstanceId: string;
  title?: Page["title"];
  meta?: Page["meta"];
};

export type EvaluationInstance = {
  id: string;
  component: string;
  tag?: string;
  label?: string;
  children: Array<
    | { type: "id"; value: string }
    | { type: "text"; value: string }
    | { type: "expression"; value: string; mode?: "read" | "readwrite" }
  >;
};

export type EvaluationProp = {
  id: string;
  instanceId: string;
  name: string;
  type: string;
  value: unknown;
};

export type EvaluationStyle = {
  styleSourceId: string;
  breakpointId: string;
  property: string;
  value: unknown;
};

export type EvaluationProject = {
  assets: Asset[];
  assetFolders: AssetFolder[];
  pages: EvaluationPage[];
  instances: EvaluationInstance[];
  props: EvaluationProp[];
  dataSources: Array<Record<string, unknown>>;
  resources: Array<Record<string, unknown>>;
  breakpoints: Array<{
    id: string;
    label: string;
    minWidth?: number;
    maxWidth?: number;
  }>;
  styleSources: Array<
    { type: "token"; id: string; name: string } | { type: "local"; id: string }
  >;
  styleSourceSelections: Array<{ instanceId: string; values: string[] }>;
  styles: EvaluationStyle[];
};

export type HighImpactFixture = {
  id:
    | "authenticated-page-v1"
    | "managed-form-authoring-v1"
    | "managed-form-runtime-v1"
    | "form-modes-authoring-v1"
    | "design-input-v1"
    | "font-assets-v1"
    | "markdown-blog-v1"
    | "markdown-references-discovery-v1"
    | "mdx-article-editing-v1";
  objective: string;
  project: EvaluationProject;
  agent: EvaluationAgentConfig;
  assetSources?: Readonly<Record<string, string>>;
};

const homePage: EvaluationPage = {
  id: "home",
  name: "Home",
  path: "",
  rootInstanceId: "home-root",
  title: "Home",
  meta: {},
};

const emptyProject = (): EvaluationProject => ({
  assets: [],
  assetFolders: [],
  pages: [homePage],
  instances: [
    {
      id: "home-root",
      component: "Body",
      tag: "body",
      children: [{ type: "id", value: "home-main" }],
    },
    {
      id: "home-main",
      component: "Box",
      tag: "main",
      children: [{ type: "text", value: "Northstar workspace" }],
    },
  ],
  props: [],
  dataSources: [],
  resources: [],
  breakpoints: [
    { id: "base", label: "Base" },
    { id: "tablet", label: "Tablet", maxWidth: 991 },
    { id: "mobile", label: "Mobile portrait", maxWidth: 479 },
  ],
  styleSources: [],
  styleSourceSelections: [],
  styles: [],
});

const markdownBlogAgentInputs: EvaluationAgentInputs = {
  assets: {
    directory: ".webstudio/assets",
    files: markdownBlogFixtureDocuments.map(({ name, format }) => ({
      name,
      type: "file",
      format,
      meta: {},
    })),
  },
};

export const authenticatedPageFixture: HighImpactFixture = {
  id: "authenticated-page-v1",
  objective:
    "Add an editable /account page using the project's existing authentication convention. Cover signed-out, loading, signed-in, and failed-auth UI states, then audit and visually verify the states. Do not persist credentials or private session data.",
  agent: {
    reasoningEffort: "low",
    guidance: { workflow: "authenticated-page" },
  },
  project: {
    ...emptyProject(),
    resources: [
      {
        id: "supabase-session",
        name: "Supabase session via server",
        method: "get",
        url: '"/api/auth/session"',
        headers: [],
        searchParams: [],
      },
    ],
    dataSources: [
      {
        id: "supabase-session-data",
        type: "resource",
        name: "Supabase session",
        scopeInstanceId: "home-root",
        resourceId: "supabase-session",
      },
      {
        id: "auth-preview-state",
        type: "variable",
        name: "Auth preview state",
        scopeInstanceId: "home-root",
        value: { type: "string", value: "signed-out" },
      },
    ],
  },
};

export const managedFormAuthoringFixture: HighImpactFixture = {
  id: "managed-form-authoring-v1",
  objective:
    'Create a polished /contact page for Northstar workspace with an accessible contact form. First search the component registry for Form (Beta), inspect the returned template, then insert it with insert-component using the form component. Do not recreate the Form manually or use insert-fragment as a substitute. If registry discovery fails, report that instead of building a substitute. Keep the template\'s named required name, email, subject, and message fields, submit button, success and error feedback, and default Resource actions connected. Add an h1 reading "Contact Northstar" and a short introductory sentence. Change the submit button label to "Send message", set the success message to "Thanks, we received your message. Our team will reply within one business day.", and set the error message to "We couldn\'t send your message. Please try again." Keep the existing form actions unchanged. Do not submit the form or trigger any Resource action. Audit the result and verify the page at desktop and mobile sizes.',
  agent: {
    reasoningEffort: "low",
    guidance: { workflow: "general" },
  },
  project: emptyProject(),
};

export const managedFormRuntimeFixture: HighImpactFixture = {
  id: "managed-form-runtime-v1",
  objective:
    "Create /contact-runtime with the registered Form (Beta) template. Replace every default Email action before any submission: create one POST HTTP Resource at the fixed test destination https://mcp-delivery.example/submit with scopeInstanceId set to the inserted Form and exposeAsDataSource true. Select its returned dataSourceId (not resourceId) as the Form action. Keep named name, email, subject, and message fields and visible success/error feedback. Once the Action update succeeds, call forms.simulate-submission immediately; do not keep inspecting the Form. Supply the saved Form instance ID, path /contact-runtime, and synthetic name, email, subject, and message values. Check success, change only the Resource URL to https://mcp-delivery.example/reject, simulate again and check error, then restore /submit and simulate a third time to check recovery. Use only these two test destinations; do not start Preview, publish, send Email, open a browser tab, attach files, or use real personal data.",
  agent: { reasoningEffort: "low", guidance: { workflow: "general" } },
  project: emptyProject(),
};

export const formModesAuthoringFixture: HighImpactFixture = {
  id: "form-modes-authoring-v1",
  objective:
    'Create /form-modes to demonstrate the three Form behaviors. Use focused component-registry searches for "Form (Beta)" and "Element", and inspect the returned metadata/templates to distinguish them. Insert a Form (Beta) labeled "Contact" and retain its default Resource actions. Insert a second Form (Beta) labeled "Search", remove all its Action selections, and use a named search input so submitting it performs client-side query navigation for page search/filter bindings. Insert an Element with tag form labeled "Native" for native browser submission. Explain which rendered action markers identify the Contact and Search forms and why the Element is native. Do not send emails or trigger Resource actions. Audit and verify /form-modes at desktop and mobile sizes.',
  agent: {
    reasoningEffort: "low",
    guidance: { workflow: "general" },
  },
  project: emptyProject(),
};

export const designInputFixture: HighImpactFixture & {
  designReference: {
    desktop: Record<string, unknown>;
    mobile: Record<string, unknown>;
  };
} = {
  id: "design-input-v1",
  objective:
    "Build an editable /summer page from the supplied desktop and mobile design reference. Preserve and reuse the existing design system, implement responsive behavior with the project's breakpoints, inspect desktop/mobile screenshots, then run a static route audit without duplicating the rendered captures.",
  agent: {
    reasoningEffort: "low",
    guidance: { workflow: "design-input" },
  },
  project: {
    ...emptyProject(),
    styleSources: [
      { type: "token", id: "token-brand", name: "Brand / Coral" },
      { type: "token", id: "token-ink", name: "Text / Ink" },
      { type: "token", id: "token-heading", name: "Type / Heading" },
      { type: "local", id: "home-main-local" },
    ],
    styleSourceSelections: [
      {
        instanceId: "home-main",
        values: ["token-ink", "token-heading", "home-main-local"],
      },
    ],
    styles: [
      {
        styleSourceId: "token-brand",
        breakpointId: "base",
        property: "background-color",
        value: { type: "rgb", r: 238, g: 101, b: 82, alpha: 1 },
      },
      {
        styleSourceId: "token-ink",
        breakpointId: "base",
        property: "color",
        value: { type: "rgb", r: 30, g: 35, b: 41, alpha: 1 },
      },
      {
        styleSourceId: "token-heading",
        breakpointId: "base",
        property: "font-family",
        value: { type: "fontFamily", value: ["Manrope", "sans-serif"] },
      },
      {
        styleSourceId: "home-main-local",
        breakpointId: "base",
        property: "max-width",
        value: { type: "unit", unit: "px", value: 1200 },
      },
    ],
  },
  designReference: {
    desktop: {
      viewport: { width: 1440, height: 900 },
      structure: ["header", "main", "hero", "featured trips", "footer"],
      hero: {
        heading: "Find your latitude",
        alignment: "left",
        columns: 2,
      },
      featuredTrips: { columns: 3 },
    },
    mobile: {
      viewport: { width: 390, height: 844 },
      structure: ["header", "main", "hero", "featured trips", "footer"],
      hero: { alignment: "left", columns: 1 },
      featuredTrips: { columns: 1 },
      navigation: "collapsed",
    },
  },
};

export const fontAssetsFixture: HighImpactFixture = {
  id: "font-assets-v1",
  objective: `Upload ${fontAssetFixtureFiles.map(({ name }) => name).join(" and ")} from .webstudio/assets as font assets, initially using family Imported Rajdhani, style normal, and weight 400. Then use update-asset exactly once on each uploaded asset to set values.meta to family Rajdhani, style normal, and weight 600. Verify both returned asset ids with verify-font-assets, run the audit, and do not change the page.`,
  agent: {
    reasoningEffort: "low",
    guidance: { workflow: "font-assets" },
    inputs: {
      assets: {
        directory: ".webstudio/assets",
        files: fontAssetFixtureFiles.map(({ name, format }) => ({
          name,
          type: "font",
          format,
          meta: fontAssetFixtureUploadMeta,
        })),
      },
    },
  },
  project: emptyProject(),
};

export const markdownBlogFixture: HighImpactFixture = {
  id: "markdown-blog-v1",
  objective: `Upload the ${markdownBlogFixtureArticles.length} provided Markdown articles from .webstudio/assets into one Blog asset folder. Build an editable, size-optimal blog overview at /blog and a dynamic detail page at /blog/:slug using exactly one fully configured scoped Assets resource per page, one overview Collection, and a directly bound detail Markdown Embed. Include the complete structured query in each initial resource creation; never create a default, placeholder, duplicate, preview copy, or repair replacement. Both queries must read the Markdown files directly. The overview query must be fully static and bounded, exclude drafts, sort newest first with a deterministic ID tie-breaker, select only the rendered title, slug, excerpt, publication date, and author frontmatter, and load no bodies so it can be materialized. The detail query must use only the dynamic slug, select only rendered metadata, and return one Markdown body without embedding it in the published database. The compiled database must include all articles without truncation, no embedded bodies, and only the intended materialized overview. Render the author name on both pages. Verify both /blog and /blog/aurora-trails at desktop and mobile sizes.`,
  agent: {
    reasoningEffort: "low",
    guidance: { workflow: "markdown-blog" },
    inputs: markdownBlogAgentInputs,
  },
  project: emptyProject(),
};

export const markdownReferencesDiscoveryFixture: HighImpactFixture = {
  id: "markdown-references-discovery-v1",
  objective: `Upload the supplied ${markdownBlogFixtureArticles.map(({ name }) => name).join(", ")} Markdown articles from .webstudio/assets. Build an editable blog overview at /blog and one dynamic article page at /blog/:slug. Query the Markdown files directly so draft articles are excluded, posts are ordered newest first, each post displays its frontmatter author, and only the selected Markdown body is fetched from Asset storage without being embedded in the published database. Verify the overview and /blog/aurora-trails at desktop and mobile sizes. Discover the supported workflow and data shapes from Webstudio MCP guidance.`,
  agent: {
    reasoningEffort: "low",
    guidance: { workflow: "markdown-blog" },
    inputs: markdownBlogAgentInputs,
  },
  project: emptyProject(),
};

export const mdxArticleSource = `---
title: Aurora trails
author:
  name: Mira Chen
readingTime: 6
draft: false
---

## Plan your route

Follow the marked trail and carry a map.
`;

export const mdxArticleFixture: HighImpactFixture = {
  id: "mdx-article-editing-v1",
  objective:
    "Use the content-block-source MCP workflow to connect the existing article.mdx Asset (article-file) to the Content Block article-block on Home. Inspect the connected source and use its actual document parameter. Before editing frontmatter, call update-text once for each of article-title, article-author, and article-reading-time with childIndex 0, mode expression, and expressionBindingMode readwrite. Bind them respectively to the direct paths document.frontmatter.title, document.frontmatter.author.name, and document.frontmatter.readingTime. Keep these designed header instances outside the MDX body and leave the existing static ‘ min read’ suffix separate. Then change only author.name to Noor Silva in the complete frontmatter map, preserving the title, readingTime, draft value, and body. Reload the connected source, inspect the saved result, and audit the page. No visual verification is requested.",
  agent: {
    reasoningEffort: "low",
    guidance: { workflow: "content-block-source" },
  },
  assetSources: { "article-file": mdxArticleSource },
  project: {
    ...emptyProject(),
    assets: [
      {
        id: "article-file",
        projectId: "high-impact-evaluation-project",
        name: "article.mdx",
        filename: "article",
        type: "file",
        format: "mdx",
        size: new TextEncoder().encode(mdxArticleSource).byteLength,
        createdAt: "2026-01-01T00:00:00.000Z",
        meta: {},
      },
    ],
    instances: [
      {
        id: "home-root",
        component: "Body",
        tag: "body",
        children: [{ type: "id", value: "article-block" }],
      },
      {
        id: "article-block",
        component: blockComponent,
        children: [
          { type: "id", value: "article-templates" },
          { type: "id", value: "article-title" },
          { type: "id", value: "article-author" },
          { type: "id", value: "article-reading-time" },
          { type: "id", value: "article-reading-suffix" },
          { type: "id", value: "article-body" },
        ],
      },
      {
        id: "article-templates",
        component: blockTemplateComponent,
        children: [],
      },
      { id: "article-body", component: blockBodyComponent, children: [] },
      {
        id: "article-title",
        component: "Heading",
        tag: "h1",
        children: [{ type: "text", value: "Title" }],
      },
      {
        id: "article-author",
        component: "Text",
        tag: "span",
        children: [{ type: "text", value: "Author" }],
      },
      {
        id: "article-reading-time",
        component: "Text",
        tag: "span",
        children: [{ type: "text", value: "0" }],
      },
      {
        id: "article-reading-suffix",
        component: "Text",
        tag: "span",
        children: [{ type: "text", value: " min read" }],
      },
    ],
  },
};

export const highImpactFixtures = [
  authenticatedPageFixture,
  managedFormAuthoringFixture,
  managedFormRuntimeFixture,
  formModesAuthoringFixture,
  designInputFixture,
  fontAssetsFixture,
  markdownBlogFixture,
  markdownReferencesDiscoveryFixture,
  mdxArticleFixture,
] as const;

export const validateHighImpactFixture = (fixture: HighImpactFixture) => {
  const failures: string[] = [];
  const instanceIds = new Set(
    fixture.project.instances.map((instance) => instance.id)
  );
  if (
    fixture.project.pages.some(
      (page) => instanceIds.has(page.rootInstanceId) === false
    )
  ) {
    failures.push(
      "Every fixture page must reference an existing root instance."
    );
  }
  if (
    fixture.project.breakpoints.some(
      (breakpoint) => breakpoint.id === "base"
    ) === false
  ) {
    failures.push("Every fixture must define a base breakpoint.");
  }
  if (fixture.id === "authenticated-page-v1") {
    const source = JSON.stringify(fixture.project).toLowerCase();
    if (
      source.includes("supabase") === false ||
      source.includes("/api/auth/session") === false
    ) {
      failures.push(
        "The auth fixture must expose its existing server-mediated Supabase convention."
      );
    }
  }
  if (fixture.id === "design-input-v1") {
    const designFixture = fixture as typeof designInputFixture;
    const desktopWidth = Number(
      (designFixture.designReference.desktop.viewport as { width?: unknown })
        .width
    );
    const mobileWidth = Number(
      (designFixture.designReference.mobile.viewport as { width?: unknown })
        .width
    );
    if (desktopWidth < 1200 || mobileWidth > 479) {
      failures.push(
        "The design fixture must include familiar desktop and mobile references."
      );
    }
    if (
      fixture.project.styleSources.some((source) => source.type === "token") ===
      false
    ) {
      failures.push(
        "The design fixture must include an existing design-token baseline."
      );
    }
  }
  return { valid: failures.length === 0, failures };
};