Skip to content

Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.

fix: Improve MCP authoring and screenshot reliability (#5957)

Oleg Isonene80ca3b54c1f
diff --git a/packages/asset-uploader/src/asset-patch-core.ts b/packages/asset-uploader/src/asset-patch-core.ts
index 195bc75565..6d892ea4f9 100644
--- a/packages/asset-uploader/src/asset-patch-core.ts
+++ b/packages/asset-uploader/src/asset-patch-core.ts
@@ -9,6 +9,8 @@ import {
} from "./patch-utils";
import { swapAssetFileWithClient } from "./revision";
+const serializeAssetMeta = (meta: Asset["meta"]) => JSON.stringify(meta);
+
export const createAssetRows = (assets: Iterable<Asset>, projectId: string) =>
Array.from(assets, (asset) => ({
id: asset.id,
@@ -154,7 +156,8 @@ export const patchAssetsWithClient = async (
previous.name === asset.name &&
previous.filename === asset.filename &&
previous.description === asset.description &&
- previous.folderId === asset.folderId
+ previous.folderId === asset.folderId &&
+ serializeAssetMeta(previous.meta) === serializeAssetMeta(asset.meta)
);
if (deletedAssetIds.length !== 0) {
await deleteAssetsWithClient({ projectId, ids: deletedAssetIds }, client);
@@ -176,27 +179,46 @@ export const patchAssetsWithClient = async (
client
);
}
- const { filename, description, folderId } = asset;
- const updatedAsset = await client
- .from("Asset")
- .update({
- filename: filename ?? null,
- description: description ?? null,
- folderId: folderId ?? null,
- })
- .eq("id", asset.id)
- .eq("projectId", asset.projectId)
- .select("filename, description, folderId")
- .single();
- assertPostgrestSuccess(updatedAsset);
if (
- updatedAsset.data?.filename !== (filename ?? null) ||
- updatedAsset.data?.description !== (description ?? null) ||
- (updatedAsset.data?.folderId ?? null) !== (folderId ?? null)
+ previous.filename !== asset.filename ||
+ previous.description !== asset.description ||
+ previous.folderId !== asset.folderId
) {
- throw new Error(
- `Asset metadata update was not persisted for ${asset.id}`
- );
+ const { filename, description, folderId } = asset;
+ const updatedAsset = await client
+ .from("Asset")
+ .update({
+ filename: filename ?? null,
+ description: description ?? null,
+ folderId: folderId ?? null,
+ })
+ .eq("id", asset.id)
+ .eq("projectId", asset.projectId)
+ .select("filename, description, folderId")
+ .single();
+ assertPostgrestSuccess(updatedAsset);
+ if (
+ updatedAsset.data?.filename !== (filename ?? null) ||
+ updatedAsset.data?.description !== (description ?? null) ||
+ (updatedAsset.data?.folderId ?? null) !== (folderId ?? null)
+ ) {
+ throw new Error(`Asset update was not persisted for ${asset.id}`);
+ }
+ }
+ if (serializeAssetMeta(previous.meta) !== serializeAssetMeta(asset.meta)) {
+ const meta = serializeAssetMeta(asset.meta);
+ const updatedFile = await client
+ .from("File")
+ .update({ meta })
+ .eq("name", asset.name)
+ .select("meta")
+ .single();
+ assertPostgrestSuccess(updatedFile);
+ if (updatedFile.data?.meta !== meta) {
+ throw new Error(
+ `Asset metadata update was not persisted for ${asset.id}`
+ );
+ }
}
}
diff --git a/packages/asset-uploader/src/patch.test.ts b/packages/asset-uploader/src/patch.test.ts
index b082c9066c..b069c0bd1c 100644
--- a/packages/asset-uploader/src/patch.test.ts
+++ b/packages/asset-uploader/src/patch.test.ts
@@ -288,6 +288,62 @@ describe("patchAssets (msw)", () => {
expect(updatedDescription).toBe("New description");
});
+ test("persists asset metadata via patch", async () => {
+ const projectId = uid();
+ let localAssetRow = {
+ ...assetRow,
+ projectId,
+ file: {
+ ...assetRow.file,
+ name: "font.woff2",
+ format: "woff2",
+ meta: JSON.stringify({
+ family: "Rajdhani",
+ style: "normal",
+ weight: 400,
+ }),
+ },
+ };
+ let updatedMeta: unknown;
+
+ server.use(
+ ownershipHandler,
+ db.get("Asset", () => json([localAssetRow])),
+ db.patch("File", async ({ request }) => {
+ updatedMeta = ((await request.json()) as { meta: unknown }).meta;
+ localAssetRow = {
+ ...localAssetRow,
+ file: { ...localAssetRow.file, meta: String(updatedMeta) },
+ };
+ return json({ meta: updatedMeta });
+ })
+ );
+
+ await patchAssets(
+ { projectId },
+ [
+ {
+ op: "replace",
+ path: ["asset-1", "meta"],
+ value: { family: "Rajdhani", style: "normal", weight: 600 },
+ },
+ ],
+ createContext()
+ );
+
+ expect(updatedMeta).toBe(
+ JSON.stringify({ family: "Rajdhani", style: "normal", weight: 600 })
+ );
+ await expect(
+ loadAssetsByProjectWithClient(projectId, testContext.postgrest.client)
+ ).resolves.toEqual([
+ expect.objectContaining({
+ id: "asset-1",
+ meta: { family: "Rajdhani", style: "normal", weight: 600 },
+ }),
+ ]);
+ });
+
test("updates asset filename via patch", async () => {
const projectId = uid();
let localAssetRow = { ...assetRow, projectId };
diff --git a/packages/cli/evaluations/high-impact/agent-runner.test.ts b/packages/cli/evaluations/high-impact/agent-runner.test.ts
index 301abd8488..ac03b6fb2e 100644
--- a/packages/cli/evaluations/high-impact/agent-runner.test.ts
+++ b/packages/cli/evaluations/high-impact/agent-runner.test.ts
@@ -27,6 +27,7 @@ describe("high-impact agent runner", () => {
expect.any(String),
expect.stringContaining("meta.guide"),
expect.any(String),
+ expect.stringContaining("Never use broad project reads"),
expect.any(String),
expect.stringContaining("meta.next"),
],
@@ -49,32 +50,62 @@ describe("high-impact agent runner", () => {
try {
const resultPath = join(directory, "result.json");
const taskPath = join(directory, "task.json");
+ const usageEvent = JSON.stringify({
+ type: "turn.completed",
+ usage: {
+ input_tokens: 1_000,
+ cached_input_tokens: 400,
+ cache_write_input_tokens: 50,
+ output_tokens: 200,
+ reasoning_output_tokens: 75,
+ },
+ });
const result = await runHighImpactAgentEvaluation({
fixture: authenticatedPageFixture,
target: { kind: "packaged", executable: "/tmp/webstudio" },
- agentCommand: "node -e 'process.exit(0)'",
+ agentCommand: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(
+ `console.log(${JSON.stringify(usageEvent)})`
+ )}`,
cwd: directory,
taskPath,
resultPath,
provider: "test-provider",
model: "test-model",
- getCallSequence: () => ["meta.guide", "audit"],
+ getToolCalls: () => [
+ { name: "meta.guide", startedAtMs: 100, durationMs: 25 },
+ { name: "audit", startedAtMs: 500, durationMs: 75 },
+ ],
evaluate: async () => ({
passed: true,
checks: { privacy: "passed", audit: "passed" },
failures: [],
- metrics: { toolCallCount: 6, focusedReadCount: 2 },
}),
});
expect(result).toMatchObject({
+ schemaVersion: 2,
outcome: "passed",
cli: "packaged",
- toolCallCount: 6,
- focusedReadCount: 2,
+ metrics: {
+ durationMs: expect.any(Number),
+ tokens: {
+ input: 1_000,
+ cachedInput: 400,
+ output: 200,
+ total: 1_200,
+ },
+ toolCalls: {
+ total: 2,
+ failed: 0,
+ verifications: 1,
+ timeToFirstVerificationMs: 500,
+ },
+ },
callSequence: ["meta.guide", "audit"],
+ checks: { usageCaptured: "passed" },
});
const source = await readFile(resultPath, "utf8");
expect(source).not.toMatch(/transcript|stdout|stderr|credential/i);
+ expect(source).not.toContain(usageEvent);
expect(JSON.parse(await readFile(taskPath, "utf8"))).toMatchObject({
fixtureId: "authenticated-page-v1",
mcp: { args: ["mcp"] },
diff --git a/packages/cli/evaluations/high-impact/agent-runner.ts b/packages/cli/evaluations/high-impact/agent-runner.ts
index ebd6543eef..db3f568107 100644
--- a/packages/cli/evaluations/high-impact/agent-runner.ts
+++ b/packages/cli/evaluations/high-impact/agent-runner.ts
@@ -2,7 +2,17 @@ import { createHash } from "node:crypto";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import type { HighImpactFixture } from "./fixtures";
-import type { HighImpactEvaluationResult } from "./validate";
+import type {
+ EvaluationToolCall,
+ HighImpactEvaluationResult,
+} from "./validate";
+import {
+ addAgentUsage,
+ getAgentUsageEvent,
+ getMcpEvaluationMetrics,
+ type AgentUsage,
+ type McpEvaluationMetrics,
+} from "./evaluation-metrics";
import { runAgentCommand } from "../../scripts/run-agent-command";
import { boundedIdentifierPattern } from "../../src/type-utils";
@@ -11,7 +21,7 @@ export type AgentCliTarget =
| { kind: "packaged"; executable: string };
export type AgentEvaluationResult = {
- schemaVersion: 1;
+ schemaVersion: 2;
kind: "high-impact-minimal-context-agent-evaluation-result";
fixtureId: HighImpactFixture["id"];
outcome: "passed" | "failed";
@@ -19,16 +29,19 @@ export type AgentEvaluationResult = {
provider: string;
model: string;
commandSha256: string;
- durationMs: number;
exitCode: number;
- toolCallCount: number;
- focusedReadCount: number;
+ metrics: {
+ durationMs: number;
+ tokens?: AgentUsage;
+ toolCalls: McpEvaluationMetrics;
+ };
callSequence: string[];
checks: Record<string, "passed" | "failed">;
};
const forbiddenResultKeys =
- /(?:prompt|transcript|stdout|stderr|token|secret|credential|payload)/i;
+ /(?:prompt|transcript|stdout|stderr|secret|credential|payload)/i;
+const forbiddenTokenKey = /token/i;
export const getCliInvocation = (target: AgentCliTarget) =>
target.kind === "source"
@@ -57,6 +70,7 @@ export const createMinimalAgentTask = (
"Use the configured Webstudio project and local CLI.",
"Begin with meta.guide for the objective and follow its workflow.",
"Choose focused reads and semantic edits yourself.",
+ "Never use broad project reads: snapshot, components.list, or components.coverage-plan.",
"Do not persist or report credentials or private session data.",
"Treat mutation meta.next steps as required. Do not report completion until audit and requested visual evidence pass.",
],
@@ -70,15 +84,20 @@ const assertBoundedResult = (result: AgentEvaluationResult) => {
) {
throw new Error("Agent provider and model must be bounded identifiers.");
}
- const visit = (value: unknown): void => {
+ const visit = (value: unknown, path: string[] = []): void => {
if (typeof value !== "object" || value === null) {
return;
}
for (const [key, child] of Object.entries(value)) {
- if (forbiddenResultKeys.test(key)) {
+ const isAggregateUsage =
+ path.length === 1 && path[0] === "metrics" && key === "tokens";
+ if (
+ forbiddenResultKeys.test(key) ||
+ (forbiddenTokenKey.test(key) && isAggregateUsage === false)
+ ) {
throw new Error(`Agent result contains forbidden field ${key}.`);
}
- visit(child);
+ visit(child, [...path, key]);
}
};
visit(result);
@@ -96,7 +115,7 @@ export const runHighImpactAgentEvaluation = async ({
resultPath,
provider,
model,
- getCallSequence,
+ getToolCalls,
evaluate,
env = process.env,
timeoutMs = 10 * 60_000,
@@ -109,7 +128,7 @@ export const runHighImpactAgentEvaluation = async ({
resultPath: string;
provider: string;
model: string;
- getCallSequence: () => string[];
+ getToolCalls: () => EvaluationToolCall[];
evaluate: () => Promise<HighImpactEvaluationResult>;
env?: NodeJS.ProcessEnv;
timeoutMs?: number;
@@ -120,29 +139,52 @@ export const runHighImpactAgentEvaluation = async ({
JSON.stringify(createMinimalAgentTask(fixture, target), undefined, 2),
"utf8"
);
+ let usage: AgentUsage | undefined;
const execution = await runAgentCommand({
command: agentCommand,
cwd,
env: { ...env, WEBSTUDIO_HIGH_IMPACT_AGENT_TASK: taskPath },
timeoutMs,
+ onStdoutLine: (line) => {
+ let value: unknown;
+ try {
+ value = JSON.parse(line);
+ } catch {
+ return;
+ }
+ const eventUsage = getAgentUsageEvent(value);
+ if (eventUsage !== undefined) {
+ usage = addAgentUsage(usage, eventUsage);
+ }
+ },
});
const evaluation = await evaluate();
+ const toolCalls = getToolCalls();
+ const checks = {
+ ...evaluation.checks,
+ usageCaptured:
+ usage === undefined ? ("failed" as const) : ("passed" as const),
+ };
const result: AgentEvaluationResult = {
- schemaVersion: 1,
+ schemaVersion: 2,
kind: "high-impact-minimal-context-agent-evaluation-result",
fixtureId: fixture.id,
outcome:
- execution.exitCode === 0 && evaluation.passed ? "passed" : "failed",
+ execution.exitCode === 0 && evaluation.passed && usage !== undefined
+ ? "passed"
+ : "failed",
cli: target.kind,
provider,
model,
commandSha256: createHash("sha256").update(agentCommand).digest("hex"),
- durationMs: execution.durationMs,
exitCode: execution.exitCode,
- toolCallCount: evaluation.metrics.toolCallCount,
- focusedReadCount: evaluation.metrics.focusedReadCount,
- callSequence: getCallSequence(),
- checks: evaluation.checks,
+ metrics: {
+ durationMs: execution.durationMs,
+ ...(usage === undefined ? {} : { tokens: usage }),
+ toolCalls: getMcpEvaluationMetrics(toolCalls),
+ },
+ callSequence: toolCalls.map(({ name }) => name),
+ checks,
};
assertBoundedResult(result);
await mkdir(dirname(resultPath), { recursive: true });
diff --git a/packages/cli/evaluations/high-impact/evaluation-metrics.test.ts b/packages/cli/evaluations/high-impact/evaluation-metrics.test.ts
new file mode 100644
index 0000000000..a19d3b14b9
--- /dev/null
+++ b/packages/cli/evaluations/high-impact/evaluation-metrics.test.ts
@@ -0,0 +1,91 @@
+import { describe, expect, test } from "vitest";
+import {
+ addAgentUsage,
+ getAgentUsageEvent,
+ getMcpEvaluationMetrics,
+} from "./evaluation-metrics";
+
+describe("evaluation metrics", () => {
+ test("retains only aggregate usage from Codex completion events", () => {
+ const usage = getAgentUsageEvent({
+ type: "turn.completed",
+ usage: {
+ input_tokens: 1_000,
+ cached_input_tokens: 400,
+ cache_write_input_tokens: 50,
+ output_tokens: 200,
+ reasoning_output_tokens: 75,
+ private_message: "discard me",
+ },
+ });
+ expect(usage).toEqual({
+ input: 1_000,
+ cachedInput: 400,
+ cacheWriteInput: 50,
+ output: 200,
+ reasoningOutput: 75,
+ total: 1_200,
+ cachedInputRate: 0.4,
+ });
+ if (usage === undefined) {
+ throw new Error("Expected aggregate usage");
+ }
+ expect(addAgentUsage(usage, usage)).toEqual({
+ input: 2_000,
+ cachedInput: 800,
+ cacheWriteInput: 100,
+ output: 400,
+ reasoningOutput: 150,
+ total: 2_400,
+ cachedInputRate: 0.4,
+ });
+ expect(getAgentUsageEvent({ type: "item.completed", usage: {} })).toBe(
+ undefined
+ );
+ });
+
+ test("measures MCP errors, retries, mutations, verification, and latency", () => {
+ expect(
+ getMcpEvaluationMetrics([
+ { name: "meta.guide", startedAtMs: 10, durationMs: 10 },
+ { name: "snapshot", startedAtMs: 20, durationMs: 20 },
+ {
+ name: "create-page",
+ startedAtMs: 50,
+ durationMs: 30,
+ isError: true,
+ },
+ {
+ name: "create-page",
+ startedAtMs: 90,
+ durationMs: 40,
+ committed: true,
+ },
+ {
+ name: "update-page",
+ arguments: { dryRun: true },
+ startedAtMs: 110,
+ durationMs: 50,
+ planned: true,
+ },
+ { name: "audit", startedAtMs: 170, durationMs: 100 },
+ ])
+ ).toEqual({
+ total: 6,
+ succeeded: 5,
+ failed: 1,
+ retries: 1,
+ focusedReads: 0,
+ broadReads: 1,
+ dryRuns: 1,
+ plannedMutations: 1,
+ committedMutations: 1,
+ verifications: 1,
+ totalDurationMs: 250,
+ p50DurationMs: 30,
+ p95DurationMs: 100,
+ timeToFirstMutationMs: 90,
+ timeToFirstVerificationMs: 170,
+ });
+ });
+});
diff --git a/packages/cli/evaluations/high-impact/evaluation-metrics.ts b/packages/cli/evaluations/high-impact/evaluation-metrics.ts
new file mode 100644
index 0000000000..91c682b2f2
--- /dev/null
+++ b/packages/cli/evaluations/high-impact/evaluation-metrics.ts
@@ -0,0 +1,183 @@
+import { isPlainRecord } from "../../src/type-utils";
+import type { EvaluationToolCall } from "./validate";
+
+export type AgentUsage = {
+ input: number;
+ cachedInput: number;
+ cacheWriteInput: number;
+ output: number;
+ reasoningOutput: number;
+ total: number;
+ cachedInputRate: number;
+};
+
+const getNonNegativeInteger = (value: unknown) =>
+ typeof value === "number" && Number.isSafeInteger(value) && value >= 0
+ ? value
+ : undefined;
+
+const createAgentUsage = ({
+ input,
+ cachedInput,
+ cacheWriteInput,
+ output,
+ reasoningOutput,
+}: Omit<AgentUsage, "total" | "cachedInputRate">): AgentUsage => ({
+ input,
+ cachedInput,
+ cacheWriteInput,
+ output,
+ reasoningOutput,
+ total: input + output,
+ cachedInputRate:
+ input === 0 ? 0 : Math.round((cachedInput / input) * 10_000) / 10_000,
+});
+
+export const getAgentUsageEvent = (value: unknown): AgentUsage | undefined => {
+ if (
+ isPlainRecord(value) === false ||
+ value.type !== "turn.completed" ||
+ isPlainRecord(value.usage) === false
+ ) {
+ return;
+ }
+ const input = getNonNegativeInteger(value.usage.input_tokens);
+ const cachedInput = getNonNegativeInteger(value.usage.cached_input_tokens);
+ const cacheWriteInput = getNonNegativeInteger(
+ value.usage.cache_write_input_tokens
+ );
+ const output = getNonNegativeInteger(value.usage.output_tokens);
+ const reasoningOutput = getNonNegativeInteger(
+ value.usage.reasoning_output_tokens
+ );
+ if (
+ input === undefined ||
+ cachedInput === undefined ||
+ cacheWriteInput === undefined ||
+ output === undefined ||
+ reasoningOutput === undefined
+ ) {
+ return;
+ }
+ return createAgentUsage({
+ input,
+ cachedInput,
+ cacheWriteInput,
+ output,
+ reasoningOutput,
+ });
+};
+
+export const addAgentUsage = (
+ left: AgentUsage | undefined,
+ right: AgentUsage
+) =>
+ createAgentUsage({
+ input: (left?.input ?? 0) + right.input,
+ cachedInput: (left?.cachedInput ?? 0) + right.cachedInput,
+ cacheWriteInput: (left?.cacheWriteInput ?? 0) + right.cacheWriteInput,
+ output: (left?.output ?? 0) + right.output,
+ reasoningOutput: (left?.reasoningOutput ?? 0) + right.reasoningOutput,
+ });
+
+export type McpEvaluationMetrics = {
+ total: number;
+ succeeded: number;
+ failed: number;
+ retries: number;
+ focusedReads: number;
+ broadReads: number;
+ dryRuns: number;
+ plannedMutations: number;
+ committedMutations: number;
+ verifications: number;
+ totalDurationMs: number;
+ p50DurationMs: number;
+ p95DurationMs: number;
+ timeToFirstMutationMs?: number;
+ timeToFirstVerificationMs?: number;
+};
+
+const broadReadNames = new Set([
+ "snapshot",
+ "components.list",
+ "components.coverage-plan",
+]);
+
+const verificationNames = new Set([
+ "audit",
+ "screenshot",
+ "screenshot.diff",
+ "verify-bindings",
+]);
+
+export const isFocusedRead = (name: string) =>
+ /^(?:list-|get-|inspect-|search-)/.test(name);
+
+export const isBroadRead = (name: string) => broadReadNames.has(name);
+
+const percentile = (values: number[], proportion: number) => {
+ if (values.length === 0) {
+ return 0;
+ }
+ const sorted = values.toSorted((left, right) => left - right);
+ const index = Math.max(0, Math.ceil(sorted.length * proportion) - 1);
+ return sorted[index] ?? 0;
+};
+
+const countRetries = (calls: readonly EvaluationToolCall[]) => {
+ const failedNames = new Set<string>();
+ let retries = 0;
+ for (const call of calls) {
+ if (failedNames.has(call.name)) {
+ retries += 1;
+ }
+ if (call.isError === true) {
+ failedNames.add(call.name);
+ } else {
+ failedNames.delete(call.name);
+ }
+ }
+ return retries;
+};
+
+export const getMcpEvaluationMetrics = (
+ calls: readonly EvaluationToolCall[]
+): McpEvaluationMetrics => {
+ const durations = calls.flatMap(({ durationMs }) =>
+ durationMs === undefined ? [] : [durationMs]
+ );
+ const firstMutation = calls.find(
+ (call) => call.committed === true && call.isError !== true
+ );
+ const firstVerification = calls.find(
+ (call) => verificationNames.has(call.name) && call.isError !== true
+ );
+ return {
+ total: calls.length,
+ succeeded: calls.filter((call) => call.isError !== true).length,
+ failed: calls.filter((call) => call.isError === true).length,
+ retries: countRetries(calls),
+ focusedReads: calls.filter((call) => isFocusedRead(call.name)).length,
+ broadReads: calls.filter((call) => isBroadRead(call.name)).length,
+ dryRuns: calls.filter((call) => call.arguments?.dryRun === true).length,
+ plannedMutations: calls.filter(
+ (call) => call.planned === true && call.isError !== true
+ ).length,
+ committedMutations: calls.filter(
+ (call) => call.committed === true && call.isError !== true
+ ).length,
+ verifications: calls.filter(
+ (call) => verificationNames.has(call.name) && call.isError !== true
+ ).length,
+ totalDurationMs: durations.reduce((total, value) => total + value, 0),
+ p50DurationMs: percentile(durations, 0.5),
+ p95DurationMs: percentile(durations, 0.95),
+ ...(firstMutation?.startedAtMs === undefined
+ ? {}
+ : { timeToFirstMutationMs: firstMutation.startedAtMs }),
+ ...(firstVerification?.startedAtMs === undefined
+ ? {}
+ : { timeToFirstVerificationMs: firstVerification.startedAtMs }),
+ };
+};
diff --git a/packages/cli/evaluations/high-impact/evaluation-regression.test.ts b/packages/cli/evaluations/high-impact/evaluation-regression.test.ts
new file mode 100644
index 0000000000..9a97680faf
--- /dev/null
+++ b/packages/cli/evaluations/high-impact/evaluation-regression.test.ts
@@ -0,0 +1,208 @@
+import { describe, expect, test } from "vitest";
+import type { AgentEvaluationResult } from "./agent-runner";
+import {
+ compareEvaluationResult,
+ isEvaluationComparisonAccepted,
+} from "./evaluation-regression";
+
+const createResult = (
+ values: Partial<AgentEvaluationResult> = {}
+): AgentEvaluationResult => ({
+ schemaVersion: 2,
+ kind: "high-impact-minimal-context-agent-evaluation-result",
+ fixtureId: "authenticated-page-v1",
+ outcome: "passed",
+ cli: "source",
+ provider: "openai",
+ model: "test-model",
+ commandSha256: "a".repeat(64),
+ exitCode: 0,
+ metrics: {
+ durationMs: 1_000,
+ tokens: {
+ input: 800,
+ cachedInput: 400,
+ cacheWriteInput: 0,
+ output: 200,
+ reasoningOutput: 50,
+ total: 1_000,
+ cachedInputRate: 0.5,
+ },
+ toolCalls: {
+ total: 10,
+ succeeded: 10,
+ failed: 0,
+ retries: 0,
+ focusedReads: 2,
+ broadReads: 0,
+ dryRuns: 1,
+ plannedMutations: 1,
+ committedMutations: 1,
+ verifications: 2,
+ totalDurationMs: 500,
+ p50DurationMs: 30,
+ p95DurationMs: 100,
+ timeToFirstMutationMs: 400,
+ timeToFirstVerificationMs: 800,
+ },
+ },
+ callSequence: ["meta.guide", "create-page", "audit"],
+ checks: { guidance: "passed", audit: "passed", usageCaptured: "passed" },
+ ...values,
+});
+
+describe("evaluation regression comparison", () => {
+ test("reports every numeric delta and blocks configured regressions", () => {
+ const baseline = createResult();
+ const baselineTokens = baseline.metrics.tokens;
+ if (baselineTokens === undefined) {
+ throw new Error("Expected baseline token metrics");
+ }
+ const current = createResult({
+ metrics: {
+ ...baseline.metrics,
+ durationMs: 1_100,
+ tokens: { ...baselineTokens, total: 1_200 },
+ },
+ });
+ const comparison = compareEvaluationResult(current, baseline);
+ expect(comparison.status).toBe("regressed");
+ expect(comparison.deltas).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ metric: "metrics.durationMs",
+ baseline: 1_000,
+ current: 1_100,
+ }),
+ expect.objectContaining({
+ metric: "metrics.tokens.total",
+ baseline: 1_000,
+ current: 1_200,
+ }),
+ ])
+ );
+ expect(comparison.regressions).toEqual([
+ expect.objectContaining({
+ metric: "metrics.tokens.total",
+ allowed: 1_150,
+ }),
+ ]);
+ });
+
+ test("blocks correctness regressions and skips incompatible baselines", () => {
+ const baseline = createResult();
+ expect(
+ compareEvaluationResult(
+ createResult({ checks: { ...baseline.checks, audit: "failed" } }),
+ baseline
+ ).regressions
+ ).toEqual([
+ {
+ metric: "checks.audit",
+ baseline: "passed",
+ current: "failed",
+ },
+ ]);
+ expect(
+ compareEvaluationResult(
+ createResult({ model: "different-model" }),
+ baseline
+ )
+ ).toMatchObject({ status: "incompatible", deltas: [], regressions: [] });
+ expect(compareEvaluationResult(createResult(), undefined)).toMatchObject({
+ status: "missing",
+ deltas: [],
+ regressions: [],
+ });
+ });
+
+ test("does not round count tolerances beyond the configured percentage", () => {
+ const baseline = createResult();
+ baseline.metrics.toolCalls.focusedReads = 9;
+ const current = structuredClone(baseline);
+ current.metrics.toolCalls.focusedReads = 11;
+ expect(compareEvaluationResult(current, baseline).regressions).toEqual([
+ {
+ metric: "metrics.toolCalls.focusedReads",
+ baseline: 9,
+ current: 11,
+ allowed: 10,
+ },
+ ]);
+ });
+
+ test("ignores insignificant absolute latency jitter", () => {
+ const baseline = createResult();
+ baseline.metrics.toolCalls.totalDurationMs = 73;
+ const current = structuredClone(baseline);
+ current.metrics.toolCalls.totalDurationMs = 84;
+ expect(compareEvaluationResult(current, baseline)).toMatchObject({
+ status: "passed",
+ regressions: [],
+ });
+ });
+
+ test("reports stochastic milestone timing without blocking one run", () => {
+ const baseline = createResult();
+ const current = structuredClone(baseline);
+ current.metrics.durationMs = 20_000;
+ current.metrics.toolCalls.timeToFirstMutationMs = 20_000;
+ current.metrics.toolCalls.timeToFirstVerificationMs = 30_000;
+ const comparison = compareEvaluationResult(current, baseline);
+ expect(comparison).toMatchObject({ status: "passed", regressions: [] });
+ expect(comparison.deltas).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ metric: "metrics.toolCalls.timeToFirstMutationMs",
+ current: 20_000,
+ }),
+ expect.objectContaining({
+ metric: "metrics.toolCalls.timeToFirstVerificationMs",
+ current: 30_000,
+ }),
+ ])
+ );
+ });
+
+ test("requires accepted baselines and never accepts metric regressions", () => {
+ const passed = compareEvaluationResult(createResult(), createResult());
+ const missing = compareEvaluationResult(createResult(), undefined);
+ const regressionCurrent = createResult();
+ if (regressionCurrent.metrics.tokens === undefined) {
+ throw new Error("Expected token metrics");
+ }
+ regressionCurrent.metrics.tokens.total = 2_000;
+ const regressed = compareEvaluationResult(
+ regressionCurrent,
+ createResult()
+ );
+ expect(
+ isEvaluationComparisonAccepted({
+ comparison: passed,
+ requireBaseline: true,
+ updateBaselines: false,
+ })
+ ).toBe(true);
+ expect(
+ isEvaluationComparisonAccepted({
+ comparison: missing,
+ requireBaseline: true,
+ updateBaselines: false,
+ })
+ ).toBe(false);
+ expect(
+ isEvaluationComparisonAccepted({
+ comparison: missing,
+ requireBaseline: true,
+ updateBaselines: true,
+ })
+ ).toBe(true);
+ expect(
+ isEvaluationComparisonAccepted({
+ comparison: regressed,
+ requireBaseline: false,
+ updateBaselines: true,
+ })
+ ).toBe(false);
+ });
+});
diff --git a/packages/cli/evaluations/high-impact/evaluation-regression.ts b/packages/cli/evaluations/high-impact/evaluation-regression.ts
new file mode 100644
index 0000000000..d44102fdc7
--- /dev/null
+++ b/packages/cli/evaluations/high-impact/evaluation-regression.ts
@@ -0,0 +1,174 @@
+import type { AgentEvaluationResult } from "./agent-runner";
+
+export type EvaluationMetricDelta = {
+ metric: string;
+ baseline: number;
+ current: number;
+ delta: number;
+ deltaPercent?: number;
+};
+
+export type EvaluationRegression = {
+ metric: string;
+ baseline: number | "passed";
+ current: number | "failed";
+ allowed?: number;
+};
+
+export type EvaluationComparison = {
+ status: "passed" | "regressed" | "missing" | "incompatible";
+ baselineCommandSha256?: string;
+ maxMetricRegressionPercent: number;
+ deltas: EvaluationMetricDelta[];
+ regressions: EvaluationRegression[];
+};
+
+export const maxMetricRegressionPercent = 15;
+
+export const isEvaluationComparisonAccepted = ({
+ comparison,
+ requireBaseline,
+ updateBaselines,
+}: {
+ comparison: EvaluationComparison;
+ requireBaseline: boolean;
+ updateBaselines: boolean;
+}) => {
+ if (comparison.status === "passed") {
+ return true;
+ }
+ if (comparison.status === "regressed") {
+ return false;
+ }
+ return updateBaselines || requireBaseline === false;
+};
+
+const gatedMetricPaths = new Set([
+ "metrics.tokens.total",
+ "metrics.toolCalls.total",
+ "metrics.toolCalls.failed",
+ "metrics.toolCalls.retries",
+ "metrics.toolCalls.focusedReads",
+ "metrics.toolCalls.broadReads",
+ "metrics.toolCalls.totalDurationMs",
+ "metrics.toolCalls.p95DurationMs",
+]);
+
+const minimumAbsoluteIncrease = new Map([
+ ["metrics.toolCalls.totalDurationMs", 100],
+ ["metrics.toolCalls.p95DurationMs", 100],
+]);
+
+const getNumericMetrics = (
+ value: unknown,
+ path: string[] = [],
+ metrics = new Map<string, number>()
+) => {
+ if (typeof value === "number" && Number.isFinite(value)) {
+ metrics.set(path.join("."), value);
+ return metrics;
+ }
+ if (typeof value !== "object" || value === null) {
+ return metrics;
+ }
+ for (const [key, child] of Object.entries(value)) {
+ getNumericMetrics(child, [...path, key], metrics);
+ }
+ return metrics;
+};
+
+const isCompatibleBaseline = (
+ current: AgentEvaluationResult,
+ baseline: AgentEvaluationResult
+) =>
+ current.schemaVersion === baseline.schemaVersion &&
+ current.fixtureId === baseline.fixtureId &&
+ current.cli === baseline.cli &&
+ current.provider === baseline.provider &&
+ current.model === baseline.model;
+
+export const compareEvaluationResult = (
+ current: AgentEvaluationResult,
+ baseline: AgentEvaluationResult | undefined
+): EvaluationComparison => {
+ const empty = {
+ maxMetricRegressionPercent,
+ deltas: [],
+ regressions: [],
+ };
+ if (baseline === undefined) {
+ return { status: "missing", ...empty };
+ }
+ if (isCompatibleBaseline(current, baseline) === false) {
+ return { status: "incompatible", ...empty };
+ }
+ const baselineMetrics = getNumericMetrics(baseline.metrics, ["metrics"]);
+ const currentMetrics = getNumericMetrics(current.metrics, ["metrics"]);
+ const deltas: EvaluationMetricDelta[] = [];
+ const regressions: EvaluationRegression[] = [];
+ for (const [metric, baselineValue] of baselineMetrics) {
+ const currentValue = currentMetrics.get(metric);
+ if (currentValue === undefined) {
+ continue;
+ }
+ const rawDelta = currentValue - baselineValue;
+ const delta = Math.round(rawDelta * 10_000) / 10_000;
+ deltas.push({
+ metric,
+ baseline: baselineValue,
+ current: currentValue,
+ delta,
+ ...(baselineValue === 0
+ ? {}
+ : {
+ deltaPercent:
+ Math.round((rawDelta / baselineValue) * 100 * 100) / 100,
+ }),
+ });
+ if (gatedMetricPaths.has(metric) === false) {
+ continue;
+ }
+ const relativeIncrease = Math.floor(
+ baselineValue * (maxMetricRegressionPercent / 100) +
+ Number.EPSILON * Math.max(1, baselineValue)
+ );
+ const allowed =
+ baselineValue +
+ Math.max(relativeIncrease, minimumAbsoluteIncrease.get(metric) ?? 0);
+ if (currentValue > allowed) {
+ regressions.push({
+ metric,
+ baseline: baselineValue,
+ current: currentValue,
+ allowed,
+ });
+ }
+ }
+ for (const [check, baselineOutcome] of Object.entries(baseline.checks)) {
+ if (baselineOutcome === "passed" && current.checks[check] === "failed") {
+ regressions.push({
+ metric: `checks.${check}`,
+ baseline: "passed",
+ current: "failed",
+ });
+ }
+ }
+ if (
+ baseline.outcome === "passed" &&
+ current.outcome === "failed" &&
+ regressions.some(({ metric }) => metric.startsWith("checks.")) === false
+ ) {
+ regressions.push({
+ metric: "outcome",
+ baseline: "passed",
+ current: "failed",
+ });
+ }
+ return {
+ status: regressions.length === 0 ? "passed" : "regressed",
+ baselineCommandSha256: baseline.commandSha256,
+ maxMetricRegressionPercent,
+ deltas,
+ regressions,
+ };
+};
diff --git a/packages/cli/evaluations/high-impact/mcp-trace-proxy.test.ts b/packages/cli/evaluations/high-impact/mcp-trace-proxy.test.ts
index 1dcda13a9c..2e2a9623c2 100644
--- a/packages/cli/evaluations/high-impact/mcp-trace-proxy.test.ts
+++ b/packages/cli/evaluations/high-impact/mcp-trace-proxy.test.ts
@@ -1,46 +1,58 @@
import { describe, expect, test } from "vitest";
-import { getMcpTraceRequest, getMcpTraceResponse } from "./mcp-trace-proxy";
+import {
+ getMcpMutationToolNames,
+ getMcpToolsListRequestId,
+ getMcpTraceRequest,
+ getMcpTraceResponse,
+} from "./mcp-trace-proxy";
describe("bounded MCP tracing", () => {
test("retains only bounded verification fields", () => {
expect(
- getMcpTraceRequest({
- jsonrpc: "2.0",
- id: 7,
- method: "tools/call",
- params: {
- name: "screenshot",
- arguments: {
- viewport: { width: 390, height: 844 },
- authToken: "private",
- path: "/account",
+ getMcpTraceRequest(
+ {
+ jsonrpc: "2.0",
+ id: 7,
+ method: "tools/call",
+ params: {
+ name: "screenshot",
+ arguments: {
+ viewport: { width: 390, height: 844 },
+ authToken: "private",
+ path: "/account",
+ },
},
},
- })
+ 125
+ )
).toEqual({
id: 7,
call: {
name: "screenshot",
arguments: { viewport: { width: 390, height: 844 } },
+ startedAtMs: 125,
},
});
});
test("records confirmation flow without retaining its token", () => {
expect(
- getMcpTraceRequest({
- id: 8,
- method: "tools/call",
- params: {
- name: "delete-instance",
- arguments: {
- instanceId: "private-instance-id",
- dryRun: true,
- confirmDestructive: true,
- confirmationToken: "private-token",
+ getMcpTraceRequest(
+ {
+ id: 8,
+ method: "tools/call",
+ params: {
+ name: "delete-instance",
+ arguments: {
+ instanceId: "private-instance-id",
+ dryRun: true,
+ confirmDestructive: true,
+ confirmationToken: "private-token",
+ },
},
},
- })
+ 50
+ )
).toEqual({
id: 8,
call: {
@@ -50,22 +62,81 @@ describe("bounded MCP tracing", () => {
confirmDestructive: true,
hasConfirmationToken: true,
},
+ startedAtMs: 50,
},
});
});
- test("records success or failure after the matching response", () => {
+ test("records bounded timing, mutation, success, and failure metadata", () => {
const pending = new Map([
- [1, { name: "meta.guide" }],
- [2, { name: "verify-bindings" }],
+ [1, { name: "create-page", startedAtMs: 100, mutation: true as const }],
+ [2, { name: "verify-bindings", startedAtMs: 200 }],
]);
- expect(getMcpTraceResponse({ id: 1, result: {} }, pending)).toEqual({
- name: "meta.guide",
+ expect(
+ getMcpTraceResponse(
+ {
+ id: 1,
+ result: {
+ structuredContent: {
+ meta: { session: { committed: true } },
+ },
+ },
+ },
+ pending,
+ 175
+ )
+ ).toEqual({
+ name: "create-page",
+ startedAtMs: 100,
+ mutation: true,
+ durationMs: 75,
+ committed: true,
});
expect(
- getMcpTraceResponse({ id: 2, result: { isError: true } }, pending)
- ).toEqual({ name: "verify-bindings", isError: true });
+ getMcpTraceResponse({ id: 2, result: { isError: true } }, pending, 260)
+ ).toEqual({
+ name: "verify-bindings",
+ startedAtMs: 200,
+ durationMs: 60,
+ isError: true,
+ });
expect(pending.size).toBe(0);
});
+
+ test("classifies mutation tools from bounded MCP annotations", () => {
+ expect(getMcpToolsListRequestId({ id: 3, method: "tools/list" })).toBe(3);
+ const mutationToolNames = new Set(
+ getMcpMutationToolNames({
+ id: 3,
+ result: {
+ tools: [
+ {
+ name: "list-pages",
+ annotations: { readOnlyHint: true, private: "discard" },
+ },
+ {
+ name: "create-page",
+ annotations: { readOnlyHint: false, private: "discard" },
+ },
+ ],
+ },
+ })
+ );
+ expect(mutationToolNames).toEqual(new Set(["create-page"]));
+ expect(
+ getMcpTraceRequest(
+ {
+ id: 4,
+ method: "tools/call",
+ params: { name: "create-page", arguments: {} },
+ },
+ 25,
+ mutationToolNames
+ )
+ ).toEqual({
+ id: 4,
+ call: { name: "create-page", startedAtMs: 25, mutation: true },
+ });
+ });
});
diff --git a/packages/cli/evaluations/high-impact/mcp-trace-proxy.ts b/packages/cli/evaluations/high-impact/mcp-trace-proxy.ts
index 3fc88e58bd..c8b062db14 100644
--- a/packages/cli/evaluations/high-impact/mcp-trace-proxy.ts
+++ b/packages/cli/evaluations/high-impact/mcp-trace-proxy.ts
@@ -13,10 +13,19 @@ export type BoundedMcpCall = {
confirmDestructive?: true;
hasConfirmationToken?: true;
};
+ startedAtMs: number;
+ durationMs?: number;
+ mutation?: true;
+ planned?: true;
+ committed?: true;
isError?: true;
};
-export const getMcpTraceRequest = (value: unknown) => {
+export const getMcpTraceRequest = (
+ value: unknown,
+ startedAtMs = 0,
+ mutationToolNames: ReadonlySet<string> = new Set()
+) => {
if (isPlainRecord(value) === false || value.method !== "tools/call") {
return;
}
@@ -29,7 +38,11 @@ export const getMcpTraceRequest = (value: unknown) => {
) {
return;
}
- const call: BoundedMcpCall = { name: params.name };
+ const call: BoundedMcpCall = {
+ name: params.name,
+ startedAtMs: Math.round(startedAtMs),
+ ...(mutationToolNames.has(params.name) ? { mutation: true as const } : {}),
+ };
if (isPlainRecord(params.arguments)) {
const args: NonNullable<BoundedMcpCall["arguments"]> = {};
const viewport = params.arguments.viewport;
@@ -59,7 +72,8 @@ export const getMcpTraceRequest = (value: unknown) => {
export const getMcpTraceResponse = (
value: unknown,
- pending: Map<JsonRpcId, BoundedMcpCall>
+ pending: Map<JsonRpcId, BoundedMcpCall>,
+ completedAtMs = 0
) => {
if (isPlainRecord(value) === false) {
return;
@@ -74,10 +88,70 @@ export const getMcpTraceResponse = (
}
pending.delete(id);
const result = value.result;
- return value.error !== undefined ||
+ const isError =
+ value.error !== undefined ||
(isPlainRecord(result) && result.isError === true)
- ? { ...call, isError: true as const }
- : call;
+ ? (true as const)
+ : undefined;
+ const structuredContent =
+ isPlainRecord(result) && isPlainRecord(result.structuredContent)
+ ? result.structuredContent
+ : undefined;
+ const meta =
+ structuredContent !== undefined && isPlainRecord(structuredContent.meta)
+ ? structuredContent.meta
+ : undefined;
+ const session =
+ meta !== undefined && isPlainRecord(meta.session)
+ ? meta.session
+ : undefined;
+ const committed =
+ call.mutation === true && session?.committed === true
+ ? (true as const)
+ : undefined;
+ const planned =
+ call.mutation === true &&
+ session?.source === "dry-run" &&
+ session.transaction !== undefined
+ ? (true as const)
+ : undefined;
+ return {
+ ...call,
+ durationMs: Math.max(0, Math.round(completedAtMs - call.startedAtMs)),
+ ...(planned === undefined ? {} : { planned }),
+ ...(committed === undefined ? {} : { committed }),
+ ...(isError === undefined ? {} : { isError }),
+ };
+};
+
+export const getMcpToolsListRequestId = (value: unknown) => {
+ if (isPlainRecord(value) === false || value.method !== "tools/list") {
+ return;
+ }
+ return typeof value.id === "string" || typeof value.id === "number"
+ ? value.id
+ : undefined;
+};
+
+export const getMcpMutationToolNames = (value: unknown) => {
+ if (
+ isPlainRecord(value) === false ||
+ isPlainRecord(value.result) === false ||
+ Array.isArray(value.result.tools) === false
+ ) {
+ return;
+ }
+ return value.result.tools.flatMap((tool) => {
+ if (
+ isPlainRecord(tool) &&
+ typeof tool.name === "string" &&
+ isPlainRecord(tool.annotations) &&
+ tool.annotations.readOnlyHint === false
+ ) {
+ return [tool.name];
+ }
+ return [];
+ });
};
const observeJsonLines = (
@@ -103,18 +177,42 @@ const run = () => {
throw new Error("Expected target CLI and trace file paths.");
}
const pending = new Map<JsonRpcId, BoundedMcpCall>();
+ const pendingToolsLists = new Set<JsonRpcId>();
+ const mutationToolNames = new Set<string>();
+ const startedAt = performance.now();
const child = spawn(process.execPath, [target, "mcp"], {
env: process.env,
stdio: ["pipe", "pipe", "inherit"],
});
observeJsonLines(process.stdin, (value) => {
- const request = getMcpTraceRequest(value);
+ const toolsListId = getMcpToolsListRequestId(value);
+ if (toolsListId !== undefined) {
+ pendingToolsLists.add(toolsListId);
+ }
+ const request = getMcpTraceRequest(
+ value,
+ performance.now() - startedAt,
+ mutationToolNames
+ );
if (request !== undefined) {
pending.set(request.id, request.call);
}
});
observeJsonLines(child.stdout, (value) => {
- const call = getMcpTraceResponse(value, pending);
+ if (
+ isPlainRecord(value) &&
+ (typeof value.id === "string" || typeof value.id === "number") &&
+ pendingToolsLists.delete(value.id)
+ ) {
+ for (const name of getMcpMutationToolNames(value) ?? []) {
+ mutationToolNames.add(name);
+ }
+ }
+ const call = getMcpTraceResponse(
+ value,
+ pending,
+ performance.now() - startedAt
+ );
if (call !== undefined) {
appendFileSync(tracePath, `${JSON.stringify(call)}\n`, "utf8");
}
diff --git a/packages/cli/evaluations/high-impact/results/authenticated-page-v1.json b/packages/cli/evaluations/high-impact/results/authenticated-page-v1.json
index 7c6a4f8773..b38cc91f1e 100644
--- a/packages/cli/evaluations/high-impact/results/authenticated-page-v1.json
+++ b/packages/cli/evaluations/high-impact/results/authenticated-page-v1.json
@@ -1,37 +1,105 @@
{
- "schemaVersion": 1,
+ "schemaVersion": 2,
"kind": "high-impact-minimal-context-agent-evaluation-result",
"fixtureId": "authenticated-page-v1",
"outcome": "passed",
"cli": "source",
"provider": "openai",
- "model": "gpt-5.6-luna",
- "commandSha256": "f61daff839ad1c7579976c4f442e2b83082498c075bcea0a51b7f7879acc3086",
- "durationMs": 116865,
+ "model": "gpt-5.6-sol",
+ "commandSha256": "79baa45fea9cee997dfac56c6d33eaf4400efe5e5155db9934c4c82c9df41c83",
"exitCode": 0,
- "toolCallCount": 20,
- "focusedReadCount": 3,
+ "metrics": {
+ "durationMs": 529970,
+ "tokens": {
+ "input": 6976750,
+ "cachedInput": 6771712,
+ "cacheWriteInput": 0,
+ "output": 17054,
+ "reasoningOutput": 4582,
+ "total": 6993804,
+ "cachedInputRate": 0.9706
+ },
+ "toolCalls": {
+ "total": 62,
+ "succeeded": 51,
+ "failed": 11,
+ "retries": 10,
+ "focusedReads": 8,
+ "broadReads": 0,
+ "dryRuns": 12,
+ "plannedMutations": 7,
+ "committedMutations": 17,
+ "verifications": 10,
+ "totalDurationMs": 165266,
+ "p50DurationMs": 20,
+ "p95DurationMs": 27606,
+ "timeToFirstMutationMs": 69436,
+ "timeToFirstVerificationMs": 177942
+ }
+ },
"callSequence": [
"meta.guide",
- "meta.get_more_tools",
- "meta.get_more_tools",
- "list-resources",
+ "meta.index",
"list-pages",
- "verify-bindings",
+ "list-resources",
+ "list-variables",
"list-instances",
+ "get-project-settings",
+ "meta.get_more_tools",
+ "search-project",
"meta.get_more_tools",
"create-page",
+ "create-page",
"create-resource",
"create-resource",
+ "create-variable",
+ "create-variable",
+ "create-variable",
"insert-fragment",
- "meta.get_more_tools",
- "audit",
+ "insert-fragment",
+ "verify-bindings",
+ "list-breakpoints",
+ "update-styles",
+ "update-styles",
"preview.start",
"preview.start",
"screenshot",
+ "wrap-instance",
+ "wrap-instance",
+ "wrap-instance",
+ "wrap-instance",
+ "wrap-instance",
+ "wrap-instance",
+ "wrap-instance",
+ "wrap-instance",
+ "list-instances",
+ "wrap-instance",
+ "bind-props",
+ "bind-props",
+ "verify-bindings",
+ "screenshot",
+ "update-resource",
+ "update-resource",
+ "preview.start",
"screenshot",
+ "screenshot",
+ "update-variable",
+ "screenshot",
+ "update-resource",
+ "update-resource",
+ "update-resource",
+ "preview.start",
+ "preview.stop",
+ "preview.start",
+ "screenshot",
+ "update-variable",
+ "screenshot",
+ "update-variable",
+ "screenshot",
+ "update-variable",
+ "verify-bindings",
"audit",
- "verify-bindings"
+ "preview.stop"
],
"checks": {
"guidance": "passed",
@@ -44,6 +112,7 @@
"accountPage": "passed",
"authStates": "passed",
"providerConvention": "passed",
- "editableStructure": "passed"
+ "editableStructure": "passed",
+ "usageCaptured": "passed"
}
}
diff --git a/packages/cli/evaluations/high-impact/results/design-input-v1.json b/packages/cli/evaluations/high-impact/results/design-input-v1.json
index af2415f0f8..d06b378853 100644
--- a/packages/cli/evaluations/high-impact/results/design-input-v1.json
+++ b/packages/cli/evaluations/high-impact/results/design-input-v1.json
@@ -1,43 +1,66 @@
{
- "schemaVersion": 1,
+ "schemaVersion": 2,
"kind": "high-impact-minimal-context-agent-evaluation-result",
"fixtureId": "design-input-v1",
"outcome": "passed",
"cli": "source",
"provider": "openai",
- "model": "gpt-5.6-luna",
- "commandSha256": "a3c4c6dc81ed196ad8074222df7af560690d7334b0410e497b6e992ee63a6070",
- "durationMs": 196834,
+ "model": "gpt-5.6-sol",
+ "commandSha256": "b67b2dd38d840a331cf71a72ff8d6417b26cec0d48f1ec48bded91a00d7322e3",
"exitCode": 0,
- "toolCallCount": 26,
- "focusedReadCount": 6,
+ "metrics": {
+ "durationMs": 257624,
+ "tokens": {
+ "input": 1912580,
+ "cachedInput": 1814272,
+ "cacheWriteInput": 0,
+ "output": 12382,
+ "reasoningOutput": 1435,
+ "total": 1924962,
+ "cachedInputRate": 0.9486
+ },
+ "toolCalls": {
+ "total": 23,
+ "succeeded": 21,
+ "failed": 2,
+ "retries": 2,
+ "focusedReads": 7,
+ "broadReads": 0,
+ "dryRuns": 2,
+ "plannedMutations": 2,
+ "committedMutations": 5,
+ "verifications": 4,
+ "totalDurationMs": 37835,
+ "p50DurationMs": 27,
+ "p95DurationMs": 10457,
+ "timeToFirstMutationMs": 53985,
+ "timeToFirstVerificationMs": 187306
+ }
+ },
"callSequence": [
"meta.guide",
- "meta.index",
- "list-design-tokens",
"list-breakpoints",
- "meta.get_more_tools",
- "list-assets",
+ "list-design-tokens",
"list-pages",
+ "list-assets",
+ "meta.get_more_tools",
"list-instances",
+ "list-design-tokens",
"create-page",
+ "create-page",
+ "insert-fragment",
"insert-fragment",
"list-instances",
"attach-design-token",
"attach-design-token",
"attach-design-token",
- "audit",
- "screenshot",
- "screenshot",
+ "verify-bindings",
+ "preview.start",
+ "preview.start",
+ "preview.start",
"screenshot",
"screenshot",
- "meta.get_more_tools",
- "detach-design-token",
- "audit",
- "attach-design-token",
- "audit",
- "screenshot",
- "screenshot"
+ "audit"
],
"checks": {
"guidance": "passed",
@@ -50,6 +73,7 @@
"supportedComponents": "passed",
"designSystem": "passed",
"breakpointBehavior": "passed",
- "viewportEvidence": "passed"
+ "viewportEvidence": "passed",
+ "usageCaptured": "passed"
}
}
diff --git a/packages/cli/evaluations/high-impact/results/font-assets-v1.json b/packages/cli/evaluations/high-impact/results/font-assets-v1.json
new file mode 100644
index 0000000000..eafb7eb868
--- /dev/null
+++ b/packages/cli/evaluations/high-impact/results/font-assets-v1.json
@@ -0,0 +1,62 @@
+{
+ "schemaVersion": 2,
+ "kind": "high-impact-minimal-context-agent-evaluation-result",
+ "fixtureId": "font-assets-v1",
+ "outcome": "passed",
+ "cli": "source",
+ "provider": "openai",
+ "model": "gpt-5.6-sol",
+ "commandSha256": "53276281bd1eb5d56e4fe9fe63e44fc5c62ccade4a6f789aafc05367a8f9045d",
+ "exitCode": 0,
+ "metrics": {
+ "durationMs": 48517,
+ "tokens": {
+ "input": 465604,
+ "cachedInput": 418304,
+ "cacheWriteInput": 0,
+ "output": 1590,
+ "reasoningOutput": 324,
+ "total": 467194,
+ "cachedInputRate": 0.8984
+ },
+ "toolCalls": {
+ "total": 8,
+ "succeeded": 8,
+ "failed": 0,
+ "retries": 0,
+ "focusedReads": 2,
+ "broadReads": 0,
+ "dryRuns": 0,
+ "plannedMutations": 0,
+ "committedMutations": 3,
+ "verifications": 1,
+ "totalDurationMs": 73,
+ "p50DurationMs": 10,
+ "p95DurationMs": 21,
+ "timeToFirstMutationMs": 27832,
+ "timeToFirstVerificationMs": 43500
+ }
+ },
+ "callSequence": [
+ "meta.guide",
+ "upload-assets",
+ "update-asset",
+ "update-asset",
+ "refresh",
+ "get-asset",
+ "get-asset",
+ "audit"
+ ],
+ "checks": {
+ "guidance": "passed",
+ "privacy": "passed",
+ "expressions": "passed",
+ "boundedReads": "passed",
+ "fontUploads": "passed",
+ "fontMetadata": "passed",
+ "metadataWorkflow": "passed",
+ "fontFaceSources": "passed",
+ "pageUnchanged": "passed",
+ "usageCaptured": "passed"
+ }
+}
diff --git a/packages/cli/evaluations/high-impact/run-local-agent.ts b/packages/cli/evaluations/high-impact/run-local-agent.ts
index 66a16caf75..a00bcb7ecd 100644
--- a/packages/cli/evaluations/high-impact/run-local-agent.ts
+++ b/packages/cli/evaluations/high-impact/run-local-agent.ts
@@ -1,5 +1,5 @@
import { execFile } from "node:child_process";
-import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { createRequire } from "node:module";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
@@ -14,10 +14,18 @@ import {
} from "./fixtures";
import { startHighImpactFixtureApi } from "./fixture-api";
import { evaluateHighImpactOutcome } from "./validate";
-import { runHighImpactAgentEvaluation } from "./agent-runner";
+import {
+ runHighImpactAgentEvaluation,
+ type AgentEvaluationResult,
+} from "./agent-runner";
import { collectHighImpactArtifacts } from "./artifacts";
import type { EvaluationToolCall } from "./validate";
import { writeFontAssetFixtureFiles } from "./font-assets-fixture";
+import {
+ compareEvaluationResult,
+ isEvaluationComparisonAccepted,
+ type EvaluationComparison,
+} from "./evaluation-regression";
const execFileAsync = promisify(execFile);
const require = createRequire(import.meta.url);
@@ -26,6 +34,10 @@ const fixtureById = new Map<string, HighImpactFixture>(
highImpactFixtures.map((fixture) => [fixture.id, fixture])
);
+type AgentEvaluationReport = AgentEvaluationResult & {
+ comparison: EvaluationComparison;
+};
+
const shellQuote = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`;
const selectFixtures = (fixtureId: string | undefined) => {
@@ -89,6 +101,7 @@ const runFixture = async ({
"--dangerously-bypass-approvals-and-sandbox",
"--skip-git-repo-check",
"--ignore-rules",
+ "--json",
"--model",
shellQuote(model),
"--cd",
@@ -114,7 +127,7 @@ const runFixture = async ({
provider: "openai",
model,
env,
- getCallSequence: () => toolCalls.map(({ name }) => name),
+ getToolCalls: () => toolCalls,
evaluate: async () => {
toolCalls = await readToolCalls();
return evaluateHighImpactOutcome({
@@ -147,10 +160,30 @@ const run = async () => {
choices: fixtureIds,
description: "Run one evaluation fixture instead of the complete suite",
})
+ .option("baseline-directory", {
+ type: "string",
+ description: "Directory containing accepted per-fixture result baselines",
+ })
+ .option("require-baseline", {
+ type: "boolean",
+ default: true,
+ description: "Fail when a compatible accepted baseline is unavailable",
+ })
+ .option("update-baselines", {
+ type: "boolean",
+ default: false,
+ description:
+ "Replace accepted baselines after every fixture passes; requires the complete suite",
+ })
.strict()
.help()
.parse();
const fixtures = selectFixtures(options.fixture);
+ if (options.updateBaselines && options.fixture !== undefined) {
+ throw new Error(
+ "Accepted baselines can only be updated by the complete suite."
+ );
+ }
if (
fixtures.length > 1 &&
process.env.WEBSTUDIO_HIGH_IMPACT_RESULT !== undefined
@@ -164,19 +197,74 @@ const run = async () => {
process.env.WEBSTUDIO_HIGH_IMPACT_RESULTS_DIR ??
join(repositoryRoot, ".temp/evaluations/high-impact")
);
- const results = [];
+ const baselineDirectory = resolve(
+ options.baselineDirectory ??
+ process.env.WEBSTUDIO_HIGH_IMPACT_BASELINE_DIR ??
+ join(import.meta.dirname, "results")
+ );
+ const results: AgentEvaluationReport[] = [];
+ const rawResults: AgentEvaluationResult[] = [];
for (const fixture of fixtures) {
const resultPath = resolve(
process.env.WEBSTUDIO_HIGH_IMPACT_RESULT ??
join(resultsDirectory, `${fixture.id}.json`)
);
- results.push(await runFixture({ fixture, repositoryRoot, resultPath }));
+ const result = await runFixture({ fixture, repositoryRoot, resultPath });
+ rawResults.push(result);
+ const baseline = await readFile(
+ join(baselineDirectory, `${fixture.id}.json`),
+ "utf8"
+ )
+ .then((source) => JSON.parse(source) as AgentEvaluationResult)
+ .catch(() => undefined);
+ const report = {
+ ...result,
+ comparison: compareEvaluationResult(result, baseline),
+ };
+ await writeFile(
+ resultPath,
+ `${JSON.stringify(report, undefined, 2)}\n`,
+ "utf8"
+ );
+ results.push(report);
+ }
+ const evaluationsPassed = rawResults.every(
+ (result) => result.outcome === "passed"
+ );
+ if (options.updateBaselines && evaluationsPassed) {
+ await mkdir(baselineDirectory, { recursive: true });
+ await Promise.all(
+ rawResults.map((result) =>
+ writeFile(
+ join(baselineDirectory, `${result.fixtureId}.json`),
+ `${JSON.stringify(result, undefined, 2)}\n`,
+ "utf8"
+ )
+ )
+ );
}
- const outcome = results.every((result) => result.outcome === "passed")
- ? "passed"
- : "failed";
+ const comparisonsPassed = results.every(({ comparison }) =>
+ isEvaluationComparisonAccepted({
+ comparison,
+ requireBaseline: options.requireBaseline,
+ updateBaselines: options.updateBaselines,
+ })
+ );
+ const outcome = evaluationsPassed && comparisonsPassed ? "passed" : "failed";
process.stdout.write(
- `${JSON.stringify({ outcome, results }, undefined, 2)}\n`
+ `${JSON.stringify(
+ {
+ outcome,
+ baselines: options.updateBaselines
+ ? evaluationsPassed
+ ? "updated"
+ : "not-updated"
+ : "compared",
+ results,
+ },
+ undefined,
+ 2
+ )}\n`
);
if (outcome === "failed") {
process.exitCode = 1;
diff --git a/packages/cli/evaluations/high-impact/validate.ts b/packages/cli/evaluations/high-impact/validate.ts
index f0ddb1d780..29ad1fabed 100644
--- a/packages/cli/evaluations/high-impact/validate.ts
+++ b/packages/cli/evaluations/high-impact/validate.ts
@@ -14,10 +14,15 @@ import {
fontAssetFixtureMeta,
fontAssetFixtureSource,
} from "./font-assets-fixture";
+import { isBroadRead } from "./evaluation-metrics";
export type EvaluationToolCall = {
name: string;
arguments?: Record<string, unknown>;
+ startedAtMs?: number;
+ durationMs?: number;
+ planned?: true;
+ committed?: true;
isError?: boolean;
};
@@ -39,7 +44,6 @@ export type HighImpactEvaluationResult = {
passed: boolean;
checks: Record<string, "passed" | "failed">;
failures: string[];
- metrics: { toolCallCount: number; focusedReadCount: number };
};
const secretPatterns = [
@@ -48,12 +52,6 @@ const secretPatterns = [
/\b(?:eyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{10,}|sk-[A-Za-z0-9_-]{20,})\b/,
];
-const broadReadNames = new Set([
- "snapshot",
- "components.list",
- "components.coverage-plan",
-]);
-
const allowedComponents = new Set([
"body",
"box",
@@ -233,11 +231,8 @@ const validateCommon = (
checks,
failures,
"boundedReads",
- input.toolCalls.every((call) => broadReadNames.has(call.name) === false) &&
- input.toolCalls.filter((call) =>
- /^(?:list-|get-|inspect-)/.test(call.name)
- ).length <= 8,
- "Evaluation used a broad project dump or more than eight focused project reads."
+ input.toolCalls.every((call) => isBroadRead(call.name) === false),
+ "Evaluation used a broad project dump instead of focused project reads."
);
};
@@ -534,11 +529,5 @@ export const evaluateHighImpactOutcome = (
passed: failures.length === 0,
checks,
failures,
- metrics: {
- toolCallCount: input.toolCalls.length,
- focusedReadCount: input.toolCalls.filter((call) =>
- /^(?:list-|get-|inspect-)/.test(call.name)
- ).length,
- },
};
};
diff --git a/packages/cli/scripts/run-agent-command.test.ts b/packages/cli/scripts/run-agent-command.test.ts
index 495ec47e43..cae8fdde30 100644
--- a/packages/cli/scripts/run-agent-command.test.ts
+++ b/packages/cli/scripts/run-agent-command.test.ts
@@ -13,6 +13,22 @@ describe("runAgentCommand", () => {
).resolves.toMatchObject({ exitCode: 7, durationMs: expect.any(Number) });
});
+ test("observes stdout one line at a time without retaining it", async () => {
+ const lines: string[] = [];
+ await expect(
+ runAgentCommand({
+ command: `${JSON.stringify(process.execPath)} -e ${JSON.stringify(
+ 'process.stdout.write("first\\nsecond\\n")'
+ )}`,
+ cwd: process.cwd(),
+ env: process.env,
+ timeoutMs: 5_000,
+ onStdoutLine: (line) => lines.push(line),
+ })
+ ).resolves.toMatchObject({ exitCode: 0, durationMs: expect.any(Number) });
+ expect(lines).toEqual(["first", "second"]);
+ });
+
test("terminates a timed-out process group", async () => {
await expect(
runAgentCommand({
diff --git a/packages/cli/scripts/run-agent-command.ts b/packages/cli/scripts/run-agent-command.ts
index 21c9d2cc16..2becac421a 100644
--- a/packages/cli/scripts/run-agent-command.ts
+++ b/packages/cli/scripts/run-agent-command.ts
@@ -5,20 +5,38 @@ export const runAgentCommand = ({
cwd,
env,
timeoutMs,
+ onStdoutLine,
}: {
command: string;
cwd: string;
env: NodeJS.ProcessEnv;
timeoutMs: number;
+ onStdoutLine?: (line: string) => void;
}) =>
new Promise<{ exitCode: number; durationMs: number }>((resolve, reject) => {
const startedAt = Date.now();
const child = spawn(process.env.SHELL ?? "/bin/sh", ["-c", command], {
cwd,
env,
- stdio: "ignore",
+ stdio: [
+ "ignore",
+ onStdoutLine === undefined ? "ignore" : "pipe",
+ "ignore",
+ ],
detached: process.platform !== "win32",
});
+ let stdoutBuffer = "";
+ if (onStdoutLine !== undefined && child.stdout !== null) {
+ child.stdout.setEncoding("utf8");
+ child.stdout.on("data", (chunk: string) => {
+ stdoutBuffer += chunk;
+ const lines = stdoutBuffer.split("\n");
+ stdoutBuffer = lines.pop() ?? "";
+ for (const line of lines) {
+ onStdoutLine(line);
+ }
+ });
+ }
const timeout = setTimeout(() => {
if (child.pid !== undefined) {
try {
@@ -31,8 +49,11 @@ export const runAgentCommand = ({
clearTimeout(timeout);
reject(error);
});
- child.once("exit", (code, signal) => {
+ child.once("close", (code, signal) => {
clearTimeout(timeout);
+ if (onStdoutLine !== undefined && stdoutBuffer.length > 0) {
+ onStdoutLine(stdoutBuffer);
+ }
if (signal !== null) {
reject(new Error(`Agent command exited from signal ${signal}.`));
return;
diff --git a/packages/cli/src/commands/mcp-preview.test.ts b/packages/cli/src/commands/mcp-preview.test.ts
index 60b194bb5b..41f352ad55 100644
--- a/packages/cli/src/commands/mcp-preview.test.ts
+++ b/packages/cli/src/commands/mcp-preview.test.ts
@@ -701,6 +701,40 @@ test("captures path screenshots through an existing base URL without preview", a
]);
});
+test.each([
+ ["absolute URL", { url: "http://127.0.0.1:5177/design-system" }],
+ ["base URL", { baseUrl: "http://127.0.0.1:5177", path: "/design-system" }],
+] as const)(
+ "closes the browser after capturing an external %s",
+ async (_, target) => {
+ const captureScreenshot = createCaptureScreenshotMock([]);
+ const createCaptureSession = vi.fn();
+ const handlers = createMcpPreviewHandlers({
+ preview: {
+ status: vi.fn(),
+ startAndWait: vi.fn(),
+ resolveUrl: vi.fn(),
+ },
+ captureScreenshot,
+ createCaptureSession,
+ });
+
+ await handlers.captureScreenshot({
+ ...target,
+ timeout: 45_000,
+ viewport: { width: 1440, height: 900 },
+ });
+
+ expect(captureScreenshot).toHaveBeenCalledWith(
+ expect.objectContaining({
+ url: "http://127.0.0.1:5177/design-system",
+ timeout: 45_000,
+ })
+ );
+ expect(createCaptureSession).not.toHaveBeenCalled();
+ }
+);
+
test("rejects authenticated Builder URLs as generated preview targets", async () => {
const preview = {
status: vi.fn(),
diff --git a/packages/cli/src/commands/mcp-preview.ts b/packages/cli/src/commands/mcp-preview.ts
index 1013b5ed74..08302eae25 100644
--- a/packages/cli/src/commands/mcp-preview.ts
+++ b/packages/cli/src/commands/mcp-preview.ts
@@ -104,6 +104,11 @@ const defaultPreviewSource = "session" satisfies PreviewSource;
const getPreviewSource = (source: PreviewSource | undefined): PreviewSource =>
source ?? defaultPreviewSource;
+const isManagedSessionPreviewCapture = (input: McpScreenshotInput) =>
+ input.url === undefined &&
+ input.baseUrl === undefined &&
+ getPreviewSource(input.source) !== "local";
+
const getPreviewTarget = (input: McpPreviewInput) => ({
source: getPreviewSource(input.source),
mode: input.mode ?? "iterative",
@@ -399,7 +404,7 @@ export const createMcpPreviewHandlers = ({
const captureOptions = getCaptureOptions(input, url);
let result: Awaited<ReturnType<typeof captureScreenshot>>;
if (
- getPreviewSource(input.source) !== "local" &&
+ isManagedSessionPreviewCapture(input) &&
createCaptureSession !== undefined
) {
result = await (
diff --git a/packages/cli/src/commands/mcp.test.ts b/packages/cli/src/commands/mcp.test.ts
index 868c9ebf28..178a2eb424 100644
--- a/packages/cli/src/commands/mcp.test.ts
+++ b/packages/cli/src/commands/mcp.test.ts
@@ -384,6 +384,8 @@ test("preserves structured input issues in MCP shell errors", () => {
ok: false,
error: {
code: "INVALID_INPUT",
+ message:
+ "Audit input is invalid.\npagePath: pageId and pagePath are mutually exclusive (use_page_id_or_page_path).",
issues: [
{
code: "mutually_exclusive_fields",
diff --git a/packages/cli/src/commands/mcp.ts b/packages/cli/src/commands/mcp.ts
index f881fcfc18..220a23b88d 100644
--- a/packages/cli/src/commands/mcp.ts
+++ b/packages/cli/src/commands/mcp.ts
@@ -23,7 +23,6 @@ import { resolveApiConnection } from "../api-connection";
import {
getCliErrorIssues,
getCliErrorMessage,
- getCliErrorSummary,
getStableErrorCode,
isMissingApiAccessError,
} from "../error-codes";
@@ -683,14 +682,14 @@ const getMcpRunError = (error: unknown) => {
code: isMissingApiAccessError(error)
? "UNAUTHORIZED"
: (error.code ?? "MCP_TOOL_FAILED"),
- message: getCliErrorSummary(error),
+ message: getCliErrorMessage(error),
...(issues === undefined ? {} : { issues }),
};
}
const code = isMissingApiAccessError(error)
? "UNAUTHORIZED"
: (getStableErrorCode(error) ?? "MCP_TOOL_FAILED");
- const message = getCliErrorSummary(error);
+ const message = getCliErrorMessage(error);
return { code, message, ...(issues === undefined ? {} : { issues }) };
};
diff --git a/packages/cli/src/error-codes.ts b/packages/cli/src/error-codes.ts
index 44944167c8..85720668b4 100644
--- a/packages/cli/src/error-codes.ts
+++ b/packages/cli/src/error-codes.ts
@@ -3,7 +3,7 @@ import {
getApiErrorCode,
} from "@webstudio-is/http-client";
import {
- formatValidationIssues,
+ formatValidationErrorMessage,
getValidationIssues,
} from "@webstudio-is/project-build/runtime";
@@ -58,7 +58,7 @@ export const getCliErrorMessage = (error: unknown, command = "mcp") => {
const issues = getValidationIssues(error);
return issues === undefined || issues.length === 0
? message
- : `${message}\n${formatValidationIssues(issues)}`;
+ : formatValidationErrorMessage(message, issues);
};
export const getCliErrorIssues = getValidationIssues;
diff --git a/packages/project-build/src/contracts/__generated__/runtime-operation-contracts.ts b/packages/project-build/src/contracts/__generated__/runtime-operation-contracts.ts
index 8de29d4841..14d28d09dd 100644
--- a/packages/project-build/src/contracts/__generated__/runtime-operation-contracts.ts
+++ b/packages/project-build/src/contracts/__generated__/runtime-operation-contracts.ts
@@ -62044,7 +62044,23 @@ export const runtimeOperationContractData = [
property: {
type: "string",
},
- value: {},
+ value: {
+ description: "Typed CSS StyleValue object.",
+ type: "object",
+ properties: {
+ type: {
+ type: "string",
+ },
+ },
+ required: ["type"],
+ additionalProperties: {},
+ examples: [
+ {
+ type: "keyword",
+ value: "red",
+ },
+ ],
+ },
breakpoint: {
type: "string",
},
@@ -62058,7 +62074,7 @@ export const runtimeOperationContractData = [
type: "boolean",
},
},
- required: ["instanceId", "property"],
+ required: ["instanceId", "property", "value"],
},
},
},
@@ -62166,7 +62182,23 @@ export const runtimeOperationContractData = [
property: {
type: "string",
},
- value: {},
+ value: {
+ description: "Typed CSS StyleValue object.",
+ type: "object",
+ properties: {
+ type: {
+ type: "string",
+ },
+ },
+ required: ["type"],
+ additionalProperties: {},
+ examples: [
+ {
+ type: "keyword",
+ value: "red",
+ },
+ ],
+ },
breakpoint: {
type: "string",
},
@@ -62180,7 +62212,7 @@ export const runtimeOperationContractData = [
type: "string",
},
},
- required: ["instanceId", "property", "styleSourceId"],
+ required: ["instanceId", "property", "value", "styleSourceId"],
},
},
},
@@ -62477,6 +62509,7 @@ export const runtimeOperationContractData = [
type: "string",
},
{
+ description: "Typed CSS StyleValue object.",
type: "object",
properties: {
type: {
@@ -62485,6 +62518,12 @@ export const runtimeOperationContractData = [
},
required: ["type"],
additionalProperties: {},
+ examples: [
+ {
+ type: "keyword",
+ value: "red",
+ },
+ ],
},
],
},
@@ -62503,6 +62542,7 @@ export const runtimeOperationContractData = [
type: "string",
},
{
+ description: "Typed CSS StyleValue object.",
type: "object",
properties: {
type: {
@@ -62511,6 +62551,12 @@ export const runtimeOperationContractData = [
},
required: ["type"],
additionalProperties: {},
+ examples: [
+ {
+ type: "keyword",
+ value: "red",
+ },
+ ],
},
],
},
@@ -62834,6 +62880,7 @@ export const runtimeOperationContractData = [
type: "string",
},
{
+ description: "Typed CSS StyleValue object.",
type: "object",
properties: {
type: {
@@ -62842,6 +62889,12 @@ export const runtimeOperationContractData = [
},
required: ["type"],
additionalProperties: {},
+ examples: [
+ {
+ type: "keyword",
+ value: "red",
+ },
+ ],
},
],
},
@@ -62860,6 +62913,7 @@ export const runtimeOperationContractData = [
type: "string",
},
{
+ description: "Typed CSS StyleValue object.",
type: "object",
properties: {
type: {
@@ -62868,6 +62922,12 @@ export const runtimeOperationContractData = [
},
required: ["type"],
additionalProperties: {},
+ examples: [
+ {
+ type: "keyword",
+ value: "red",
+ },
+ ],
},
],
},
@@ -62949,6 +63009,7 @@ export const runtimeOperationContractData = [
type: "string",
},
{
+ description: "Typed CSS StyleValue object.",
type: "object",
properties: {
type: {
@@ -62957,6 +63018,12 @@ export const runtimeOperationContractData = [
},
required: ["type"],
additionalProperties: {},
+ examples: [
+ {
+ type: "keyword",
+ value: "red",
+ },
+ ],
},
],
},
diff --git a/packages/project-build/src/mcp.test.ts b/packages/project-build/src/mcp.test.ts
index 9e4d4b8abd..e1aa53a4eb 100644
--- a/packages/project-build/src/mcp.test.ts
+++ b/packages/project-build/src/mcp.test.ts
@@ -7265,7 +7265,9 @@ describe("project session mcp adapter", () => {
structuredContent: {
ok: false,
error: {
- message: "Tool input is invalid.",
+ message: expect.stringContaining(
+ 'email: Invalid email address (format:email). Example: "[email protected]".'
+ ),
code: "INVALID_INPUT",
issues: expect.arrayContaining([
expect.objectContaining({
@@ -7322,7 +7324,8 @@ describe("project session mcp adapter", () => {
ok: false,
error: {
code: "INVALID_INPUT",
- message: "Page input is invalid.",
+ message:
+ 'Page input is invalid.\nvalues.title: Invalid Webstudio expression (valid_webstudio_expression). Example: "pageTitle ?? \\"Pricing\\"". Detail: Unexpected token at 1:4',
issues: [issue],
},
meta: {},
diff --git a/packages/project-build/src/mcp.ts b/packages/project-build/src/mcp.ts
index 5809a74b4f..15045a0e8d 100644
--- a/packages/project-build/src/mcp.ts
+++ b/packages/project-build/src/mcp.ts
@@ -79,6 +79,7 @@ import {
import { parseWebstudioJsxFragment } from "./runtime/jsx";
import { webstudioJsxFragmentInputDescription } from "./runtime/jsx/bindings";
import {
+ formatValidationErrorMessage,
getValidationIssues,
getZodValidationIssues,
semanticValidationIssuesJsonSchema,
@@ -6849,7 +6850,7 @@ const toToolErrorResult = (
error instanceof z.ZodError
? getZodValidationIssues(error, inputSchema)
: getValidationIssues(error);
- const message =
+ const baseMessage =
code === "PROJECT_SESSION_BUSY"
? projectSessionBusyMessage
: error instanceof z.ZodError
@@ -6857,6 +6858,10 @@ const toToolErrorResult = (
: error instanceof Error
? error.message
: String(error);
+ const message =
+ validationIssues === undefined || validationIssues.length === 0
+ ? baseMessage
+ : formatValidationErrorMessage(baseMessage, validationIssues);
const structuredContent = {
ok: false as const,
error: {
diff --git a/packages/project-build/src/runtime/errors.ts b/packages/project-build/src/runtime/errors.ts
index 941b9b1a00..a08d91fe40 100644
--- a/packages/project-build/src/runtime/errors.ts
+++ b/packages/project-build/src/runtime/errors.ts
@@ -340,6 +340,17 @@ export const formatValidationIssues = (
})
.join("\n");
+export const formatValidationErrorMessage = (
+ message: string,
+ issues: readonly SemanticValidationIssue[]
+) => {
+ const formattedIssues = formatValidationIssues(issues);
+ if (formattedIssues.length === 0 || message.includes(formattedIssues)) {
+ return message;
+ }
+ return `${message}\n${formattedIssues}`;
+};
+
export const formatValidationIssueMessages = (
issues: readonly SemanticValidationIssue[]
) =>
diff --git a/packages/project-build/src/runtime/pages.test.ts b/packages/project-build/src/runtime/pages.test.ts
index 5fec4b5f00..e6f6a8f5a9 100644
--- a/packages/project-build/src/runtime/pages.test.ts
+++ b/packages/project-build/src/runtime/pages.test.ts
@@ -1874,6 +1874,50 @@ describe("createPage", () => {
]);
});
+ test("accepts multi-sentence fixed page text", () => {
+ const mutation = createPage(
+ { pages: createPages() },
+ {
+ name: "Marques",
+ path: "/marques",
+ title: "Marques. Toutes nos marques",
+ meta: {
+ description:
+ "Toutes nos marques moto. Concessionnaire multimarque en Gironde.",
+ },
+ },
+ { createId: createIdFactory() }
+ );
+
+ expect(mutation.payload[0]?.patches[0]).toEqual(
+ expect.objectContaining({
+ value: expect.objectContaining({
+ title: `"Marques. Toutes nos marques"`,
+ meta: expect.objectContaining({
+ description: `"Toutes nos marques moto. Concessionnaire multimarque en Gironde."`,
+ }),
+ }),
+ })
+ );
+
+ const computedMutation = createPage(
+ { pages: createPages() },
+ {
+ name: "Computed marques",
+ path: "/computed-marques",
+ title: `pageTitle ?? "Marques. Toutes nos marques"`,
+ },
+ { createId: createIdFactory() }
+ );
+ expect(computedMutation.payload[0]?.patches[0]).toEqual(
+ expect.objectContaining({
+ value: expect.objectContaining({
+ title: `pageTitle ?? "Marques. Toutes nos marques"`,
+ }),
+ })
+ );
+ });
+
test("still rejects invalid JavaScript-looking page metadata expressions", () => {
const pages = createPages();
@@ -1989,6 +2033,35 @@ describe("updatePage", () => {
]);
});
+ test("accepts multi-sentence fixed page text", () => {
+ const mutation = updatePage(
+ { pages: createPages() },
+ {
+ pageId: "page",
+ values: {
+ title: "Marques. Toutes nos marques",
+ meta: {
+ description:
+ "L'atelier présente toutes nos marques moto. Concessionnaire multimarque en Gironde.",
+ },
+ },
+ }
+ );
+
+ expect(mutation.payload[0]?.patches).toEqual([
+ {
+ op: "replace",
+ path: ["pages", "page", "title"],
+ value: `"Marques. Toutes nos marques"`,
+ },
+ {
+ op: "add",
+ path: ["pages", "page", "meta", "description"],
+ value: `"L'atelier présente toutes nos marques moto. Concessionnaire multimarque en Gironde."`,
+ },
+ ]);
+ });
+
test("rejects invalid JavaScript-looking page metadata expressions before patching", () => {
const pages = createPages();
diff --git a/packages/project-build/src/runtime/pages.ts b/packages/project-build/src/runtime/pages.ts
index bdcd7d348d..29cd6ddc20 100644
--- a/packages/project-build/src/runtime/pages.ts
+++ b/packages/project-build/src/runtime/pages.ts
@@ -24,6 +24,7 @@ import {
isPageDraft,
} from "@webstudio-is/sdk";
import { serializePages } from "@webstudio-is/project-migrations/pages";
+import { tokenizer } from "acorn";
import * as bcp47 from "bcp-47";
import slugify from "slugify";
import { z } from "zod";
@@ -859,10 +860,56 @@ const jsExpressionStartPattern =
/^\s*(?:["'`[{(]|(?:await|new|typeof|void)\b|(?:undefined|null|true|false)\s*$)/;
const jsExpressionOperatorPattern =
/(?:\?\?|&&|\|\||=>|\?\s*.+\s*:|\.\s*[A-Za-z_$]|\[[^\]]*\]|\s(?:[=!<>]=?|[+\-*/%])\s)/;
-const urlStringPattern = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//;
+
+const pageTextSentenceSegmenter = new Intl.Segmenter(undefined, {
+ granularity: "sentence",
+});
+
+const hasMultipleSentences = (value: string) => {
+ let count = 0;
+ for (const { segment } of pageTextSentenceSegmenter.segment(value)) {
+ if (segment.trim().length > 0) {
+ count += 1;
+ }
+ if (count > 1) {
+ return true;
+ }
+ }
+ return false;
+};
+
+const hasWhitespaceAfterDot = (value: string) => {
+ try {
+ const tokens = tokenizer(value, { ecmaVersion: "latest" });
+ let token = tokens.getToken();
+ while (token.type.label !== "eof") {
+ const nextToken = tokens.getToken();
+ if (
+ token.type.label === "." &&
+ nextToken.type.label === "name" &&
+ token.end < nextToken.start
+ ) {
+ return true;
+ }
+ token = nextToken;
+ }
+ } catch {
+ return hasMultipleSentences(value);
+ }
+ return false;
+};
+
+const isAbsoluteUrl = (value: string) => {
+ try {
+ new URL(value);
+ return true;
+ } catch {
+ return false;
+ }
+};
const normalizePageExpressionInput = (value: string) => {
- if (urlStringPattern.test(value)) {
+ if (isAbsoluteUrl(value) || hasWhitespaceAfterDot(value)) {
return JSON.stringify(value);
}
if (
diff --git a/packages/project-build/src/runtime/styles.test.ts b/packages/project-build/src/runtime/styles.test.ts
index 93a4935843..f64522f3da 100644
--- a/packages/project-build/src/runtime/styles.test.ts
+++ b/packages/project-build/src/runtime/styles.test.ts
@@ -59,6 +59,7 @@ import {
createSelectedStyleDeclarationUpdatePayload,
styleDeleteInput,
styleReplaceInput,
+ styleUpdateDeclarationsInput,
styleUpdateInput,
defineCssVariables,
deleteDesignTokenStyles,
@@ -79,6 +80,8 @@ import {
updateVarReferencesInProps,
} from "./styles";
import { createLocalStyleSourceClonePlan } from "./style-utils";
+import { getInputSchemaMetadata } from "../contracts/input-schema";
+import { getZodValidationIssues } from "./errors";
import {
addStyleSourceToInstanceMutable,
createLocalStyleSourcePatchPlan,
@@ -2024,6 +2027,64 @@ describe("style declaration helpers", () => {
},
],
});
+
+ const invalidStyleUpdate = styleUpdateDeclarationsInput.safeParse({
+ updates: [
+ {
+ instanceId: "box",
+ property: "color",
+ value: "red",
+ },
+ ],
+ });
+ expect(invalidStyleUpdate.success).toBe(false);
+ if (invalidStyleUpdate.success) {
+ throw new Error("Expected the primitive style value to be rejected");
+ }
+ expect(
+ getZodValidationIssues(
+ invalidStyleUpdate.error,
+ getInputSchemaMetadata(styleUpdateDeclarationsInput).inputJsonSchema
+ )
+ ).toEqual([
+ expect.objectContaining({
+ code: "invalid_type",
+ path: ["updates", "0", "value"],
+ message: "Invalid input: expected object, received string",
+ example: { type: "keyword", value: "red" },
+ }),
+ ]);
+ expect(
+ styleUpdateInput.safeParse({
+ instanceId: "box",
+ property: "color",
+ value: "red",
+ }).success
+ ).toBe(false);
+ expect(
+ designTokenCreateInput.safeParse({
+ name: "Primary",
+ styles: { color: "red" },
+ }).success
+ ).toBe(true);
+ expect(
+ designTokenCreateInput.safeParse({
+ name: "Primary",
+ styles: { color: { type: "unsupported", value: "red" } },
+ }).success
+ ).toBe(false);
+
+ expect(
+ getInputSchemaMetadata(styleUpdateInput).inputJsonSchema.properties?.value
+ ).toMatchObject({
+ description: "Typed CSS StyleValue object.",
+ examples: [{ type: "keyword", value: "red" }],
+ type: "object",
+ properties: {
+ type: { type: "string" },
+ },
+ required: ["type"],
+ });
});
test("reject empty design token names", () => {
diff --git a/packages/project-build/src/runtime/styles.ts b/packages/project-build/src/runtime/styles.ts
index fb6457d280..d3b99cadc3 100644
--- a/packages/project-build/src/runtime/styles.ts
+++ b/packages/project-build/src/runtime/styles.ts
@@ -38,7 +38,7 @@ import {
import type { BuilderState } from "../state/builder-state";
import { findSerializedPageByInput, getSerializedPages } from "./pages";
import { getInstanceDepths } from "./instances";
-import { throwBuilderRuntimeError } from "./errors";
+import { addZodValidationIssue, throwBuilderRuntimeError } from "./errors";
import { runtimeGeneratedIdInput } from "./generated-id-input";
import { createRuntimeMutation } from "./mutation";
import { serializeStyleDeclarations } from "./style-utils";
@@ -301,10 +301,32 @@ const styleStateInput = z.string().superRefine((state, context) => {
}
});
+const styleValueExample = { type: "keyword", value: "red" } as const;
+const typedStyleValueInput = z
+ .object({ type: z.string() })
+ .passthrough()
+ .meta({
+ description: "Typed CSS StyleValue object.",
+ examples: [styleValueExample],
+ })
+ .superRefine((value, context) => {
+ if (styleValue.safeParse(value).success === false) {
+ addZodValidationIssue(context, {
+ code: "invalid_style_value",
+ path: [],
+ message: "Expected a typed CSS StyleValue object",
+ constraint: "typed_css_style_value",
+ example: styleValueExample,
+ });
+ }
+ })
+ .transform((value) => value as StyleValue)
+ .describe("Typed CSS StyleValue object.");
+
export const styleUpdateInput = z.object({
instanceId: z.string(),
property: z.string(),
- value: z.unknown(),
+ value: typedStyleValueInput,
breakpoint: z.string().optional(),
state: styleStateInput.optional(),
listed: z.boolean().optional(),
@@ -1161,10 +1183,7 @@ export const serializeCssVariables = ({
};
};
-const designTokenStyleValueInput = z.union([
- z.string(),
- z.object({ type: z.string() }).passthrough(),
-]);
+const designTokenStyleValueInput = z.union([z.string(), typedStyleValueInput]);
export const designTokenStyleInput = z.object({
property: z.string(),