AI-native design editor. Open-source Figma alternative.
feat(ai): add isolated visual inspection
Danila Poyarkov8294e32f3398
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 09f6ba0cb..1049eac36 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -5,6 +5,7 @@### Added- Add local crash recovery for unsaved and pathless documents, including MCP-created documents. (#487)+- Add isolated visual inspection that sends bounded selection renders to the configured Vision model and returns text findings without retaining image data in Design chat history. (#232, #471)- Allow supported AI model profiles to set a provider-specific reasoning effort. (#454)- Show unavailable or substituted document fonts with affected-layer selection and retry actions, and expose font fidelity through the Figma API and MCP tooling. (#503)diff --git a/packages/core/src/tools/vector/export.ts b/packages/core/src/tools/vector/export.tsindex faeea55b5..cb92a5195 100644--- a/packages/core/src/tools/vector/export.ts+++ b/packages/core/src/tools/vector/export.ts@@ -70,11 +70,19 @@ export const exportImage = defineTool({},scale: {type: 'number',- description: 'Export scale multiplier (default: 1)',+ description: 'Export scale multiplier before the maximum-edge limit is applied (default: 1)',default: 1,min: 0.1,max: 4},+ maxEdge: {+ type: 'number',+ description:+ 'Maximum output width or height in pixels. Preserves aspect ratio and never upscales. Defaults to 1280 for bounded model input.',+ default: 1280,+ min: 64,+ max: 4096+ },path: {type: 'string',description:@@ -88,8 +96,33 @@ export const exportImage = defineTool({const ids =args.ids && args.ids.length > 0 ? args.ids : figma.currentPage.children.map((node) => node.id)const format = (args.format ?? 'PNG').toUpperCase() as RasterExportFormat+ const requestedScale = args.scale ?? 1+ const maxEdge = args.maxEdge ?? 1280+ const nodes = ids.map((id) => figma.getNodeById(id)).filter((node) => node !== null)+ if (nodes.length === 0) return { error: 'No visible nodes to export' }+ const bounds = nodes.reduce(+ (result, node) => {+ const box = node.absoluteBoundingBox+ const minX = Math.min(result.minX, box.x)+ const minY = Math.min(result.minY, box.y)+ const maxX = Math.max(result.maxX, box.x + box.width)+ const maxY = Math.max(result.maxY, box.y + box.height)+ return { minX, minY, maxX, maxY }+ },+ {+ minX: Number.POSITIVE_INFINITY,+ minY: Number.POSITIVE_INFINITY,+ maxX: Number.NEGATIVE_INFINITY,+ maxY: Number.NEGATIVE_INFINITY+ }+ )+ const width = bounds.maxX - bounds.minX+ const height = bounds.maxY - bounds.minY+ const longestEdge = Math.max(width, height)+ const boundedScale = longestEdge > 0 ? Math.min(requestedScale, maxEdge / longestEdge) : 0+ if (boundedScale <= 0) return { error: 'No visible nodes to export' }const data = await figma.exportImage(ids, {- scale: args.scale ?? 1,+ scale: boundedScale,format})if (!data || data.length === 0) return { error: 'No visible nodes to export' }@@ -98,7 +131,10 @@ export const exportImage = defineTool({return {mimeType: mimeMap[format],base64,- byteLength: data.length+ byteLength: data.length,+ width: Math.ceil(width * boundedScale),+ height: Math.ceil(height * boundedScale),+ scale: boundedScale}}})diff --git a/src/app/ai/chat/system-prompt.md b/src/app/ai/chat/system-prompt.mdindex 5870ef673..c897044fd 100644--- a/src/app/ai/chat/system-prompt.md+++ b/src/app/ai/chat/system-prompt.md@@ -211,7 +211,7 @@ Common warnings:β **Don't call `viewport_zoom_to_fit` or `describe` with the same arguments as a previous call in the same conversation.** Check your last calls before repeating.-π« **Never use `export_image`** β slow and wastes tokens. Use `describe` instead.+ποΈ **Use `export_image` only when visual evidence is necessary** β for an explicit visual review, a rendering problem, or a user request to compare appearance. Prefer selected node IDs over the whole page, use PNG at the default 1Γ scale, and do not render repeatedly unless the design changed. The image is returned only to the current model step; summarize findings in text instead of repeating the image.## Step budgetdiff --git a/src/app/ai/tools/index.ts b/src/app/ai/tools/index.tsindex 6cbdfd1f4..47d56dd5d 100644--- a/src/app/ai/tools/index.ts+++ b/src/app/ai/tools/index.ts@@ -3,7 +3,7 @@ import { tool } from 'ai'import * as v from 'valibot'import { computeAllLayouts } from '@open-pencil/core/layout'-import { CORE_TOOLS, toolsToAI } from '@open-pencil/core/tools'+import { CORE_TOOLS, EXTENDED_TOOLS, toolsToAI } from '@open-pencil/core/tools'import type { StepBudget, ToolLogEntry } from '@open-pencil/core/tools'import type { SceneNode } from '@open-pencil/scene-graph'@@ -12,8 +12,16 @@ import { getActiveEditorStore } from '@/app/editor/active-store'import type { EditorStore } from '@/app/editor/active-store'import { ensureGraphFonts } from '@/app/editor/fonts'+import { createVisualInspectionTool } from './vision'+export const MAX_AGENT_STEPS = 50+const VISUAL_INSPECTION_TOOL_NAMES = new Set(['export_image'])+const AI_CHAT_TOOLS = [+ ...CORE_TOOLS,+ ...EXTENDED_TOOLS.filter((definition) => VISUAL_INSPECTION_TOOL_NAMES.has(definition.name))+]+export interface StepUsage {inputTokens: numberoutputTokens: number@@ -86,50 +94,53 @@ export function createAITools(store: EditorStore) {let beforeSnapshot: Map<string, SceneNode> | null = nullconst runState = getRunState(store)- return toolsToAI(- CORE_TOOLS,- {- getFigma: () => makeFigmaFromStore(store),- onBeforeExecute: (def) => {- if (def.mutates) {- beforeSnapshot = store.snapshotPage()- }- },- onAfterExecute: async (def) => {- if (def.mutates) {- const pageId = store.state.currentPageId- const pageNode = store.graph.getNode(pageId)- if (pageNode) await ensureGraphFonts(store.graph, pageNode.childIds, store.renderer)- computeAllLayouts(store.graph, pageId)- store.requestRender()- if (beforeSnapshot) {- const before = beforeSnapshot- const after = store.snapshotPage()- store.pushUndoEntry({- label: `AI: ${def.name}`,- forward: () => store.restorePageFromSnapshot(after),- inverse: () => store.restorePageFromSnapshot(before)- })- beforeSnapshot = null+ return {+ ...toolsToAI(+ AI_CHAT_TOOLS,+ {+ getFigma: () => makeFigmaFromStore(store),+ onBeforeExecute: (def) => {+ if (def.mutates) {+ beforeSnapshot = store.snapshotPage()}- }- },- onFlashNodes: (nodeIds) => {- store.renderer?.aiClearActive()- if (nodeIds.length > 0) {- store.aiFlashDone(nodeIds)- }- },- onToolLog: (entry) => {- runState.toolLog.push(entry)+ },+ onAfterExecute: async (def) => {+ if (def.mutates) {+ const pageId = store.state.currentPageId+ const pageNode = store.graph.getNode(pageId)+ if (pageNode) await ensureGraphFonts(store.graph, pageNode.childIds, store.renderer)+ computeAllLayouts(store.graph, pageId)+ store.requestRender()+ if (beforeSnapshot) {+ const before = beforeSnapshot+ const after = store.snapshotPage()+ store.pushUndoEntry({+ label: `AI: ${def.name}`,+ forward: () => store.restorePageFromSnapshot(after),+ inverse: () => store.restorePageFromSnapshot(before)+ })+ beforeSnapshot = null+ }+ }+ },+ onFlashNodes: (nodeIds) => {+ store.renderer?.aiClearActive()+ if (nodeIds.length > 0) {+ store.aiFlashDone(nodeIds)+ }+ },+ onToolLog: (entry) => {+ runState.toolLog.push(entry)+ },+ getStepBudget: (): StepBudget => ({+ current: runState.currentSteps,+ max: MAX_AGENT_STEPS+ })},- getStepBudget: (): StepBudget => ({- current: runState.currentSteps,- max: MAX_AGENT_STEPS- })- },- { v, valibotSchema, tool }- )+ { v, valibotSchema, tool }+ ),+ inspect_visual: createVisualInspectionTool(store)+ }}export type AITools = ReturnType<typeof createAITools>diff --git a/src/app/ai/tools/vision.ts b/src/app/ai/tools/vision.tsnew file mode 100644index 000000000..3027ae2a1--- /dev/null+++ b/src/app/ai/tools/vision.ts@@ -0,0 +1,112 @@+import { valibotSchema } from '@ai-sdk/valibot'+import { generateText, tool } from 'ai'+import * as v from 'valibot'++import { computeContentBounds } from '@open-pencil/core/io'++import { createAIModelRuntime } from '@/app/ai/models'+import type { EditorStore } from '@/app/editor/active-store'++const DEFAULT_VISION_MAX_EDGE = 1280+const MAX_VISION_MAX_EDGE = 4096+const MAX_VISION_OUTPUT_TOKENS = 1200++export type VisualInspectionRequest = {+ ids?: string[]+ question?: string+ maxEdge?: number+}++export type VisualInspectionResult = {+ analysis: string+ inspectedNodeIds: string[]+ image: { width: number; height: number }+}++export type VisualInspectionDependencies = {+ createRuntime: typeof createAIModelRuntime+ inspect: typeof generateText+}++export function boundedImageScale(+ width: number,+ height: number,+ maxEdge = DEFAULT_VISION_MAX_EDGE+): number {+ const longestEdge = Math.max(width, height)+ if (longestEdge <= 0) return 0+ return Math.min(1, maxEdge / longestEdge)+}++export async function inspectRenderedDesign(+ store: EditorStore,+ request: VisualInspectionRequest,+ dependencies: VisualInspectionDependencies = {+ createRuntime: createAIModelRuntime,+ inspect: generateText+ }+): Promise<VisualInspectionResult | { error: string }> {+ const runtime = await dependencies.createRuntime('vision')+ if (runtime?.kind !== 'direct') {+ return { error: 'Configure a vision-capable model in Settings to inspect rendered designs.' }+ }++ const pageId = store.state.currentPageId+ let nodeIds = request.ids ?? []+ if (nodeIds.length === 0) nodeIds = [...store.state.selectedIds]+ if (nodeIds.length === 0) {+ nodeIds = store.graph.getChildren(pageId).map((node) => node.id)+ }+ const bounds = computeContentBounds(store.graph, nodeIds)+ if (!bounds) return { error: 'No visible design content to inspect.' }+ const width = bounds.maxX - bounds.minX+ const height = bounds.maxY - bounds.minY+ const scale = boundedImageScale(width, height, request.maxEdge)+ if (scale <= 0) return { error: 'No visible design content to inspect.' }++ const image = await store.renderExportImage(nodeIds, scale, 'PNG', pageId)+ if (!image) return { error: 'Could not render the design for visual inspection.' }++ const result = await dependencies.inspect({+ model: runtime.model,+ maxOutputTokens: Math.min(runtime.role.profile.maxOutputTokens, MAX_VISION_OUTPUT_TOKENS),+ messages: [+ {+ role: 'user',+ content: [+ {+ type: 'text',+ text:+ request.question?.trim() ||+ 'Review this rendered design. Concisely identify visual hierarchy, alignment, spacing, clipping, contrast, and rendering problems. Return actionable findings only.'+ },+ { type: 'file', mediaType: 'image/png', data: image }+ ]+ }+ ]+ })++ return {+ analysis: result.text,+ inspectedNodeIds: nodeIds,+ image: {+ width: Math.ceil(width * scale),+ height: Math.ceil(height * scale)+ }+ }+}++export function createVisualInspectionTool(store: EditorStore) {+ return tool({+ description:+ 'Render the current selection or specified nodes and ask the isolated Vision model to inspect their visual appearance. Returns text findings only; the image is not added to the Design chat history. Use only for explicit visual review or rendering diagnosis.',+ inputSchema: valibotSchema(+ v.object({+ ids: v.optional(v.array(v.string())),+ question: v.optional(v.string()),+ maxEdge: v.optional(v.pipe(v.number(), v.minValue(64), v.maxValue(MAX_VISION_MAX_EDGE)))+ })+ ),+ execute: (request) => inspectRenderedDesign(store, request)+ })+}diff --git a/tests/engine/app/ai/vision.test.ts b/tests/engine/app/ai/vision.test.tsnew file mode 100644index 000000000..b4db6ff22--- /dev/null+++ b/tests/engine/app/ai/vision.test.ts@@ -0,0 +1,68 @@+import { describe, expect, test } from 'bun:test'++import type { LanguageModel } from 'ai'++import { SceneGraph } from '@open-pencil/scene-graph'++import {+ boundedImageScale,+ inspectRenderedDesign,+ type VisualInspectionDependencies+} from '@/app/ai/tools/vision'+import type { EditorStore } from '@/app/editor/session/create'++describe('isolated visual inspection', () => {+ test('bounds renders and forwards only textual findings to the caller', async () => {+ const graph = new SceneGraph()+ const page = graph.getPages()[0]+ const frame = graph.createNode('FRAME', page.id, { width: 2560, height: 1600 })+ const rendered: Array<{ ids: string[]; scale: number }> = []+ const inspections: unknown[] = []+ const store = {+ graph,+ state: { currentPageId: page.id, selectedIds: new Set([frame.id]) },+ renderExportImage: async (ids: string[], scale: number) => {+ rendered.push({ ids, scale })+ return new Uint8Array([1, 2, 3])+ }+ } as EditorStore+ const dependencies: VisualInspectionDependencies = {+ createRuntime: async () =>+ ({+ kind: 'direct',+ model: {} as LanguageModel,+ role: {+ requestedRole: 'vision',+ profile: { maxOutputTokens: 8000 },+ connection: {}+ }+ }) as never,+ inspect: async (options) => {+ inspections.push(options)+ return { text: 'Align the button with the form edge.' } as never+ }+ }++ const result = await inspectRenderedDesign(store, {}, dependencies)++ expect(rendered).toEqual([{ ids: [frame.id], scale: 0.5 }])+ expect(result).toEqual({+ analysis: 'Align the button with the form edge.',+ inspectedNodeIds: [frame.id],+ image: { width: 1280, height: 800 }+ })+ const request = inspections[0] as {+ maxOutputTokens: number+ messages: Array<{ content: Array<{ type: string; data?: Uint8Array }> }>+ }+ expect(request.maxOutputTokens).toBe(1200)+ expect(request.messages[0]?.content[1]?.type).toBe('file')+ expect(request.messages[0]?.content[1]?.data).toEqual(new Uint8Array([1, 2, 3]))+ expect(result).not.toHaveProperty('base64')+ })++ test('never upscales bounded images', () => {+ expect(boundedImageScale(640, 400)).toBe(1)+ expect(boundedImageScale(2560, 1600)).toBe(0.5)+ })+})diff --git a/tests/engine/tools/export-image.test.ts b/tests/engine/tools/export-image.test.tsnew file mode 100644index 000000000..d07be795e--- /dev/null+++ b/tests/engine/tools/export-image.test.ts@@ -0,0 +1,59 @@+import { describe, expect, test } from 'bun:test'++import { getTool, setupToolTest } from '#tests/helpers/tools'++type ExportImageResult = {+ width?: unknown+ height?: unknown+ scale?: unknown+}++function exportImageResult(value: unknown): ExportImageResult {+ return typeof value === 'object' && value !== null ? value : {}+}++describe('export_image tool', () => {+ test('bounds model-facing image output by its longest edge without upscaling', async () => {+ const { figma } = setupToolTest()+ const frame = figma.createFrame()+ frame.resize(2560, 1600)+ const calls: Array<{ scale?: number; format?: string }> = []+ figma.exportImage = async (_ids, options) => {+ calls.push(options)+ return new Uint8Array([1, 2, 3])+ }++ const result = exportImageResult(+ await getTool('export_image').execute(figma, {+ ids: [frame.id],+ format: 'PNG',+ scale: 2,+ maxEdge: 1280+ })+ )++ expect(calls).toEqual([{ scale: 0.5, format: 'PNG' }])+ expect(result.width).toBe(1280)+ expect(result.height).toBe(800)+ expect(result.scale).toBe(0.5)+ })++ test('keeps the requested scale when the image already fits', async () => {+ const { figma } = setupToolTest()+ const frame = figma.createFrame()+ frame.resize(640, 400)+ const calls: Array<{ scale?: number; format?: string }> = []+ figma.exportImage = async (_ids, options) => {+ calls.push(options)+ return new Uint8Array([1])+ }++ await getTool('export_image').execute(figma, {+ ids: [frame.id],+ scale: 1,+ maxEdge: 1280+ })++ expect(calls[0]?.scale).toBe(1)+ })+})