Build and revise procedural 3D assets with your coding agent. Local MCP server, CLI and TypeScript engine with rendered review, editable source and GLB export.
JavaScript 3d3d-assets3d-modelingagent-skillsagent-toolsai-agentsasset-pipelinecligame-assetsglbgltfmcpmcp-servermodel-context-protocolprocedural-generationprocedural-modelingtext-to-3dthreejstypescript
feat(agent): add host visual observation port
Matt Kissingerc55f67a8c45c
diff --git a/src/agent/index.ts b/src/agent/index.tsindex cff1ccd2..2162404c 100644--- a/src/agent/index.ts+++ b/src/agent/index.ts@@ -27,7 +27,12 @@ export { resolveToolSurface, buildAgentTools } from './surface';export type { KilnToolSurface, RefineMode } from './surface';export { DEFAULT_INLOOP_VIEW_RENDER_TIMEOUT_MS } from '../tools/registry';-export type { InLoopViewRender } from '../tools/registry';+export type {+ InLoopViewRender,+ RenderObservationInput,+ RenderObservationPort,+ RenderObservationValue,+} from '../tools/registry';export {generateKilnAsset,diff --git a/src/agent/run-surface.test.ts b/src/agent/run-surface.test.tsindex 55644381..2a56ea92 100644--- a/src/agent/run-surface.test.ts+++ b/src/agent/run-surface.test.ts@@ -11,7 +11,7 @@ import { describe, expect, test } from 'bun:test';import { buildAgentTools, resolveToolSurface, type ToolBuildOptions } from './surface';import type { SubmitSink, EditSink, UnifiedSink } from './tools';-import type { InLoopViewRender, KilnToolContext } from '../tools/registry';+import type { InLoopViewRender, KilnToolContext, RenderObservationInput } from '../tools/registry';function freshSinks() {return {@@ -125,6 +125,24 @@ describe('buildAgentTools', () => {expect(events[0]?.degraded).toBe(true);expect(events[0]?.degradedReason).toContain('timed out after 5ms');});++ test('threads a host visual observer into unified kiln_render without returning pixels', async () => {+ const observed: RenderObservationInput[] = [];+ const context: KilnToolContext = {+ renderObservationPort: async (input) => {+ observed.push(input);+ return { schemaVersion: 1, verdict: 'ready', findings: [] };+ },+ };+ const tools = buildAgentTools('unified', context, freshSinks());++ await findTool(tools, 'kiln_draft').invoke({ code: METAL_CODE });+ const rendered = (await findTool(tools, 'kiln_render').invoke({})) as unknown[];++ expect(rendered).toHaveLength(1);+ expect(observed).toHaveLength(1);+ expect(observed[0]?.toolName).toBe('kiln_render');+ });});describe('resolveToolSurface', () => {diff --git a/src/agent/run.ts b/src/agent/run.tsindex 2a6f198b..e56078f2 100644--- a/src/agent/run.ts+++ b/src/agent/run.ts@@ -89,7 +89,10 @@ function imageFormatFromMime(mime: string): 'png' | 'jpeg' | 'gif' | 'webp' {}export interface RunKilnAgentOptions- extends Pick<KilnToolContext, 'viewRenderPort' | 'viewRenderTimeoutMs' | 'onViewsRendered'> {+ extends Pick<+ KilnToolContext,+ 'viewRenderPort' | 'viewRenderTimeoutMs' | 'onViewsRendered' | 'renderObservationPort'+ > {/** A constructed Strands `Model` instance (provider-agnostic). */model: unknown;/** Natural-language description of the asset to build. */diff --git a/src/agent/surface.ts b/src/agent/surface.tsindex e530d1b4..8a30b468 100644--- a/src/agent/surface.ts+++ b/src/agent/surface.ts@@ -76,6 +76,7 @@ export function buildAgentTools(? { viewRenderTimeoutMs: opts.viewRenderTimeoutMs }: {}),...(opts.onViewsRendered ? { onViewsRendered: opts.onViewsRendered } : {}),+ ...(opts.renderObservationPort ? { renderObservationPort: opts.renderObservationPort } : {}),};if (surface === 'unified') {return makeKilnUnifiedTools({diff --git a/src/agent/tools-media.test.ts b/src/agent/tools-media.test.tsindex 1709142b..380efc3d 100644--- a/src/agent/tools-media.test.ts+++ b/src/agent/tools-media.test.ts@@ -51,6 +51,58 @@ describe('makeKilnTools media handling', () => {expect('pngBase64' in json).toBe(false);});+ test('a host observer replaces screenshot pixels with a structured visual observation', async () => {+ const calls: Array<{ toolName: string; pngs: readonly Uint8Array[] }> = [];+ const tools = makeKilnTools(+ {},+ {+ renderObservationPort: async (input) => {+ calls.push({ toolName: input.toolName, pngs: input.pngs });+ return {+ schemaVersion: 1,+ verdict: 'continue',+ findings: [{ criterionId: 'silhouette', summary: 'The box is visible.' }],+ };+ },+ },+ );+ const out = (await findTool(tools, 'kiln_screenshot').invoke({ code: BOX_CODE })) as unknown[];++ expect(out).toHaveLength(1);+ expect(out[0]).toBeInstanceOf(JsonBlock);+ const json = (out[0] as JsonBlock).json as Record<string, unknown>;+ expect(json['ok']).toBe(true);+ expect(json['visualObservation']).toEqual({+ ok: true,+ value: {+ schemaVersion: 1,+ verdict: 'continue',+ findings: [{ criterionId: 'silhouette', summary: 'The box is visible.' }],+ },+ });+ expect(calls).toHaveLength(1);+ expect(calls[0]?.toolName).toBe('kiln_screenshot');+ expect(calls[0]?.pngs).toHaveLength(1);+ });++ test('an observer failure degrades to explicit unavailable JSON and never leaks pixels', async () => {+ const tools = makeKilnTools(+ {},+ {+ renderObservationPort: async () => {+ throw new Error('provider detail must stay host-only');+ },+ },+ );+ const out = (await findTool(tools, 'kiln_screenshot').invoke({ code: BOX_CODE })) as unknown[];++ expect(out).toHaveLength(1);+ expect(out[0]).toBeInstanceOf(JsonBlock);+ const json = (out[0] as JsonBlock).json as Record<string, unknown>;+ expect(json['visualObservation']).toEqual({ ok: false, reason: 'observer-unavailable' });+ expect(JSON.stringify(json)).not.toContain('provider detail');+ });+test('kiln_screenshot on broken code falls back to the plain JSON error output', async () => {const tools = makeKilnTools({});const out = (await findTool(tools, 'kiln_screenshot').invoke({ code: 'nope(' })) as {diff --git a/src/agent/tools.ts b/src/agent/tools.tsindex 08cf6113..86da0ae2 100644--- a/src/agent/tools.ts+++ b/src/agent/tools.ts@@ -62,9 +62,53 @@ const submitInput = z.object({* tool-result coercion passes block arrays through as-is — so it is cast past* JSONValue.*/-function toCallbackResult(def: KilnToolDef, output: unknown): JSONValue {+function withVisualObservation(json: unknown, visualObservation: JSONValue): JSONValue {+ if (json && typeof json === 'object' && !Array.isArray(json)) {+ return { ...(json as Record<string, JSONValue>), visualObservation } as JSONValue;+ }+ return { result: json as JSONValue, visualObservation } as JSONValue;+}++async function observedMediaResult(+ def: KilnToolDef,+ pngs: readonly Uint8Array[],+ json: unknown,+ context: KilnToolContext,+): Promise<JSONValue> {+ try {+ const value = await context.renderObservationPort!({+ toolName: def.name,+ pngs,+ json,+ ...(context.intent ? { intent: context.intent } : {}),+ });+ return [+ new JsonBlock({+ json: withVisualObservation(json, { ok: true, value } as JSONValue),+ }),+ ] as unknown as JSONValue;+ } catch {+ return [+ new JsonBlock({+ json: withVisualObservation(json, {+ ok: false,+ reason: 'observer-unavailable',+ } as JSONValue),+ }),+ ] as unknown as JSONValue;+ }+}++async function toCallbackResult(+ def: KilnToolDef,+ output: unknown,+ context: KilnToolContext = {},+): Promise<JSONValue> {const multi = def.mediaMulti?.(output);if (multi) {+ if (context.renderObservationPort) {+ return observedMediaResult(def, multi.pngs, multi.json, context);+ }return [...multi.pngs.map((png) => new ImageBlock({ format: 'png', source: { bytes: png } })),new JsonBlock({ json: multi.json as JSONValue }),@@ -72,6 +116,9 @@ function toCallbackResult(def: KilnToolDef, output: unknown): JSONValue {}const media = def.media?.(output);if (media) {+ if (context.renderObservationPort) {+ return observedMediaResult(def, [media.png], media.json, context);+ }return [new ImageBlock({ format: 'png', source: { bytes: media.png } }),new JsonBlock({ json: media.json as JSONValue }),@@ -97,7 +144,11 @@ const animationBufferInput = z.object({/** Build a buffer-aware kiln_screenshot_animation tool that runs against `getCode()`* (the working buffer) instead of a `code` argument. Same behavior as the registry* def otherwise; image transports attach the frame(s) via toCallbackResult. */-function makeBufferAnimationTool(getCode: () => string, def: KilnToolDef): Tool {+function makeBufferAnimationTool(+ getCode: () => string,+ def: KilnToolDef,+ context: KilnToolContext = {},+): Tool {return tool({name: def.name,description: `${def.description} Operates on your current working buffer (omit code).`,@@ -112,17 +163,18 @@ function makeBufferAnimationTool(getCode: () => string, def: KilnToolDef): Tool...(i.camera ? { camera: i.camera } : {}),...(i.perFrame ? { perFrame: true } : {}),}),+ context,);},});}-function toStrandsTool(def: KilnToolDef): Tool {+function toStrandsTool(def: KilnToolDef, context: KilnToolContext = {}): Tool {return tool({name: def.name,description: def.description,inputSchema: def.inputSchema as z.ZodType,- callback: async (input) => toCallbackResult(def, await def.run(input)),+ callback: async (input) => toCallbackResult(def, await def.run(input), context),});}@@ -134,10 +186,10 @@ function toStrandsTool(def: KilnToolDef): Tool {* @param sink - Receives the submitted final code. Read `sink.code` after invoke.*/export function makeKilnTools(sink: SubmitSink, context: KilnToolContext = {}): Tool[] {- const kilnTools = createKilnToolRegistry(context).map(toStrandsTool);+ const kilnTools = createKilnToolRegistry(context).map((def) => toStrandsTool(def, context));// kiln_screenshot_animation rides alongside the four registry tools (it takes a// `code` arg like them) so a from-scratch animated build can SEE its motion.- const animationTool = toStrandsTool(createKilnScreenshotAnimationDef(context));+ const animationTool = toStrandsTool(createKilnScreenshotAnimationDef(context), context);const submitTool: Tool = tool({name: KILN_SUBMIT_TOOL_NAME,description:@@ -345,7 +397,7 @@ export function makeKilnEditTools(const renderDef = find('kiln_render');const screenshotDef = find('kiln_screenshot');- const listTool = toStrandsTool(find('kiln_list_primitives'));+ const listTool = toStrandsTool(find('kiln_list_primitives'), opts);const viewTool: Tool = tool({name: 'kiln_view',@@ -399,12 +451,13 @@ export function makeKilnEditTools(toCallbackResult(screenshotDef,await screenshotDef.run({ code: (input as { code?: string }).code ?? buffer.code }),+ opts,),});// Buffer-aware motion view: a refine that touches an animated character can SEE// the clip move and confirm the edit didn't break it.- const animationTool = makeBufferAnimationTool(() => buffer.code, animationDef);+ const animationTool = makeBufferAnimationTool(() => buffer.code, animationDef, opts);const submitTool: Tool = tool({name: KILN_SUBMIT_TOOL_NAME,@@ -615,7 +668,7 @@ export function makeKilnUnifiedTools(}}}- return toCallbackResult(renderViewsDef, out);+ return toCallbackResult(renderViewsDef, out, opts);},});@@ -644,13 +697,13 @@ export function makeKilnUnifiedTools(...(i.zoom !== undefined ? { zoom: i.zoom } : {}),...(i.isolate !== undefined ? { isolate: i.isolate } : {}),});- return toCallbackResult(inspectDef, out);+ return toCallbackResult(inspectDef, out, opts);},});// Buffer-aware motion view: after drafting/editing an animated character, SEE a// clip move (sideways walk, reverse knee, backward swing, item not tracking).- const animationTool = makeBufferAnimationTool(() => buffer.code, animationDef);+ const animationTool = makeBufferAnimationTool(() => buffer.code, animationDef, opts);// Buffer-aware interior view: after drafting/editing a BUILDING, SEE inside it// (roof off — open floor, real doorway gap, fixtures grounded, nothing buried).@@ -672,7 +725,7 @@ export function makeKilnUnifiedTools(code: buffer.code,...(n ? { nodeName: n } : {}),});- return toCallbackResult(interiorDef, out);+ return toCallbackResult(interiorDef, out, opts);},});diff --git a/src/tools/registry.ts b/src/tools/registry.tsindex c379031a..89f4dac0 100644--- a/src/tools/registry.ts+++ b/src/tools/registry.ts@@ -98,8 +98,46 @@ export interface KilnToolContext {* Never throws into the render path: a host callback that throws is swallowed.*/onViewsRendered?(event: InLoopViewRender): void;+ /**+ * Optional host-owned visual observer for authors that cannot consume image+ * input. When present, media-bearing tool results are sent here out of band+ * and the author receives only the returned structured observation plus the+ * ordinary JSON metrics. Raw pixels never enter the author transcript.+ *+ * The engine does not construct a model or network client. Studio owns the+ * observer model, prompt, schema validation, deadline, accounting, and retry+ * policy. A port failure degrades to an explicit unavailable marker rather+ * than leaking the image or failing the render tool.+ */+ renderObservationPort?: RenderObservationPort;}+/** JSON-safe value accepted from a host visual observer. */+export type RenderObservationValue =+ | null+ | boolean+ | number+ | string+ | RenderObservationValue[]+ | { [key: string]: RenderObservationValue };++/** Evidence passed to a host visual observer, never to the author model. */+export interface RenderObservationInput {+ /** Model-facing tool that produced the evidence. */+ toolName: string;+ /** One grid PNG for ordinary render/inspect, or multiple animation frames. */+ pngs: readonly Uint8Array[];+ /** The same base64-free metrics JSON a direct-vision author would receive. */+ json: unknown;+ /** Trusted request intent, when the host supplied one. */+ intent?: AssetIntentV1;+}++/** Host-injected, model-free engine seam for a bounded VLM observer. */+export type RenderObservationPort = (+ input: RenderObservationInput,+) => Promise<RenderObservationValue>;+/** One in-loop grid, and who drew it. Counted by the host, never shown to the model. */export interface InLoopViewRender {/** The port's own renderer id when the GPU drew it; `cpu-raster` when it did not. */