Skip to content

armgit / kiln

Public Imported from GitHub
0

Build and revise procedural 3D assets with your coding agent. Local MCP server, CLI and TypeScript engine with rendered review, editable source and GLB export.

feat(agent): add host visual observation port

Matt Kissingerc55f67a8c45c
diff --git a/src/agent/index.ts b/src/agent/index.ts
index cff1ccd2..2162404c 100644
--- a/src/agent/index.ts
+++ b/src/agent/index.ts
@@ -27,7 +27,12 @@ export { resolveToolSurface, buildAgentTools } from './surface';
export type { KilnToolSurface, RefineMode } from './surface';
export { DEFAULT_INLOOP_VIEW_RENDER_TIMEOUT_MS } from '../tools/registry';
-export type { InLoopViewRender } from '../tools/registry';
+export type {
+ InLoopViewRender,
+ RenderObservationInput,
+ RenderObservationPort,
+ RenderObservationValue,
+} from '../tools/registry';
export {
generateKilnAsset,
diff --git a/src/agent/run-surface.test.ts b/src/agent/run-surface.test.ts
index 55644381..2a56ea92 100644
--- a/src/agent/run-surface.test.ts
+++ b/src/agent/run-surface.test.ts
@@ -11,7 +11,7 @@ import { describe, expect, test } from 'bun:test';
import { buildAgentTools, resolveToolSurface, type ToolBuildOptions } from './surface';
import type { SubmitSink, EditSink, UnifiedSink } from './tools';
-import type { InLoopViewRender, KilnToolContext } from '../tools/registry';
+import type { InLoopViewRender, KilnToolContext, RenderObservationInput } from '../tools/registry';
function freshSinks() {
return {
@@ -125,6 +125,24 @@ describe('buildAgentTools', () => {
expect(events[0]?.degraded).toBe(true);
expect(events[0]?.degradedReason).toContain('timed out after 5ms');
});
+
+ test('threads a host visual observer into unified kiln_render without returning pixels', async () => {
+ const observed: RenderObservationInput[] = [];
+ const context: KilnToolContext = {
+ renderObservationPort: async (input) => {
+ observed.push(input);
+ return { schemaVersion: 1, verdict: 'ready', findings: [] };
+ },
+ };
+ const tools = buildAgentTools('unified', context, freshSinks());
+
+ await findTool(tools, 'kiln_draft').invoke({ code: METAL_CODE });
+ const rendered = (await findTool(tools, 'kiln_render').invoke({})) as unknown[];
+
+ expect(rendered).toHaveLength(1);
+ expect(observed).toHaveLength(1);
+ expect(observed[0]?.toolName).toBe('kiln_render');
+ });
});
describe('resolveToolSurface', () => {
diff --git a/src/agent/run.ts b/src/agent/run.ts
index 2a6f198b..e56078f2 100644
--- a/src/agent/run.ts
+++ b/src/agent/run.ts
@@ -89,7 +89,10 @@ function imageFormatFromMime(mime: string): 'png' | 'jpeg' | 'gif' | 'webp' {
}
export interface RunKilnAgentOptions
- extends Pick<KilnToolContext, 'viewRenderPort' | 'viewRenderTimeoutMs' | 'onViewsRendered'> {
+ extends Pick<
+ KilnToolContext,
+ 'viewRenderPort' | 'viewRenderTimeoutMs' | 'onViewsRendered' | 'renderObservationPort'
+ > {
/** A constructed Strands `Model` instance (provider-agnostic). */
model: unknown;
/** Natural-language description of the asset to build. */
diff --git a/src/agent/surface.ts b/src/agent/surface.ts
index e530d1b4..8a30b468 100644
--- a/src/agent/surface.ts
+++ b/src/agent/surface.ts
@@ -76,6 +76,7 @@ export function buildAgentTools(
? { viewRenderTimeoutMs: opts.viewRenderTimeoutMs }
: {}),
...(opts.onViewsRendered ? { onViewsRendered: opts.onViewsRendered } : {}),
+ ...(opts.renderObservationPort ? { renderObservationPort: opts.renderObservationPort } : {}),
};
if (surface === 'unified') {
return makeKilnUnifiedTools({
diff --git a/src/agent/tools-media.test.ts b/src/agent/tools-media.test.ts
index 1709142b..380efc3d 100644
--- a/src/agent/tools-media.test.ts
+++ b/src/agent/tools-media.test.ts
@@ -51,6 +51,58 @@ describe('makeKilnTools media handling', () => {
expect('pngBase64' in json).toBe(false);
});
+ test('a host observer replaces screenshot pixels with a structured visual observation', async () => {
+ const calls: Array<{ toolName: string; pngs: readonly Uint8Array[] }> = [];
+ const tools = makeKilnTools(
+ {},
+ {
+ renderObservationPort: async (input) => {
+ calls.push({ toolName: input.toolName, pngs: input.pngs });
+ return {
+ schemaVersion: 1,
+ verdict: 'continue',
+ findings: [{ criterionId: 'silhouette', summary: 'The box is visible.' }],
+ };
+ },
+ },
+ );
+ const out = (await findTool(tools, 'kiln_screenshot').invoke({ code: BOX_CODE })) as unknown[];
+
+ expect(out).toHaveLength(1);
+ expect(out[0]).toBeInstanceOf(JsonBlock);
+ const json = (out[0] as JsonBlock).json as Record<string, unknown>;
+ expect(json['ok']).toBe(true);
+ expect(json['visualObservation']).toEqual({
+ ok: true,
+ value: {
+ schemaVersion: 1,
+ verdict: 'continue',
+ findings: [{ criterionId: 'silhouette', summary: 'The box is visible.' }],
+ },
+ });
+ expect(calls).toHaveLength(1);
+ expect(calls[0]?.toolName).toBe('kiln_screenshot');
+ expect(calls[0]?.pngs).toHaveLength(1);
+ });
+
+ test('an observer failure degrades to explicit unavailable JSON and never leaks pixels', async () => {
+ const tools = makeKilnTools(
+ {},
+ {
+ renderObservationPort: async () => {
+ throw new Error('provider detail must stay host-only');
+ },
+ },
+ );
+ const out = (await findTool(tools, 'kiln_screenshot').invoke({ code: BOX_CODE })) as unknown[];
+
+ expect(out).toHaveLength(1);
+ expect(out[0]).toBeInstanceOf(JsonBlock);
+ const json = (out[0] as JsonBlock).json as Record<string, unknown>;
+ expect(json['visualObservation']).toEqual({ ok: false, reason: 'observer-unavailable' });
+ expect(JSON.stringify(json)).not.toContain('provider detail');
+ });
+
test('kiln_screenshot on broken code falls back to the plain JSON error output', async () => {
const tools = makeKilnTools({});
const out = (await findTool(tools, 'kiln_screenshot').invoke({ code: 'nope(' })) as {
diff --git a/src/agent/tools.ts b/src/agent/tools.ts
index 08cf6113..86da0ae2 100644
--- a/src/agent/tools.ts
+++ b/src/agent/tools.ts
@@ -62,9 +62,53 @@ const submitInput = z.object({
* tool-result coercion passes block arrays through as-is — so it is cast past
* JSONValue.
*/
-function toCallbackResult(def: KilnToolDef, output: unknown): JSONValue {
+function withVisualObservation(json: unknown, visualObservation: JSONValue): JSONValue {
+ if (json && typeof json === 'object' && !Array.isArray(json)) {
+ return { ...(json as Record<string, JSONValue>), visualObservation } as JSONValue;
+ }
+ return { result: json as JSONValue, visualObservation } as JSONValue;
+}
+
+async function observedMediaResult(
+ def: KilnToolDef,
+ pngs: readonly Uint8Array[],
+ json: unknown,
+ context: KilnToolContext,
+): Promise<JSONValue> {
+ try {
+ const value = await context.renderObservationPort!({
+ toolName: def.name,
+ pngs,
+ json,
+ ...(context.intent ? { intent: context.intent } : {}),
+ });
+ return [
+ new JsonBlock({
+ json: withVisualObservation(json, { ok: true, value } as JSONValue),
+ }),
+ ] as unknown as JSONValue;
+ } catch {
+ return [
+ new JsonBlock({
+ json: withVisualObservation(json, {
+ ok: false,
+ reason: 'observer-unavailable',
+ } as JSONValue),
+ }),
+ ] as unknown as JSONValue;
+ }
+}
+
+async function toCallbackResult(
+ def: KilnToolDef,
+ output: unknown,
+ context: KilnToolContext = {},
+): Promise<JSONValue> {
const multi = def.mediaMulti?.(output);
if (multi) {
+ if (context.renderObservationPort) {
+ return observedMediaResult(def, multi.pngs, multi.json, context);
+ }
return [
...multi.pngs.map((png) => new ImageBlock({ format: 'png', source: { bytes: png } })),
new JsonBlock({ json: multi.json as JSONValue }),
@@ -72,6 +116,9 @@ function toCallbackResult(def: KilnToolDef, output: unknown): JSONValue {
}
const media = def.media?.(output);
if (media) {
+ if (context.renderObservationPort) {
+ return observedMediaResult(def, [media.png], media.json, context);
+ }
return [
new ImageBlock({ format: 'png', source: { bytes: media.png } }),
new JsonBlock({ json: media.json as JSONValue }),
@@ -97,7 +144,11 @@ const animationBufferInput = z.object({
/** Build a buffer-aware kiln_screenshot_animation tool that runs against `getCode()`
* (the working buffer) instead of a `code` argument. Same behavior as the registry
* def otherwise; image transports attach the frame(s) via toCallbackResult. */
-function makeBufferAnimationTool(getCode: () => string, def: KilnToolDef): Tool {
+function makeBufferAnimationTool(
+ getCode: () => string,
+ def: KilnToolDef,
+ context: KilnToolContext = {},
+): Tool {
return tool({
name: def.name,
description: `${def.description} Operates on your current working buffer (omit code).`,
@@ -112,17 +163,18 @@ function makeBufferAnimationTool(getCode: () => string, def: KilnToolDef): Tool
...(i.camera ? { camera: i.camera } : {}),
...(i.perFrame ? { perFrame: true } : {}),
}),
+ context,
);
},
});
}
-function toStrandsTool(def: KilnToolDef): Tool {
+function toStrandsTool(def: KilnToolDef, context: KilnToolContext = {}): Tool {
return tool({
name: def.name,
description: def.description,
inputSchema: def.inputSchema as z.ZodType,
- callback: async (input) => toCallbackResult(def, await def.run(input)),
+ callback: async (input) => toCallbackResult(def, await def.run(input), context),
});
}
@@ -134,10 +186,10 @@ function toStrandsTool(def: KilnToolDef): Tool {
* @param sink - Receives the submitted final code. Read `sink.code` after invoke.
*/
export function makeKilnTools(sink: SubmitSink, context: KilnToolContext = {}): Tool[] {
- const kilnTools = createKilnToolRegistry(context).map(toStrandsTool);
+ const kilnTools = createKilnToolRegistry(context).map((def) => toStrandsTool(def, context));
// kiln_screenshot_animation rides alongside the four registry tools (it takes a
// `code` arg like them) so a from-scratch animated build can SEE its motion.
- const animationTool = toStrandsTool(createKilnScreenshotAnimationDef(context));
+ const animationTool = toStrandsTool(createKilnScreenshotAnimationDef(context), context);
const submitTool: Tool = tool({
name: KILN_SUBMIT_TOOL_NAME,
description:
@@ -345,7 +397,7 @@ export function makeKilnEditTools(
const renderDef = find('kiln_render');
const screenshotDef = find('kiln_screenshot');
- const listTool = toStrandsTool(find('kiln_list_primitives'));
+ const listTool = toStrandsTool(find('kiln_list_primitives'), opts);
const viewTool: Tool = tool({
name: 'kiln_view',
@@ -399,12 +451,13 @@ export function makeKilnEditTools(
toCallbackResult(
screenshotDef,
await screenshotDef.run({ code: (input as { code?: string }).code ?? buffer.code }),
+ opts,
),
});
// Buffer-aware motion view: a refine that touches an animated character can SEE
// the clip move and confirm the edit didn't break it.
- const animationTool = makeBufferAnimationTool(() => buffer.code, animationDef);
+ const animationTool = makeBufferAnimationTool(() => buffer.code, animationDef, opts);
const submitTool: Tool = tool({
name: KILN_SUBMIT_TOOL_NAME,
@@ -615,7 +668,7 @@ export function makeKilnUnifiedTools(
}
}
}
- return toCallbackResult(renderViewsDef, out);
+ return toCallbackResult(renderViewsDef, out, opts);
},
});
@@ -644,13 +697,13 @@ export function makeKilnUnifiedTools(
...(i.zoom !== undefined ? { zoom: i.zoom } : {}),
...(i.isolate !== undefined ? { isolate: i.isolate } : {}),
});
- return toCallbackResult(inspectDef, out);
+ return toCallbackResult(inspectDef, out, opts);
},
});
// Buffer-aware motion view: after drafting/editing an animated character, SEE a
// clip move (sideways walk, reverse knee, backward swing, item not tracking).
- const animationTool = makeBufferAnimationTool(() => buffer.code, animationDef);
+ const animationTool = makeBufferAnimationTool(() => buffer.code, animationDef, opts);
// Buffer-aware interior view: after drafting/editing a BUILDING, SEE inside it
// (roof off — open floor, real doorway gap, fixtures grounded, nothing buried).
@@ -672,7 +725,7 @@ export function makeKilnUnifiedTools(
code: buffer.code,
...(n ? { nodeName: n } : {}),
});
- return toCallbackResult(interiorDef, out);
+ return toCallbackResult(interiorDef, out, opts);
},
});
diff --git a/src/tools/registry.ts b/src/tools/registry.ts
index c379031a..89f4dac0 100644
--- a/src/tools/registry.ts
+++ b/src/tools/registry.ts
@@ -98,8 +98,46 @@ export interface KilnToolContext {
* Never throws into the render path: a host callback that throws is swallowed.
*/
onViewsRendered?(event: InLoopViewRender): void;
+ /**
+ * Optional host-owned visual observer for authors that cannot consume image
+ * input. When present, media-bearing tool results are sent here out of band
+ * and the author receives only the returned structured observation plus the
+ * ordinary JSON metrics. Raw pixels never enter the author transcript.
+ *
+ * The engine does not construct a model or network client. Studio owns the
+ * observer model, prompt, schema validation, deadline, accounting, and retry
+ * policy. A port failure degrades to an explicit unavailable marker rather
+ * than leaking the image or failing the render tool.
+ */
+ renderObservationPort?: RenderObservationPort;
}
+/** JSON-safe value accepted from a host visual observer. */
+export type RenderObservationValue =
+ | null
+ | boolean
+ | number
+ | string
+ | RenderObservationValue[]
+ | { [key: string]: RenderObservationValue };
+
+/** Evidence passed to a host visual observer, never to the author model. */
+export interface RenderObservationInput {
+ /** Model-facing tool that produced the evidence. */
+ toolName: string;
+ /** One grid PNG for ordinary render/inspect, or multiple animation frames. */
+ pngs: readonly Uint8Array[];
+ /** The same base64-free metrics JSON a direct-vision author would receive. */
+ json: unknown;
+ /** Trusted request intent, when the host supplied one. */
+ intent?: AssetIntentV1;
+}
+
+/** Host-injected, model-free engine seam for a bounded VLM observer. */
+export type RenderObservationPort = (
+ input: RenderObservationInput,
+) => Promise<RenderObservationValue>;
+
/** One in-loop grid, and who drew it. Counted by the host, never shown to the model. */
export interface InLoopViewRender {
/** The port's own renderer id when the GPU drew it; `cpu-raster` when it did not. */