Skip to content

armgit / kiln

Public Imported from GitHub
0

Build and revise procedural 3D assets with your coding agent. Local MCP server, CLI and TypeScript engine with rendered review, editable source and GLB export.

feat(agent): enforce generation-global call budget

Matt Kissingerf2dbc41bbb23
diff --git a/src/agent/call-budget.test.ts b/src/agent/call-budget.test.ts
new file mode 100644
index 00000000..b68accc5
--- /dev/null
+++ b/src/agent/call-budget.test.ts
@@ -0,0 +1,57 @@
+import { describe, expect, test } from 'bun:test';
+import {
+ DEFAULT_GENERATION_MODEL_CALL_LIMIT,
+ createGenerationCallBudget,
+ generationModelCallLimitFromEnv,
+} from './call-budget';
+
+describe('generation-global model-call budget', () => {
+ test('defaults to the existing aggregate 40-call ceiling', () => {
+ expect(createGenerationCallBudget().receipt()).toMatchObject({
+ limit: DEFAULT_GENERATION_MODEL_CALL_LIMIT,
+ consumed: 0,
+ remaining: 40,
+ exhausted: false,
+ });
+ });
+
+ test('attributes one shared allowance across author, observer, repair, retry, and fallback', () => {
+ const budget = createGenerationCallBudget(5);
+ for (const role of ['author', 'observer', 'repair', 'retry', 'fallback'] as const) {
+ expect(budget.tryConsume(role)).toBe(true);
+ }
+ expect(budget.tryConsume('repair')).toBe(false);
+ expect(budget.receipt()).toEqual({
+ limit: 5,
+ consumed: 5,
+ remaining: 0,
+ exhausted: true,
+ denied: 1,
+ byRole: { author: 1, observer: 1, repair: 1, retry: 1, fallback: 1 },
+ });
+ });
+
+ test('zero preserves the existing explicit unlimited configuration while retaining attribution', () => {
+ const budget = createGenerationCallBudget(0);
+ for (let i = 0; i < 50; i++) expect(budget.tryConsume('author')).toBe(true);
+ expect(budget.receipt()).toMatchObject({
+ limit: 0,
+ consumed: 50,
+ remaining: null,
+ exhausted: false,
+ denied: 0,
+ byRole: { author: 50 },
+ });
+ });
+
+ test('the generation-global env wins while the legacy step env remains a fallback', () => {
+ expect(
+ generationModelCallLimitFromEnv({
+ KILN_GENERATION_MAX_CALLS: '12',
+ KILN_AGENT_MAX_STEPS: '7',
+ }),
+ ).toBe(12);
+ expect(generationModelCallLimitFromEnv({ KILN_AGENT_MAX_STEPS: '7' })).toBe(7);
+ expect(generationModelCallLimitFromEnv({ KILN_GENERATION_MAX_CALLS: 'invalid' })).toBe(40);
+ });
+});
diff --git a/src/agent/call-budget.ts b/src/agent/call-budget.ts
new file mode 100644
index 00000000..38148440
--- /dev/null
+++ b/src/agent/call-budget.ts
@@ -0,0 +1,78 @@
+/** One deterministic model-call allowance shared by every paid role in a generation. */
+
+export const DEFAULT_GENERATION_MODEL_CALL_LIMIT = 40;
+
+export type GenerationModelCallRole = 'author' | 'observer' | 'repair' | 'retry' | 'fallback';
+
+export interface GenerationCallBudgetReceipt {
+ /** Aggregate ceiling. Zero preserves the legacy explicit unlimited setting. */
+ limit: number;
+ consumed: number;
+ /** Null only for an unlimited budget. */
+ remaining: number | null;
+ exhausted: boolean;
+ /** Dispatches refused before reaching a provider. */
+ denied: number;
+ byRole: Partial<Record<GenerationModelCallRole, number>>;
+}
+
+export interface GenerationCallBudget {
+ /** Atomically reserve one call for a role. False means do not dispatch. */
+ tryConsume(role: GenerationModelCallRole): boolean;
+ /** Immutable JSON-safe snapshot for metrics, persistence, and tests. */
+ receipt(): GenerationCallBudgetReceipt;
+}
+
+/** Invalid configuration must not turn the cost guard off. */
+export function resolveGenerationModelCallLimit(raw: unknown): number {
+ if (raw === undefined || raw === null || raw === '') return DEFAULT_GENERATION_MODEL_CALL_LIMIT;
+ const parsed = typeof raw === 'number' ? raw : Number(raw);
+ if (!Number.isFinite(parsed) || parsed < 0 || !Number.isInteger(parsed)) {
+ return DEFAULT_GENERATION_MODEL_CALL_LIMIT;
+ }
+ return parsed;
+}
+
+export function generationModelCallLimitFromEnv(
+ env: Readonly<{
+ KILN_GENERATION_MAX_CALLS?: string;
+ KILN_AGENT_MAX_STEPS?: string;
+ }> = process.env as {
+ KILN_GENERATION_MAX_CALLS?: string;
+ KILN_AGENT_MAX_STEPS?: string;
+ },
+): number {
+ return resolveGenerationModelCallLimit(env.KILN_GENERATION_MAX_CALLS ?? env.KILN_AGENT_MAX_STEPS);
+}
+
+export function createGenerationCallBudget(
+ limit = DEFAULT_GENERATION_MODEL_CALL_LIMIT,
+): GenerationCallBudget {
+ const resolvedLimit = resolveGenerationModelCallLimit(limit);
+ let consumed = 0;
+ let denied = 0;
+ const byRole: Partial<Record<GenerationModelCallRole, number>> = {};
+
+ return {
+ tryConsume(role) {
+ if (resolvedLimit > 0 && consumed >= resolvedLimit) {
+ denied += 1;
+ return false;
+ }
+ consumed += 1;
+ byRole[role] = (byRole[role] ?? 0) + 1;
+ return true;
+ },
+ receipt() {
+ const remaining = resolvedLimit === 0 ? null : Math.max(0, resolvedLimit - consumed);
+ return {
+ limit: resolvedLimit,
+ consumed,
+ remaining,
+ exhausted: resolvedLimit > 0 && remaining === 0,
+ denied,
+ byRole: { ...byRole },
+ };
+ },
+ };
+}
diff --git a/src/agent/hooks.test.ts b/src/agent/hooks.test.ts
index b361b4c6..4cb03af2 100644
--- a/src/agent/hooks.test.ts
+++ b/src/agent/hooks.test.ts
@@ -11,6 +11,7 @@ import {
AfterModelCallEvent,
} from '@strands-agents/sdk';
import { MetricsCollector } from './hooks';
+import { createGenerationCallBudget } from './call-budget';
type Cb = (event: unknown) => void;
@@ -86,6 +87,31 @@ describe('MetricsCollector', () => {
expect(m.readMetrics().steps).toBe(3);
});
+ test('two collectors debit one aggregate budget before author and repair dispatch', () => {
+ const budget = createGenerationCallBudget(3);
+ const author = new MetricsCollector(undefined, 40, budget, 'author');
+ const repair = new MetricsCollector(undefined, 40, budget, 'repair');
+ const authorAgent = fakeAgent();
+ const repairAgent = fakeAgent();
+ author.attach(authorAgent.agent as never);
+ repair.attach(repairAgent.agent as never);
+
+ expect(authorAgent.modelCall().cancelled).toBe(false);
+ expect(authorAgent.modelCall().cancelled).toBe(false);
+ expect(repairAgent.modelCall().cancelled).toBe(false);
+ const exhausted = repairAgent.modelCall();
+
+ expect(exhausted.cancelled).toBe(true);
+ expect(exhausted.cancelText).toContain('generation model-call budget');
+ expect(repair.wasCapped()).toBe(true);
+ expect(budget.receipt()).toMatchObject({
+ consumed: 3,
+ remaining: 0,
+ denied: 1,
+ byRole: { author: 2, repair: 1 },
+ });
+ });
+
test('records token usage from the agent result', () => {
const m = new MetricsCollector();
m.recordResultUsage({ inputTokens: 1200, outputTokens: 340 });
diff --git a/src/agent/hooks.ts b/src/agent/hooks.ts
index 354234ad..0f1418f5 100644
--- a/src/agent/hooks.ts
+++ b/src/agent/hooks.ts
@@ -17,6 +17,7 @@ import {
BeforeModelCallEvent,
AfterModelCallEvent,
} from '@strands-agents/sdk';
+import type { GenerationCallBudget, GenerationModelCallRole } from './call-budget';
/** Token-usage subset we surface. The cache fields mirror the Strands `Usage`
* type (dist/src/models/streaming.d.ts): the Anthropic adapter fills them from
@@ -81,15 +82,16 @@ export class MetricsCollector {
* @param onEvent - Optional live sink, fired as the loop runs (tool calls,
* model calls). Errors thrown by the sink are swallowed — a broken progress
* listener must never fail a generation.
- * @param maxSteps - Optional hard cap on model calls (agent-loop iterations).
- * When the loop reaches this many calls, the NEXT model call is cancelled
- * (SDK `BeforeModelCallEvent.cancel`), ending the loop with a clean stop
- * instead of an unbounded tool/edit/render loop. A pure cost backstop — set
- * well above normal runs (~10 calls); only catches runaways. 0/undefined = off.
+ * @param maxSteps - Legacy per-collector cap, retained for direct callers that
+ * do not inject a generation-global budget.
+ * @param generationCallBudget - Shared allowance across author, observer, and
+ * repair work. It supersedes maxSteps and is consumed before dispatch.
*/
constructor(
private readonly onEvent?: (event: KilnAgentEvent) => void,
private readonly maxSteps?: number,
+ private readonly generationCallBudget?: GenerationCallBudget,
+ private readonly modelCallRole: GenerationModelCallRole = 'author',
) {}
private emit(event: KilnAgentEvent): void {
@@ -111,12 +113,22 @@ export class MetricsCollector {
this.steps += 1;
this.emit({ type: 'model_call', step: this.steps });
});
- // Hard step cap: before a model call that would exceed maxSteps, cancel it.
+ // Hard cost cap: reserve from the shared budget before dispatch. Direct
+ // legacy collectors without one retain their local maxSteps behavior.
// The SDK ends the loop with stopReason 'endTurn' (no exception) — `capped`
// lets the caller turn that bounded stop into a clear failed generation.
const offCap =
- this.maxSteps && this.maxSteps > 0
+ this.generationCallBudget || (this.maxSteps && this.maxSteps > 0)
? agent.addHook(BeforeModelCallEvent, (event) => {
+ if (this.generationCallBudget) {
+ if (this.generationCallBudget.tryConsume(this.modelCallRole)) return;
+ this.capped = true;
+ const receipt = this.generationCallBudget.receipt();
+ event.cancel =
+ `kiln: generation model-call budget reached (${receipt.limit} aggregate calls) — ` +
+ 'aborted before provider dispatch to bound cost';
+ return;
+ }
if (this.steps >= this.maxSteps!) {
this.capped = true;
event.cancel = `kiln: agent step cap reached (${this.maxSteps} model calls) — aborted to bound cost`;
diff --git a/src/agent/index.ts b/src/agent/index.ts
index 2162404c..73560d7f 100644
--- a/src/agent/index.ts
+++ b/src/agent/index.ts
@@ -120,3 +120,15 @@ export type {
export { MetricsCollector } from './hooks';
export type { CollectedMetrics, AgentUsage, KilnAgentEvent } from './hooks';
+
+export {
+ createGenerationCallBudget,
+ generationModelCallLimitFromEnv,
+ resolveGenerationModelCallLimit,
+ DEFAULT_GENERATION_MODEL_CALL_LIMIT,
+} from './call-budget';
+export type {
+ GenerationCallBudget,
+ GenerationCallBudgetReceipt,
+ GenerationModelCallRole,
+} from './call-budget';
diff --git a/src/agent/run.ts b/src/agent/run.ts
index e56078f2..00bb6241 100644
--- a/src/agent/run.ts
+++ b/src/agent/run.ts
@@ -47,6 +47,12 @@ import {
} from './surface';
import { unifiedDiff } from './diff';
import { MetricsCollector, type AgentUsage, type KilnAgentEvent } from './hooks';
+import {
+ createGenerationCallBudget,
+ generationModelCallLimitFromEnv,
+ type GenerationCallBudget,
+ type GenerationModelCallRole,
+} from './call-budget';
import {
installRenderImageCompaction,
installGenerationCounters,
@@ -91,7 +97,11 @@ function imageFormatFromMime(mime: string): 'png' | 'jpeg' | 'gif' | 'webp' {
export interface RunKilnAgentOptions
extends Pick<
KilnToolContext,
- 'viewRenderPort' | 'viewRenderTimeoutMs' | 'onViewsRendered' | 'renderObservationPort'
+ | 'viewRenderPort'
+ | 'viewRenderTimeoutMs'
+ | 'onViewsRendered'
+ | 'renderObservationPort'
+ | 'generationCallBudget'
> {
/** A constructed Strands `Model` instance (provider-agnostic). */
model: unknown;
@@ -165,6 +175,8 @@ export interface RunKilnAgentOptions
* consolidate, and the refined program is kept only if its grade improves.
* 'off' skips the check (no extra model turn, no assessment bake). */
gradeRefine?: 'auto' | 'off';
+ /** Attribution role for this invocation within the shared generation budget. */
+ modelCallRole?: GenerationModelCallRole;
}
export interface RunKilnAgentResult {
@@ -218,14 +230,22 @@ function lastMessageText(message: Message | undefined): string | undefined {
* on `result.error` with whatever metrics were collected beforehand.
*/
export async function runKilnAgent(opts: RunKilnAgentOptions): Promise<RunKilnAgentResult> {
- // Hard agent-loop cap (runaway backstop, NOT a throttle): a stuck model could
+ // Hard generation-global cap (runaway backstop, NOT a throttle): a stuck model could
// otherwise loop draft/edit/render forever, re-feeding a growing transcript
// (input tokens scale ~linearly with step count). Default 40 — ~4x the normal
// run (~10 model calls) and clear of the slowest legit runs observed (~23); only
- // catches genuine infinite loops. Env-overridable (set 0 to disable, raise for
- // an unusually deep build).
- const maxSteps = Number(process.env['KILN_AGENT_MAX_STEPS'] ?? 40) || 0;
- const metrics = new MetricsCollector(opts.onEvent, maxSteps);
+ // catches genuine infinite loops. The same object can span author, observer,
+ // retry/fallback, and bounded repair calls. Env-overridable (set 0 to disable).
+ const configuredLimit = generationModelCallLimitFromEnv();
+ const generationCallBudget: GenerationCallBudget =
+ opts.generationCallBudget ?? createGenerationCallBudget(configuredLimit);
+ const maxSteps = generationCallBudget.receipt().limit;
+ const metrics = new MetricsCollector(
+ opts.onEvent,
+ maxSteps,
+ generationCallBudget,
+ opts.modelCallRole ?? 'author',
+ );
const surface = resolveToolSurface(opts.toolSurface);
const trustedCategory = opts.intent?.category ?? opts.category ?? 'prop';
const gradeAssessmentOptions = {
@@ -245,7 +265,7 @@ export async function runKilnAgent(opts: RunKilnAgentOptions): Promise<RunKilnAg
try {
const tools: Tool[] = buildAgentTools(
surface,
- { ...opts, category: trustedCategory },
+ { ...opts, category: trustedCategory, generationCallBudget },
{ sink, editSink, unifiedSink },
);
const allTools: unknown[] = opts.extraTools ? [...tools, ...opts.extraTools] : tools;
@@ -428,8 +448,8 @@ export async function runKilnAgent(opts: RunKilnAgentOptions): Promise<RunKilnAg
const trigger = {
mode: 'auto' as const,
report: assess.report,
- steps: metrics.readMetrics().steps,
- maxSteps,
+ steps: generationCallBudget.receipt().consumed,
+ maxSteps: generationCallBudget.receipt().limit,
};
if (assess.ok && assess.report && shouldGradeRefine(trigger)) {
const report = assess.report;
diff --git a/src/agent/surface.ts b/src/agent/surface.ts
index 8a30b468..31a2378d 100644
--- a/src/agent/surface.ts
+++ b/src/agent/surface.ts
@@ -77,6 +77,7 @@ export function buildAgentTools(
: {}),
...(opts.onViewsRendered ? { onViewsRendered: opts.onViewsRendered } : {}),
...(opts.renderObservationPort ? { renderObservationPort: opts.renderObservationPort } : {}),
+ ...(opts.generationCallBudget ? { generationCallBudget: opts.generationCallBudget } : {}),
};
if (surface === 'unified') {
return makeKilnUnifiedTools({
diff --git a/src/agent/tools-media.test.ts b/src/agent/tools-media.test.ts
index 380efc3d..f176c760 100644
--- a/src/agent/tools-media.test.ts
+++ b/src/agent/tools-media.test.ts
@@ -6,6 +6,7 @@ import { describe, expect, test } from 'bun:test';
import { ImageBlock, JsonBlock } from '@strands-agents/sdk';
import { makeKilnTools, makeKilnEditTools, type SubmitSink, type EditSink } from './tools';
+import { createGenerationCallBudget } from './call-budget';
const BOX_CODE = `
const meta = { name: 'test-box', category: 'prop' };
@@ -25,6 +26,28 @@ function findTool(tools: ReturnType<typeof makeKilnTools>, name: string) {
}
describe('makeKilnTools media handling', () => {
+ test('forwards the global budget so the observer port can debit actual dispatches', async () => {
+ const budget = createGenerationCallBudget(1);
+ expect(budget.tryConsume('author')).toBe(true);
+ let observerCalls = 0;
+ const tools = makeKilnTools(
+ {},
+ {
+ generationCallBudget: budget,
+ renderObservationPort: async (input) => {
+ observerCalls++;
+ expect(input.generationCallBudget).toBe(budget);
+ return { verdict: 'ready' };
+ },
+ },
+ );
+
+ const result = await findTool(tools, 'kiln_screenshot').invoke({ code: BOX_CODE });
+ expect(observerCalls).toBe(1);
+ expect(JSON.stringify(result)).toContain('ready');
+ expect(budget.receipt()).toMatchObject({ consumed: 1, denied: 0 });
+ });
+
test('exposes the four registry tools plus the animation view and kiln_submit', () => {
const sink: SubmitSink = {};
const tools = makeKilnTools(sink);
diff --git a/src/agent/tools.ts b/src/agent/tools.ts
index 86da0ae2..f3be992d 100644
--- a/src/agent/tools.ts
+++ b/src/agent/tools.ts
@@ -81,6 +81,9 @@ async function observedMediaResult(
pngs,
json,
...(context.intent ? { intent: context.intent } : {}),
+ ...(context.generationCallBudget
+ ? { generationCallBudget: context.generationCallBudget }
+ : {}),
});
return [
new JsonBlock({
diff --git a/src/tools/registry.ts b/src/tools/registry.ts
index 89f4dac0..6e0c8d13 100644
--- a/src/tools/registry.ts
+++ b/src/tools/registry.ts
@@ -28,6 +28,7 @@ import type { AssetQaReportV1 } from '../qa';
import type { PbrRenderPort } from '../composer/render-port';
import type { ViewGridResult } from '../views';
import { sceneNeedsPbrShading } from '../material-resources';
+import type { GenerationCallBudget } from '../agent/call-budget';
// =============================================================================
// Tool definition contract
@@ -110,6 +111,8 @@ export interface KilnToolContext {
* than leaking the image or failing the render tool.
*/
renderObservationPort?: RenderObservationPort;
+ /** Shared generation-global allowance, forwarded to the host observer port. */
+ generationCallBudget?: GenerationCallBudget;
}
/** JSON-safe value accepted from a host visual observer. */
@@ -131,6 +134,12 @@ export interface RenderObservationInput {
json: unknown;
/** Trusted request intent, when the host supplied one. */
intent?: AssetIntentV1;
+ /**
+ * Shared generation-global allowance. The host must debit role `observer`
+ * immediately before each actual provider dispatch; image preparation,
+ * sub-cap rejection, and other pre-dispatch failures must not consume it.
+ */
+ generationCallBudget?: GenerationCallBudget;
}
/** Host-injected, model-free engine seam for a bounded VLM observer. */