3D engine that AI agents drive
Add an opt-in MCP call log and a summary script (#746)
Nguyen Xuan An68563badfa05
diff --git a/changes/746.md b/changes/746.mdnew file mode 100644index 00000000..9c621d68--- /dev/null+++ b/changes/746.md@@ -0,0 +1 @@+- Set `SIXTYSEVENJS_MCP_LOG=<file>` to log each MCP tool call (tool, duration, ok, result size, never arguments), and run `scripts/summarize-mcp-log.mjs` for calls, error rate and median duration per tool (#746)diff --git a/docs/mcp.md b/docs/mcp.mdindex bb1bdc4f..2b9d45ad 100644--- a/docs/mcp.md+++ b/docs/mcp.md@@ -201,6 +201,17 @@ References must resolve after the complete batch. Removing a parent leaves its cReparent or remove those children before the parent removal command to avoid rejection.The [scene contract](agent-scene.md) defines the geometry fields, transforms, materials, and atomic validation rules.+## Call log++To see which tools agents use, set `SIXTYSEVENJS_MCP_LOG=<file path>` before starting the server. It appends one JSON line per tool call: `time`, `tool`, `durationMs`, `ok`, `error` (`tool_error` or `thrown`, only on failure) and `resultBytes`. It never writes arguments, scene content or paths. With the variable unset nothing is written.++```sh+SIXTYSEVENJS_MCP_LOG=calls.jsonl npx 67js-mcp+node scripts/summarize-mcp-log.mjs calls.jsonl other.jsonl+```++The script prints calls, errors, error rate and median duration per tool, most called first.+## Limits| Budget | Limit |diff --git a/scripts/mcp-call-log.mjs b/scripts/mcp-call-log.mjsnew file mode 100644index 00000000..8088aeb7--- /dev/null+++ b/scripts/mcp-call-log.mjs@@ -0,0 +1,52 @@+import { appendFileSync } from 'node:fs';++export const CALL_LOG_ENVIRONMENT_VARIABLE = 'SIXTYSEVENJS_MCP_LOG';++// Result size is the UTF-8 length of the text content plus the base64 length of images: what the agent has to read.+function resultByteCount(result) {+ let total = 0;+ for (const item of result?.content ?? []) {+ if (typeof item.text === 'string') {+ total += Buffer.byteLength(item.text);+ } else if (typeof item.data === 'string') {+ total += item.data.length;+ }+ }+ return total;+}++// Returns null when no path is set, so the server pays for nothing. A line holds the time, tool name, duration, outcome+// and result size, never the arguments, the scene or any path the caller passed.+export function createCallLogger(logPath, { now = () => performance.now(), clock = () => new Date() } = {}) {+ if (!logPath) {+ return null;+ }+ return async function logCall(toolName, run) {+ const startedAt = now();+ let result;+ let thrown;+ try {+ result = await run();+ } catch (error) {+ thrown = error;+ }+ const failed = thrown !== undefined || result?.isError === true;+ const line = {+ time: clock().toISOString(),+ tool: toolName,+ durationMs: Math.round((now() - startedAt) * 10) / 10,+ ok: !failed,+ ...(failed ? { error: thrown !== undefined ? 'thrown' : 'tool_error' } : {}),+ resultBytes: resultByteCount(result),+ };+ try {+ appendFileSync(logPath, `${JSON.stringify(line)}\n`);+ } catch (error) {+ console.error(`67js call log write failed: ${error.message}`);+ }+ if (thrown !== undefined) {+ throw thrown;+ }+ return result;+ };+}diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjsindex f42918ba..219218bc 100644--- a/scripts/mcp-server.mjs+++ b/scripts/mcp-server.mjs@@ -15,6 +15,7 @@ import { BUNDLED_ENVIRONMENT_NAMES, BUNDLED_CUBE_LOOK_NAMES, CATALOG_FILE_ID } fimport { MATERIAL_PATTERN_TYPES, MATERIAL_MAPPINGS, LIGHTING_SETUP_NAMES, RENDER_PASS_NAMES, RENDER_OUTPUT_FORMATS, MAXIMUM_JOB_FPS, VIDEO_ENCODERS, MAXIMUM_JOB_SIDE, MAXIMUM_MEMORY_MIB } from '../src/agent-scene-capabilities.js';import { MATERIAL_LAYER_SCALAR_RANGES, MATERIAL_LAYER_COLOR_RANGES } from '../src/material-layers.js';import { SUBSURFACE_RANGES } from '../src/subsurface-limits.js';+import { createCallLogger, CALL_LOG_ENVIRONMENT_VARIABLE } from './mcp-call-log.mjs';import { MAX_POINT_LIGHTS, MAX_SPOT_LIGHTS, MAX_AREA_LIGHTS } from '../src/light-limits.js';describeSchemaInputsWithReceivedValues();@@ -835,7 +836,8 @@ function jsonResult(value) {return { content: [{ type: 'text', text: JSON.stringify(value) }], structuredContent: value };}-export function createAgentMcpServer(runtime, { version = packageMetadata.version, branches = true, relations = true } = {}) {+export function createAgentMcpServer(runtime, { version = packageMetadata.version, branches = true, relations = true, callLogPath = process.env[CALL_LOG_ENVIRONMENT_VARIABLE] } = {}) {+ const logCall = createCallLogger(callLogPath);// Without branches the schemas carry no branch field and the instructions do not mention variants (a benchmark switch).const branchField = branches ? { branch: branchName.optional().describe('Fork: store the result as this branch (up to 8), leaving the scene unchanged.') } : {};const reviewBranchField = branches ? { branch: branchName.optional().describe('Render this scene_apply branch.') } : {};@@ -865,6 +867,7 @@ export function createAgentMcpServer(runtime, { version = packageMetadata.versio}function register(name, description, inputSchema, annotations, run) {server.registerTool(name, { title: TOOL_TITLES[name], description, inputSchema, annotations: { title: TOOL_TITLES[name], ...annotations }, ...(OUTPUT_SCHEMAS[name] ? { outputSchema: OUTPUT_SCHEMAS[name] } : {}) }, async argumentsValue => {+ const call = async () => {try {return await run(withActiveSession(argumentsValue, inputSchema));} catch (error) {@@ -873,6 +876,8 @@ export function createAgentMcpServer(runtime, { version = packageMetadata.versioconst sceneMissing = CREATE_TOOLS.has(name) && !message.startsWith('Dispose the current scene');return { isError: true, content: [{ type: 'text', text: `${name} failed: ${message}${sceneMissing ? ` ${SCENE_NOT_CREATED_ADVICE}` : ''}` }] };}+ };+ return logCall ? logCall(name, call) : call();});}register('scene_capabilities', 'Index of supported recipe features and runtime budgets; pass topic for the detail of one. Read it before authoring. Does not launch a browser.', z.strictObject({ topic: z.string().min(1).max(64).optional().describe('A name from the index, or name.branch such as meshEdit.bevel. Leave out for the index.') }), readOnly, async ({ topic }) => jsonResult(answerCapabilities(relations ? await runtime.capabilities() : withoutRelationsAdvice(await runtime.capabilities()), topic, outlineSources)));diff --git a/scripts/summarize-mcp-log.mjs b/scripts/summarize-mcp-log.mjsnew file mode 100644index 00000000..415511d1--- /dev/null+++ b/scripts/summarize-mcp-log.mjs@@ -0,0 +1,58 @@+#!/usr/bin/env node+import { readFileSync } from 'node:fs';+import { pathToFileURL } from 'node:url';++export function readCalls(logPaths) {+ const calls = [];+ for (const logPath of logPaths) {+ for (const line of readFileSync(logPath, 'utf8').split('\n')) {+ if (line.trim() === '') {+ continue;+ }+ try {+ calls.push(JSON.parse(line));+ } catch {+ console.error(`Skipped a line in ${logPath} that is not JSON.`);+ }+ }+ }+ return calls;+}++function median(values) {+ const sorted = [...values].sort((first, second) => first - second);+ const middle = Math.floor(sorted.length / 2);+ return sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;+}++export function summarizeCalls(calls) {+ const byTool = new Map();+ for (const call of calls) {+ const entry = byTool.get(call.tool) ?? { tool: call.tool, calls: 0, errors: 0, durations: [] };+ entry.calls += 1;+ entry.errors += call.ok === false ? 1 : 0;+ entry.durations.push(call.durationMs);+ byTool.set(call.tool, entry);+ }+ return [...byTool.values()]+ .map(entry => ({ tool: entry.tool, calls: entry.calls, errors: entry.errors, errorRate: entry.errors / entry.calls, medianMs: median(entry.durations) }))+ .sort((first, second) => second.calls - first.calls || first.tool.localeCompare(second.tool));+}++export function formatSummary(rows) {+ const header = ['tool', 'calls', 'errors', 'error rate', 'median ms'];+ const body = rows.map(row => [row.tool, String(row.calls), String(row.errors), `${(row.errorRate * 100).toFixed(1)}%`, row.medianMs.toFixed(1)]);+ const widths = header.map((title, column) => Math.max(title.length, ...body.map(cells => cells[column].length)));+ const render = cells => cells.map((cell, column) => (column === 0 ? cell.padEnd(widths[column]) : cell.padStart(widths[column]))).join(' ');+ return [render(header), ...body.map(render)].join('\n');+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ const logPaths = process.argv.slice(2);+ if (logPaths.length === 0) {+ console.error('Usage: node scripts/summarize-mcp-log.mjs <log file...>');+ process.exitCode = 1;+ } else {+ console.log(formatSummary(summarizeCalls(readCalls(logPaths))));+ }+}diff --git a/tests/mcp-call-log.test.js b/tests/mcp-call-log.test.jsnew file mode 100644index 00000000..808f6a43--- /dev/null+++ b/tests/mcp-call-log.test.js@@ -0,0 +1,99 @@+import './support/load-agent-scene-modules.mjs';+import test from 'node:test';+import assert from 'node:assert/strict';+import { existsSync, readFileSync } from 'node:fs';+import { mkdtemp, rm, writeFile } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { spawnSync } from 'node:child_process';+import { Client } from '@modelcontextprotocol/client';+import { InMemoryTransport } from '@modelcontextprotocol/server';+import { createAgentMcpServer } from '../scripts/mcp-server.mjs';+import { formatSummary, summarizeCalls } from '../scripts/summarize-mcp-log.mjs';++const SECRET_MARKER = 'secret-topic-marker';++const runtime = {+ activeSessionId: () => null,+ async close() {},+ async inspect() {+ return { assets: [] };+ },+ async dispose() {+ throw new Error('No scene is active.');+ },+};++async function connect(context, callLogPath) {+ const server = createAgentMcpServer(runtime, { callLogPath });+ const client = new Client({ name: 'call-log-test', version: '1' });+ const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();+ await server.connect(serverTransport);+ await client.connect(clientTransport);+ context.after(async () => {+ await client.close();+ await server.close();+ });+ return client;+}++async function temporaryLogPath(context) {+ const directory = await mkdtemp(join(tmpdir(), '67js-call-log-'));+ context.after(() => rm(directory, { recursive: true, force: true }));+ return join(directory, 'calls.jsonl');+}++test('with no log path the server writes no file', async context => {+ const logPath = await temporaryLogPath(context);+ const client = await connect(context, undefined);+ await client.callTool({ name: 'scene_asset_catalog', arguments: {} });+ assert.equal(existsSync(logPath), false);+});++test('with a log path each call adds one line that holds no arguments', async context => {+ const logPath = await temporaryLogPath(context);+ const client = await connect(context, logPath);+ await client.callTool({ name: 'scene_inspect', arguments: { ids: [SECRET_MARKER] } });+ const failed = await client.callTool({ name: 'scene_dispose', arguments: {} });+ assert.equal(failed.isError, true);++ const text = readFileSync(logPath, 'utf8');+ assert.ok(!text.includes(SECRET_MARKER), 'argument values are never written');+ const lines = text.trim().split('\n').map(line => JSON.parse(line));+ assert.equal(lines.length, 2);+ assert.deepEqual(Object.keys(lines[0]).sort(), ['durationMs', 'ok', 'resultBytes', 'time', 'tool']);+ assert.equal(lines[0].tool, 'scene_inspect');+ assert.equal(lines[0].ok, true);+ assert.ok(lines[0].resultBytes > 0);+ assert.ok(!Number.isNaN(Date.parse(lines[0].time)));+ assert.equal(lines[1].tool, 'scene_dispose');+ assert.equal(lines[1].ok, false);+ assert.equal(lines[1].error, 'tool_error');+});++test('the summary counts calls, error rate and median duration per tool, most called first', () => {+ const calls = [+ { tool: 'scene_apply', ok: true, durationMs: 10 },+ { tool: 'scene_apply', ok: false, durationMs: 30 },+ { tool: 'scene_apply', ok: true, durationMs: 20 },+ { tool: 'scene_apply', ok: true, durationMs: 40 },+ { tool: 'scene_look', ok: true, durationMs: 5 },+ ];+ const rows = summarizeCalls(calls);+ assert.deepEqual(rows, [+ { tool: 'scene_apply', calls: 4, errors: 1, errorRate: 0.25, medianMs: 25 },+ { tool: 'scene_look', calls: 1, errors: 0, errorRate: 0, medianMs: 5 },+ ]);+ assert.match(formatSummary(rows), /scene_apply\s+4\s+1\s+25\.0%\s+25\.0/);+});++test('the summary script reads several log files and prints a table', async context => {+ const first = await temporaryLogPath(context);+ const second = join(first, '..', 'second.jsonl');+ await writeFile(first, `${JSON.stringify({ tool: 'scene_look', ok: true, durationMs: 4 })}\nnot json\n`);+ await writeFile(second, `${JSON.stringify({ tool: 'scene_look', ok: false, durationMs: 8 })}\n`);+ const script = new URL('../scripts/summarize-mcp-log.mjs', import.meta.url);+ const run = spawnSync(process.execPath, [script.pathname.replace(/^\/(\w:)/, '$1'), first, second], { encoding: 'utf8' });+ assert.equal(run.status, 0, run.stderr);+ assert.match(run.stdout, /scene_look\s+2\s+1\s+50\.0%\s+6\.0/);+});