Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Add an opt-in MCP call log and a summary script (#746)

Nguyen Xuan An68563badfa05
diff --git a/changes/746.md b/changes/746.md
new file mode 100644
index 00000000..9c621d68
--- /dev/null
+++ b/changes/746.md
@@ -0,0 +1 @@
+- Set `SIXTYSEVENJS_MCP_LOG=<file>` to log each MCP tool call (tool, duration, ok, result size, never arguments), and run `scripts/summarize-mcp-log.mjs` for calls, error rate and median duration per tool (#746)
diff --git a/docs/mcp.md b/docs/mcp.md
index bb1bdc4f..2b9d45ad 100644
--- a/docs/mcp.md
+++ b/docs/mcp.md
@@ -201,6 +201,17 @@ References must resolve after the complete batch. Removing a parent leaves its c
Reparent or remove those children before the parent removal command to avoid rejection.
The [scene contract](agent-scene.md) defines the geometry fields, transforms, materials, and atomic validation rules.
+## Call log
+
+To see which tools agents use, set `SIXTYSEVENJS_MCP_LOG=<file path>` before starting the server. It appends one JSON line per tool call: `time`, `tool`, `durationMs`, `ok`, `error` (`tool_error` or `thrown`, only on failure) and `resultBytes`. It never writes arguments, scene content or paths. With the variable unset nothing is written.
+
+```sh
+SIXTYSEVENJS_MCP_LOG=calls.jsonl npx 67js-mcp
+node scripts/summarize-mcp-log.mjs calls.jsonl other.jsonl
+```
+
+The script prints calls, errors, error rate and median duration per tool, most called first.
+
## Limits
| Budget | Limit |
diff --git a/scripts/mcp-call-log.mjs b/scripts/mcp-call-log.mjs
new file mode 100644
index 00000000..8088aeb7
--- /dev/null
+++ b/scripts/mcp-call-log.mjs
@@ -0,0 +1,52 @@
+import { appendFileSync } from 'node:fs';
+
+export const CALL_LOG_ENVIRONMENT_VARIABLE = 'SIXTYSEVENJS_MCP_LOG';
+
+// Result size is the UTF-8 length of the text content plus the base64 length of images: what the agent has to read.
+function resultByteCount(result) {
+ let total = 0;
+ for (const item of result?.content ?? []) {
+ if (typeof item.text === 'string') {
+ total += Buffer.byteLength(item.text);
+ } else if (typeof item.data === 'string') {
+ total += item.data.length;
+ }
+ }
+ return total;
+}
+
+// Returns null when no path is set, so the server pays for nothing. A line holds the time, tool name, duration, outcome
+// and result size, never the arguments, the scene or any path the caller passed.
+export function createCallLogger(logPath, { now = () => performance.now(), clock = () => new Date() } = {}) {
+ if (!logPath) {
+ return null;
+ }
+ return async function logCall(toolName, run) {
+ const startedAt = now();
+ let result;
+ let thrown;
+ try {
+ result = await run();
+ } catch (error) {
+ thrown = error;
+ }
+ const failed = thrown !== undefined || result?.isError === true;
+ const line = {
+ time: clock().toISOString(),
+ tool: toolName,
+ durationMs: Math.round((now() - startedAt) * 10) / 10,
+ ok: !failed,
+ ...(failed ? { error: thrown !== undefined ? 'thrown' : 'tool_error' } : {}),
+ resultBytes: resultByteCount(result),
+ };
+ try {
+ appendFileSync(logPath, `${JSON.stringify(line)}\n`);
+ } catch (error) {
+ console.error(`67js call log write failed: ${error.message}`);
+ }
+ if (thrown !== undefined) {
+ throw thrown;
+ }
+ return result;
+ };
+}
diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjs
index f42918ba..219218bc 100644
--- a/scripts/mcp-server.mjs
+++ b/scripts/mcp-server.mjs
@@ -15,6 +15,7 @@ import { BUNDLED_ENVIRONMENT_NAMES, BUNDLED_CUBE_LOOK_NAMES, CATALOG_FILE_ID } f
import { MATERIAL_PATTERN_TYPES, MATERIAL_MAPPINGS, LIGHTING_SETUP_NAMES, RENDER_PASS_NAMES, RENDER_OUTPUT_FORMATS, MAXIMUM_JOB_FPS, VIDEO_ENCODERS, MAXIMUM_JOB_SIDE, MAXIMUM_MEMORY_MIB } from '../src/agent-scene-capabilities.js';
import { MATERIAL_LAYER_SCALAR_RANGES, MATERIAL_LAYER_COLOR_RANGES } from '../src/material-layers.js';
import { SUBSURFACE_RANGES } from '../src/subsurface-limits.js';
+import { createCallLogger, CALL_LOG_ENVIRONMENT_VARIABLE } from './mcp-call-log.mjs';
import { MAX_POINT_LIGHTS, MAX_SPOT_LIGHTS, MAX_AREA_LIGHTS } from '../src/light-limits.js';
describeSchemaInputsWithReceivedValues();
@@ -835,7 +836,8 @@ function jsonResult(value) {
return { content: [{ type: 'text', text: JSON.stringify(value) }], structuredContent: value };
}
-export function createAgentMcpServer(runtime, { version = packageMetadata.version, branches = true, relations = true } = {}) {
+export function createAgentMcpServer(runtime, { version = packageMetadata.version, branches = true, relations = true, callLogPath = process.env[CALL_LOG_ENVIRONMENT_VARIABLE] } = {}) {
+ const logCall = createCallLogger(callLogPath);
// Without branches the schemas carry no branch field and the instructions do not mention variants (a benchmark switch).
const branchField = branches ? { branch: branchName.optional().describe('Fork: store the result as this branch (up to 8), leaving the scene unchanged.') } : {};
const reviewBranchField = branches ? { branch: branchName.optional().describe('Render this scene_apply branch.') } : {};
@@ -865,6 +867,7 @@ export function createAgentMcpServer(runtime, { version = packageMetadata.versio
}
function register(name, description, inputSchema, annotations, run) {
server.registerTool(name, { title: TOOL_TITLES[name], description, inputSchema, annotations: { title: TOOL_TITLES[name], ...annotations }, ...(OUTPUT_SCHEMAS[name] ? { outputSchema: OUTPUT_SCHEMAS[name] } : {}) }, async argumentsValue => {
+ const call = async () => {
try {
return await run(withActiveSession(argumentsValue, inputSchema));
} catch (error) {
@@ -873,6 +876,8 @@ export function createAgentMcpServer(runtime, { version = packageMetadata.versio
const sceneMissing = CREATE_TOOLS.has(name) && !message.startsWith('Dispose the current scene');
return { isError: true, content: [{ type: 'text', text: `${name} failed: ${message}${sceneMissing ? ` ${SCENE_NOT_CREATED_ADVICE}` : ''}` }] };
}
+ };
+ return logCall ? logCall(name, call) : call();
});
}
register('scene_capabilities', 'Index of supported recipe features and runtime budgets; pass topic for the detail of one. Read it before authoring. Does not launch a browser.', z.strictObject({ topic: z.string().min(1).max(64).optional().describe('A name from the index, or name.branch such as meshEdit.bevel. Leave out for the index.') }), readOnly, async ({ topic }) => jsonResult(answerCapabilities(relations ? await runtime.capabilities() : withoutRelationsAdvice(await runtime.capabilities()), topic, outlineSources)));
diff --git a/scripts/summarize-mcp-log.mjs b/scripts/summarize-mcp-log.mjs
new file mode 100644
index 00000000..415511d1
--- /dev/null
+++ b/scripts/summarize-mcp-log.mjs
@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+import { readFileSync } from 'node:fs';
+import { pathToFileURL } from 'node:url';
+
+export function readCalls(logPaths) {
+ const calls = [];
+ for (const logPath of logPaths) {
+ for (const line of readFileSync(logPath, 'utf8').split('\n')) {
+ if (line.trim() === '') {
+ continue;
+ }
+ try {
+ calls.push(JSON.parse(line));
+ } catch {
+ console.error(`Skipped a line in ${logPath} that is not JSON.`);
+ }
+ }
+ }
+ return calls;
+}
+
+function median(values) {
+ const sorted = [...values].sort((first, second) => first - second);
+ const middle = Math.floor(sorted.length / 2);
+ return sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
+}
+
+export function summarizeCalls(calls) {
+ const byTool = new Map();
+ for (const call of calls) {
+ const entry = byTool.get(call.tool) ?? { tool: call.tool, calls: 0, errors: 0, durations: [] };
+ entry.calls += 1;
+ entry.errors += call.ok === false ? 1 : 0;
+ entry.durations.push(call.durationMs);
+ byTool.set(call.tool, entry);
+ }
+ return [...byTool.values()]
+ .map(entry => ({ tool: entry.tool, calls: entry.calls, errors: entry.errors, errorRate: entry.errors / entry.calls, medianMs: median(entry.durations) }))
+ .sort((first, second) => second.calls - first.calls || first.tool.localeCompare(second.tool));
+}
+
+export function formatSummary(rows) {
+ const header = ['tool', 'calls', 'errors', 'error rate', 'median ms'];
+ const body = rows.map(row => [row.tool, String(row.calls), String(row.errors), `${(row.errorRate * 100).toFixed(1)}%`, row.medianMs.toFixed(1)]);
+ const widths = header.map((title, column) => Math.max(title.length, ...body.map(cells => cells[column].length)));
+ const render = cells => cells.map((cell, column) => (column === 0 ? cell.padEnd(widths[column]) : cell.padStart(widths[column]))).join(' ');
+ return [render(header), ...body.map(render)].join('\n');
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ const logPaths = process.argv.slice(2);
+ if (logPaths.length === 0) {
+ console.error('Usage: node scripts/summarize-mcp-log.mjs <log file...>');
+ process.exitCode = 1;
+ } else {
+ console.log(formatSummary(summarizeCalls(readCalls(logPaths))));
+ }
+}
diff --git a/tests/mcp-call-log.test.js b/tests/mcp-call-log.test.js
new file mode 100644
index 00000000..808f6a43
--- /dev/null
+++ b/tests/mcp-call-log.test.js
@@ -0,0 +1,99 @@
+import './support/load-agent-scene-modules.mjs';
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { existsSync, readFileSync } from 'node:fs';
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { Client } from '@modelcontextprotocol/client';
+import { InMemoryTransport } from '@modelcontextprotocol/server';
+import { createAgentMcpServer } from '../scripts/mcp-server.mjs';
+import { formatSummary, summarizeCalls } from '../scripts/summarize-mcp-log.mjs';
+
+const SECRET_MARKER = 'secret-topic-marker';
+
+const runtime = {
+ activeSessionId: () => null,
+ async close() {},
+ async inspect() {
+ return { assets: [] };
+ },
+ async dispose() {
+ throw new Error('No scene is active.');
+ },
+};
+
+async function connect(context, callLogPath) {
+ const server = createAgentMcpServer(runtime, { callLogPath });
+ const client = new Client({ name: 'call-log-test', version: '1' });
+ const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
+ await server.connect(serverTransport);
+ await client.connect(clientTransport);
+ context.after(async () => {
+ await client.close();
+ await server.close();
+ });
+ return client;
+}
+
+async function temporaryLogPath(context) {
+ const directory = await mkdtemp(join(tmpdir(), '67js-call-log-'));
+ context.after(() => rm(directory, { recursive: true, force: true }));
+ return join(directory, 'calls.jsonl');
+}
+
+test('with no log path the server writes no file', async context => {
+ const logPath = await temporaryLogPath(context);
+ const client = await connect(context, undefined);
+ await client.callTool({ name: 'scene_asset_catalog', arguments: {} });
+ assert.equal(existsSync(logPath), false);
+});
+
+test('with a log path each call adds one line that holds no arguments', async context => {
+ const logPath = await temporaryLogPath(context);
+ const client = await connect(context, logPath);
+ await client.callTool({ name: 'scene_inspect', arguments: { ids: [SECRET_MARKER] } });
+ const failed = await client.callTool({ name: 'scene_dispose', arguments: {} });
+ assert.equal(failed.isError, true);
+
+ const text = readFileSync(logPath, 'utf8');
+ assert.ok(!text.includes(SECRET_MARKER), 'argument values are never written');
+ const lines = text.trim().split('\n').map(line => JSON.parse(line));
+ assert.equal(lines.length, 2);
+ assert.deepEqual(Object.keys(lines[0]).sort(), ['durationMs', 'ok', 'resultBytes', 'time', 'tool']);
+ assert.equal(lines[0].tool, 'scene_inspect');
+ assert.equal(lines[0].ok, true);
+ assert.ok(lines[0].resultBytes > 0);
+ assert.ok(!Number.isNaN(Date.parse(lines[0].time)));
+ assert.equal(lines[1].tool, 'scene_dispose');
+ assert.equal(lines[1].ok, false);
+ assert.equal(lines[1].error, 'tool_error');
+});
+
+test('the summary counts calls, error rate and median duration per tool, most called first', () => {
+ const calls = [
+ { tool: 'scene_apply', ok: true, durationMs: 10 },
+ { tool: 'scene_apply', ok: false, durationMs: 30 },
+ { tool: 'scene_apply', ok: true, durationMs: 20 },
+ { tool: 'scene_apply', ok: true, durationMs: 40 },
+ { tool: 'scene_look', ok: true, durationMs: 5 },
+ ];
+ const rows = summarizeCalls(calls);
+ assert.deepEqual(rows, [
+ { tool: 'scene_apply', calls: 4, errors: 1, errorRate: 0.25, medianMs: 25 },
+ { tool: 'scene_look', calls: 1, errors: 0, errorRate: 0, medianMs: 5 },
+ ]);
+ assert.match(formatSummary(rows), /scene_apply\s+4\s+1\s+25\.0%\s+25\.0/);
+});
+
+test('the summary script reads several log files and prints a table', async context => {
+ const first = await temporaryLogPath(context);
+ const second = join(first, '..', 'second.jsonl');
+ await writeFile(first, `${JSON.stringify({ tool: 'scene_look', ok: true, durationMs: 4 })}\nnot json\n`);
+ await writeFile(second, `${JSON.stringify({ tool: 'scene_look', ok: false, durationMs: 8 })}\n`);
+ const script = new URL('../scripts/summarize-mcp-log.mjs', import.meta.url);
+ const run = spawnSync(process.execPath, [script.pathname.replace(/^\/(\w:)/, '$1'), first, second], { encoding: 'utf8' });
+ assert.equal(run.status, 0, run.stderr);
+ assert.match(run.stdout, /scene_look\s+2\s+1\s+50\.0%\s+6\.0/);
+});