Skip to content

Chrome DevTools for coding agents

feat(telemetry): log devtools data. (#2460)

yulunzdf2c753006ce
diff --git a/src/ToolHandler.ts b/src/ToolHandler.ts
index a9094175..074652f5 100644
--- a/src/ToolHandler.ts
+++ b/src/ToolHandler.ts
@@ -11,7 +11,7 @@ import type {DataFormat} from './McpResponse.js';
import {McpResponse} from './McpResponse.js';
import {SlimMcpResponse} from './SlimMcpResponse.js';
import {ClearcutLogger} from './telemetry/ClearcutLogger.js';
-import {bucketizeLatency} from './telemetry/transformation.js';
+import {bucketizeLatency, buildContext} from './telemetry/transformation.js';
import type {CallToolResult} from './third_party/index.js';
import {zod} from './third_party/index.js';
import type {ToolCategory} from './tools/categories.js';
@@ -316,17 +316,14 @@ export class ToolHandler {
isError: true,
};
} finally {
- const isDevToolsOpen = devToolsData
- ? Object.keys(devToolsData).length > 0
- : undefined;
+ const context = buildContext(devToolsData, pageUrl);
void ClearcutLogger.get()?.logToolInvocation({
toolName: this.tool.name,
params,
schema: this.inputSchema,
success,
latencyMs: bucketizeLatency(Date.now() - startTime),
- isDevToolsOpen,
- pageUrl,
+ context,
});
guard[Symbol.dispose]();
}
diff --git a/src/telemetry/ClearcutLogger.ts b/src/telemetry/ClearcutLogger.ts
index 3a6606ac..35f711ef 100644
--- a/src/telemetry/ClearcutLogger.ts
+++ b/src/telemetry/ClearcutLogger.ts
@@ -9,7 +9,6 @@ import process from 'node:process';
import {DAEMON_CLIENT_NAME} from '../daemon/utils.js';
import type {zod, ShapeOutput} from '../third_party/index.js';
import {logger} from '../utils/logger.js';
-import {isLocalhost} from '../utils/url.js';
import type {ErrorCode} from './errors.js';
import type {LocalState, Persistence} from './persistence.js';
@@ -114,8 +113,7 @@ export class ClearcutLogger {
schema: zod.ZodRawShape;
success: boolean;
latencyMs: number;
- isDevToolsOpen?: boolean;
- pageUrl?: string;
+ context: ToolInvocationContext;
}): Promise<void> {
const sanitizedToolName = stripUnderscoreBeforeNumber(args.toolName);
const tool_invocation: ToolInvocation = {
@@ -123,15 +121,8 @@ export class ClearcutLogger {
success: args.success,
latency_ms: args.latencyMs,
};
- const context: ToolInvocationContext = {};
- if (args.isDevToolsOpen !== undefined) {
- context.is_devtools_open = args.isDevToolsOpen;
- }
- if (args.pageUrl !== undefined) {
- context.is_localhost = isLocalhost(args.pageUrl);
- }
- if (Object.keys(context).length > 0) {
- tool_invocation.context = context;
+ if (Object.keys(args.context).length > 0) {
+ tool_invocation.context = args.context;
}
if (Object.keys(args.params).length > 0) {
tool_invocation.tool_params = {
diff --git a/src/telemetry/transformation.ts b/src/telemetry/transformation.ts
index 5fd2b6ba..a57f44a2 100644
--- a/src/telemetry/transformation.ts
+++ b/src/telemetry/transformation.ts
@@ -5,6 +5,10 @@
*/
import type {zod, ShapeOutput} from '../third_party/index.js';
+import type {DevToolsData} from '../tools/ToolDefinition.js';
+import {isLocalhost} from '../utils/url.js';
+
+import type {LoggedDevToolsData, ToolInvocationContext} from './types.js';
const LATENCY_BUCKETS = [50, 100, 250, 500, 1000, 2500, 5000, 10000];
@@ -182,3 +186,39 @@ export function sanitizeParams(
}
return transformed;
}
+
+function transformDevToolsData(devToolsData: DevToolsData): LoggedDevToolsData {
+ const logged: LoggedDevToolsData = {};
+ if (devToolsData.cdpBackendNodeId !== undefined) {
+ logged.is_dom_element_selected = true;
+ }
+ if (devToolsData.cdpRequestId !== undefined) {
+ logged.is_network_request_selected = true;
+ }
+ return logged;
+}
+
+export function buildContext(
+ devToolsData?: DevToolsData,
+ pageUrl?: string,
+): ToolInvocationContext {
+ let context: ToolInvocationContext;
+ if (devToolsData === undefined) {
+ context = {is_devtools_open: false};
+ } else {
+ context = {
+ is_devtools_open: Object.keys(devToolsData).length > 0,
+ };
+
+ const loggedDevtoolsData = transformDevToolsData(devToolsData);
+ if (Object.keys(loggedDevtoolsData).length > 0) {
+ context.devtools_data = loggedDevtoolsData;
+ }
+ }
+
+ if (pageUrl !== undefined) {
+ context.is_localhost = isLocalhost(pageUrl);
+ }
+
+ return context;
+}
diff --git a/src/telemetry/types.ts b/src/telemetry/types.ts
index 855d4d69..cac3f242 100644
--- a/src/telemetry/types.ts
+++ b/src/telemetry/types.ts
@@ -26,9 +26,15 @@ export interface ServerError {
export type ServerShutdown = Record<string, never>;
+export interface LoggedDevToolsData {
+ is_dom_element_selected?: boolean;
+ is_network_request_selected?: boolean;
+}
+
export interface ToolInvocationContext {
is_devtools_open?: boolean;
is_localhost?: boolean;
+ devtools_data?: LoggedDevToolsData;
}
export interface ToolInvocation {
diff --git a/tests/ToolHandler.test.ts b/tests/ToolHandler.test.ts
index 381fe338..bacdb06e 100644
--- a/tests/ToolHandler.test.ts
+++ b/tests/ToolHandler.test.ts
@@ -110,12 +110,10 @@ describe('ToolHandler', () => {
assert.strictEqual(result.isError, undefined);
});
- it('logs isDevToolsOpen telemetry based on getDevToolsData', async () => {
- let handlerCalled = false;
- const tool: DefinedPageTool = {
- name: 'page_tool',
- description: 'A page tool',
- pageScoped: true,
+ it('appends correct context to tool call logs', async () => {
+ const baseTool: ToolDefinition = {
+ name: 'test_tool',
+ description: 'A test tool',
annotations: {
category: ToolCategory.NAVIGATION,
readOnlyHint: true,
@@ -124,46 +122,92 @@ describe('ToolHandler', () => {
blockedByDialog: false,
verifyFilesSchema: [],
handler: async () => {
- handlerCalled = true;
+ return;
},
};
- const mockContext = sinon.createStubInstance(McpContext);
- mockContext.getDevToolsData.resolves({cdpBackendNodeId: 1});
- const mockPage = {
- pptrPage: {
- isClosed: () => false,
- url: () => 'http://localhost:9222/',
+ const testCases: Array<{
+ tool: ToolDefinition | DefinedPageTool;
+ devToolsData: Record<string, unknown>;
+ pageUrl?: string;
+ expectedContext: Record<string, unknown>;
+ }> = [
+ {
+ tool: {
+ ...baseTool,
+ name: 'page_tool',
+ pageScoped: true,
+ },
+ devToolsData: {cdpBackendNodeId: 1},
+ pageUrl: 'http://localhost:9222/',
+ expectedContext: {
+ is_devtools_open: true,
+ is_localhost: true,
+ devtools_data: {
+ is_dom_element_selected: true,
+ },
+ },
},
- } as unknown as McpPage;
- mockContext.getSelectedMcpPage.returns(mockPage);
-
- const logSpy = sinon.spy();
- sinon.stub(ClearcutLogger, 'get').returns({
- logToolInvocation: logSpy,
- } as unknown as ClearcutLogger);
-
- const toolMutex = new Mutex();
- const serverArgs = parseArguments('1.0.0', ['node', 'script.js'], {
- CHROME_DEVTOOLS_MCP_NO_USAGE_STATISTICS: 'true',
- });
-
- const toolHandler = new ToolHandler(
- tool,
- serverArgs,
- async () => mockContext,
- toolMutex,
- );
-
- await toolHandler.handle({});
+ {
+ tool: {
+ ...baseTool,
+ name: 'global_tool',
+ },
+ devToolsData: {},
+ pageUrl: undefined,
+ expectedContext: {
+ is_devtools_open: false,
+ },
+ },
+ ];
- assert.strictEqual(logSpy.calledOnce, true);
- assert.strictEqual(logSpy.firstCall.args[0].isDevToolsOpen, true);
- assert.strictEqual(
- logSpy.firstCall.args[0].pageUrl,
- 'http://localhost:9222/',
- );
- assert.strictEqual(handlerCalled, true);
+ for (const testCase of testCases) {
+ let handlerCalled = false;
+ testCase.tool.handler = async () => {
+ handlerCalled = true;
+ };
+
+ const mockContext = sinon.createStubInstance(McpContext);
+ mockContext.getDevToolsData.resolves(testCase.devToolsData);
+ if (testCase.pageUrl) {
+ const mockPage = {
+ pptrPage: {
+ isClosed: () => false,
+ url: () => testCase.pageUrl,
+ },
+ } as unknown as McpPage;
+ mockContext.getSelectedMcpPage.returns(mockPage);
+ }
+
+ const logSpy = sinon.spy();
+ sinon.stub(ClearcutLogger, 'get').returns({
+ logToolInvocation: logSpy,
+ } as unknown as ClearcutLogger);
+
+ const toolMutex = new Mutex();
+ const serverArgs = parseArguments('1.0.0', ['node', 'script.js'], {
+ CHROME_DEVTOOLS_MCP_NO_USAGE_STATISTICS: 'true',
+ });
+
+ const toolHandler = new ToolHandler(
+ testCase.tool,
+ serverArgs,
+ async () => mockContext,
+ toolMutex,
+ );
+
+ await toolHandler.handle({});
+
+ assert.strictEqual(logSpy.calledOnce, true);
+ assert.deepStrictEqual(
+ logSpy.firstCall.args[0].context,
+ testCase.expectedContext,
+ );
+ assert.strictEqual(handlerCalled, true);
+
+ sinon.restore();
+ ClearcutLogger.resetForTesting();
+ }
});
it('reports unknown registered tool arguments clearly', async () => {
diff --git a/tests/telemetry/ClearcutLogger.test.ts b/tests/telemetry/ClearcutLogger.test.ts
index 65585a3a..90196e8e 100644
--- a/tests/telemetry/ClearcutLogger.test.ts
+++ b/tests/telemetry/ClearcutLogger.test.ts
@@ -50,6 +50,7 @@ describe('ClearcutLogger', () => {
schema: {},
success: true,
latencyMs: 123,
+ context: {},
});
assert(mockWatchdogClient.send.calledOnce);
@@ -59,7 +60,7 @@ describe('ClearcutLogger', () => {
assert.strictEqual(msg.payload.tool_invocation?.success, true);
assert.strictEqual(msg.payload.tool_invocation?.latency_ms, 123);
});
- it('sends context with is_devtools_open', async () => {
+ it('sends context when provided', async () => {
const logger = ClearcutLogger.initialize({
persistence: mockPersistence,
appVersion: '1.0.0',
@@ -71,7 +72,10 @@ describe('ClearcutLogger', () => {
schema: {},
success: true,
latencyMs: 123,
- isDevToolsOpen: true,
+ context: {
+ is_devtools_open: true,
+ is_localhost: false,
+ },
});
assert(mockWatchdogClient.send.calledOnce);
@@ -79,37 +83,9 @@ describe('ClearcutLogger', () => {
assert.strictEqual(msg.type, WatchdogMessageType.LOG_EVENT);
assert.deepStrictEqual(msg.payload.tool_invocation?.context, {
is_devtools_open: true,
+ is_localhost: false,
});
});
- it('sends context with correct is_localhost based on the URL', async () => {
- const logger = ClearcutLogger.initialize({
- persistence: mockPersistence,
- appVersion: '1.0.0',
- watchdogClient: mockWatchdogClient,
- });
-
- for (const {pageUrl, isLocalhost} of [
- {pageUrl: 'http://localhost:9222/test', isLocalhost: true},
- {pageUrl: 'https://example.com/test', isLocalhost: false},
- ]) {
- mockWatchdogClient.send.resetHistory();
- await logger.logToolInvocation({
- toolName: 'test_tool',
- params: {},
- schema: {},
- success: true,
- latencyMs: 123,
- pageUrl,
- });
-
- assert(mockWatchdogClient.send.calledOnce);
- const msg = mockWatchdogClient.send.firstCall.args[0];
- assert.strictEqual(msg.type, WatchdogMessageType.LOG_EVENT);
- assert.deepStrictEqual(msg.payload.tool_invocation?.context, {
- is_localhost: isLocalhost,
- });
- }
- });
it('sends sanitized params', async () => {
const logger = ClearcutLogger.initialize({
persistence: mockPersistence,
@@ -133,6 +109,7 @@ describe('ClearcutLogger', () => {
schema,
success: true,
latencyMs: 123,
+ context: {},
});
assert(mockWatchdogClient.send.calledOnce);
diff --git a/tests/telemetry/transformation.test.ts b/tests/telemetry/transformation.test.ts
index cb111fdb..690f7f38 100644
--- a/tests/telemetry/transformation.test.ts
+++ b/tests/telemetry/transformation.test.ts
@@ -9,6 +9,7 @@ import {describe, it} from 'node:test';
import {
bucketizeLatency,
+ buildContext,
getEnumValues,
sanitizeParams,
stripUnderscoreBeforeNumber,
@@ -198,3 +199,57 @@ describe('transformArgName', () => {
);
});
});
+
+describe('buildContext', () => {
+ it('should set is_devtools_open based on devToolsData', () => {
+ assert.deepStrictEqual(buildContext(undefined, undefined), {
+ is_devtools_open: false,
+ });
+ assert.deepStrictEqual(buildContext({}, undefined), {
+ is_devtools_open: false,
+ });
+ assert.deepStrictEqual(buildContext({cdpBackendNodeId: 1}, undefined), {
+ is_devtools_open: true,
+ devtools_data: {
+ is_dom_element_selected: true,
+ },
+ });
+ });
+
+ it('should set is_localhost based on pageUrl', () => {
+ assert.deepStrictEqual(
+ buildContext(undefined, 'http://localhost:9222/test'),
+ {
+ is_devtools_open: false,
+ is_localhost: true,
+ },
+ );
+ assert.deepStrictEqual(
+ buildContext(undefined, 'https://example.com/test'),
+ {
+ is_devtools_open: false,
+ is_localhost: false,
+ },
+ );
+ });
+
+ it('should include devtools_data when present', () => {
+ assert.deepStrictEqual(
+ buildContext(
+ {
+ cdpBackendNodeId: 1,
+ cdpRequestId: 'req-1',
+ },
+ 'http://localhost:9222/',
+ ),
+ {
+ is_devtools_open: true,
+ is_localhost: true,
+ devtools_data: {
+ is_dom_element_selected: true,
+ is_network_request_selected: true,
+ },
+ },
+ );
+ });
+});