Chrome DevTools for coding agents
fix: discover third-party developer tools before tool lookup (#3208)
Wolfgang Beyer2777fa05b296
diff --git a/src/McpPage.ts b/src/McpPage.tsindex 53da2fa5..554e68fb 100644--- a/src/McpPage.ts+++ b/src/McpPage.ts@@ -174,8 +174,6 @@ export class McpPage implements ContextPage {// Screencast#screenRecorder: ScreenRecorderData | null = null;- eventBasedThirdPartyTools: ToolGroups = [];-#networkCollector?: NetworkCollector;#consoleCollector?: ConsoleCollector;@@ -411,14 +409,7 @@ export class McpPage implements ContextPage {];}- getThirdPartyDeveloperTools(): ToolGroups {- return [- ...this.#getDebuggingWebMcpToolGroups(),- ...this.eventBasedThirdPartyTools,- ];- }-- async getToolGroups(): Promise<ToolGroups> {+ async discoverThirdPartyDeveloperTools(): Promise<ToolGroups> {// Check if there is a `devtoolstooldiscovery` event listenerusing windowHandle = await this.pptrPage.evaluateHandle(() => window);// @ts-expect-error internal API@@ -428,8 +419,7 @@ export class McpPage implements ContextPage {objectId: windowHandle.remoteObject().objectId,});if (listeners.find(l => l.type === 'devtoolstooldiscovery') === undefined) {- this.eventBasedThirdPartyTools = [];- return this.getThirdPartyDeveloperTools();+ return this.#getDebuggingWebMcpToolGroups();}const eventToolGroups = await this.pptrPage.evaluate(() => {@@ -515,9 +505,7 @@ export class McpPage implements ContextPage {}}- this.eventBasedThirdPartyTools = eventToolGroups;-- return this.getThirdPartyDeveloperTools();+ return [...this.#getDebuggingWebMcpToolGroups(), ...eventToolGroups];}getWebMcpTools(): WebMCPTool[] {diff --git a/src/McpResponse.ts b/src/McpResponse.tsindex 3adeb072..d3f3c08d 100644--- a/src/McpResponse.ts+++ b/src/McpResponse.ts@@ -624,7 +624,7 @@ export class McpResponse implements Response {this.#listThirdPartyDeveloperTools &&this.#page) {- return await this.#page.getToolGroups();+ return await this.#page.discoverThirdPartyDeveloperTools();}return undefined;}diff --git a/src/tools/ToolDefinition.ts b/src/tools/ToolDefinition.tsindex 3c2f5271..02de6634 100644--- a/src/tools/ToolDefinition.ts+++ b/src/tools/ToolDefinition.ts@@ -475,7 +475,7 @@ export type ContextPage = Readonly<{},): Promise<WaitForEventsResult>;getWebMcpTools(): WebMCPTool[];- getThirdPartyDeveloperTools(): ToolGroups;+ discoverThirdPartyDeveloperTools(): Promise<ToolGroups>;executeThirdPartyDeveloperTool(toolName: string,diff --git a/src/tools/thirdPartyDeveloper.ts b/src/tools/thirdPartyDeveloper.tsindex 83c16bf3..0e385472 100644--- a/src/tools/thirdPartyDeveloper.ts+++ b/src/tools/thirdPartyDeveloper.ts@@ -101,7 +101,7 @@ export const executeThirdPartyDeveloperTool = definePageTool(() => ({}}- const toolGroups = request.page.getThirdPartyDeveloperTools();+ const toolGroups = await request.page.discoverThirdPartyDeveloperTools();let tool;if (toolGroups) {for (const group of toolGroups) {diff --git a/tests/McpPage.test.ts b/tests/McpPage.test.tsindex 0df230f6..e75da8e3 100644--- a/tests/McpPage.test.ts+++ b/tests/McpPage.test.ts@@ -327,8 +327,8 @@ describe('McpPage', () => {sinon.assert.notCalled(target.page);sinon.assert.notCalled(target.asPage);assert.throws(() => mcpPage.pptrPage, /not initialized/);- assert.throws(- () => mcpPage.getThirdPartyDeveloperTools(),+ await assert.rejects(+ () => mcpPage.discoverThirdPartyDeveloperTools(),/not initialized/,);assert.throws(() => mcpPage.getWebMcpTools(), /not initialized/);@@ -1476,7 +1476,7 @@ describe('McpPage', () => {sinon.restore();});- it('filters WebMCP tools by annotations.debugging in getToolGroups()', async () => {+ it('filters WebMCP tools by annotations.debugging in discoverThirdPartyDeveloperTools()', async () => {const {mcpPage, pptrPage} = await createMcpPage();const debuggingTool = createMockWebMCPTool({@@ -1502,7 +1502,7 @@ describe('McpPage', () => {unannotatedTool,]);- const groups = await mcpPage.getToolGroups();+ const groups = await mcpPage.discoverThirdPartyDeveloperTools();assert.deepStrictEqual(groups, [{name: 'WebMCP Tools',@@ -1516,10 +1516,9 @@ describe('McpPage', () => {],},]);- assert.deepStrictEqual(mcpPage.getThirdPartyDeveloperTools(), groups);});- it('combines WebMCP debugging tools and devtoolstooldiscovery tools in getThirdPartyDeveloperTools()', async () => {+ it('combines WebMCP debugging tools and devtoolstooldiscovery tools in discoverThirdPartyDeveloperTools()', async () => {const {mcpPage, pptrPage} = await createMcpPage();const debuggingTool = createMockWebMCPTool({@@ -1528,7 +1527,10 @@ describe('McpPage', () => {annotations: {debugging: true},});pptrPage.webmcp.tools.returns([debuggingTool]);- mcpPage.eventBasedThirdPartyTools = [+ pptrPage.cdpSession.send.resolves({+ listeners: [{type: 'devtoolstooldiscovery'}],+ });+ pptrPage.evaluate.resolves([{name: 'Custom Tools',description: 'Tools exposed via event',@@ -1540,17 +1542,16 @@ describe('McpPage', () => {type: 'object',properties: {el: {- properties: {uid: {type: 'string'}},- required: ['uid'],+ 'x-mcp-type': 'HTMLElement',},},},},],},- ];+ ]);- const groups = mcpPage.getThirdPartyDeveloperTools();+ const groups = await mcpPage.discoverThirdPartyDeveloperTools();assert.deepStrictEqual(groups, [{name: 'WebMCP Tools',@@ -1574,6 +1575,7 @@ describe('McpPage', () => {type: 'object',properties: {el: {+ 'x-mcp-type': 'HTMLElement',properties: {uid: {type: 'string'}},required: ['uid'],},@@ -1585,10 +1587,13 @@ describe('McpPage', () => {]);});- it('reflects dynamically added and removed WebMCP debugging tools in getThirdPartyDeveloperTools()', async () => {+ it('reflects dynamically added and removed WebMCP debugging tools in discoverThirdPartyDeveloperTools()', async () => {const {mcpPage, pptrPage} = await createMcpPage();- assert.deepStrictEqual(mcpPage.getThirdPartyDeveloperTools(), []);+ assert.deepStrictEqual(+ await mcpPage.discoverThirdPartyDeveloperTools(),+ [],+ );const debuggingTool = createMockWebMCPTool({name: 'dynamic_debug_tool',@@ -1598,7 +1603,7 @@ describe('McpPage', () => {});pptrPage.webmcp.tools.returns([debuggingTool]);- assert.deepStrictEqual(mcpPage.getThirdPartyDeveloperTools(), [+ assert.deepStrictEqual(await mcpPage.discoverThirdPartyDeveloperTools(), [{name: 'WebMCP Tools',description: 'Tools exposed via WebMCP',@@ -1613,7 +1618,10 @@ describe('McpPage', () => {]);pptrPage.webmcp.tools.returns([]);- assert.deepStrictEqual(mcpPage.getThirdPartyDeveloperTools(), []);+ assert.deepStrictEqual(+ await mcpPage.discoverThirdPartyDeveloperTools(),+ [],+ );});it('executes a WebMCP tool in executeThirdPartyDeveloperTool()', async () => {diff --git a/tests/McpResponse.test.ts b/tests/McpResponse.test.tsindex eb151ca1..0cce0010 100644--- a/tests/McpResponse.test.ts+++ b/tests/McpResponse.test.ts@@ -1127,7 +1127,7 @@ describe('third-party developer tools', () => {categoryExperimentalThirdParty: true,});const page = createMockMcpPage();- page.getToolGroups.resolves([+ page.discoverThirdPartyDeveloperTools.resolves([{name: 'My Tool Group',description: 'A group of tools',@@ -1165,7 +1165,7 @@ describe('third-party developer tools', () => {createMockParsedArguments({categoryExperimentalThirdParty: true}),);const page = createMockMcpPage();- page.getToolGroups.resolves([]);+ page.discoverThirdPartyDeveloperTools.resolves([]);response.setPage(page);response.setListThirdPartyDeveloperTools();@@ -1192,7 +1192,7 @@ describe('third-party developer tools', () => {const page = createMockMcpPage();Object.defineProperty(page, 'id', {value: 1});page.url.returns('about:blank');- page.getToolGroups.resolves([+ page.discoverThirdPartyDeveloperTools.resolves([{name: 'Tool group name',description: 'Test tools',diff --git a/tests/mocks.ts b/tests/mocks.tsindex 59a6d679..e0a33345 100644--- a/tests/mocks.ts+++ b/tests/mocks.ts@@ -81,6 +81,13 @@ export type MockPuppeteerPage = sinon.SinonStubbedInstance<Page> & {tracing: sinon.SinonStubbedInstance<Tracing>;mouse: sinon.SinonStubbedInstance<Mouse>;accessibility: sinon.SinonStubbedInstance<Accessibility>;+ cdpSession: {+ on: sinon.SinonStub;+ off: sinon.SinonStub;+ send: sinon.SinonStub;+ target: sinon.SinonStub;+ emit: (eventName: string | symbol | number, data?: unknown) => void;+ };};export type MockMcpPage = sinon.SinonStubbedInstance<McpPage> & {@@ -346,7 +353,13 @@ export function createMockPuppeteerPage(): MockPuppeteerPage {page.emulateFocusedPage.resolves();page.evaluateHandle.resolves(createMockElementHandle().handle);- return Object.assign(page, {webmcp, tracing, mouse, accessibility});+ return Object.assign(page, {+ webmcp,+ tracing,+ mouse,+ accessibility,+ cdpSession,+ });}export function createMockWebMCPTool(options: {@@ -438,6 +451,7 @@ export function createMockMcpPage(return {};});page.getWebMcpTools.returns([]);+ page.discoverThirdPartyDeveloperTools.resolves([]);Object.defineProperty(page, 'pptrPage', {value: pptrPage,writable: true,diff --git a/tests/tools/thirdPartyDeveloper.test.ts b/tests/tools/thirdPartyDeveloper.test.tsindex 5e322e2b..0c18bf2d 100644--- a/tests/tools/thirdPartyDeveloper.test.ts+++ b/tests/tools/thirdPartyDeveloper.test.ts@@ -243,7 +243,7 @@ describe('thirdPartyDeveloperTools', () => {);});- it('clears window.__dtmcp.toolGroups on subsequent getToolGroups calls', async () => {+ it('clears window.__dtmcp.toolGroups on subsequent discoverThirdPartyDeveloperTools calls', async () => {await withMcpContext(async (response, context, args) => {const page = await context.newPage();@@ -320,7 +320,7 @@ describe('thirdPartyDeveloperTools', () => {const {page, context, response, args} = createHandlerMocks({categoryExperimentalThirdParty: true,});- page.getThirdPartyDeveloperTools.returns([+ page.discoverThirdPartyDeveloperTools.resolves([{name: 'test-group',description: 'test description',@@ -352,19 +352,24 @@ describe('thirdPartyDeveloperTools', () => {context,);+ sinon.assert.calledOnceWithExactly(page.discoverThirdPartyDeveloperTools);sinon.assert.calledOnceWithExactly(page.executeThirdPartyDeveloperTool,'test-tool',{arg: 'value'},response,);+ sinon.assert.callOrder(+ page.discoverThirdPartyDeveloperTools,+ page.executeThirdPartyDeveloperTool,+ );});it('throws if tool not found in list', async () => {const {page, context, response, args} = createHandlerMocks({categoryExperimentalThirdParty: true,});- page.getThirdPartyDeveloperTools.returns([+ page.discoverThirdPartyDeveloperTools.resolves([{name: 'test-group',description: 'test description',@@ -389,6 +394,7 @@ describe('thirdPartyDeveloperTools', () => {{message: /Tool missing-tool not found/},);+ sinon.assert.calledOnceWithExactly(page.discoverThirdPartyDeveloperTools);sinon.assert.notCalled(page.executeThirdPartyDeveloperTool);});@@ -414,7 +420,7 @@ describe('thirdPartyDeveloperTools', () => {const {page, context, response, args} = createHandlerMocks({categoryExperimentalThirdParty: true,});- page.getThirdPartyDeveloperTools.returns([+ page.discoverThirdPartyDeveloperTools.resolves([{name: 'test-group',description: 'test description',@@ -451,6 +457,7 @@ describe('thirdPartyDeveloperTools', () => {{message: /Invalid parameters for tool test-tool/},);+ sinon.assert.calledOnceWithExactly(page.discoverThirdPartyDeveloperTools);sinon.assert.notCalled(page.executeThirdPartyDeveloperTool);});@@ -502,8 +509,8 @@ describe('thirdPartyDeveloperTools', () => {const page = await context.newPage();response.setPage(page);- page.eventBasedThirdPartyTools = [- {+ await page.pptrPage.evaluate(() => {+ const mockToolGroup = {name: 'test-group',description: 'test description',tools: [@@ -517,32 +524,29 @@ describe('thirdPartyDeveloperTools', () => {},required: ['element'],},+ execute: async (toolArgs: Record<string, unknown>) => {+ const el = toolArgs.element;+ if (el instanceof HTMLElement) {+ return {+ isElement: true,+ tagName: el.tagName,+ id: el.id,+ };+ }+ return {+ isElement: false,+ tagName: '',+ id: '',+ };+ },},],- },- ];-- await page.pptrPage.evaluate(() => {- window.__dtmcp = {- executeTool: async (- _name: string,- args: Record<string, unknown>,- ) => {- const el = args.element;- if (el instanceof HTMLElement) {- return {- isElement: true,- tagName: el.tagName,- id: el.id,- };- }- return {- isElement: false,- tagName: '',- id: '',- };- },};+ window.addEventListener('devtoolstooldiscovery', (e: Event) => {+ if ('respondWith' in e && typeof e.respondWith === 'function') {+ e.respondWith(mockToolGroup);+ }+ });});await page.pptrPage.evaluate(() => {@@ -742,8 +746,8 @@ describe('thirdPartyDeveloperTools', () => {const page = await context.newPage();response.setPage(page);- page.eventBasedThirdPartyTools = [- {+ await page.pptrPage.evaluate(() => {+ const mockToolGroup = {name: 'test-group',description: 'test description',tools: [@@ -751,32 +755,32 @@ describe('thirdPartyDeveloperTools', () => {name: 'test-tool',description: 'test tool description',inputSchema: {},+ execute: async () => {+ const circularArr: unknown[] = ['item'];+ circularArr.push(circularArr);+ const sharedObj = {sharedKey: 'sharedVal'};+ const nullProtoObj: Record<string, unknown> =+ Object.create(null);+ nullProtoObj.key = 'val';+ const AnonymousClass = (() => class {})();+ return {+ sym: Symbol('test-sym'),+ big: BigInt(42),+ circularArr,+ shared1: sharedObj,+ shared2: sharedObj,+ nullProtoObj,+ anonInstance: new AnonymousClass(),+ };+ },},],- },- ];-- await page.pptrPage.evaluate(() => {- window.__dtmcp = {- executeTool: async () => {- const circularArr: unknown[] = ['item'];- circularArr.push(circularArr);- const sharedObj = {sharedKey: 'sharedVal'};- const nullProtoObj: Record<string, unknown> =- Object.create(null);- nullProtoObj.key = 'val';- const AnonymousClass = (() => class {})();- return {- sym: Symbol('test-sym'),- big: BigInt(42),- circularArr,- shared1: sharedObj,- shared2: sharedObj,- nullProtoObj,- anonInstance: new AnonymousClass(),- };- },};+ window.addEventListener('devtoolstooldiscovery', (e: Event) => {+ if ('respondWith' in e && typeof e.respondWith === 'function') {+ e.respondWith(mockToolGroup);+ }+ });});await executeThirdPartyDeveloperTool(args).handler(@@ -818,8 +822,8 @@ describe('thirdPartyDeveloperTools', () => {const page = await context.newPage();response.setPage(page);- page.eventBasedThirdPartyTools = [- {+ await page.pptrPage.evaluate(() => {+ const mockToolGroup = {name: 'test-group',description: 'test description',tools: [@@ -827,20 +831,20 @@ describe('thirdPartyDeveloperTools', () => {name: 'test-tool',description: 'test tool description',inputSchema: {},+ execute: async () => {+ const div = document.createElement('div');+ div.id = 'test-element';+ document.body.appendChild(div);+ return div;+ },},],- },- ];-- await page.pptrPage.evaluate(() => {- window.__dtmcp = {- executeTool: async () => {- const div = document.createElement('div');- div.id = 'test-element';- document.body.appendChild(div);- return div;- },};+ window.addEventListener('devtoolstooldiscovery', (e: Event) => {+ if ('respondWith' in e && typeof e.respondWith === 'function') {+ e.respondWith(mockToolGroup);+ }+ });});await executeThirdPartyDeveloperTool(args).handler(@@ -871,8 +875,8 @@ describe('thirdPartyDeveloperTools', () => {const page = await context.newPage();response.setPage(page);- page.eventBasedThirdPartyTools = [- {+ await page.pptrPage.evaluate(() => {+ const mockToolGroup = {name: 'test-group',description: 'test description',tools: [@@ -880,20 +884,20 @@ describe('thirdPartyDeveloperTools', () => {name: 'test-tool',description: 'test tool description',inputSchema: {},+ execute: async () => {+ const div = document.createElement('div');+ div.id = 'test-element';+ document.body.appendChild(div);+ return div;+ },},],- },- ];-- await page.pptrPage.evaluate(() => {- window.__dtmcp = {- executeTool: async () => {- const div = document.createElement('div');- div.id = 'test-element';- document.body.appendChild(div);- return div;- },};+ window.addEventListener('devtoolstooldiscovery', (e: Event) => {+ if ('respondWith' in e && typeof e.respondWith === 'function') {+ e.respondWith(mockToolGroup);+ }+ });});await executeThirdPartyDeveloperTool(args).handler(@@ -924,8 +928,8 @@ describe('thirdPartyDeveloperTools', () => {const page = await context.newPage();response.setPage(page);- page.eventBasedThirdPartyTools = [- {+ await page.pptrPage.evaluate(() => {+ const mockToolGroup = {name: 'test-group',description: 'test description',tools: [@@ -933,22 +937,22 @@ describe('thirdPartyDeveloperTools', () => {name: 'test-tool',description: 'test tool description',inputSchema: {},+ execute: async () => {+ return 'simple-result';+ },},],- },- ];-- await page.pptrPage.evaluate(() => {- window.__dtmcp = {- executeTool: async () => {- return 'simple-result';- },};+ window.addEventListener('devtoolstooldiscovery', (e: Event) => {+ if ('respondWith' in e && typeof e.respondWith === 'function') {+ e.respondWith(mockToolGroup);+ }+ });});const stubSnapshot = sinon.stub(TextSnapshot, 'create')- .resolves({} as TextSnapshot);+ .resolves(sinon.createStubInstance(TextSnapshot));await executeThirdPartyDeveloperTool(args).handler({@@ -978,6 +982,111 @@ describe('thirdPartyDeveloperTools', () => {);});+ it('executes a tool after the page reloads itself', async () => {+ await withMcpContext(+ async (response, context, args) => {+ const page = await context.newPage();+ response.setPage(page);++ await page.pptrPage.evaluateOnNewDocument(() => {+ const mockToolGroup = {+ name: 'test-group',+ description: 'test description',+ tools: [+ {+ name: 'test-tool',+ description: 'test tool description',+ inputSchema: {},+ execute: () => ({reloaded: true}),+ },+ ],+ };+ window.addEventListener('devtoolstooldiscovery', (e: Event) => {+ if ('respondWith' in e && typeof e.respondWith === 'function') {+ e.respondWith(mockToolGroup);+ }+ });+ });+ await page.pptrPage.reload();++ await listThirdPartyDeveloperTools(args).handler(+ {params: {}, page},+ response,+ context,+ );+ await response.handle(context);++ await page.pptrPage.reload();++ await executeThirdPartyDeveloperTool(args).handler(+ {+ params: {+ toolName: 'test-tool',+ params: JSON.stringify({}),+ },+ page,+ },+ response,+ context,+ );+ assert.strictEqual(+ response.responseLines[0],+ JSON.stringify({reloaded: true}, null, 2),+ );+ },+ undefined,+ {categoryExperimentalThirdParty: true},+ );+ });++ it('throws if tool is no longer exposed after the page reloads itself', async () => {+ await withMcpContext(+ async (response, context, args) => {+ await setupThirdPartyDeveloperTools(response, context, args, () => {+ const mockToolGroup = {+ name: 'test-group',+ description: 'test description',+ tools: [+ {+ name: 'test-tool',+ description: 'test tool description',+ inputSchema: {},+ execute: () => ({foo: 'bar'}),+ },+ ],+ };+ window.addEventListener('devtoolstooldiscovery', (e: Event) => {+ if ('respondWith' in e && typeof e.respondWith === 'function') {+ e.respondWith(mockToolGroup);+ }+ });+ });++ const page = context.getSelectedMcpPage();+ await page.pptrPage.reload();++ await assert.rejects(+ async () => {+ await executeThirdPartyDeveloperTool(args).handler(+ {+ params: {+ toolName: 'test-tool',+ params: JSON.stringify({}),+ },+ page,+ },+ response,+ context,+ );+ },+ {message: /Tool test-tool not found/},+ );+ },+ undefined,+ {categoryExperimentalThirdParty: true},+ );+ });+it('disposes old handles when executing third party developer tools', async () => {await withMcpContext(async (response, context, args) => {