Skip to content

Chrome DevTools for coding agents

chore: make tool_call_metrics.json append only against new changes. (#1778)

yulunzed9e4dd17dd3
diff --git a/scripts/update_tool_call_metrics.ts b/scripts/update_tool_call_metrics.ts
index bbf017f4..f28e2688 100644
--- a/scripts/update_tool_call_metrics.ts
+++ b/scripts/update_tool_call_metrics.ts
@@ -8,7 +8,11 @@ import * as fs from 'node:fs';
import * as path from 'node:path';
import type {ParsedArguments} from '../build/src/bin/chrome-devtools-mcp-cli-options.js';
-import {generateToolMetrics} from '../build/src/telemetry/toolMetricsUtils.js';
+import {
+ applyToExistingMetrics,
+ generateToolMetrics,
+ type ToolMetric,
+} from '../build/src/telemetry/toolMetricsUtils.js';
import type {ToolDefinition} from '../build/src/tools/ToolDefinition.js';
import {createTools} from '../build/src/tools/tools.js';
@@ -35,16 +39,26 @@ function writeToolCallMetricsConfig() {
throw new Error('Error: Duplicate tool names found.');
}
- // Map tools to their metadata
- const toolData = generateToolMetrics(allTools);
+ let existingMetrics: ToolMetric[] = [];
+ if (fs.existsSync(outputPath)) {
+ try {
+ existingMetrics = JSON.parse(
+ fs.readFileSync(outputPath, 'utf8'),
+ ) as ToolMetric[];
+ } catch {
+ console.warn(
+ `Warning: Failed to parse existing metrics from ${outputPath}. Starting fresh.`,
+ );
+ }
+ }
- // Sort by name for determinism
- toolData.sort((a, b) => a.name.localeCompare(b.name));
+ const newMetrics = generateToolMetrics(allTools);
+ const mergedMetrics = applyToExistingMetrics(existingMetrics, newMetrics);
- fs.writeFileSync(outputPath, JSON.stringify(toolData, null, 2) + '\n');
+ fs.writeFileSync(outputPath, JSON.stringify(mergedMetrics, null, 2) + '\n');
console.log(
- `Successfully wrote ${toolData.length} tool names with arguments to ${outputPath}`,
+ `Successfully wrote ${mergedMetrics.length} total tool metrics (including deprecated ones) to ${outputPath}`,
);
}
diff --git a/src/telemetry/toolMetricsUtils.ts b/src/telemetry/toolMetricsUtils.ts
index 124913a6..30ba81b5 100644
--- a/src/telemetry/toolMetricsUtils.ts
+++ b/src/telemetry/toolMetricsUtils.ts
@@ -30,11 +30,61 @@ export function validateEnumHomogeneity(values: unknown[]): string {
export interface ArgMetric {
name: string;
argType: string;
+ isDeprecated?: boolean;
}
export interface ToolMetric {
name: string;
args: ArgMetric[];
+ isDeprecated?: boolean;
+}
+
+export function applyToExistingMetrics(
+ existing: ToolMetric[],
+ update: ToolMetric[],
+): ToolMetric[] {
+ const updated = applyToExisting<ToolMetric>(existing, update);
+ const existingByName = new Map(existing.map(tool => [tool.name, tool]));
+ const updatedByName = new Map(update.map(tool => [tool.name, tool]));
+
+ return updated.map(tool => {
+ const existingTool = existingByName.get(tool.name);
+ const updatedTool = updatedByName.get(tool.name);
+ // If the tool still exists in the update, we will update the args.
+ if (existingTool && updatedTool) {
+ const updatedArgs = applyToExisting<ArgMetric>(
+ existingTool.args,
+ updatedTool.args,
+ );
+ return {...tool, args: updatedArgs};
+ }
+ return tool;
+ });
+}
+
+function applyToExisting<T extends {name: string; isDeprecated?: boolean}>(
+ existing: T[],
+ update: T[],
+): T[] {
+ const existingNames = new Set(existing.map(item => item.name));
+ const updatedNames = new Set(update.map(item => item.name));
+
+ const result: T[] = [];
+ // Keep the original ordering.
+ for (const entry of existing) {
+ const toAdd = {...entry};
+ if (!updatedNames.has(entry.name)) {
+ toAdd.isDeprecated = true;
+ }
+ result.push(toAdd);
+ }
+ // New entries must be added to the very back of the list.
+ for (const entry of update) {
+ if (!existingNames.has(entry.name)) {
+ result.push({...entry});
+ }
+ }
+ return result;
}
/**
diff --git a/tests/telemetry/toolMetricsUtils.test.ts b/tests/telemetry/toolMetricsUtils.test.ts
index a984c428..0c369aae 100644
--- a/tests/telemetry/toolMetricsUtils.test.ts
+++ b/tests/telemetry/toolMetricsUtils.test.ts
@@ -8,6 +8,7 @@ import assert from 'node:assert';
import {describe, it} from 'node:test';
import {
+ applyToExistingMetrics,
generateToolMetrics,
validateEnumHomogeneity,
} from '../../src/telemetry/toolMetricsUtils.js';
@@ -80,4 +81,183 @@ describe('toolMetricsUtils', () => {
assert.strictEqual(metrics[0].args[0].argType, 'string');
});
});
+
+ describe('applyToExistingMetrics', () => {
+ it('should return the same metrics if existing and update are the same', () => {
+ const existing = [{name: 'foo', args: []}];
+ const update = [{name: 'foo', args: []}];
+ const result = applyToExistingMetrics(existing, update);
+ const expected = [{name: 'foo', args: []}];
+ assert.deepStrictEqual(result, expected);
+ });
+
+ it('should append new entries to the end of the array', () => {
+ const existing = [{name: 'foo', args: []}];
+ const update = [
+ {name: 'foo', args: []},
+ {name: 'bar', args: []},
+ ];
+ const result = applyToExistingMetrics(existing, update);
+ const expected = [
+ {name: 'foo', args: []},
+ {name: 'bar', args: []},
+ ];
+ assert.deepStrictEqual(result, expected);
+ });
+
+ it('should mark missing entries as deprecated and preserve their order', () => {
+ const existing = [
+ {name: 'foo', args: []},
+ {name: 'bar', args: []},
+ ];
+ const update = [{name: 'foo', args: []}];
+ const result = applyToExistingMetrics(existing, update);
+ const expected = [
+ {name: 'foo', args: []},
+ {name: 'bar', args: [], isDeprecated: true},
+ ];
+ assert.deepStrictEqual(result, expected);
+ });
+
+ it('should handle adding new entries and deprecating old ones simultaneously', () => {
+ const existing = [
+ {name: 'foo', args: []},
+ {name: 'bar', args: []},
+ ];
+ const update = [
+ {name: 'bar', args: []},
+ {name: 'baz', args: []},
+ ];
+ const result = applyToExistingMetrics(existing, update);
+ const expected = [
+ {name: 'foo', args: [], isDeprecated: true},
+ {name: 'bar', args: []},
+ {name: 'baz', args: []},
+ ];
+ assert.deepStrictEqual(result, expected);
+ });
+
+ it('should append new arguments to the back', () => {
+ const existing = [
+ {name: 'foo', args: [{name: 'arg_a', argType: 'string'}]},
+ ];
+ const update = [
+ {
+ name: 'foo',
+ args: [
+ {name: 'arg_a', argType: 'string'},
+ {name: 'arg_b', argType: 'string'},
+ ],
+ },
+ ];
+ const result = applyToExistingMetrics(existing, update);
+ const expected = [
+ {
+ name: 'foo',
+ args: [
+ {name: 'arg_a', argType: 'string'},
+ {name: 'arg_b', argType: 'string'},
+ ],
+ },
+ ];
+ assert.deepStrictEqual(result, expected);
+ });
+
+ it('should mark removed arguments as deprecated', () => {
+ const existing = [
+ {
+ name: 'foo',
+ args: [
+ {name: 'arg_a', argType: 'string'},
+ {name: 'arg_b', argType: 'string'},
+ ],
+ },
+ ];
+ const update = [
+ {name: 'foo', args: [{name: 'arg_a', argType: 'string'}]},
+ ];
+ const result = applyToExistingMetrics(existing, update);
+ const expected = [
+ {
+ name: 'foo',
+ args: [
+ {name: 'arg_a', argType: 'string'},
+ {name: 'arg_b', argType: 'string', isDeprecated: true},
+ ],
+ },
+ ];
+ assert.deepStrictEqual(result, expected);
+ });
+
+ it('should not change args if they are the same', () => {
+ const existing = [
+ {name: 'foo', args: [{name: 'arg_a', argType: 'string'}]},
+ ];
+ const update = [
+ {name: 'foo', args: [{name: 'arg_a', argType: 'string'}]},
+ ];
+ const result = applyToExistingMetrics(existing, update);
+ const expected = [
+ {name: 'foo', args: [{name: 'arg_a', argType: 'string'}]},
+ ];
+ assert.deepStrictEqual(result, expected);
+ });
+
+ it('should handle adding and removing arguments simultaneously', () => {
+ const existing = [
+ {
+ name: 'foo',
+ args: [
+ {name: 'arg_a', argType: 'string'},
+ {name: 'arg_b', argType: 'string'},
+ ],
+ },
+ ];
+ const update = [
+ {
+ name: 'foo',
+ args: [
+ {name: 'arg_b', argType: 'string'},
+ {name: 'arg_c', argType: 'string'},
+ ],
+ },
+ ];
+ const result = applyToExistingMetrics(existing, update);
+ const expected = [
+ {
+ name: 'foo',
+ args: [
+ {name: 'arg_a', argType: 'string', isDeprecated: true},
+ {name: 'arg_b', argType: 'string'},
+ {name: 'arg_c', argType: 'string'},
+ ],
+ },
+ ];
+ assert.deepStrictEqual(result, expected);
+ });
+
+ it('should handle tool and argument changes simultaneously', () => {
+ const existing = [
+ {name: 'foo', args: [{name: 'arg_a', argType: 'string'}]},
+ {name: 'bar', args: []},
+ ];
+ const update = [
+ {name: 'foo', args: [{name: 'arg_b', argType: 'string'}]},
+ {name: 'baz', args: []},
+ ];
+ const result = applyToExistingMetrics(existing, update);
+ const expected = [
+ {
+ name: 'foo',
+ args: [
+ {name: 'arg_a', argType: 'string', isDeprecated: true},
+ {name: 'arg_b', argType: 'string'},
+ ],
+ },
+ {name: 'bar', args: [], isDeprecated: true},
+ {name: 'baz', args: []},
+ ];
+ assert.deepStrictEqual(result, expected);
+ });
+ });
});