:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(mcp): send help to search, and stop the catalog hint firing on a stray match (#125426)
Meikel Ratzf5f070d6f5dd
diff --git a/services/mcp/src/tools/exec.ts b/services/mcp/src/tools/exec.tsindex 14f42a162572..3ad544fa89bf 100644--- a/services/mcp/src/tools/exec.ts+++ b/services/mcp/src/tools/exec.ts@@ -112,13 +112,28 @@ export function markNoncanonicalMetricRun(toolName: string, result: unknown): un}}-function catalogDiscoveryHint(allTools: Tool<ZodObjectAny>[], matches: string[]): string | undefined {+// A tool whose description happens to mention the search term says nothing about what the+// search was for: `search persons-` reaches query-logs that way. Only a name match counts.+function namesMatchingSearch(names: string[], term: string): string[] {+ if (isRegexPattern(term)) {+ try {+ const regex = new RegExp(term, 'i')+ return names.filter((name) => regex.test(name))+ } catch {+ return []+ }+ }+ const tokens = term.toLowerCase().split(/\s+/).filter(Boolean)+ return names.filter((name) => tokens.some((token) => name.toLowerCase().includes(token)))+}++function catalogDiscoveryHint(allTools: Tool<ZodObjectAny>[], nameMatches: string[]): string | undefined {const availableToolNames = new Set(allTools.map((tool) => tool.name))const hasMetricCatalog =availableToolNames.has('metric-list') &&availableToolNames.has('metric-describe') &&availableToolNames.has('data-catalog-metric-run')- const hasDataDomainMatch = matches.some((name) =>+ const hasDataDomainMatch = nameMatches.some((name) =>DATA_DOMAIN_TOOL_PREFIXES.some((prefix) => name.startsWith(prefix)))if (!hasMetricCatalog || !hasDataDomainMatch) {@@ -328,8 +343,10 @@ function execHelp(command: string, learnEnabled: boolean): string {}const help = commands[command]if (typeof help !== 'string') {+ // `help <prefix>` is a natural way to ask "what can I do with workflows", so the+ // refusal names the command that answers it rather than only listing the verbs.throw new ExecCommandError(- `Unknown command: "${command}". Run "help" to list available commands.`,+ `Unknown command: "${command}". Run "help" to list available commands, or "search ${command}" to find tools for it.`,'unknown_command')}@@ -1879,7 +1896,11 @@ export function createExecTool(let gatedMatches: ScopeGatedTool[] = []let truncatedFrom = 0let fallbackFrom: string | undefined+ // The word fallback searches something other than what the agent typed, so+ // the catalog hint has to read the matches against the term that found them.+ let searchTerm = queryconst rankByWords = (words: string): void => {+ searchTerm = wordsconst ranked = searchToolsRanked(searchableTools, words)truncatedFrom = ranked.length > MAX_RANKED_SEARCH_RESULTS ? ranked.length : 0matchedNames = ranked.map((r) => r.name)@@ -1946,7 +1967,7 @@ export function createExecTool(? undefined: `No tool is named "${fallbackFrom}". These are the closest matches by its words.`if (truncatedFrom > 0) {- const catalogHint = catalogDiscoveryHint(allTools, matches)+ const catalogHint = catalogDiscoveryHint(allTools, namesMatchingSearch(matchedNames, searchTerm))return JSON.stringify({matches,truncated: true,@@ -1959,7 +1980,7 @@ export function createExecTool(.join(' '),})}- const catalogHint = catalogDiscoveryHint(allTools, matches)+ const catalogHint = catalogDiscoveryHint(allTools, namesMatchingSearch(matchedNames, searchTerm))const hint = [fallbackHint, catalogHint].filter(Boolean).join(' ')if (hint) {return JSON.stringify({ matches, hint })diff --git a/services/mcp/tests/unit/exec.test.ts b/services/mcp/tests/unit/exec.test.tsindex 3759ed96121b..2304fc963f65 100644--- a/services/mcp/tests/unit/exec.test.ts+++ b/services/mcp/tests/unit/exec.test.ts@@ -161,10 +161,11 @@ describe('exec tool', () => {await expect(exec.handler(mockContext, { command: 'help learn' })).resolves.toContain('learn <topic...>')})- it('directs unknown help topics to the command list', async () => {- await expect(createExec().handler(mockContext, { command: 'help unknown' })).rejects.toMatchObject({+ it('directs unknown help topics to the command list and to search', async () => {+ await expect(createExec().handler(mockContext, { command: 'help workflows' })).rejects.toMatchObject({reason: 'unknown_command',- message: 'Unknown command: "unknown". Run "help" to list available commands.',+ message:+ 'Unknown command: "workflows". Run "help" to list available commands, or "search workflows" to find tools for it.',})})})@@ -1775,6 +1776,54 @@ describe('exec tool', () => {expect(JSON.parse(result as string)).toEqual(['feature-flag-get-all'])})+ const catalogTools = [+ makeMockTool({ name: 'metric-list', title: 'List metrics' }),+ makeMockTool({ name: 'metric-describe', title: 'Describe a metric' }),+ makeMockTool({ name: 'data-catalog-metric-run', title: 'Run a metric' }),+ ]++ it('leaves out the metric catalog hint when the only data-domain match is incidental', async () => {+ const exec = createExec([+ makeMockTool({ name: 'persons-list', title: 'List persons' }),+ makeMockTool({ name: 'persons-retrieve', title: 'Get person' }),+ makeMockTool({ name: 'persons-property-set', title: 'Set person properties' }),+ makeMockTool({+ name: 'query-logs',+ title: 'Query logs',+ description: 'Query logs for persons-related events.',+ }),+ ...catalogTools,+ ])++ const result = JSON.parse((await exec.handler(mockContext, { command: 'search persons-' })) as string)++ expect(result.hint).toBeUndefined()+ })++ it('keeps the metric catalog hint when catalog order puts other matches first', async () => {+ // searchToolsRegex filters without ranking, so description matches on unrelated+ // tools come back ahead of the query tools a `query-` search is actually about.+ const exec = createExec([+ makeMockTool({ name: 'cohorts-create', title: 'Create cohort', description: 'Cohorts for a query-.' }),+ makeMockTool({ name: 'cohorts-partial-update', title: 'Update cohort', description: 'A query- edit.' }),+ makeMockTool({ name: 'insight-create', title: 'Create insight', description: 'Insights use query-.' }),+ makeMockTool({ name: 'query-logs', title: 'Query logs' }),+ ...catalogTools,+ ])++ const result = JSON.parse((await exec.handler(mockContext, { command: 'search query-' })) as string)++ expect(result.hint).toContain('metric-list')+ })++ it('keeps the metric catalog hint when a data-domain tool is a top match', async () => {+ const exec = createExec([makeMockTool({ name: 'billing-usage', title: 'Billing usage' }), ...catalogTools])++ const result = JSON.parse((await exec.handler(mockContext, { command: 'search billing' })) as string)++ expect(result.hint).toContain('metric-list')+ })+it('rejects an overly long search pattern before compiling the regex', async () => {const exec = createExec([makeMockTool()])const longPattern = 'a'.repeat(801)