Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(mcp): send help to search, and stop the catalog hint firing on a stray match (#125426)

Meikel Ratzf5f070d6f5dd
diff --git a/services/mcp/src/tools/exec.ts b/services/mcp/src/tools/exec.ts
index 14f42a162572..3ad544fa89bf 100644
--- a/services/mcp/src/tools/exec.ts
+++ b/services/mcp/src/tools/exec.ts
@@ -112,13 +112,28 @@ export function markNoncanonicalMetricRun(toolName: string, result: unknown): un
}
}
-function catalogDiscoveryHint(allTools: Tool<ZodObjectAny>[], matches: string[]): string | undefined {
+// A tool whose description happens to mention the search term says nothing about what the
+// search was for: `search persons-` reaches query-logs that way. Only a name match counts.
+function namesMatchingSearch(names: string[], term: string): string[] {
+ if (isRegexPattern(term)) {
+ try {
+ const regex = new RegExp(term, 'i')
+ return names.filter((name) => regex.test(name))
+ } catch {
+ return []
+ }
+ }
+ const tokens = term.toLowerCase().split(/\s+/).filter(Boolean)
+ return names.filter((name) => tokens.some((token) => name.toLowerCase().includes(token)))
+}
+
+function catalogDiscoveryHint(allTools: Tool<ZodObjectAny>[], nameMatches: string[]): string | undefined {
const availableToolNames = new Set(allTools.map((tool) => tool.name))
const hasMetricCatalog =
availableToolNames.has('metric-list') &&
availableToolNames.has('metric-describe') &&
availableToolNames.has('data-catalog-metric-run')
- const hasDataDomainMatch = matches.some((name) =>
+ const hasDataDomainMatch = nameMatches.some((name) =>
DATA_DOMAIN_TOOL_PREFIXES.some((prefix) => name.startsWith(prefix))
)
if (!hasMetricCatalog || !hasDataDomainMatch) {
@@ -328,8 +343,10 @@ function execHelp(command: string, learnEnabled: boolean): string {
}
const help = commands[command]
if (typeof help !== 'string') {
+ // `help <prefix>` is a natural way to ask "what can I do with workflows", so the
+ // refusal names the command that answers it rather than only listing the verbs.
throw new ExecCommandError(
- `Unknown command: "${command}". Run "help" to list available commands.`,
+ `Unknown command: "${command}". Run "help" to list available commands, or "search ${command}" to find tools for it.`,
'unknown_command'
)
}
@@ -1879,7 +1896,11 @@ export function createExecTool(
let gatedMatches: ScopeGatedTool[] = []
let truncatedFrom = 0
let fallbackFrom: string | undefined
+ // The word fallback searches something other than what the agent typed, so
+ // the catalog hint has to read the matches against the term that found them.
+ let searchTerm = query
const rankByWords = (words: string): void => {
+ searchTerm = words
const ranked = searchToolsRanked(searchableTools, words)
truncatedFrom = ranked.length > MAX_RANKED_SEARCH_RESULTS ? ranked.length : 0
matchedNames = ranked.map((r) => r.name)
@@ -1946,7 +1967,7 @@ export function createExecTool(
? undefined
: `No tool is named "${fallbackFrom}". These are the closest matches by its words.`
if (truncatedFrom > 0) {
- const catalogHint = catalogDiscoveryHint(allTools, matches)
+ const catalogHint = catalogDiscoveryHint(allTools, namesMatchingSearch(matchedNames, searchTerm))
return JSON.stringify({
matches,
truncated: true,
@@ -1959,7 +1980,7 @@ export function createExecTool(
.join(' '),
})
}
- const catalogHint = catalogDiscoveryHint(allTools, matches)
+ const catalogHint = catalogDiscoveryHint(allTools, namesMatchingSearch(matchedNames, searchTerm))
const hint = [fallbackHint, catalogHint].filter(Boolean).join(' ')
if (hint) {
return JSON.stringify({ matches, hint })
diff --git a/services/mcp/tests/unit/exec.test.ts b/services/mcp/tests/unit/exec.test.ts
index 3759ed96121b..2304fc963f65 100644
--- a/services/mcp/tests/unit/exec.test.ts
+++ b/services/mcp/tests/unit/exec.test.ts
@@ -161,10 +161,11 @@ describe('exec tool', () => {
await expect(exec.handler(mockContext, { command: 'help learn' })).resolves.toContain('learn <topic...>')
})
- it('directs unknown help topics to the command list', async () => {
- await expect(createExec().handler(mockContext, { command: 'help unknown' })).rejects.toMatchObject({
+ it('directs unknown help topics to the command list and to search', async () => {
+ await expect(createExec().handler(mockContext, { command: 'help workflows' })).rejects.toMatchObject({
reason: 'unknown_command',
- message: 'Unknown command: "unknown". Run "help" to list available commands.',
+ message:
+ 'Unknown command: "workflows". Run "help" to list available commands, or "search workflows" to find tools for it.',
})
})
})
@@ -1775,6 +1776,54 @@ describe('exec tool', () => {
expect(JSON.parse(result as string)).toEqual(['feature-flag-get-all'])
})
+ const catalogTools = [
+ makeMockTool({ name: 'metric-list', title: 'List metrics' }),
+ makeMockTool({ name: 'metric-describe', title: 'Describe a metric' }),
+ makeMockTool({ name: 'data-catalog-metric-run', title: 'Run a metric' }),
+ ]
+
+ it('leaves out the metric catalog hint when the only data-domain match is incidental', async () => {
+ const exec = createExec([
+ makeMockTool({ name: 'persons-list', title: 'List persons' }),
+ makeMockTool({ name: 'persons-retrieve', title: 'Get person' }),
+ makeMockTool({ name: 'persons-property-set', title: 'Set person properties' }),
+ makeMockTool({
+ name: 'query-logs',
+ title: 'Query logs',
+ description: 'Query logs for persons-related events.',
+ }),
+ ...catalogTools,
+ ])
+
+ const result = JSON.parse((await exec.handler(mockContext, { command: 'search persons-' })) as string)
+
+ expect(result.hint).toBeUndefined()
+ })
+
+ it('keeps the metric catalog hint when catalog order puts other matches first', async () => {
+ // searchToolsRegex filters without ranking, so description matches on unrelated
+ // tools come back ahead of the query tools a `query-` search is actually about.
+ const exec = createExec([
+ makeMockTool({ name: 'cohorts-create', title: 'Create cohort', description: 'Cohorts for a query-.' }),
+ makeMockTool({ name: 'cohorts-partial-update', title: 'Update cohort', description: 'A query- edit.' }),
+ makeMockTool({ name: 'insight-create', title: 'Create insight', description: 'Insights use query-.' }),
+ makeMockTool({ name: 'query-logs', title: 'Query logs' }),
+ ...catalogTools,
+ ])
+
+ const result = JSON.parse((await exec.handler(mockContext, { command: 'search query-' })) as string)
+
+ expect(result.hint).toContain('metric-list')
+ })
+
+ it('keeps the metric catalog hint when a data-domain tool is a top match', async () => {
+ const exec = createExec([makeMockTool({ name: 'billing-usage', title: 'Billing usage' }), ...catalogTools])
+
+ const result = JSON.parse((await exec.handler(mockContext, { command: 'search billing' })) as string)
+
+ expect(result.hint).toContain('metric-list')
+ })
+
it('rejects an overly long search pattern before compiling the regex', async () => {
const exec = createExec([makeMockTool()])
const longPattern = 'a'.repeat(801)