:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(mcp): add flag-gated guidance notices on tool calls (#123607)
Adam Bowker18db5b6cd0f7
diff --git a/services/mcp/src/hono/dispatcher.ts b/services/mcp/src/hono/dispatcher.tsindex a00318a248c4..d5b3ee1b892e 100644--- a/services/mcp/src/hono/dispatcher.ts+++ b/services/mcp/src/hono/dispatcher.ts@@ -42,6 +42,7 @@ import {import { trackInitEvent } from './analytics'import type { RedisLike } from './cache/RedisCache'import { getEnv, MCP_EXEC_SKILLS_FEATURE_FLAG } from './constants'+import { GuidanceEngine } from './guidance/engine'import { InstructionsBuilder } from './instructions'import { InterviewRecruitment, InterviewRequestError } from './interview-recruitment'import { initDurationSeconds, initTotal } from './metrics'@@ -169,7 +170,12 @@ class McpDispatcher {archiveUrl: env.POSTHOG_MCP_SKILLS_URL,expectedArchiveSha: env.POSTHOG_MCP_SKILLS_SHA256,})- this.toolExecutor = new ToolExecutor(catalog, this.instructionsBuilder, this.skillCatalogService)+ this.toolExecutor = new ToolExecutor(+ catalog,+ this.instructionsBuilder,+ this.skillCatalogService,+ new GuidanceEngine(redis)+ )this.interviewRecruitment = new InterviewRecruitment(redis)}diff --git a/services/mcp/src/hono/guidance/engine.ts b/services/mcp/src/hono/guidance/engine.tsnew file mode 100644index 000000000000..bd0e206ff1b8--- /dev/null+++ b/services/mcp/src/hono/guidance/engine.ts@@ -0,0 +1,247 @@+import { createHash } from 'node:crypto'++import { resolveScopePreset } from '@/lib/scope-preset'++import type { RedisLike } from '../cache/RedisCache'+import { resolveSessionKey } from '../mcp-context'+import { guidanceEvaluationErrorsTotal, guidanceNoticesTotal } from '../metrics'+import type { ResolvedState } from '../request-state-resolver'+import { GUIDANCE_RULES, type GuidanceRule } from './rules'++// A conversation handle outlives any single request, so its counters keep the session store's+// idle TTL. Without a handle the counters key on the caller and expire like a 30-minute idle+// session, which is the same sessionization the analytics queries fall back to.+const SESSION_TTL_SECONDS = 24 * 60 * 60+const CALLER_TTL_SECONDS = 30 * 60++export interface GuidanceCall {+ /** The effective tool name: the inner tool for an exec `call`, `exec` for other verbs. */+ toolName: string+ execVerb?: string | undefined+ args: Record<string, unknown> | undefined+ isError: boolean+}++export type GuidanceOutcome = 'shown' | 'flag_off' | 'suppressed' | 'no_consent'++export interface GuidanceNotice {+ ruleId: string+ outcome: GuidanceOutcome+ /** Only present when the notice is shown to the agent. */+ text?: string+}++export function matchesToolPattern(pattern: string, toolName: string): boolean {+ return pattern.endsWith('*') ? toolName.startsWith(pattern.slice(0, -1)) : pattern === toolName+}++function readArgPath(args: Record<string, unknown> | undefined, path: string): unknown {+ return path.split('.').reduce<unknown>((current, segment) => {+ if (typeof current !== 'object' || current === null) {+ return undefined+ }+ return (current as Record<string, unknown>)[segment]+ }, args)+}++export function triggerMatches(rule: GuidanceRule, call: GuidanceCall): boolean {+ const { trigger } = rule+ if (trigger.tool !== undefined && !matchesToolPattern(trigger.tool, call.toolName)) {+ return false+ }+ if (trigger.execVerb !== undefined && trigger.execVerb !== call.execVerb) {+ return false+ }+ if (trigger.outcome === 'success' && call.isError) {+ return false+ }+ if (trigger.outcome === 'error' && !call.isError) {+ return false+ }+ if (trigger.argPattern !== undefined) {+ const value = trigger.argPath === undefined ? call.args : readArgPath(call.args, trigger.argPath)+ const text = typeof value === 'string' ? value : value === undefined ? '' : JSON.stringify(value)+ return trigger.argPattern.test(text)+ }+ return true+}++// Server-minted tokens are not excluded here: scouts carry the same internal scopes as sandbox+// runs, so each rule's `presets` decides which of them it reaches.+function isGuidanceAudience(state: ResolvedState): boolean {+ return !(state.isImpersonated || state.suppressAnalytics || state.clientProfile.consumer === 'posthog_ai')+}++// The flag and org consent decide only whether the text goes out: an eligible caller with the+// flag off, or without AI consent, is counted the same way, so the stamped outcome gives a+// control group and a measure of how often consent blocks the notice.+function ruleApplies(rule: GuidanceRule, state: ResolvedState, preset: string): boolean {+ return (+ rule.trigger.presets.includes(preset as GuidanceRule['trigger']['presets'][number]) &&+ rule.requiresTool.every((name) => state.allTools.some((tool) => tool.name === name))+ )+}++// One script per rule, so a concurrent batch cannot reserve the same notice twice. The shown+// key counts reports of either kind, so a suppressed outcome that loses the call to another+// rule stays pending and is reported on the next matching call instead of being dropped.+const RULE_STATE_SCRIPT = `+local ttl = tonumber(ARGV[1])+if ARGV[4] == 'suppress' then+ redis.call('SET', KEYS[2], '1', 'EX', ttl)+ redis.call('EXPIRE', KEYS[1], ttl)+ redis.call('EXPIRE', KEYS[3], ttl)+ return 0+end+local matched = redis.call('INCR', KEYS[1])+redis.call('EXPIRE', KEYS[1], ttl)+redis.call('EXPIRE', KEYS[2], ttl)+redis.call('EXPIRE', KEYS[3], ttl)+if ARGV[5] ~= '1' or matched < tonumber(ARGV[2]) then+ return 0+end+local shown = redis.call('INCR', KEYS[3])+redis.call('EXPIRE', KEYS[3], ttl)+if shown > tonumber(ARGV[3]) then+ return 0+end+if redis.call('EXISTS', KEYS[2]) == 1 then+ return 2+end+return 1+`++enum RuleResult {+ None = 0,+ Fired = 1,+ Suppressed = 2,+}++export class GuidanceEngine {+ constructor(+ private readonly redis: RedisLike,+ private readonly rules: readonly GuidanceRule[] = GUIDANCE_RULES+ ) {}++ // Never throws: a Redis problem silently yields no notice, so `tools/call` keeps working.+ async evaluate(state: ResolvedState, call: GuidanceCall): Promise<GuidanceNotice | undefined> {+ if (!isGuidanceAudience(state)) {+ return undefined+ }+ const preset = resolveScopePreset(state.apiKeyScopes)+ const rules = this.rules.filter((rule) => ruleApplies(rule, state, preset))+ if (rules.length === 0) {+ return undefined+ }+ try {+ const sessionKey = await this.resolveCountingKey(state)+ const ttl = sessionKey ? SESSION_TTL_SECONDS : CALLER_TTL_SECONDS+ // The braces are a cluster hash tag, so the script can touch a rule's three keys together.+ const scope = `mcp:guidance:{${hashKey(`${state.distinctId}|${sessionKey ?? ''}`)}}`+ let winner: GuidanceNotice | undefined+ for (const rule of rules) {+ const result = await this.touchRule(rule, scope, ttl, call, winner === undefined)+ if (result === RuleResult.Fired) {+ const outcome = firedOutcome(rule, state)+ winner = {+ ruleId: rule.id,+ outcome,+ ...(outcome === 'shown' ? { text: rule.text } : {}),+ }+ } else if (result === RuleResult.Suppressed && !winner) {+ winner = { ruleId: rule.id, outcome: 'suppressed' }+ }+ }+ if (winner) {+ guidanceNoticesTotal.inc({ rule: winner.ruleId, outcome: winner.outcome })+ }+ return winner+ } catch (error) {+ guidanceEvaluationErrorsTotal.inc()+ console.warn('[Guidance] evaluation failed:', error instanceof Error ? error.message : String(error))+ return undefined+ }+ }++ // A call that carries no handle counts under the caller, because the SDK mints a handle on+ // every such call and a client that never echoes one would otherwise never reach a threshold.+ // The minted handle is aliased to the caller, so a client that does echo it keeps the counters+ // and suppression the conversation started with instead of restarting under the handle.+ private async resolveCountingKey(state: ResolvedState): Promise<string | undefined> {+ const handle = resolveSessionKey(state.requestContext)+ if (!handle) {+ return undefined+ }+ const aliasKey = `mcp:guidance:alias:{${hashKey(`${state.distinctId}|${handle}`)}}`+ if (state.carriedContextHandle === false) {+ await this.redis.set(aliasKey, '1', 'EX', SESSION_TTL_SECONDS)+ return undefined+ }+ return (await this.redis.get(aliasKey)) === '1' ? undefined : handle+ }++ private async touchRule(+ rule: GuidanceRule,+ scope: string,+ ttl: number,+ call: GuidanceCall,+ canFire: boolean+ ): Promise<RuleResult> {+ const suppressing = rule.suppressIfCalled.some((pattern) => matchesToolPattern(pattern, call.toolName))+ if (!suppressing && !triggerMatches(rule, call)) {+ return RuleResult.None+ }+ const key = (suffix: string): string => `${scope}:${rule.id}:${suffix}`+ const fired = await this.redis.eval(+ RULE_STATE_SCRIPT,+ 3,+ key('matched'),+ key('suppressed'),+ key('shown'),+ ttl,+ rule.threshold,+ rule.maxPerSession,+ suppressing ? 'suppress' : 'match',+ canFire ? '1' : '0'+ )+ return Number(fired) as RuleResult+ }+}++function firedOutcome(rule: GuidanceRule, state: ResolvedState): GuidanceOutcome {+ if (rule.requiresAiConsent && state.aiConsentGiven !== true) {+ return 'no_consent'+ }+ return state.toolFeatureFlags?.[rule.flag] === true ? 'shown' : 'flag_off'+}++function hashKey(value: string): string {+ return createHash('sha256').update(value).digest('base64url').slice(0, 22)+}++// Attached on errors too: a notice that fires on repeated failures must reach the agent. A JSON+// call keeps its result as the first block, because callers parse `content[0]` as raw JSON.+export function attachGuidanceNotice(+ result: unknown,+ notice: GuidanceNotice | undefined,+ options: { jsonOutput?: boolean } = {}+): unknown {+ if (+ !notice?.text ||+ typeof result !== 'object' ||+ result === null ||+ !Array.isArray((result as { content?: unknown }).content)+ ) {+ return result+ }+ const withContent = result as { content: unknown[] }+ const block = { type: 'text', text: notice.text }+ return {+ ...withContent,+ content: options.jsonOutput ? [...withContent.content, block] : [block, ...withContent.content],+ }+}++export function guidanceAnalyticsProperties(notice: GuidanceNotice | undefined): Record<string, unknown> {+ return notice ? { mcp_guidance_rule: notice.ruleId, mcp_guidance_outcome: notice.outcome } : {}+}diff --git a/services/mcp/src/hono/guidance/rules.ts b/services/mcp/src/hono/guidance/rules.tsnew file mode 100644index 000000000000..df5e883d415c--- /dev/null+++ b/services/mcp/src/hono/guidance/rules.ts@@ -0,0 +1,94 @@+import { z } from 'zod'++import { MCP_SCOPE_PRESET, type McpScopePreset } from '@/lib/scope-preset'++const scopePresets = Object.values(MCP_SCOPE_PRESET) as [McpScopePreset, ...McpScopePreset[]]++const guidanceRuleSchema = z.object({+ id: z.string().regex(/^[a-z0-9-]+$/),+ /** The feature flag that turns the rule on for a caller. Every rule ships off. */+ flag: z.string().min(1),+ trigger: z.object({+ /** Effective tool name (the inner tool for exec calls). A trailing `*` matches a prefix. */+ tool: z.string().min(1).optional(),+ /** A non-call exec verb such as `search`. The effective tool name is `exec` for those. */+ execVerb: z.string().min(1).optional(),+ /** Dot path into the call arguments whose string value `argPattern` must match. */+ argPath: z.string().min(1).optional(),+ argPattern: z.instanceof(RegExp).optional(),+ outcome: z.enum(['any', 'success', 'error']),+ presets: z.array(z.enum(scopePresets)).min(1),+ }),+ /** The notice is shown once the session's matching-call count reaches this. */+ threshold: z.number().int().min(1),+ maxPerSession: z.number().int().min(1),+ /** Once the session has called any of these (same `*` prefix syntax), the rule stays silent. */+ suppressIfCalled: z.array(z.string().min(1)),+ /** Every tool the notice names, so it never points at one the caller's roster lacks. */+ requiresTool: z.array(z.string().min(1)).min(1),+ /** The notice sends the agent to an AI feature, so it is withheld (and stamped) without org consent. */+ requiresAiConsent: z.boolean().optional(),+ text: z.string().min(1).max(2000),+})++export type GuidanceRule = z.infer<typeof guidanceRuleSchema>++export const MCP_GUIDANCE_REPLAY_VISION_FLAG = 'mcp-guidance-replay-vision'+export const MCP_GUIDANCE_SQL_SCHEMA_FLAG = 'mcp-guidance-sql-schema'++const RULES: GuidanceRule[] = [+ {+ id: 'replay-sql-to-vision',+ flag: MCP_GUIDANCE_REPLAY_VISION_FLAG,+ trigger: {+ tool: 'execute-sql',+ argPath: 'query',+ argPattern:+ /\b(raw_)?session_replay_(events|features)\b|\bconsole_logs_log_entries\b|\bsystem\.session_recordings\b/i,+ outcome: 'any',+ presets: ['user'],+ },+ threshold: 2,+ maxPerSession: 1,+ suppressIfCalled: ['vision-*', 'inline-scan'],+ requiresTool: [+ 'vision-scanners-inline-scan',+ 'vision-requests-create',+ 'vision-requests-get',+ 'vision-observations-search',+ ],+ requiresAiConsent: true,+ text: [+ 'SQL over session replay tables returns recording metadata (duration, event counts, URLs), not what happened in the session.',+ 'To answer a question about what users did or saw, Replay Vision can watch the recordings for you:',+ 'call `vision-scanners-inline-scan` with `session_ids` and either `built_in: "summarize_recording"` for a',+ 'general summary or a `prompt` for a specific question, for sessions you already have,',+ 'or `vision-requests-create` for up to 200 sessions and poll `vision-requests-get` for the results.',+ '`vision-observations-search` finds observations from earlier scans by meaning.',+ 'Keep using SQL when the user only needs recording metadata.',+ ].join(' '),+ },+ {+ id: 'sql-failures-to-schema',+ flag: MCP_GUIDANCE_SQL_SCHEMA_FLAG,+ trigger: {+ tool: 'execute-sql',+ outcome: 'error',+ presets: ['user', 'scout'],+ },+ threshold: 3,+ maxPerSession: 1,+ suppressIfCalled: ['read-data-schema'],+ requiresTool: ['read-data-schema'],+ text: [+ 'Three SQL queries in this session have failed.',+ 'Before another attempt, check the schema: query `system.information_schema.tables` and',+ '`system.information_schema.columns` for table and column names, or call `read-data-schema` for',+ 'event and property names. Then rewrite the query against that schema.',+ ].join(' '),+ },+]++export const GUIDANCE_RULES: readonly GuidanceRule[] = z.array(guidanceRuleSchema).parse(RULES)++export const GUIDANCE_RULE_FLAGS: readonly string[] = [...new Set(GUIDANCE_RULES.map((rule) => rule.flag))]diff --git a/services/mcp/src/hono/metrics.ts b/services/mcp/src/hono/metrics.tsindex adee38d8f50f..07cd8c9ae823 100644--- a/services/mcp/src/hono/metrics.ts+++ b/services/mcp/src/hono/metrics.ts@@ -219,4 +219,15 @@ export function routeLabel(pathname: string): string {return 'other'}+export const guidanceNoticesTotal = new Counter({+ name: 'mcp_guidance_notices_total',+ help: 'Guidance rule outcomes on tool calls (shown, flag_off, suppressed, no_consent), by rule.',+ labelNames: ['rule', 'outcome'] as const,+})++export const guidanceEvaluationErrorsTotal = new Counter({+ name: 'mcp_guidance_evaluation_errors_total',+ help: 'Guidance evaluations that failed (Redis errors or timeouts) and yielded no notice.',+})+export { register }diff --git a/services/mcp/src/hono/request-state-resolver.ts b/services/mcp/src/hono/request-state-resolver.tsindex 85143eb079c2..9029de7f4a23 100644--- a/services/mcp/src/hono/request-state-resolver.ts+++ b/services/mcp/src/hono/request-state-resolver.ts@@ -29,6 +29,7 @@ import type { Context, Env, PinnedActiveContext, Tool, ZodObjectAny } from '@/toimport { McpSessionRedisStore } from './cache/McpSessionRedisStore'import type { RedisLike } from './cache/RedisCache'import { MCP_EXEC_SKILLS_FEATURE_FLAG } from './constants'+import { GUIDANCE_RULE_FLAGS } from './guidance/rules'import { MCP_INTENT_PROMPT_FLAG, resolveIntentPrompt } from './intent-prompt'import {type InterviewRecruitmentConfig,@@ -71,6 +72,10 @@ export interface ResolvedState {analyticsContext?: MCPAnalyticsContext | undefined/** Set on the per-call state when the no-data notice blocked or warned this call. */noDataAction?: NoDataAction | undefined+ /** Set on the per-call state: whether the client sent a session or conversation handle. */+ carriedContextHandle?: boolean | undefined+ /** The organization's AI data processing consent; undefined when the org could not be read. */+ aiConsentGiven?: boolean | undefinedoauthClientId: string | undefinedclientProfile: MCPClientProfilerequestContext: MCPRequestContext@@ -202,6 +207,7 @@ export class RequestStateResolver {MCP_INTERVIEW_RECRUITMENT_FLAG,MCP_INTENT_PROMPT_FLAG,MCP_NO_DATA_INSTALL_NOTICE_FLAG,+ ...GUIDANCE_RULE_FLAGS,]),]@@ -350,6 +356,7 @@ export class RequestStateResolver {suppressAnalytics: props.suppressAnalytics,projectHasData: flagAnalyticsContext?.projectHasData,analyticsContext: flagAnalyticsContext,+ aiConsentGiven: aiConsentGiven ?? undefined,oauthClientId: oauthClientId || undefined,clientProfile,requestContext,@@ -481,13 +488,15 @@ export class RequestStateResolver {return { flags: {}, payloads: {} }}// Enable product surfaces locally because the analytics flag client is disabled there.- // Recruitment and intent experiments stay off so ordinary development keeps the default behavior.+ // Recruitment, intent and guidance experiments stay off so ordinary development keeps the default behavior.if (isLocalApi()) {return {flags: Object.fromEntries(flagKeys.map((key) => [key,- key !== MCP_INTERVIEW_RECRUITMENT_FLAG && key !== MCP_INTENT_PROMPT_FLAG,+ key !== MCP_INTERVIEW_RECRUITMENT_FLAG &&+ key !== MCP_INTENT_PROMPT_FLAG &&+ !GUIDANCE_RULE_FLAGS.includes(key),])),payloads: {},diff --git a/services/mcp/src/hono/tool-executor.ts b/services/mcp/src/hono/tool-executor.tsindex 6184b1791078..f50cf385d6bc 100644--- a/services/mcp/src/hono/tool-executor.ts+++ b/services/mcp/src/hono/tool-executor.ts@@ -55,6 +55,7 @@ import {repairArgumentNesting,UNRECOGNIZED_EXEC_TOKEN,type ExecCommandMeta,+ type ExecInnerCallProperties,type ExecInnerCallTracker,} from '@/tools/exec'import { EXECUTE_SQL_TOOL_NAME } from '@/tools/posthogAiTools/executeSql'@@ -72,6 +73,12 @@ import {trackToolsList,type ToolCallAnalyticsMeta,} from './analytics'+import {+ type GuidanceEngine,+ type GuidanceNotice,+ guidanceAnalyticsProperties,+ attachGuidanceNotice,+} from './guidance/engine'import type { InstructionsBuilder } from './instructions'import { getEffectiveMCPClientContext, getEffectiveMCPClientIdentity, resolveSessionKey } from './mcp-context'import { toolCallDurationSeconds, toolCallsTotal, toolErrorsTotal } from './metrics'@@ -88,6 +95,7 @@ interface ResolvedTool {interface ExecMetricState {innerToolName: string | undefined+ innerOutputFormat: ExecInnerCallProperties['output_format'] | undefined/** What the agent asked for, merged onto the event whichever verb ran. */commandMeta: ExecCommandMeta | undefined/**@@ -158,15 +166,25 @@ export class ToolExecutor {private readonly catalog: ToolCatalogprivate readonly instructionsBuilder: InstructionsBuilderprivate readonly skillCatalogService: SkillCatalogService | undefined+ private readonly guidance: GuidanceEngine | undefinedconstructor(catalog: ToolCatalog,instructionsBuilder: InstructionsBuilder,- skillCatalogService?: SkillCatalogService+ skillCatalogService?: SkillCatalogService,+ guidance?: GuidanceEngine) {this.catalog = catalogthis.instructionsBuilder = instructionsBuilderthis.skillCatalogService = skillCatalogService+ this.guidance = guidance+ }++ private async guidanceFor(+ state: ResolvedState,+ call: Parameters<GuidanceEngine['evaluate']>[1]+ ): Promise<GuidanceNotice | undefined> {+ return this.guidance?.evaluate(state, call)}async handleToolsList(state: ResolvedState): Promise<ListToolsResult> {@@ -266,6 +284,7 @@ export class ToolExecutor {}}const callState = stateForToolCall(state, analyticsMeta.intent)+ callState.carriedContextHandle = carriedContextHandleconst callParams = { ...params, arguments: args }const targetToolName =@@ -514,6 +533,11 @@ export class ToolExecutor {stop({ status: 'success' })const duration = Date.now() - startMs+ const guidance = await this.guidanceFor(state, {+ toolName: tool.name,+ args: validation.data,+ isError: false,+ })let response: ToolResultPayloadif (isToolCallPayload(handlerResult)) {@@ -551,6 +575,7 @@ export class ToolExecutor {input_tokens: estimateTokens(validation.data),output_tokens: estimateResponseTokens(response),...toolResultAnalyticsProperties(response),+ ...guidanceAnalyticsProperties(guidance),},analyticsMeta,this.servedToolDescription(tool.name)@@ -573,11 +598,16 @@ export class ToolExecutor {output: response,})- return response+ return attachGuidanceNotice(response, guidance)} catch (error: unknown) {toolCallsTotal.inc({ tool: tool.name, status: 'error' })stop({ status: 'error' })const classification = classifyToolError(error, tool.name)+ const guidance = await this.guidanceFor(state, {+ toolName: tool.name,+ args: validation.data,+ isError: true,+ })// A skill lookup that misses is not a failure the agent should read as// one. The exec dispatcher rewrites the same miss, and a tools-mode@@ -602,6 +632,7 @@ export class ToolExecutor {...inputShape,...errorAnalyticsProperties(classification, error),...(lookupMiss ? skillLookupMissProperties(lookupMiss.kind) : {}),+ ...guidanceAnalyticsProperties(guidance),},analyticsMeta,this.servedToolDescription(tool.name)@@ -630,16 +661,19 @@ export class ToolExecutor {})if (lookupMiss) {- return { content: [{ type: 'text', text: lookupMiss.message }] }+ return attachGuidanceNotice({ content: [{ type: 'text', text: lookupMiss.message }] }, guidance)}const sessionUuid = await sessionUuidForError(state)- return handleToolError(- error,- tool.name,- state.distinctId,- sessionUuid,- state.suppressAnalytics || isPrivateScoutTrialTool(tool.name)+ return attachGuidanceNotice(+ handleToolError(+ error,+ tool.name,+ state.distinctId,+ sessionUuid,+ state.suppressAnalytics || isPrivateScoutTrialTool(tool.name)+ ),+ guidance)}}@@ -659,6 +693,7 @@ export class ToolExecutor {): Promise<unknown> {const execMetrics: ExecMetricState = {innerToolName: undefined,+ innerOutputFormat: undefined,commandMeta: undefined,innerFailure: undefined,skillLookupMissKind: undefined,@@ -747,6 +782,12 @@ export class ToolExecutor {const failureShape = innerFailure? errorAnalyticsProperties(classifyToolError(innerFailure.error, execToolName()), innerFailure.error): undefined+ const guidance = await this.guidanceFor(state, {+ toolName: execToolName(),+ execVerb: typeof execShape.$mcp_exec_verb === 'string' ? execShape.$mcp_exec_verb : undefined,+ args: execMetrics.innerToolName ? parseExecCallInnerArgs(validation.data.command) : validation.data,+ isError: innerFailure !== undefined,+ })void trackToolCall(execToolName(),@@ -765,12 +806,16 @@ export class ToolExecutor {...execMetrics.commandMeta,...(execMetrics.resultEmpty ? { mcp_result_empty: true } : {}),...toolResultAnalyticsProperties(response),+ ...guidanceAnalyticsProperties(guidance),},analyticsMeta,this.servedToolDescription(execToolName()))- return execMetrics.noDataNotice ? prependNoDataNotice(response, execMetrics.noDataNotice.text) : response+ const guided = attachGuidanceNotice(response, guidance, {+ jsonOutput: execMetrics.innerOutputFormat === 'json',+ })+ return execMetrics.noDataNotice ? prependNoDataNotice(guided, execMetrics.noDataNotice.text) : guided} catch (error: unknown) {const metricTool = execToolName()if (error instanceof NoDataInstallNoticeError) {@@ -783,6 +828,12 @@ export class ToolExecutor {)}const classification = classifyToolError(error, metricTool)+ const guidance = await this.guidanceFor(state, {+ toolName: metricTool,+ execVerb: typeof execShape.$mcp_exec_verb === 'string' ? execShape.$mcp_exec_verb : undefined,+ args: execMetrics.innerToolName ? parseExecCallInnerArgs(validation.data.command) : validation.data,+ isError: true,+ })if (!execMetrics.innerToolName) {// Match the inner-tool path, which labels rejected input `validation_error`.const status = classification.errorType === 'validation' ? 'validation_error' : 'error'@@ -799,6 +850,7 @@ export class ToolExecutor {...execInputShape,...errorAnalyticsProperties(classification, error),...execMetrics.commandMeta,+ ...guidanceAnalyticsProperties(guidance),},analyticsMeta,this.servedToolDescription(metricTool)@@ -809,14 +861,17 @@ export class ToolExecutor {// not the `exec` wrapper — so the agent-facing `[tool]` label and the 5xx// exception fingerprint point at the real source instead of collapsing every// exec-routed failure into one opaque `exec` bucket.- return handleToolError(- error,- metricTool,- state.distinctId,- sessionUuid,- state.suppressAnalytics ||- isPrivateScoutTrialTool(metricTool) ||- isPrivateScoutTrialTool(execShape.$mcp_exec_target_tool)+ return attachGuidanceNotice(+ handleToolError(+ error,+ metricTool,+ state.distinctId,+ sessionUuid,+ state.suppressAnalytics ||+ isPrivateScoutTrialTool(metricTool) ||+ isPrivateScoutTrialTool(execShape.$mcp_exec_target_tool)+ ),+ guidance)}}@@ -933,6 +988,7 @@ export class ToolExecutor {// event (now relabelled to the inner tool name, with the inner tool's category// derived from it) already carries this call, so a second emit would double-count.execMetrics.innerToolName = toolName+ execMetrics.innerOutputFormat = properties.output_formatif (!properties.success) {execMetrics.innerFailure = { error: properties.error }}diff --git a/services/mcp/tests/hono/guidance-engine.test.ts b/services/mcp/tests/hono/guidance-engine.test.tsnew file mode 100644index 000000000000..86978a1a6ee6--- /dev/null+++ b/services/mcp/tests/hono/guidance-engine.test.ts@@ -0,0 +1,254 @@+import { describe, expect, it, vi } from 'vitest'++import { GuidanceEngine, triggerMatches, type GuidanceCall } from '@/hono/guidance/engine'+import { GUIDANCE_RULES, type GuidanceRule } from '@/hono/guidance/rules'+import type { ResolvedState } from '@/hono/request-state-resolver'++import { makeToolExecutorState } from '../shared/test-utils'+import { makeGuidanceRedis } from './helpers/guidance-redis-stubs'++const rule: GuidanceRule = {+ id: 'test-rule',+ flag: 'mcp-guidance-test',+ trigger: {+ tool: 'execute-sql',+ argPath: 'query',+ argPattern: /session_replay_events/i,+ outcome: 'any',+ presets: ['user'],+ },+ threshold: 2,+ maxPerSession: 1,+ suppressIfCalled: ['vision-*'],+ requiresTool: ['vision-scanners-inline-scan', 'vision-requests-get'],+ text: 'Try Replay Vision.',+}++const replayQuery: GuidanceCall = {+ toolName: 'execute-sql',+ args: { query: 'select session_id from session_replay_events' },+ isError: false,+}+const otherQuery: GuidanceCall = { toolName: 'execute-sql', args: { query: 'select 1 from events' }, isError: false }+const visionCall: GuidanceCall = { toolName: 'vision-scanners-inline-scan', args: {}, isError: false }++function makeState(overrides: Partial<ResolvedState> = {}): ResolvedState {+ return makeToolExecutorState(+ [{ name: 'execute-sql' }, { name: 'vision-scanners-inline-scan' }, { name: 'vision-requests-get' }],+ {+ toolFeatureFlags: { [rule.flag]: true },+ suppressAnalytics: false,+ ...overrides,+ }+ )+}++async function run(+ engine: GuidanceEngine,+ state: ResolvedState,+ calls: GuidanceCall[]+): Promise<(string | undefined)[]> {+ const shown: (string | undefined)[] = []+ for (const call of calls) {+ const notice = await engine.evaluate(state, call)+ shown.push(notice && `${notice.ruleId}:${notice.outcome}`)+ }+ return shown+}++describe('GuidanceEngine', () => {+ it('shows the notice once the session reaches the threshold, and only once', async () => {+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])+ await expect(run(engine, makeState(), [replayQuery, otherQuery, replayQuery, replayQuery])).resolves.toEqual([+ undefined,+ undefined,+ 'test-rule:shown',+ undefined,+ ])+ })++ it('stamps a suppressed outcome once when the session already called a suggested tool', async () => {+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])+ await expect(run(engine, makeState(), [visionCall, replayQuery, replayQuery, replayQuery])).resolves.toEqual([+ undefined,+ undefined,+ 'test-rule:suppressed',+ undefined,+ ])+ })++ it.each([+ ['without org consent', false, 'no_consent'],+ ['when consent is unknown', undefined, 'no_consent'],+ ['with org consent', true, 'shown'],+ ])('stamps the consent outcome %s for a rule that needs it', async (_label, aiConsentGiven, outcome) => {+ const engine = new GuidanceEngine(makeGuidanceRedis(), [{ ...rule, requiresAiConsent: true }])+ const state = makeState({ aiConsentGiven })+ await engine.evaluate(state, replayQuery)+ const notice = await engine.evaluate(state, replayQuery)+ expect(notice).toMatchObject({ ruleId: 'test-rule', outcome })+ expect(notice?.text === undefined).toBe(outcome !== 'shown')+ })++ it('stamps a flag_off outcome without text when the flag is off', async () => {+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])+ const state = makeState({ toolFeatureFlags: {} })+ await engine.evaluate(state, replayQuery)+ await expect(engine.evaluate(state, replayQuery)).resolves.toEqual({+ ruleId: 'test-rule',+ outcome: 'flag_off',+ })+ })++ it.each([+ [+ 'sessions',+ (first: ResolvedState): Partial<ResolvedState> => ({+ requestContext: { ...first.requestContext, mcpSessionId: 'sess-2' },+ }),+ ],+ ['callers echoing the same session handle', (): Partial<ResolvedState> => ({ distinctId: 'other-caller' })],+ ])('keeps %s apart', async (_label, overrides) => {+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])+ const first = makeState({ requestContext: { ...makeState().requestContext, mcpSessionId: 'sess-1' } })+ const second = makeState({ ...first, ...overrides(first) })+ await run(engine, first, [replayQuery])+ await expect(run(engine, second, [replayQuery, replayQuery])).resolves.toEqual([undefined, 'test-rule:shown'])+ })++ describe('a client that sends no handle', () => {+ const withHandle = (id: string, carriedContextHandle: boolean): ResolvedState =>+ makeState({+ carriedContextHandle,+ requestContext: { ...makeState().requestContext, mcpConversationId: id },+ })++ it('counts minted handles under the caller', async () => {+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])+ await run(engine, withHandle('minted-1', false), [replayQuery])+ await expect(run(engine, withHandle('minted-2', false), [replayQuery])).resolves.toEqual([+ 'test-rule:shown',+ ])+ })++ it.each([+ ['the match count', replayQuery, ['test-rule:shown', undefined]],+ ['suppression', visionCall, [undefined, 'test-rule:suppressed']],+ ])('keeps %s when the client later echoes the minted handle', async (_label, firstCall, expected) => {+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])+ await run(engine, withHandle('minted-1', false), [firstCall])+ await expect(run(engine, withHandle('minted-1', true), [replayQuery, replayQuery])).resolves.toEqual(+ expected+ )+ })+ })++ it('reports a suppressed outcome on a later call when another rule wins the threshold call', async () => {+ const second: GuidanceRule = {+ ...rule,+ id: 'second-rule',+ flag: 'mcp-guidance-second',+ suppressIfCalled: ['read-data-schema'],+ }+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule, second])+ const state = makeState({ toolFeatureFlags: { [rule.flag]: true, [second.flag]: true } })+ const schemaCall: GuidanceCall = { toolName: 'read-data-schema', args: {}, isError: false }+ await expect(run(engine, state, [schemaCall, replayQuery, replayQuery, replayQuery])).resolves.toEqual([+ undefined,+ undefined,+ 'test-rule:shown',+ 'second-rule:suppressed',+ ])+ })++ it('reserves the allowance once when matching calls run concurrently', async () => {+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])+ const state = makeState()+ await engine.evaluate(state, replayQuery)+ const shown = await Promise.all([engine.evaluate(state, replayQuery), engine.evaluate(state, replayQuery)])+ expect(shown.filter(Boolean)).toHaveLength(1)+ })++ it('lets a later rule keep its allowance when an earlier rule fires on the same call', async () => {+ const second: GuidanceRule = { ...rule, id: 'second-rule', flag: 'mcp-guidance-second' }+ const redis = makeGuidanceRedis()+ const engine = new GuidanceEngine(redis, [rule, second])+ const state = makeState({ toolFeatureFlags: { [rule.flag]: true, [second.flag]: true } })+ await run(engine, state, [replayQuery, replayQuery])+ expect([...redis.store.keys()].filter((key) => key.endsWith(':shown'))).toEqual([+ expect.stringContaining(':test-rule:shown'),+ ])+ })++ it.each([+ ['impersonated session', { isImpersonated: true }],+ ['analytics suppressed', { suppressAnalytics: true }],+ ['sandbox token', { apiKeyScopes: ['internal_run:read'] }],+ ['scout token', { apiKeyScopes: ['signal_scout_internal:write'] }],+ ['suggested tool not in roster', { allTools: [{ name: 'execute-sql' }] as ResolvedState['allTools'] }],+ [+ 'one named tool missing from roster',+ {+ allTools: [+ { name: 'execute-sql' },+ { name: 'vision-scanners-inline-scan' },+ ] as ResolvedState['allTools'],+ },+ ],+ ])('shows nothing for %s', async (_label, overrides) => {+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])+ await expect(run(engine, makeState(overrides), [replayQuery, replayQuery])).resolves.toEqual([+ undefined,+ undefined,+ ])+ })++ it('shows the notice to a scout when the rule lists the scout preset', async () => {+ const scoutRule: GuidanceRule = { ...rule, trigger: { ...rule.trigger, presets: ['user', 'scout'] } }+ const engine = new GuidanceEngine(makeGuidanceRedis(), [scoutRule])+ const state = makeState({ apiKeyScopes: ['signal_scout_internal:write', 'internal_run:read'] })+ await expect(run(engine, state, [replayQuery, replayQuery])).resolves.toEqual([undefined, 'test-rule:shown'])+ })++ it('matches only the configured outcome', async () => {+ const errorRule: GuidanceRule = { ...rule, trigger: { ...rule.trigger, outcome: 'error' }, threshold: 1 }+ const engine = new GuidanceEngine(makeGuidanceRedis(), [errorRule])+ await expect(run(engine, makeState(), [replayQuery, { ...replayQuery, isError: true }])).resolves.toEqual([+ undefined,+ 'test-rule:shown',+ ])+ })++ it('shows nothing and logs when Redis fails', async () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const engine = new GuidanceEngine(makeGuidanceRedis(true), [{ ...rule, threshold: 1 }])+ await expect(engine.evaluate(makeState(), replayQuery)).resolves.toBeUndefined()+ expect(warn).toHaveBeenCalledWith('[Guidance] evaluation failed:', expect.any(String))+ warn.mockRestore()+ })++ it.each([+ ['session_replay_events', true],+ ['raw_session_replay_events', true],+ ['session_replay_features', true],+ ['console_logs_log_entries', true],+ ['system.session_recordings', true],+ ['log_entries', false],+ ['batch_export_log_entries', false],+ ['sessions', false],+ ['events', false],+ ])('the replay rule matches SQL over %s: %s', (table, expected) => {+ const replayRule = GUIDANCE_RULES.find((r) => r.id === 'replay-sql-to-vision')!+ const call: GuidanceCall = {+ toolName: 'execute-sql',+ args: { query: `select 1 from ${table}` },+ isError: false,+ }+ expect(triggerMatches(replayRule, call)).toBe(expected)+ })++ it('ships every rule with a distinct id and flag', () => {+ expect(new Set(GUIDANCE_RULES.map((r) => r.id)).size).toBe(GUIDANCE_RULES.length)+ expect(new Set(GUIDANCE_RULES.map((r) => r.flag)).size).toBe(GUIDANCE_RULES.length)+ })+})diff --git a/services/mcp/tests/hono/helpers/guidance-redis-stubs.ts b/services/mcp/tests/hono/helpers/guidance-redis-stubs.tsnew file mode 100644index 000000000000..651f93cdcc75--- /dev/null+++ b/services/mcp/tests/hono/helpers/guidance-redis-stubs.ts@@ -0,0 +1,53 @@+import type { RedisLike } from '@/hono/cache/RedisCache'++export interface GuidanceRedisStub extends RedisLike {+ store: Map<string, string>+ evalCalls: (string | number)[][]+}++export function makeGuidanceRedis(failing = false): GuidanceRedisStub {+ const store = new Map<string, string>()+ const evalCalls: (string | number)[][] = []+ const guard = async <T>(value: T): Promise<T> => {+ if (failing) {+ throw new Error('redis down')+ }+ return value+ }+ const incr = (key: string): number => {+ const next = Number(store.get(key) ?? 0) + 1+ store.set(key, String(next))+ return next+ }+ return {+ store,+ evalCalls,+ get: (key) => guard(store.get(key) ?? null),+ set: (key, value) => {+ store.set(key, value)+ return guard('OK')+ },+ del: async (...keys) => keys.filter((key) => store.delete(key)).length,+ scan: async () => ['0', []],+ incr: (key) => guard(incr(key)),+ expire: () => guard(1),+ ttl: () => guard(-1),+ eval: (_script, _numberOfKeys, ...args) => {+ evalCalls.push(args)+ const [matchedKey, suppressedKey, shownKey, _ttl, threshold, maxPerSession, mode, canFire] =+ args.map(String)+ if (mode === 'suppress') {+ store.set(suppressedKey!, '1')+ return guard(0)+ }+ const matched = incr(matchedKey!)+ if (canFire !== '1' || matched < Number(threshold)) {+ return guard(0)+ }+ if (incr(shownKey!) > Number(maxPerSession)) {+ return guard(0)+ }+ return guard(store.has(suppressedKey!) ? 2 : 1)+ },+ }+}diff --git a/services/mcp/tests/hono/tool-executor-guidance.test.ts b/services/mcp/tests/hono/tool-executor-guidance.test.tsnew file mode 100644index 000000000000..5e1ac4601efc--- /dev/null+++ b/services/mcp/tests/hono/tool-executor-guidance.test.ts@@ -0,0 +1,164 @@+import { afterEach, beforeAll, describe, expect, it, type MockInstance, vi } from 'vitest'+import { z } from 'zod'++vi.mock('@/resources/internals', () => ({+ fetchContextMillResources: vi.fn().mockRejectedValue(new Error('mocked')),+ filterValidEntries: vi.fn().mockReturnValue([]),+ loadManifestFromArchive: vi.fn().mockReturnValue({ resources: [] }),+}))++vi.mock('@/resources', () => ({+ getPromptsFromManifest: vi.fn().mockResolvedValue([]),+}))++vi.mock('@/lib/posthog', async () => {+ const { PostHogMCP } = await import('@posthog/mcp-analytics')+ const client = new PostHogMCP('phc_test', { disabled: true, captureModel: true })+ return { getPostHogClient: () => client }+})++import { GuidanceEngine } from '@/hono/guidance/engine'+import { MCP_GUIDANCE_REPLAY_VISION_FLAG, MCP_GUIDANCE_SQL_SCHEMA_FLAG } from '@/hono/guidance/rules'+import { InstructionsBuilder } from '@/hono/instructions'+import { ToolCatalog } from '@/hono/tool-catalog'+import { ToolExecutor } from '@/hono/tool-executor'+import { getPostHogClient } from '@/lib/posthog'++import { makeToolExecutorState } from '../shared/test-utils'+import { makeGuidanceRedis } from './helpers/guidance-redis-stubs'++function makeTool(name: string, handler: (args: unknown) => unknown): any {+ return {+ name,+ title: name,+ description: `Synthetic ${name}`,+ scopes: [],+ annotations: { readOnlyHint: true },+ schema: z.object({ query: z.string().optional() }),+ handler: vi.fn(async (_context: unknown, args: unknown) => handler(args)),+ }+}++describe('ToolExecutor guidance notices', () => {+ let catalog: ToolCatalog++ beforeAll(async () => {+ catalog = new ToolCatalog()+ await catalog.warmup()+ })++ afterEach(() => {+ vi.restoreAllMocks()+ })++ type SqlResult = { isError?: boolean; content: { type: string; text?: string }[] }++ function setup(+ mode: string,+ sqlHandler: (args: unknown) => unknown,+ flag: string,+ callFlags = ''+ ): {+ callSql: (query: string) => Promise<SqlResult>+ capture: MockInstance<ReturnType<typeof getPostHogClient>['captureToolCall']>+ } {+ const sql = makeTool('execute-sql', sqlHandler)+ const visionTools = [+ 'vision-scanners-inline-scan',+ 'vision-requests-create',+ 'vision-requests-get',+ 'vision-observations-search',+ ].map((name) => makeTool(name, () => ({})))+ const schema = makeTool('read-data-schema', () => ({}))+ const tools = [sql, ...visionTools, schema]+ vi.spyOn(catalog, 'getToolByName').mockImplementation((name) => {+ const tool = tools.find((t) => t.name === name)+ return tool+ ? { build: () => tool, meta: undefined, rawInputSchema: undefined, definition: undefined }+ : undefined+ })+ const executor = new ToolExecutor(+ catalog,+ new InstructionsBuilder(''),+ undefined,+ new GuidanceEngine(makeGuidanceRedis())+ )+ const state = makeToolExecutorState(tools, {+ useSingleExec: mode === 'exec',+ suppressAnalytics: false,+ aiConsentGiven: true,+ toolFeatureFlags: { [flag]: true },+ })+ const capture = vi.spyOn(getPostHogClient(), 'captureToolCall').mockImplementation(() => {})+ const callSql = async (+ query: string+ ): Promise<{ isError?: boolean; content: { type: string; text?: string }[] }> => {+ const args =+ mode === 'exec' ? { command: `call ${callFlags}execute-sql ${JSON.stringify({ query })}` } : { query }+ const result = await executor.handleToolCall(+ { name: mode === 'exec' ? 'exec' : 'execute-sql', arguments: args },+ state+ )+ await new Promise((resolve) => setImmediate(resolve))+ return result as any+ }+ return { callSql, capture }+ }++ it.each(['direct', 'exec'])('prepends the replay notice to the second replay query in %s mode', async (mode) => {+ const { callSql, capture } = setup(mode, () => ({ results: [] }), MCP_GUIDANCE_REPLAY_VISION_FLAG)+ const first = await callSql('select 1 from session_replay_events')+ const second = await callSql('select 2 from session_replay_events')++ expect(first.content[0]?.text).not.toContain('Replay Vision')+ expect(second.content[0]?.text).toContain('Replay Vision')+ expect(second.content).toHaveLength(first.content.length + 1)+ expect(capture.mock.calls.map(([call]) => call.properties?.mcp_guidance_rule)).toEqual([+ undefined,+ 'replay-sql-to-vision',+ ])+ expect(capture.mock.calls[1]?.[0].properties?.mcp_guidance_outcome).toBe('shown')+ })++ it('keeps the JSON result as the first block when guidance fires on a --json call', async () => {+ const { callSql } = setup('exec', () => ({ results: [[1]] }), MCP_GUIDANCE_REPLAY_VISION_FLAG, '--json ')+ await callSql('select 1 from session_replay_events')+ const second = await callSql('select 2 from session_replay_events')++ expect(() => JSON.parse(second.content[0]?.text ?? '')).not.toThrow()+ expect(second.content[1]?.text).toContain('Replay Vision')+ })++ it('stamps flag_off on the event and leaves the result alone when the flag is off', async () => {+ const { callSql, capture } = setup('direct', () => ({ results: [] }), 'some-other-flag')+ const first = await callSql('select 1 from session_replay_events')+ const second = await callSql('select 2 from session_replay_events')++ expect(second.content).toEqual(first.content)+ expect(capture.mock.calls[1]?.[0].properties).toMatchObject({+ mcp_guidance_rule: 'replay-sql-to-vision',+ mcp_guidance_outcome: 'flag_off',+ })+ })++ it.each(['direct', 'exec'])('prepends the schema notice to the third failed query in %s mode', async (mode) => {+ const { callSql, capture } = setup(+ mode,+ () => {+ throw new Error('Unknown column')+ },+ MCP_GUIDANCE_SQL_SCHEMA_FLAG+ )+ await callSql('select a from b')+ await callSql('select a from b')+ const third = await callSql('select a from b')++ expect(third.isError).toBe(true)+ expect(third.content[0]?.text).toContain('read-data-schema')+ expect(capture.mock.calls.map(([call]) => call.properties?.mcp_guidance_rule)).toEqual([+ undefined,+ undefined,+ 'sql-failures-to-schema',+ ])+ })+})