Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(mcp): add flag-gated guidance notices on tool calls (#123607)

Adam Bowker18db5b6cd0f7
diff --git a/services/mcp/src/hono/dispatcher.ts b/services/mcp/src/hono/dispatcher.ts
index a00318a248c4..d5b3ee1b892e 100644
--- a/services/mcp/src/hono/dispatcher.ts
+++ b/services/mcp/src/hono/dispatcher.ts
@@ -42,6 +42,7 @@ import {
import { trackInitEvent } from './analytics'
import type { RedisLike } from './cache/RedisCache'
import { getEnv, MCP_EXEC_SKILLS_FEATURE_FLAG } from './constants'
+import { GuidanceEngine } from './guidance/engine'
import { InstructionsBuilder } from './instructions'
import { InterviewRecruitment, InterviewRequestError } from './interview-recruitment'
import { initDurationSeconds, initTotal } from './metrics'
@@ -169,7 +170,12 @@ class McpDispatcher {
archiveUrl: env.POSTHOG_MCP_SKILLS_URL,
expectedArchiveSha: env.POSTHOG_MCP_SKILLS_SHA256,
})
- this.toolExecutor = new ToolExecutor(catalog, this.instructionsBuilder, this.skillCatalogService)
+ this.toolExecutor = new ToolExecutor(
+ catalog,
+ this.instructionsBuilder,
+ this.skillCatalogService,
+ new GuidanceEngine(redis)
+ )
this.interviewRecruitment = new InterviewRecruitment(redis)
}
diff --git a/services/mcp/src/hono/guidance/engine.ts b/services/mcp/src/hono/guidance/engine.ts
new file mode 100644
index 000000000000..bd0e206ff1b8
--- /dev/null
+++ b/services/mcp/src/hono/guidance/engine.ts
@@ -0,0 +1,247 @@
+import { createHash } from 'node:crypto'
+
+import { resolveScopePreset } from '@/lib/scope-preset'
+
+import type { RedisLike } from '../cache/RedisCache'
+import { resolveSessionKey } from '../mcp-context'
+import { guidanceEvaluationErrorsTotal, guidanceNoticesTotal } from '../metrics'
+import type { ResolvedState } from '../request-state-resolver'
+import { GUIDANCE_RULES, type GuidanceRule } from './rules'
+
+// A conversation handle outlives any single request, so its counters keep the session store's
+// idle TTL. Without a handle the counters key on the caller and expire like a 30-minute idle
+// session, which is the same sessionization the analytics queries fall back to.
+const SESSION_TTL_SECONDS = 24 * 60 * 60
+const CALLER_TTL_SECONDS = 30 * 60
+
+export interface GuidanceCall {
+ /** The effective tool name: the inner tool for an exec `call`, `exec` for other verbs. */
+ toolName: string
+ execVerb?: string | undefined
+ args: Record<string, unknown> | undefined
+ isError: boolean
+}
+
+export type GuidanceOutcome = 'shown' | 'flag_off' | 'suppressed' | 'no_consent'
+
+export interface GuidanceNotice {
+ ruleId: string
+ outcome: GuidanceOutcome
+ /** Only present when the notice is shown to the agent. */
+ text?: string
+}
+
+export function matchesToolPattern(pattern: string, toolName: string): boolean {
+ return pattern.endsWith('*') ? toolName.startsWith(pattern.slice(0, -1)) : pattern === toolName
+}
+
+function readArgPath(args: Record<string, unknown> | undefined, path: string): unknown {
+ return path.split('.').reduce<unknown>((current, segment) => {
+ if (typeof current !== 'object' || current === null) {
+ return undefined
+ }
+ return (current as Record<string, unknown>)[segment]
+ }, args)
+}
+
+export function triggerMatches(rule: GuidanceRule, call: GuidanceCall): boolean {
+ const { trigger } = rule
+ if (trigger.tool !== undefined && !matchesToolPattern(trigger.tool, call.toolName)) {
+ return false
+ }
+ if (trigger.execVerb !== undefined && trigger.execVerb !== call.execVerb) {
+ return false
+ }
+ if (trigger.outcome === 'success' && call.isError) {
+ return false
+ }
+ if (trigger.outcome === 'error' && !call.isError) {
+ return false
+ }
+ if (trigger.argPattern !== undefined) {
+ const value = trigger.argPath === undefined ? call.args : readArgPath(call.args, trigger.argPath)
+ const text = typeof value === 'string' ? value : value === undefined ? '' : JSON.stringify(value)
+ return trigger.argPattern.test(text)
+ }
+ return true
+}
+
+// Server-minted tokens are not excluded here: scouts carry the same internal scopes as sandbox
+// runs, so each rule's `presets` decides which of them it reaches.
+function isGuidanceAudience(state: ResolvedState): boolean {
+ return !(state.isImpersonated || state.suppressAnalytics || state.clientProfile.consumer === 'posthog_ai')
+}
+
+// The flag and org consent decide only whether the text goes out: an eligible caller with the
+// flag off, or without AI consent, is counted the same way, so the stamped outcome gives a
+// control group and a measure of how often consent blocks the notice.
+function ruleApplies(rule: GuidanceRule, state: ResolvedState, preset: string): boolean {
+ return (
+ rule.trigger.presets.includes(preset as GuidanceRule['trigger']['presets'][number]) &&
+ rule.requiresTool.every((name) => state.allTools.some((tool) => tool.name === name))
+ )
+}
+
+// One script per rule, so a concurrent batch cannot reserve the same notice twice. The shown
+// key counts reports of either kind, so a suppressed outcome that loses the call to another
+// rule stays pending and is reported on the next matching call instead of being dropped.
+const RULE_STATE_SCRIPT = `
+local ttl = tonumber(ARGV[1])
+if ARGV[4] == 'suppress' then
+ redis.call('SET', KEYS[2], '1', 'EX', ttl)
+ redis.call('EXPIRE', KEYS[1], ttl)
+ redis.call('EXPIRE', KEYS[3], ttl)
+ return 0
+end
+local matched = redis.call('INCR', KEYS[1])
+redis.call('EXPIRE', KEYS[1], ttl)
+redis.call('EXPIRE', KEYS[2], ttl)
+redis.call('EXPIRE', KEYS[3], ttl)
+if ARGV[5] ~= '1' or matched < tonumber(ARGV[2]) then
+ return 0
+end
+local shown = redis.call('INCR', KEYS[3])
+redis.call('EXPIRE', KEYS[3], ttl)
+if shown > tonumber(ARGV[3]) then
+ return 0
+end
+if redis.call('EXISTS', KEYS[2]) == 1 then
+ return 2
+end
+return 1
+`
+
+enum RuleResult {
+ None = 0,
+ Fired = 1,
+ Suppressed = 2,
+}
+
+export class GuidanceEngine {
+ constructor(
+ private readonly redis: RedisLike,
+ private readonly rules: readonly GuidanceRule[] = GUIDANCE_RULES
+ ) {}
+
+ // Never throws: a Redis problem silently yields no notice, so `tools/call` keeps working.
+ async evaluate(state: ResolvedState, call: GuidanceCall): Promise<GuidanceNotice | undefined> {
+ if (!isGuidanceAudience(state)) {
+ return undefined
+ }
+ const preset = resolveScopePreset(state.apiKeyScopes)
+ const rules = this.rules.filter((rule) => ruleApplies(rule, state, preset))
+ if (rules.length === 0) {
+ return undefined
+ }
+ try {
+ const sessionKey = await this.resolveCountingKey(state)
+ const ttl = sessionKey ? SESSION_TTL_SECONDS : CALLER_TTL_SECONDS
+ // The braces are a cluster hash tag, so the script can touch a rule's three keys together.
+ const scope = `mcp:guidance:{${hashKey(`${state.distinctId}|${sessionKey ?? ''}`)}}`
+ let winner: GuidanceNotice | undefined
+ for (const rule of rules) {
+ const result = await this.touchRule(rule, scope, ttl, call, winner === undefined)
+ if (result === RuleResult.Fired) {
+ const outcome = firedOutcome(rule, state)
+ winner = {
+ ruleId: rule.id,
+ outcome,
+ ...(outcome === 'shown' ? { text: rule.text } : {}),
+ }
+ } else if (result === RuleResult.Suppressed && !winner) {
+ winner = { ruleId: rule.id, outcome: 'suppressed' }
+ }
+ }
+ if (winner) {
+ guidanceNoticesTotal.inc({ rule: winner.ruleId, outcome: winner.outcome })
+ }
+ return winner
+ } catch (error) {
+ guidanceEvaluationErrorsTotal.inc()
+ console.warn('[Guidance] evaluation failed:', error instanceof Error ? error.message : String(error))
+ return undefined
+ }
+ }
+
+ // A call that carries no handle counts under the caller, because the SDK mints a handle on
+ // every such call and a client that never echoes one would otherwise never reach a threshold.
+ // The minted handle is aliased to the caller, so a client that does echo it keeps the counters
+ // and suppression the conversation started with instead of restarting under the handle.
+ private async resolveCountingKey(state: ResolvedState): Promise<string | undefined> {
+ const handle = resolveSessionKey(state.requestContext)
+ if (!handle) {
+ return undefined
+ }
+ const aliasKey = `mcp:guidance:alias:{${hashKey(`${state.distinctId}|${handle}`)}}`
+ if (state.carriedContextHandle === false) {
+ await this.redis.set(aliasKey, '1', 'EX', SESSION_TTL_SECONDS)
+ return undefined
+ }
+ return (await this.redis.get(aliasKey)) === '1' ? undefined : handle
+ }
+
+ private async touchRule(
+ rule: GuidanceRule,
+ scope: string,
+ ttl: number,
+ call: GuidanceCall,
+ canFire: boolean
+ ): Promise<RuleResult> {
+ const suppressing = rule.suppressIfCalled.some((pattern) => matchesToolPattern(pattern, call.toolName))
+ if (!suppressing && !triggerMatches(rule, call)) {
+ return RuleResult.None
+ }
+ const key = (suffix: string): string => `${scope}:${rule.id}:${suffix}`
+ const fired = await this.redis.eval(
+ RULE_STATE_SCRIPT,
+ 3,
+ key('matched'),
+ key('suppressed'),
+ key('shown'),
+ ttl,
+ rule.threshold,
+ rule.maxPerSession,
+ suppressing ? 'suppress' : 'match',
+ canFire ? '1' : '0'
+ )
+ return Number(fired) as RuleResult
+ }
+}
+
+function firedOutcome(rule: GuidanceRule, state: ResolvedState): GuidanceOutcome {
+ if (rule.requiresAiConsent && state.aiConsentGiven !== true) {
+ return 'no_consent'
+ }
+ return state.toolFeatureFlags?.[rule.flag] === true ? 'shown' : 'flag_off'
+}
+
+function hashKey(value: string): string {
+ return createHash('sha256').update(value).digest('base64url').slice(0, 22)
+}
+
+// Attached on errors too: a notice that fires on repeated failures must reach the agent. A JSON
+// call keeps its result as the first block, because callers parse `content[0]` as raw JSON.
+export function attachGuidanceNotice(
+ result: unknown,
+ notice: GuidanceNotice | undefined,
+ options: { jsonOutput?: boolean } = {}
+): unknown {
+ if (
+ !notice?.text ||
+ typeof result !== 'object' ||
+ result === null ||
+ !Array.isArray((result as { content?: unknown }).content)
+ ) {
+ return result
+ }
+ const withContent = result as { content: unknown[] }
+ const block = { type: 'text', text: notice.text }
+ return {
+ ...withContent,
+ content: options.jsonOutput ? [...withContent.content, block] : [block, ...withContent.content],
+ }
+}
+
+export function guidanceAnalyticsProperties(notice: GuidanceNotice | undefined): Record<string, unknown> {
+ return notice ? { mcp_guidance_rule: notice.ruleId, mcp_guidance_outcome: notice.outcome } : {}
+}
diff --git a/services/mcp/src/hono/guidance/rules.ts b/services/mcp/src/hono/guidance/rules.ts
new file mode 100644
index 000000000000..df5e883d415c
--- /dev/null
+++ b/services/mcp/src/hono/guidance/rules.ts
@@ -0,0 +1,94 @@
+import { z } from 'zod'
+
+import { MCP_SCOPE_PRESET, type McpScopePreset } from '@/lib/scope-preset'
+
+const scopePresets = Object.values(MCP_SCOPE_PRESET) as [McpScopePreset, ...McpScopePreset[]]
+
+const guidanceRuleSchema = z.object({
+ id: z.string().regex(/^[a-z0-9-]+$/),
+ /** The feature flag that turns the rule on for a caller. Every rule ships off. */
+ flag: z.string().min(1),
+ trigger: z.object({
+ /** Effective tool name (the inner tool for exec calls). A trailing `*` matches a prefix. */
+ tool: z.string().min(1).optional(),
+ /** A non-call exec verb such as `search`. The effective tool name is `exec` for those. */
+ execVerb: z.string().min(1).optional(),
+ /** Dot path into the call arguments whose string value `argPattern` must match. */
+ argPath: z.string().min(1).optional(),
+ argPattern: z.instanceof(RegExp).optional(),
+ outcome: z.enum(['any', 'success', 'error']),
+ presets: z.array(z.enum(scopePresets)).min(1),
+ }),
+ /** The notice is shown once the session's matching-call count reaches this. */
+ threshold: z.number().int().min(1),
+ maxPerSession: z.number().int().min(1),
+ /** Once the session has called any of these (same `*` prefix syntax), the rule stays silent. */
+ suppressIfCalled: z.array(z.string().min(1)),
+ /** Every tool the notice names, so it never points at one the caller's roster lacks. */
+ requiresTool: z.array(z.string().min(1)).min(1),
+ /** The notice sends the agent to an AI feature, so it is withheld (and stamped) without org consent. */
+ requiresAiConsent: z.boolean().optional(),
+ text: z.string().min(1).max(2000),
+})
+
+export type GuidanceRule = z.infer<typeof guidanceRuleSchema>
+
+export const MCP_GUIDANCE_REPLAY_VISION_FLAG = 'mcp-guidance-replay-vision'
+export const MCP_GUIDANCE_SQL_SCHEMA_FLAG = 'mcp-guidance-sql-schema'
+
+const RULES: GuidanceRule[] = [
+ {
+ id: 'replay-sql-to-vision',
+ flag: MCP_GUIDANCE_REPLAY_VISION_FLAG,
+ trigger: {
+ tool: 'execute-sql',
+ argPath: 'query',
+ argPattern:
+ /\b(raw_)?session_replay_(events|features)\b|\bconsole_logs_log_entries\b|\bsystem\.session_recordings\b/i,
+ outcome: 'any',
+ presets: ['user'],
+ },
+ threshold: 2,
+ maxPerSession: 1,
+ suppressIfCalled: ['vision-*', 'inline-scan'],
+ requiresTool: [
+ 'vision-scanners-inline-scan',
+ 'vision-requests-create',
+ 'vision-requests-get',
+ 'vision-observations-search',
+ ],
+ requiresAiConsent: true,
+ text: [
+ 'SQL over session replay tables returns recording metadata (duration, event counts, URLs), not what happened in the session.',
+ 'To answer a question about what users did or saw, Replay Vision can watch the recordings for you:',
+ 'call `vision-scanners-inline-scan` with `session_ids` and either `built_in: "summarize_recording"` for a',
+ 'general summary or a `prompt` for a specific question, for sessions you already have,',
+ 'or `vision-requests-create` for up to 200 sessions and poll `vision-requests-get` for the results.',
+ '`vision-observations-search` finds observations from earlier scans by meaning.',
+ 'Keep using SQL when the user only needs recording metadata.',
+ ].join(' '),
+ },
+ {
+ id: 'sql-failures-to-schema',
+ flag: MCP_GUIDANCE_SQL_SCHEMA_FLAG,
+ trigger: {
+ tool: 'execute-sql',
+ outcome: 'error',
+ presets: ['user', 'scout'],
+ },
+ threshold: 3,
+ maxPerSession: 1,
+ suppressIfCalled: ['read-data-schema'],
+ requiresTool: ['read-data-schema'],
+ text: [
+ 'Three SQL queries in this session have failed.',
+ 'Before another attempt, check the schema: query `system.information_schema.tables` and',
+ '`system.information_schema.columns` for table and column names, or call `read-data-schema` for',
+ 'event and property names. Then rewrite the query against that schema.',
+ ].join(' '),
+ },
+]
+
+export const GUIDANCE_RULES: readonly GuidanceRule[] = z.array(guidanceRuleSchema).parse(RULES)
+
+export const GUIDANCE_RULE_FLAGS: readonly string[] = [...new Set(GUIDANCE_RULES.map((rule) => rule.flag))]
diff --git a/services/mcp/src/hono/metrics.ts b/services/mcp/src/hono/metrics.ts
index adee38d8f50f..07cd8c9ae823 100644
--- a/services/mcp/src/hono/metrics.ts
+++ b/services/mcp/src/hono/metrics.ts
@@ -219,4 +219,15 @@ export function routeLabel(pathname: string): string {
return 'other'
}
+export const guidanceNoticesTotal = new Counter({
+ name: 'mcp_guidance_notices_total',
+ help: 'Guidance rule outcomes on tool calls (shown, flag_off, suppressed, no_consent), by rule.',
+ labelNames: ['rule', 'outcome'] as const,
+})
+
+export const guidanceEvaluationErrorsTotal = new Counter({
+ name: 'mcp_guidance_evaluation_errors_total',
+ help: 'Guidance evaluations that failed (Redis errors or timeouts) and yielded no notice.',
+})
+
export { register }
diff --git a/services/mcp/src/hono/request-state-resolver.ts b/services/mcp/src/hono/request-state-resolver.ts
index 85143eb079c2..9029de7f4a23 100644
--- a/services/mcp/src/hono/request-state-resolver.ts
+++ b/services/mcp/src/hono/request-state-resolver.ts
@@ -29,6 +29,7 @@ import type { Context, Env, PinnedActiveContext, Tool, ZodObjectAny } from '@/to
import { McpSessionRedisStore } from './cache/McpSessionRedisStore'
import type { RedisLike } from './cache/RedisCache'
import { MCP_EXEC_SKILLS_FEATURE_FLAG } from './constants'
+import { GUIDANCE_RULE_FLAGS } from './guidance/rules'
import { MCP_INTENT_PROMPT_FLAG, resolveIntentPrompt } from './intent-prompt'
import {
type InterviewRecruitmentConfig,
@@ -71,6 +72,10 @@ export interface ResolvedState {
analyticsContext?: MCPAnalyticsContext | undefined
/** Set on the per-call state when the no-data notice blocked or warned this call. */
noDataAction?: NoDataAction | undefined
+ /** Set on the per-call state: whether the client sent a session or conversation handle. */
+ carriedContextHandle?: boolean | undefined
+ /** The organization's AI data processing consent; undefined when the org could not be read. */
+ aiConsentGiven?: boolean | undefined
oauthClientId: string | undefined
clientProfile: MCPClientProfile
requestContext: MCPRequestContext
@@ -202,6 +207,7 @@ export class RequestStateResolver {
MCP_INTERVIEW_RECRUITMENT_FLAG,
MCP_INTENT_PROMPT_FLAG,
MCP_NO_DATA_INSTALL_NOTICE_FLAG,
+ ...GUIDANCE_RULE_FLAGS,
]),
]
@@ -350,6 +356,7 @@ export class RequestStateResolver {
suppressAnalytics: props.suppressAnalytics,
projectHasData: flagAnalyticsContext?.projectHasData,
analyticsContext: flagAnalyticsContext,
+ aiConsentGiven: aiConsentGiven ?? undefined,
oauthClientId: oauthClientId || undefined,
clientProfile,
requestContext,
@@ -481,13 +488,15 @@ export class RequestStateResolver {
return { flags: {}, payloads: {} }
}
// Enable product surfaces locally because the analytics flag client is disabled there.
- // Recruitment and intent experiments stay off so ordinary development keeps the default behavior.
+ // Recruitment, intent and guidance experiments stay off so ordinary development keeps the default behavior.
if (isLocalApi()) {
return {
flags: Object.fromEntries(
flagKeys.map((key) => [
key,
- key !== MCP_INTERVIEW_RECRUITMENT_FLAG && key !== MCP_INTENT_PROMPT_FLAG,
+ key !== MCP_INTERVIEW_RECRUITMENT_FLAG &&
+ key !== MCP_INTENT_PROMPT_FLAG &&
+ !GUIDANCE_RULE_FLAGS.includes(key),
])
),
payloads: {},
diff --git a/services/mcp/src/hono/tool-executor.ts b/services/mcp/src/hono/tool-executor.ts
index 6184b1791078..f50cf385d6bc 100644
--- a/services/mcp/src/hono/tool-executor.ts
+++ b/services/mcp/src/hono/tool-executor.ts
@@ -55,6 +55,7 @@ import {
repairArgumentNesting,
UNRECOGNIZED_EXEC_TOKEN,
type ExecCommandMeta,
+ type ExecInnerCallProperties,
type ExecInnerCallTracker,
} from '@/tools/exec'
import { EXECUTE_SQL_TOOL_NAME } from '@/tools/posthogAiTools/executeSql'
@@ -72,6 +73,12 @@ import {
trackToolsList,
type ToolCallAnalyticsMeta,
} from './analytics'
+import {
+ type GuidanceEngine,
+ type GuidanceNotice,
+ guidanceAnalyticsProperties,
+ attachGuidanceNotice,
+} from './guidance/engine'
import type { InstructionsBuilder } from './instructions'
import { getEffectiveMCPClientContext, getEffectiveMCPClientIdentity, resolveSessionKey } from './mcp-context'
import { toolCallDurationSeconds, toolCallsTotal, toolErrorsTotal } from './metrics'
@@ -88,6 +95,7 @@ interface ResolvedTool {
interface ExecMetricState {
innerToolName: string | undefined
+ innerOutputFormat: ExecInnerCallProperties['output_format'] | undefined
/** What the agent asked for, merged onto the event whichever verb ran. */
commandMeta: ExecCommandMeta | undefined
/**
@@ -158,15 +166,25 @@ export class ToolExecutor {
private readonly catalog: ToolCatalog
private readonly instructionsBuilder: InstructionsBuilder
private readonly skillCatalogService: SkillCatalogService | undefined
+ private readonly guidance: GuidanceEngine | undefined
constructor(
catalog: ToolCatalog,
instructionsBuilder: InstructionsBuilder,
- skillCatalogService?: SkillCatalogService
+ skillCatalogService?: SkillCatalogService,
+ guidance?: GuidanceEngine
) {
this.catalog = catalog
this.instructionsBuilder = instructionsBuilder
this.skillCatalogService = skillCatalogService
+ this.guidance = guidance
+ }
+
+ private async guidanceFor(
+ state: ResolvedState,
+ call: Parameters<GuidanceEngine['evaluate']>[1]
+ ): Promise<GuidanceNotice | undefined> {
+ return this.guidance?.evaluate(state, call)
}
async handleToolsList(state: ResolvedState): Promise<ListToolsResult> {
@@ -266,6 +284,7 @@ export class ToolExecutor {
}
}
const callState = stateForToolCall(state, analyticsMeta.intent)
+ callState.carriedContextHandle = carriedContextHandle
const callParams = { ...params, arguments: args }
const targetToolName =
@@ -514,6 +533,11 @@ export class ToolExecutor {
stop({ status: 'success' })
const duration = Date.now() - startMs
+ const guidance = await this.guidanceFor(state, {
+ toolName: tool.name,
+ args: validation.data,
+ isError: false,
+ })
let response: ToolResultPayload
if (isToolCallPayload(handlerResult)) {
@@ -551,6 +575,7 @@ export class ToolExecutor {
input_tokens: estimateTokens(validation.data),
output_tokens: estimateResponseTokens(response),
...toolResultAnalyticsProperties(response),
+ ...guidanceAnalyticsProperties(guidance),
},
analyticsMeta,
this.servedToolDescription(tool.name)
@@ -573,11 +598,16 @@ export class ToolExecutor {
output: response,
})
- return response
+ return attachGuidanceNotice(response, guidance)
} catch (error: unknown) {
toolCallsTotal.inc({ tool: tool.name, status: 'error' })
stop({ status: 'error' })
const classification = classifyToolError(error, tool.name)
+ const guidance = await this.guidanceFor(state, {
+ toolName: tool.name,
+ args: validation.data,
+ isError: true,
+ })
// A skill lookup that misses is not a failure the agent should read as
// one. The exec dispatcher rewrites the same miss, and a tools-mode
@@ -602,6 +632,7 @@ export class ToolExecutor {
...inputShape,
...errorAnalyticsProperties(classification, error),
...(lookupMiss ? skillLookupMissProperties(lookupMiss.kind) : {}),
+ ...guidanceAnalyticsProperties(guidance),
},
analyticsMeta,
this.servedToolDescription(tool.name)
@@ -630,16 +661,19 @@ export class ToolExecutor {
})
if (lookupMiss) {
- return { content: [{ type: 'text', text: lookupMiss.message }] }
+ return attachGuidanceNotice({ content: [{ type: 'text', text: lookupMiss.message }] }, guidance)
}
const sessionUuid = await sessionUuidForError(state)
- return handleToolError(
- error,
- tool.name,
- state.distinctId,
- sessionUuid,
- state.suppressAnalytics || isPrivateScoutTrialTool(tool.name)
+ return attachGuidanceNotice(
+ handleToolError(
+ error,
+ tool.name,
+ state.distinctId,
+ sessionUuid,
+ state.suppressAnalytics || isPrivateScoutTrialTool(tool.name)
+ ),
+ guidance
)
}
}
@@ -659,6 +693,7 @@ export class ToolExecutor {
): Promise<unknown> {
const execMetrics: ExecMetricState = {
innerToolName: undefined,
+ innerOutputFormat: undefined,
commandMeta: undefined,
innerFailure: undefined,
skillLookupMissKind: undefined,
@@ -747,6 +782,12 @@ export class ToolExecutor {
const failureShape = innerFailure
? errorAnalyticsProperties(classifyToolError(innerFailure.error, execToolName()), innerFailure.error)
: undefined
+ const guidance = await this.guidanceFor(state, {
+ toolName: execToolName(),
+ execVerb: typeof execShape.$mcp_exec_verb === 'string' ? execShape.$mcp_exec_verb : undefined,
+ args: execMetrics.innerToolName ? parseExecCallInnerArgs(validation.data.command) : validation.data,
+ isError: innerFailure !== undefined,
+ })
void trackToolCall(
execToolName(),
@@ -765,12 +806,16 @@ export class ToolExecutor {
...execMetrics.commandMeta,
...(execMetrics.resultEmpty ? { mcp_result_empty: true } : {}),
...toolResultAnalyticsProperties(response),
+ ...guidanceAnalyticsProperties(guidance),
},
analyticsMeta,
this.servedToolDescription(execToolName())
)
- return execMetrics.noDataNotice ? prependNoDataNotice(response, execMetrics.noDataNotice.text) : response
+ const guided = attachGuidanceNotice(response, guidance, {
+ jsonOutput: execMetrics.innerOutputFormat === 'json',
+ })
+ return execMetrics.noDataNotice ? prependNoDataNotice(guided, execMetrics.noDataNotice.text) : guided
} catch (error: unknown) {
const metricTool = execToolName()
if (error instanceof NoDataInstallNoticeError) {
@@ -783,6 +828,12 @@ export class ToolExecutor {
)
}
const classification = classifyToolError(error, metricTool)
+ const guidance = await this.guidanceFor(state, {
+ toolName: metricTool,
+ execVerb: typeof execShape.$mcp_exec_verb === 'string' ? execShape.$mcp_exec_verb : undefined,
+ args: execMetrics.innerToolName ? parseExecCallInnerArgs(validation.data.command) : validation.data,
+ isError: true,
+ })
if (!execMetrics.innerToolName) {
// Match the inner-tool path, which labels rejected input `validation_error`.
const status = classification.errorType === 'validation' ? 'validation_error' : 'error'
@@ -799,6 +850,7 @@ export class ToolExecutor {
...execInputShape,
...errorAnalyticsProperties(classification, error),
...execMetrics.commandMeta,
+ ...guidanceAnalyticsProperties(guidance),
},
analyticsMeta,
this.servedToolDescription(metricTool)
@@ -809,14 +861,17 @@ export class ToolExecutor {
// not the `exec` wrapper — so the agent-facing `[tool]` label and the 5xx
// exception fingerprint point at the real source instead of collapsing every
// exec-routed failure into one opaque `exec` bucket.
- return handleToolError(
- error,
- metricTool,
- state.distinctId,
- sessionUuid,
- state.suppressAnalytics ||
- isPrivateScoutTrialTool(metricTool) ||
- isPrivateScoutTrialTool(execShape.$mcp_exec_target_tool)
+ return attachGuidanceNotice(
+ handleToolError(
+ error,
+ metricTool,
+ state.distinctId,
+ sessionUuid,
+ state.suppressAnalytics ||
+ isPrivateScoutTrialTool(metricTool) ||
+ isPrivateScoutTrialTool(execShape.$mcp_exec_target_tool)
+ ),
+ guidance
)
}
}
@@ -933,6 +988,7 @@ export class ToolExecutor {
// event (now relabelled to the inner tool name, with the inner tool's category
// derived from it) already carries this call, so a second emit would double-count.
execMetrics.innerToolName = toolName
+ execMetrics.innerOutputFormat = properties.output_format
if (!properties.success) {
execMetrics.innerFailure = { error: properties.error }
}
diff --git a/services/mcp/tests/hono/guidance-engine.test.ts b/services/mcp/tests/hono/guidance-engine.test.ts
new file mode 100644
index 000000000000..86978a1a6ee6
--- /dev/null
+++ b/services/mcp/tests/hono/guidance-engine.test.ts
@@ -0,0 +1,254 @@
+import { describe, expect, it, vi } from 'vitest'
+
+import { GuidanceEngine, triggerMatches, type GuidanceCall } from '@/hono/guidance/engine'
+import { GUIDANCE_RULES, type GuidanceRule } from '@/hono/guidance/rules'
+import type { ResolvedState } from '@/hono/request-state-resolver'
+
+import { makeToolExecutorState } from '../shared/test-utils'
+import { makeGuidanceRedis } from './helpers/guidance-redis-stubs'
+
+const rule: GuidanceRule = {
+ id: 'test-rule',
+ flag: 'mcp-guidance-test',
+ trigger: {
+ tool: 'execute-sql',
+ argPath: 'query',
+ argPattern: /session_replay_events/i,
+ outcome: 'any',
+ presets: ['user'],
+ },
+ threshold: 2,
+ maxPerSession: 1,
+ suppressIfCalled: ['vision-*'],
+ requiresTool: ['vision-scanners-inline-scan', 'vision-requests-get'],
+ text: 'Try Replay Vision.',
+}
+
+const replayQuery: GuidanceCall = {
+ toolName: 'execute-sql',
+ args: { query: 'select session_id from session_replay_events' },
+ isError: false,
+}
+const otherQuery: GuidanceCall = { toolName: 'execute-sql', args: { query: 'select 1 from events' }, isError: false }
+const visionCall: GuidanceCall = { toolName: 'vision-scanners-inline-scan', args: {}, isError: false }
+
+function makeState(overrides: Partial<ResolvedState> = {}): ResolvedState {
+ return makeToolExecutorState(
+ [{ name: 'execute-sql' }, { name: 'vision-scanners-inline-scan' }, { name: 'vision-requests-get' }],
+ {
+ toolFeatureFlags: { [rule.flag]: true },
+ suppressAnalytics: false,
+ ...overrides,
+ }
+ )
+}
+
+async function run(
+ engine: GuidanceEngine,
+ state: ResolvedState,
+ calls: GuidanceCall[]
+): Promise<(string | undefined)[]> {
+ const shown: (string | undefined)[] = []
+ for (const call of calls) {
+ const notice = await engine.evaluate(state, call)
+ shown.push(notice && `${notice.ruleId}:${notice.outcome}`)
+ }
+ return shown
+}
+
+describe('GuidanceEngine', () => {
+ it('shows the notice once the session reaches the threshold, and only once', async () => {
+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])
+ await expect(run(engine, makeState(), [replayQuery, otherQuery, replayQuery, replayQuery])).resolves.toEqual([
+ undefined,
+ undefined,
+ 'test-rule:shown',
+ undefined,
+ ])
+ })
+
+ it('stamps a suppressed outcome once when the session already called a suggested tool', async () => {
+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])
+ await expect(run(engine, makeState(), [visionCall, replayQuery, replayQuery, replayQuery])).resolves.toEqual([
+ undefined,
+ undefined,
+ 'test-rule:suppressed',
+ undefined,
+ ])
+ })
+
+ it.each([
+ ['without org consent', false, 'no_consent'],
+ ['when consent is unknown', undefined, 'no_consent'],
+ ['with org consent', true, 'shown'],
+ ])('stamps the consent outcome %s for a rule that needs it', async (_label, aiConsentGiven, outcome) => {
+ const engine = new GuidanceEngine(makeGuidanceRedis(), [{ ...rule, requiresAiConsent: true }])
+ const state = makeState({ aiConsentGiven })
+ await engine.evaluate(state, replayQuery)
+ const notice = await engine.evaluate(state, replayQuery)
+ expect(notice).toMatchObject({ ruleId: 'test-rule', outcome })
+ expect(notice?.text === undefined).toBe(outcome !== 'shown')
+ })
+
+ it('stamps a flag_off outcome without text when the flag is off', async () => {
+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])
+ const state = makeState({ toolFeatureFlags: {} })
+ await engine.evaluate(state, replayQuery)
+ await expect(engine.evaluate(state, replayQuery)).resolves.toEqual({
+ ruleId: 'test-rule',
+ outcome: 'flag_off',
+ })
+ })
+
+ it.each([
+ [
+ 'sessions',
+ (first: ResolvedState): Partial<ResolvedState> => ({
+ requestContext: { ...first.requestContext, mcpSessionId: 'sess-2' },
+ }),
+ ],
+ ['callers echoing the same session handle', (): Partial<ResolvedState> => ({ distinctId: 'other-caller' })],
+ ])('keeps %s apart', async (_label, overrides) => {
+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])
+ const first = makeState({ requestContext: { ...makeState().requestContext, mcpSessionId: 'sess-1' } })
+ const second = makeState({ ...first, ...overrides(first) })
+ await run(engine, first, [replayQuery])
+ await expect(run(engine, second, [replayQuery, replayQuery])).resolves.toEqual([undefined, 'test-rule:shown'])
+ })
+
+ describe('a client that sends no handle', () => {
+ const withHandle = (id: string, carriedContextHandle: boolean): ResolvedState =>
+ makeState({
+ carriedContextHandle,
+ requestContext: { ...makeState().requestContext, mcpConversationId: id },
+ })
+
+ it('counts minted handles under the caller', async () => {
+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])
+ await run(engine, withHandle('minted-1', false), [replayQuery])
+ await expect(run(engine, withHandle('minted-2', false), [replayQuery])).resolves.toEqual([
+ 'test-rule:shown',
+ ])
+ })
+
+ it.each([
+ ['the match count', replayQuery, ['test-rule:shown', undefined]],
+ ['suppression', visionCall, [undefined, 'test-rule:suppressed']],
+ ])('keeps %s when the client later echoes the minted handle', async (_label, firstCall, expected) => {
+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])
+ await run(engine, withHandle('minted-1', false), [firstCall])
+ await expect(run(engine, withHandle('minted-1', true), [replayQuery, replayQuery])).resolves.toEqual(
+ expected
+ )
+ })
+ })
+
+ it('reports a suppressed outcome on a later call when another rule wins the threshold call', async () => {
+ const second: GuidanceRule = {
+ ...rule,
+ id: 'second-rule',
+ flag: 'mcp-guidance-second',
+ suppressIfCalled: ['read-data-schema'],
+ }
+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule, second])
+ const state = makeState({ toolFeatureFlags: { [rule.flag]: true, [second.flag]: true } })
+ const schemaCall: GuidanceCall = { toolName: 'read-data-schema', args: {}, isError: false }
+ await expect(run(engine, state, [schemaCall, replayQuery, replayQuery, replayQuery])).resolves.toEqual([
+ undefined,
+ undefined,
+ 'test-rule:shown',
+ 'second-rule:suppressed',
+ ])
+ })
+
+ it('reserves the allowance once when matching calls run concurrently', async () => {
+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])
+ const state = makeState()
+ await engine.evaluate(state, replayQuery)
+ const shown = await Promise.all([engine.evaluate(state, replayQuery), engine.evaluate(state, replayQuery)])
+ expect(shown.filter(Boolean)).toHaveLength(1)
+ })
+
+ it('lets a later rule keep its allowance when an earlier rule fires on the same call', async () => {
+ const second: GuidanceRule = { ...rule, id: 'second-rule', flag: 'mcp-guidance-second' }
+ const redis = makeGuidanceRedis()
+ const engine = new GuidanceEngine(redis, [rule, second])
+ const state = makeState({ toolFeatureFlags: { [rule.flag]: true, [second.flag]: true } })
+ await run(engine, state, [replayQuery, replayQuery])
+ expect([...redis.store.keys()].filter((key) => key.endsWith(':shown'))).toEqual([
+ expect.stringContaining(':test-rule:shown'),
+ ])
+ })
+
+ it.each([
+ ['impersonated session', { isImpersonated: true }],
+ ['analytics suppressed', { suppressAnalytics: true }],
+ ['sandbox token', { apiKeyScopes: ['internal_run:read'] }],
+ ['scout token', { apiKeyScopes: ['signal_scout_internal:write'] }],
+ ['suggested tool not in roster', { allTools: [{ name: 'execute-sql' }] as ResolvedState['allTools'] }],
+ [
+ 'one named tool missing from roster',
+ {
+ allTools: [
+ { name: 'execute-sql' },
+ { name: 'vision-scanners-inline-scan' },
+ ] as ResolvedState['allTools'],
+ },
+ ],
+ ])('shows nothing for %s', async (_label, overrides) => {
+ const engine = new GuidanceEngine(makeGuidanceRedis(), [rule])
+ await expect(run(engine, makeState(overrides), [replayQuery, replayQuery])).resolves.toEqual([
+ undefined,
+ undefined,
+ ])
+ })
+
+ it('shows the notice to a scout when the rule lists the scout preset', async () => {
+ const scoutRule: GuidanceRule = { ...rule, trigger: { ...rule.trigger, presets: ['user', 'scout'] } }
+ const engine = new GuidanceEngine(makeGuidanceRedis(), [scoutRule])
+ const state = makeState({ apiKeyScopes: ['signal_scout_internal:write', 'internal_run:read'] })
+ await expect(run(engine, state, [replayQuery, replayQuery])).resolves.toEqual([undefined, 'test-rule:shown'])
+ })
+
+ it('matches only the configured outcome', async () => {
+ const errorRule: GuidanceRule = { ...rule, trigger: { ...rule.trigger, outcome: 'error' }, threshold: 1 }
+ const engine = new GuidanceEngine(makeGuidanceRedis(), [errorRule])
+ await expect(run(engine, makeState(), [replayQuery, { ...replayQuery, isError: true }])).resolves.toEqual([
+ undefined,
+ 'test-rule:shown',
+ ])
+ })
+
+ it('shows nothing and logs when Redis fails', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const engine = new GuidanceEngine(makeGuidanceRedis(true), [{ ...rule, threshold: 1 }])
+ await expect(engine.evaluate(makeState(), replayQuery)).resolves.toBeUndefined()
+ expect(warn).toHaveBeenCalledWith('[Guidance] evaluation failed:', expect.any(String))
+ warn.mockRestore()
+ })
+
+ it.each([
+ ['session_replay_events', true],
+ ['raw_session_replay_events', true],
+ ['session_replay_features', true],
+ ['console_logs_log_entries', true],
+ ['system.session_recordings', true],
+ ['log_entries', false],
+ ['batch_export_log_entries', false],
+ ['sessions', false],
+ ['events', false],
+ ])('the replay rule matches SQL over %s: %s', (table, expected) => {
+ const replayRule = GUIDANCE_RULES.find((r) => r.id === 'replay-sql-to-vision')!
+ const call: GuidanceCall = {
+ toolName: 'execute-sql',
+ args: { query: `select 1 from ${table}` },
+ isError: false,
+ }
+ expect(triggerMatches(replayRule, call)).toBe(expected)
+ })
+
+ it('ships every rule with a distinct id and flag', () => {
+ expect(new Set(GUIDANCE_RULES.map((r) => r.id)).size).toBe(GUIDANCE_RULES.length)
+ expect(new Set(GUIDANCE_RULES.map((r) => r.flag)).size).toBe(GUIDANCE_RULES.length)
+ })
+})
diff --git a/services/mcp/tests/hono/helpers/guidance-redis-stubs.ts b/services/mcp/tests/hono/helpers/guidance-redis-stubs.ts
new file mode 100644
index 000000000000..651f93cdcc75
--- /dev/null
+++ b/services/mcp/tests/hono/helpers/guidance-redis-stubs.ts
@@ -0,0 +1,53 @@
+import type { RedisLike } from '@/hono/cache/RedisCache'
+
+export interface GuidanceRedisStub extends RedisLike {
+ store: Map<string, string>
+ evalCalls: (string | number)[][]
+}
+
+export function makeGuidanceRedis(failing = false): GuidanceRedisStub {
+ const store = new Map<string, string>()
+ const evalCalls: (string | number)[][] = []
+ const guard = async <T>(value: T): Promise<T> => {
+ if (failing) {
+ throw new Error('redis down')
+ }
+ return value
+ }
+ const incr = (key: string): number => {
+ const next = Number(store.get(key) ?? 0) + 1
+ store.set(key, String(next))
+ return next
+ }
+ return {
+ store,
+ evalCalls,
+ get: (key) => guard(store.get(key) ?? null),
+ set: (key, value) => {
+ store.set(key, value)
+ return guard('OK')
+ },
+ del: async (...keys) => keys.filter((key) => store.delete(key)).length,
+ scan: async () => ['0', []],
+ incr: (key) => guard(incr(key)),
+ expire: () => guard(1),
+ ttl: () => guard(-1),
+ eval: (_script, _numberOfKeys, ...args) => {
+ evalCalls.push(args)
+ const [matchedKey, suppressedKey, shownKey, _ttl, threshold, maxPerSession, mode, canFire] =
+ args.map(String)
+ if (mode === 'suppress') {
+ store.set(suppressedKey!, '1')
+ return guard(0)
+ }
+ const matched = incr(matchedKey!)
+ if (canFire !== '1' || matched < Number(threshold)) {
+ return guard(0)
+ }
+ if (incr(shownKey!) > Number(maxPerSession)) {
+ return guard(0)
+ }
+ return guard(store.has(suppressedKey!) ? 2 : 1)
+ },
+ }
+}
diff --git a/services/mcp/tests/hono/tool-executor-guidance.test.ts b/services/mcp/tests/hono/tool-executor-guidance.test.ts
new file mode 100644
index 000000000000..5e1ac4601efc
--- /dev/null
+++ b/services/mcp/tests/hono/tool-executor-guidance.test.ts
@@ -0,0 +1,164 @@
+import { afterEach, beforeAll, describe, expect, it, type MockInstance, vi } from 'vitest'
+import { z } from 'zod'
+
+vi.mock('@/resources/internals', () => ({
+ fetchContextMillResources: vi.fn().mockRejectedValue(new Error('mocked')),
+ filterValidEntries: vi.fn().mockReturnValue([]),
+ loadManifestFromArchive: vi.fn().mockReturnValue({ resources: [] }),
+}))
+
+vi.mock('@/resources', () => ({
+ getPromptsFromManifest: vi.fn().mockResolvedValue([]),
+}))
+
+vi.mock('@/lib/posthog', async () => {
+ const { PostHogMCP } = await import('@posthog/mcp-analytics')
+ const client = new PostHogMCP('phc_test', { disabled: true, captureModel: true })
+ return { getPostHogClient: () => client }
+})
+
+import { GuidanceEngine } from '@/hono/guidance/engine'
+import { MCP_GUIDANCE_REPLAY_VISION_FLAG, MCP_GUIDANCE_SQL_SCHEMA_FLAG } from '@/hono/guidance/rules'
+import { InstructionsBuilder } from '@/hono/instructions'
+import { ToolCatalog } from '@/hono/tool-catalog'
+import { ToolExecutor } from '@/hono/tool-executor'
+import { getPostHogClient } from '@/lib/posthog'
+
+import { makeToolExecutorState } from '../shared/test-utils'
+import { makeGuidanceRedis } from './helpers/guidance-redis-stubs'
+
+function makeTool(name: string, handler: (args: unknown) => unknown): any {
+ return {
+ name,
+ title: name,
+ description: `Synthetic ${name}`,
+ scopes: [],
+ annotations: { readOnlyHint: true },
+ schema: z.object({ query: z.string().optional() }),
+ handler: vi.fn(async (_context: unknown, args: unknown) => handler(args)),
+ }
+}
+
+describe('ToolExecutor guidance notices', () => {
+ let catalog: ToolCatalog
+
+ beforeAll(async () => {
+ catalog = new ToolCatalog()
+ await catalog.warmup()
+ })
+
+ afterEach(() => {
+ vi.restoreAllMocks()
+ })
+
+ type SqlResult = { isError?: boolean; content: { type: string; text?: string }[] }
+
+ function setup(
+ mode: string,
+ sqlHandler: (args: unknown) => unknown,
+ flag: string,
+ callFlags = ''
+ ): {
+ callSql: (query: string) => Promise<SqlResult>
+ capture: MockInstance<ReturnType<typeof getPostHogClient>['captureToolCall']>
+ } {
+ const sql = makeTool('execute-sql', sqlHandler)
+ const visionTools = [
+ 'vision-scanners-inline-scan',
+ 'vision-requests-create',
+ 'vision-requests-get',
+ 'vision-observations-search',
+ ].map((name) => makeTool(name, () => ({})))
+ const schema = makeTool('read-data-schema', () => ({}))
+ const tools = [sql, ...visionTools, schema]
+ vi.spyOn(catalog, 'getToolByName').mockImplementation((name) => {
+ const tool = tools.find((t) => t.name === name)
+ return tool
+ ? { build: () => tool, meta: undefined, rawInputSchema: undefined, definition: undefined }
+ : undefined
+ })
+ const executor = new ToolExecutor(
+ catalog,
+ new InstructionsBuilder(''),
+ undefined,
+ new GuidanceEngine(makeGuidanceRedis())
+ )
+ const state = makeToolExecutorState(tools, {
+ useSingleExec: mode === 'exec',
+ suppressAnalytics: false,
+ aiConsentGiven: true,
+ toolFeatureFlags: { [flag]: true },
+ })
+ const capture = vi.spyOn(getPostHogClient(), 'captureToolCall').mockImplementation(() => {})
+ const callSql = async (
+ query: string
+ ): Promise<{ isError?: boolean; content: { type: string; text?: string }[] }> => {
+ const args =
+ mode === 'exec' ? { command: `call ${callFlags}execute-sql ${JSON.stringify({ query })}` } : { query }
+ const result = await executor.handleToolCall(
+ { name: mode === 'exec' ? 'exec' : 'execute-sql', arguments: args },
+ state
+ )
+ await new Promise((resolve) => setImmediate(resolve))
+ return result as any
+ }
+ return { callSql, capture }
+ }
+
+ it.each(['direct', 'exec'])('prepends the replay notice to the second replay query in %s mode', async (mode) => {
+ const { callSql, capture } = setup(mode, () => ({ results: [] }), MCP_GUIDANCE_REPLAY_VISION_FLAG)
+ const first = await callSql('select 1 from session_replay_events')
+ const second = await callSql('select 2 from session_replay_events')
+
+ expect(first.content[0]?.text).not.toContain('Replay Vision')
+ expect(second.content[0]?.text).toContain('Replay Vision')
+ expect(second.content).toHaveLength(first.content.length + 1)
+ expect(capture.mock.calls.map(([call]) => call.properties?.mcp_guidance_rule)).toEqual([
+ undefined,
+ 'replay-sql-to-vision',
+ ])
+ expect(capture.mock.calls[1]?.[0].properties?.mcp_guidance_outcome).toBe('shown')
+ })
+
+ it('keeps the JSON result as the first block when guidance fires on a --json call', async () => {
+ const { callSql } = setup('exec', () => ({ results: [[1]] }), MCP_GUIDANCE_REPLAY_VISION_FLAG, '--json ')
+ await callSql('select 1 from session_replay_events')
+ const second = await callSql('select 2 from session_replay_events')
+
+ expect(() => JSON.parse(second.content[0]?.text ?? '')).not.toThrow()
+ expect(second.content[1]?.text).toContain('Replay Vision')
+ })
+
+ it('stamps flag_off on the event and leaves the result alone when the flag is off', async () => {
+ const { callSql, capture } = setup('direct', () => ({ results: [] }), 'some-other-flag')
+ const first = await callSql('select 1 from session_replay_events')
+ const second = await callSql('select 2 from session_replay_events')
+
+ expect(second.content).toEqual(first.content)
+ expect(capture.mock.calls[1]?.[0].properties).toMatchObject({
+ mcp_guidance_rule: 'replay-sql-to-vision',
+ mcp_guidance_outcome: 'flag_off',
+ })
+ })
+
+ it.each(['direct', 'exec'])('prepends the schema notice to the third failed query in %s mode', async (mode) => {
+ const { callSql, capture } = setup(
+ mode,
+ () => {
+ throw new Error('Unknown column')
+ },
+ MCP_GUIDANCE_SQL_SCHEMA_FLAG
+ )
+ await callSql('select a from b')
+ await callSql('select a from b')
+ const third = await callSql('select a from b')
+
+ expect(third.isError).toBe(true)
+ expect(third.content[0]?.text).toContain('read-data-schema')
+ expect(capture.mock.calls.map(([call]) => call.properties?.mcp_guidance_rule)).toEqual([
+ undefined,
+ undefined,
+ 'sql-failures-to-schema',
+ ])
+ })
+})