:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { createHash } from 'node:crypto'
import { resolveScopePreset } from '@/lib/scope-preset'
import type { RedisLike } from '../cache/RedisCache'
import { resolveSessionKey } from '../mcp-context'
import { guidanceEvaluationErrorsTotal, guidanceNoticesTotal } from '../metrics'
import type { ResolvedState } from '../request-state-resolver'
import { GuidanceProductState } from './product-state'
import { GUIDANCE_RULES, type GuidanceRule } from './rules'
// A conversation handle outlives any single request, so its counters keep the session store's
// idle TTL. Without a handle the counters key on the caller and expire like a 30-minute idle
// session, which is the same sessionization the analytics queries fall back to.
const SESSION_TTL_SECONDS = 24 * 60 * 60
const CALLER_TTL_SECONDS = 30 * 60
interface CountingScopes {
primary: string
ttl: number
mirror?: string
}
export interface GuidanceCall {
/** The effective tool name: the inner tool for an exec `call`, `exec` for other verbs. */
toolName: string
execVerb?: string | undefined
args: Record<string, unknown> | undefined
isError: boolean
resultEmpty?: boolean | undefined
}
export type GuidanceOutcome = 'shown' | 'flag_off' | 'suppressed' | 'no_consent'
export interface GuidanceNotice {
ruleId: string
outcome: GuidanceOutcome
/** Only present when the notice is shown to the agent. */
text?: string
}
export function matchesToolPattern(pattern: string, toolName: string): boolean {
return pattern.endsWith('*') ? toolName.startsWith(pattern.slice(0, -1)) : pattern === toolName
}
function readArgPath(args: Record<string, unknown> | undefined, path: string): unknown {
return path.split('.').reduce<unknown>((current, segment) => {
if (typeof current !== 'object' || current === null) {
return undefined
}
return (current as Record<string, unknown>)[segment]
}, args)
}
function isSet(value: unknown): boolean {
return value !== undefined && value !== null && value !== ''
}
export function triggerMatches(rule: GuidanceRule, call: GuidanceCall): boolean {
const { trigger } = rule
if (trigger.tool !== undefined && !matchesToolPattern(trigger.tool, call.toolName)) {
return false
}
if (trigger.execVerb !== undefined && trigger.execVerb !== call.execVerb) {
return false
}
if (trigger.outcome === 'success' && call.isError) {
return false
}
if (trigger.outcome === 'error' && !call.isError) {
return false
}
if (trigger.resultEmpty !== undefined && trigger.resultEmpty !== (call.resultEmpty === true)) {
return false
}
if (trigger.excludeArgs?.some((path) => isSet(readArgPath(call.args, path)))) {
return false
}
if (trigger.argPattern !== undefined) {
const value = trigger.argPath === undefined ? call.args : readArgPath(call.args, trigger.argPath)
const text = typeof value === 'string' ? value : value === undefined ? '' : JSON.stringify(value)
if (trigger.argMaxLength !== undefined && text.length > trigger.argMaxLength) {
return false
}
return trigger.argPattern.test(text)
}
return true
}
// Server-minted tokens are not excluded here: scouts carry the same internal scopes as sandbox
// runs, so each rule's `presets` decides which of them it reaches.
function isGuidanceAudience(state: ResolvedState): boolean {
return !(state.isImpersonated || state.suppressAnalytics || state.clientProfile.consumer === 'posthog_ai')
}
// The flag and org consent decide only whether the text goes out: an eligible caller with the
// flag off, or without AI consent, is counted the same way, so the stamped outcome gives a
// control group and a measure of how often consent blocks the notice.
function ruleApplies(rule: GuidanceRule, state: ResolvedState, preset: string): boolean {
// The roster drops consent-gated tools without org consent, so it cannot decide these callers.
const withheldForConsent = rule.requiresAiConsent === true && state.aiConsentGiven !== true
return (
rule.trigger.presets.includes(preset as GuidanceRule['trigger']['presets'][number]) &&
(withheldForConsent || rule.requiresTool.every((name) => state.allTools.some((tool) => tool.name === name)))
)
}
// One script per rule, so a concurrent batch cannot reserve the same notice twice. The shown
// key counts reports of either kind, so a suppressed outcome that loses the call to another
// rule stays pending and is reported on the next matching call instead of being dropped.
// Mirror mode only records a call that another scope decided, so it never reports a notice itself.
const RULE_STATE_SCRIPT = `
local ttl = tonumber(ARGV[1])
if ARGV[4] == 'touch' then
redis.call('EXPIRE', KEYS[1], ttl)
redis.call('EXPIRE', KEYS[2], ttl)
redis.call('EXPIRE', KEYS[3], ttl)
return tonumber(redis.call('GET', KEYS[3]) or 0)
end
if ARGV[4] == 'suppress' then
redis.call('SET', KEYS[2], '1', 'EX', ttl)
redis.call('EXPIRE', KEYS[1], ttl)
redis.call('EXPIRE', KEYS[3], ttl)
return 0
end
local matched = redis.call('INCR', KEYS[1])
redis.call('EXPIRE', KEYS[1], ttl)
redis.call('EXPIRE', KEYS[2], ttl)
redis.call('EXPIRE', KEYS[3], ttl)
if ARGV[4] == 'mirror' then
if ARGV[5] == '1' then
redis.call('INCR', KEYS[3])
redis.call('EXPIRE', KEYS[3], ttl)
end
return 0
end
if ARGV[5] ~= '1' or matched < tonumber(ARGV[2]) then
return 0
end
local shown = redis.call('INCR', KEYS[3])
redis.call('EXPIRE', KEYS[3], ttl)
if shown > tonumber(ARGV[3]) then
return 0
end
if redis.call('EXISTS', KEYS[2]) == 1 then
return 2
end
return 1
`
enum RuleResult {
None = 0,
Fired = 1,
Suppressed = 2,
}
type RuleMode = 'match' | 'suppress' | 'mirror' | 'touch'
export class GuidanceEngine {
constructor(
private readonly redis: RedisLike,
private readonly rules: readonly GuidanceRule[] = GUIDANCE_RULES
) {}
// Never throws: a Redis problem silently yields no notice, so `tools/call` keeps working.
async evaluate(state: ResolvedState, call: GuidanceCall): Promise<GuidanceNotice | undefined> {
if (!isGuidanceAudience(state)) {
return undefined
}
const preset = resolveScopePreset(state.apiKeyScopes)
const rules = this.rules.filter((rule) => ruleApplies(rule, state, preset))
if (rules.length === 0) {
return undefined
}
try {
const scopes = countingScopes(state)
const productState = new GuidanceProductState(this.redis, state)
let winner: GuidanceNotice | undefined
for (const rule of rules) {
const result = await this.touchRule(rule, scopes, call, winner === undefined, productState)
if (result === RuleResult.Fired) {
const outcome = firedOutcome(rule, state)
winner = {
ruleId: rule.id,
outcome,
...(outcome === 'shown'
? { text: typeof rule.text === 'string' ? rule.text : rule.text(call) }
: {}),
}
} else if (result === RuleResult.Suppressed && !winner) {
winner = { ruleId: rule.id, outcome: 'suppressed' }
}
}
if (winner) {
guidanceNoticesTotal.inc({ rule: winner.ruleId, outcome: winner.outcome })
}
return winner
} catch (error) {
guidanceEvaluationErrorsTotal.inc()
console.warn('[Guidance] evaluation failed:', error instanceof Error ? error.message : String(error))
return undefined
}
}
private async touchRule(
rule: GuidanceRule,
scopes: CountingScopes,
call: GuidanceCall,
canFire: boolean,
productState: GuidanceProductState
): Promise<RuleResult> {
const suppressing = rule.suppressIfCalled.some((pattern) => matchesToolPattern(pattern, call.toolName))
if (!suppressing && !triggerMatches(rule, call)) {
return RuleResult.None
}
const { condition } = rule
if (suppressing || !condition) {
const mode = suppressing ? 'suppress' : 'match'
const result = await this.runRuleScript(rule, scopes.primary, scopes.ttl, mode, canFire)
await this.mirrorRule(rule, scopes, suppressing ? 'suppress' : 'mirror', result !== RuleResult.None)
return result as RuleResult
}
// `touch` keeps a capped rule's counters alive without the `match` count, so a rejected call
// spends no `maxPerSession`. `match` is a separate step so that a gate that outlives the budget
// cannot reserve a notice that nobody receives.
const admitted = await productState.withinBudget('rule_gate', async () => {
const shown = await this.runRuleScript(rule, scopes.primary, scopes.ttl, 'touch', canFire)
return shown < rule.maxPerSession && (await condition({ call, productState }))
})
const result =
admitted === true && productState.hasBudgetLeft()
? await productState.withinBudget('rule_match', () =>
this.runRuleScript(rule, scopes.primary, scopes.ttl, 'match', canFire)
)
: undefined
if (result === undefined) {
await productState.withinBudget('rule_mirror', () => this.mirrorRule(rule, scopes, 'touch', false))
return RuleResult.None
}
await productState.withinBudget('rule_mirror', () =>
this.mirrorRule(rule, scopes, 'mirror', result !== RuleResult.None)
)
return result as RuleResult
}
private async mirrorRule(rule: GuidanceRule, scopes: CountingScopes, mode: RuleMode, flag: boolean): Promise<void> {
if (scopes.mirror) {
await this.runRuleScript(rule, scopes.mirror, SESSION_TTL_SECONDS, mode, flag)
}
}
private async runRuleScript(
rule: GuidanceRule,
scope: string,
ttl: number,
mode: RuleMode,
flag: boolean
): Promise<number> {
const key = (suffix: string): string => `${scope}:${rule.id}:${suffix}`
const fired = await this.redis.eval(
RULE_STATE_SCRIPT,
3,
key('matched'),
key('suppressed'),
key('shown'),
ttl,
rule.threshold,
rule.maxPerSession,
mode,
flag ? '1' : '0'
)
return Number(fired)
}
}
// A call with a minted handle counts under the caller, because a client that never echoes gets a new
// handle on every call. It also records under the handle, so a client that echoes keeps its own counters.
function countingScopes(state: ResolvedState): CountingScopes {
const handle = resolveSessionKey(state.requestContext)
const callerScope = scopeKey(state.distinctId, undefined)
if (!handle) {
return { primary: callerScope, ttl: CALLER_TTL_SECONDS }
}
const handleScope = scopeKey(state.distinctId, handle)
return state.carriedContextHandle === false
? { primary: callerScope, ttl: CALLER_TTL_SECONDS, mirror: handleScope }
: { primary: handleScope, ttl: SESSION_TTL_SECONDS }
}
// The braces are a cluster hash tag, so the script can touch a rule's three keys together.
function scopeKey(distinctId: string, handle: string | undefined): string {
return `mcp:guidance:{${hashKey(`${distinctId}|${handle ?? ''}`)}}`
}
function firedOutcome(rule: GuidanceRule, state: ResolvedState): GuidanceOutcome {
if (rule.requiresAiConsent && state.aiConsentGiven !== true) {
return 'no_consent'
}
return state.toolFeatureFlags?.[rule.flag] === true ? 'shown' : 'flag_off'
}
function hashKey(value: string): string {
return createHash('sha256').update(value).digest('base64url').slice(0, 22)
}
// Attached on errors too: a notice that fires on repeated failures must reach the agent. A JSON
// call keeps its result as the first block, because callers parse `content[0]` as raw JSON.
export function attachGuidanceNotice(
result: unknown,
notice: GuidanceNotice | undefined,
options: { jsonOutput?: boolean } = {}
): unknown {
if (
!notice?.text ||
typeof result !== 'object' ||
result === null ||
!Array.isArray((result as { content?: unknown }).content)
) {
return result
}
const withContent = result as { content: unknown[] }
const block = { type: 'text', text: notice.text }
return {
...withContent,
content: options.jsonOutput ? [...withContent.content, block] : [block, ...withContent.content],
}
}
export function guidanceAnalyticsProperties(notice: GuidanceNotice | undefined): Record<string, unknown> {
return notice ? { mcp_guidance_rule: notice.ruleId, mcp_guidance_outcome: notice.outcome } : {}
}