Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

12.9 KB
Raw Blame
import { createHash } from 'node:crypto'

import { resolveScopePreset } from '@/lib/scope-preset'

import type { RedisLike } from '../cache/RedisCache'
import { resolveSessionKey } from '../mcp-context'
import { guidanceEvaluationErrorsTotal, guidanceNoticesTotal } from '../metrics'
import type { ResolvedState } from '../request-state-resolver'
import { GuidanceProductState } from './product-state'
import { GUIDANCE_RULES, type GuidanceRule } from './rules'

// A conversation handle outlives any single request, so its counters keep the session store's
// idle TTL. Without a handle the counters key on the caller and expire like a 30-minute idle
// session, which is the same sessionization the analytics queries fall back to.
const SESSION_TTL_SECONDS = 24 * 60 * 60
const CALLER_TTL_SECONDS = 30 * 60

interface CountingScopes {
    primary: string
    ttl: number
    mirror?: string
}

export interface GuidanceCall {
    /** The effective tool name: the inner tool for an exec `call`, `exec` for other verbs. */
    toolName: string
    execVerb?: string | undefined
    args: Record<string, unknown> | undefined
    isError: boolean
    resultEmpty?: boolean | undefined
}

export type GuidanceOutcome = 'shown' | 'flag_off' | 'suppressed' | 'no_consent'

export interface GuidanceNotice {
    ruleId: string
    outcome: GuidanceOutcome
    /** Only present when the notice is shown to the agent. */
    text?: string
}

export function matchesToolPattern(pattern: string, toolName: string): boolean {
    return pattern.endsWith('*') ? toolName.startsWith(pattern.slice(0, -1)) : pattern === toolName
}

function readArgPath(args: Record<string, unknown> | undefined, path: string): unknown {
    return path.split('.').reduce<unknown>((current, segment) => {
        if (typeof current !== 'object' || current === null) {
            return undefined
        }
        return (current as Record<string, unknown>)[segment]
    }, args)
}

function isSet(value: unknown): boolean {
    return value !== undefined && value !== null && value !== ''
}

export function triggerMatches(rule: GuidanceRule, call: GuidanceCall): boolean {
    const { trigger } = rule
    if (trigger.tool !== undefined && !matchesToolPattern(trigger.tool, call.toolName)) {
        return false
    }
    if (trigger.execVerb !== undefined && trigger.execVerb !== call.execVerb) {
        return false
    }
    if (trigger.outcome === 'success' && call.isError) {
        return false
    }
    if (trigger.outcome === 'error' && !call.isError) {
        return false
    }
    if (trigger.resultEmpty !== undefined && trigger.resultEmpty !== (call.resultEmpty === true)) {
        return false
    }
    if (trigger.excludeArgs?.some((path) => isSet(readArgPath(call.args, path)))) {
        return false
    }
    if (trigger.argPattern !== undefined) {
        const value = trigger.argPath === undefined ? call.args : readArgPath(call.args, trigger.argPath)
        const text = typeof value === 'string' ? value : value === undefined ? '' : JSON.stringify(value)
        if (trigger.argMaxLength !== undefined && text.length > trigger.argMaxLength) {
            return false
        }
        return trigger.argPattern.test(text)
    }
    return true
}

// Server-minted tokens are not excluded here: scouts carry the same internal scopes as sandbox
// runs, so each rule's `presets` decides which of them it reaches.
function isGuidanceAudience(state: ResolvedState): boolean {
    return !(state.isImpersonated || state.suppressAnalytics || state.clientProfile.consumer === 'posthog_ai')
}

// The flag and org consent decide only whether the text goes out: an eligible caller with the
// flag off, or without AI consent, is counted the same way, so the stamped outcome gives a
// control group and a measure of how often consent blocks the notice.
function ruleApplies(rule: GuidanceRule, state: ResolvedState, preset: string): boolean {
    // The roster drops consent-gated tools without org consent, so it cannot decide these callers.
    const withheldForConsent = rule.requiresAiConsent === true && state.aiConsentGiven !== true
    return (
        rule.trigger.presets.includes(preset as GuidanceRule['trigger']['presets'][number]) &&
        (withheldForConsent || rule.requiresTool.every((name) => state.allTools.some((tool) => tool.name === name)))
    )
}

// One script per rule, so a concurrent batch cannot reserve the same notice twice. The shown
// key counts reports of either kind, so a suppressed outcome that loses the call to another
// rule stays pending and is reported on the next matching call instead of being dropped.
// Mirror mode only records a call that another scope decided, so it never reports a notice itself.
const RULE_STATE_SCRIPT = `
local ttl = tonumber(ARGV[1])
if ARGV[4] == 'touch' then
    redis.call('EXPIRE', KEYS[1], ttl)
    redis.call('EXPIRE', KEYS[2], ttl)
    redis.call('EXPIRE', KEYS[3], ttl)
    return tonumber(redis.call('GET', KEYS[3]) or 0)
end
if ARGV[4] == 'suppress' then
    redis.call('SET', KEYS[2], '1', 'EX', ttl)
    redis.call('EXPIRE', KEYS[1], ttl)
    redis.call('EXPIRE', KEYS[3], ttl)
    return 0
end
local matched = redis.call('INCR', KEYS[1])
redis.call('EXPIRE', KEYS[1], ttl)
redis.call('EXPIRE', KEYS[2], ttl)
redis.call('EXPIRE', KEYS[3], ttl)
if ARGV[4] == 'mirror' then
    if ARGV[5] == '1' then
        redis.call('INCR', KEYS[3])
        redis.call('EXPIRE', KEYS[3], ttl)
    end
    return 0
end
if ARGV[5] ~= '1' or matched < tonumber(ARGV[2]) then
    return 0
end
local shown = redis.call('INCR', KEYS[3])
redis.call('EXPIRE', KEYS[3], ttl)
if shown > tonumber(ARGV[3]) then
    return 0
end
if redis.call('EXISTS', KEYS[2]) == 1 then
    return 2
end
return 1
`

enum RuleResult {
    None = 0,
    Fired = 1,
    Suppressed = 2,
}

type RuleMode = 'match' | 'suppress' | 'mirror' | 'touch'

export class GuidanceEngine {
    constructor(
        private readonly redis: RedisLike,
        private readonly rules: readonly GuidanceRule[] = GUIDANCE_RULES
    ) {}

    // Never throws: a Redis problem silently yields no notice, so `tools/call` keeps working.
    async evaluate(state: ResolvedState, call: GuidanceCall): Promise<GuidanceNotice | undefined> {
        if (!isGuidanceAudience(state)) {
            return undefined
        }
        const preset = resolveScopePreset(state.apiKeyScopes)
        const rules = this.rules.filter((rule) => ruleApplies(rule, state, preset))
        if (rules.length === 0) {
            return undefined
        }
        try {
            const scopes = countingScopes(state)
            const productState = new GuidanceProductState(this.redis, state)
            let winner: GuidanceNotice | undefined
            for (const rule of rules) {
                const result = await this.touchRule(rule, scopes, call, winner === undefined, productState)
                if (result === RuleResult.Fired) {
                    const outcome = firedOutcome(rule, state)
                    winner = {
                        ruleId: rule.id,
                        outcome,
                        ...(outcome === 'shown'
                            ? { text: typeof rule.text === 'string' ? rule.text : rule.text(call) }
                            : {}),
                    }
                } else if (result === RuleResult.Suppressed && !winner) {
                    winner = { ruleId: rule.id, outcome: 'suppressed' }
                }
            }
            if (winner) {
                guidanceNoticesTotal.inc({ rule: winner.ruleId, outcome: winner.outcome })
            }
            return winner
        } catch (error) {
            guidanceEvaluationErrorsTotal.inc()
            console.warn('[Guidance] evaluation failed:', error instanceof Error ? error.message : String(error))
            return undefined
        }
    }

    private async touchRule(
        rule: GuidanceRule,
        scopes: CountingScopes,
        call: GuidanceCall,
        canFire: boolean,
        productState: GuidanceProductState
    ): Promise<RuleResult> {
        const suppressing = rule.suppressIfCalled.some((pattern) => matchesToolPattern(pattern, call.toolName))
        if (!suppressing && !triggerMatches(rule, call)) {
            return RuleResult.None
        }
        const { condition } = rule
        if (suppressing || !condition) {
            const mode = suppressing ? 'suppress' : 'match'
            const result = await this.runRuleScript(rule, scopes.primary, scopes.ttl, mode, canFire)
            await this.mirrorRule(rule, scopes, suppressing ? 'suppress' : 'mirror', result !== RuleResult.None)
            return result as RuleResult
        }
        // `touch` keeps a capped rule's counters alive without the `match` count, so a rejected call
        // spends no `maxPerSession`. `match` is a separate step so that a gate that outlives the budget
        // cannot reserve a notice that nobody receives.
        const admitted = await productState.withinBudget('rule_gate', async () => {
            const shown = await this.runRuleScript(rule, scopes.primary, scopes.ttl, 'touch', canFire)
            return shown < rule.maxPerSession && (await condition({ call, productState }))
        })
        const result =
            admitted === true && productState.hasBudgetLeft()
                ? await productState.withinBudget('rule_match', () =>
                      this.runRuleScript(rule, scopes.primary, scopes.ttl, 'match', canFire)
                  )
                : undefined
        if (result === undefined) {
            await productState.withinBudget('rule_mirror', () => this.mirrorRule(rule, scopes, 'touch', false))
            return RuleResult.None
        }
        await productState.withinBudget('rule_mirror', () =>
            this.mirrorRule(rule, scopes, 'mirror', result !== RuleResult.None)
        )
        return result as RuleResult
    }

    private async mirrorRule(rule: GuidanceRule, scopes: CountingScopes, mode: RuleMode, flag: boolean): Promise<void> {
        if (scopes.mirror) {
            await this.runRuleScript(rule, scopes.mirror, SESSION_TTL_SECONDS, mode, flag)
        }
    }

    private async runRuleScript(
        rule: GuidanceRule,
        scope: string,
        ttl: number,
        mode: RuleMode,
        flag: boolean
    ): Promise<number> {
        const key = (suffix: string): string => `${scope}:${rule.id}:${suffix}`
        const fired = await this.redis.eval(
            RULE_STATE_SCRIPT,
            3,
            key('matched'),
            key('suppressed'),
            key('shown'),
            ttl,
            rule.threshold,
            rule.maxPerSession,
            mode,
            flag ? '1' : '0'
        )
        return Number(fired)
    }
}

// A call with a minted handle counts under the caller, because a client that never echoes gets a new
// handle on every call. It also records under the handle, so a client that echoes keeps its own counters.
function countingScopes(state: ResolvedState): CountingScopes {
    const handle = resolveSessionKey(state.requestContext)
    const callerScope = scopeKey(state.distinctId, undefined)
    if (!handle) {
        return { primary: callerScope, ttl: CALLER_TTL_SECONDS }
    }
    const handleScope = scopeKey(state.distinctId, handle)
    return state.carriedContextHandle === false
        ? { primary: callerScope, ttl: CALLER_TTL_SECONDS, mirror: handleScope }
        : { primary: handleScope, ttl: SESSION_TTL_SECONDS }
}

// The braces are a cluster hash tag, so the script can touch a rule's three keys together.
function scopeKey(distinctId: string, handle: string | undefined): string {
    return `mcp:guidance:{${hashKey(`${distinctId}|${handle ?? ''}`)}}`
}

function firedOutcome(rule: GuidanceRule, state: ResolvedState): GuidanceOutcome {
    if (rule.requiresAiConsent && state.aiConsentGiven !== true) {
        return 'no_consent'
    }
    return state.toolFeatureFlags?.[rule.flag] === true ? 'shown' : 'flag_off'
}

function hashKey(value: string): string {
    return createHash('sha256').update(value).digest('base64url').slice(0, 22)
}

// Attached on errors too: a notice that fires on repeated failures must reach the agent. A JSON
// call keeps its result as the first block, because callers parse `content[0]` as raw JSON.
export function attachGuidanceNotice(
    result: unknown,
    notice: GuidanceNotice | undefined,
    options: { jsonOutput?: boolean } = {}
): unknown {
    if (
        !notice?.text ||
        typeof result !== 'object' ||
        result === null ||
        !Array.isArray((result as { content?: unknown }).content)
    ) {
        return result
    }
    const withContent = result as { content: unknown[] }
    const block = { type: 'text', text: notice.text }
    return {
        ...withContent,
        content: options.jsonOutput ? [...withContent.content, block] : [block, ...withContent.content],
    }
}

export function guidanceAnalyticsProperties(notice: GuidanceNotice | undefined): Record<string, unknown> {
    return notice ? { mcp_guidance_rule: notice.ruleId, mcp_guidance_outcome: notice.outcome } : {}
}