:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(mcp): add guidance for empty error tracking results (#128493)
Adam Bowker5bb89d485cb9
diff --git a/services/mcp/src/hono/guidance/rules.ts b/services/mcp/src/hono/guidance/rules.tsindex 98e3cbe50332..87d16cc46aa3 100644--- a/services/mcp/src/hono/guidance/rules.ts+++ b/services/mcp/src/hono/guidance/rules.ts@@ -52,6 +52,47 @@ export type GuidanceRule = z.infer<typeof guidanceRuleSchema>export const MCP_GUIDANCE_REPLAY_VISION_FLAG = 'mcp-guidance-replay-vision'export const MCP_GUIDANCE_SQL_SCHEMA_FLAG = 'mcp-guidance-sql-schema'export const MCP_GUIDANCE_SQL_TYPED_RUNNER_FLAG = 'mcp-guidance-sql-typed-runner'+export const MCP_GUIDANCE_ERROR_TRACKING_EMPTY_SETUP_FLAG = 'mcp-guidance-error-tracking-empty-setup'+export const MCP_GUIDANCE_ERROR_TRACKING_EMPTY_FILTERS_FLAG = 'mcp-guidance-error-tracking-empty-filters'+export const MCP_GUIDANCE_ERROR_TRACKING_EMPTY_RANGE_FLAG = 'mcp-guidance-error-tracking-empty-range'++const ERROR_TRACKING_ISSUES_LIST_TOOL = 'query-error-tracking-issues-list'++// `status` and `dateRange` are absent because the default query also sets them, and the range rule covers them.+const ERROR_TRACKING_NARROWING_FILTERS = [+ 'searchQuery',+ 'library',+ 'release',+ 'fingerprint',+ 'url',+ 'personId',+ 'user',+ 'filePath',+ 'filterGroup',+ 'assignee',+ 'firstSeenAfter',+]++function errorTrackingFiltersIn(call: GuidanceCall): string[] {+ return ERROR_TRACKING_NARROWING_FILTERS.filter((filter) => {+ const value = call.args?.[filter]+ return value !== undefined && value !== null && value !== '' && !(Array.isArray(value) && value.length === 0)+ })+}++// An empty later page only means the agent read past the last row.+function isLaterPage(call: GuidanceCall): boolean {+ const offset = call.args?.offset+ return typeof offset === 'number' && offset > 0+}++async function projectHasErrorTrackingIssues(productState: GuidanceProductState): Promise<boolean | undefined> {+ // A project with no data at all gets no notice, so the issue lookup never runs for it.+ if ((await productState.hasIngestedData()) !== true) {+ return undefined+ }+ return productState.hasErrorTrackingIssues()+}const TYPED_RUNNER_TOOLS = ['query-trends',@@ -159,6 +200,76 @@ const RULES: GuidanceRule[] = ['To only check that an event or property exists, use `read-data-schema`.',].join(' '),},+ {+ id: 'error-tracking-empty-to-setup',+ flag: MCP_GUIDANCE_ERROR_TRACKING_EMPTY_SETUP_FLAG,+ trigger: {+ tool: ERROR_TRACKING_ISSUES_LIST_TOOL,+ resultEmpty: true,+ outcome: 'success',+ presets: ['user'],+ },+ threshold: 1,+ maxPerSession: 1,+ suppressIfCalled: ['docs-search'],+ requiresTool: ['docs-search'],+ condition: async ({ call, productState }) =>+ !isLaterPage(call) && (await projectHasErrorTrackingIssues(productState)) === false,+ text: [+ 'This project has no error tracking issues at all, so exception capture is probably not set up.',+ 'If the user wants error tracking, call `docs-search` to find the setup steps for their SDK.',+ 'If the user expected errors here, tell them that no exceptions have reached PostHog.',+ ].join(' '),+ },+ {+ id: 'error-tracking-empty-to-filters',+ flag: MCP_GUIDANCE_ERROR_TRACKING_EMPTY_FILTERS_FLAG,+ trigger: {+ tool: ERROR_TRACKING_ISSUES_LIST_TOOL,+ resultEmpty: true,+ outcome: 'success',+ presets: ['user'],+ },+ threshold: 1,+ maxPerSession: 2,+ suppressIfCalled: [],+ requiresTool: [ERROR_TRACKING_ISSUES_LIST_TOOL],+ condition: async ({ call, productState }) =>+ !isLaterPage(call) &&+ errorTrackingFiltersIn(call).length > 0 &&+ (await projectHasErrorTrackingIssues(productState)) === true,+ text: (call) =>+ [+ 'This project has error tracking issues, but none matched this query.',+ `This call filtered on ${errorTrackingFiltersIn(call)+ .map((filter) => `\`${filter}\``)+ .join(', ')}.`,+ 'Remove or loosen these filters and try again before you report that no matching errors exist.',+ ].join(' '),+ },+ {+ id: 'error-tracking-empty-to-range',+ flag: MCP_GUIDANCE_ERROR_TRACKING_EMPTY_RANGE_FLAG,+ trigger: {+ tool: ERROR_TRACKING_ISSUES_LIST_TOOL,+ resultEmpty: true,+ outcome: 'success',+ presets: ['user'],+ },+ threshold: 1,+ maxPerSession: 1,+ suppressIfCalled: [],+ requiresTool: [ERROR_TRACKING_ISSUES_LIST_TOOL],+ condition: async ({ call, productState }) =>+ !isLaterPage(call) &&+ errorTrackingFiltersIn(call).length === 0 &&+ (await projectHasErrorTrackingIssues(productState)) === true,+ text: [+ 'This project has error tracking issues, but none matched this status, date range and test account filter.',+ 'This tool returns only issues seen during `dateRange`. `status` defaults to `active`, and `filterTestAccounts` defaults to true.',+ 'Try a wider `dateRange`, `status: "all"` or `filterTestAccounts: false` before you report that no errors exist.',+ ].join(' '),+ },]export const GUIDANCE_RULES: readonly GuidanceRule[] = z.array(guidanceRuleSchema).parse(RULES)diff --git a/services/mcp/tests/hono/guidance-error-tracking.test.ts b/services/mcp/tests/hono/guidance-error-tracking.test.tsnew file mode 100644index 000000000000..8fa6f5523e9a--- /dev/null+++ b/services/mcp/tests/hono/guidance-error-tracking.test.ts@@ -0,0 +1,135 @@+import { describe, expect, it, vi } from 'vitest'++import { GuidanceEngine, type GuidanceCall } from '@/hono/guidance/engine'+import {+ MCP_GUIDANCE_ERROR_TRACKING_EMPTY_FILTERS_FLAG,+ MCP_GUIDANCE_ERROR_TRACKING_EMPTY_RANGE_FLAG,+ MCP_GUIDANCE_ERROR_TRACKING_EMPTY_SETUP_FLAG,+} from '@/hono/guidance/rules'+import type { ResolvedState } from '@/hono/request-state-resolver'++import { makeToolExecutorState } from '../shared/test-utils'+import { makeGuidanceRedis } from './helpers/guidance-redis-stubs'++const ALL_FLAGS = {+ [MCP_GUIDANCE_ERROR_TRACKING_EMPTY_SETUP_FLAG]: true,+ [MCP_GUIDANCE_ERROR_TRACKING_EMPTY_FILTERS_FLAG]: true,+ [MCP_GUIDANCE_ERROR_TRACKING_EMPTY_RANGE_FLAG]: true,+}++function emptyList(args: Record<string, unknown> = {}): GuidanceCall {+ return { toolName: 'query-error-tracking-issues-list', args, isError: false, resultEmpty: true }+}++function makeState(+ options: {+ project?: () => Promise<unknown>+ request?: () => Promise<unknown>+ flags?: Record<string, boolean>+ } = {}+): {+ state: ResolvedState+ getProject: ReturnType<typeof vi.fn>+ request: ReturnType<typeof vi.fn>+} {+ const state = makeToolExecutorState([{ name: 'query-error-tracking-issues-list' }, { name: 'docs-search' }], {+ suppressAnalytics: false,+ toolFeatureFlags: options.flags ?? ALL_FLAGS,+ })+ const getProject = vi.fn(options.project ?? (async () => ({ ingested_event: true })))+ const request = vi.fn(options.request ?? (async () => ({ exists: true })))+ state.context.stateManager = {+ getCachedOrFetchProject: getProject,+ getProjectId: vi.fn().mockResolvedValue('42'),+ } as any+ state.context.api = { request } as any+ return { state, getProject, request }+}++describe('error tracking empty-result guidance', () => {+ it.each([+ ['a project with no data', { ingested_event: false }, { exists: true }, {}, undefined],+ ['a later page', { ingested_event: true }, { exists: true }, { offset: 10 }, undefined],+ ['the first page at offset 0', { ingested_event: true }, { exists: true }, { offset: 0 }, 'range'],+ ['a project with no issues', { ingested_event: true }, { exists: false }, {}, 'setup'],+ ['filters on a project with no issues', { ingested_event: true }, { exists: false }, { url: 'a' }, 'setup'],+ ['a narrowing filter', { ingested_event: true }, { exists: true }, { searchQuery: 'x' }, 'filters'],+ [+ 'status and an empty filter group',+ { ingested_event: true },+ { exists: true },+ { status: 'all', filterGroup: [] },+ 'range',+ ],+ ])('handles %s', async (_label, project, exists, args, expected) => {+ const { state } = makeState({ project: async () => project, request: async () => exists })+ const notice = await new GuidanceEngine(makeGuidanceRedis()).evaluate(state, emptyList(args))+ expect(notice?.ruleId).toBe(expected && `error-tracking-empty-to-${expected}`)+ })++ it('names the filters the call used and no others', async () => {+ const { state } = makeState()+ const notice = await new GuidanceEngine(makeGuidanceRedis()).evaluate(+ state,+ emptyList({ searchQuery: 'secret-value', library: 'posthog-js', status: 'active' })+ )+ expect(notice?.text).toContain('`searchQuery`, `library`')+ expect(notice?.text).not.toContain('secret-value')+ expect(notice?.text).not.toContain('`status`')+ })++ it('skips the issue lookup for a project with no data', async () => {+ const { state, request } = makeState({ project: async () => ({ ingested_event: false }) })+ await new GuidanceEngine(makeGuidanceRedis()).evaluate(state, emptyList())+ expect(request).not.toHaveBeenCalled()+ })++ it('ignores a populated result without reading product state', async () => {+ const { state, getProject } = makeState()+ const notice = await new GuidanceEngine(makeGuidanceRedis()).evaluate(state, {+ ...emptyList(),+ resultEmpty: false,+ })+ expect(notice).toBeUndefined()+ expect(getProject).not.toHaveBeenCalled()+ })++ it('stamps flag_off without text after the same checks', async () => {+ const { state, request } = makeState({ flags: {} })+ await expect(new GuidanceEngine(makeGuidanceRedis()).evaluate(state, emptyList())).resolves.toEqual({+ ruleId: 'error-tracking-empty-to-range',+ outcome: 'flag_off',+ })+ expect(request).toHaveBeenCalledTimes(1)+ })++ it('repeats neither the notice nor the API lookup once the session reaches the cap', async () => {+ const { state, request } = makeState()+ const engine = new GuidanceEngine(makeGuidanceRedis())+ await engine.evaluate(state, emptyList())+ await expect(engine.evaluate(state, emptyList())).resolves.toBeUndefined()+ expect(request).toHaveBeenCalledTimes(1)+ })++ it('shows the filters notice at most twice in a session', async () => {+ const { state } = makeState()+ const engine = new GuidanceEngine(makeGuidanceRedis())+ const call = emptyList({ library: 'posthog-js' })+ const notices = [+ await engine.evaluate(state, call),+ await engine.evaluate(state, call),+ await engine.evaluate(state, call),+ ]+ expect(notices.map((notice) => notice?.outcome)).toEqual(['shown', 'shown', undefined])+ })++ it('reports a suppressed setup notice after the session searched the docs', async () => {+ const { state } = makeState({ request: async () => ({ exists: false }) })+ const engine = new GuidanceEngine(makeGuidanceRedis())+ await engine.evaluate(state, { toolName: 'docs-search', args: {}, isError: false })+ await expect(engine.evaluate(state, emptyList())).resolves.toEqual({+ ruleId: 'error-tracking-empty-to-setup',+ outcome: 'suppressed',+ })+ })+})