Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(mcp): add guidance for empty error tracking results (#128493)

Adam Bowker5bb89d485cb9
diff --git a/services/mcp/src/hono/guidance/rules.ts b/services/mcp/src/hono/guidance/rules.ts
index 98e3cbe50332..87d16cc46aa3 100644
--- a/services/mcp/src/hono/guidance/rules.ts
+++ b/services/mcp/src/hono/guidance/rules.ts
@@ -52,6 +52,47 @@ export type GuidanceRule = z.infer<typeof guidanceRuleSchema>
export const MCP_GUIDANCE_REPLAY_VISION_FLAG = 'mcp-guidance-replay-vision'
export const MCP_GUIDANCE_SQL_SCHEMA_FLAG = 'mcp-guidance-sql-schema'
export const MCP_GUIDANCE_SQL_TYPED_RUNNER_FLAG = 'mcp-guidance-sql-typed-runner'
+export const MCP_GUIDANCE_ERROR_TRACKING_EMPTY_SETUP_FLAG = 'mcp-guidance-error-tracking-empty-setup'
+export const MCP_GUIDANCE_ERROR_TRACKING_EMPTY_FILTERS_FLAG = 'mcp-guidance-error-tracking-empty-filters'
+export const MCP_GUIDANCE_ERROR_TRACKING_EMPTY_RANGE_FLAG = 'mcp-guidance-error-tracking-empty-range'
+
+const ERROR_TRACKING_ISSUES_LIST_TOOL = 'query-error-tracking-issues-list'
+
+// `status` and `dateRange` are absent because the default query also sets them, and the range rule covers them.
+const ERROR_TRACKING_NARROWING_FILTERS = [
+ 'searchQuery',
+ 'library',
+ 'release',
+ 'fingerprint',
+ 'url',
+ 'personId',
+ 'user',
+ 'filePath',
+ 'filterGroup',
+ 'assignee',
+ 'firstSeenAfter',
+]
+
+function errorTrackingFiltersIn(call: GuidanceCall): string[] {
+ return ERROR_TRACKING_NARROWING_FILTERS.filter((filter) => {
+ const value = call.args?.[filter]
+ return value !== undefined && value !== null && value !== '' && !(Array.isArray(value) && value.length === 0)
+ })
+}
+
+// An empty later page only means the agent read past the last row.
+function isLaterPage(call: GuidanceCall): boolean {
+ const offset = call.args?.offset
+ return typeof offset === 'number' && offset > 0
+}
+
+async function projectHasErrorTrackingIssues(productState: GuidanceProductState): Promise<boolean | undefined> {
+ // A project with no data at all gets no notice, so the issue lookup never runs for it.
+ if ((await productState.hasIngestedData()) !== true) {
+ return undefined
+ }
+ return productState.hasErrorTrackingIssues()
+}
const TYPED_RUNNER_TOOLS = [
'query-trends',
@@ -159,6 +200,76 @@ const RULES: GuidanceRule[] = [
'To only check that an event or property exists, use `read-data-schema`.',
].join(' '),
},
+ {
+ id: 'error-tracking-empty-to-setup',
+ flag: MCP_GUIDANCE_ERROR_TRACKING_EMPTY_SETUP_FLAG,
+ trigger: {
+ tool: ERROR_TRACKING_ISSUES_LIST_TOOL,
+ resultEmpty: true,
+ outcome: 'success',
+ presets: ['user'],
+ },
+ threshold: 1,
+ maxPerSession: 1,
+ suppressIfCalled: ['docs-search'],
+ requiresTool: ['docs-search'],
+ condition: async ({ call, productState }) =>
+ !isLaterPage(call) && (await projectHasErrorTrackingIssues(productState)) === false,
+ text: [
+ 'This project has no error tracking issues at all, so exception capture is probably not set up.',
+ 'If the user wants error tracking, call `docs-search` to find the setup steps for their SDK.',
+ 'If the user expected errors here, tell them that no exceptions have reached PostHog.',
+ ].join(' '),
+ },
+ {
+ id: 'error-tracking-empty-to-filters',
+ flag: MCP_GUIDANCE_ERROR_TRACKING_EMPTY_FILTERS_FLAG,
+ trigger: {
+ tool: ERROR_TRACKING_ISSUES_LIST_TOOL,
+ resultEmpty: true,
+ outcome: 'success',
+ presets: ['user'],
+ },
+ threshold: 1,
+ maxPerSession: 2,
+ suppressIfCalled: [],
+ requiresTool: [ERROR_TRACKING_ISSUES_LIST_TOOL],
+ condition: async ({ call, productState }) =>
+ !isLaterPage(call) &&
+ errorTrackingFiltersIn(call).length > 0 &&
+ (await projectHasErrorTrackingIssues(productState)) === true,
+ text: (call) =>
+ [
+ 'This project has error tracking issues, but none matched this query.',
+ `This call filtered on ${errorTrackingFiltersIn(call)
+ .map((filter) => `\`${filter}\``)
+ .join(', ')}.`,
+ 'Remove or loosen these filters and try again before you report that no matching errors exist.',
+ ].join(' '),
+ },
+ {
+ id: 'error-tracking-empty-to-range',
+ flag: MCP_GUIDANCE_ERROR_TRACKING_EMPTY_RANGE_FLAG,
+ trigger: {
+ tool: ERROR_TRACKING_ISSUES_LIST_TOOL,
+ resultEmpty: true,
+ outcome: 'success',
+ presets: ['user'],
+ },
+ threshold: 1,
+ maxPerSession: 1,
+ suppressIfCalled: [],
+ requiresTool: [ERROR_TRACKING_ISSUES_LIST_TOOL],
+ condition: async ({ call, productState }) =>
+ !isLaterPage(call) &&
+ errorTrackingFiltersIn(call).length === 0 &&
+ (await projectHasErrorTrackingIssues(productState)) === true,
+ text: [
+ 'This project has error tracking issues, but none matched this status, date range and test account filter.',
+ 'This tool returns only issues seen during `dateRange`. `status` defaults to `active`, and `filterTestAccounts` defaults to true.',
+ 'Try a wider `dateRange`, `status: "all"` or `filterTestAccounts: false` before you report that no errors exist.',
+ ].join(' '),
+ },
]
export const GUIDANCE_RULES: readonly GuidanceRule[] = z.array(guidanceRuleSchema).parse(RULES)
diff --git a/services/mcp/tests/hono/guidance-error-tracking.test.ts b/services/mcp/tests/hono/guidance-error-tracking.test.ts
new file mode 100644
index 000000000000..8fa6f5523e9a
--- /dev/null
+++ b/services/mcp/tests/hono/guidance-error-tracking.test.ts
@@ -0,0 +1,135 @@
+import { describe, expect, it, vi } from 'vitest'
+
+import { GuidanceEngine, type GuidanceCall } from '@/hono/guidance/engine'
+import {
+ MCP_GUIDANCE_ERROR_TRACKING_EMPTY_FILTERS_FLAG,
+ MCP_GUIDANCE_ERROR_TRACKING_EMPTY_RANGE_FLAG,
+ MCP_GUIDANCE_ERROR_TRACKING_EMPTY_SETUP_FLAG,
+} from '@/hono/guidance/rules'
+import type { ResolvedState } from '@/hono/request-state-resolver'
+
+import { makeToolExecutorState } from '../shared/test-utils'
+import { makeGuidanceRedis } from './helpers/guidance-redis-stubs'
+
+const ALL_FLAGS = {
+ [MCP_GUIDANCE_ERROR_TRACKING_EMPTY_SETUP_FLAG]: true,
+ [MCP_GUIDANCE_ERROR_TRACKING_EMPTY_FILTERS_FLAG]: true,
+ [MCP_GUIDANCE_ERROR_TRACKING_EMPTY_RANGE_FLAG]: true,
+}
+
+function emptyList(args: Record<string, unknown> = {}): GuidanceCall {
+ return { toolName: 'query-error-tracking-issues-list', args, isError: false, resultEmpty: true }
+}
+
+function makeState(
+ options: {
+ project?: () => Promise<unknown>
+ request?: () => Promise<unknown>
+ flags?: Record<string, boolean>
+ } = {}
+): {
+ state: ResolvedState
+ getProject: ReturnType<typeof vi.fn>
+ request: ReturnType<typeof vi.fn>
+} {
+ const state = makeToolExecutorState([{ name: 'query-error-tracking-issues-list' }, { name: 'docs-search' }], {
+ suppressAnalytics: false,
+ toolFeatureFlags: options.flags ?? ALL_FLAGS,
+ })
+ const getProject = vi.fn(options.project ?? (async () => ({ ingested_event: true })))
+ const request = vi.fn(options.request ?? (async () => ({ exists: true })))
+ state.context.stateManager = {
+ getCachedOrFetchProject: getProject,
+ getProjectId: vi.fn().mockResolvedValue('42'),
+ } as any
+ state.context.api = { request } as any
+ return { state, getProject, request }
+}
+
+describe('error tracking empty-result guidance', () => {
+ it.each([
+ ['a project with no data', { ingested_event: false }, { exists: true }, {}, undefined],
+ ['a later page', { ingested_event: true }, { exists: true }, { offset: 10 }, undefined],
+ ['the first page at offset 0', { ingested_event: true }, { exists: true }, { offset: 0 }, 'range'],
+ ['a project with no issues', { ingested_event: true }, { exists: false }, {}, 'setup'],
+ ['filters on a project with no issues', { ingested_event: true }, { exists: false }, { url: 'a' }, 'setup'],
+ ['a narrowing filter', { ingested_event: true }, { exists: true }, { searchQuery: 'x' }, 'filters'],
+ [
+ 'status and an empty filter group',
+ { ingested_event: true },
+ { exists: true },
+ { status: 'all', filterGroup: [] },
+ 'range',
+ ],
+ ])('handles %s', async (_label, project, exists, args, expected) => {
+ const { state } = makeState({ project: async () => project, request: async () => exists })
+ const notice = await new GuidanceEngine(makeGuidanceRedis()).evaluate(state, emptyList(args))
+ expect(notice?.ruleId).toBe(expected && `error-tracking-empty-to-${expected}`)
+ })
+
+ it('names the filters the call used and no others', async () => {
+ const { state } = makeState()
+ const notice = await new GuidanceEngine(makeGuidanceRedis()).evaluate(
+ state,
+ emptyList({ searchQuery: 'secret-value', library: 'posthog-js', status: 'active' })
+ )
+ expect(notice?.text).toContain('`searchQuery`, `library`')
+ expect(notice?.text).not.toContain('secret-value')
+ expect(notice?.text).not.toContain('`status`')
+ })
+
+ it('skips the issue lookup for a project with no data', async () => {
+ const { state, request } = makeState({ project: async () => ({ ingested_event: false }) })
+ await new GuidanceEngine(makeGuidanceRedis()).evaluate(state, emptyList())
+ expect(request).not.toHaveBeenCalled()
+ })
+
+ it('ignores a populated result without reading product state', async () => {
+ const { state, getProject } = makeState()
+ const notice = await new GuidanceEngine(makeGuidanceRedis()).evaluate(state, {
+ ...emptyList(),
+ resultEmpty: false,
+ })
+ expect(notice).toBeUndefined()
+ expect(getProject).not.toHaveBeenCalled()
+ })
+
+ it('stamps flag_off without text after the same checks', async () => {
+ const { state, request } = makeState({ flags: {} })
+ await expect(new GuidanceEngine(makeGuidanceRedis()).evaluate(state, emptyList())).resolves.toEqual({
+ ruleId: 'error-tracking-empty-to-range',
+ outcome: 'flag_off',
+ })
+ expect(request).toHaveBeenCalledTimes(1)
+ })
+
+ it('repeats neither the notice nor the API lookup once the session reaches the cap', async () => {
+ const { state, request } = makeState()
+ const engine = new GuidanceEngine(makeGuidanceRedis())
+ await engine.evaluate(state, emptyList())
+ await expect(engine.evaluate(state, emptyList())).resolves.toBeUndefined()
+ expect(request).toHaveBeenCalledTimes(1)
+ })
+
+ it('shows the filters notice at most twice in a session', async () => {
+ const { state } = makeState()
+ const engine = new GuidanceEngine(makeGuidanceRedis())
+ const call = emptyList({ library: 'posthog-js' })
+ const notices = [
+ await engine.evaluate(state, call),
+ await engine.evaluate(state, call),
+ await engine.evaluate(state, call),
+ ]
+ expect(notices.map((notice) => notice?.outcome)).toEqual(['shown', 'shown', undefined])
+ })
+
+ it('reports a suppressed setup notice after the session searched the docs', async () => {
+ const { state } = makeState({ request: async () => ({ exists: false }) })
+ const engine = new GuidanceEngine(makeGuidanceRedis())
+ await engine.evaluate(state, { toolName: 'docs-search', args: {}, isError: false })
+ await expect(engine.evaluate(state, emptyList())).resolves.toEqual({
+ ruleId: 'error-tracking-empty-to-setup',
+ outcome: 'suppressed',
+ })
+ })
+})