Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(mcp-analytics): track next steps and fix the alert destinations copy (#118205)

Lucas Fariab92a51dc58ff
diff --git a/products/mcp_analytics/frontend/earlyData/ActivityNextSteps.tsx b/products/mcp_analytics/frontend/earlyData/ActivityNextSteps.tsx
index 6320d4a09112..f8012f4efc46 100644
--- a/products/mcp_analytics/frontend/earlyData/ActivityNextSteps.tsx
+++ b/products/mcp_analytics/frontend/earlyData/ActivityNextSteps.tsx
@@ -1,4 +1,4 @@
-import { useValues } from 'kea'
+import { useActions, useValues } from 'kea'
import { useState } from 'react'
import { IconSparkles } from '@posthog/icons'
@@ -20,6 +20,7 @@ function AgentPromptAction({
prompt: string
docsUrl: string
}): JSX.Element {
+ const { nextStepClicked } = useActions(mcpEarlyDataLogic)
const [open, setOpen] = useState(false)
return (
<>
@@ -27,7 +28,10 @@ function AgentPromptAction({
type="secondary"
size="small"
icon={<IconSparkles />}
- onClick={() => setOpen(true)}
+ onClick={() => {
+ nextStepClicked(step.key, 'open')
+ setOpen(true)
+ }}
data-attr={`mcp-analytics-activity-next-step-${step.key}`}
>
Fix with your agent
@@ -37,6 +41,7 @@ function AgentPromptAction({
size="small"
to={docsUrl}
targetBlank
+ onClick={() => nextStepClicked(step.key, 'docs')}
data-attr={`mcp-analytics-activity-next-step-${step.key}-docs`}
>
Docs
@@ -55,12 +60,14 @@ function AgentPromptAction({
}
function NextStepActions({ step }: { step: NextStep }): JSX.Element {
+ const { nextStepClicked } = useActions(mcpEarlyDataLogic)
if (step.action.kind === 'link') {
return (
<LemonButton
type="secondary"
size="small"
to={step.action.to}
+ onClick={() => nextStepClicked(step.key, 'open')}
data-attr={`mcp-analytics-activity-next-step-${step.key}`}
>
{step.action.label}
@@ -73,7 +80,7 @@ function NextStepActions({ step }: { step: NextStep }): JSX.Element {
export function ActivityNextSteps(): JSX.Element | null {
const { nextSteps } = useValues(mcpEarlyDataLogic)
- if (nextSteps.length === 0) {
+ if (!nextSteps?.length) {
return null
}
diff --git a/products/mcp_analytics/frontend/earlyData/earlyData.test.ts b/products/mcp_analytics/frontend/earlyData/earlyData.test.ts
index dc7e545046e8..710c5dcae16d 100644
--- a/products/mcp_analytics/frontend/earlyData/earlyData.test.ts
+++ b/products/mcp_analytics/frontend/earlyData/earlyData.test.ts
@@ -1,3 +1,5 @@
+import { DESTINATION_OPTIONS } from 'scenes/hog-functions/list/newNotificationDialogLogic'
+
import { buildActivitySummary } from './activitySummary'
import { EarlyStats, buildNextSteps } from './nextSteps'
@@ -112,4 +114,20 @@ describe('early data derivations', () => {
})
expect(steps[1].action).toMatchObject({ to: '/mcp-analytics/notifications' })
})
+
+ describe('alert step copy', () => {
+ const alertStep = buildNextSteps(
+ stats({ totalCalls: 40, callsWithIntent: 35, distinctSessions: 8, errorCalls: 3 }),
+ false
+ )[0]
+
+ // Hog function alerts can only go to the destinations the new-alert dialog lists.
+ it.each(DESTINATION_OPTIONS.map(({ label }) => label))('offers %s', (label) => {
+ expect(alertStep.detail.toLowerCase()).toContain(label.toLowerCase())
+ })
+
+ it('does not offer email', () => {
+ expect(alertStep.detail).not.toMatch(/email/i)
+ })
+ })
})
diff --git a/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.test.ts b/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.test.ts
index aa9bafb0cf0a..8972a6059669 100644
--- a/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.test.ts
+++ b/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.test.ts
@@ -1,18 +1,48 @@
import { expectLogic } from 'kea-test-utils'
import api from 'lib/api'
+import posthog from 'lib/posthog-typed'
import { initKeaTests } from '~/test/init'
import { AnyPropertyFilter, PropertyFilterType, PropertyOperator } from '~/types'
import { mcpAnalyticsSessionsActivityOverview } from '../generated/api'
+import type { MCPActivityOverviewApi } from '../generated/api.schemas'
import { mcpAnalyticsFiltersLogic } from '../mcpAnalyticsFiltersLogic'
import { mcpEarlyDataLogic } from './mcpEarlyDataLogic'
jest.mock('lib/api')
+jest.mock('lib/posthog-typed', () => ({ __esModule: true, default: { capture: jest.fn(), captureRaw: jest.fn() } }))
jest.mock('../generated/api', () => ({ mcpAnalyticsSessionsActivityOverview: jest.fn() }))
const overviewMock = mcpAnalyticsSessionsActivityOverview as jest.Mock
+const captureRawMock = posthog.captureRaw as jest.Mock
+
+function overviewWith(stats: Partial<MCPActivityOverviewApi['stats']>): MCPActivityOverviewApi {
+ return {
+ stats: {
+ total_calls: 40,
+ distinct_tools: 5,
+ distinct_sessions: 8,
+ distinct_clients: 2,
+ calls_with_intent: 35,
+ error_calls: 0,
+ missing_capability_reports: 0,
+ ...stats,
+ },
+ top_tools: [],
+ clients: [],
+ recent_calls: [],
+ }
+}
+
+const INTENT_AND_ALERT_STEPS = overviewWith({ calls_with_intent: 2, error_calls: 3 })
+const ALERT_STEP_ONLY = overviewWith({ error_calls: 3 })
+const NO_STEPS = overviewWith({})
+
+function eventsNamed(name: string): unknown[][] {
+ return captureRawMock.mock.calls.filter(([event]) => event === name).map(([, properties]) => properties)
+}
const TOOL_FILTER: AnyPropertyFilter = {
key: '$mcp_tool_name',
value: ['create_insight'],
@@ -72,6 +102,96 @@ describe('mcpEarlyDataLogic', () => {
expect(logic.values.activityQuery.source).toMatchObject(expectedQuery)
})
+ it.each([
+ ['one refresh with steps', [INTENT_AND_ALERT_STEPS], [['intent', 'notification']]],
+ [
+ 'the same steps after the 60s refresh',
+ [INTENT_AND_ALERT_STEPS, overviewWith({ calls_with_intent: 2, error_calls: 3 })],
+ [['intent', 'notification']],
+ ],
+ [
+ 'a step done in between',
+ [INTENT_AND_ALERT_STEPS, ALERT_STEP_ONLY],
+ [['intent', 'notification'], ['notification']],
+ ],
+ ['no steps to show', [NO_STEPS], []],
+ [
+ 'the steps coming back after none',
+ [INTENT_AND_ALERT_STEPS, NO_STEPS, INTENT_AND_ALERT_STEPS],
+ [
+ ['intent', 'notification'],
+ ['intent', 'notification'],
+ ],
+ ],
+ ])('reports the steps shown for %s', async (_label, overviews, expectedShown) => {
+ await expectLogic(logic).toFinishAllListeners()
+ captureRawMock.mockClear()
+
+ for (const overview of overviews) {
+ overviewMock.mockResolvedValue(overview)
+ await expectLogic(logic, () => logic.actions.loadOverview()).toFinishAllListeners()
+ }
+
+ expect(eventsNamed('mcp analytics next steps shown').map((properties: any) => properties.step_keys)).toEqual(
+ expectedShown
+ )
+ })
+
+ it.each([
+ ['loads', false, ['intent', 'notification']],
+ ['fails to load', true, ['intent']],
+ ])('holds the card until the alert list %s, then reports it once', async (_label, listFails, expectedSteps) => {
+ logic.unmount()
+ let settleList!: () => void
+ const list = new Promise<Awaited<ReturnType<typeof api.hogFunctions.list>>>((resolve, reject) => {
+ settleList = () =>
+ listFails ? reject(new Error('down')) : resolve({ count: 0, next: null, previous: null, results: [] })
+ })
+ jest.spyOn(api.hogFunctions, 'list').mockReturnValue(list)
+ overviewMock.mockResolvedValue(INTENT_AND_ALERT_STEPS)
+ captureRawMock.mockClear()
+ logic = mcpEarlyDataLogic()
+ logic.mount()
+
+ await expectLogic(logic).toDispatchActions(['loadOverviewSuccess'])
+
+ expect(logic.values.nextSteps).toBeNull()
+ expect(eventsNamed('mcp analytics next steps shown')).toEqual([])
+
+ settleList()
+ await expectLogic(logic).toFinishAllListeners()
+
+ expect(eventsNamed('mcp analytics next steps shown').map((properties: any) => properties.step_keys)).toEqual([
+ expectedSteps,
+ ])
+ })
+
+ it('reports how much data stood behind the steps shown', async () => {
+ await expectLogic(logic).toFinishAllListeners()
+ captureRawMock.mockClear()
+ overviewMock.mockResolvedValue(INTENT_AND_ALERT_STEPS)
+
+ await expectLogic(logic, () => logic.actions.loadOverview()).toFinishAllListeners()
+
+ expect(eventsNamed('mcp analytics next steps shown')).toEqual([
+ { step_keys: ['intent', 'notification'], total_calls: 40, error_calls: 3 },
+ ])
+ })
+
+ it.each([
+ ['intent', 'open', 0],
+ ['intent', 'docs', 0],
+ ['notification', 'open', 1],
+ ] as const)('reports a %s click with interaction %s at position %s', async (stepKey, interaction, position) => {
+ overviewMock.mockResolvedValue(INTENT_AND_ALERT_STEPS)
+ await expectLogic(logic, () => logic.actions.loadOverview()).toFinishAllListeners()
+ captureRawMock.mockClear()
+
+ logic.actions.nextStepClicked(stepKey, interaction)
+
+ expect(eventsNamed('mcp analytics next step clicked')).toEqual([{ step_key: stepKey, interaction, position }])
+ })
+
const MINUTE = 60_000
it.each([
diff --git a/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.ts b/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.ts
index 96a31748499c..2adff7e6da84 100644
--- a/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.ts
+++ b/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.ts
@@ -1,7 +1,20 @@
-import { MakeLogicType, actions, afterMount, connect, kea, listeners, path, reducers, selectors } from 'kea'
+import {
+ MakeLogicType,
+ actions,
+ afterMount,
+ connect,
+ kea,
+ listeners,
+ path,
+ reducers,
+ selectors,
+ sharedListeners,
+} from 'kea'
+import type { BreakPointFunction } from 'kea'
import { loaders } from 'kea-loaders'
import api from 'lib/api'
+import posthog from 'lib/posthog-typed'
import { teamLogic } from 'scenes/teamLogic'
import { dataNodeCollectionLogic } from '~/queries/nodes/DataNode/dataNodeCollectionLogic'
@@ -28,10 +41,12 @@ import type { MCPOnboardingSignals } from '../mcpAnalyticsOnboardingLogic'
import type { HarnessRow, ModelRow } from '../mcpDashboardOverviewLogic'
import { mcpAnalyticsNotificationsLogic } from '../notifications/mcpAnalyticsNotificationsLogic'
import { ActivitySummary, buildActivitySummary } from './activitySummary'
-import { EarlyStats, NextStep, buildNextSteps } from './nextSteps'
+import { EarlyStats, NextStep, NextStepInteraction, NextStepKey, buildNextSteps } from './nextSteps'
export type { EarlyStats, NextStep } from './nextSteps'
+export type NotificationsLoad = 'pending' | 'loaded' | 'failed'
+
export interface EarlyToolRow {
tool: string
calls: number
@@ -75,8 +90,8 @@ export interface mcpEarlyDataLogicValues {
inputSinceRefresh: boolean
modelRows: ModelRow[]
modelRowsLoading: boolean
- nextSteps: NextStep[]
- notificationsResolved: boolean
+ nextSteps: NextStep[] | null
+ notificationsLoad: NotificationsLoad
overview: MCPActivityOverviewApi | null
overviewLoading: boolean
refreshDelayMs: number
@@ -103,6 +118,13 @@ export interface mcpEarlyDataLogicActions {
properties: import('~/types').AnyPropertyFilter[]
} // mcpAnalyticsFiltersLogic
loadNotifications: () => any // mcpAnalyticsNotificationsLogic
+ loadNotificationsFailure: (
+ error: string,
+ errorObject?: any
+ ) => {
+ error: string
+ errorObject?: any
+ } // mcpAnalyticsNotificationsLogic
loadNotificationsSuccess: (
notifications: HogFunctionType[],
payload?: any
@@ -156,6 +178,13 @@ export interface mcpEarlyDataLogicActions {
overview: MCPActivityOverviewApi | null
payload?: void
}
+ nextStepClicked: (
+ stepKey: NextStepKey,
+ interaction: NextStepInteraction
+ ) => {
+ interaction: NextStepInteraction
+ stepKey: NextStepKey
+ }
noteUserInput: () => {
value: true
}
@@ -184,12 +213,31 @@ export interface mcpEarlyDataLogicActions {
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface mcpEarlyDataLogicMeta {
+ sharedListeners: {
+ captureNextStepsShown: (
+ payload: any,
+ breakpoint: BreakPointFunction,
+ action: {
+ type: string
+ payload: any
+ },
+ previousState: any
+ ) => void | Promise<void>
+ }
__keaTypeGenInternalSelectorTypes: {
stats: (overview: MCPActivityOverviewApi | null) => EarlyStats
topTools: (overview: MCPActivityOverviewApi | null) => EarlyToolRow[]
summary: (signals: MCPOnboardingSignals | null, stats: EarlyStats, topTools: EarlyToolRow[]) => ActivitySummary
- hasFailureNotification: (notificationsResolved: boolean, notifications: HogFunctionType[]) => boolean | null
- nextSteps: (stats: EarlyStats, hasFailureNotification: boolean | null) => NextStep[]
+ hasFailureNotification: (
+ notificationsLoad: NotificationsLoad,
+ notifications: HogFunctionType[]
+ ) => boolean | null
+ nextSteps: (
+ overview: MCPActivityOverviewApi | null,
+ stats: EarlyStats,
+ notificationsLoad: NotificationsLoad,
+ hasFailureNotification: boolean | null
+ ) => NextStep[] | null
hasModelData: (modelRows: ModelRow[]) => boolean
refreshDelayMs: (refreshesWithoutInput: number) => number
}
@@ -223,7 +271,7 @@ export const mcpEarlyDataLogic = kea<mcpEarlyDataLogicType>([
mcpAnalyticsFiltersLogic,
['hydrateFilters', 'setFilterTestAccounts', 'setPropertyFilters'],
mcpAnalyticsNotificationsLogic,
- ['loadNotifications', 'loadNotificationsSuccess'],
+ ['loadNotifications', 'loadNotificationsSuccess', 'loadNotificationsFailure'],
dataNodeCollectionLogic({ key: MCP_ACTIVITY_DATA_COLLECTION_ID }),
['reloadAll as reloadActivityQuery'],
],
@@ -237,11 +285,17 @@ export const mcpEarlyDataLogic = kea<mcpEarlyDataLogicType>([
scheduleRefresh: true,
setActivityQuery: (query: DataTableNode) => ({ query }),
showFailedCallsOnly: true,
+ nextStepClicked: (stepKey: NextStepKey, interaction: NextStepInteraction) => ({ stepKey, interaction }),
}),
reducers({
activityQuery: [DEFAULT_MCP_ACTIVITY_QUERY, { setActivityQuery: (_, { query }) => query }],
// The list defaults to empty before its first load, which would read as "no alert set up".
- notificationsResolved: [false, { loadNotificationsSuccess: () => true }],
+ // A failed load also releases the card, so it never waits forever on a list that will not arrive,
+ // but leaves the alert question unanswered rather than reading the empty default as "no alert".
+ notificationsLoad: [
+ 'pending' as NotificationsLoad,
+ { loadNotificationsSuccess: () => 'loaded', loadNotificationsFailure: () => 'failed' },
+ ],
inputSinceRefresh: [false, { noteUserInput: () => true, refreshOnSchedule: () => false }],
refreshesWithoutInput: [
0,
@@ -335,14 +389,20 @@ export const mcpEarlyDataLogic = kea<mcpEarlyDataLogicType>([
],
// A paused alert notifies nobody, so only enabled ones count as "set up".
hasFailureNotification: [
- (s) => [s.notificationsResolved, s.notifications],
- (resolved: boolean, notifications: HogFunctionType[]): boolean | null =>
- resolved ? notifications.some((notification) => notification.enabled) : null,
+ (s) => [s.notificationsLoad, s.notifications],
+ (notificationsLoad: NotificationsLoad, notifications: HogFunctionType[]): boolean | null =>
+ notificationsLoad === 'loaded' ? notifications.some((notification) => notification.enabled) : null,
],
+ // Null until the overview and the notification list both resolve, so steps never reorder under the reader.
nextSteps: [
- (s) => [s.stats, s.hasFailureNotification],
- (stats: EarlyStats, hasFailureNotification: boolean | null): NextStep[] =>
- buildNextSteps(stats, hasFailureNotification),
+ (s) => [s.overview, s.stats, s.notificationsLoad, s.hasFailureNotification],
+ (
+ overview: MCPActivityOverviewApi | null,
+ stats: EarlyStats,
+ notificationsLoad: NotificationsLoad,
+ hasFailureNotification: boolean | null
+ ): NextStep[] | null =>
+ overview && notificationsLoad !== 'pending' ? buildNextSteps(stats, hasFailureNotification) : null,
],
hasModelData: [
(s) => [s.modelRows],
@@ -354,7 +414,37 @@ export const mcpEarlyDataLogic = kea<mcpEarlyDataLogicType>([
Math.min(REFRESH_INTERVAL_MS * 2 ** refreshesWithoutInput, MAX_REFRESH_INTERVAL_MS),
],
}),
- listeners(({ actions, values, cache }) => ({
+ sharedListeners(({ cache, values }) => ({
+ // Bound to each action that resolves an input of `nextSteps` (reacting-to-changes.md prefers
+ // sharedListeners to kea-subscriptions). Compares step keys: the 60s refresh rebuilds the same steps.
+ captureNextStepsShown: () => {
+ const stepKeys = (values.nextSteps ?? []).map((step) => step.key)
+ const shown = stepKeys.join()
+ if (shown === cache.shownStepKeys) {
+ return
+ }
+ // An empty card is remembered too, so steps coming back after none are shown again.
+ cache.shownStepKeys = shown
+ if (stepKeys.length > 0) {
+ posthog.captureRaw('mcp analytics next steps shown', {
+ step_keys: stepKeys,
+ total_calls: values.stats.totalCalls,
+ error_calls: values.stats.errorCalls,
+ })
+ }
+ },
+ })),
+ listeners(({ actions, values, cache, sharedListeners }) => ({
+ loadOverviewSuccess: sharedListeners.captureNextStepsShown,
+ loadNotificationsSuccess: sharedListeners.captureNextStepsShown,
+ loadNotificationsFailure: sharedListeners.captureNextStepsShown,
+ nextStepClicked: ({ stepKey, interaction }) => {
+ posthog.captureRaw('mcp analytics next step clicked', {
+ step_key: stepKey,
+ interaction,
+ position: values.nextSteps?.findIndex((step) => step.key === stepKey) ?? -1,
+ })
+ },
scheduleRefresh: () => {
cache.disposables.add(() => {
const id = window.setTimeout(
diff --git a/products/mcp_analytics/frontend/earlyData/nextSteps.ts b/products/mcp_analytics/frontend/earlyData/nextSteps.ts
index 70225f462974..c3ce7eb389e5 100644
--- a/products/mcp_analytics/frontend/earlyData/nextSteps.ts
+++ b/products/mcp_analytics/frontend/earlyData/nextSteps.ts
@@ -15,8 +15,12 @@ export type NextStepAction =
/** A self-contained instruction the user pastes into their coding agent, with the docs as the fallback. */
| { kind: 'agent-prompt'; prompt: string; docsUrl: string }
+export type NextStepKey = 'intent' | 'notification' | 'sessions'
+
+export type NextStepInteraction = 'open' | 'docs'
+
export interface NextStep {
- key: 'intent' | 'notification' | 'sessions'
+ key: NextStepKey
title: string
detail: string
action: NextStepAction
@@ -62,7 +66,7 @@ export function buildNextSteps(stats: EarlyStats, hasFailureNotification: boolea
steps.push({
key: 'notification',
title: 'Get alerted when a tool call fails',
- detail: `${stats.errorCalls} tool call${stats.errorCalls === 1 ? '' : 's'} failed in the last 30 days. Send new failures to Slack, email, or a webhook.`,
+ detail: `${stats.errorCalls} tool call${stats.errorCalls === 1 ? '' : 's'} failed in the last 30 days. Send new failures to Slack, Discord, Microsoft Teams, or a webhook.`,
action: { kind: 'link', label: 'Set up an alert', to: urls.mcpAnalyticsNotifications() },
})
}