:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(mcp-analytics): track next steps and fix the alert destinations copy (#118205)
Lucas Fariab92a51dc58ff
diff --git a/products/mcp_analytics/frontend/earlyData/ActivityNextSteps.tsx b/products/mcp_analytics/frontend/earlyData/ActivityNextSteps.tsxindex 6320d4a09112..f8012f4efc46 100644--- a/products/mcp_analytics/frontend/earlyData/ActivityNextSteps.tsx+++ b/products/mcp_analytics/frontend/earlyData/ActivityNextSteps.tsx@@ -1,4 +1,4 @@-import { useValues } from 'kea'+import { useActions, useValues } from 'kea'import { useState } from 'react'import { IconSparkles } from '@posthog/icons'@@ -20,6 +20,7 @@ function AgentPromptAction({prompt: stringdocsUrl: string}): JSX.Element {+ const { nextStepClicked } = useActions(mcpEarlyDataLogic)const [open, setOpen] = useState(false)return (<>@@ -27,7 +28,10 @@ function AgentPromptAction({type="secondary"size="small"icon={<IconSparkles />}- onClick={() => setOpen(true)}+ onClick={() => {+ nextStepClicked(step.key, 'open')+ setOpen(true)+ }}data-attr={`mcp-analytics-activity-next-step-${step.key}`}>Fix with your agent@@ -37,6 +41,7 @@ function AgentPromptAction({size="small"to={docsUrl}targetBlank+ onClick={() => nextStepClicked(step.key, 'docs')}data-attr={`mcp-analytics-activity-next-step-${step.key}-docs`}>Docs@@ -55,12 +60,14 @@ function AgentPromptAction({}function NextStepActions({ step }: { step: NextStep }): JSX.Element {+ const { nextStepClicked } = useActions(mcpEarlyDataLogic)if (step.action.kind === 'link') {return (<LemonButtontype="secondary"size="small"to={step.action.to}+ onClick={() => nextStepClicked(step.key, 'open')}data-attr={`mcp-analytics-activity-next-step-${step.key}`}>{step.action.label}@@ -73,7 +80,7 @@ function NextStepActions({ step }: { step: NextStep }): JSX.Element {export function ActivityNextSteps(): JSX.Element | null {const { nextSteps } = useValues(mcpEarlyDataLogic)- if (nextSteps.length === 0) {+ if (!nextSteps?.length) {return null}diff --git a/products/mcp_analytics/frontend/earlyData/earlyData.test.ts b/products/mcp_analytics/frontend/earlyData/earlyData.test.tsindex dc7e545046e8..710c5dcae16d 100644--- a/products/mcp_analytics/frontend/earlyData/earlyData.test.ts+++ b/products/mcp_analytics/frontend/earlyData/earlyData.test.ts@@ -1,3 +1,5 @@+import { DESTINATION_OPTIONS } from 'scenes/hog-functions/list/newNotificationDialogLogic'+import { buildActivitySummary } from './activitySummary'import { EarlyStats, buildNextSteps } from './nextSteps'@@ -112,4 +114,20 @@ describe('early data derivations', () => {})expect(steps[1].action).toMatchObject({ to: '/mcp-analytics/notifications' })})++ describe('alert step copy', () => {+ const alertStep = buildNextSteps(+ stats({ totalCalls: 40, callsWithIntent: 35, distinctSessions: 8, errorCalls: 3 }),+ false+ )[0]++ // Hog function alerts can only go to the destinations the new-alert dialog lists.+ it.each(DESTINATION_OPTIONS.map(({ label }) => label))('offers %s', (label) => {+ expect(alertStep.detail.toLowerCase()).toContain(label.toLowerCase())+ })++ it('does not offer email', () => {+ expect(alertStep.detail).not.toMatch(/email/i)+ })+ })})diff --git a/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.test.ts b/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.test.tsindex aa9bafb0cf0a..8972a6059669 100644--- a/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.test.ts+++ b/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.test.ts@@ -1,18 +1,48 @@import { expectLogic } from 'kea-test-utils'import api from 'lib/api'+import posthog from 'lib/posthog-typed'import { initKeaTests } from '~/test/init'import { AnyPropertyFilter, PropertyFilterType, PropertyOperator } from '~/types'import { mcpAnalyticsSessionsActivityOverview } from '../generated/api'+import type { MCPActivityOverviewApi } from '../generated/api.schemas'import { mcpAnalyticsFiltersLogic } from '../mcpAnalyticsFiltersLogic'import { mcpEarlyDataLogic } from './mcpEarlyDataLogic'jest.mock('lib/api')+jest.mock('lib/posthog-typed', () => ({ __esModule: true, default: { capture: jest.fn(), captureRaw: jest.fn() } }))jest.mock('../generated/api', () => ({ mcpAnalyticsSessionsActivityOverview: jest.fn() }))const overviewMock = mcpAnalyticsSessionsActivityOverview as jest.Mock+const captureRawMock = posthog.captureRaw as jest.Mock++function overviewWith(stats: Partial<MCPActivityOverviewApi['stats']>): MCPActivityOverviewApi {+ return {+ stats: {+ total_calls: 40,+ distinct_tools: 5,+ distinct_sessions: 8,+ distinct_clients: 2,+ calls_with_intent: 35,+ error_calls: 0,+ missing_capability_reports: 0,+ ...stats,+ },+ top_tools: [],+ clients: [],+ recent_calls: [],+ }+}++const INTENT_AND_ALERT_STEPS = overviewWith({ calls_with_intent: 2, error_calls: 3 })+const ALERT_STEP_ONLY = overviewWith({ error_calls: 3 })+const NO_STEPS = overviewWith({})++function eventsNamed(name: string): unknown[][] {+ return captureRawMock.mock.calls.filter(([event]) => event === name).map(([, properties]) => properties)+}const TOOL_FILTER: AnyPropertyFilter = {key: '$mcp_tool_name',value: ['create_insight'],@@ -72,6 +102,96 @@ describe('mcpEarlyDataLogic', () => {expect(logic.values.activityQuery.source).toMatchObject(expectedQuery)})+ it.each([+ ['one refresh with steps', [INTENT_AND_ALERT_STEPS], [['intent', 'notification']]],+ [+ 'the same steps after the 60s refresh',+ [INTENT_AND_ALERT_STEPS, overviewWith({ calls_with_intent: 2, error_calls: 3 })],+ [['intent', 'notification']],+ ],+ [+ 'a step done in between',+ [INTENT_AND_ALERT_STEPS, ALERT_STEP_ONLY],+ [['intent', 'notification'], ['notification']],+ ],+ ['no steps to show', [NO_STEPS], []],+ [+ 'the steps coming back after none',+ [INTENT_AND_ALERT_STEPS, NO_STEPS, INTENT_AND_ALERT_STEPS],+ [+ ['intent', 'notification'],+ ['intent', 'notification'],+ ],+ ],+ ])('reports the steps shown for %s', async (_label, overviews, expectedShown) => {+ await expectLogic(logic).toFinishAllListeners()+ captureRawMock.mockClear()++ for (const overview of overviews) {+ overviewMock.mockResolvedValue(overview)+ await expectLogic(logic, () => logic.actions.loadOverview()).toFinishAllListeners()+ }++ expect(eventsNamed('mcp analytics next steps shown').map((properties: any) => properties.step_keys)).toEqual(+ expectedShown+ )+ })++ it.each([+ ['loads', false, ['intent', 'notification']],+ ['fails to load', true, ['intent']],+ ])('holds the card until the alert list %s, then reports it once', async (_label, listFails, expectedSteps) => {+ logic.unmount()+ let settleList!: () => void+ const list = new Promise<Awaited<ReturnType<typeof api.hogFunctions.list>>>((resolve, reject) => {+ settleList = () =>+ listFails ? reject(new Error('down')) : resolve({ count: 0, next: null, previous: null, results: [] })+ })+ jest.spyOn(api.hogFunctions, 'list').mockReturnValue(list)+ overviewMock.mockResolvedValue(INTENT_AND_ALERT_STEPS)+ captureRawMock.mockClear()+ logic = mcpEarlyDataLogic()+ logic.mount()++ await expectLogic(logic).toDispatchActions(['loadOverviewSuccess'])++ expect(logic.values.nextSteps).toBeNull()+ expect(eventsNamed('mcp analytics next steps shown')).toEqual([])++ settleList()+ await expectLogic(logic).toFinishAllListeners()++ expect(eventsNamed('mcp analytics next steps shown').map((properties: any) => properties.step_keys)).toEqual([+ expectedSteps,+ ])+ })++ it('reports how much data stood behind the steps shown', async () => {+ await expectLogic(logic).toFinishAllListeners()+ captureRawMock.mockClear()+ overviewMock.mockResolvedValue(INTENT_AND_ALERT_STEPS)++ await expectLogic(logic, () => logic.actions.loadOverview()).toFinishAllListeners()++ expect(eventsNamed('mcp analytics next steps shown')).toEqual([+ { step_keys: ['intent', 'notification'], total_calls: 40, error_calls: 3 },+ ])+ })++ it.each([+ ['intent', 'open', 0],+ ['intent', 'docs', 0],+ ['notification', 'open', 1],+ ] as const)('reports a %s click with interaction %s at position %s', async (stepKey, interaction, position) => {+ overviewMock.mockResolvedValue(INTENT_AND_ALERT_STEPS)+ await expectLogic(logic, () => logic.actions.loadOverview()).toFinishAllListeners()+ captureRawMock.mockClear()++ logic.actions.nextStepClicked(stepKey, interaction)++ expect(eventsNamed('mcp analytics next step clicked')).toEqual([{ step_key: stepKey, interaction, position }])+ })+const MINUTE = 60_000it.each([diff --git a/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.ts b/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.tsindex 96a31748499c..2adff7e6da84 100644--- a/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.ts+++ b/products/mcp_analytics/frontend/earlyData/mcpEarlyDataLogic.ts@@ -1,7 +1,20 @@-import { MakeLogicType, actions, afterMount, connect, kea, listeners, path, reducers, selectors } from 'kea'+import {+ MakeLogicType,+ actions,+ afterMount,+ connect,+ kea,+ listeners,+ path,+ reducers,+ selectors,+ sharedListeners,+} from 'kea'+import type { BreakPointFunction } from 'kea'import { loaders } from 'kea-loaders'import api from 'lib/api'+import posthog from 'lib/posthog-typed'import { teamLogic } from 'scenes/teamLogic'import { dataNodeCollectionLogic } from '~/queries/nodes/DataNode/dataNodeCollectionLogic'@@ -28,10 +41,12 @@ import type { MCPOnboardingSignals } from '../mcpAnalyticsOnboardingLogic'import type { HarnessRow, ModelRow } from '../mcpDashboardOverviewLogic'import { mcpAnalyticsNotificationsLogic } from '../notifications/mcpAnalyticsNotificationsLogic'import { ActivitySummary, buildActivitySummary } from './activitySummary'-import { EarlyStats, NextStep, buildNextSteps } from './nextSteps'+import { EarlyStats, NextStep, NextStepInteraction, NextStepKey, buildNextSteps } from './nextSteps'export type { EarlyStats, NextStep } from './nextSteps'+export type NotificationsLoad = 'pending' | 'loaded' | 'failed'+export interface EarlyToolRow {tool: stringcalls: number@@ -75,8 +90,8 @@ export interface mcpEarlyDataLogicValues {inputSinceRefresh: booleanmodelRows: ModelRow[]modelRowsLoading: boolean- nextSteps: NextStep[]- notificationsResolved: boolean+ nextSteps: NextStep[] | null+ notificationsLoad: NotificationsLoadoverview: MCPActivityOverviewApi | nulloverviewLoading: booleanrefreshDelayMs: number@@ -103,6 +118,13 @@ export interface mcpEarlyDataLogicActions {properties: import('~/types').AnyPropertyFilter[]} // mcpAnalyticsFiltersLogicloadNotifications: () => any // mcpAnalyticsNotificationsLogic+ loadNotificationsFailure: (+ error: string,+ errorObject?: any+ ) => {+ error: string+ errorObject?: any+ } // mcpAnalyticsNotificationsLogicloadNotificationsSuccess: (notifications: HogFunctionType[],payload?: any@@ -156,6 +178,13 @@ export interface mcpEarlyDataLogicActions {overview: MCPActivityOverviewApi | nullpayload?: void}+ nextStepClicked: (+ stepKey: NextStepKey,+ interaction: NextStepInteraction+ ) => {+ interaction: NextStepInteraction+ stepKey: NextStepKey+ }noteUserInput: () => {value: true}@@ -184,12 +213,31 @@ export interface mcpEarlyDataLogicActions {// Generated by kea-typegen. Update if you're an agent, ignore if you're human.export interface mcpEarlyDataLogicMeta {+ sharedListeners: {+ captureNextStepsShown: (+ payload: any,+ breakpoint: BreakPointFunction,+ action: {+ type: string+ payload: any+ },+ previousState: any+ ) => void | Promise<void>+ }__keaTypeGenInternalSelectorTypes: {stats: (overview: MCPActivityOverviewApi | null) => EarlyStatstopTools: (overview: MCPActivityOverviewApi | null) => EarlyToolRow[]summary: (signals: MCPOnboardingSignals | null, stats: EarlyStats, topTools: EarlyToolRow[]) => ActivitySummary- hasFailureNotification: (notificationsResolved: boolean, notifications: HogFunctionType[]) => boolean | null- nextSteps: (stats: EarlyStats, hasFailureNotification: boolean | null) => NextStep[]+ hasFailureNotification: (+ notificationsLoad: NotificationsLoad,+ notifications: HogFunctionType[]+ ) => boolean | null+ nextSteps: (+ overview: MCPActivityOverviewApi | null,+ stats: EarlyStats,+ notificationsLoad: NotificationsLoad,+ hasFailureNotification: boolean | null+ ) => NextStep[] | nullhasModelData: (modelRows: ModelRow[]) => booleanrefreshDelayMs: (refreshesWithoutInput: number) => number}@@ -223,7 +271,7 @@ export const mcpEarlyDataLogic = kea<mcpEarlyDataLogicType>([mcpAnalyticsFiltersLogic,['hydrateFilters', 'setFilterTestAccounts', 'setPropertyFilters'],mcpAnalyticsNotificationsLogic,- ['loadNotifications', 'loadNotificationsSuccess'],+ ['loadNotifications', 'loadNotificationsSuccess', 'loadNotificationsFailure'],dataNodeCollectionLogic({ key: MCP_ACTIVITY_DATA_COLLECTION_ID }),['reloadAll as reloadActivityQuery'],],@@ -237,11 +285,17 @@ export const mcpEarlyDataLogic = kea<mcpEarlyDataLogicType>([scheduleRefresh: true,setActivityQuery: (query: DataTableNode) => ({ query }),showFailedCallsOnly: true,+ nextStepClicked: (stepKey: NextStepKey, interaction: NextStepInteraction) => ({ stepKey, interaction }),}),reducers({activityQuery: [DEFAULT_MCP_ACTIVITY_QUERY, { setActivityQuery: (_, { query }) => query }],// The list defaults to empty before its first load, which would read as "no alert set up".- notificationsResolved: [false, { loadNotificationsSuccess: () => true }],+ // A failed load also releases the card, so it never waits forever on a list that will not arrive,+ // but leaves the alert question unanswered rather than reading the empty default as "no alert".+ notificationsLoad: [+ 'pending' as NotificationsLoad,+ { loadNotificationsSuccess: () => 'loaded', loadNotificationsFailure: () => 'failed' },+ ],inputSinceRefresh: [false, { noteUserInput: () => true, refreshOnSchedule: () => false }],refreshesWithoutInput: [0,@@ -335,14 +389,20 @@ export const mcpEarlyDataLogic = kea<mcpEarlyDataLogicType>([],// A paused alert notifies nobody, so only enabled ones count as "set up".hasFailureNotification: [- (s) => [s.notificationsResolved, s.notifications],- (resolved: boolean, notifications: HogFunctionType[]): boolean | null =>- resolved ? notifications.some((notification) => notification.enabled) : null,+ (s) => [s.notificationsLoad, s.notifications],+ (notificationsLoad: NotificationsLoad, notifications: HogFunctionType[]): boolean | null =>+ notificationsLoad === 'loaded' ? notifications.some((notification) => notification.enabled) : null,],+ // Null until the overview and the notification list both resolve, so steps never reorder under the reader.nextSteps: [- (s) => [s.stats, s.hasFailureNotification],- (stats: EarlyStats, hasFailureNotification: boolean | null): NextStep[] =>- buildNextSteps(stats, hasFailureNotification),+ (s) => [s.overview, s.stats, s.notificationsLoad, s.hasFailureNotification],+ (+ overview: MCPActivityOverviewApi | null,+ stats: EarlyStats,+ notificationsLoad: NotificationsLoad,+ hasFailureNotification: boolean | null+ ): NextStep[] | null =>+ overview && notificationsLoad !== 'pending' ? buildNextSteps(stats, hasFailureNotification) : null,],hasModelData: [(s) => [s.modelRows],@@ -354,7 +414,37 @@ export const mcpEarlyDataLogic = kea<mcpEarlyDataLogicType>([Math.min(REFRESH_INTERVAL_MS * 2 ** refreshesWithoutInput, MAX_REFRESH_INTERVAL_MS),],}),- listeners(({ actions, values, cache }) => ({+ sharedListeners(({ cache, values }) => ({+ // Bound to each action that resolves an input of `nextSteps` (reacting-to-changes.md prefers+ // sharedListeners to kea-subscriptions). Compares step keys: the 60s refresh rebuilds the same steps.+ captureNextStepsShown: () => {+ const stepKeys = (values.nextSteps ?? []).map((step) => step.key)+ const shown = stepKeys.join()+ if (shown === cache.shownStepKeys) {+ return+ }+ // An empty card is remembered too, so steps coming back after none are shown again.+ cache.shownStepKeys = shown+ if (stepKeys.length > 0) {+ posthog.captureRaw('mcp analytics next steps shown', {+ step_keys: stepKeys,+ total_calls: values.stats.totalCalls,+ error_calls: values.stats.errorCalls,+ })+ }+ },+ })),+ listeners(({ actions, values, cache, sharedListeners }) => ({+ loadOverviewSuccess: sharedListeners.captureNextStepsShown,+ loadNotificationsSuccess: sharedListeners.captureNextStepsShown,+ loadNotificationsFailure: sharedListeners.captureNextStepsShown,+ nextStepClicked: ({ stepKey, interaction }) => {+ posthog.captureRaw('mcp analytics next step clicked', {+ step_key: stepKey,+ interaction,+ position: values.nextSteps?.findIndex((step) => step.key === stepKey) ?? -1,+ })+ },scheduleRefresh: () => {cache.disposables.add(() => {const id = window.setTimeout(diff --git a/products/mcp_analytics/frontend/earlyData/nextSteps.ts b/products/mcp_analytics/frontend/earlyData/nextSteps.tsindex 70225f462974..c3ce7eb389e5 100644--- a/products/mcp_analytics/frontend/earlyData/nextSteps.ts+++ b/products/mcp_analytics/frontend/earlyData/nextSteps.ts@@ -15,8 +15,12 @@ export type NextStepAction =/** A self-contained instruction the user pastes into their coding agent, with the docs as the fallback. */| { kind: 'agent-prompt'; prompt: string; docsUrl: string }+export type NextStepKey = 'intent' | 'notification' | 'sessions'++export type NextStepInteraction = 'open' | 'docs'+export interface NextStep {- key: 'intent' | 'notification' | 'sessions'+ key: NextStepKeytitle: stringdetail: stringaction: NextStepAction@@ -62,7 +66,7 @@ export function buildNextSteps(stats: EarlyStats, hasFailureNotification: booleasteps.push({key: 'notification',title: 'Get alerted when a tool call fails',- detail: `${stats.errorCalls} tool call${stats.errorCalls === 1 ? '' : 's'} failed in the last 30 days. Send new failures to Slack, email, or a webhook.`,+ detail: `${stats.errorCalls} tool call${stats.errorCalls === 1 ? '' : 's'} failed in the last 30 days. Send new failures to Slack, Discord, Microsoft Teams, or a webhook.`,action: { kind: 'link', label: 'Set up an alert', to: urls.mcpAnalyticsNotifications() },})}