:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(posthog-ai): track context picker use and context source per turn (#130210)
Adam Leith1b081c52678f
diff --git a/frontend/src/scenes/max/maxContextLogic.ts b/frontend/src/scenes/max/maxContextLogic.tsindex cf966c6cfa39..833ea110ed5b 100644--- a/frontend/src/scenes/max/maxContextLogic.ts+++ b/frontend/src/scenes/max/maxContextLogic.ts@@ -69,6 +69,15 @@ const addOrUpdateEntity = <TContext extends EntityWithIdAndType>(state: TContextconst removeEntity = <TContext extends EntityWithIdAndType>(state: TContext[], id: string | number): TContext[] =>state.filter((item) => item.id !== id)+const PICKER_GROUP_CONTEXT_TYPES: Partial<Record<TaxonomicFilterGroupType, MaxContextType>> = {+ [TaxonomicFilterGroupType.Events]: MaxContextType.EVENT,+ [TaxonomicFilterGroupType.Actions]: MaxContextType.ACTION,+ [TaxonomicFilterGroupType.Insights]: MaxContextType.INSIGHT,+ [TaxonomicFilterGroupType.Dashboards]: MaxContextType.DASHBOARD,+ [TaxonomicFilterGroupType.Notebooks]: MaxContextType.NOTEBOOK,+ [TaxonomicFilterGroupType.ErrorTrackingIssues]: MaxContextType.ERROR_TRACKING_ISSUE,+}+// A throw while building scene context blanks out ALL of PostHog AI's auto-collected context// (e.g. PostHog AI reporting it has no dashboard and falling back to search). The selector that// catches it runs on every read, so dedupe by message to avoid spamming when it keeps@@ -434,8 +443,11 @@ export const maxContextLogic = kea<maxContextLogicType>([},],}),- listeners(({ actions, cache }) => ({+ listeners(({ actions, cache, values }) => ({openContextPicker: () => {+ posthog.capture('max context picker opened', {+ has_page_context: values.contextOptions.length > 0,+ })dashboardsModel.mount()dashboardsModel.actions.loadDashboardsIfNeeded()},@@ -573,6 +585,14 @@ export const maxContextLogic = kea<maxContextLogicType>([groupType: TaxonomicFilterGroupTypeitem: TaxonomicItem}) => {+ posthog.capture('max context added', {+ group_type: groupType,+ context_type:+ groupType === TaxonomicFilterGroupType.MaxAIContext+ ? (item as MaxContextTaxonomicFilterOption).type+ : PICKER_GROUP_CONTEXT_TYPES[groupType],+ from_page_tab: groupType === TaxonomicFilterGroupType.MaxAIContext,+ })try {if (groupType === TaxonomicFilterGroupType.Events) {actions.addOrUpdateContextEvent(item as EventDefinition)diff --git a/frontend/src/scenes/max/maxThreadLogic.test.ts b/frontend/src/scenes/max/maxThreadLogic.test.tsindex 1cbba2080f5b..ffc4c1e3c243 100644--- a/frontend/src/scenes/max/maxThreadLogic.test.ts+++ b/frontend/src/scenes/max/maxThreadLogic.test.ts@@ -52,7 +52,7 @@ import { maxContextLogic } from './maxContextLogic'import { maxGlobalLogic } from './maxGlobalLogic'import { maxLogic } from './maxLogic'import { MAX_DASHBOARD_CONTEXT_WAIT_MS, maxThreadLogic } from './maxThreadLogic'-import { MaxContextType } from './maxTypes'+import { MaxContextType, MaxInsightContext } from './maxTypes'import {MOCK_CONVERSATION,MOCK_CONVERSATION_ID,@@ -1421,6 +1421,31 @@ describe('maxThreadLogic', () => {expect.objectContaining({ status: 'failure', error_status_code: 400, prompt_length: 5 }))})++ it('keeps the context source of the send on a reconnect, which sends no ui_context', async () => {+ const captureSpy = jest.spyOn(posthog, 'capture').mockImplementation(() => undefined as any)+ jest.spyOn(api.conversations, 'stream').mockRejectedValue(new ApiError('error', 500))++ logic.unmount()+ maxLogicInstance.actions.setConversationId(MOCK_TEMP_CONVERSATION_ID)+ logic = maxThreadLogic({ conversationId: MOCK_TEMP_CONVERSATION_ID, panelId: 'test' })+ logic.mount()++ await expectLogic(logic, () => {+ logic.actions.askMax('hello', true, { insights: [{ id: 'abc' } as MaxInsightContext] })+ }).toDispatchActions(['askMax', 'completeThreadGeneration'])+ await expectLogic(logic, () => {+ logic.actions.streamConversation(+ { conversation: MOCK_TEMP_CONVERSATION_ID, content: null, agent_mode: null },+ 0+ )+ }).toDispatchActions(['streamConversation', 'completeThreadGeneration'])++ const turnSources = captureSpy.mock.calls+ .filter(([event]) => event === 'max conversation turn completed')+ .map(([, properties]) => properties?.context_source)+ expect(turnSources).toEqual(['added', 'added'])+ })})describe('error tracking capture gating', () => {diff --git a/frontend/src/scenes/max/maxThreadLogic.tsx b/frontend/src/scenes/max/maxThreadLogic.tsxindex a881dfb72126..003d9fb34e2e 100644--- a/frontend/src/scenes/max/maxThreadLogic.tsx+++ b/frontend/src/scenes/max/maxThreadLogic.tsx@@ -96,7 +96,7 @@ import { PENDING_AI_PROMPT_KEY } from './max-storage-keys'import { MaxBillingContext, maxBillingContextLogic } from './maxBillingContextLogic'import { maxGlobalLogic } from './maxGlobalLogic'import { SCENE_PANEL_ID, maxLogic } from './maxLogic'-import { AttachedContext, MaxUIContext } from './maxTypes'+import { AttachedContext, MaxContextItem, MaxUIContext } from './maxTypes'import { posthogAiContextLogic } from './posthogAiContextLogic'import { MAX_SLASH_COMMANDS, SlashCommand, SlashCommandName } from './slash-commands'import { canCreateSupportTicket, isTicketCommand } from './ticketUtils'@@ -105,6 +105,7 @@ import {activeSceneLogicHasMaxContext,findPendingClientToolCall,getAgentModeForScene,+ getContextSourceProperties,isAssistantMessage,isAssistantToolCallMessage,isHumanMessage,@@ -167,6 +168,7 @@ export interface maxThreadLogicValues {featureFlags: FeatureFlagsSet // featureFlagLogicbillingContext: MaxBillingContext | null // maxBillingContextLogiccompiledContext: MaxUIContext | null // maxContextLogic+ sceneContext: MaxContextItem[] // maxContextLogicavailableStaticTools: ToolRegistration[] // maxGlobalLogicdataProcessingAccepted: boolean // maxGlobalLogictoolMap: {@@ -783,7 +785,7 @@ export const maxThreadLogic = kea<maxThreadLogicType>(['pendingBindTaskId',],maxContextLogic,- ['compiledContext'],+ ['compiledContext', 'sceneContext'],maxBillingContextLogic,['billingContext'],billingLogic,@@ -1287,6 +1289,16 @@ export const maxThreadLogic = kea<maxThreadLogicType>([// otherwise invisible outside session recordings, because no send event records a length.// Null content means resume/continue rather than a new message.const promptLength = typeof streamData.content === 'string' ? messageLength(streamData.content) : undefined+ // Snapshot on the user's send only. A retry may run after the user navigated away, and a+ // reconnect or resume sends no `ui_context`, yet both continue the same turn.+ if (generationAttempt === 0 && streamData.content !== null) {+ cache.turnContextSourceProperties = getContextSourceProperties(+ streamData.ui_context,+ values.sceneContext+ )+ }+ const contextSourceProperties: ReturnType<typeof getContextSourceProperties> =+ cache.turnContextSourceProperties ?? getContextSourceProperties(undefined, [])// Sandbox runtime: route the message to a non-streaming products/tasks Run, then hand the// SSE connection off to runStreamLogic. The LangGraph EventSource loop below is never@@ -1528,6 +1540,7 @@ export const maxThreadLogic = kea<maxThreadLogicType>([agent_mode: agentMode,generation_attempt: generationAttempt,prompt_length: promptLength,+ ...contextSourceProperties,})}@@ -1643,6 +1656,7 @@ export const maxThreadLogic = kea<maxThreadLogicType>([? 'api_error': 'unknown_error',prompt_length: promptLength,+ ...contextSourceProperties,})// Remove streaming messages and reload from server (source of truth)actions.finalizeStreamingMessages()@@ -1668,6 +1682,7 @@ export const maxThreadLogic = kea<maxThreadLogicType>([agent_mode: agentMode,generation_attempt: generationAttempt,prompt_length: promptLength,+ ...contextSourceProperties,})}actions.completeThreadGeneration()diff --git a/frontend/src/scenes/max/utils.test.ts b/frontend/src/scenes/max/utils.test.tsindex 48a2637ca33f..1e8b4e386bdc 100644--- a/frontend/src/scenes/max/utils.test.ts+++ b/frontend/src/scenes/max/utils.test.ts@@ -10,11 +10,13 @@ import { DataVisualizationNode, NodeKind } from '~/queries/schema/schema-generalimport { DashboardType, InsightShortId, InsightModel } from '~/types'import { EnhancedToolCall } from './max-constants'+import { MaxContextItem, MaxContextType, MaxUIContext } from './maxTypes'import {activeSceneLogicHasMaxContext,dashboardToMaxContext,insightToMaxContext,findPendingClientToolCall,+ getContextSourceProperties,isMultiQuestionFormMessage,threadEndsWithMultiQuestionForm,} from './utils'@@ -361,4 +363,32 @@ describe('max/utils', () => {expect(activeSceneLogicHasMaxContext(logic)).toBe(expected)})})++ describe('getContextSourceProperties()', () => {+ const sceneContext = [+ { type: MaxContextType.DASHBOARD, id: 1 },+ { type: MaxContextType.EVENT, id: 'pageview' },+ ] as MaxContextItem[]++ it.each([+ ['no context', undefined, { context_source: 'none', page_context_count: 0, added_context_count: 0 }],+ [+ 'only items from the page',+ { dashboards: [{ id: 1 }], events: [{ id: 'pageview' }] },+ { context_source: 'page', page_context_count: 2, added_context_count: 0 },+ ],+ [+ 'only items the user added',+ { insights: [{ id: 'abc' }], events: [{ id: 'signup' }] },+ { context_source: 'added', page_context_count: 0, added_context_count: 2 },+ ],+ [+ 'page and added items, with an id shared across types',+ { dashboards: [{ id: 1 }], actions: [{ id: 1 }] },+ { context_source: 'mixed', page_context_count: 1, added_context_count: 1 },+ ],+ ])('classifies %s', (_, uiContext, expected) => {+ expect(getContextSourceProperties(uiContext as MaxUIContext | undefined, sceneContext)).toEqual(expected)+ })+ })})diff --git a/frontend/src/scenes/max/utils.ts b/frontend/src/scenes/max/utils.tsindex 46a5f0d9203c..3631c9ca7b05 100644--- a/frontend/src/scenes/max/utils.ts+++ b/frontend/src/scenes/max/utils.ts@@ -44,6 +44,7 @@ import { SuggestionGroup } from './maxLogic'import {EvaluationRuntime,MaxActionContext,+ MaxContextItem,MaxContextType,MaxContextDashboard,MaxDashboardContext,@@ -406,3 +407,40 @@ export function activeSceneLogicHasMaxContext(): activeSceneLogic is BuiltLogic {return !!activeSceneLogic?.isMounted() && 'maxContext' in activeSceneLogic.selectors}++const UI_CONTEXT_FIELD_TYPES = {+ dashboards: MaxContextType.DASHBOARD,+ insights: MaxContextType.INSIGHT,+ events: MaxContextType.EVENT,+ actions: MaxContextType.ACTION,+ error_tracking_issues: MaxContextType.ERROR_TRACKING_ISSUE,+ evaluations: MaxContextType.EVALUATION,+ notebooks: MaxContextType.NOTEBOOK,+} as const satisfies Partial<Record<keyof MaxUIContext, MaxContextType>>++export type ContextSource = 'none' | 'page' | 'added' | 'mixed'++/**+ * Splits the context sent with a message into items the open page attached on its own and items+ * the user added (through the picker or an "ask AI" entry point), for turn telemetry.+ */+export function getContextSourceProperties(+ uiContext: MaxUIContext | null | undefined,+ sceneContext: MaxContextItem[]+): { context_source: ContextSource; page_context_count: number; added_context_count: number } {+ const sceneKeys = new Set(sceneContext.map((item) => `${item.type}:${item.id}`))+ let pageCount = 0+ let addedCount = 0+ for (const [field, type] of Object.entries(UI_CONTEXT_FIELD_TYPES)) {+ for (const item of uiContext?.[field as keyof typeof UI_CONTEXT_FIELD_TYPES] ?? []) {+ if (sceneKeys.has(`${type}:${item.id}`)) {+ pageCount+++ } else {+ addedCount+++ }+ }+ }+ const contextSource: ContextSource =+ pageCount > 0 && addedCount > 0 ? 'mixed' : pageCount > 0 ? 'page' : addedCount > 0 ? 'added' : 'none'+ return { context_source: contextSource, page_context_count: pageCount, added_context_count: addedCount }+}