:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-vision): track scanner entry points and ai draft edits (#129443)
Kim Svatos Dugan189bcc572cc0
diff --git a/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.test.ts b/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.test.tsindex 54751eb8b10f..5aba09c98f6e 100644--- a/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.test.ts+++ b/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.test.ts@@ -252,7 +252,9 @@ describe('createExperimentLogic', () => {button.action()// The retry must reopen the experiment scanner wizard on this experiment, not the generic// template list, or the person starts over without the experiment.- expect(routerPushSpy).toHaveBeenLastCalledWith('/replay-vision/new/details?experiment=123')+ expect(routerPushSpy).toHaveBeenLastCalledWith(+ '/replay-vision/new/details?experiment=123&entry_point=experiment_create_wizard_retry'+ )expect(captureExceptionSpy).toHaveBeenCalledTimes(shouldCapture ? 1 : 0)captureExceptionSpy.mockRestore()})diff --git a/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.ts b/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.tsindex fe17d6f0d124..8255ff2d0b94 100644--- a/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.ts+++ b/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.ts@@ -538,13 +538,13 @@ export const createExperimentLogic = kea<createExperimentLogicType>([label: 'Set up experiment scanner',action: () =>router.actions.push(- combineUrl(- urls.replayVisionScannerDetails('new'),- experimentScannerParams({+ combineUrl(urls.replayVisionScannerDetails('new'), {+ ...experimentScannerParams({experimentId: response.id as number,variantKey: null,- })- ).url+ }),+ entry_point: 'experiment_create_wizard_retry',+ }).url),},})diff --git a/frontend/src/scenes/experiments/ExperimentView/ExperimentScannerEntryPoint.tsx b/frontend/src/scenes/experiments/ExperimentView/ExperimentScannerEntryPoint.tsxindex b07b8d29a010..31cdfd04765f 100644--- a/frontend/src/scenes/experiments/ExperimentView/ExperimentScannerEntryPoint.tsx+++ b/frontend/src/scenes/experiments/ExperimentView/ExperimentScannerEntryPoint.tsx@@ -183,7 +183,11 @@ export function ExperimentScannerEntryPoint({// Straight into the wizard: the experiment fixes the type, so there is no template to pick.const setUpUrl = combineUrl(urls.replayVisionScannerDetails('new'),- experimentScannerParams({ experimentId: experiment.id as number, variantKey })+ // Pinned: the editor reports this `entry_point` on replay_vision_scanner_created.+ {+ ...experimentScannerParams({ experimentId: experiment.id as number, variantKey }),+ entry_point: 'experiment_recordings_tab',+ }).urlif (linkedScannersLoading) {diff --git a/frontend/src/scenes/experiments/replayVisionScanner.ts b/frontend/src/scenes/experiments/replayVisionScanner.tsindex d4194d8c5d52..fde986fb3140 100644--- a/frontend/src/scenes/experiments/replayVisionScanner.ts+++ b/frontend/src/scenes/experiments/replayVisionScanner.ts@@ -39,5 +39,7 @@ export function experimentScannerBody(experiment: Experiment): ReplayScannerApimodel: DEFAULT_MODEL,query: experimentScannerQuery(experiment),enabled: false,+ // Pinned: reported on replay_vision_scanner_created to attribute scanners to this wizard.+ entry_point: 'experiment_create_wizard',})}diff --git a/products/replay_vision/frontend/replay_scanners/draftChanges.ts b/products/replay_vision/frontend/replay_scanners/draftChanges.tsnew file mode 100644index 000000000000..43e58d472f7a--- /dev/null+++ b/products/replay_vision/frontend/replay_scanners/draftChanges.ts@@ -0,0 +1,43 @@+import { objectsEqual } from 'lib/utils/objects'++import type { ScannerFormValues } from './types'++export type DraftChangedField =+ | 'name'+ | 'description'+ | 'prompt'+ | 'scanner_type'+ | 'type_config'+ | 'query'+ | 'sampling'+ | 'model'+ | 'budget'++function configWithoutPrompt(scanner: ScannerFormValues): object {+ if (!scanner.scanner_config) {+ return {}+ }+ const { prompt: _prompt, ...rest } = scanner.scanner_config+ return rest+}++/** Which parts of the drafted form the person changed before saving. Empty when they saved the draft as drafted. */+export function draftChanges(drafted: ScannerFormValues, saved: ScannerFormValues): DraftChangedField[] {+ const changed: Record<DraftChangedField, boolean> = {+ name: (drafted.name ?? '').trim() !== (saved.name ?? '').trim(),+ description: (drafted.description ?? '').trim() !== (saved.description ?? '').trim(),+ prompt: (drafted.scanner_config?.prompt ?? '').trim() !== (saved.scanner_config?.prompt ?? '').trim(),+ scanner_type: drafted.scanner_type !== saved.scanner_type,+ type_config: !objectsEqual(configWithoutPrompt(drafted), configWithoutPrompt(saved)),+ query:+ !objectsEqual(drafted.query ?? null, saved.query ?? null) ||+ !objectsEqual(drafted.experiment_targeting ?? null, saved.experiment_targeting ?? null),+ sampling: drafted.sampling_mode !== saved.sampling_mode || drafted.sampling_rate !== saved.sampling_rate,+ model: drafted.model !== saved.model,+ budget:+ !!drafted.free_only !== !!saved.free_only ||+ !!drafted.credit_limit_enabled !== !!saved.credit_limit_enabled ||+ (drafted.credit_limit ?? null) !== (saved.credit_limit ?? null),+ }+ return (Object.keys(changed) as DraftChangedField[]).filter((field) => changed[field])+}diff --git a/products/replay_vision/frontend/replay_scanners/highlightPicksStarterLogic.ts b/products/replay_vision/frontend/replay_scanners/highlightPicksStarterLogic.tsindex e66458301bda..ad8433be4adc 100644--- a/products/replay_vision/frontend/replay_scanners/highlightPicksStarterLogic.ts+++ b/products/replay_vision/frontend/replay_scanners/highlightPicksStarterLogic.ts@@ -383,6 +383,7 @@ export const highlightPicksStarterLogic: LogicWrapper<highlightPicksStarterLogicfree_only: free,credit_limit: free ? FREE_SCANNER_CREDIT_LIMIT : values.budget,creation_method: 'template',+ entry_point: 'highlight_picks_starter',}))} catch {diff --git a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.ts b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.tsindex 782d09f3d411..9c8b341274be 100644--- a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.ts+++ b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.ts@@ -2050,6 +2050,97 @@ describe('replayScannerLogic', () => {logic.actions.startFromScratch()expect(logic.values.creationMethod).toEqual('scratch')})++ it.each([+ ['saved as drafted', {}, false, []],+ [+ 'saved with an edited prompt and model',+ {+ scanner_config: {+ prompt: 'Did the visitor browse or buy?',+ tags: ['browsing'],+ multi_label: false,+ },+ model: ScannerModelEnumApi.Gemini38Flash,+ },+ true,+ ['prompt', 'model'],+ ],+ [+ 'saved with a changed experiment variant',+ { experiment_targeting: { experiment_id: 7, variant: 'test' } },+ true,+ ['query'],+ ],+ ])('an AI draft %s reports what the person changed before create', async (_, edits, edited, changes) => {+ draftSpy.mockReturnValue([+ 200,+ {+ name: 'User intent',+ description: 'Tags each session by intent.',+ scanner_type: 'classifier',+ scanner_config: {+ prompt: 'Classify the session by intent.',+ tags: ['browsing'],+ multi_label: false,+ },+ },+ ])+ router.actions.push(urls.replayVisionScannerGoal('new'))+ await expectLogic(logic, () =>+ logic.actions.draftScannerFromGoal({+ goal: 'why do people visit',+ spend: { monthlyCreditBudget: 2500 },+ })+ ).toFinishAllListeners()+ logic.actions.setScannerValues(edits)+ scannerEditorSceneLogic.actions.setStep('overview')+ const captureSpy = jest.spyOn(posthog, 'capture')++ await expectLogic(logic, () => logic.actions.submitScanner()).toFinishAllListeners()++ expect(createSpy).toHaveBeenCalledTimes(1)+ expect(captureSpy).toHaveBeenCalledWith('replay_vision_scanner_draft_saved', {+ scanner_id: 'created-scanner',+ draft_edited: edited,+ draft_changes: changes,+ drafted_scanner_type: 'classifier',+ scanner_type: 'classifier',+ entry_point: null,+ })+ })++ it('an AI draft resumed after a reload still reports its changes', async () => {+ draftSpy.mockReturnValue([+ 200,+ {+ name: 'User intent',+ scanner_type: 'monitor',+ scanner_config: { prompt: 'Did the visitor buy?' },+ },+ ])+ router.actions.push(urls.replayVisionScannerGoal('new'))+ await expectLogic(logic, () =>+ logic.actions.draftScannerFromGoal({+ goal: 'why do people visit',+ spend: { monthlyCreditBudget: 2500 },+ })+ ).toFinishAllListeners()+ logic.actions.setScannerValues({ scanner_config: { prompt: 'Did the visitor buy twice?' } })+ logic.unmount()+ logic = replayScannerLogic({ id: 'new' })+ logic.mount()+ await expectLogic(logic, () => logic.actions.loadScanner()).toFinishAllListeners()+ scannerEditorSceneLogic.actions.setStep('overview')+ const captureSpy = jest.spyOn(posthog, 'capture')++ await expectLogic(logic, () => logic.actions.submitScanner()).toFinishAllListeners()++ expect(captureSpy).toHaveBeenCalledWith(+ 'replay_vision_scanner_draft_saved',+ expect.objectContaining({ draft_edited: true, draft_changes: ['prompt'] })+ )+ })})describe('rebuildExperimentContext', () => {diff --git a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.ts b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.tsindex 8be891e0c1bf..a48b3cf4fdda 100644--- a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.ts+++ b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.ts@@ -70,6 +70,7 @@ import {parseSortParam,serializeSortParam,} from '../utils/urlParams'+import { draftChanges } from './draftChanges'import { clampDurationFilter, durationFilterError } from './durationBounds'import {ExperimentScannerContext,@@ -548,6 +549,7 @@ export interface replayScannerLogicValues {experimentContextLinked: booleangoalBudgetInput: number | nullgoalDraft: DraftScannerResponseApi | null+ goalDraftBaseline: ScannerFormValues | nullgoalDraftInput: stringgoalDraftLoading: booleangoalDraftQuery: RecordingsQuery | null@@ -797,6 +799,9 @@ export interface replayScannerLogicActions {dateFrom: string | nulldateTo: string | null}+ setCreationMethod: (creationMethod: ScannerCreationMethodEnumApi) => {+ creationMethod: ScannerCreationMethodEnumApi+ }setCreditLimitUnit: (unit: CreditLimitUnit) => {unit: CreditLimitUnit}@@ -812,6 +817,9 @@ export interface replayScannerLogicActions {setGoalBudgetInput: (budget: number | null) => {budget: number | null}+ setGoalDraftBaseline: (baseline: ScannerFormValues | null) => {+ baseline: ScannerFormValues | null+ }setGoalDraftInput: (goal: string) => {goal: string}@@ -1073,6 +1081,8 @@ export const replayScannerLogic = kea<replayScannerLogicType>([setGoalSpendMode: (mode: GoalSpendMode) => ({ mode }),setGoalDraftQuery: (query: RecordingsQuery | null) => ({ query }),setEntryPoint: (entryPoint: string | null) => ({ entryPoint }),+ setGoalDraftBaseline: (baseline: ScannerFormValues | null) => ({ baseline }),+ setCreationMethod: (creationMethod: ScannerCreationMethodEnumApi) => ({ creationMethod }),loadObservations: (background = false) => ({ background }),loadObservationsSuccess: (observations: ReplayObservationApi[], total: number) => ({ observations, total }),loadObservationsFailure: true,@@ -1228,6 +1238,20 @@ export const replayScannerLogic = kea<replayScannerLogicType>([entry_point: values.entryPoint,}))+ if (values.creationMethod === 'ai') {+ // Null changes mean the drafted form is unknown, not that it was saved unchanged.+ const changes = values.goalDraftBaseline+ ? draftChanges(values.goalDraftBaseline, scanner)+ : null+ posthog.capture('replay_vision_scanner_draft_saved', {+ scanner_id: response.id,+ draft_edited: changes ? changes.length > 0 : null,+ draft_changes: changes,+ drafted_scanner_type: values.goalDraftBaseline?.scanner_type ?? null,+ scanner_type: scanner.scanner_type,+ entry_point: values.entryPoint,+ })+ }actions.scannerSaved(scanner)leaveScannerEditor(props.id, urls.replayVision(response.id))// First scheduled results are minutes away, so the copy matches the Overview's@@ -1408,6 +1432,17 @@ export const replayScannerLogic = kea<replayScannerLogicType>([startFromScratch: () => null,},],+ // The form as the AI draft filled it, so the save can report what the person changed on review.+ goalDraftBaseline: [+ null as ScannerFormValues | null,+ {+ setGoalDraftBaseline: (_, { baseline }) => baseline,+ draftScannerFromGoal: () => null,+ startFromScratch: () => null,+ discardScannerDraft: () => null,+ scannerSaved: () => null,+ },+ ],// How the form standing in front of the user was built, sent with the create request so the// saved scanner can be attributed. Last path taken wins, because each of these replaces the// form: someone who drafts with AI and then starts from scratch saved a hand-built scanner.@@ -1416,6 +1451,7 @@ export const replayScannerLogic = kea<replayScannerLogicType>([{draftScannerFromGoal: () => 'ai' as ScannerCreationMethodEnumApi,startFromScratch: () => 'scratch' as ScannerCreationMethodEnumApi,+ setCreationMethod: (_, { creationMethod }) => creationMethod,},],savingCohortKey: [@@ -2012,7 +2048,8 @@ export const replayScannerLogic = kea<replayScannerLogicType>([const savedAt = writeScannerDraft(teamId,values.scanner,- typeof linkedExperimentId === 'number' ? linkedExperimentId : null+ typeof linkedExperimentId === 'number' ? linkedExperimentId : null,+ { creationMethod: values.creationMethod, goalDraftBaseline: values.goalDraftBaseline })if (savedAt === null) {// A failed write leaves any older draft behind; drop it so it can't resurrect stale edits.@@ -2195,6 +2232,10 @@ export const replayScannerLogic = kea<replayScannerLogicType>([if (draft) {actions.setScannerValues(draft.scanner)actions.setScannerDraftSavedAt(draft.savedAt)+ if (draft.provenance) {+ actions.setCreationMethod(draft.provenance.creationMethod)+ actions.setGoalDraftBaseline(draft.provenance.goalDraftBaseline)+ }// A draft made from an experiment prefill carries targeting the// loadScannerSuccess above (a bare newScanner) didn't see. A draft started// from an experiment link restores that link, so the wizard stays locked.@@ -2466,6 +2507,7 @@ export const replayScannerLogic = kea<replayScannerLogicType>([actions.rebuildExperimentContext()// The form now carries the drafted filter, so count what it will actually watch.actions.loadScannerEstimate()+ actions.setGoalDraftBaseline(values.scanner)},draftScannerFromGoalFailure: ({ errorObject }) => {@@ -2500,6 +2542,9 @@ export const replayScannerLogic = kea<replayScannerLogicType>([actions.requestScannerEstimate()persistDraft()},+ setGoalDraftBaseline: () => {+ persistDraft()+ },startFromScratch: () => {// Counterpart of the AI capture, so the creation funnel can split by path.posthog.capture('replay_vision_scanner_creation_started', {diff --git a/products/replay_vision/frontend/replay_scanners/scannerDraft.ts b/products/replay_vision/frontend/replay_scanners/scannerDraft.tsindex bde4c5d3804d..e8213e143ed2 100644--- a/products/replay_vision/frontend/replay_scanners/scannerDraft.ts+++ b/products/replay_vision/frontend/replay_scanners/scannerDraft.ts@@ -1,3 +1,4 @@+import { ScannerCreationMethodEnumApi } from '../generated/api.schemas'import { ScannerFormValues } from './types'const DRAFT_STORAGE_KEY = 'replay-vision.new-scanner-draft'@@ -11,18 +12,41 @@ interface StoredScannerDraft {scanner: ScannerFormValues// The experiment the wizard was entered from. Optional, so drafts saved without it still read.linkedExperimentId?: number | null+ provenance?: ScannerDraftProvenance+}++/** How the form was built, so a resumed draft still reports its creation path and its AI-drafted baseline. */+export interface ScannerDraftProvenance {+ creationMethod: ScannerCreationMethodEnumApi+ goalDraftBaseline: ScannerFormValues | null}export interface ScannerDraft {scanner: ScannerFormValuessavedAt: numberlinkedExperimentId: number | null+ provenance: ScannerDraftProvenance | null+}++function readProvenance(value: unknown): ScannerDraftProvenance | null {+ if (!value || typeof value !== 'object') {+ return null+ }+ const { creationMethod, goalDraftBaseline } = value as Partial<ScannerDraftProvenance>+ if (!Object.values(ScannerCreationMethodEnumApi).includes(creationMethod as ScannerCreationMethodEnumApi)) {+ return null+ }+ return {+ creationMethod: creationMethod as ScannerCreationMethodEnumApi,+ goalDraftBaseline: goalDraftBaseline && typeof goalDraftBaseline === 'object' ? goalDraftBaseline : null,+ }}export function writeScannerDraft(teamId: number,scanner: ScannerFormValues,- linkedExperimentId: number | null = null+ linkedExperimentId: number | null = null,+ provenance: ScannerDraftProvenance | null = null): number | null {const savedAt = Date.now()const draft: StoredScannerDraft = {@@ -31,6 +55,7 @@ export function writeScannerDraft(savedAt,scanner,linkedExperimentId,+ ...(provenance ? { provenance } : {}),}try {localStorage.setItem(DRAFT_STORAGE_KEY, JSON.stringify(draft))@@ -60,6 +85,7 @@ export function readScannerDraft(teamId: number): ScannerDraft | null {scanner: draft.scanner,savedAt: draft.savedAt,linkedExperimentId: typeof draft.linkedExperimentId === 'number' ? draft.linkedExperimentId : null,+ provenance: readProvenance(draft.provenance),}} catch {return null