Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-vision): track scanner entry points and ai draft edits (#129443)

Kim Svatos Dugan189bcc572cc0
diff --git a/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.test.ts b/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.test.ts
index 54751eb8b10f..5aba09c98f6e 100644
--- a/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.test.ts
+++ b/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.test.ts
@@ -252,7 +252,9 @@ describe('createExperimentLogic', () => {
button.action()
// The retry must reopen the experiment scanner wizard on this experiment, not the generic
// template list, or the person starts over without the experiment.
- expect(routerPushSpy).toHaveBeenLastCalledWith('/replay-vision/new/details?experiment=123')
+ expect(routerPushSpy).toHaveBeenLastCalledWith(
+ '/replay-vision/new/details?experiment=123&entry_point=experiment_create_wizard_retry'
+ )
expect(captureExceptionSpy).toHaveBeenCalledTimes(shouldCapture ? 1 : 0)
captureExceptionSpy.mockRestore()
})
diff --git a/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.ts b/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.ts
index fe17d6f0d124..8255ff2d0b94 100644
--- a/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.ts
+++ b/frontend/src/scenes/experiments/ExperimentForm/createExperimentLogic.ts
@@ -538,13 +538,13 @@ export const createExperimentLogic = kea<createExperimentLogicType>([
label: 'Set up experiment scanner',
action: () =>
router.actions.push(
- combineUrl(
- urls.replayVisionScannerDetails('new'),
- experimentScannerParams({
+ combineUrl(urls.replayVisionScannerDetails('new'), {
+ ...experimentScannerParams({
experimentId: response.id as number,
variantKey: null,
- })
- ).url
+ }),
+ entry_point: 'experiment_create_wizard_retry',
+ }).url
),
},
})
diff --git a/frontend/src/scenes/experiments/ExperimentView/ExperimentScannerEntryPoint.tsx b/frontend/src/scenes/experiments/ExperimentView/ExperimentScannerEntryPoint.tsx
index b07b8d29a010..31cdfd04765f 100644
--- a/frontend/src/scenes/experiments/ExperimentView/ExperimentScannerEntryPoint.tsx
+++ b/frontend/src/scenes/experiments/ExperimentView/ExperimentScannerEntryPoint.tsx
@@ -183,7 +183,11 @@ export function ExperimentScannerEntryPoint({
// Straight into the wizard: the experiment fixes the type, so there is no template to pick.
const setUpUrl = combineUrl(
urls.replayVisionScannerDetails('new'),
- experimentScannerParams({ experimentId: experiment.id as number, variantKey })
+ // Pinned: the editor reports this `entry_point` on replay_vision_scanner_created.
+ {
+ ...experimentScannerParams({ experimentId: experiment.id as number, variantKey }),
+ entry_point: 'experiment_recordings_tab',
+ }
).url
if (linkedScannersLoading) {
diff --git a/frontend/src/scenes/experiments/replayVisionScanner.ts b/frontend/src/scenes/experiments/replayVisionScanner.ts
index d4194d8c5d52..fde986fb3140 100644
--- a/frontend/src/scenes/experiments/replayVisionScanner.ts
+++ b/frontend/src/scenes/experiments/replayVisionScanner.ts
@@ -39,5 +39,7 @@ export function experimentScannerBody(experiment: Experiment): ReplayScannerApi
model: DEFAULT_MODEL,
query: experimentScannerQuery(experiment),
enabled: false,
+ // Pinned: reported on replay_vision_scanner_created to attribute scanners to this wizard.
+ entry_point: 'experiment_create_wizard',
})
}
diff --git a/products/replay_vision/frontend/replay_scanners/draftChanges.ts b/products/replay_vision/frontend/replay_scanners/draftChanges.ts
new file mode 100644
index 000000000000..43e58d472f7a
--- /dev/null
+++ b/products/replay_vision/frontend/replay_scanners/draftChanges.ts
@@ -0,0 +1,43 @@
+import { objectsEqual } from 'lib/utils/objects'
+
+import type { ScannerFormValues } from './types'
+
+export type DraftChangedField =
+ | 'name'
+ | 'description'
+ | 'prompt'
+ | 'scanner_type'
+ | 'type_config'
+ | 'query'
+ | 'sampling'
+ | 'model'
+ | 'budget'
+
+function configWithoutPrompt(scanner: ScannerFormValues): object {
+ if (!scanner.scanner_config) {
+ return {}
+ }
+ const { prompt: _prompt, ...rest } = scanner.scanner_config
+ return rest
+}
+
+/** Which parts of the drafted form the person changed before saving. Empty when they saved the draft as drafted. */
+export function draftChanges(drafted: ScannerFormValues, saved: ScannerFormValues): DraftChangedField[] {
+ const changed: Record<DraftChangedField, boolean> = {
+ name: (drafted.name ?? '').trim() !== (saved.name ?? '').trim(),
+ description: (drafted.description ?? '').trim() !== (saved.description ?? '').trim(),
+ prompt: (drafted.scanner_config?.prompt ?? '').trim() !== (saved.scanner_config?.prompt ?? '').trim(),
+ scanner_type: drafted.scanner_type !== saved.scanner_type,
+ type_config: !objectsEqual(configWithoutPrompt(drafted), configWithoutPrompt(saved)),
+ query:
+ !objectsEqual(drafted.query ?? null, saved.query ?? null) ||
+ !objectsEqual(drafted.experiment_targeting ?? null, saved.experiment_targeting ?? null),
+ sampling: drafted.sampling_mode !== saved.sampling_mode || drafted.sampling_rate !== saved.sampling_rate,
+ model: drafted.model !== saved.model,
+ budget:
+ !!drafted.free_only !== !!saved.free_only ||
+ !!drafted.credit_limit_enabled !== !!saved.credit_limit_enabled ||
+ (drafted.credit_limit ?? null) !== (saved.credit_limit ?? null),
+ }
+ return (Object.keys(changed) as DraftChangedField[]).filter((field) => changed[field])
+}
diff --git a/products/replay_vision/frontend/replay_scanners/highlightPicksStarterLogic.ts b/products/replay_vision/frontend/replay_scanners/highlightPicksStarterLogic.ts
index e66458301bda..ad8433be4adc 100644
--- a/products/replay_vision/frontend/replay_scanners/highlightPicksStarterLogic.ts
+++ b/products/replay_vision/frontend/replay_scanners/highlightPicksStarterLogic.ts
@@ -383,6 +383,7 @@ export const highlightPicksStarterLogic: LogicWrapper<highlightPicksStarterLogic
free_only: free,
credit_limit: free ? FREE_SCANNER_CREDIT_LIMIT : values.budget,
creation_method: 'template',
+ entry_point: 'highlight_picks_starter',
})
)
} catch {
diff --git a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.ts b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.ts
index 782d09f3d411..9c8b341274be 100644
--- a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.ts
+++ b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.test.ts
@@ -2050,6 +2050,97 @@ describe('replayScannerLogic', () => {
logic.actions.startFromScratch()
expect(logic.values.creationMethod).toEqual('scratch')
})
+
+ it.each([
+ ['saved as drafted', {}, false, []],
+ [
+ 'saved with an edited prompt and model',
+ {
+ scanner_config: {
+ prompt: 'Did the visitor browse or buy?',
+ tags: ['browsing'],
+ multi_label: false,
+ },
+ model: ScannerModelEnumApi.Gemini38Flash,
+ },
+ true,
+ ['prompt', 'model'],
+ ],
+ [
+ 'saved with a changed experiment variant',
+ { experiment_targeting: { experiment_id: 7, variant: 'test' } },
+ true,
+ ['query'],
+ ],
+ ])('an AI draft %s reports what the person changed before create', async (_, edits, edited, changes) => {
+ draftSpy.mockReturnValue([
+ 200,
+ {
+ name: 'User intent',
+ description: 'Tags each session by intent.',
+ scanner_type: 'classifier',
+ scanner_config: {
+ prompt: 'Classify the session by intent.',
+ tags: ['browsing'],
+ multi_label: false,
+ },
+ },
+ ])
+ router.actions.push(urls.replayVisionScannerGoal('new'))
+ await expectLogic(logic, () =>
+ logic.actions.draftScannerFromGoal({
+ goal: 'why do people visit',
+ spend: { monthlyCreditBudget: 2500 },
+ })
+ ).toFinishAllListeners()
+ logic.actions.setScannerValues(edits)
+ scannerEditorSceneLogic.actions.setStep('overview')
+ const captureSpy = jest.spyOn(posthog, 'capture')
+
+ await expectLogic(logic, () => logic.actions.submitScanner()).toFinishAllListeners()
+
+ expect(createSpy).toHaveBeenCalledTimes(1)
+ expect(captureSpy).toHaveBeenCalledWith('replay_vision_scanner_draft_saved', {
+ scanner_id: 'created-scanner',
+ draft_edited: edited,
+ draft_changes: changes,
+ drafted_scanner_type: 'classifier',
+ scanner_type: 'classifier',
+ entry_point: null,
+ })
+ })
+
+ it('an AI draft resumed after a reload still reports its changes', async () => {
+ draftSpy.mockReturnValue([
+ 200,
+ {
+ name: 'User intent',
+ scanner_type: 'monitor',
+ scanner_config: { prompt: 'Did the visitor buy?' },
+ },
+ ])
+ router.actions.push(urls.replayVisionScannerGoal('new'))
+ await expectLogic(logic, () =>
+ logic.actions.draftScannerFromGoal({
+ goal: 'why do people visit',
+ spend: { monthlyCreditBudget: 2500 },
+ })
+ ).toFinishAllListeners()
+ logic.actions.setScannerValues({ scanner_config: { prompt: 'Did the visitor buy twice?' } })
+ logic.unmount()
+ logic = replayScannerLogic({ id: 'new' })
+ logic.mount()
+ await expectLogic(logic, () => logic.actions.loadScanner()).toFinishAllListeners()
+ scannerEditorSceneLogic.actions.setStep('overview')
+ const captureSpy = jest.spyOn(posthog, 'capture')
+
+ await expectLogic(logic, () => logic.actions.submitScanner()).toFinishAllListeners()
+
+ expect(captureSpy).toHaveBeenCalledWith(
+ 'replay_vision_scanner_draft_saved',
+ expect.objectContaining({ draft_edited: true, draft_changes: ['prompt'] })
+ )
+ })
})
describe('rebuildExperimentContext', () => {
diff --git a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.ts b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.ts
index 8be891e0c1bf..a48b3cf4fdda 100644
--- a/products/replay_vision/frontend/replay_scanners/replayScannerLogic.ts
+++ b/products/replay_vision/frontend/replay_scanners/replayScannerLogic.ts
@@ -70,6 +70,7 @@ import {
parseSortParam,
serializeSortParam,
} from '../utils/urlParams'
+import { draftChanges } from './draftChanges'
import { clampDurationFilter, durationFilterError } from './durationBounds'
import {
ExperimentScannerContext,
@@ -548,6 +549,7 @@ export interface replayScannerLogicValues {
experimentContextLinked: boolean
goalBudgetInput: number | null
goalDraft: DraftScannerResponseApi | null
+ goalDraftBaseline: ScannerFormValues | null
goalDraftInput: string
goalDraftLoading: boolean
goalDraftQuery: RecordingsQuery | null
@@ -797,6 +799,9 @@ export interface replayScannerLogicActions {
dateFrom: string | null
dateTo: string | null
}
+ setCreationMethod: (creationMethod: ScannerCreationMethodEnumApi) => {
+ creationMethod: ScannerCreationMethodEnumApi
+ }
setCreditLimitUnit: (unit: CreditLimitUnit) => {
unit: CreditLimitUnit
}
@@ -812,6 +817,9 @@ export interface replayScannerLogicActions {
setGoalBudgetInput: (budget: number | null) => {
budget: number | null
}
+ setGoalDraftBaseline: (baseline: ScannerFormValues | null) => {
+ baseline: ScannerFormValues | null
+ }
setGoalDraftInput: (goal: string) => {
goal: string
}
@@ -1073,6 +1081,8 @@ export const replayScannerLogic = kea<replayScannerLogicType>([
setGoalSpendMode: (mode: GoalSpendMode) => ({ mode }),
setGoalDraftQuery: (query: RecordingsQuery | null) => ({ query }),
setEntryPoint: (entryPoint: string | null) => ({ entryPoint }),
+ setGoalDraftBaseline: (baseline: ScannerFormValues | null) => ({ baseline }),
+ setCreationMethod: (creationMethod: ScannerCreationMethodEnumApi) => ({ creationMethod }),
loadObservations: (background = false) => ({ background }),
loadObservationsSuccess: (observations: ReplayObservationApi[], total: number) => ({ observations, total }),
loadObservationsFailure: true,
@@ -1228,6 +1238,20 @@ export const replayScannerLogic = kea<replayScannerLogicType>([
entry_point: values.entryPoint,
})
)
+ if (values.creationMethod === 'ai') {
+ // Null changes mean the drafted form is unknown, not that it was saved unchanged.
+ const changes = values.goalDraftBaseline
+ ? draftChanges(values.goalDraftBaseline, scanner)
+ : null
+ posthog.capture('replay_vision_scanner_draft_saved', {
+ scanner_id: response.id,
+ draft_edited: changes ? changes.length > 0 : null,
+ draft_changes: changes,
+ drafted_scanner_type: values.goalDraftBaseline?.scanner_type ?? null,
+ scanner_type: scanner.scanner_type,
+ entry_point: values.entryPoint,
+ })
+ }
actions.scannerSaved(scanner)
leaveScannerEditor(props.id, urls.replayVision(response.id))
// First scheduled results are minutes away, so the copy matches the Overview's
@@ -1408,6 +1432,17 @@ export const replayScannerLogic = kea<replayScannerLogicType>([
startFromScratch: () => null,
},
],
+ // The form as the AI draft filled it, so the save can report what the person changed on review.
+ goalDraftBaseline: [
+ null as ScannerFormValues | null,
+ {
+ setGoalDraftBaseline: (_, { baseline }) => baseline,
+ draftScannerFromGoal: () => null,
+ startFromScratch: () => null,
+ discardScannerDraft: () => null,
+ scannerSaved: () => null,
+ },
+ ],
// How the form standing in front of the user was built, sent with the create request so the
// saved scanner can be attributed. Last path taken wins, because each of these replaces the
// form: someone who drafts with AI and then starts from scratch saved a hand-built scanner.
@@ -1416,6 +1451,7 @@ export const replayScannerLogic = kea<replayScannerLogicType>([
{
draftScannerFromGoal: () => 'ai' as ScannerCreationMethodEnumApi,
startFromScratch: () => 'scratch' as ScannerCreationMethodEnumApi,
+ setCreationMethod: (_, { creationMethod }) => creationMethod,
},
],
savingCohortKey: [
@@ -2012,7 +2048,8 @@ export const replayScannerLogic = kea<replayScannerLogicType>([
const savedAt = writeScannerDraft(
teamId,
values.scanner,
- typeof linkedExperimentId === 'number' ? linkedExperimentId : null
+ typeof linkedExperimentId === 'number' ? linkedExperimentId : null,
+ { creationMethod: values.creationMethod, goalDraftBaseline: values.goalDraftBaseline }
)
if (savedAt === null) {
// A failed write leaves any older draft behind; drop it so it can't resurrect stale edits.
@@ -2195,6 +2232,10 @@ export const replayScannerLogic = kea<replayScannerLogicType>([
if (draft) {
actions.setScannerValues(draft.scanner)
actions.setScannerDraftSavedAt(draft.savedAt)
+ if (draft.provenance) {
+ actions.setCreationMethod(draft.provenance.creationMethod)
+ actions.setGoalDraftBaseline(draft.provenance.goalDraftBaseline)
+ }
// A draft made from an experiment prefill carries targeting the
// loadScannerSuccess above (a bare newScanner) didn't see. A draft started
// from an experiment link restores that link, so the wizard stays locked.
@@ -2466,6 +2507,7 @@ export const replayScannerLogic = kea<replayScannerLogicType>([
actions.rebuildExperimentContext()
// The form now carries the drafted filter, so count what it will actually watch.
actions.loadScannerEstimate()
+ actions.setGoalDraftBaseline(values.scanner)
},
draftScannerFromGoalFailure: ({ errorObject }) => {
@@ -2500,6 +2542,9 @@ export const replayScannerLogic = kea<replayScannerLogicType>([
actions.requestScannerEstimate()
persistDraft()
},
+ setGoalDraftBaseline: () => {
+ persistDraft()
+ },
startFromScratch: () => {
// Counterpart of the AI capture, so the creation funnel can split by path.
posthog.capture('replay_vision_scanner_creation_started', {
diff --git a/products/replay_vision/frontend/replay_scanners/scannerDraft.ts b/products/replay_vision/frontend/replay_scanners/scannerDraft.ts
index bde4c5d3804d..e8213e143ed2 100644
--- a/products/replay_vision/frontend/replay_scanners/scannerDraft.ts
+++ b/products/replay_vision/frontend/replay_scanners/scannerDraft.ts
@@ -1,3 +1,4 @@
+import { ScannerCreationMethodEnumApi } from '../generated/api.schemas'
import { ScannerFormValues } from './types'
const DRAFT_STORAGE_KEY = 'replay-vision.new-scanner-draft'
@@ -11,18 +12,41 @@ interface StoredScannerDraft {
scanner: ScannerFormValues
// The experiment the wizard was entered from. Optional, so drafts saved without it still read.
linkedExperimentId?: number | null
+ provenance?: ScannerDraftProvenance
+}
+
+/** How the form was built, so a resumed draft still reports its creation path and its AI-drafted baseline. */
+export interface ScannerDraftProvenance {
+ creationMethod: ScannerCreationMethodEnumApi
+ goalDraftBaseline: ScannerFormValues | null
}
export interface ScannerDraft {
scanner: ScannerFormValues
savedAt: number
linkedExperimentId: number | null
+ provenance: ScannerDraftProvenance | null
+}
+
+function readProvenance(value: unknown): ScannerDraftProvenance | null {
+ if (!value || typeof value !== 'object') {
+ return null
+ }
+ const { creationMethod, goalDraftBaseline } = value as Partial<ScannerDraftProvenance>
+ if (!Object.values(ScannerCreationMethodEnumApi).includes(creationMethod as ScannerCreationMethodEnumApi)) {
+ return null
+ }
+ return {
+ creationMethod: creationMethod as ScannerCreationMethodEnumApi,
+ goalDraftBaseline: goalDraftBaseline && typeof goalDraftBaseline === 'object' ? goalDraftBaseline : null,
+ }
}
export function writeScannerDraft(
teamId: number,
scanner: ScannerFormValues,
- linkedExperimentId: number | null = null
+ linkedExperimentId: number | null = null,
+ provenance: ScannerDraftProvenance | null = null
): number | null {
const savedAt = Date.now()
const draft: StoredScannerDraft = {
@@ -31,6 +55,7 @@ export function writeScannerDraft(
savedAt,
scanner,
linkedExperimentId,
+ ...(provenance ? { provenance } : {}),
}
try {
localStorage.setItem(DRAFT_STORAGE_KEY, JSON.stringify(draft))
@@ -60,6 +85,7 @@ export function readScannerDraft(teamId: number): ScannerDraft | null {
scanner: draft.scanner,
savedAt: draft.savedAt,
linkedExperimentId: typeof draft.linkedExperimentId === 'number' ? draft.linkedExperimentId : null,
+ provenance: readProvenance(draft.provenance),
}
} catch {
return null