:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-vision): show the scan's question and answer on highlight cards (#127464)
Tue Haulunda2e939d25959
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex 79d0dadcffe3..482b855e9396 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -14915,9 +14915,9 @@ snapshots:scenes-app-replay-vision--experiment-variants-waiting-for-observations--light:hash: v1.k794b7964.a1555dab4c0006028e8c240451c56c6aed5e31c8a8e68c893e3d6a07b562bba6.VYeYPsjZ_p0Rt4cbZzF7lbxjxurMjLXff9kJRsy49gUscenes-app-replay-vision--home-highlight-feed--dark:- hash: v1.k794b7964.bdefa12f8b1fd3c372d13eed61076a004f3bb4259d737bec9c09bd061fbd9359.DY7EZQgzN_Fsjqen3WwoyFAW4Wxli-3CTT1m7g6mfNg+ hash: v1.k794b7964.ce0a030f18d1d88ba525b3cceb99792c4d9bd91e486320881eae81f0f8bc1bb7.fmqlFtZxZRvva22AOIfiqZiXgs1WZ44Z080MR0nBpWsscenes-app-replay-vision--home-highlight-feed--light:- hash: v1.k794b7964.2c2b41d8fdf182d402a7b63b53fdd49048b7cbd4638d34ee76ead066a537c653.579SnEEXXsj4wAf31vrZmwJv6CETMTgDOos_NgdrCwU+ hash: v1.k794b7964.2f1cc4e6d66d677a25fc5b8ccd7f15fb392e4e85796d067319f61ddd31dd3b94.67g0YG_Oc_-vrb65Vk1use4JG3o9kyL1jV-50kgwjRwscenes-app-replay-vision--home-highlight-feed-empty--dark:hash: v1.k794b7964.aa2a535d2294cab79f8090ef68ef9b8a74c930928c50c011df3eabb0ee7f455b.LrXy7q5Igei_2wtEYnbLH-pTAC_cBJSgkLKp5xAiNcQscenes-app-replay-vision--home-highlight-feed-empty--light:diff --git a/products/replay_vision/frontend/observations/ObservationHeadline.tsx b/products/replay_vision/frontend/observations/ObservationHeadline.tsxindex 4be29a1a630a..d020359b8626 100644--- a/products/replay_vision/frontend/observations/ObservationHeadline.tsx+++ b/products/replay_vision/frontend/observations/ObservationHeadline.tsx@@ -1,12 +1,10 @@-import { IconCheckCircle, IconQuestion, IconSparkles, IconXCircle } from '@posthog/icons'+import { IconSparkles } from '@posthog/icons'import { Tooltip } from '@posthog/lemon-ui'import { LabeledRow } from '../components/LabeledRow'import { ObservationPrimaryOutput } from '../components/ObservationCard'-import type { PromptValenceEnumApi, ReplayObservationApi, ScannerTypeEnumApi } from '../generated/api.schemas'-import { configFromSnapshot, type ScorerScannerConfig } from '../replay_scanners/types'+import type { ReplayObservationApi, ScannerTypeEnumApi } from '../generated/api.schemas'import {- type MonitorVerdict,VERDICT_LABEL,readFixedTags,readFreeformTags,@@ -14,6 +12,7 @@ import {readScore,readVerdict,} from '../utils/observation'+import { VERDICT_ICON, scoreColorFor, scorerScale, verdictClassFor } from '../utils/resultValence'import { ConfidenceBadge } from './ConfidenceBadge'// Filled, so each category stays visible in dark mode, where a tag's outline alone fades into the card.@@ -29,41 +28,6 @@ const HEADLINE_LABEL: Record<ScannerTypeEnumApi, string> = {experiment: 'Summary',}-// The `--success`/`--danger` family LemonTag uses. The `text-success` utility maps to a different, brighter green.-const GOOD_CLASS = 'text-success-dark dark:text-success-light'-const BAD_CLASS = 'text-danger-dark dark:text-danger-light'--const VERDICT_ICON: Record<MonitorVerdict, typeof IconCheckCircle> = {- yes: IconCheckCircle,- no: IconXCircle,- inconclusive: IconQuestion,-}--type Direction = 'good' | 'bad'--function direction(valence: PromptValenceEnumApi | null): Direction | null {- return valence === 'good' || valence === 'bad' ? valence : null-}--function verdictClass(verdict: MonitorVerdict, yesIs: Direction | null): string {- if (verdict === 'inconclusive') {- return 'text-secondary dark:text-default'- }- if (!yesIs) {- return 'text-default'- }- return (verdict === 'yes') === (yesIs === 'good') ? GOOD_CLASS : BAD_CLASS-}--function scorerScale(observation: ReplayObservationApi): { min: number; max: number | null; label: string | null } {- const scale = (configFromSnapshot(observation.scanner_snapshot) as ScorerScannerConfig | null)?.scale- return {- min: typeof scale?.min === 'number' ? scale.min : 0,- max: typeof scale?.max === 'number' ? scale.max : null,- label: scale?.label ?? null,- }-}-/** The scale's name, stamped on each result from the scanner config. Older results may lack it. */function scoreLabel(observation: ReplayObservationApi): string | null {const resultLabel = readModelOutput(observation)?.label@@ -86,15 +50,6 @@ function headlineLabel(observation: ReplayObservationApi, scannerType: ScannerTyreturn `${label} · ${shown}`}-/** Red at the bad end of the scale through amber to green at the good end. */-function scoreColor(score: number, min: number, max: number, highIs: Direction): string {- const fromMin = max > min ? Math.min(1, Math.max(0, (score - min) / (max - min))) : 1- const position = highIs === 'good' ? fromMin : 1 - fromMin- return position < 0.5- ? `color-mix(in oklab, var(--warning) ${Math.round(position * 200)}%, var(--danger))`- : `color-mix(in oklab, var(--success) ${Math.round((position - 0.5) * 200)}%, var(--warning))`-}-function HeadlineValue({observation,scannerType,@@ -112,7 +67,7 @@ function HeadlineValue({const Icon = VERDICT_ICON[verdict]return (<span- className={`inline-flex items-center gap-1.5 text-xl font-bold ${verdictClass(verdict, direction(observation.prompt_valence))}`}+ className={`inline-flex items-center gap-1.5 text-xl font-bold ${verdictClassFor(observation, verdict)}`}data-attr="vision-observation-verdict"><Icon className="text-2xl" />@@ -123,9 +78,8 @@ function HeadlineValue({if (scannerType === 'scorer') {const score = readScore(observation)- const { min, max } = scorerScale(observation)- const highIs = direction(observation.prompt_valence)- const color = score !== null && max !== null && highIs ? scoreColor(score, min, max, highIs) : null+ const { max } = scorerScale(observation)+ const color = score !== null ? scoreColorFor(observation, score) : nullreturn (<span className="text-3xl font-bold tabular-nums"><spandiff --git a/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.stories.tsx b/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.stories.tsxindex 801d03f69ae3..cf9b1f1e53bf 100644--- a/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.stories.tsx+++ b/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.stories.tsx@@ -1210,8 +1210,9 @@ const checkoutGroupMembers = [}))-// The home page lands on the Highlights feed. Each pick names its category, and the first stands for seven similar-// sessions. The viewer created one scanner, so the feed offers the scope tabs.+// The home page lands on the Highlights feed. There is one pick per scanner type, so each answer label shows. Each pick+// names its category, and the first stands for seven similar sessions. The viewer created one scanner, so the feed+// offers the scope tabs.export const HomeHighlightFeed: StoryObj = {decorators: [mswDecorator({@@ -1238,6 +1239,7 @@ export const HomeHighlightFeed: StoryObj = {observation: observation({id: '00000000-0000-0000-0000-0000000000a1',prompt_question: 'Did the user hesitate at checkout?',+ prompt_valence: 'bad',scanner_id: scanners.results[0].id,scanner_snapshot: {name: 'Confused checkout',@@ -1254,7 +1256,7 @@ export const HomeHighlightFeed: StoryObj = {verdict: 'yes',confidence: 0.92,reasoning:- 'The card form rejected a valid card three times before the user left.',+ 'The card form rejected a valid card three times before the user left. They retried with the same card, opened the help chat to ask why it was declined, then closed the tab without paying.',key_moment_ms: 154000,},signals_count: 0,@@ -1307,6 +1309,69 @@ export const HomeHighlightFeed: StoryObj = {}),reason: { kind: 'embedding_pick', watch_category: 'revenue', similar_count: 0 },},+ {+ observation: observation({+ id: '00000000-0000-0000-0000-0000000000a3',+ prompt_question: 'Which frustration patterns appear in this session?',+ scanner_id: scanners.results[1].id,+ scanner_snapshot: {+ name: 'Frustration tags',+ scanner_type: 'classifier',+ scanner_version: 1,+ model: 'gemini-3.8-flash',+ provider: 'google',+ emits_signals: false,+ scanner_config: {+ prompt: 'Tag this session.',+ tags: ['rage-click', 'dead-end'],+ multi_label: true,+ },+ },+ scanner_result: {+ model_output: {+ scanner_type: 'classifier',+ tags: ['rage-click', 'dead-end'],+ confidence: 0.64,+ reasoning:+ 'The user clicked the disabled Save button eleven times. The settings page gave no reason why, so they left for the docs.',+ key_moment_ms: 61000,+ },+ signals_count: 0,+ },+ viewed: true,+ }),+ reason: { kind: 'embedding_pick', watch_category: 'struggle', similar_count: 3 },+ },+ {+ observation: observation({+ id: '00000000-0000-0000-0000-0000000000a4',+ prompt_question: 'How strong is the buying intent in this session?',+ prompt_valence: 'good',+ scanner_id: scanners.results[3].id,+ scanner_snapshot: {+ name: 'Intent score',+ scanner_type: 'scorer',+ scanner_version: 1,+ model: 'gemini-3.8-flash',+ provider: 'google',+ emits_signals: false,+ scanner_config: { prompt: 'Score this session.', scale: { min: 0, max: 10 } },+ },+ scanner_result: {+ model_output: {+ scanner_type: 'scorer',+ score: 9,+ confidence: 0.88,+ reasoning:+ 'The user compared plans twice and opened the billing page. They invited a teammate before checking out.',+ key_moment_ms: 95000,+ },+ signals_count: 0,+ },+ viewed: true,+ }),+ reason: { kind: 'embedding_pick', watch_category: 'revenue', similar_count: 0 },+ },],},},diff --git a/products/replay_vision/frontend/replay_scanners/components/HighlightFeedCard.tsx b/products/replay_vision/frontend/replay_scanners/components/HighlightFeedCard.tsxindex 9e5fb31e0539..14641fce6b3c 100644--- a/products/replay_vision/frontend/replay_scanners/components/HighlightFeedCard.tsx+++ b/products/replay_vision/frontend/replay_scanners/components/HighlightFeedCard.tsx@@ -4,7 +4,15 @@ import { combineUrl } from 'kea-router'import { useCallback, useEffect, useMemo, useState } from 'react'import { useInView } from 'react-intersection-observer'-import { IconChevronLeft, IconChevronRight, IconPlayFilled } from '@posthog/icons'+import {+ IconCheck,+ IconChevronLeft,+ IconChevronRight,+ IconMinus,+ IconPlayFilled,+ IconSparkles,+ IconX,+} from '@posthog/icons'import { LemonButton, Link, Tooltip } from '@posthog/lemon-ui'import { useAutoMarkRead } from 'lib/components/NotificationsMenu/useAutoMarkRead'@@ -31,8 +39,21 @@ import type {} from '../../generated/api.schemas'import { OBSERVATION_ORIGIN_PARAM, HIGHLIGHT_FEED_ORIGIN } from '../../utils/breadcrumbs'import { citedTextToPlainText } from '../../utils/citations'-import { hasScannerPage, scannerLabel } from '../../utils/observation'+import {+ type ConfidenceLevel,+ VERDICT_LABEL,+ confidenceLevel,+ hasScannerPage,+ isSummaryScannerType,+ readConfidence,+ readFixedTags,+ readFreeformTags,+ readScore,+ readVerdict,+ scannerLabel,+} from '../../utils/observation'import { TimelineChapter, readChapters } from '../../utils/recordingTimeline'+import { CONFIDENCE_CLASS, VERDICT_ICON, scoreColorFor, scorerScale, verdictClassFor } from '../../utils/resultValence'import { feedCategoriesParam, highlightFeedLogic } from '../highlightFeedLogic'import { CARD_SEEN_DWELL_MS, cardProperties } from '../highlightFeedTelemetry'import { ScannerType } from '../types'@@ -166,12 +187,15 @@ function ChapterCarousel({slide,onSlide,retryKey,+ below,}: {observation: ReplayObservationApichapters: TimelineChapter[]slide: numberonSlide: (slide: number) => voidretryKey: unknown+ /** Sits under the chapter caption, inside the poster. */+ below?: React.ReactNode}): JSX.Element {const { currentTeamId } = useValues(teamLogic)const [reached, setReached] = useState(1)@@ -266,6 +290,7 @@ function ChapterCarousel({/>))}</div>+ {below}</div></>)@@ -367,6 +392,145 @@ function HighlightFeedScannerLink({)}+// Filled, so each category stays visible on the grey box in dark mode.+const CATEGORY_CLASS =+ 'inline-flex items-center gap-0.5 rounded border border-primary bg-surface-primary px-1.5 font-semibold text-default'++// More pills than this overflow a narrow card's answer line, so the rest collapse into a count.+const MAX_CATEGORY_PILLS = 3++const CONFIDENCE_ICON: Record<ConfidenceLevel['type'], typeof IconCheck> = {+ success: IconCheck,+ warning: IconMinus,+ danger: IconX,+}++/** What the scan answered. Only the answer itself takes the valence color, and a summary has none. */+function highlightCardResult(+ observation: ReplayObservationApi,+ scannerType: ScannerType | undefined+): JSX.Element | null {+ if (scannerType === 'monitor') {+ const verdict = readVerdict(observation)+ if (!verdict) {+ return null+ }+ const Icon = VERDICT_ICON[verdict]+ return (+ <span className={clsx('flex items-center gap-0.5', verdictClassFor(observation, verdict))}>+ <Icon className="shrink-0" aria-hidden />+ {VERDICT_LABEL[verdict]}+ </span>+ )+ }+ if (scannerType === 'scorer') {+ const score = readScore(observation)+ if (score === null) {+ return null+ }+ const { max } = scorerScale(observation)+ const color = scoreColorFor(observation, score)+ return (+ <span+ className="tabular-nums"+ // The color is a position on a continuous scale, which Tailwind classes can't express.+ // eslint-disable-next-line react/forbid-dom-props+ style={color ? { color } : undefined}+ >+ {max !== null ? `${score} / ${max}` : score}+ </span>+ )+ }+ if (scannerType === 'classifier') {+ const tags = readFixedTags(observation)+ const freeform = readFreeformTags(observation)+ const categories = [+ ...tags.map((tag) => ({ tag, isFreeform: false })),+ ...freeform.map((tag) => ({ tag, isFreeform: true })),+ ]+ if (categories.length === 0) {+ return <span className="font-normal text-secondary">None</span>+ }+ const hidden = categories.length - MAX_CATEGORY_PILLS+ return (+ // Pills shrink and truncate, and the count stays outside them, so a narrow card still shows how many are hidden.+ <span className="flex min-w-0 items-center gap-1">+ {categories.slice(0, MAX_CATEGORY_PILLS).map(({ tag, isFreeform }, index) => (+ <span key={`${index}-${tag}`} className={clsx(CATEGORY_CLASS, 'min-w-0')}>+ {isFreeform && <IconSparkles className="shrink-0" aria-hidden />}+ <span className="truncate">{tag}</span>+ </span>+ ))}+ {hidden > 0 && <span className={clsx(CATEGORY_CLASS, 'shrink-0')}>+{hidden}</span>}+ </span>+ )+ }+ return null+}++function HighlightCardConfidence({ observation }: { observation: ReplayObservationApi }): JSX.Element | null {+ const confidence = readConfidence(observation)+ if (confidence === null) {+ return null+ }+ const level = confidenceLevel(confidence)+ const Icon = CONFIDENCE_ICON[level.type]+ return (+ <span className="flex shrink-0 items-center gap-1">+ <span className="text-secondary">Confidence</span>+ <span+ className={clsx('flex items-center gap-0.5 font-semibold tabular-nums', CONFIDENCE_CLASS[level.type])}+ >+ <Icon className="shrink-0" role="img" aria-label={`${level.label} confidence`} />+ {Math.round(confidence * 100)}%+ </span>+ </span>+ )+}++/**+ * The scanner's question, then its answer with the confidence beside it. The box sits over the+ * poster, so its height never changes the card's. A summary has no answer, so it gets no box.+ */+function HighlightCardAnswer({+ observation,+ scannerType,+}: {+ observation: ReplayObservationApi+ scannerType: ScannerType | undefined+}): JSX.Element | null {+ if (isSummaryScannerType(scannerType)) {+ return null+ }+ const question = observation.prompt_question+ const result = highlightCardResult(observation, scannerType)+ if (!question && !result) {+ return null+ }+ return (+ <div+ className="grid min-w-0 grid-cols-[max-content_minmax(0,1fr)] items-center gap-x-2 gap-y-1 rounded-md border border-primary bg-surface-secondary px-3 py-2 text-xs"+ data-attr="vision-highlight-feed-answer"+ >+ {question && (+ <>+ <span className="text-secondary">Question</span>+ <span className="min-w-0 truncate">{question}</span>+ </>+ )}+ <span className="text-secondary">Answer</span>+ <span className="flex h-5 min-w-0 items-center gap-3">+ <span className="flex min-w-0 items-center overflow-hidden font-semibold">+ {result ?? <span className="font-normal text-secondary">None</span>}+ </span>+ <span className="ml-auto">+ <HighlightCardConfidence observation={observation} />+ </span>+ </span>+ </div>+ )+}+/*** The title of a row. A summarizer's authored title is already short, so it leads. Other scans* lead with highlightFeedRowSentence.@@ -454,6 +618,7 @@ export function HighlightFeedRow({chapter_slide: slide,})}+ const answerBox = <HighlightCardAnswer observation={observation} scannerType={scannerType} />return (<HighlightFeedCardLayoutref={setRowRef}@@ -501,13 +666,17 @@ export function HighlightFeedRow({setSlide(next)}}retryKey={feedLoad?.feedLoadId}+ below={answerBox}/>) : (- <ObservationThumbnail- observation={observation}- className="rounded-none border-0"- retryKey={feedLoad?.feedLoadId}- />+ <>+ <ObservationThumbnail+ observation={observation}+ className="rounded-none border-0"+ retryKey={feedLoad?.feedLoadId}+ />+ <div className="absolute inset-x-0 bottom-0 z-[5] px-2.5 pb-2">{answerBox}</div>+ </>)}{!observation.viewed && <UnviewedObservationTag className="absolute top-2 left-2 z-10" />}</>diff --git a/products/replay_vision/frontend/utils/resultValence.ts b/products/replay_vision/frontend/utils/resultValence.tsnew file mode 100644index 000000000000..a38436e82d1f--- /dev/null+++ b/products/replay_vision/frontend/utils/resultValence.ts@@ -0,0 +1,72 @@+import { IconCheckCircle, IconQuestion, IconXCircle } from '@posthog/icons'++import type { PromptValenceEnumApi, ReplayObservationApi } from '../generated/api.schemas'+import { configFromSnapshot, type ScorerScannerConfig } from '../replay_scanners/types'+import type { ConfidenceLevel, MonitorVerdict } from './observation'++// The `--success`/`--danger` family LemonTag uses. The `text-success` utility maps to a different, brighter green.+const GOOD_CLASS = 'text-success-dark dark:text-success-light'+const BAD_CLASS = 'text-danger-dark dark:text-danger-light'++/** Text color for each confidence level, from the same color family as the verdicts. */+export const CONFIDENCE_CLASS: Record<ConfidenceLevel['type'], string> = {+ success: GOOD_CLASS,+ warning: 'text-warning-dark dark:text-warning',+ danger: BAD_CLASS,+}++export const VERDICT_ICON: Record<MonitorVerdict, typeof IconCheckCircle> = {+ yes: IconCheckCircle,+ no: IconXCircle,+ inconclusive: IconQuestion,+}++type Direction = 'good' | 'bad'++function direction(valence: PromptValenceEnumApi | null): Direction | null {+ return valence === 'good' || valence === 'bad' ? valence : null+}++function verdictClass(verdict: MonitorVerdict, yesIs: Direction | null): string {+ if (verdict === 'inconclusive') {+ return 'text-secondary dark:text-default'+ }+ if (!yesIs) {+ return 'text-default'+ }+ return (verdict === 'yes') === (yesIs === 'good') ? GOOD_CLASS : BAD_CLASS+}++export function scorerScale(observation: ReplayObservationApi): {+ min: number+ max: number | null+ label: string | null+} {+ const scale = (configFromSnapshot(observation.scanner_snapshot) as ScorerScannerConfig | null)?.scale+ return {+ min: typeof scale?.min === 'number' ? scale.min : 0,+ max: typeof scale?.max === 'number' ? scale.max : null,+ label: scale?.label ?? null,+ }+}++/** Red at the bad end of the scale through amber to green at the good end. */+function scoreColor(score: number, min: number, max: number, highIs: Direction): string {+ const fromMin = max > min ? Math.min(1, Math.max(0, (score - min) / (max - min))) : 1+ const position = highIs === 'good' ? fromMin : 1 - fromMin+ return position < 0.5+ ? `color-mix(in oklab, var(--warning) ${Math.round(position * 200)}%, var(--danger))`+ : `color-mix(in oklab, var(--success) ${Math.round((position - 0.5) * 200)}%, var(--warning))`+}++/** The verdict's text color, given whether the scanner's question is good or bad news. */+export function verdictClassFor(observation: ReplayObservationApi, verdict: MonitorVerdict): string {+ return verdictClass(verdict, direction(observation.prompt_valence))+}++/** The score's color on the scanner's scale, or null when the scale or the question's valence is unknown. */+export function scoreColorFor(observation: ReplayObservationApi, score: number): string | null {+ const { min, max } = scorerScale(observation)+ const highIs = direction(observation.prompt_valence)+ return max !== null && highIs ? scoreColor(score, min, max, highIs) : null+}