Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-vision): show the scan's question and answer on highlight cards (#127464)

Tue Haulunda2e939d25959
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 79d0dadcffe3..482b855e9396 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -14915,9 +14915,9 @@ snapshots:
scenes-app-replay-vision--experiment-variants-waiting-for-observations--light:
hash: v1.k794b7964.a1555dab4c0006028e8c240451c56c6aed5e31c8a8e68c893e3d6a07b562bba6.VYeYPsjZ_p0Rt4cbZzF7lbxjxurMjLXff9kJRsy49gU
scenes-app-replay-vision--home-highlight-feed--dark:
- hash: v1.k794b7964.bdefa12f8b1fd3c372d13eed61076a004f3bb4259d737bec9c09bd061fbd9359.DY7EZQgzN_Fsjqen3WwoyFAW4Wxli-3CTT1m7g6mfNg
+ hash: v1.k794b7964.ce0a030f18d1d88ba525b3cceb99792c4d9bd91e486320881eae81f0f8bc1bb7.fmqlFtZxZRvva22AOIfiqZiXgs1WZ44Z080MR0nBpWs
scenes-app-replay-vision--home-highlight-feed--light:
- hash: v1.k794b7964.2c2b41d8fdf182d402a7b63b53fdd49048b7cbd4638d34ee76ead066a537c653.579SnEEXXsj4wAf31vrZmwJv6CETMTgDOos_NgdrCwU
+ hash: v1.k794b7964.2f1cc4e6d66d677a25fc5b8ccd7f15fb392e4e85796d067319f61ddd31dd3b94.67g0YG_Oc_-vrb65Vk1use4JG3o9kyL1jV-50kgwjRw
scenes-app-replay-vision--home-highlight-feed-empty--dark:
hash: v1.k794b7964.aa2a535d2294cab79f8090ef68ef9b8a74c930928c50c011df3eabb0ee7f455b.LrXy7q5Igei_2wtEYnbLH-pTAC_cBJSgkLKp5xAiNcQ
scenes-app-replay-vision--home-highlight-feed-empty--light:
diff --git a/products/replay_vision/frontend/observations/ObservationHeadline.tsx b/products/replay_vision/frontend/observations/ObservationHeadline.tsx
index 4be29a1a630a..d020359b8626 100644
--- a/products/replay_vision/frontend/observations/ObservationHeadline.tsx
+++ b/products/replay_vision/frontend/observations/ObservationHeadline.tsx
@@ -1,12 +1,10 @@
-import { IconCheckCircle, IconQuestion, IconSparkles, IconXCircle } from '@posthog/icons'
+import { IconSparkles } from '@posthog/icons'
import { Tooltip } from '@posthog/lemon-ui'
import { LabeledRow } from '../components/LabeledRow'
import { ObservationPrimaryOutput } from '../components/ObservationCard'
-import type { PromptValenceEnumApi, ReplayObservationApi, ScannerTypeEnumApi } from '../generated/api.schemas'
-import { configFromSnapshot, type ScorerScannerConfig } from '../replay_scanners/types'
+import type { ReplayObservationApi, ScannerTypeEnumApi } from '../generated/api.schemas'
import {
- type MonitorVerdict,
VERDICT_LABEL,
readFixedTags,
readFreeformTags,
@@ -14,6 +12,7 @@ import {
readScore,
readVerdict,
} from '../utils/observation'
+import { VERDICT_ICON, scoreColorFor, scorerScale, verdictClassFor } from '../utils/resultValence'
import { ConfidenceBadge } from './ConfidenceBadge'
// Filled, so each category stays visible in dark mode, where a tag's outline alone fades into the card.
@@ -29,41 +28,6 @@ const HEADLINE_LABEL: Record<ScannerTypeEnumApi, string> = {
experiment: 'Summary',
}
-// The `--success`/`--danger` family LemonTag uses. The `text-success` utility maps to a different, brighter green.
-const GOOD_CLASS = 'text-success-dark dark:text-success-light'
-const BAD_CLASS = 'text-danger-dark dark:text-danger-light'
-
-const VERDICT_ICON: Record<MonitorVerdict, typeof IconCheckCircle> = {
- yes: IconCheckCircle,
- no: IconXCircle,
- inconclusive: IconQuestion,
-}
-
-type Direction = 'good' | 'bad'
-
-function direction(valence: PromptValenceEnumApi | null): Direction | null {
- return valence === 'good' || valence === 'bad' ? valence : null
-}
-
-function verdictClass(verdict: MonitorVerdict, yesIs: Direction | null): string {
- if (verdict === 'inconclusive') {
- return 'text-secondary dark:text-default'
- }
- if (!yesIs) {
- return 'text-default'
- }
- return (verdict === 'yes') === (yesIs === 'good') ? GOOD_CLASS : BAD_CLASS
-}
-
-function scorerScale(observation: ReplayObservationApi): { min: number; max: number | null; label: string | null } {
- const scale = (configFromSnapshot(observation.scanner_snapshot) as ScorerScannerConfig | null)?.scale
- return {
- min: typeof scale?.min === 'number' ? scale.min : 0,
- max: typeof scale?.max === 'number' ? scale.max : null,
- label: scale?.label ?? null,
- }
-}
-
/** The scale's name, stamped on each result from the scanner config. Older results may lack it. */
function scoreLabel(observation: ReplayObservationApi): string | null {
const resultLabel = readModelOutput(observation)?.label
@@ -86,15 +50,6 @@ function headlineLabel(observation: ReplayObservationApi, scannerType: ScannerTy
return `${label} · ${shown}`
}
-/** Red at the bad end of the scale through amber to green at the good end. */
-function scoreColor(score: number, min: number, max: number, highIs: Direction): string {
- const fromMin = max > min ? Math.min(1, Math.max(0, (score - min) / (max - min))) : 1
- const position = highIs === 'good' ? fromMin : 1 - fromMin
- return position < 0.5
- ? `color-mix(in oklab, var(--warning) ${Math.round(position * 200)}%, var(--danger))`
- : `color-mix(in oklab, var(--success) ${Math.round((position - 0.5) * 200)}%, var(--warning))`
-}
-
function HeadlineValue({
observation,
scannerType,
@@ -112,7 +67,7 @@ function HeadlineValue({
const Icon = VERDICT_ICON[verdict]
return (
<span
- className={`inline-flex items-center gap-1.5 text-xl font-bold ${verdictClass(verdict, direction(observation.prompt_valence))}`}
+ className={`inline-flex items-center gap-1.5 text-xl font-bold ${verdictClassFor(observation, verdict)}`}
data-attr="vision-observation-verdict"
>
<Icon className="text-2xl" />
@@ -123,9 +78,8 @@ function HeadlineValue({
if (scannerType === 'scorer') {
const score = readScore(observation)
- const { min, max } = scorerScale(observation)
- const highIs = direction(observation.prompt_valence)
- const color = score !== null && max !== null && highIs ? scoreColor(score, min, max, highIs) : null
+ const { max } = scorerScale(observation)
+ const color = score !== null ? scoreColorFor(observation, score) : null
return (
<span className="text-3xl font-bold tabular-nums">
<span
diff --git a/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.stories.tsx b/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.stories.tsx
index 801d03f69ae3..cf9b1f1e53bf 100644
--- a/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.stories.tsx
+++ b/products/replay_vision/frontend/replay_scanners/ReplayScannersScene.stories.tsx
@@ -1210,8 +1210,9 @@ const checkoutGroupMembers = [
})
)
-// The home page lands on the Highlights feed. Each pick names its category, and the first stands for seven similar
-// sessions. The viewer created one scanner, so the feed offers the scope tabs.
+// The home page lands on the Highlights feed. There is one pick per scanner type, so each answer label shows. Each pick
+// names its category, and the first stands for seven similar sessions. The viewer created one scanner, so the feed
+// offers the scope tabs.
export const HomeHighlightFeed: StoryObj = {
decorators: [
mswDecorator({
@@ -1238,6 +1239,7 @@ export const HomeHighlightFeed: StoryObj = {
observation: observation({
id: '00000000-0000-0000-0000-0000000000a1',
prompt_question: 'Did the user hesitate at checkout?',
+ prompt_valence: 'bad',
scanner_id: scanners.results[0].id,
scanner_snapshot: {
name: 'Confused checkout',
@@ -1254,7 +1256,7 @@ export const HomeHighlightFeed: StoryObj = {
verdict: 'yes',
confidence: 0.92,
reasoning:
- 'The card form rejected a valid card three times before the user left.',
+ 'The card form rejected a valid card three times before the user left. They retried with the same card, opened the help chat to ask why it was declined, then closed the tab without paying.',
key_moment_ms: 154000,
},
signals_count: 0,
@@ -1307,6 +1309,69 @@ export const HomeHighlightFeed: StoryObj = {
}),
reason: { kind: 'embedding_pick', watch_category: 'revenue', similar_count: 0 },
},
+ {
+ observation: observation({
+ id: '00000000-0000-0000-0000-0000000000a3',
+ prompt_question: 'Which frustration patterns appear in this session?',
+ scanner_id: scanners.results[1].id,
+ scanner_snapshot: {
+ name: 'Frustration tags',
+ scanner_type: 'classifier',
+ scanner_version: 1,
+ model: 'gemini-3.8-flash',
+ provider: 'google',
+ emits_signals: false,
+ scanner_config: {
+ prompt: 'Tag this session.',
+ tags: ['rage-click', 'dead-end'],
+ multi_label: true,
+ },
+ },
+ scanner_result: {
+ model_output: {
+ scanner_type: 'classifier',
+ tags: ['rage-click', 'dead-end'],
+ confidence: 0.64,
+ reasoning:
+ 'The user clicked the disabled Save button eleven times. The settings page gave no reason why, so they left for the docs.',
+ key_moment_ms: 61000,
+ },
+ signals_count: 0,
+ },
+ viewed: true,
+ }),
+ reason: { kind: 'embedding_pick', watch_category: 'struggle', similar_count: 3 },
+ },
+ {
+ observation: observation({
+ id: '00000000-0000-0000-0000-0000000000a4',
+ prompt_question: 'How strong is the buying intent in this session?',
+ prompt_valence: 'good',
+ scanner_id: scanners.results[3].id,
+ scanner_snapshot: {
+ name: 'Intent score',
+ scanner_type: 'scorer',
+ scanner_version: 1,
+ model: 'gemini-3.8-flash',
+ provider: 'google',
+ emits_signals: false,
+ scanner_config: { prompt: 'Score this session.', scale: { min: 0, max: 10 } },
+ },
+ scanner_result: {
+ model_output: {
+ scanner_type: 'scorer',
+ score: 9,
+ confidence: 0.88,
+ reasoning:
+ 'The user compared plans twice and opened the billing page. They invited a teammate before checking out.',
+ key_moment_ms: 95000,
+ },
+ signals_count: 0,
+ },
+ viewed: true,
+ }),
+ reason: { kind: 'embedding_pick', watch_category: 'revenue', similar_count: 0 },
+ },
],
},
},
diff --git a/products/replay_vision/frontend/replay_scanners/components/HighlightFeedCard.tsx b/products/replay_vision/frontend/replay_scanners/components/HighlightFeedCard.tsx
index 9e5fb31e0539..14641fce6b3c 100644
--- a/products/replay_vision/frontend/replay_scanners/components/HighlightFeedCard.tsx
+++ b/products/replay_vision/frontend/replay_scanners/components/HighlightFeedCard.tsx
@@ -4,7 +4,15 @@ import { combineUrl } from 'kea-router'
import { useCallback, useEffect, useMemo, useState } from 'react'
import { useInView } from 'react-intersection-observer'
-import { IconChevronLeft, IconChevronRight, IconPlayFilled } from '@posthog/icons'
+import {
+ IconCheck,
+ IconChevronLeft,
+ IconChevronRight,
+ IconMinus,
+ IconPlayFilled,
+ IconSparkles,
+ IconX,
+} from '@posthog/icons'
import { LemonButton, Link, Tooltip } from '@posthog/lemon-ui'
import { useAutoMarkRead } from 'lib/components/NotificationsMenu/useAutoMarkRead'
@@ -31,8 +39,21 @@ import type {
} from '../../generated/api.schemas'
import { OBSERVATION_ORIGIN_PARAM, HIGHLIGHT_FEED_ORIGIN } from '../../utils/breadcrumbs'
import { citedTextToPlainText } from '../../utils/citations'
-import { hasScannerPage, scannerLabel } from '../../utils/observation'
+import {
+ type ConfidenceLevel,
+ VERDICT_LABEL,
+ confidenceLevel,
+ hasScannerPage,
+ isSummaryScannerType,
+ readConfidence,
+ readFixedTags,
+ readFreeformTags,
+ readScore,
+ readVerdict,
+ scannerLabel,
+} from '../../utils/observation'
import { TimelineChapter, readChapters } from '../../utils/recordingTimeline'
+import { CONFIDENCE_CLASS, VERDICT_ICON, scoreColorFor, scorerScale, verdictClassFor } from '../../utils/resultValence'
import { feedCategoriesParam, highlightFeedLogic } from '../highlightFeedLogic'
import { CARD_SEEN_DWELL_MS, cardProperties } from '../highlightFeedTelemetry'
import { ScannerType } from '../types'
@@ -166,12 +187,15 @@ function ChapterCarousel({
slide,
onSlide,
retryKey,
+ below,
}: {
observation: ReplayObservationApi
chapters: TimelineChapter[]
slide: number
onSlide: (slide: number) => void
retryKey: unknown
+ /** Sits under the chapter caption, inside the poster. */
+ below?: React.ReactNode
}): JSX.Element {
const { currentTeamId } = useValues(teamLogic)
const [reached, setReached] = useState(1)
@@ -266,6 +290,7 @@ function ChapterCarousel({
/>
))}
</div>
+ {below}
</div>
</>
)
@@ -367,6 +392,145 @@ function HighlightFeedScannerLink({
)
}
+// Filled, so each category stays visible on the grey box in dark mode.
+const CATEGORY_CLASS =
+ 'inline-flex items-center gap-0.5 rounded border border-primary bg-surface-primary px-1.5 font-semibold text-default'
+
+// More pills than this overflow a narrow card's answer line, so the rest collapse into a count.
+const MAX_CATEGORY_PILLS = 3
+
+const CONFIDENCE_ICON: Record<ConfidenceLevel['type'], typeof IconCheck> = {
+ success: IconCheck,
+ warning: IconMinus,
+ danger: IconX,
+}
+
+/** What the scan answered. Only the answer itself takes the valence color, and a summary has none. */
+function highlightCardResult(
+ observation: ReplayObservationApi,
+ scannerType: ScannerType | undefined
+): JSX.Element | null {
+ if (scannerType === 'monitor') {
+ const verdict = readVerdict(observation)
+ if (!verdict) {
+ return null
+ }
+ const Icon = VERDICT_ICON[verdict]
+ return (
+ <span className={clsx('flex items-center gap-0.5', verdictClassFor(observation, verdict))}>
+ <Icon className="shrink-0" aria-hidden />
+ {VERDICT_LABEL[verdict]}
+ </span>
+ )
+ }
+ if (scannerType === 'scorer') {
+ const score = readScore(observation)
+ if (score === null) {
+ return null
+ }
+ const { max } = scorerScale(observation)
+ const color = scoreColorFor(observation, score)
+ return (
+ <span
+ className="tabular-nums"
+ // The color is a position on a continuous scale, which Tailwind classes can't express.
+ // eslint-disable-next-line react/forbid-dom-props
+ style={color ? { color } : undefined}
+ >
+ {max !== null ? `${score} / ${max}` : score}
+ </span>
+ )
+ }
+ if (scannerType === 'classifier') {
+ const tags = readFixedTags(observation)
+ const freeform = readFreeformTags(observation)
+ const categories = [
+ ...tags.map((tag) => ({ tag, isFreeform: false })),
+ ...freeform.map((tag) => ({ tag, isFreeform: true })),
+ ]
+ if (categories.length === 0) {
+ return <span className="font-normal text-secondary">None</span>
+ }
+ const hidden = categories.length - MAX_CATEGORY_PILLS
+ return (
+ // Pills shrink and truncate, and the count stays outside them, so a narrow card still shows how many are hidden.
+ <span className="flex min-w-0 items-center gap-1">
+ {categories.slice(0, MAX_CATEGORY_PILLS).map(({ tag, isFreeform }, index) => (
+ <span key={`${index}-${tag}`} className={clsx(CATEGORY_CLASS, 'min-w-0')}>
+ {isFreeform && <IconSparkles className="shrink-0" aria-hidden />}
+ <span className="truncate">{tag}</span>
+ </span>
+ ))}
+ {hidden > 0 && <span className={clsx(CATEGORY_CLASS, 'shrink-0')}>+{hidden}</span>}
+ </span>
+ )
+ }
+ return null
+}
+
+function HighlightCardConfidence({ observation }: { observation: ReplayObservationApi }): JSX.Element | null {
+ const confidence = readConfidence(observation)
+ if (confidence === null) {
+ return null
+ }
+ const level = confidenceLevel(confidence)
+ const Icon = CONFIDENCE_ICON[level.type]
+ return (
+ <span className="flex shrink-0 items-center gap-1">
+ <span className="text-secondary">Confidence</span>
+ <span
+ className={clsx('flex items-center gap-0.5 font-semibold tabular-nums', CONFIDENCE_CLASS[level.type])}
+ >
+ <Icon className="shrink-0" role="img" aria-label={`${level.label} confidence`} />
+ {Math.round(confidence * 100)}%
+ </span>
+ </span>
+ )
+}
+
+/**
+ * The scanner's question, then its answer with the confidence beside it. The box sits over the
+ * poster, so its height never changes the card's. A summary has no answer, so it gets no box.
+ */
+function HighlightCardAnswer({
+ observation,
+ scannerType,
+}: {
+ observation: ReplayObservationApi
+ scannerType: ScannerType | undefined
+}): JSX.Element | null {
+ if (isSummaryScannerType(scannerType)) {
+ return null
+ }
+ const question = observation.prompt_question
+ const result = highlightCardResult(observation, scannerType)
+ if (!question && !result) {
+ return null
+ }
+ return (
+ <div
+ className="grid min-w-0 grid-cols-[max-content_minmax(0,1fr)] items-center gap-x-2 gap-y-1 rounded-md border border-primary bg-surface-secondary px-3 py-2 text-xs"
+ data-attr="vision-highlight-feed-answer"
+ >
+ {question && (
+ <>
+ <span className="text-secondary">Question</span>
+ <span className="min-w-0 truncate">{question}</span>
+ </>
+ )}
+ <span className="text-secondary">Answer</span>
+ <span className="flex h-5 min-w-0 items-center gap-3">
+ <span className="flex min-w-0 items-center overflow-hidden font-semibold">
+ {result ?? <span className="font-normal text-secondary">None</span>}
+ </span>
+ <span className="ml-auto">
+ <HighlightCardConfidence observation={observation} />
+ </span>
+ </span>
+ </div>
+ )
+}
+
/**
* The title of a row. A summarizer's authored title is already short, so it leads. Other scans
* lead with highlightFeedRowSentence.
@@ -454,6 +618,7 @@ export function HighlightFeedRow({
chapter_slide: slide,
})
}
+ const answerBox = <HighlightCardAnswer observation={observation} scannerType={scannerType} />
return (
<HighlightFeedCardLayout
ref={setRowRef}
@@ -501,13 +666,17 @@ export function HighlightFeedRow({
setSlide(next)
}}
retryKey={feedLoad?.feedLoadId}
+ below={answerBox}
/>
) : (
- <ObservationThumbnail
- observation={observation}
- className="rounded-none border-0"
- retryKey={feedLoad?.feedLoadId}
- />
+ <>
+ <ObservationThumbnail
+ observation={observation}
+ className="rounded-none border-0"
+ retryKey={feedLoad?.feedLoadId}
+ />
+ <div className="absolute inset-x-0 bottom-0 z-[5] px-2.5 pb-2">{answerBox}</div>
+ </>
)}
{!observation.viewed && <UnviewedObservationTag className="absolute top-2 left-2 z-10" />}
</>
diff --git a/products/replay_vision/frontend/utils/resultValence.ts b/products/replay_vision/frontend/utils/resultValence.ts
new file mode 100644
index 000000000000..a38436e82d1f
--- /dev/null
+++ b/products/replay_vision/frontend/utils/resultValence.ts
@@ -0,0 +1,72 @@
+import { IconCheckCircle, IconQuestion, IconXCircle } from '@posthog/icons'
+
+import type { PromptValenceEnumApi, ReplayObservationApi } from '../generated/api.schemas'
+import { configFromSnapshot, type ScorerScannerConfig } from '../replay_scanners/types'
+import type { ConfidenceLevel, MonitorVerdict } from './observation'
+
+// The `--success`/`--danger` family LemonTag uses. The `text-success` utility maps to a different, brighter green.
+const GOOD_CLASS = 'text-success-dark dark:text-success-light'
+const BAD_CLASS = 'text-danger-dark dark:text-danger-light'
+
+/** Text color for each confidence level, from the same color family as the verdicts. */
+export const CONFIDENCE_CLASS: Record<ConfidenceLevel['type'], string> = {
+ success: GOOD_CLASS,
+ warning: 'text-warning-dark dark:text-warning',
+ danger: BAD_CLASS,
+}
+
+export const VERDICT_ICON: Record<MonitorVerdict, typeof IconCheckCircle> = {
+ yes: IconCheckCircle,
+ no: IconXCircle,
+ inconclusive: IconQuestion,
+}
+
+type Direction = 'good' | 'bad'
+
+function direction(valence: PromptValenceEnumApi | null): Direction | null {
+ return valence === 'good' || valence === 'bad' ? valence : null
+}
+
+function verdictClass(verdict: MonitorVerdict, yesIs: Direction | null): string {
+ if (verdict === 'inconclusive') {
+ return 'text-secondary dark:text-default'
+ }
+ if (!yesIs) {
+ return 'text-default'
+ }
+ return (verdict === 'yes') === (yesIs === 'good') ? GOOD_CLASS : BAD_CLASS
+}
+
+export function scorerScale(observation: ReplayObservationApi): {
+ min: number
+ max: number | null
+ label: string | null
+} {
+ const scale = (configFromSnapshot(observation.scanner_snapshot) as ScorerScannerConfig | null)?.scale
+ return {
+ min: typeof scale?.min === 'number' ? scale.min : 0,
+ max: typeof scale?.max === 'number' ? scale.max : null,
+ label: scale?.label ?? null,
+ }
+}
+
+/** Red at the bad end of the scale through amber to green at the good end. */
+function scoreColor(score: number, min: number, max: number, highIs: Direction): string {
+ const fromMin = max > min ? Math.min(1, Math.max(0, (score - min) / (max - min))) : 1
+ const position = highIs === 'good' ? fromMin : 1 - fromMin
+ return position < 0.5
+ ? `color-mix(in oklab, var(--warning) ${Math.round(position * 200)}%, var(--danger))`
+ : `color-mix(in oklab, var(--success) ${Math.round((position - 0.5) * 200)}%, var(--warning))`
+}
+
+/** The verdict's text color, given whether the scanner's question is good or bad news. */
+export function verdictClassFor(observation: ReplayObservationApi, verdict: MonitorVerdict): string {
+ return verdictClass(verdict, direction(observation.prompt_valence))
+}
+
+/** The score's color on the scanner's scale, or null when the scale or the question's valence is unknown. */
+export function scoreColorFor(observation: ReplayObservationApi, score: number): string | null {
+ const { min, max } = scorerScale(observation)
+ const highIs = direction(observation.prompt_valence)
+ return max !== null && highIs ? scoreColor(score, min, max, highIs) : null
+}