Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay): quill twins for summarize and the observations empty state (#128966)

Hayne Ryu2eb8c1463941
diff --git a/frontend/src/scenes/session-recordings/player/watch/WatchPanelTabsWithObservations.tsx b/frontend/src/scenes/session-recordings/player/watch/WatchPanelTabsWithObservations.tsx
index 10f68c9657bd..55b5032ed2b7 100644
--- a/frontend/src/scenes/session-recordings/player/watch/WatchPanelTabsWithObservations.tsx
+++ b/frontend/src/scenes/session-recordings/player/watch/WatchPanelTabsWithObservations.tsx
@@ -4,8 +4,8 @@ import { Skeleton } from '@posthog/quill'
import { sessionRecordingPlayerLogic } from 'scenes/session-recordings/player/sessionRecordingPlayerLogic'
+import { QuillRecordingMoments } from 'products/replay_vision/frontend/components/QuillRecordingMoments'
import { QuillRecordingObservationsEmptyState } from 'products/replay_vision/frontend/components/QuillRecordingObservationsEmptyState'
-import { RecordingMoments } from 'products/replay_vision/frontend/components/RecordingMoments'
import { observationsDockLogic } from 'products/replay_vision/frontend/logics/observationsDockLogic'
import { momentCount } from 'products/replay_vision/frontend/utils/recordingTimeline'
import { recordingScanBlock } from 'products/replay_vision/frontend/utils/scanEligibility'
@@ -31,7 +31,7 @@ export function WatchPanelTabsWithObservations({
/>
) : (
<div className="flex flex-col max-h-[60vh]">
- <RecordingMoments key={sessionId} sessionId={sessionId} playerLogicProps={logicProps} />
+ <QuillRecordingMoments key={sessionId} sessionId={sessionId} playerLogicProps={logicProps} />
</div>
)
diff --git a/products/replay_vision/frontend/components/QuillObservationThumbnail.tsx b/products/replay_vision/frontend/components/QuillObservationThumbnail.tsx
new file mode 100644
index 000000000000..ae5142d07db9
--- /dev/null
+++ b/products/replay_vision/frontend/components/QuillObservationThumbnail.tsx
@@ -0,0 +1,107 @@
+import clsx from 'clsx'
+import { useValues } from 'kea'
+import { useEffect, useRef, useState } from 'react'
+
+import { IconVideoCamera } from '@posthog/icons'
+import { Skeleton } from '@posthog/quill'
+
+import { teamLogic } from 'scenes/teamLogic'
+
+import { ReplayObservationApi, VisionObservationsThumbnailRetrieveSize } from '../generated/api.schemas'
+import { observationThumbnailSrc } from './ObservationThumbnail'
+
+const RETRY_DELAY_MS = 2000
+
+/** Just under the endpoint's redirect cache, after which a remount refetches the poster. */
+const DECODED_TTL_MS = 9 * 60 * 1000
+
+const decodedThumbnails = new Map<string, number>()
+
+function isDecoded(src: string): boolean {
+ const decodedAt = decodedThumbnails.get(src)
+ if (decodedAt !== undefined && Date.now() - decodedAt < DECODED_TTL_MS) {
+ return true
+ }
+ decodedThumbnails.delete(src)
+ return false
+}
+
+/** The frame the scan picked out of the session, as a 16:9 poster. */
+export function QuillObservationThumbnail({
+ observation,
+ chapter,
+ className,
+ size = VisionObservationsThumbnailRetrieveSize.Large,
+ retryKey,
+ children,
+}: {
+ observation: ReplayObservationApi
+ chapter?: number
+ className?: string
+ size?: VisionObservationsThumbnailRetrieveSize
+ /** A new value clears a failed load, so a caller that keeps the poster mounted across data reloads can retry it. */
+ retryKey?: unknown
+ children?: React.ReactNode
+}): JSX.Element {
+ const { currentTeamId } = useValues(teamLogic)
+ const [attempt, setAttempt] = useState(0)
+ const [failedId, setFailedId] = useState<string | null>(null)
+ const [loadedSrc, setLoadedSrc] = useState<string | null>(null)
+ const retryTimer = useRef<ReturnType<typeof setTimeout> | null>(null)
+
+ // A row's instance is reused as the table pages, so a failure can outlive the observation it was for.
+ useEffect(() => {
+ setAttempt(0)
+ setFailedId(null)
+ }, [observation.id, chapter, retryKey])
+
+ useEffect(() => {
+ return () => {
+ if (retryTimer.current !== null) {
+ clearTimeout(retryTimer.current)
+ }
+ }
+ }, [])
+
+ const failed = failedId === observation.id
+ const baseSrc = failed ? undefined : observationThumbnailSrc(currentTeamId, observation, { chapter, size })
+ const src = baseSrc !== undefined && attempt > 0 ? `${baseSrc}&retry=${attempt}` : baseSrc
+ const loaded = src !== undefined && (loadedSrc === src || isDecoded(src))
+
+ const onError = (): void => {
+ // One retry: the redirect target can blink, and the poster would stay blank for the whole mount.
+ if (attempt === 0) {
+ retryTimer.current = setTimeout(() => setAttempt(1), RETRY_DELAY_MS)
+ return
+ }
+ setFailedId(observation.id)
+ }
+
+ return (
+ <div className={clsx('relative aspect-video overflow-hidden rounded border border-border bg-muted', className)}>
+ {src && (
+ <>
+ {!loaded && <Skeleton className="absolute inset-0 size-full rounded-none" />}
+ <img
+ src={src}
+ alt=""
+ className={clsx(
+ 'absolute inset-0 size-full object-cover transition-opacity duration-300 motion-reduce:transition-none',
+ loaded ? 'opacity-100' : 'opacity-0'
+ )}
+ loading={loaded ? 'eager' : 'lazy'}
+ decoding="async"
+ onLoad={() => {
+ decodedThumbnails.set(src, Date.now())
+ setLoadedSrc(src)
+ }}
+ onError={onError}
+ />
+ </>
+ )}
+ <div className="absolute inset-0 flex items-center justify-center">
+ {children ?? (!src && <IconVideoCamera className="text-xl text-muted-foreground" aria-hidden />)}
+ </div>
+ </div>
+ )
+}
diff --git a/products/replay_vision/frontend/components/QuillRecordingMoments.tsx b/products/replay_vision/frontend/components/QuillRecordingMoments.tsx
new file mode 100644
index 000000000000..91b5722719c1
--- /dev/null
+++ b/products/replay_vision/frontend/components/QuillRecordingMoments.tsx
@@ -0,0 +1,184 @@
+import { useActions, useValues } from 'kea'
+import { useEffect, useMemo, useRef } from 'react'
+
+import { IconChevronRight } from '@posthog/icons'
+import { Button, Switch } from '@posthog/quill'
+
+import { useResizeObserver } from 'lib/hooks/useResizeObserver'
+import {
+ SessionRecordingPlayerLogicProps,
+ sessionRecordingPlayerLogic,
+} from 'scenes/session-recordings/player/sessionRecordingPlayerLogic'
+
+import { observationsDockLogic } from '../logics/observationsDockLogic'
+import { visionQuotaLogic } from '../logics/visionQuotaLogic'
+import { quotaUx } from '../utils/quotaProjection'
+import {
+ currentRowIndex,
+ nextTimelineStopMs,
+ timelineEndMs,
+ timelineRows as buildTimelineRows,
+} from '../utils/recordingTimeline'
+import { recordingScanBlock } from '../utils/scanEligibility'
+import { ChapterFilmstrip } from './ChapterFilmstrip'
+import { QuillRecordingTimeline } from './QuillRecordingTimeline'
+import { QuillScannerPicker } from './QuillScannerPicker'
+import { ScanList } from './ScanList'
+
+const CURRENT_MOMENT_SELECTOR = '[data-current-moment]'
+const FILMSTRIP_MIN_WIDTH_PX = 840
+
+function SectionHeader({ label }: { label: string }): JSX.Element {
+ return (
+ <div className="flex items-center gap-2 px-3 py-1.5 border-y border-border bg-muted text-xs font-semibold uppercase tracking-wide text-muted-foreground">
+ {label}
+ </div>
+ )
+}
+
+// Aligned to the card's start, a snap point; a smaller scroll gets snapped back and the card stays clipped.
+function scrollIntoStrip(card: HTMLElement): void {
+ const strip = card.parentElement
+ if (!strip) {
+ return
+ }
+ const cardRect = card.getBoundingClientRect()
+ const stripRect = strip.getBoundingClientRect()
+ if (cardRect.left < stripRect.left || cardRect.right > stripRect.right) {
+ strip.scrollBy({ left: cardRect.left - stripRect.left })
+ }
+}
+
+/** The observations tab body once observations exist; the host shows the empty and loading states. */
+export function QuillRecordingMoments({
+ sessionId,
+ playerLogicProps,
+}: {
+ sessionId: string
+ playerLogicProps: SessionRecordingPlayerLogicProps
+}): JSX.Element {
+ const logic = observationsDockLogic({ sessionId })
+ const { observations, timeline, followMoments, summarizePending } = useValues(logic)
+ const { setFollowMoments, summarize } = useActions(logic)
+ const { quota } = useValues(visionQuotaLogic)
+ const { disabledReason: quotaDisabledReason } = quotaUx(quota)
+ const lastJumpMs = useRef<number | null>(null)
+ const timelineRef = useRef<HTMLDivElement>(null)
+ const followedIndex = useRef<number | null>(null)
+ const { width, ref: rootRef } = useResizeObserver<HTMLDivElement>()
+ const { logicProps, sessionPlayerMetaData, currentPlayerTime, sessionPlayerData } = useValues(
+ sessionRecordingPlayerLogic(playerLogicProps)
+ )
+ const durationMs = sessionPlayerData.durationMs > 0 ? sessionPlayerData.durationMs : null
+ const timelineRows = useMemo(() => buildTimelineRows(timeline, durationMs), [timeline, durationMs])
+ const seekToTime = (ms: number): void => {
+ sessionRecordingPlayerLogic.findMounted(logicProps)?.actions.seekToTime(ms)
+ }
+ const scanBlock = recordingScanBlock(sessionPlayerMetaData)
+ const filmstrip = width !== undefined && width >= FILMSTRIP_MIN_WIDTH_PX
+ // A seek can land short of its target. Step from the last jump while still on it.
+ const nextFrom =
+ lastJumpMs.current !== null && Math.abs(currentPlayerTime - lastJumpMs.current) < 1000
+ ? lastJumpMs.current
+ : currentPlayerTime
+ const nextStopMs = nextTimelineStopMs(timeline, nextFrom)
+ const jumpToNext = (): void => {
+ if (nextStopMs === null) {
+ return
+ }
+ lastJumpMs.current = nextStopMs
+ seekToTime(nextStopMs)
+ }
+ const currentIndex = currentRowIndex(timelineRows, currentPlayerTime)
+
+ useEffect(() => {
+ followedIndex.current = null
+ }, [filmstrip])
+
+ useEffect(() => {
+ if (!followMoments) {
+ followedIndex.current = null
+ return
+ }
+ if (currentIndex < 0 || followedIndex.current === currentIndex) {
+ return
+ }
+ const current = timelineRef.current?.querySelector<HTMLElement>(CURRENT_MOMENT_SELECTOR)
+ if (!current) {
+ return
+ }
+ followedIndex.current = currentIndex
+ if (filmstrip) {
+ scrollIntoStrip(current)
+ } else {
+ current.scrollIntoView({ block: 'nearest' })
+ }
+ }, [currentIndex, followMoments, filmstrip, width])
+
+ const timelineProps = {
+ timeline,
+ rows: timelineRows,
+ currentTimeMs: currentPlayerTime,
+ onSeek: seekToTime,
+ onSummarize: () => summarize(true),
+ summarizing: summarizePending,
+ summarizeDisabledReason: scanBlock?.reason ?? quotaDisabledReason,
+ }
+
+ return (
+ // A container, so long observation text wraps at the panel's width instead of widening it.
+ <div ref={rootRef} className="@container flex flex-col flex-1 min-h-0" data-attr="vision-observations-tab">
+ <div className="flex items-center gap-2 p-2 border-b border-border bg-muted">
+ <QuillScannerPicker sessionId={sessionId} scanBlock={scanBlock} variant="outline" />
+ {timelineRows.length > 0 && (
+ <div className="ml-auto flex items-center gap-2">
+ <label className="flex items-center gap-1.5 text-xs">
+ <Switch
+ size="sm"
+ checked={followMoments}
+ onCheckedChange={setFollowMoments}
+ data-attr="vision-follow-moments"
+ />
+ Follow
+ </label>
+ <span className="hidden @[400px]:block">
+ <Button
+ size="sm"
+ variant="outline"
+ disabledReason={nextStopMs !== null ? undefined : 'Nothing later in the recording'}
+ onClick={jumpToNext}
+ data-attr="vision-next-moment"
+ >
+ Next
+ <IconChevronRight />
+ </Button>
+ </span>
+ </div>
+ )}
+ </div>
+ <div className="flex-1 min-h-0 overflow-y-auto" ref={timelineRef}>
+ {/* Nothing before the width is known, so posters are fetched once. */}
+ {width === undefined ? null : filmstrip ? (
+ <div data-not-quill className="contents">
+ <ChapterFilmstrip {...timelineProps} />
+ </div>
+ ) : (
+ <>
+ <SectionHeader label="Timeline" />
+ <QuillRecordingTimeline {...timelineProps} />
+ </>
+ )}
+ <SectionHeader label="Scans" />
+ <div data-not-quill className="contents">
+ <ScanList
+ sessionId={sessionId}
+ observations={observations}
+ endMs={timelineEndMs(timeline, durationMs)}
+ currentTimeMs={currentPlayerTime}
+ onSeek={seekToTime}
+ />
+ </div>
+ </div>
+ </div>
+ )
+}
diff --git a/products/replay_vision/frontend/components/QuillRecordingTimeline.tsx b/products/replay_vision/frontend/components/QuillRecordingTimeline.tsx
new file mode 100644
index 000000000000..9f872c763ff8
--- /dev/null
+++ b/products/replay_vision/frontend/components/QuillRecordingTimeline.tsx
@@ -0,0 +1,214 @@
+import { Fragment } from 'react'
+
+import { IconSparkles } from '@posthog/icons'
+import { Button, Spinner, Tooltip, TooltipContent, TooltipTrigger } from '@posthog/quill'
+
+import { cn } from 'lib/utils/css-classes'
+import { colonDelimitedDuration, humanFriendlyDuration } from 'lib/utils/durations'
+
+import type { ReplayObservationApi } from '../generated/api.schemas'
+import type { RecordingTimeline as RecordingTimelineData, TimelineRow } from '../utils/recordingTimeline'
+import { MIN_INACTIVE_ROW_MS, currentRowIndex, rowStartMs, timelineGapPx } from '../utils/recordingTimeline'
+import { QuillObservationThumbnail } from './QuillObservationThumbnail'
+import { QuillTimelineRail } from './QuillTimelineRail'
+
+export interface QuillRecordingTimelineProps {
+ timeline: RecordingTimelineData
+ rows: TimelineRow[]
+ currentTimeMs: number
+ onSeek: (timestampMs: number) => void
+ onSummarize: () => void
+ summarizing: boolean
+ summarizeDisabledReason?: string | null
+}
+
+const ROW_GRID = 'grid grid-cols-[2.5rem_1rem_minmax(0,1fr)] gap-x-1.5 px-2'
+const ROW_GRID_HOURS = 'grid grid-cols-[4rem_1rem_minmax(0,1fr)] gap-x-1.5 px-2'
+const HOUR_MS = 3_600_000
+
+const formatSpan = (ms: number): string => humanFriendlyDuration(ms / 1000, { maxUnits: 1 })
+
+export function QuillSummaryNotice({
+ timeline,
+ onSummarize,
+ summarizing,
+ summarizeDisabledReason,
+}: QuillRecordingTimelineProps): JSX.Element | null {
+ if (timeline.summaryState === 'ready') {
+ return null
+ }
+ if (timeline.summaryState === 'pending') {
+ return (
+ <div
+ className="flex items-center gap-2 px-3 py-2 text-xs text-muted-foreground"
+ data-attr="vision-timeline-pending"
+ >
+ <Spinner /> Building the timeline of this recording…
+ </div>
+ )
+ }
+ const outdated = timeline.summaryState === 'outdated'
+ return (
+ <div className="flex flex-col items-start gap-2 px-3 py-3" data-attr="vision-timeline-cta">
+ <span className="text-xs text-muted-foreground">
+ {outdated
+ ? 'This recording was summarized before timelines existed.'
+ : 'Summarize this recording to get a timeline of what happened.'}
+ </span>
+ <Button
+ size="sm"
+ variant="outline"
+ onClick={onSummarize}
+ loading={summarizing}
+ disabledReason={summarizeDisabledReason}
+ data-attr={outdated ? 'vision-timeline-rebuild' : 'vision-timeline-summarize'}
+ >
+ <IconSparkles />
+ {outdated ? 'Rebuild the timeline' : 'Summarize this recording'}
+ </Button>
+ </div>
+ )
+}
+
+/** The recording's timeline on a rail: the summary's chapters and the idle gaps between them. */
+export function QuillRecordingTimeline(props: QuillRecordingTimelineProps): JSX.Element {
+ const { timeline, rows, currentTimeMs, onSeek } = props
+ const currentIndex = currentRowIndex(rows, currentTimeMs)
+ const hasHours = rows.some((row) => rowStartMs(row) >= HOUR_MS)
+ const rowGrid = hasHours ? ROW_GRID_HOURS : ROW_GRID
+ const formatTime = (ms: number): string => colonDelimitedDuration(Math.floor(ms / 1000), hasHours ? 3 : 2)
+ const summary = timeline.summary as ReplayObservationApi
+
+ return (
+ <div className="flex flex-col py-1" data-attr="vision-recording-timeline">
+ {rows.map((row, index) => {
+ const startMs = rowStartMs(row)
+ const isCurrent = index === currentIndex
+ const passedAbove = startMs <= currentTimeMs
+ const passedBelow = index < currentIndex
+ const gapPx = index === 0 ? 0 : timelineGapPx(startMs - rowStartMs(rows[index - 1]))
+ const inactive = row.kind === 'inactive'
+ const rail = { first: index === 0, last: index === rows.length - 1, passedAbove, passedBelow }
+ const timeCell = (
+ <span
+ className={cn(
+ 'text-xs font-mono text-right',
+ row.kind === 'chapter' ? 'pt-2' : 'self-center py-1',
+ isCurrent ? 'text-primary font-semibold' : 'text-muted-foreground',
+ inactive && !isCurrent && 'opacity-70'
+ )}
+ >
+ {formatTime(startMs)}
+ </span>
+ )
+ return (
+ <Fragment key={`${row.kind}-${startMs}-${index}`}>
+ {gapPx > 0 && (
+ <div
+ className={rowGrid}
+ // eslint-disable-next-line react/forbid-dom-props
+ style={{ height: gapPx }}
+ >
+ <span />
+ <QuillTimelineRail
+ dot="none"
+ passedAbove={passedAbove}
+ passedBelow={passedAbove}
+ isCurrent={false}
+ dashed={rows[index - 1].kind === 'inactive'}
+ />
+ </div>
+ )}
+ {row.kind === 'chapter' ? (
+ <div
+ data-current-moment={isCurrent ? true : undefined}
+ className={cn(rowGrid, 'transition-colors', isCurrent && 'bg-fill-selected')}
+ >
+ {timeCell}
+ <QuillTimelineRail dot="chapter" isCurrent={isCurrent} alignTop {...rail} />
+ <Button
+ size="row"
+ left
+ onClick={() => onSeek(row.chapter.startMs)}
+ className="h-auto items-start gap-2 min-w-0 p-1 -mx-1 my-0.5"
+ data-attr="vision-timeline-chapter"
+ >
+ {row.chapter.hasFrame ? (
+ <Tooltip>
+ <TooltipTrigger render={<span className="flex shrink-0" />}>
+ <QuillObservationThumbnail
+ observation={summary}
+ chapter={row.chapter.position}
+ className="w-20 shrink-0"
+ size="small"
+ />
+ </TooltipTrigger>
+ <TooltipContent side="left" className="max-w-none p-1">
+ <QuillObservationThumbnail
+ observation={summary}
+ chapter={row.chapter.position}
+ className="w-128 border-0"
+ />
+ </TooltipContent>
+ </Tooltip>
+ ) : (
+ <QuillObservationThumbnail
+ observation={summary}
+ chapter={row.chapter.position}
+ className="w-20 shrink-0"
+ size="small"
+ />
+ )}
+ <span className="flex flex-col min-w-0 gap-0.5 text-left whitespace-normal">
+ <span className="text-sm font-medium line-clamp-2">{row.chapter.title}</span>
+ <span className="text-xs text-muted-foreground">
+ {formatSpan(
+ row.chapter.endMs - row.chapter.startMs - row.chapter.inactiveMs
+ )}
+ {row.chapter.inactiveMs >= MIN_INACTIVE_ROW_MS && (
+ <span className="opacity-70">
+ {' '}
+ · {formatSpan(row.chapter.inactiveMs)} inactive
+ </span>
+ )}
+ </span>
+ </span>
+ </Button>
+ </div>
+ ) : row.kind === 'boundary' ? (
+ <div
+ data-current-moment={isCurrent ? true : undefined}
+ className={cn(rowGrid, isCurrent && 'bg-fill-selected')}
+ data-attr="vision-timeline-boundary"
+ >
+ {timeCell}
+ <QuillTimelineRail dot="boundary" isCurrent={false} {...rail} />
+ <Button
+ variant="link-muted"
+ size="inline"
+ className="justify-start text-xs py-1"
+ onClick={() => onSeek(row.atMs)}
+ >
+ {row.edge === 'start' ? 'Session start' : 'Session end'}
+ </Button>
+ </div>
+ ) : (
+ <div
+ data-current-moment={isCurrent ? true : undefined}
+ className={rowGrid}
+ data-attr="vision-timeline-inactive"
+ >
+ {timeCell}
+ <QuillTimelineRail dot="none" isCurrent={false} dashed {...rail} />
+ <span className="text-xs text-muted-foreground italic py-1">
+ Inactive for {formatSpan(row.endMs - row.startMs)}
+ </span>
+ </div>
+ )}
+ </Fragment>
+ )
+ })}
+ <QuillSummaryNotice {...props} />
+ </div>
+ )
+}
diff --git a/products/replay_vision/frontend/components/QuillTimelineRail.tsx b/products/replay_vision/frontend/components/QuillTimelineRail.tsx
new file mode 100644
index 000000000000..207d1c6843f9
--- /dev/null
+++ b/products/replay_vision/frontend/components/QuillTimelineRail.tsx
@@ -0,0 +1,54 @@
+import { cn } from 'lib/utils/css-classes'
+
+import type { TimelineRailDot } from './TimelineRail'
+
+/** The rail cell of one timeline row: the line through it, and the dot that marks where the row sits. */
+export function QuillTimelineRail({
+ dot,
+ passedAbove,
+ passedBelow,
+ isCurrent,
+ dashed = false,
+ alignTop = false,
+ first = false,
+ last = false,
+}: {
+ dot: TimelineRailDot
+ passedAbove: boolean
+ passedBelow: boolean
+ isCurrent: boolean
+ dashed?: boolean
+ alignTop?: boolean
+ first?: boolean
+ last?: boolean
+}): JSX.Element {
+ const segment = (passed: boolean, grow: boolean, hidden: boolean): string =>
+ cn(
+ grow ? 'flex-1' : 'h-3.5 shrink-0',
+ hidden
+ ? 'w-0.5'
+ : dashed
+ ? cn('w-0 border-l-2 border-dotted', passed ? 'border-primary' : 'border-border')
+ : cn('w-0.5', passed ? 'bg-primary' : 'bg-border')
+ )
+ return (
+ <div className="flex flex-col items-center self-stretch">
+ <span className={segment(passedAbove, !alignTop, first)} />
+ {dot !== 'none' && (
+ <span
+ className={cn(
+ 'rounded-full shrink-0',
+ dot === 'chapter' ? 'w-3 h-3' : 'w-2 h-2',
+ dot === 'boundary'
+ ? cn('rounded-sm', passedAbove ? 'bg-primary' : 'bg-border')
+ : passedAbove
+ ? 'bg-primary'
+ : 'bg-background border-2 border-border',
+ isCurrent && 'ring-2 ring-primary ring-offset-1'
+ )}
+ />
+ )}
+ <span className={segment(passedBelow, true, last)} />
+ </div>
+ )
+}