:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { useMountedLogic, useValues } from 'kea'
import { LemonBanner } from '@posthog/lemon-ui'
import { lemonDialogLogic } from 'lib/lemon-ui/LemonDialog/lemonDialogLogic'
import { LemonField } from 'lib/lemon-ui/LemonField'
import { LemonInputSelect } from 'lib/lemon-ui/LemonInputSelect'
import { LemonRadio, LemonRadioOption } from 'lib/lemon-ui/LemonRadio'
import { LemonSelect } from 'lib/lemon-ui/LemonSelect'
import { LemonSkeleton } from 'lib/lemon-ui/LemonSkeleton'
import { LemonTag } from 'lib/lemon-ui/LemonTag'
import { DataWarehouseSavedQueryIncrementalCheck } from '~/types'
import { saveViewIncrementalLogic } from 'products/data_warehouse/frontend/sql_editor/saveViewIncrementalLogic'
export const LOOKBACK_OPTIONS = [
{ value: 0, label: 'No lookback' },
{ value: 60 * 60, label: '1 hour' },
{ value: 60 * 60 * 24, label: '1 day' },
{ value: 60 * 60 * 24 * 7, label: '7 days' },
]
type RefreshMode = 'full_refresh' | 'incremental'
const REFRESH_MODE_OPTIONS: LemonRadioOption<RefreshMode>[] = [
{
value: 'full_refresh',
label: 'Full refresh',
description: 'Each run rebuilds the whole table from scratch.',
},
{
value: 'incremental',
label: 'Incremental',
description: 'Each run adds or updates only rows that are new since the last run.',
},
]
const INCREMENTAL_KEY_LABEL = 'Incremental column'
const INCREMENTAL_KEY_HELP =
"The column that tracks which rows are new: its value grows as rows arrive, like a timestamp or a sequential ID. Each run reads only rows at or after the last run's highest value."
const UNIQUE_KEY_LABEL = 'Unique key'
const UNIQUE_KEY_HELP =
'The columns that together identify a row, like a primary key. Rows that match on them are updated in place. Include every column the query groups by. Values here can never be empty.'
const LOOKBACK_HELP = 'Include this much previously processed data in each run to catch late arrivals and updates.'
interface IncrementalConfigFieldsProps {
queries: string[]
}
/** What the user has picked so far, before it becomes a saved incremental config. */
export interface IncrementalConfigDraft {
enabled: boolean
incrementalKey: string | null
uniqueKey: string[]
lookbackSeconds: number
}
export const EMPTY_INCREMENTAL_DRAFT: IncrementalConfigDraft = {
enabled: false,
incrementalKey: null,
uniqueKey: [],
lookbackSeconds: 0,
}
function IneligibleBanner({ check }: { check: DataWarehouseSavedQueryIncrementalCheck }): JSX.Element {
return (
<LemonBanner type="info" className="mt-2">
<span className="text-xs">
This query is always refreshed in full.{' '}
{check.blockers[0] ?? 'It has no column that can track which rows are new.'}
</span>
</LemonBanner>
)
}
function WarningsBanner({ warnings }: { warnings: string[] }): JSX.Element | null {
if (warnings.length === 0) {
return null
}
return (
<LemonBanner type="warning">
<ul className="text-xs deprecated-space-y-1">
{warnings.map((warning) => (
<li key={warning}>{warning}</li>
))}
</ul>
</LemonBanner>
)
}
function RefreshModeRadio({
incremental,
onChange,
dataAttr,
}: {
incremental: boolean
onChange: (incremental: boolean) => void
dataAttr: string
}): JSX.Element {
return (
<LemonRadio
value={incremental ? 'incremental' : 'full_refresh'}
onChange={(mode: RefreshMode) => onChange(mode === 'incremental')}
options={REFRESH_MODE_OPTIONS.map((option) => ({
...option,
'data-attr': `${dataAttr}-${option.value}`,
}))}
radioPosition="top"
/>
)
}
function ColumnWithType({ column, columnType }: { column: string; columnType?: string }): JSX.Element {
return (
<span className="flex items-center gap-2">
{column}
{columnType && <LemonTag type="muted">{columnType}</LemonTag>}
</span>
)
}
function IncrementalKeyInput({
check,
value,
onChange,
dataAttr,
}: {
check: DataWarehouseSavedQueryIncrementalCheck
value: string | null
onChange: (value: string | null) => void
dataAttr: string
}): JSX.Element {
return (
<LemonSelect
value={value}
onChange={onChange}
options={check.key_candidates.map((column) => ({
value: column,
label: <ColumnWithType column={column} columnType={check.key_candidate_types?.[column]} />,
}))}
placeholder="Select a column"
data-attr={dataAttr}
size="small"
renderButtonContent={(option) => option?.value ?? 'Select a column'}
fullWidth
/>
)
}
function UniqueKeyInput({
check,
value,
onChange,
dataAttr,
}: {
check: DataWarehouseSavedQueryIncrementalCheck
value: string[]
onChange: (value: string[]) => void
dataAttr: string
}): JSX.Element {
// Wider than key_candidates: identifying a row only needs equality, so strings qualify here
// even though they cannot track new rows.
const candidates = check.unique_key_candidates ?? check.key_candidates
return (
<LemonInputSelect
mode="multiple"
size="small"
fullWidth
transparentBackground
value={value}
onChange={onChange}
options={candidates.map((column) => ({
key: column,
label: column,
tooltip: check.key_candidate_types?.[column],
}))}
placeholder="Add column"
data-attr={dataAttr}
/>
)
}
function LookbackInput({ value, onChange }: { value: number; onChange: (value: number) => void }): JSX.Element {
return <LemonSelect value={value} onChange={onChange} options={LOOKBACK_OPTIONS} size="small" fullWidth />
}
interface IncrementalConfigOptionsProps {
/** Result of the backend eligibility check for the view's query. */
check: DataWarehouseSavedQueryIncrementalCheck | null
draft: IncrementalConfigDraft
onChange: (draft: Partial<IncrementalConfigDraft>) => void
}
export const INCREMENTAL_CHECK_LOADING: DataWarehouseSavedQueryIncrementalCheck = {
eligible: false,
key_candidates: [],
blockers: [],
warnings: [],
}
/**
* Controlled version of the incremental settings, for surfaces without a kea form —
* the materialization panel and modal.
*/
export function IncrementalConfigOptions({
check,
draft,
onChange,
}: IncrementalConfigOptionsProps): JSX.Element | null {
if (check === INCREMENTAL_CHECK_LOADING) {
return (
<div className="mt-4 deprecated-space-y-2" data-attr="materialization-refresh-mode-loading">
<LemonSkeleton className="h-8 w-full" repeat={2} />
</div>
)
}
if (!check) {
return null
}
if (!check.eligible || check.key_candidates.length === 0) {
return <IneligibleBanner check={check} />
}
return (
<div className="mt-4 deprecated-space-y-4">
<RefreshModeRadio
incremental={draft.enabled}
onChange={(enabled) => onChange({ enabled })}
dataAttr="materialization-refresh-mode"
/>
{draft.enabled && (
<div className="@container/incremental mt-4 ml-6 max-w-120 space-y-3">
<LemonField.Pure
className="@min-[400px]/incremental:flex-row @min-[400px]/incremental:items-center"
labelClassName="@min-[400px]/incremental:w-44 @min-[400px]/incremental:shrink-0"
label={INCREMENTAL_KEY_LABEL}
info={INCREMENTAL_KEY_HELP}
>
<div className="min-w-0 w-full flex-1">
<IncrementalKeyInput
check={check}
value={draft.incrementalKey}
onChange={(incrementalKey) => onChange({ incrementalKey })}
dataAttr="materialization-incremental-key"
/>
</div>
</LemonField.Pure>
<LemonField.Pure
className="@min-[400px]/incremental:flex-row @min-[400px]/incremental:items-center"
labelClassName="@min-[400px]/incremental:w-44 @min-[400px]/incremental:shrink-0"
label={UNIQUE_KEY_LABEL}
info={UNIQUE_KEY_HELP}
>
<div className="min-w-0 w-full flex-1">
<UniqueKeyInput
check={check}
value={draft.uniqueKey}
onChange={(uniqueKey) => onChange({ uniqueKey })}
dataAttr="materialization-incremental-unique-key"
/>
</div>
</LemonField.Pure>
<LemonField.Pure
className="@min-[400px]/incremental:flex-row @min-[400px]/incremental:items-center"
labelClassName="@min-[400px]/incremental:w-44 @min-[400px]/incremental:shrink-0"
label="Lookback window"
info={LOOKBACK_HELP}
>
<div className="min-w-0 w-full flex-1">
<LookbackInput
value={draft.lookbackSeconds}
onChange={(lookbackSeconds) => onChange({ lookbackSeconds })}
/>
</div>
</LemonField.Pure>
<WarningsBanner warnings={check.warnings} />
</div>
)}
</div>
)
}
/**
* Incremental settings inside the save-as-view form.
*
* Rendered only when the view is being materialized, since incremental describes how a
* materialized table is refreshed and means nothing without one.
*/
export function IncrementalConfigFields({ queries }: IncrementalConfigFieldsProps): JSX.Element | null {
const formLogic = useMountedLogic(lemonDialogLogic)
const { form } = useValues(formLogic)
const { check, checkLoading } = useValues(
saveViewIncrementalLogic({
query: queries[Number(form.saveTarget)] ?? '',
dialogKey: String(formLogic.key),
enabled: Boolean(form.materializeAfterSave),
})
)
if (!form.materializeAfterSave) {
return null
}
if (checkLoading) {
return (
<div className="mt-4 deprecated-space-y-2" data-attr="sql-editor-save-view-incremental-loading">
<LemonSkeleton className="h-8 w-full" repeat={2} />
</div>
)
}
if (!check) {
return (
<LemonBanner type="info" className="mt-2">
Incremental options are unavailable. You can still save this view with a full refresh.
</LemonBanner>
)
}
if (!check.eligible || check.key_candidates.length === 0) {
return <IneligibleBanner check={check} />
}
return (
<LemonField name="incrementalEnabled" className="mt-2">
{({ value, onChange }) => (
<>
<RefreshModeRadio
incremental={!!value}
onChange={onChange}
dataAttr="sql-editor-save-view-refresh-mode"
/>
{value && (
<div className="mt-4 deprecated-space-y-4">
<LemonField name="incrementalKey" label={INCREMENTAL_KEY_LABEL} info={INCREMENTAL_KEY_HELP}>
{({ value: key, onChange: onKeyChange }) => (
<IncrementalKeyInput
check={check}
value={key}
onChange={onKeyChange}
dataAttr="sql-editor-input-save-view-incremental-key"
/>
)}
</LemonField>
<LemonField name="incrementalUniqueKey" label={UNIQUE_KEY_LABEL} info={UNIQUE_KEY_HELP}>
{({ value: uniqueKey, onChange: onUniqueKeyChange }) => (
<UniqueKeyInput
check={check}
value={uniqueKey}
onChange={onUniqueKeyChange}
dataAttr="sql-editor-input-save-view-incremental-unique-key"
/>
)}
</LemonField>
<LemonField name="incrementalLookbackSeconds" label="Lookback window" info={LOOKBACK_HELP}>
{({ value: lookback, onChange: onLookbackChange }) => (
<LookbackInput value={lookback} onChange={onLookbackChange} />
)}
</LemonField>
<WarningsBanner warnings={check.warnings} />
</div>
)}
</>
)}
</LemonField>
)
}