:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { useActions, useValues } from 'kea'
import { IconRefresh } from '@posthog/icons'
import { LemonBanner, LemonCard, LemonSelect, LemonSkeleton, LemonTable, Link } from '@posthog/lemon-ui'
import { TZLabel } from 'lib/components/TZLabel'
import { FEATURE_FLAGS } from 'lib/constants'
import { LemonButton } from 'lib/lemon-ui/LemonButton'
import { LemonProgress } from 'lib/lemon-ui/LemonProgress'
import { LemonTag, LemonTagType } from 'lib/lemon-ui/LemonTag'
import { Tooltip } from 'lib/lemon-ui/Tooltip'
import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { getAccessControlDisabledReason } from 'lib/utils/accessControlUtils'
import { humanFriendlyDetailedTime } from 'lib/utils/datetime'
import { humanFriendlyNumber } from 'lib/utils/numbers'
import { LogsViewer } from 'scenes/hog-functions/logs/LogsViewer'
import { teamLogic } from 'scenes/teamLogic'
import { userLogic } from 'scenes/userLogic'
import { AccessControlLevel, AccessControlResourceType, LogEntryLevel } from '~/types'
import { SERVING_ENGINE, suspendedScheduleMessage } from 'products/data_modeling/frontend/suspension'
import {
DataModelingJobsListEngine,
type DataModelingJobApi,
} from 'products/data_warehouse/frontend/generated/api.schemas'
import { MaterializationLoading } from 'products/data_warehouse/frontend/shared/components/MaterializationLoading'
import { MaterializationRunActions } from 'products/data_warehouse/frontend/shared/components/MaterializationRunActions'
import { MaterializationRunError } from 'products/data_warehouse/frontend/shared/components/MaterializationRunError'
import { IncrementalConfigOptions } from '../editor/IncrementalConfigFields'
import { dataWarehouseViewsLogic } from './dataWarehouseViewsLogic'
import { DEFAULT_JOBS_PAGE_SIZE, materializationJobsLogic } from './materializationJobsLogic'
import { computeJobDuration, jobLogsWindow } from './materializationJobUtils'
import {
SyncFrequencySelect,
SyncFrequencyValue,
defaultCadenceWithin,
modeDisabledReason,
} from './SyncFrequencySelect'
const STATUS_TAG_TYPES: Record<string, LemonTagType> = {
Completed: 'success',
Failed: 'danger',
Running: 'warning',
Cancelled: 'muted',
Skipped: 'muted',
}
const LOG_LEVELS: LogEntryLevel[] = ['LOG', 'INFO', 'WARN', 'WARNING', 'ERROR']
interface MaterializationStatusPanelProps {
viewId: string
/**
* The product surface this panel is rendered in. Endpoints own the materialization lifecycle of their
* backing saved query, so destructive saved-query controls (revert, sync frequency) must be hidden
* when `kind === 'endpoint'` — those mutations bypass the endpoint's `_disable_materialization` flow.
*/
kind?: 'view' | 'endpoint'
/** Drops the "Materialization" heading where the surface already names the panel, such as a tab. */
hideTitle?: boolean
showRunActions?: boolean
showStatusSummary?: boolean
}
// Watermarks are only ISO strings for date/datetime incremental keys. Numeric and arbitrary
// string keys must render as-is: pushing them through a date formatter shows a bogus timestamp.
const ISO_DATE_PREFIX = /^\d{4}-\d{2}-\d{2}([T ]|$)/
function formatWatermark(watermark: string | null | undefined): string {
if (watermark == null || watermark === '') {
return 'the last run'
}
return ISO_DATE_PREFIX.test(watermark) ? humanFriendlyDetailedTime(watermark) : watermark
}
function getMaterializationStatusMessage(
rowsMaterialized: number,
progressPercentage: number,
rowsExpected: number
): string {
const percentComplete = Math.round(Math.min(100, (rowsMaterialized / rowsExpected) * 100))
switch (true) {
case rowsMaterialized === 0:
return `Spinning up spikes — starting materialization job... ${percentComplete}% complete.`
case progressPercentage < 10:
return `Digging into SQL... executing your query now... ${percentComplete}% complete.`
case progressPercentage < 25:
return `First ${humanFriendlyNumber(rowsMaterialized)} rows tucked away... ${percentComplete}% complete.`
case progressPercentage < 50:
return `${humanFriendlyNumber(rowsMaterialized)} rows shipped to storage... ${percentComplete}% complete.`
case progressPercentage < 90:
return `Still going — ${humanFriendlyNumber(
rowsMaterialized
)} rows written... ${percentComplete}% complete.`
case progressPercentage === 100:
return `Wrapping up — ${humanFriendlyNumber(
rowsMaterialized
)} rows processed... ${percentComplete}% complete.`
default:
return `Almost there — ${humanFriendlyNumber(
rowsMaterialized
)} rows processed... ${percentComplete}% complete.`
}
}
function getMaterializationDisabledReasons(
currentJobStatus: string | null,
startingMaterialization: boolean
): {
sync: string | false
cancel: string | false
revert: string | false
} {
return {
sync:
currentJobStatus === 'Running'
? 'Materialization is currently running'
: startingMaterialization
? 'Materialization is starting'
: false,
cancel: currentJobStatus !== 'Running' ? 'Materialization is not running' : false,
revert: currentJobStatus === 'Running' ? 'Cannot revert while materialization is running' : false,
}
}
export function MaterializationStatusPanel({
viewId,
kind = 'view',
hideTitle,
showRunActions = true,
showStatusSummary = true,
}: MaterializationStatusPanelProps): JSX.Element {
const jobsLogic = materializationJobsLogic({ viewId, kind })
const {
dataModelingJobs,
dataModelingJobsLoading,
jobsPage,
jobsPageResults,
runHistoryLoading,
runHistoryError,
selectedRunHistoryEngine,
lastSuccessfulSyncAt,
startingMaterialization,
savedQuery,
savedQueryLoading,
initialSyncFrequency,
incrementalCheck,
incrementalDraft,
syncFrequencyDraft,
savingMaterialization,
cadencePin,
nodeFrequency,
nodeFrequencyLoading,
} = useValues(jobsLogic)
const {
loadDataModelingJobs,
loadRunHistoryJobs,
setJobsPage,
loadOlderJobsPage,
setSelectedRunHistoryEngine,
setInitialSyncFrequency,
setIncrementalDraft,
setSyncFrequencyDraft,
setPinDraft,
} = useActions(jobsLogic)
const { updatingDataWarehouseSavedQuery, materializationActionLoading } = useValues(dataWarehouseViewsLogic)
const { featureFlags } = useValues(featureFlagLogic)
const managedWarehouseEnabled = !!featureFlags[FEATURE_FLAGS.MANAGED_TRINO_QUERY]
const { timezone } = useValues(teamLogic)
const { user } = useValues(userLogic)
const showDebugLogs = user?.is_staff || user?.is_impersonated
const materializationAccessReason = getAccessControlDisabledReason(
AccessControlResourceType.WarehouseObjects,
AccessControlLevel.Editor,
savedQuery?.user_access_level
)
if (!savedQuery) {
return (
<div className="min-h-64" data-attr="materialization-status-panel">
{savedQueryLoading ? <MaterializationLoading /> : null}
</div>
)
}
const currentJobStatus = dataModelingJobs?.results?.[0]?.status || null
const { sync } = getMaterializationDisabledReasons(currentJobStatus, startingMaterialization)
const supportsIncremental = kind !== 'endpoint'
const showIncremental = supportsIncremental && !!savedQuery.incremental?.enabled
const lastRunMode = savedQuery.incremental_state?.last_run_mode
const savedIncremental = savedQuery.incremental
// Key or unique-key edits change what the stored rows mean, so the next run rebuilds (via the
// definition fingerprint). A lookback-only change is operational and does not.
const structuralChange = savedIncremental?.enabled
? !incrementalDraft.enabled ||
incrementalDraft.incrementalKey !== savedIncremental.incremental_key ||
[...incrementalDraft.uniqueKey].sort().join(',') !== [...savedIncremental.unique_key].sort().join(',')
: incrementalDraft.enabled
const lookbackChanged =
!!savedIncremental?.enabled &&
incrementalDraft.enabled &&
incrementalDraft.lookbackSeconds !== (savedIncremental.lookback_seconds ?? 0)
const refreshModeChanged = structuralChange || lookbackChanged
const startingFrequency = defaultCadenceWithin(savedQuery.sync_frequency_bounds, initialSyncFrequency)
const cadenceOwnedElsewhere = modeDisabledReason(savedQuery.sync_frequency_bounds)
const isPaused = !cadenceOwnedElsewhere && (!savedQuery.sync_frequency || savedQuery.sync_frequency === 'never')
// Only ClickHouse serves queries, so a marker on a shadow engine means the comparison
// run stopped, not that this model stopped refreshing.
const suspension = savedQuery.suspended?.[SERVING_ENGINE]
const showSuspendedBanner = !!suspension && !!savedQuery.is_materialized
return (
<div className="@container/materialization" data-attr="materialization-status-panel">
<div className="flex flex-col flex-1 gap-4">
<div>
<div className="flex flex-row items-center gap-2">
{!hideTitle && <h3 className="mb-0">Materialization</h3>}
</div>
{showStatusSummary && showSuspendedBanner && suspension && (
<LemonBanner type="error" className="mt-2">
<div data-attr="materialization-suspended-banner">
<div>{suspendedScheduleMessage(kind === 'endpoint' ? 'endpoint' : 'view')}</div>
<Tooltip title={suspension.reason} interactive>
<div className="mt-1 text-xs font-normal line-clamp-2">
Suspended {humanFriendlyDetailedTime(suspension.at)} · {suspension.reason}
</div>
</Tooltip>
</div>
</LemonBanner>
)}
<div>
{savedQuery?.is_materialized ? (
<div>
{showStatusSummary && (
<LemonCard hoverEffect={false} className="!p-3 mb-4 w-fit max-w-full">
<div className="flex flex-wrap items-center gap-2 mb-2">
<span className="font-semibold">
{currentJobStatus === 'Running' ? 'Current run' : 'Last run'}
</span>
{currentJobStatus ? (
<LemonTag type={STATUS_TAG_TYPES[currentJobStatus] ?? 'default'}>
{currentJobStatus === 'Cancelled' ? 'Canceled' : currentJobStatus}
</LemonTag>
) : dataModelingJobsLoading ? (
<LemonSkeleton className="h-5 w-20" />
) : (
<LemonTag type="muted">
{dataModelingJobs ? 'Not run yet' : 'Status unavailable'}
</LemonTag>
)}
</div>
<div className="flex flex-wrap items-center gap-x-3 gap-y-1 text-sm">
<span className="text-secondary">Last successful refresh</span>
{lastSuccessfulSyncAt ? (
<TZLabel time={lastSuccessfulSyncAt} />
) : dataModelingJobsLoading ? (
<LemonSkeleton className="h-4 w-24" />
) : (
<span>
{dataModelingJobs
? 'No successful run in recent history'
: 'Run history unavailable'}
</span>
)}
</div>
</LemonCard>
)}
{showIncremental && lastRunMode === 'incremental' && (
<div className="text-xs text-secondary mt-1">
Updating new rows only, up to{' '}
{formatWatermark(savedQuery.incremental_state?.watermark)}
</div>
)}
{showStatusSummary && isPaused && kind !== 'endpoint' && (
<div className="text-xs text-secondary mt-1">
Scheduled refreshes are paused. Pick a cadence to resume, or use Sync now to
refresh it once.
</div>
)}
<div className="flex flex-col gap-2 items-start mt-2">
{kind !== 'endpoint' && nodeFrequencyLoading && !nodeFrequency ? (
// Without the pin state the bar would first render as unpinned, then jump.
<LemonSkeleton className="h-8 w-80" />
) : kind !== 'endpoint' ? (
<SyncFrequencySelect
bounds={savedQuery.sync_frequency_bounds}
disabledReason={
sync ||
(savingMaterialization
? 'Saving materialization settings'
: undefined) ||
materializationAccessReason ||
(materializationActionLoading ? 'Updating materialization' : undefined)
}
value={
syncFrequencyDraft ??
(savedQuery.sync_frequency as SyncFrequencyValue) ??
'never'
}
onChange={setSyncFrequencyDraft}
loading={
savingMaterialization ||
updatingDataWarehouseSavedQuery ||
materializationActionLoading
}
pin={cadencePin}
onUnpin={() => setPinDraft(false)}
/>
) : null}
{showRunActions && (
<div className="flex flex-wrap items-center gap-2">
<MaterializationRunActions viewId={viewId} kind={kind} />
</div>
)}
</div>
{supportsIncremental && !savedQuery.managed_viewset_kind && incrementalCheck && (
<fieldset
className="mt-4 max-w-160"
disabled={
!!(
sync ||
savingMaterialization ||
materializationAccessReason ||
materializationActionLoading
)
}
>
<h4 className="mb-0">Refresh mode</h4>
<IncrementalConfigOptions
check={incrementalCheck}
draft={incrementalDraft}
onChange={setIncrementalDraft}
/>
{refreshModeChanged && incrementalDraft.enabled && (
<div className="mt-2">
<div className="text-xs text-secondary mt-1">
{structuralChange
? 'Changing these settings rebuilds the whole table on the next run. After that, runs update only new rows.'
: 'The new lookback applies from the next run.'}
</div>
</div>
)}
</fieldset>
)}
</div>
) : (
<div>
<p className="text-xs">
Materialized views are a way to pre-compute data in your data warehouse. This allows
you to run queries faster and more efficiently.
<br />
<Link
data-attr="materializing-help"
to="https://posthog.com/docs/data-warehouse/views#materializing-and-scheduling-a-view"
target="_blank"
>
Learn more about materialization
</Link>
.
</p>
<div className="flex flex-col gap-2 items-start">
{kind !== 'endpoint' && (
<SyncFrequencySelect
data-attr="initial-sync-frequency"
bounds={savedQuery.sync_frequency_bounds}
disabledReason={
materializationAccessReason ||
(materializationActionLoading ? 'Updating materialization' : undefined)
}
value={startingFrequency}
onChange={(newValue) => setInitialSyncFrequency(newValue)}
/>
)}
{showRunActions && <MaterializationRunActions viewId={viewId} kind={kind} />}
</div>
{supportsIncremental && (
<div className="max-w-160">
<IncrementalConfigOptions
check={incrementalCheck}
draft={incrementalDraft}
onChange={setIncrementalDraft}
/>
</div>
)}
</div>
)}
</div>
</div>
<div className="border-t pt-6 mt-4 flex items-start justify-between gap-2">
<div>
<h3 className="mb-0">Run history</h3>
</div>
<div className="flex items-center gap-2">
{managedWarehouseEnabled ? (
<LemonSelect
value={selectedRunHistoryEngine}
onChange={setSelectedRunHistoryEngine}
options={[
{ value: DataModelingJobsListEngine.Clickhouse, label: 'ClickHouse' },
{
value: DataModelingJobsListEngine.ManagedWarehouse,
label: 'Managed warehouse',
},
]}
size="small"
data-attr="materialization-run-history-engine-filter"
/>
) : null}
<LemonButton
icon={<IconRefresh />}
size="small"
type="secondary"
onClick={() =>
jobsPage > 1
? loadOlderJobsPage()
: selectedRunHistoryEngine === DataModelingJobsListEngine.Clickhouse
? loadDataModelingJobs()
: loadRunHistoryJobs()
}
loading={runHistoryLoading}
disabledReason={startingMaterialization ? 'Materialization is starting' : undefined}
tooltip="Refresh runs"
aria-label="Refresh runs"
/>
</div>
</div>
{runHistoryError && (
<LemonBanner type="error">Couldn't load runs. Use Refresh runs to try again.</LemonBanner>
)}
<LemonTable
rowKey="id"
pagination={{
controlled: true,
useUrl: false,
pageSize: DEFAULT_JOBS_PAGE_SIZE,
currentPage: jobsPage,
entryCount: jobsPageResults?.count,
// PaginationControl decides whether an arrow is enabled from the entry count, not
// from these handlers, so withholding one while a page loads would leave a live
// arrow that does nothing. Let the click through: the page loader breakpoints, so
// the superseded response is discarded.
onBackward: jobsPage > 1 ? () => setJobsPage(jobsPage - 1) : undefined,
onForward: jobsPageResults?.next ? () => setJobsPage(jobsPage + 1) : undefined,
}}
size="small"
// A timer reloads page 1 in the background, and LemonTable's loading overlay blocks
// pointer events over the rows and the pager, so page 1 shows the loader only before it
// has rows. Only a user action loads an older page, so that loader always shows.
loading={runHistoryLoading && (jobsPage > 1 || !jobsPageResults?.results?.length)}
dataSource={jobsPage > 1 && runHistoryError ? [] : jobsPageResults?.results || []}
columns={[
{
title: 'Status',
dataIndex: 'status',
render: (_, job: DataModelingJobApi) => {
const { status, rows_materialized, rows_expected } = job
const type = STATUS_TAG_TYPES[status] || 'warning'
const progressPercentage =
rows_expected && rows_expected > 0
? Math.min(100, (rows_materialized / rows_expected) * 100)
: 0
if (status === 'Running' && progressPercentage > 0 && rows_expected !== null) {
return (
<Tooltip
placement="right"
title={getMaterializationStatusMessage(
rows_materialized,
progressPercentage,
rows_expected
)}
>
<div className="w-[68px]">
<LemonProgress percent={progressPercentage} />
</div>
</Tooltip>
)
}
const tag = (
<LemonTag type={type}>{status === 'Cancelled' ? 'Canceled' : status}</LemonTag>
)
return job.error ? (
<Tooltip
title={<div className="whitespace-pre-wrap font-mono text-xs">{job.error}</div>}
interactive
>
<span tabIndex={0} aria-label={`${status}. ${job.error}`}>
{tag}
</span>
</Tooltip>
) : (
tag
)
},
},
{
title: 'Refresh mode',
dataIndex: 'run_mode',
isHidden: !supportsIncremental || !savedQuery.has_incremental_history,
render: (_, { run_mode, full_refresh_reason }: DataModelingJobApi) => {
if (run_mode === 'incremental') {
return 'Incremental'
}
if (run_mode === 'full_refresh') {
return full_refresh_reason ? (
<Tooltip title={`Full refresh. Reason: ${full_refresh_reason}.`}>
<span
className="border-b border-dotted cursor-help"
tabIndex={0}
aria-label={`Full refresh. Reason: ${full_refresh_reason}.`}
>
Full refresh
</span>
</Tooltip>
) : (
'Full refresh'
)
}
return '-'
},
},
{
title: 'Error',
dataIndex: 'error',
render: (_, { error, status }: DataModelingJobApi) => (
<div className="max-w-28 @min-[48rem]/materialization:max-w-60">
<span
className={`block truncate ${status === 'Failed' ? 'text-danger' : 'text-secondary'}`}
>
{error?.split('\n')[0]}
</span>
</div>
),
},
{
title: 'Rows',
dataIndex: 'rows_materialized',
render: (_, { rows_materialized, status, run_mode }: DataModelingJobApi) => {
if (
(status === 'Running' || status === 'Cancelled' || status === 'Skipped') &&
rows_materialized === 0
) {
return '~'
}
const count = humanFriendlyNumber(rows_materialized)
if (!run_mode) {
return count
}
return (
<Tooltip
title={
run_mode === 'incremental'
? 'Rows this run synced, including the re-read lookback window.'
: 'This run rebuilt the whole table. This is its full row count.'
}
>
<span>{count}</span>
</Tooltip>
)
},
},
{
title: 'Updated',
dataIndex: 'last_run_at',
render: (_, { last_run_at }: DataModelingJobApi) =>
last_run_at ? <TZLabel time={last_run_at} /> : '-',
},
{
title: 'Duration',
render: (_, job: DataModelingJobApi) => computeJobDuration(job),
},
]}
expandable={
jobsPageResults?.results?.length
? {
expandedRowRender: (job: DataModelingJobApi) => (
<div className="p-4 min-w-0">
<MaterializationRunError error={job.error} status={job.status} />
<LogsViewer
logicKey={`data_modeling_run:${job.id}`}
sourceType="data_modeling_run"
sourceId={viewId}
groupByInstanceId={false}
hideDateFilter
hideLevelsFilter
hideInstanceIdColumn
defaultFilters={{
instanceId: job.workflow_run_id ?? undefined,
...jobLogsWindow(job, timezone),
levels: showDebugLogs ? ['DEBUG', ...LOG_LEVELS] : LOG_LEVELS,
}}
/>
</div>
),
rowExpandable: () => true,
noIndent: true,
}
: undefined
}
nouns={['run', 'runs']}
emptyState="No runs available"
/>
</div>
</div>
)
}