:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import {
MakeLogicType,
actions,
afterMount,
connect,
kea,
key,
listeners,
path,
props,
propsChanged,
reducers,
selectors,
} from 'kea'
import { loaders } from 'kea-loaders'
import posthog from 'posthog-js'
import { lemonToast } from '@posthog/lemon-ui'
import api, { ApiConfig, ApiError } from 'lib/api'
import type { DataWarehouseSavedQuerySummary } from 'scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic'
import type { DataWarehouseSavedQuery } from '~/types'
import { DataModelingSyncInterval, DataWarehouseSavedQueryIncrementalCheck } from '~/types'
import {
dataModelingNodesFrequencyBoundsRetrieve,
dataModelingNodesPartialUpdate,
dataModelingNodesRetrieve,
} from 'products/data_modeling/frontend/generated/api'
import {
DataModelingNodesRetrieveLookup,
type NodeFrequencyBoundsApi,
} from 'products/data_modeling/frontend/generated/api.schemas'
import { dataModelingJobsList, warehouseSavedQueriesResumeCreate } from 'products/data_warehouse/frontend/generated/api'
import { DataModelingJobsListEngine } from 'products/data_warehouse/frontend/generated/api.schemas'
import type {
DataModelingJobApi,
PaginatedDataModelingJobListApi,
} from 'products/data_warehouse/frontend/generated/api.schemas'
import {
EMPTY_INCREMENTAL_DRAFT,
INCREMENTAL_CHECK_LOADING,
IncrementalConfigDraft,
} from '../editor/IncrementalConfigFields'
import { DEFAULT_MATERIALIZE_SYNC_FREQUENCY, dataWarehouseViewsLogic } from './dataWarehouseViewsLogic'
import { latestSuccessfulSyncAt } from './materializationJobUtils'
import { CadencePin, needsPin, SyncFrequencyValue } from './SyncFrequencySelect'
const ACTIVE_REFRESH_INTERVAL_MS = 5000
const IDLE_REFRESH_INTERVAL_MS = 60000
// The run request returns before the workflow writes the job row, so a run can take a while to show up.
// Give up waiting after this long so the controls do not stay blocked when no job ever appears.
const STARTING_TIMEOUT_MS = 120000
export const DEFAULT_JOBS_PAGE_SIZE = 10
interface OlderJobsPageResult {
engine: DataModelingJobsListEngine
page: number
jobs: PaginatedDataModelingJobListApi
}
export interface NodeFrequency {
nodeId: string
bounds: NodeFrequencyBoundsApi
}
function apiErrorDetail(error: unknown): string {
return error instanceof ApiError && error.detail ? error.detail : 'Try again.'
}
function runHistoryEngineParams(engine: DataModelingJobsListEngine): {
all_engines?: boolean
engine?: DataModelingJobsListEngine
} {
return engine === DataModelingJobsListEngine.ManagedWarehouse ? { all_engines: true, engine } : {}
}
function isNewRunVisible(newestJob: DataModelingJobApi | undefined, previousJobId: string | null | undefined): boolean {
return !!newestJob && newestJob.id !== previousJobId
}
export interface MaterializationJobsLogicProps {
viewId: string
/** Endpoints own their saved query's materialization lifecycle, so incremental setup is hidden for them. */
kind?: 'view' | 'endpoint'
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface materializationJobsLogicValues {
cadencePin: CadencePin | null
dataModelingJobs: PaginatedDataModelingJobListApi | null
dataModelingJobsError: boolean
dataModelingJobsLoading: boolean
deletingView: boolean
hasMaterializationChanges: boolean
incrementalCheck: DataWarehouseSavedQueryIncrementalCheck | null
incrementalCheckLoading: boolean
incrementalDraft: IncrementalConfigDraft
incrementalDraftTouched: boolean
initialSyncFrequency: DataModelingSyncInterval
jobsPage: number
jobsPageResults: PaginatedDataModelingJobListApi | null
jobsRefreshPending: boolean
lastSuccessfulSyncAt: string | null
latestCompletedJob: PaginatedDataModelingJobListApi | null
latestCompletedJobLoading: boolean
materializationRefreshPending: boolean
nodeFrequency: NodeFrequency | null
nodeFrequencyLoading: boolean
olderJobsPage: OlderJobsPageResult | null
olderJobsPageError: boolean
olderJobsPageLoading: boolean
pinChanged: boolean
pinDraft: boolean | null
refreshModeChanged: boolean
resumingMaterialization: boolean
runHistoryError: boolean
runHistoryJobs: PaginatedDataModelingJobListApi | null
runHistoryJobsError: boolean
runHistoryJobsLoading: boolean
runHistoryLoading: boolean
savedQuery: DataWarehouseSavedQuery | null
savedQueryError: boolean
savedQueryLoading: boolean
savedQueryRefreshPending: boolean
savingMaterialization: boolean
selectedRunHistoryEngine: DataModelingJobsListEngine
startingMaterialization: boolean
syncFrequencyDraft: SyncFrequencyValue | null
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface materializationJobsLogicActions {
deleteDataWarehouseSavedQuery: (viewId: string) => string // dataWarehouseViewsLogic
deleteDataWarehouseSavedQueryFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
} // dataWarehouseViewsLogic
deleteDataWarehouseSavedQuerySuccess: (
dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[],
payload?: string | undefined
) => {
dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[]
payload?: string
} // dataWarehouseViewsLogic
loadDataWarehouseSavedQueries: () => any // dataWarehouseViewsLogic
materializationChanged: (viewId: string) => {
viewId: string
} // dataWarehouseViewsLogic
runDataWarehouseSavedQueryFailure: (viewId: string) => {
viewId: string
} // dataWarehouseViewsLogic
runDataWarehouseSavedQuerySuccess: (viewId: string) => {
viewId: string
} // dataWarehouseViewsLogic
updateDataWarehouseSavedQuery: (
view: import('scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic').DataWarehouseSavedQueryUpdate
) => import('scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic').DataWarehouseSavedQueryUpdate // dataWarehouseViewsLogic
updateDataWarehouseSavedQueryFailed: (viewId: string) => {
viewId: string
} // dataWarehouseViewsLogic
updateDataWarehouseSavedQuerySuccess: (
dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[],
payload?:
| import('scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic').DataWarehouseSavedQueryUpdate
| undefined
) => {
dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[]
payload?: import('scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic').DataWarehouseSavedQueryUpdate
} // dataWarehouseViewsLogic
clearSyncFrequencyDraft: () => {
value: true
}
deleteView: () => {
value: true
}
discardMaterializationChanges: () => {
value: true
}
finishDeletingView: () => {
value: true
}
finishSavingMaterialization: () => {
value: true
}
loadDataModelingJobs: (_: void) => void
loadDataModelingJobsFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
loadDataModelingJobsSuccess: (
dataModelingJobs: PaginatedDataModelingJobListApi,
payload?: void
) => {
dataModelingJobs: PaginatedDataModelingJobListApi
payload?: void
}
loadIncrementalCheck: () => any
loadIncrementalCheckFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
loadIncrementalCheckSuccess: (
incrementalCheck: DataWarehouseSavedQueryIncrementalCheck | null,
payload?: any
) => {
incrementalCheck: DataWarehouseSavedQueryIncrementalCheck | null
payload?: any
}
loadLatestCompletedJob: (_: void) => void
loadLatestCompletedJobFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
loadLatestCompletedJobSuccess: (
latestCompletedJob: PaginatedDataModelingJobListApi,
payload?: void
) => {
latestCompletedJob: PaginatedDataModelingJobListApi
payload?: void
}
loadNodeFrequency: (_: void) => void
loadNodeFrequencyFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
loadNodeFrequencySuccess: (
nodeFrequency: NodeFrequency | null,
payload?: void
) => {
nodeFrequency: NodeFrequency | null
payload?: void
}
loadOlderJobsPage: (_: void) => void
loadOlderJobsPageFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
loadOlderJobsPageSuccess: (
olderJobsPage: {
engine: DataModelingJobsListEngine
jobs: PaginatedDataModelingJobListApi
page: number
},
payload?: void
) => {
olderJobsPage: {
engine: DataModelingJobsListEngine
jobs: PaginatedDataModelingJobListApi
page: number
}
payload?: void
}
loadRunHistoryJobs: (_: void) => void
loadRunHistoryJobsFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
loadRunHistoryJobsSuccess: (
runHistoryJobs: PaginatedDataModelingJobListApi,
payload?: void
) => {
runHistoryJobs: PaginatedDataModelingJobListApi
payload?: void
}
loadSavedQuery: (_: void) => void
loadSavedQueryFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
loadSavedQuerySuccess: (
savedQuery: DataWarehouseSavedQuery | null,
payload?: void
) => {
savedQuery: DataWarehouseSavedQuery | null
payload?: void
}
refreshMaterialization: () => {
value: true
}
resumeMaterialization: () => {
value: true
}
saveMaterializationChanges: () => {
value: true
}
scheduleJobsRefresh: (delay: number) => {
delay: number
}
seedIncrementalDraft: (draft: IncrementalConfigDraft) => {
draft: IncrementalConfigDraft
}
setIncrementalDraft: (draft: Partial<IncrementalConfigDraft>) => {
draft: Partial<IncrementalConfigDraft>
}
setInitialSyncFrequency: (syncFrequency: DataModelingSyncInterval) => {
syncFrequency: DataModelingSyncInterval
}
setJobsPage: (page: number) => {
page: number
}
setPinDraft: (pinned: boolean | null) => {
pinned: boolean | null
}
setResumingMaterialization: (resuming: boolean) => {
resuming: boolean
}
setSelectedRunHistoryEngine: (engine: DataModelingJobsListEngine) => {
engine: DataModelingJobsListEngine
}
setStartingMaterialization: (starting: boolean) => {
starting: boolean
}
setSyncFrequencyDraft: (frequency: SyncFrequencyValue) => {
frequency: SyncFrequencyValue
}
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface materializationJobsLogicMeta {
key: string
__keaTypeGenInternalSelectorTypes: {
materializationRefreshPending: (savedQueryRefreshPending: boolean, jobsRefreshPending: boolean) => boolean
refreshModeChanged: (
savedQuery: DataWarehouseSavedQuery | null,
incrementalDraft: IncrementalConfigDraft,
incrementalDraftTouched: boolean
) => boolean
pinChanged: (pinDraft: boolean | null, nodeFrequency: NodeFrequency | null) => boolean
hasMaterializationChanges: (
savedQuery: DataWarehouseSavedQuery | null,
syncFrequencyDraft: SyncFrequencyValue | null,
refreshModeChanged: boolean,
pinChanged: boolean
) => boolean
cadencePin: (
nodeFrequency: NodeFrequency | null,
pinDraft: boolean | null,
pinChanged: boolean,
syncFrequencyDraft: SyncFrequencyValue | null,
savedQuery: DataWarehouseSavedQuery | null
) => CadencePin | null
jobsPageResults: (
jobsPage: number,
selectedRunHistoryEngine: DataModelingJobsListEngine,
dataModelingJobs: PaginatedDataModelingJobListApi | null,
runHistoryJobs: PaginatedDataModelingJobListApi | null,
olderJobsPage: OlderJobsPageResult | null
) => PaginatedDataModelingJobListApi | null
runHistoryError: (
selectedRunHistoryEngine: DataModelingJobsListEngine,
dataModelingJobsError: boolean,
runHistoryJobsError: boolean,
olderJobsPageError: boolean,
jobsPage: number
) => boolean
runHistoryLoading: (
selectedRunHistoryEngine: DataModelingJobsListEngine,
dataModelingJobsLoading: boolean,
runHistoryJobsLoading: boolean,
olderJobsPageLoading: boolean,
jobsPage: number
) => boolean
lastSuccessfulSyncAt: (
dataModelingJobs: PaginatedDataModelingJobListApi | null,
latestCompletedJob: PaginatedDataModelingJobListApi | null
) => string | null
}
}
export type materializationJobsLogicType = MakeLogicType<
materializationJobsLogicValues,
materializationJobsLogicActions,
MaterializationJobsLogicProps,
materializationJobsLogicMeta
>
export const materializationJobsLogic = kea<materializationJobsLogicType>([
path(['scenes', 'data-warehouse', 'saved_queries', 'materializationJobsLogic']),
props({} as MaterializationJobsLogicProps),
key((props) => props.viewId),
connect(() => ({
actions: [
dataWarehouseViewsLogic,
[
'updateDataWarehouseSavedQuery',
'updateDataWarehouseSavedQuerySuccess',
'updateDataWarehouseSavedQueryFailed',
'runDataWarehouseSavedQueryFailure',
'runDataWarehouseSavedQuerySuccess',
'materializationChanged',
'deleteDataWarehouseSavedQuery',
'deleteDataWarehouseSavedQuerySuccess',
'deleteDataWarehouseSavedQueryFailure',
'loadDataWarehouseSavedQueries',
],
],
})),
actions({
refreshMaterialization: true,
setJobsPage: (page: number) => ({ page }),
setSelectedRunHistoryEngine: (engine: DataModelingJobsListEngine) => ({ engine }),
deleteView: true,
finishDeletingView: true,
scheduleJobsRefresh: (delay: number) => ({ delay }),
setStartingMaterialization: (starting: boolean) => ({ starting }),
resumeMaterialization: true,
setSyncFrequencyDraft: (frequency: SyncFrequencyValue) => ({ frequency }),
clearSyncFrequencyDraft: true,
setPinDraft: (pinned: boolean | null) => ({ pinned }),
discardMaterializationChanges: true,
saveMaterializationChanges: true,
finishSavingMaterialization: true,
setResumingMaterialization: (resuming: boolean) => ({ resuming }),
setInitialSyncFrequency: (syncFrequency: DataModelingSyncInterval) => ({ syncFrequency }),
setIncrementalDraft: (draft: Partial<IncrementalConfigDraft>) => ({ draft }),
// Distinct from setIncrementalDraft so that reflecting the saved config into the draft
// doesn't count as the user touching it.
seedIncrementalDraft: (draft: IncrementalConfigDraft) => ({ draft }),
}),
loaders(({ values, props, cache }) => ({
savedQuery: [
null as DataWarehouseSavedQuery | null,
{
loadSavedQuery: async (_: void, breakpoint) => {
if (!props.viewId) {
return null
}
const savedQuery = await api.dataWarehouseSavedQueries.get(props.viewId)
breakpoint()
return savedQuery
},
},
],
// The cadence pin lives on the view's DAG node, which the saved query payload does not carry.
nodeFrequency: [
null as NodeFrequency | null,
{
loadNodeFrequency: async (_: void, breakpoint) => {
const teamId = String(ApiConfig.getCurrentTeamId())
let frequency: NodeFrequency
try {
const node = await dataModelingNodesRetrieve(teamId, props.viewId, {
lookup: DataModelingNodesRetrieveLookup.SavedQuery,
})
const bounds = await dataModelingNodesFrequencyBoundsRetrieve(teamId, node.id)
frequency = { nodeId: node.id, bounds }
} catch (e) {
// A view with no node has nothing to pin, and the picker then refuses slow cadences as before.
if (!(e instanceof ApiError && e.status === 404)) {
posthog.captureException(e)
// Not an answer about the view, so the next saved query reload gets to try again.
cache.nodeFrequencyRequested = false
}
return null
}
// Outside the try: a breakpoint throws to cancel a superseded load, and the catch would turn that into "no node".
breakpoint()
return frequency
},
},
],
incrementalCheck: [
INCREMENTAL_CHECK_LOADING as DataWarehouseSavedQueryIncrementalCheck | null,
{
loadIncrementalCheck: async () => {
const sql = values.savedQuery?.query?.query
if (!sql) {
return null
}
// A rejected check is not the user's error, so never toast: the panel just omits the
// incremental option. A 4xx is a deterministic refusal, so it is not retried. Anything
// else is ours to look at, and the next savedQuery reload gets to try again.
try {
return await api.dataWarehouseSavedQueries.checkIncremental({ query: sql })
} catch (e) {
if (!(e instanceof ApiError && e.status && e.status >= 400 && e.status < 500)) {
posthog.captureException(e)
cache.incrementalCheckRequested = false
}
return null
}
},
},
],
dataModelingJobs: [
null as PaginatedDataModelingJobListApi | null,
{
loadDataModelingJobs: async (_: void, breakpoint) => {
const jobs = await dataModelingJobsList(String(ApiConfig.getCurrentTeamId()), {
saved_query_id: props.viewId,
limit: DEFAULT_JOBS_PAGE_SIZE,
offset: 0,
})
breakpoint()
return jobs
},
},
],
runHistoryJobs: [
null as PaginatedDataModelingJobListApi | null,
{
loadRunHistoryJobs: async (_: void, breakpoint) => {
const jobs = await dataModelingJobsList(String(ApiConfig.getCurrentTeamId()), {
saved_query_id: props.viewId,
limit: DEFAULT_JOBS_PAGE_SIZE,
offset: 0,
...runHistoryEngineParams(values.selectedRunHistoryEngine),
})
breakpoint()
return jobs
},
},
],
// The newest completed run when the first page holds none. The panel reports when the data the
// view serves was built, and a view whose last ten runs all failed still serves an older build.
latestCompletedJob: [
null as PaginatedDataModelingJobListApi | null,
{
loadLatestCompletedJob: async (_: void, breakpoint) => {
const jobs = await dataModelingJobsList(String(ApiConfig.getCurrentTeamId()), {
saved_query_id: props.viewId,
status: 'Completed',
limit: 1,
offset: 0,
})
breakpoint()
return jobs
},
},
],
olderJobsPage: [
null as OlderJobsPageResult | null,
{
loadOlderJobsPage: async (_: void, breakpoint) => {
const engine = values.selectedRunHistoryEngine
const page = values.jobsPage
const jobs = await dataModelingJobsList(String(ApiConfig.getCurrentTeamId()), {
saved_query_id: props.viewId,
limit: DEFAULT_JOBS_PAGE_SIZE,
offset: (page - 1) * DEFAULT_JOBS_PAGE_SIZE,
...runHistoryEngineParams(engine),
})
breakpoint()
return { engine, page, jobs }
},
},
],
})),
reducers(({ props }) => ({
jobsPage: [1, { setJobsPage: (_, { page }) => page, setSelectedRunHistoryEngine: () => 1 }],
selectedRunHistoryEngine: [
DataModelingJobsListEngine.Clickhouse as DataModelingJobsListEngine,
{ setSelectedRunHistoryEngine: (_, { engine }) => engine },
],
runHistoryJobsError: [false, { loadRunHistoryJobs: () => false, loadRunHistoryJobsFailure: () => true }],
olderJobsPageError: [false, { loadOlderJobsPage: () => false, loadOlderJobsPageFailure: () => true }],
deletingView: [false, { deleteView: () => true, finishDeletingView: () => false }],
savedQueryError: [
false,
{
loadSavedQueryFailure: () => true,
loadSavedQuerySuccess: () => false,
},
],
dataModelingJobsError: [
false,
{
loadDataModelingJobsFailure: () => true,
loadDataModelingJobsSuccess: () => false,
},
],
savedQueryRefreshPending: [
false,
{
refreshMaterialization: () => true,
loadSavedQuerySuccess: () => false,
},
],
jobsRefreshPending: [
false,
{
refreshMaterialization: () => true,
loadDataModelingJobsSuccess: () => false,
},
],
syncFrequencyDraft: [
null as SyncFrequencyValue | null,
{
setSyncFrequencyDraft: (_, { frequency }) => frequency,
clearSyncFrequencyDraft: () => null,
discardMaterializationChanges: () => null,
},
],
pinDraft: [
null as boolean | null,
{
setPinDraft: (_, { pinned }) => pinned,
discardMaterializationChanges: () => null,
},
],
savingMaterialization: [
false,
{
saveMaterializationChanges: () => true,
finishSavingMaterialization: () => false,
},
],
// What the not-yet-materialized panel will ask for. Kept here rather than in component state
// so it survives the panel remounting while the saved query reloads.
initialSyncFrequency: [
DEFAULT_MATERIALIZE_SYNC_FREQUENCY,
{
setInitialSyncFrequency: (_, { syncFrequency }: { syncFrequency: DataModelingSyncInterval }) =>
syncFrequency,
},
],
incrementalDraft: [
EMPTY_INCREMENTAL_DRAFT,
{
setIncrementalDraft: (
state: IncrementalConfigDraft,
{ draft }: { draft: Partial<IncrementalConfigDraft> }
) => ({
...state,
...draft,
}),
seedIncrementalDraft: (_: IncrementalConfigDraft, { draft }: { draft: IncrementalConfigDraft }) =>
draft,
// Pre-select the first candidate so enabling incremental starts from a working config.
loadIncrementalCheckSuccess: (
state: IncrementalConfigDraft,
{ incrementalCheck }: { incrementalCheck: DataWarehouseSavedQueryIncrementalCheck | null }
) =>
state.incrementalKey
? state
: { ...state, incrementalKey: incrementalCheck?.key_candidates[0] ?? null },
},
],
// Blocks the saved config from overwriting in-progress edits when the saved query repolls.
incrementalDraftTouched: [
false,
{
setIncrementalDraft: () => true,
discardMaterializationChanges: () => false,
},
],
resumingMaterialization: [
false,
{
resumeMaterialization: () => true,
setResumingMaterialization: (_, { resuming }: { resuming: boolean }) => resuming,
},
],
startingMaterialization: [
false,
{
setStartingMaterialization: (_, { starting }: { starting: boolean }) => starting,
// No job exists when the request itself failed, so the reset on job arrival never
// fires and the controls would stay disabled until the starting timeout.
runDataWarehouseSavedQueryFailure: (state: boolean, { viewId }: { viewId: string }) =>
viewId === props.viewId ? false : state,
},
],
})),
selectors({
materializationRefreshPending: [
(s) => [s.savedQueryRefreshPending, s.jobsRefreshPending],
(savedQueryPending: boolean, jobsPending: boolean): boolean => savedQueryPending || jobsPending,
],
refreshModeChanged: [
(s) => [s.savedQuery, s.incrementalDraft, s.incrementalDraftTouched],
(savedQuery: DataWarehouseSavedQuery | null, draft: IncrementalConfigDraft, touched: boolean): boolean => {
const saved = savedQuery?.incremental
return (
touched &&
!!savedQuery &&
(draft.enabled !== !!saved?.enabled ||
(!!saved?.enabled &&
draft.enabled &&
(draft.incrementalKey !== saved.incremental_key ||
[...draft.uniqueKey].sort().join(',') !== [...saved.unique_key].sort().join(',') ||
draft.lookbackSeconds !== (saved.lookback_seconds ?? 0))))
)
},
],
pinChanged: [
(s) => [s.pinDraft, s.nodeFrequency],
(pinDraft: boolean | null, nodeFrequency: NodeFrequency | null): boolean =>
pinDraft !== null && !!nodeFrequency && pinDraft !== nodeFrequency.bounds.pinned,
],
hasMaterializationChanges: [
(s) => [s.savedQuery, s.syncFrequencyDraft, s.refreshModeChanged, s.pinChanged],
(
savedQuery: DataWarehouseSavedQuery | null,
frequency: SyncFrequencyValue | null,
modeChanged: boolean,
pinChanged: boolean
): boolean =>
!!savedQuery?.is_materialized &&
(modeChanged ||
pinChanged ||
(frequency !== null && frequency !== (savedQuery.sync_frequency || 'never'))),
],
cadencePin: [
(s) => [s.nodeFrequency, s.pinDraft, s.pinChanged, s.syncFrequencyDraft, s.savedQuery],
(
nodeFrequency: NodeFrequency | null,
pinDraft: boolean | null,
pinChanged: boolean,
frequency: SyncFrequencyValue | null,
savedQuery: DataWarehouseSavedQuery | null
): CadencePin | null => {
if (!nodeFrequency) {
return null
}
const cadence = frequency ?? savedQuery?.sync_frequency
return {
pinned: pinDraft ?? nodeFrequency.bounds.pinned,
pending: pinChanged,
delayed:
nodeFrequency.bounds.options.find((option) => option.cadence === cadence)?.delays_if_pinned ??
[],
blockedBy: Object.fromEntries(
nodeFrequency.bounds.options.flatMap((option) =>
option.pin_blocked_by ? [[option.cadence, option.pin_blocked_by]] : []
)
),
delaysBy: Object.fromEntries(
nodeFrequency.bounds.options.map((option) => [option.cadence, option.delays_if_pinned])
),
unpinnedBy: Object.fromEntries(
nodeFrequency.bounds.options.map((option) => [option.cadence, option.unpinned_cadence])
),
}
},
],
jobsPageResults: [
(s) => [s.jobsPage, s.selectedRunHistoryEngine, s.dataModelingJobs, s.runHistoryJobs, s.olderJobsPage],
(
page: number,
engine: DataModelingJobsListEngine,
latest: PaginatedDataModelingJobListApi | null,
filtered: PaginatedDataModelingJobListApi | null,
older: OlderJobsPageResult | null
): PaginatedDataModelingJobListApi | null => {
if (page === 1) {
return engine === DataModelingJobsListEngine.Clickhouse ? latest : filtered
}
return older?.engine === engine && older.page === page ? older.jobs : null
},
],
runHistoryError: [
(s) => [
s.selectedRunHistoryEngine,
s.dataModelingJobsError,
s.runHistoryJobsError,
s.olderJobsPageError,
s.jobsPage,
],
(
engine: DataModelingJobsListEngine,
clickhouseError: boolean,
managedError: boolean,
olderError: boolean,
page: number
): boolean =>
page > 1
? olderError
: engine === DataModelingJobsListEngine.Clickhouse
? clickhouseError
: managedError,
],
runHistoryLoading: [
(s) => [
s.selectedRunHistoryEngine,
s.dataModelingJobsLoading,
s.runHistoryJobsLoading,
s.olderJobsPageLoading,
s.jobsPage,
],
(
engine: DataModelingJobsListEngine,
clickhouseLoading: boolean,
managedLoading: boolean,
olderLoading: boolean,
page: number
): boolean =>
page > 1
? olderLoading
: engine === DataModelingJobsListEngine.Clickhouse
? clickhouseLoading
: managedLoading,
],
lastSuccessfulSyncAt: [
(s) => [s.dataModelingJobs, s.latestCompletedJob],
(
dataModelingJobs: PaginatedDataModelingJobListApi | null,
latestCompletedJob: PaginatedDataModelingJobListApi | null
) => latestSuccessfulSyncAt([...(dataModelingJobs?.results ?? []), ...(latestCompletedJob?.results ?? [])]),
],
}),
afterMount(({ actions, cache, props, values }) => {
if (props.viewId) {
actions.loadDataModelingJobs()
actions.loadSavedQuery()
}
// Background tabs pause the poll timer, and it restarts with its full delay. Reload at once
// when the user comes back, so the panel does not show a finished run as still running.
cache.disposables.add(
() => {
const onVisibilityChange = (): void => {
if (!document.hidden && props.viewId) {
actions.loadDataModelingJobs()
if (
values.jobsPage === 1 &&
values.selectedRunHistoryEngine === DataModelingJobsListEngine.ManagedWarehouse
) {
actions.loadRunHistoryJobs()
}
}
}
document.addEventListener('visibilitychange', onVisibilityChange)
return () => document.removeEventListener('visibilitychange', onVisibilityChange)
},
'visibilityRefresh',
{ pauseOnPageHidden: false }
)
}),
propsChanged(({ actions, props }, oldProps) => {
if (props.viewId && props.viewId !== oldProps.viewId) {
actions.loadDataModelingJobs()
actions.loadSavedQuery()
}
}),
listeners(({ actions, cache, props, values }) => ({
setJobsPage: () => {
if (values.jobsPage === 1 && values.selectedRunHistoryEngine === DataModelingJobsListEngine.Clickhouse) {
actions.loadDataModelingJobs()
} else if (values.jobsPage === 1) {
actions.loadRunHistoryJobs()
} else {
actions.loadOlderJobsPage()
}
},
setSelectedRunHistoryEngine: () => {
if (values.selectedRunHistoryEngine === DataModelingJobsListEngine.Clickhouse) {
actions.loadDataModelingJobs()
} else {
actions.loadRunHistoryJobs()
}
},
deleteView: () => actions.deleteDataWarehouseSavedQuery(props.viewId),
deleteDataWarehouseSavedQuerySuccess: ({ payload }) => {
if (payload === props.viewId && values.deletingView) {
actions.finishDeletingView()
}
},
deleteDataWarehouseSavedQueryFailure: () => actions.finishDeletingView(),
materializationChanged: ({ viewId }) => {
if (viewId === props.viewId) {
actions.refreshMaterialization()
}
},
runDataWarehouseSavedQuerySuccess: ({ viewId }) => {
if (viewId === props.viewId) {
actions.refreshMaterialization()
}
},
refreshMaterialization: () => {
actions.loadSavedQuery()
actions.loadDataModelingJobs()
if (
values.jobsPage === 1 &&
values.selectedRunHistoryEngine === DataModelingJobsListEngine.ManagedWarehouse
) {
actions.loadRunHistoryJobs()
}
},
loadDataModelingJobs: () => {
cache.disposables.dispose('dataModelingJobsRefreshTimeout')
},
scheduleJobsRefresh: ({ delay }) => {
cache.disposables.add(() => {
const timeoutId = setTimeout(() => {
actions.loadDataModelingJobs()
if (
values.jobsPage === 1 &&
values.selectedRunHistoryEngine === DataModelingJobsListEngine.ManagedWarehouse
) {
actions.loadRunHistoryJobs()
}
}, delay)
return () => clearTimeout(timeoutId)
}, 'dataModelingJobsRefreshTimeout')
},
setStartingMaterialization: ({ starting }) => {
if (!starting) {
cache.disposables.dispose('startingMaterializationTimeout')
return
}
cache.newestJobIdBeforeStart = values.dataModelingJobs?.results[0]?.id ?? null
cache.disposables.add(
() => {
const timeoutId = setTimeout(() => actions.setStartingMaterialization(false), STARTING_TIMEOUT_MS)
return () => clearTimeout(timeoutId)
},
'startingMaterializationTimeout',
{ pauseOnPageHidden: false }
)
},
runDataWarehouseSavedQueryFailure: ({ viewId }) => {
if (viewId === props.viewId) {
cache.disposables.dispose('startingMaterializationTimeout')
}
},
loadDataModelingJobsFailure: () => {
actions.scheduleJobsRefresh(IDLE_REFRESH_INTERVAL_MS)
},
setSyncFrequencyDraft: ({ frequency }) => {
// A cadence slower than a downstream view needs is only valid pinned, so the pick carries the pin.
if (values.cadencePin && needsPin(values.cadencePin.unpinnedBy, frequency)) {
actions.setPinDraft(true)
} else if (values.pinDraft) {
actions.setPinDraft(null)
}
// The cadence bar reports a change on every click, including the segment that is already
// selected, so a draft can hold the cadence that is already saved. Such a draft is invisible
// because it is not a change, until something else moves the saved cadence. Pausing sets the
// saved cadence to 'never', which makes the stale draft look like an edit, and saving it
// resumes the schedule the user just paused.
if (frequency === (values.savedQuery?.sync_frequency || 'never')) {
actions.clearSyncFrequencyDraft()
}
},
discardMaterializationChanges: () => {
const saved = values.savedQuery?.incremental
actions.seedIncrementalDraft(
saved?.enabled
? {
enabled: true,
incrementalKey: saved.incremental_key,
uniqueKey: saved.unique_key,
lookbackSeconds: saved.lookback_seconds ?? 0,
}
: { ...EMPTY_INCREMENTAL_DRAFT, incrementalKey: values.incrementalCheck?.key_candidates[0] ?? null }
)
},
saveMaterializationChanges: async () => {
const draft = values.incrementalDraft
if (!values.savedQuery || !values.hasMaterializationChanges) {
actions.finishSavingMaterialization()
return
}
let cadenceWritten = false
if (values.pinChanged && values.nodeFrequency) {
// The cadence and the pin go in one node write: a cadence slower than a downstream view
// needs is refused unless it is pinned in the same request.
try {
await dataModelingNodesPartialUpdate(
String(ApiConfig.getCurrentTeamId()),
values.nodeFrequency.nodeId,
{
...(values.syncFrequencyDraft !== null
? { sync_frequency: values.syncFrequencyDraft }
: {}),
pinned: !!values.pinDraft,
}
)
} catch (e) {
lemonToast.error(`Couldn't save the cadence. ${apiErrorDetail(e)}`)
actions.finishSavingMaterialization()
return
}
cadenceWritten = true
if (!values.refreshModeChanged) {
// Keep the draft on screen until both reloads land, so the picker never shows the old
// pin state between the write and the fresh bounds.
// Through the loader, so its breakpoint drops any poll that started before the write.
const savedQueryReload = actions.loadSavedQuery()
try {
const bounds = await dataModelingNodesFrequencyBoundsRetrieve(
String(ApiConfig.getCurrentTeamId()),
values.nodeFrequency.nodeId
)
actions.loadNodeFrequencySuccess({ nodeId: values.nodeFrequency.nodeId, bounds })
} catch {
actions.loadNodeFrequency()
}
await savedQueryReload
actions.discardMaterializationChanges()
actions.finishSavingMaterialization()
return
}
actions.loadNodeFrequency()
}
actions.updateDataWarehouseSavedQuery({
id: props.viewId,
...(values.syncFrequencyDraft !== null && !cadenceWritten
? { sync_frequency: values.syncFrequencyDraft }
: {}),
...(values.refreshModeChanged
? {
incremental:
draft.enabled && draft.incrementalKey
? {
enabled: true,
incremental_key: draft.incrementalKey,
unique_key: draft.uniqueKey,
lookback_seconds: draft.lookbackSeconds,
}
: null,
}
: {}),
types: [[]],
lifecycle: 'update',
})
},
updateDataWarehouseSavedQueryFailed: ({ viewId }) => {
if (viewId === props.viewId) {
actions.finishSavingMaterialization()
}
},
resumeMaterialization: async () => {
try {
await warehouseSavedQueriesResumeCreate(String(ApiConfig.getCurrentTeamId()), props.viewId)
lemonToast.success('Materialization resumed. The next scheduled run will include this view.')
actions.loadSavedQuery()
actions.loadDataModelingJobs()
// The SQL editor sidebar reads suspension from the view list, not from this view's record.
actions.loadDataWarehouseSavedQueries()
} catch {
lemonToast.error(
"Couldn't resume materialization. Try again, and contact support if it keeps happening."
)
} finally {
actions.setResumingMaterialization(false)
}
},
loadSavedQuerySuccess: ({ savedQuery }) => {
if (!savedQuery || props.kind === 'endpoint') {
return
}
if (savedQuery.is_materialized && !cache.nodeFrequencyRequested) {
cache.nodeFrequencyRequested = true
actions.loadNodeFrequency()
}
// Reflect the saved config so a materialized view's editor starts from what is
// actually configured, not from the empty draft.
if (!values.incrementalDraftTouched && savedQuery.incremental?.enabled) {
actions.seedIncrementalDraft({
enabled: true,
incrementalKey: savedQuery.incremental.incremental_key,
uniqueKey: savedQuery.incremental.unique_key,
lookbackSeconds: savedQuery.incremental.lookback_seconds ?? 0,
})
}
// One eligibility check per mount: savedQuery reloads on every jobs poll, and the
// query text can't change while this panel is open.
if (cache.incrementalCheckRequested) {
return
}
cache.incrementalCheckRequested = true
actions.loadIncrementalCheck()
},
updateDataWarehouseSavedQuerySuccess: ({ payload }) => {
// The sync frequency dropdown mutates this saved query through dataWarehouseViewsLogic, which
// doesn't own our `savedQuery`. Reload it so the displayed frequency reflects the new value
// immediately instead of reverting until the next poll.
if (payload?.id === props.viewId) {
// Pausing a view clears its pin on the server, so the pin held here may be out of date.
if (values.nodeFrequency) {
actions.loadNodeFrequency()
}
if (values.savingMaterialization && values.savedQuery) {
actions.loadSavedQuerySuccess({ ...values.savedQuery, ...payload } as DataWarehouseSavedQuery)
actions.discardMaterializationChanges()
actions.finishSavingMaterialization()
}
actions.loadSavedQuery()
}
},
loadDataModelingJobsSuccess: ({ dataModelingJobs }) => {
// Refresh saved query alongside jobs so latest_error / status / sync_frequency stay in sync.
actions.loadSavedQuery()
if (!dataModelingJobs.results.some((job) => job.status === 'Completed')) {
actions.loadLatestCompletedJob()
}
// The SQL editor sidebar reads run status from the view list, so reload it once when a run ends.
// Compare the newest run's id and status, because a run can start and end between idle polls.
const newestJob = dataModelingJobs.results[0]
const running = newestJob?.status === 'Running'
const newestJobState = newestJob ? `${newestJob.id}:${newestJob.status}` : null
if (cache.newestJobState !== undefined && cache.newestJobState !== newestJobState && !running) {
actions.loadDataWarehouseSavedQueries()
}
cache.newestJobState = newestJobState
// Until the new run's job row exists, the newest job is the run before the click.
if (values.startingMaterialization && isNewRunVisible(newestJob, cache.newestJobIdBeforeStart)) {
actions.setStartingMaterialization(false)
}
const active = values.startingMaterialization || running
actions.scheduleJobsRefresh(active ? ACTIVE_REFRESH_INTERVAL_MS : IDLE_REFRESH_INTERVAL_MS)
},
})),
])