:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { MakeLogicType, actions, afterMount, connect, kea, listeners, path, reducers, selectors } from 'kea'
import { urlToAction } from 'kea-router'
import posthog from 'posthog-js'
import { FEATURE_FLAGS } from 'lib/constants'
import { featureFlagLogic, FeatureFlagsSet } from 'lib/logic/featureFlagLogic'
import type { DataWarehouseSavedQuerySummary } from 'scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic'
import { dataWarehouseViewsLogic } from 'scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic'
import { urls } from 'scenes/urls'
import { DataModelingEdge, DataModelingNode, DataModelingNodeType } from '~/types'
import { BehindScheduleModel, modelsBehindSchedule } from './freshness'
import { NodeSuspensionApi } from './generated/api.schemas'
import { lineageDataLogic } from './lineage/lineageDataLogic'
import { buildAdjacencyMaps, traverseLineage } from './lineage/lineageSearch'
import { servingSuspension } from './suspension'
const MODEL_NODE_TYPES = new Set<DataModelingNodeType>(['view', 'matview', 'endpoint'])
export type ModelsSceneTab = 'overview' | 'models' | 'lineage' | 'data-quality' | 'usage'
const MODELS_SCENE_TABS: ModelsSceneTab[] = ['overview', 'models', 'lineage', 'data-quality', 'usage']
function isModelsSceneTab(tab: unknown): tab is ModelsSceneTab {
return typeof tab === 'string' && (MODELS_SCENE_TABS as string[]).includes(tab)
}
/** One row of the overview's attention table: a broken model, why, and what it holds up. */
export interface AttentionModel {
node: DataModelingNode
problem: 'Failed' | 'Suspended'
reason: string | null
downstreamCount: number
skippedCount: number
}
export interface modelsSceneLogicValues {
featureFlags: FeatureFlagsSet // featureFlagLogic
dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[] // dataWarehouseViewsLogic
dataWarehouseSavedQueriesFailed: boolean // dataWarehouseViewsLogic
dataWarehouseSavedQueriesLoaded: boolean // dataWarehouseViewsLogic
dataWarehouseSavedQueriesLoading: boolean // dataWarehouseViewsLogic
activeTab: ModelsSceneTab
lineageTabVisited: boolean
nodes: DataModelingNode[] // lineageDataLogic
nodesLoading: boolean // lineageDataLogic
nodesLoaded: boolean // lineageDataLogic
nodesFailed: boolean // lineageDataLogic
edges: DataModelingEdge[] // lineageDataLogic
now: number
savedQueryIdToNodeId: Record<string, string>
failingNodes: DataModelingNode[]
suspendedNodes: DataModelingNode[]
suspensionBySavedQueryId: Record<string, NodeSuspensionApi | undefined>
attentionModels: AttentionModel[]
behindSchedule: BehindScheduleModel[]
modelsResolved: boolean
modelsFailed: boolean
noModelsYet: boolean
usageEnabled: boolean
}
export interface modelsSceneLogicActions {
loadDataWarehouseSavedQueries: () => any // dataWarehouseViewsLogic
loadNodes: () => any // lineageDataLogic
setActiveTab: (tab: ModelsSceneTab) => { tab: ModelsSceneTab }
setNow: (now: number) => { now: number }
}
export interface modelsSceneLogicMeta {
__keaTypeGenInternalSelectorTypes: {
savedQueryIdToNodeId: (nodes: DataModelingNode[]) => Record<string, string>
failingNodes: (nodes: DataModelingNode[]) => DataModelingNode[]
suspendedNodes: (nodes: DataModelingNode[]) => DataModelingNode[]
suspensionBySavedQueryId: (nodes: DataModelingNode[]) => Record<string, NodeSuspensionApi | undefined>
attentionModels: (
failingNodes: DataModelingNode[],
suspendedNodes: DataModelingNode[],
edges: DataModelingEdge[],
nodes: DataModelingNode[],
savedQueries: DataWarehouseSavedQuerySummary[]
) => AttentionModel[]
behindSchedule: (
nodes: DataModelingNode[],
attentionModels: AttentionModel[],
now: number
) => BehindScheduleModel[]
modelsResolved: (nodesLoaded: boolean, dataWarehouseSavedQueriesLoaded: boolean) => boolean
modelsFailed: (nodesFailed: boolean, dataWarehouseSavedQueriesFailed: boolean) => boolean
noModelsYet: (nodes: DataModelingNode[], savedQueries: DataWarehouseSavedQuerySummary[]) => boolean
usageEnabled: (featureFlags: FeatureFlagsSet) => boolean
}
}
export type modelsSceneLogicType = MakeLogicType<
modelsSceneLogicValues,
modelsSceneLogicActions,
Record<string, any>,
modelsSceneLogicMeta
>
export const modelsSceneLogic = kea<modelsSceneLogicType>([
path(['scenes', 'models', 'modelsSceneLogic']),
connect(() => ({
values: [
featureFlagLogic,
['featureFlags'],
dataWarehouseViewsLogic,
[
'dataWarehouseSavedQueries',
'dataWarehouseSavedQueriesFailed',
'dataWarehouseSavedQueriesLoaded',
'dataWarehouseSavedQueriesLoading',
],
lineageDataLogic,
['nodes', 'nodesLoading', 'nodesLoaded', 'nodesFailed', 'edges'],
],
actions: [dataWarehouseViewsLogic, ['loadDataWarehouseSavedQueries'], lineageDataLogic, ['loadNodes']],
})),
actions({
setActiveTab: (tab: ModelsSceneTab) => ({ tab }),
setNow: (now: number) => ({ now }),
}),
reducers({
activeTab: [
'overview' as ModelsSceneTab,
{
setActiveTab: (_, { tab }) => tab,
},
],
lineageTabVisited: [
false,
{
setActiveTab: (visited, { tab }) => visited || tab === 'lineage',
},
],
now: [
Date.now(),
{
setNow: (_, { now }) => now,
},
],
}),
selectors({
savedQueryIdToNodeId: [
(s) => [s.nodes],
(nodes: DataModelingNode[]): Record<string, string> => {
const map: Record<string, string> = {}
for (const node of nodes) {
if (node.saved_query_id) {
map[node.saved_query_id] = node.id
}
}
return map
},
],
failingNodes: [
(s) => [s.nodes],
(nodes: DataModelingNode[]): DataModelingNode[] =>
nodes.filter((node) => node.last_run_status === 'Failed'),
],
/**
* Only a marker on the serving engine means scheduled runs stopped. The shadow engine marks
* its own failures, and those leave the schedule firing.
*/
suspendedNodes: [
(s) => [s.nodes],
(nodes: DataModelingNode[]): DataModelingNode[] =>
nodes.filter((node) => servingSuspension(node.suspended)),
],
suspensionBySavedQueryId: [
(s) => [s.suspendedNodes],
(suspendedNodes: DataModelingNode[]): Record<string, NodeSuspensionApi | undefined> => {
const map: Record<string, NodeSuspensionApi | undefined> = {}
for (const node of suspendedNodes) {
if (node.saved_query_id) {
map[node.saved_query_id] = servingSuspension(node.suspended)
}
}
return map
},
],
/**
* What the overview leads with. Suspension outranks a failure because scheduled runs have
* stopped entirely, and within each the widest blast radius comes first.
*/
attentionModels: [
(s) => [s.failingNodes, s.suspendedNodes, s.edges, s.nodes],
(
failingNodes: DataModelingNode[],
suspendedNodes: DataModelingNode[],
edges: DataModelingEdge[],
nodes: DataModelingNode[]
): AttentionModel[] => {
const suspendedIds = new Set(suspendedNodes.map((node) => node.id))
const affected = [...suspendedNodes, ...failingNodes.filter((node) => !suspendedIds.has(node.id))]
if (affected.length === 0) {
return []
}
const nodeById: Record<string, DataModelingNode> = {}
for (const node of nodes) {
nodeById[node.id] = node
}
const maps = buildAdjacencyMaps(edges)
const rows = affected.map((node): AttentionModel => {
const suspension = suspendedIds.has(node.id) ? servingSuspension(node.suspended) : undefined
// The cone includes the node itself, which is not something it blocks.
const downstream = [...traverseLineage(node.id, maps, 'downstream')].filter((id) => id !== node.id)
return {
node,
problem: suspension ? 'Suspended' : 'Failed',
reason: suspension?.reason ?? node.last_run_error ?? null,
downstreamCount: downstream.length,
skippedCount: downstream.filter((id) => nodeById[id]?.last_run_status === 'Skipped').length,
}
})
return rows.sort((a, b) => {
if (a.problem !== b.problem) {
return a.problem === 'Suspended' ? -1 : 1
}
return b.downstreamCount - a.downstreamCount
})
},
],
behindSchedule: [
(s) => [s.nodes, s.attentionModels, s.now],
(nodes: DataModelingNode[], attentionModels: AttentionModel[], now: number): BehindScheduleModel[] => {
// A failed or suspended model is behind as a consequence, and its own row
// already says why, so reporting it here would only repeat the same problem.
const reported = new Set(attentionModels.map((row) => row.node.id))
return modelsBehindSchedule(nodes, now).filter((row) => !reported.has(row.node.id))
},
],
modelsResolved: [
(s) => [s.nodesLoaded, s.dataWarehouseSavedQueriesLoaded],
(nodesLoaded: boolean, dataWarehouseSavedQueriesLoaded: boolean): boolean =>
nodesLoaded && dataWarehouseSavedQueriesLoaded,
],
modelsFailed: [
(s) => [s.nodesFailed, s.dataWarehouseSavedQueriesFailed],
(nodesFailed: boolean, dataWarehouseSavedQueriesFailed: boolean): boolean =>
nodesFailed || dataWarehouseSavedQueriesFailed,
],
noModelsYet: [
(s) => [s.nodes, s.dataWarehouseSavedQueries],
(nodes: DataModelingNode[], savedQueries: DataWarehouseSavedQuerySummary[]): boolean =>
!nodes.some((node) => MODEL_NODE_TYPES.has(node.type)) && savedQueries.length === 0,
],
usageEnabled: [
(s) => [s.featureFlags],
(featureFlags: FeatureFlagsSet): boolean => !!featureFlags[FEATURE_FLAGS.DATA_MODELING_USAGE],
],
}),
listeners(() => ({
setActiveTab: ({ tab }) => {
posthog.capture('models tab viewed', { tab })
},
})),
urlToAction(({ actions, values }) => ({
[urls.models()]: (_, searchParams) => {
const requestedTab: ModelsSceneTab = isModelsSceneTab(searchParams.tab) ? searchParams.tab : 'overview'
const tab: ModelsSceneTab = requestedTab === 'usage' && !values.usageEnabled ? 'overview' : requestedTab
if (tab !== values.activeTab) {
actions.setActiveTab(tab)
}
},
})),
afterMount(({ actions, cache, values }) => {
posthog.capture('models tab viewed', { tab: values.activeTab })
// Mounting dataWarehouseViewsLogic already starts this load, so only refresh when it was mounted earlier
if (!values.dataWarehouseSavedQueriesLoading) {
actions.loadDataWarehouseSavedQueries()
}
actions.setNow(Date.now())
cache.disposables.add(() => {
const intervalId = window.setInterval(() => actions.setNow(Date.now()), 60_000)
return () => window.clearInterval(intervalId)
})
}),
])