Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

12.0 KB
Raw Blame
import { MakeLogicType, actions, afterMount, connect, kea, listeners, path, reducers, selectors } from 'kea'
import { urlToAction } from 'kea-router'
import posthog from 'posthog-js'

import { FEATURE_FLAGS } from 'lib/constants'
import { featureFlagLogic, FeatureFlagsSet } from 'lib/logic/featureFlagLogic'
import type { DataWarehouseSavedQuerySummary } from 'scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic'
import { dataWarehouseViewsLogic } from 'scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic'
import { urls } from 'scenes/urls'

import { DataModelingEdge, DataModelingNode, DataModelingNodeType } from '~/types'

import { BehindScheduleModel, modelsBehindSchedule } from './freshness'
import { NodeSuspensionApi } from './generated/api.schemas'
import { lineageDataLogic } from './lineage/lineageDataLogic'
import { buildAdjacencyMaps, traverseLineage } from './lineage/lineageSearch'
import { servingSuspension } from './suspension'

const MODEL_NODE_TYPES = new Set<DataModelingNodeType>(['view', 'matview', 'endpoint'])

export type ModelsSceneTab = 'overview' | 'models' | 'lineage' | 'data-quality' | 'usage'

const MODELS_SCENE_TABS: ModelsSceneTab[] = ['overview', 'models', 'lineage', 'data-quality', 'usage']

function isModelsSceneTab(tab: unknown): tab is ModelsSceneTab {
    return typeof tab === 'string' && (MODELS_SCENE_TABS as string[]).includes(tab)
}

/** One row of the overview's attention table: a broken model, why, and what it holds up. */
export interface AttentionModel {
    node: DataModelingNode
    problem: 'Failed' | 'Suspended'
    reason: string | null
    downstreamCount: number
    skippedCount: number
}

export interface modelsSceneLogicValues {
    featureFlags: FeatureFlagsSet // featureFlagLogic
    dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[] // dataWarehouseViewsLogic
    dataWarehouseSavedQueriesFailed: boolean // dataWarehouseViewsLogic
    dataWarehouseSavedQueriesLoaded: boolean // dataWarehouseViewsLogic
    dataWarehouseSavedQueriesLoading: boolean // dataWarehouseViewsLogic
    activeTab: ModelsSceneTab
    lineageTabVisited: boolean
    nodes: DataModelingNode[] // lineageDataLogic
    nodesLoading: boolean // lineageDataLogic
    nodesLoaded: boolean // lineageDataLogic
    nodesFailed: boolean // lineageDataLogic
    edges: DataModelingEdge[] // lineageDataLogic
    now: number
    savedQueryIdToNodeId: Record<string, string>
    failingNodes: DataModelingNode[]
    suspendedNodes: DataModelingNode[]
    suspensionBySavedQueryId: Record<string, NodeSuspensionApi | undefined>
    attentionModels: AttentionModel[]
    behindSchedule: BehindScheduleModel[]
    modelsResolved: boolean
    modelsFailed: boolean
    noModelsYet: boolean
    usageEnabled: boolean
}

export interface modelsSceneLogicActions {
    loadDataWarehouseSavedQueries: () => any // dataWarehouseViewsLogic
    loadNodes: () => any // lineageDataLogic
    setActiveTab: (tab: ModelsSceneTab) => { tab: ModelsSceneTab }
    setNow: (now: number) => { now: number }
}

export interface modelsSceneLogicMeta {
    __keaTypeGenInternalSelectorTypes: {
        savedQueryIdToNodeId: (nodes: DataModelingNode[]) => Record<string, string>
        failingNodes: (nodes: DataModelingNode[]) => DataModelingNode[]
        suspendedNodes: (nodes: DataModelingNode[]) => DataModelingNode[]
        suspensionBySavedQueryId: (nodes: DataModelingNode[]) => Record<string, NodeSuspensionApi | undefined>
        attentionModels: (
            failingNodes: DataModelingNode[],
            suspendedNodes: DataModelingNode[],
            edges: DataModelingEdge[],
            nodes: DataModelingNode[],
            savedQueries: DataWarehouseSavedQuerySummary[]
        ) => AttentionModel[]
        behindSchedule: (
            nodes: DataModelingNode[],
            attentionModels: AttentionModel[],
            now: number
        ) => BehindScheduleModel[]
        modelsResolved: (nodesLoaded: boolean, dataWarehouseSavedQueriesLoaded: boolean) => boolean
        modelsFailed: (nodesFailed: boolean, dataWarehouseSavedQueriesFailed: boolean) => boolean
        noModelsYet: (nodes: DataModelingNode[], savedQueries: DataWarehouseSavedQuerySummary[]) => boolean
        usageEnabled: (featureFlags: FeatureFlagsSet) => boolean
    }
}

export type modelsSceneLogicType = MakeLogicType<
    modelsSceneLogicValues,
    modelsSceneLogicActions,
    Record<string, any>,
    modelsSceneLogicMeta
>

export const modelsSceneLogic = kea<modelsSceneLogicType>([
    path(['scenes', 'models', 'modelsSceneLogic']),
    connect(() => ({
        values: [
            featureFlagLogic,
            ['featureFlags'],
            dataWarehouseViewsLogic,
            [
                'dataWarehouseSavedQueries',
                'dataWarehouseSavedQueriesFailed',
                'dataWarehouseSavedQueriesLoaded',
                'dataWarehouseSavedQueriesLoading',
            ],
            lineageDataLogic,
            ['nodes', 'nodesLoading', 'nodesLoaded', 'nodesFailed', 'edges'],
        ],
        actions: [dataWarehouseViewsLogic, ['loadDataWarehouseSavedQueries'], lineageDataLogic, ['loadNodes']],
    })),
    actions({
        setActiveTab: (tab: ModelsSceneTab) => ({ tab }),
        setNow: (now: number) => ({ now }),
    }),
    reducers({
        activeTab: [
            'overview' as ModelsSceneTab,
            {
                setActiveTab: (_, { tab }) => tab,
            },
        ],
        lineageTabVisited: [
            false,
            {
                setActiveTab: (visited, { tab }) => visited || tab === 'lineage',
            },
        ],
        now: [
            Date.now(),
            {
                setNow: (_, { now }) => now,
            },
        ],
    }),
    selectors({
        savedQueryIdToNodeId: [
            (s) => [s.nodes],
            (nodes: DataModelingNode[]): Record<string, string> => {
                const map: Record<string, string> = {}
                for (const node of nodes) {
                    if (node.saved_query_id) {
                        map[node.saved_query_id] = node.id
                    }
                }
                return map
            },
        ],
        failingNodes: [
            (s) => [s.nodes],
            (nodes: DataModelingNode[]): DataModelingNode[] =>
                nodes.filter((node) => node.last_run_status === 'Failed'),
        ],
        /**
         * Only a marker on the serving engine means scheduled runs stopped. The shadow engine marks
         * its own failures, and those leave the schedule firing.
         */
        suspendedNodes: [
            (s) => [s.nodes],
            (nodes: DataModelingNode[]): DataModelingNode[] =>
                nodes.filter((node) => servingSuspension(node.suspended)),
        ],
        suspensionBySavedQueryId: [
            (s) => [s.suspendedNodes],
            (suspendedNodes: DataModelingNode[]): Record<string, NodeSuspensionApi | undefined> => {
                const map: Record<string, NodeSuspensionApi | undefined> = {}
                for (const node of suspendedNodes) {
                    if (node.saved_query_id) {
                        map[node.saved_query_id] = servingSuspension(node.suspended)
                    }
                }
                return map
            },
        ],
        /**
         * What the overview leads with. Suspension outranks a failure because scheduled runs have
         * stopped entirely, and within each the widest blast radius comes first.
         */
        attentionModels: [
            (s) => [s.failingNodes, s.suspendedNodes, s.edges, s.nodes],
            (
                failingNodes: DataModelingNode[],
                suspendedNodes: DataModelingNode[],
                edges: DataModelingEdge[],
                nodes: DataModelingNode[]
            ): AttentionModel[] => {
                const suspendedIds = new Set(suspendedNodes.map((node) => node.id))
                const affected = [...suspendedNodes, ...failingNodes.filter((node) => !suspendedIds.has(node.id))]
                if (affected.length === 0) {
                    return []
                }

                const nodeById: Record<string, DataModelingNode> = {}
                for (const node of nodes) {
                    nodeById[node.id] = node
                }
                const maps = buildAdjacencyMaps(edges)

                const rows = affected.map((node): AttentionModel => {
                    const suspension = suspendedIds.has(node.id) ? servingSuspension(node.suspended) : undefined
                    // The cone includes the node itself, which is not something it blocks.
                    const downstream = [...traverseLineage(node.id, maps, 'downstream')].filter((id) => id !== node.id)
                    return {
                        node,
                        problem: suspension ? 'Suspended' : 'Failed',
                        reason: suspension?.reason ?? node.last_run_error ?? null,
                        downstreamCount: downstream.length,
                        skippedCount: downstream.filter((id) => nodeById[id]?.last_run_status === 'Skipped').length,
                    }
                })

                return rows.sort((a, b) => {
                    if (a.problem !== b.problem) {
                        return a.problem === 'Suspended' ? -1 : 1
                    }
                    return b.downstreamCount - a.downstreamCount
                })
            },
        ],
        behindSchedule: [
            (s) => [s.nodes, s.attentionModels, s.now],
            (nodes: DataModelingNode[], attentionModels: AttentionModel[], now: number): BehindScheduleModel[] => {
                // A failed or suspended model is behind as a consequence, and its own row
                // already says why, so reporting it here would only repeat the same problem.
                const reported = new Set(attentionModels.map((row) => row.node.id))
                return modelsBehindSchedule(nodes, now).filter((row) => !reported.has(row.node.id))
            },
        ],
        modelsResolved: [
            (s) => [s.nodesLoaded, s.dataWarehouseSavedQueriesLoaded],
            (nodesLoaded: boolean, dataWarehouseSavedQueriesLoaded: boolean): boolean =>
                nodesLoaded && dataWarehouseSavedQueriesLoaded,
        ],
        modelsFailed: [
            (s) => [s.nodesFailed, s.dataWarehouseSavedQueriesFailed],
            (nodesFailed: boolean, dataWarehouseSavedQueriesFailed: boolean): boolean =>
                nodesFailed || dataWarehouseSavedQueriesFailed,
        ],
        noModelsYet: [
            (s) => [s.nodes, s.dataWarehouseSavedQueries],
            (nodes: DataModelingNode[], savedQueries: DataWarehouseSavedQuerySummary[]): boolean =>
                !nodes.some((node) => MODEL_NODE_TYPES.has(node.type)) && savedQueries.length === 0,
        ],
        usageEnabled: [
            (s) => [s.featureFlags],
            (featureFlags: FeatureFlagsSet): boolean => !!featureFlags[FEATURE_FLAGS.DATA_MODELING_USAGE],
        ],
    }),
    listeners(() => ({
        setActiveTab: ({ tab }) => {
            posthog.capture('models tab viewed', { tab })
        },
    })),
    urlToAction(({ actions, values }) => ({
        [urls.models()]: (_, searchParams) => {
            const requestedTab: ModelsSceneTab = isModelsSceneTab(searchParams.tab) ? searchParams.tab : 'overview'
            const tab: ModelsSceneTab = requestedTab === 'usage' && !values.usageEnabled ? 'overview' : requestedTab
            if (tab !== values.activeTab) {
                actions.setActiveTab(tab)
            }
        },
    })),
    afterMount(({ actions, cache, values }) => {
        posthog.capture('models tab viewed', { tab: values.activeTab })
        // Mounting dataWarehouseViewsLogic already starts this load, so only refresh when it was mounted earlier
        if (!values.dataWarehouseSavedQueriesLoading) {
            actions.loadDataWarehouseSavedQueries()
        }
        actions.setNow(Date.now())
        cache.disposables.add(() => {
            const intervalId = window.setInterval(() => actions.setNow(Date.now()), 60_000)
            return () => window.clearInterval(intervalId)
        })
    }),
])