Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

10.8 KB
Raw Blame
import { router } from 'kea-router'
import { expectLogic } from 'kea-test-utils'

import { FEATURE_FLAGS } from 'lib/constants'
import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { dataWarehouseViewsLogic } from 'scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic'

import { useMocks } from '~/mocks/jest'
import { initKeaTests } from '~/test/init'
import { DataModelingNode } from '~/types'

import { lineageDataLogic } from './lineage/lineageDataLogic'
import { modelsSceneLogic } from './modelsSceneLogic'

function buildNode(id: string, overrides: Partial<DataModelingNode> = {}): DataModelingNode {
    return {
        id,
        name: id,
        type: 'matview',
        created_at: '2024-01-01T00:00:00Z',
        updated_at: '2024-01-01T00:00:00Z',
        upstream_count: 0,
        downstream_count: 0,
        saved_query_id: `query-${id}`,
        ...overrides,
    }
}

describe('modelsSceneLogic', () => {
    let logic: ReturnType<typeof modelsSceneLogic.build>

    const mount = async (path: string): Promise<void> => {
        router.actions.push(path)
        logic = modelsSceneLogic()
        logic.mount()
        await expectLogic(logic).toFinishAllListeners()
    }

    beforeEach(() => {
        useMocks({
            get: {
                '/api/environments/:team_id/data_modeling_nodes/': {
                    count: 7,
                    results: [
                        buildNode('healthy', {
                            last_run_status: 'Completed',
                            sync_interval: '1hour',
                            last_run_at: '2024-01-01T00:00:00Z',
                        }),
                        buildNode('broken', {
                            last_run_status: 'Failed',
                            last_run_error: 'Unknown table foo',
                            sync_interval: '1hour',
                            last_run_at: '2024-01-01T00:00:00Z',
                        }),
                        buildNode('child', { last_run_status: 'Skipped' }),
                        buildNode('grandchild', { last_run_status: 'Skipped' }),
                        buildNode('paused', {
                            last_run_status: 'Failed',
                            suspended: { clickhouse: { at: '2024-01-02T00:00:00Z', reason: 'boom', job_id: 'j1' } },
                        }),
                        buildNode('never-ran'),
                        buildNode('shadow-only', {
                            suspended: { duckgres: { at: '2024-01-02T00:00:00Z', reason: 'shadow', job_id: 'j2' } },
                        }),
                    ],
                },
                '/api/environments/:team_id/data_modeling_edges/': {
                    count: 2,
                    results: [
                        { id: 'e1', source_id: 'broken', target_id: 'child' },
                        { id: 'e2', source_id: 'child', target_id: 'grandchild' },
                    ],
                },
                '/api/environments/:team_id/warehouse_saved_queries/': {
                    count: 2,
                    results: [
                        { id: 'query-healthy', name: 'healthy', columns: [], is_materialized: true },
                        {
                            id: 'query-broken',
                            name: 'broken',
                            columns: [],
                            is_materialized: true,
                            latest_error: 'a stale copy nothing should read',
                        },
                    ],
                },
            },
        })
        initKeaTests()
        featureFlagLogic.actions.setFeatureFlags([], { [FEATURE_FLAGS.DATA_MODELING_USAGE]: true })
    })

    afterEach(() => {
        logic?.unmount()
    })

    it.each([
        ['/models', 'overview'],
        ['/models?tab=models', 'models'],
        ['/models?tab=lineage', 'lineage'],
        ['/models?tab=nope', 'overview'],
        ['/models?tab=data-quality', 'data-quality'],
        ['/models?tab=usage', 'usage'],
    ])('%s opens the %s tab', async (path, tab) => {
        await mount(path)
        expect(logic.values.activeTab).toEqual(tab)
        expect(logic.values.lineageTabVisited).toBe(tab === 'lineage')
    })

    it('falls back to overview when the usage tab is disabled', async () => {
        featureFlagLogic.actions.setFeatureFlags([], { [FEATURE_FLAGS.DATA_MODELING_USAGE]: false })

        await mount('/models?tab=usage')

        expect(logic.values.activeTab).toEqual('overview')
    })

    it('remembers that the lineage tab was visited after the user leaves it', async () => {
        await mount('/models')
        expect(logic.values.lineageTabVisited).toBe(false)

        router.actions.push('/models?tab=lineage')
        expect(logic.values.lineageTabVisited).toBe(true)

        router.actions.push('/models')
        expect(logic.values.activeTab).toEqual('overview')
        expect(logic.values.lineageTabVisited).toBe(true)
    })

    it('counts models whose last run failed and models that are suspended', async () => {
        await mount('/models')
        await expectLogic(lineageDataLogic).toDispatchActions(['loadNodesSuccess'])
        await expectLogic(dataWarehouseViewsLogic).toDispatchActions(['loadDataWarehouseSavedQueriesSuccess'])

        expect(logic.values.failingNodes.map((node) => node.id)).toEqual(['broken', 'paused'])
        expect(logic.values.suspendedNodes.map((node) => node.id)).toEqual(['paused'])
    })

    it('ignores a marker on an engine that does not serve queries', async () => {
        await mount('/models')
        await expectLogic(lineageDataLogic).toDispatchActions(['loadNodesSuccess'])

        // shadow-only carries a duckgres marker. ClickHouse still runs it on schedule.
        expect(logic.values.suspendedNodes.map((node) => node.id)).toEqual(['paused'])
    })

    it('lists broken models with their error and what they hold up', async () => {
        await mount('/models')
        await expectLogic(lineageDataLogic).toDispatchActions(['loadNodesSuccess', 'loadEdgesSuccess'])
        await expectLogic(dataWarehouseViewsLogic).toDispatchActions(['loadDataWarehouseSavedQueriesSuccess'])

        // Suspended sorts above failed, because a suspended model has stopped running altogether.
        expect(logic.values.attentionModels.map((row) => [row.node.id, row.problem])).toEqual([
            ['paused', 'Suspended'],
            ['broken', 'Failed'],
        ])

        const paused = logic.values.attentionModels[0]
        expect(paused.reason).toEqual('boom')
        expect(paused.downstreamCount).toEqual(0)

        // The whole cone counts, not just direct children, and the anchor is not one of them.
        const broken = logic.values.attentionModels[1]
        expect(broken.reason).toEqual('Unknown table foo')
        expect(broken.reason).not.toEqual('a stale copy nothing should read')
        expect(broken.downstreamCount).toEqual(2)
        expect(broken.skippedCount).toEqual(2)
    })

    it.each<[string, string, any, string]>([
        ['the model list', '/api/environments/:team_id/data_modeling_nodes/', lineageDataLogic, 'loadNodesFailure'],
        [
            'the view list',
            '/api/projects/:team_id/warehouse_saved_queries/',
            dataWarehouseViewsLogic,
            'loadDataWarehouseSavedQueriesFailure',
        ],
    ])('reports a failure instead of resolving when %s fails to load', async (_case, path, failedLogic, action) => {
        useMocks({ get: { [path]: () => [500, {}] } })
        await mount('/models')
        await expectLogic(failedLogic).toDispatchActions([action])

        expect(logic.values.modelsFailed).toBe(true)
        expect(logic.values.modelsResolved).toBe(false)
    })

    it.each<[string, DataModelingNode[], Record<string, any>[], boolean]>([
        ['neither models nor views', [], [], true],
        ['a view that is not a model yet', [], [{ id: 'query-1', name: 'a_view', columns: [] }], false],
        [
            'a warehouse source table and nothing else',
            [buildNode('source-table', { type: 'table', saved_query_id: undefined })],
            [],
            true,
        ],
    ])('reports a project with %s as new: %s', async (_case, nodes, savedQueries, expected) => {
        useMocks({
            get: {
                '/api/environments/:team_id/data_modeling_nodes/': { count: nodes.length, results: nodes },
                '/api/projects/:team_id/warehouse_saved_queries/': {
                    count: savedQueries.length,
                    results: savedQueries,
                },
            },
        })
        await mount('/models')
        await expectLogic(lineageDataLogic).toDispatchActions(['loadNodesSuccess'])
        await expectLogic(dataWarehouseViewsLogic).toDispatchActions(['loadDataWarehouseSavedQueriesSuccess'])

        expect(logic.values.modelsResolved).toBe(true)
        expect(logic.values.noModelsYet).toBe(expected)
    })

    it('stops reporting the model list resolved once a reload fails', async () => {
        await mount('/models')
        await expectLogic(lineageDataLogic).toDispatchActions(['loadNodesSuccess'])
        await expectLogic(dataWarehouseViewsLogic).toDispatchActions(['loadDataWarehouseSavedQueriesSuccess'])
        expect(logic.values.modelsResolved).toBe(true)

        useMocks({ get: { '/api/environments/:team_id/data_modeling_nodes/': () => [500, {}] } })
        lineageDataLogic.actions.loadNodes()
        await expectLogic(lineageDataLogic).toDispatchActions(['loadNodesFailure'])

        expect(logic.values.modelsResolved).toBe(false)
    })

    it('lists models behind schedule, minus the ones already listed as broken', async () => {
        await mount('/models')
        await expectLogic(lineageDataLogic).toDispatchActions(['loadNodesSuccess', 'loadEdgesSuccess'])

        // Both blew their cadence, but a broken model is already named above, and saying it
        // twice would send the reader to the same place for two different reasons.
        expect(logic.values.behindSchedule.map((row) => row.node.id)).toEqual(['healthy'])
    })

    it('updates models behind schedule when the scene clock advances without reloading nodes', async () => {
        const initialNow = new Date('2026-01-01T12:00:00Z').getTime()
        const nowSpy = jest.spyOn(Date, 'now').mockReturnValue(initialNow)
        try {
            await mount('/models')
            lineageDataLogic.actions.loadNodesSuccess([
                buildNode('nearly-behind', {
                    sync_interval: '1hour',
                    last_run_at: '2026-01-01T10:01:00Z',
                }),
            ])
            expect(logic.values.behindSchedule).toEqual([])

            nowSpy.mockReturnValue(initialNow + 2 * 60_000)
            logic.actions.setNow(Date.now())

            expect(logic.values.behindSchedule.map((row) => row.node.id)).toEqual(['nearly-behind'])
        } finally {
            nowSpy.mockRestore()
        }
    })
})