Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

21.2 KB
Raw Blame
import { MakeLogicType, actions, afterMount, connect, kea, listeners, path, reducers, selectors } from 'kea'
import { loaders } from 'kea-loaders'
import posthog from 'posthog-js'

import type { Series } from '@posthog/quill-charts'

import { FEATURE_FLAGS } from 'lib/constants'
import { dayjs } from 'lib/dayjs'
import { featureFlagLogic, FeatureFlagsSet } from 'lib/logic/featureFlagLogic'
import { getCurrentTeamId } from 'lib/utils/getAppContext'
import {
    DataWarehouseSavedQuerySummary,
    dataWarehouseViewsLogic,
} from 'scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic'

import { dataModelingJobsList } from 'products/data_warehouse/frontend/generated/api'
import {
    DataModelingJobsListEngine,
    DataModelingJobsListStatus,
    type PaginatedDataModelingJobListApi,
} from 'products/data_warehouse/frontend/generated/api.schemas'

import * as api from './generated/api'
import {
    DataModelingUsageListEngine,
    type DataModelingUsageListParams,
    type ModelUsageApi,
} from './generated/api.schemas'

export const MATERIALIZATION_JOBS_PAGE_SIZE = 20
export const DEFAULT_MODELS_USAGE_DATE_FROM = '-30d'
export const MATERIALIZATION_ENGINE_LABELS: Record<string, string> = {
    all: 'All engines',
    clickhouse: 'ClickHouse',
    duckgres: 'Duckgres',
    managed_warehouse: 'Managed warehouse',
}

const MODEL_STATUS_ORDER = [
    DataModelingJobsListStatus.Completed,
    DataModelingJobsListStatus.Running,
    DataModelingJobsListStatus.Failed,
    DataModelingJobsListStatus.Cancelled,
    DataModelingJobsListStatus.Skipped,
]

const JOB_ENGINE_BY_USAGE_ENGINE: Partial<Record<DataModelingUsageListEngine, DataModelingJobsListEngine>> = {
    [DataModelingUsageListEngine.Clickhouse]: DataModelingJobsListEngine.Clickhouse,
    [DataModelingUsageListEngine.ManagedWarehouse]: DataModelingJobsListEngine.ManagedWarehouse,
}

function normalizeModelsUsageDateRange(
    dateFrom: string | null,
    dateTo: string | null
): { dateFrom: string; dateTo: string | null } {
    const normalizedDateFrom = dateFrom ?? DEFAULT_MODELS_USAGE_DATE_FROM
    if (dateTo === null && /^-(\d+)d$/.test(normalizedDateFrom)) {
        return { dateFrom: normalizedDateFrom, dateTo }
    }

    const today = dayjs.utc().startOf('day')
    const lastCompleteDay = today.subtract(1, 'day')
    const parsedDateFrom = dayjs.utc(normalizedDateFrom).startOf('day')
    const parsedDateTo = dateTo ? dayjs.utc(dateTo).startOf('day') : null
    const effectiveDateTo = parsedDateTo?.isBefore(today) ? parsedDateTo : lastCompleteDay
    const exclusiveDateTo = effectiveDateTo.add(1, 'day')
    const earliestDateFrom = exclusiveDateTo.subtract(366, 'day')
    const latestDateFrom = exclusiveDateTo.subtract(1, 'day')
    const effectiveDateFrom = parsedDateFrom.isBefore(earliestDateFrom)
        ? earliestDateFrom
        : parsedDateFrom.isAfter(latestDateFrom)
          ? latestDateFrom
          : parsedDateFrom

    return {
        dateFrom: effectiveDateFrom.format('YYYY-MM-DD'),
        dateTo: dateTo === null ? null : effectiveDateTo.format('YYYY-MM-DD'),
    }
}

function completeUtcUsageDateRange(
    dateFrom: string,
    dateTo: string | null
): {
    days?: number
    date_from?: string
    date_to?: string
} {
    const normalizedRange = normalizeModelsUsageDateRange(dateFrom, dateTo)
    dateFrom = normalizedRange.dateFrom
    dateTo = normalizedRange.dateTo
    const relativeDays = dateTo === null ? /^-(\d+)d$/.exec(dateFrom) : null
    if (relativeDays) {
        return { days: Number(relativeDays[1]) }
    }

    const start = dayjs.utc(dateFrom).startOf('day')
    return {
        date_from: start.format('YYYY-MM-DD'),
        date_to: dateTo ? dayjs.utc(dateTo).format('YYYY-MM-DD') : undefined,
    }
}

function materializationJobsDateFrom(dateFrom: string, dateTo: string | null): string {
    const normalizedRange = normalizeModelsUsageDateRange(dateFrom, dateTo)
    const relativeDays = normalizedRange.dateTo === null ? /^-(\d+)d$/.exec(normalizedRange.dateFrom) : null
    const start = relativeDays
        ? dayjs.utc().startOf('day').subtract(Number(relativeDays[1]), 'day')
        : dayjs.utc(normalizedRange.dateFrom).startOf('day')

    return start.toISOString()
}

export type ModelsUsageSortBy = NonNullable<DataModelingUsageListParams['sort_by']>
export type ModelsUsageSortDirection = NonNullable<DataModelingUsageListParams['sort_direction']>

export interface MaterializedViewOption {
    id: string
    name: string
}

export function modelStatusLabel(status: string): string {
    return status === DataModelingJobsListStatus.Cancelled ? 'Canceled' : status
}

// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface modelsUsageLogicValues {
    dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[] // dataWarehouseViewsLogic
    dataWarehouseSavedQueriesFailed: boolean // dataWarehouseViewsLogic
    dataWarehouseSavedQueriesLoading: boolean // dataWarehouseViewsLogic
    featureFlags: FeatureFlagsSet // featureFlagLogic
    chartLabels: string[]
    chartSeries: Series[]
    dateFrom: string
    dateTo: string | null
    failed: boolean
    jobsFailed: boolean
    jobsLoading: boolean
    jobsPage: number
    managedWarehouseEnabled: boolean
    materializationJobs: PaginatedDataModelingJobListApi | null
    materializationJobsFailed: boolean
    materializationJobsLoading: boolean
    materializedViewOptions: MaterializedViewOption[]
    page: number
    selectedEngine: DataModelingUsageListEngine
    selectedJobStatus: DataModelingJobsListStatus | null
    selectedMaterializedViewOptions: MaterializedViewOption[]
    selectedSavedQueryIds: string[]
    sortBy: ModelsUsageSortBy
    sortDirection: ModelsUsageSortDirection
    statusChartLabels: string[]
    statusChartSeries: Series[]
    usage: ModelUsageApi | null
    usageLoading: boolean
    viewFilterOptions: MaterializedViewOption[]
}

// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface modelsUsageLogicActions {
    loadDataWarehouseSavedQueries: () => any // dataWarehouseViewsLogic
    loadMaterializationJobs: (_: void) => void
    loadMaterializationJobsFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    loadMaterializationJobsSuccess: (
        materializationJobs: PaginatedDataModelingJobListApi,
        payload?: void
    ) => {
        materializationJobs: PaginatedDataModelingJobListApi
        payload?: void
    }
    loadUsage: (_: void) => void
    loadUsageFailure: (
        error: string,
        errorObject?: any
    ) => {
        error: string
        errorObject?: any
    }
    loadUsageSuccess: (
        usage: ModelUsageApi,
        payload?: void
    ) => {
        usage: ModelUsageApi
        payload?: void
    }
    openModel: () => {
        value: true
    }
    refresh: () => {
        value: true
    }
    refreshJobs: () => {
        value: true
    }
    setDateRange: (
        dateFrom: string | null,
        dateTo: string | null
    ) => {
        dateFrom: string
        dateTo: string | null
    }
    setJobsPage: (page: number) => {
        page: number
    }
    setPage: (page: number) => {
        page: number
    }
    setSelectedEngine: (engine: DataModelingUsageListEngine) => {
        engine: DataModelingUsageListEngine
    }
    setSelectedJobStatus: (status: DataModelingJobsListStatus | null) => {
        status: DataModelingJobsListStatus | null
    }
    setSelectedSavedQueryIds: (savedQueryIds: string[]) => {
        savedQueryIds: string[]
    }
    sortModels: (sortBy: ModelsUsageSortBy) => {
        sortBy: ModelsUsageSortBy
        sortDirection: ModelsUsageSortDirection
    }
}

// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface modelsUsageLogicMeta {
    __keaTypeGenInternalSelectorTypes: {
        managedWarehouseEnabled: (featureFlags: FeatureFlagsSet) => boolean
        materializedViewOptions: (
            dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[]
        ) => MaterializedViewOption[]
        viewFilterOptions: (
            materializedViewOptions: MaterializedViewOption[],
            selectedSavedQueryIds: string[],
            usage: ModelUsageApi | null
        ) => MaterializedViewOption[]
        selectedMaterializedViewOptions: (
            viewFilterOptions: MaterializedViewOption[],
            selectedSavedQueryIds: string[]
        ) => MaterializedViewOption[]
        jobsFailed: (materializationJobsFailed: boolean, dataWarehouseSavedQueriesFailed: boolean) => boolean
        jobsLoading: (materializationJobsLoading: boolean, dataWarehouseSavedQueriesLoading: boolean) => boolean
        chartLabels: (usage: ModelUsageApi | null) => string[]
        chartSeries: (usage: ModelUsageApi | null) => Series[]
        statusChartLabels: (usage: ModelUsageApi | null) => string[]
        statusChartSeries: (usage: ModelUsageApi | null) => Series[]
    }
}

export type modelsUsageLogicType = MakeLogicType<
    modelsUsageLogicValues,
    modelsUsageLogicActions,
    Record<string, any>,
    modelsUsageLogicMeta
>

export const modelsUsageLogic = kea<modelsUsageLogicType>([
    path(['products', 'data_modeling', 'frontend', 'modelsUsageLogic']),
    connect(() => ({
        values: [
            featureFlagLogic,
            ['featureFlags'],
            dataWarehouseViewsLogic,
            ['dataWarehouseSavedQueries', 'dataWarehouseSavedQueriesFailed', 'dataWarehouseSavedQueriesLoading'],
        ],
        actions: [dataWarehouseViewsLogic, ['loadDataWarehouseSavedQueries']],
    })),
    actions(({ values }) => ({
        setDateRange: (dateFrom: string | null, dateTo: string | null) =>
            normalizeModelsUsageDateRange(dateFrom, dateTo),
        setPage: (page: number) => ({ page }),
        sortModels: (sortBy: ModelsUsageSortBy) => ({
            sortBy,
            sortDirection: (sortBy === values.sortBy
                ? values.sortDirection === 'asc'
                    ? 'desc'
                    : 'asc'
                : sortBy === 'name'
                  ? 'asc'
                  : 'desc') as ModelsUsageSortDirection,
        }),
        setJobsPage: (page: number) => ({ page }),
        setSelectedEngine: (engine: DataModelingUsageListEngine) => ({ engine }),
        setSelectedJobStatus: (status: DataModelingJobsListStatus | null) => ({ status }),
        setSelectedSavedQueryIds: (savedQueryIds: string[]) => ({ savedQueryIds }),
        openModel: true,
        refresh: true,
        refreshJobs: true,
    })),
    reducers({
        dateFrom: [
            DEFAULT_MODELS_USAGE_DATE_FROM as string,
            { setDateRange: (_, { dateFrom }) => dateFrom ?? DEFAULT_MODELS_USAGE_DATE_FROM },
        ],
        dateTo: [null as string | null, { setDateRange: (_, { dateTo }) => dateTo }],
        page: [
            1,
            {
                setPage: (_, { page }) => page,
                sortModels: () => 1,
                setDateRange: () => 1,
                setSelectedEngine: () => 1,
                setSelectedSavedQueryIds: () => 1,
            },
        ],
        sortBy: [
            'read_bytes' as ModelsUsageSortBy,
            { sortModels: (_, { sortBy }) => sortBy, setSelectedEngine: () => 'read_bytes' },
        ],
        sortDirection: [
            'desc' as ModelsUsageSortDirection,
            {
                sortModels: (_, { sortDirection }) => sortDirection,
                setSelectedEngine: () => 'desc',
            },
        ],
        jobsPage: [
            1,
            {
                setJobsPage: (_, { page }) => page,
                setDateRange: () => 1,
                setSelectedEngine: () => 1,
                setSelectedJobStatus: () => 1,
                setSelectedSavedQueryIds: () => 1,
            },
        ],
        selectedEngine: [
            DataModelingUsageListEngine.All as DataModelingUsageListEngine,
            { setSelectedEngine: (_, { engine }) => engine },
        ],
        selectedJobStatus: [
            null as DataModelingJobsListStatus | null,
            { setSelectedJobStatus: (_, { status }) => status },
        ],
        selectedSavedQueryIds: [[] as string[], { setSelectedSavedQueryIds: (_, { savedQueryIds }) => savedQueryIds }],
        failed: [false, { loadUsage: () => false, loadUsageFailure: () => true }],
        materializationJobsFailed: [
            false,
            { loadMaterializationJobs: () => false, loadMaterializationJobsFailure: () => true },
        ],
    }),
    loaders(({ values }) => ({
        usage: [
            null as ModelUsageApi | null,
            {
                loadUsage: async (_: void, breakpoint): Promise<ModelUsageApi> => {
                    const dateRange = completeUtcUsageDateRange(values.dateFrom, values.dateTo)
                    const result = await api.dataModelingUsageList(String(getCurrentTeamId()), {
                        days: dateRange.days,
                        date_from: dateRange.date_from,
                        date_to: dateRange.date_to,
                        engine: values.managedWarehouseEnabled
                            ? values.selectedEngine
                            : DataModelingUsageListEngine.Clickhouse,
                        page: values.page,
                        page_size: 20,
                        sort_by: values.sortBy,
                        sort_direction: values.sortDirection,
                        saved_query_ids: values.selectedSavedQueryIds.join(',') || undefined,
                    })
                    breakpoint()
                    return result
                },
            },
        ],
        materializationJobs: [
            null as PaginatedDataModelingJobListApi | null,
            {
                loadMaterializationJobs: async (_: void, breakpoint): Promise<PaginatedDataModelingJobListApi> => {
                    const result = await dataModelingJobsList(String(getCurrentTeamId()), {
                        after: materializationJobsDateFrom(values.dateFrom, values.dateTo),
                        limit: MATERIALIZATION_JOBS_PAGE_SIZE,
                        offset: (values.jobsPage - 1) * MATERIALIZATION_JOBS_PAGE_SIZE,
                        saved_query_ids:
                            values.selectedSavedQueryIds.length > 0 ? values.selectedSavedQueryIds : undefined,
                        all_engines: values.managedWarehouseEnabled || undefined,
                        engine:
                            values.managedWarehouseEnabled && values.selectedEngine !== DataModelingUsageListEngine.All
                                ? JOB_ENGINE_BY_USAGE_ENGINE[values.selectedEngine]
                                : undefined,
                        status: values.selectedJobStatus ?? undefined,
                    })
                    breakpoint()
                    return result
                },
            },
        ],
    })),
    selectors({
        managedWarehouseEnabled: [
            (s) => [s.featureFlags],
            (featureFlags: FeatureFlagsSet): boolean => !!featureFlags[FEATURE_FLAGS.MANAGED_TRINO_QUERY],
        ],
        materializedViewOptions: [
            (s) => [s.dataWarehouseSavedQueries],
            (savedQueries: DataWarehouseSavedQuerySummary[]): MaterializedViewOption[] =>
                savedQueries
                    .filter((view) => view.is_materialized)
                    .map((view) => ({ id: view.id, name: view.name ?? 'Untitled view' }))
                    .sort((a, b) => a.name.localeCompare(b.name)),
        ],
        viewFilterOptions: [
            (s) => [s.materializedViewOptions, s.selectedSavedQueryIds, s.usage],
            (
                materializedViewOptions: MaterializedViewOption[],
                selectedSavedQueryIds: string[],
                usage: ModelUsageApi | null
            ): MaterializedViewOption[] => {
                const optionsById = new Map(materializedViewOptions.map((option) => [option.id, option]))
                const selectedIds = new Set(selectedSavedQueryIds)
                for (const breakdown of usage?.status_breakdown ?? []) {
                    for (const model of breakdown.models) {
                        if (selectedIds.has(model.saved_query_id) && !optionsById.has(model.saved_query_id)) {
                            optionsById.set(model.saved_query_id, { id: model.saved_query_id, name: model.name })
                        }
                    }
                }
                for (const id of selectedSavedQueryIds) {
                    if (!optionsById.has(id)) {
                        optionsById.set(id, { id, name: 'Unavailable model' })
                    }
                }
                return [...optionsById.values()]
            },
        ],
        selectedMaterializedViewOptions: [
            (s) => [s.viewFilterOptions, s.selectedSavedQueryIds],
            (viewFilterOptions: MaterializedViewOption[], selectedSavedQueryIds: string[]) => {
                const optionsById = new Map(viewFilterOptions.map((option) => [option.id, option]))
                return selectedSavedQueryIds.map((id) => optionsById.get(id)!)
            },
        ],
        jobsFailed: [
            (s) => [s.materializationJobsFailed, s.dataWarehouseSavedQueriesFailed],
            (materializationJobsFailed: boolean, dataWarehouseSavedQueriesFailed: boolean): boolean =>
                materializationJobsFailed || dataWarehouseSavedQueriesFailed,
        ],
        jobsLoading: [
            (s) => [s.materializationJobsLoading, s.dataWarehouseSavedQueriesLoading],
            (materializationJobsLoading: boolean, dataWarehouseSavedQueriesLoading: boolean): boolean =>
                materializationJobsLoading || dataWarehouseSavedQueriesLoading,
        ],
        chartLabels: [
            (s) => [s.usage],
            (usage: ModelUsageApi | null): string[] => usage?.trends.map((day) => day.day) ?? [],
        ],
        chartSeries: [
            (s) => [s.usage],
            (usage: ModelUsageApi | null): Series[] =>
                (usage?.trends[0]?.engines ?? []).map(({ engine }) => ({
                    key: engine,
                    label: MATERIALIZATION_ENGINE_LABELS[engine] ?? engine,
                    data:
                        usage?.trends.map(
                            (day) => day.engines.find((item) => item.engine === engine)?.read_bytes ?? NaN
                        ) ?? [],
                })),
        ],
        statusChartLabels: [
            (s) => [s.usage],
            (usage: ModelUsageApi | null): string[] =>
                MODEL_STATUS_ORDER.filter((status) =>
                    usage?.status_breakdown.some((item) => item.status === status)
                ).map(modelStatusLabel),
        ],
        statusChartSeries: [
            (s) => [s.usage],
            (usage: ModelUsageApi | null): Series[] => {
                const engines = [...new Set(usage?.status_breakdown.map((item) => item.engine) ?? [])]
                const statuses = MODEL_STATUS_ORDER.filter((status) =>
                    usage?.status_breakdown.some((item) => item.status === status)
                )
                return engines.map((engine) => ({
                    key: engine,
                    label: MATERIALIZATION_ENGINE_LABELS[engine] ?? engine,
                    data: statuses.map(
                        (status) =>
                            usage?.status_breakdown.find((item) => item.engine === engine && item.status === status)
                                ?.count ?? 0
                    ),
                }))
            },
        ],
    }),
    listeners(({ actions, values }) => ({
        setDateRange: () => {
            posthog.capture('models usage date range changed', {
                date_from: values.dateFrom,
                date_to: values.dateTo,
            })
            actions.loadUsage()
            actions.loadMaterializationJobs()
        },
        setPage: () => actions.loadUsage(),
        sortModels: () => {
            posthog.capture('models usage sort changed', {
                sort_by: values.sortBy,
                sort_direction: values.sortDirection,
            })
            actions.loadUsage()
        },
        setJobsPage: () => actions.loadMaterializationJobs(),
        setSelectedEngine: () => {
            actions.loadUsage()
            actions.loadMaterializationJobs()
        },
        setSelectedJobStatus: () => actions.loadMaterializationJobs(),
        setSelectedSavedQueryIds: () => {
            actions.loadUsage()
            actions.loadMaterializationJobs()
        },
        refreshJobs: () => {
            actions.loadDataWarehouseSavedQueries()
            actions.loadMaterializationJobs()
        },
        refresh: () => {
            actions.loadUsage()
            actions.refreshJobs()
        },
        loadUsageSuccess: ({ usage }) => {
            posthog.capture('models usage loaded', {
                date_from: values.dateFrom,
                date_to: values.dateTo,
                has_runs: usage.overview.runs > 0,
                has_measurements: usage.overview.measured_runs > 0,
            })
        },
        loadUsageFailure: () => {
            posthog.capture('models usage load failed')
        },
        openModel: () => {
            posthog.capture('models usage model opened')
        },
    })),
    afterMount(({ actions }) => {
        actions.loadUsage()
        actions.loadMaterializationJobs()
    }),
])