:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { MakeLogicType, actions, afterMount, connect, kea, listeners, path, reducers, selectors } from 'kea'
import { loaders } from 'kea-loaders'
import posthog from 'posthog-js'
import type { Series } from '@posthog/quill-charts'
import { FEATURE_FLAGS } from 'lib/constants'
import { dayjs } from 'lib/dayjs'
import { featureFlagLogic, FeatureFlagsSet } from 'lib/logic/featureFlagLogic'
import { getCurrentTeamId } from 'lib/utils/getAppContext'
import {
DataWarehouseSavedQuerySummary,
dataWarehouseViewsLogic,
} from 'scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic'
import { dataModelingJobsList } from 'products/data_warehouse/frontend/generated/api'
import {
DataModelingJobsListEngine,
DataModelingJobsListStatus,
type PaginatedDataModelingJobListApi,
} from 'products/data_warehouse/frontend/generated/api.schemas'
import * as api from './generated/api'
import {
DataModelingUsageListEngine,
type DataModelingUsageListParams,
type ModelUsageApi,
} from './generated/api.schemas'
export const MATERIALIZATION_JOBS_PAGE_SIZE = 20
export const DEFAULT_MODELS_USAGE_DATE_FROM = '-30d'
export const MATERIALIZATION_ENGINE_LABELS: Record<string, string> = {
all: 'All engines',
clickhouse: 'ClickHouse',
duckgres: 'Duckgres',
managed_warehouse: 'Managed warehouse',
}
const MODEL_STATUS_ORDER = [
DataModelingJobsListStatus.Completed,
DataModelingJobsListStatus.Running,
DataModelingJobsListStatus.Failed,
DataModelingJobsListStatus.Cancelled,
DataModelingJobsListStatus.Skipped,
]
const JOB_ENGINE_BY_USAGE_ENGINE: Partial<Record<DataModelingUsageListEngine, DataModelingJobsListEngine>> = {
[DataModelingUsageListEngine.Clickhouse]: DataModelingJobsListEngine.Clickhouse,
[DataModelingUsageListEngine.ManagedWarehouse]: DataModelingJobsListEngine.ManagedWarehouse,
}
function normalizeModelsUsageDateRange(
dateFrom: string | null,
dateTo: string | null
): { dateFrom: string; dateTo: string | null } {
const normalizedDateFrom = dateFrom ?? DEFAULT_MODELS_USAGE_DATE_FROM
if (dateTo === null && /^-(\d+)d$/.test(normalizedDateFrom)) {
return { dateFrom: normalizedDateFrom, dateTo }
}
const today = dayjs.utc().startOf('day')
const lastCompleteDay = today.subtract(1, 'day')
const parsedDateFrom = dayjs.utc(normalizedDateFrom).startOf('day')
const parsedDateTo = dateTo ? dayjs.utc(dateTo).startOf('day') : null
const effectiveDateTo = parsedDateTo?.isBefore(today) ? parsedDateTo : lastCompleteDay
const exclusiveDateTo = effectiveDateTo.add(1, 'day')
const earliestDateFrom = exclusiveDateTo.subtract(366, 'day')
const latestDateFrom = exclusiveDateTo.subtract(1, 'day')
const effectiveDateFrom = parsedDateFrom.isBefore(earliestDateFrom)
? earliestDateFrom
: parsedDateFrom.isAfter(latestDateFrom)
? latestDateFrom
: parsedDateFrom
return {
dateFrom: effectiveDateFrom.format('YYYY-MM-DD'),
dateTo: dateTo === null ? null : effectiveDateTo.format('YYYY-MM-DD'),
}
}
function completeUtcUsageDateRange(
dateFrom: string,
dateTo: string | null
): {
days?: number
date_from?: string
date_to?: string
} {
const normalizedRange = normalizeModelsUsageDateRange(dateFrom, dateTo)
dateFrom = normalizedRange.dateFrom
dateTo = normalizedRange.dateTo
const relativeDays = dateTo === null ? /^-(\d+)d$/.exec(dateFrom) : null
if (relativeDays) {
return { days: Number(relativeDays[1]) }
}
const start = dayjs.utc(dateFrom).startOf('day')
return {
date_from: start.format('YYYY-MM-DD'),
date_to: dateTo ? dayjs.utc(dateTo).format('YYYY-MM-DD') : undefined,
}
}
function materializationJobsDateFrom(dateFrom: string, dateTo: string | null): string {
const normalizedRange = normalizeModelsUsageDateRange(dateFrom, dateTo)
const relativeDays = normalizedRange.dateTo === null ? /^-(\d+)d$/.exec(normalizedRange.dateFrom) : null
const start = relativeDays
? dayjs.utc().startOf('day').subtract(Number(relativeDays[1]), 'day')
: dayjs.utc(normalizedRange.dateFrom).startOf('day')
return start.toISOString()
}
export type ModelsUsageSortBy = NonNullable<DataModelingUsageListParams['sort_by']>
export type ModelsUsageSortDirection = NonNullable<DataModelingUsageListParams['sort_direction']>
export interface MaterializedViewOption {
id: string
name: string
}
export function modelStatusLabel(status: string): string {
return status === DataModelingJobsListStatus.Cancelled ? 'Canceled' : status
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface modelsUsageLogicValues {
dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[] // dataWarehouseViewsLogic
dataWarehouseSavedQueriesFailed: boolean // dataWarehouseViewsLogic
dataWarehouseSavedQueriesLoading: boolean // dataWarehouseViewsLogic
featureFlags: FeatureFlagsSet // featureFlagLogic
chartLabels: string[]
chartSeries: Series[]
dateFrom: string
dateTo: string | null
failed: boolean
jobsFailed: boolean
jobsLoading: boolean
jobsPage: number
managedWarehouseEnabled: boolean
materializationJobs: PaginatedDataModelingJobListApi | null
materializationJobsFailed: boolean
materializationJobsLoading: boolean
materializedViewOptions: MaterializedViewOption[]
page: number
selectedEngine: DataModelingUsageListEngine
selectedJobStatus: DataModelingJobsListStatus | null
selectedMaterializedViewOptions: MaterializedViewOption[]
selectedSavedQueryIds: string[]
sortBy: ModelsUsageSortBy
sortDirection: ModelsUsageSortDirection
statusChartLabels: string[]
statusChartSeries: Series[]
usage: ModelUsageApi | null
usageLoading: boolean
viewFilterOptions: MaterializedViewOption[]
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface modelsUsageLogicActions {
loadDataWarehouseSavedQueries: () => any // dataWarehouseViewsLogic
loadMaterializationJobs: (_: void) => void
loadMaterializationJobsFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
loadMaterializationJobsSuccess: (
materializationJobs: PaginatedDataModelingJobListApi,
payload?: void
) => {
materializationJobs: PaginatedDataModelingJobListApi
payload?: void
}
loadUsage: (_: void) => void
loadUsageFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
loadUsageSuccess: (
usage: ModelUsageApi,
payload?: void
) => {
usage: ModelUsageApi
payload?: void
}
openModel: () => {
value: true
}
refresh: () => {
value: true
}
refreshJobs: () => {
value: true
}
setDateRange: (
dateFrom: string | null,
dateTo: string | null
) => {
dateFrom: string
dateTo: string | null
}
setJobsPage: (page: number) => {
page: number
}
setPage: (page: number) => {
page: number
}
setSelectedEngine: (engine: DataModelingUsageListEngine) => {
engine: DataModelingUsageListEngine
}
setSelectedJobStatus: (status: DataModelingJobsListStatus | null) => {
status: DataModelingJobsListStatus | null
}
setSelectedSavedQueryIds: (savedQueryIds: string[]) => {
savedQueryIds: string[]
}
sortModels: (sortBy: ModelsUsageSortBy) => {
sortBy: ModelsUsageSortBy
sortDirection: ModelsUsageSortDirection
}
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface modelsUsageLogicMeta {
__keaTypeGenInternalSelectorTypes: {
managedWarehouseEnabled: (featureFlags: FeatureFlagsSet) => boolean
materializedViewOptions: (
dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[]
) => MaterializedViewOption[]
viewFilterOptions: (
materializedViewOptions: MaterializedViewOption[],
selectedSavedQueryIds: string[],
usage: ModelUsageApi | null
) => MaterializedViewOption[]
selectedMaterializedViewOptions: (
viewFilterOptions: MaterializedViewOption[],
selectedSavedQueryIds: string[]
) => MaterializedViewOption[]
jobsFailed: (materializationJobsFailed: boolean, dataWarehouseSavedQueriesFailed: boolean) => boolean
jobsLoading: (materializationJobsLoading: boolean, dataWarehouseSavedQueriesLoading: boolean) => boolean
chartLabels: (usage: ModelUsageApi | null) => string[]
chartSeries: (usage: ModelUsageApi | null) => Series[]
statusChartLabels: (usage: ModelUsageApi | null) => string[]
statusChartSeries: (usage: ModelUsageApi | null) => Series[]
}
}
export type modelsUsageLogicType = MakeLogicType<
modelsUsageLogicValues,
modelsUsageLogicActions,
Record<string, any>,
modelsUsageLogicMeta
>
export const modelsUsageLogic = kea<modelsUsageLogicType>([
path(['products', 'data_modeling', 'frontend', 'modelsUsageLogic']),
connect(() => ({
values: [
featureFlagLogic,
['featureFlags'],
dataWarehouseViewsLogic,
['dataWarehouseSavedQueries', 'dataWarehouseSavedQueriesFailed', 'dataWarehouseSavedQueriesLoading'],
],
actions: [dataWarehouseViewsLogic, ['loadDataWarehouseSavedQueries']],
})),
actions(({ values }) => ({
setDateRange: (dateFrom: string | null, dateTo: string | null) =>
normalizeModelsUsageDateRange(dateFrom, dateTo),
setPage: (page: number) => ({ page }),
sortModels: (sortBy: ModelsUsageSortBy) => ({
sortBy,
sortDirection: (sortBy === values.sortBy
? values.sortDirection === 'asc'
? 'desc'
: 'asc'
: sortBy === 'name'
? 'asc'
: 'desc') as ModelsUsageSortDirection,
}),
setJobsPage: (page: number) => ({ page }),
setSelectedEngine: (engine: DataModelingUsageListEngine) => ({ engine }),
setSelectedJobStatus: (status: DataModelingJobsListStatus | null) => ({ status }),
setSelectedSavedQueryIds: (savedQueryIds: string[]) => ({ savedQueryIds }),
openModel: true,
refresh: true,
refreshJobs: true,
})),
reducers({
dateFrom: [
DEFAULT_MODELS_USAGE_DATE_FROM as string,
{ setDateRange: (_, { dateFrom }) => dateFrom ?? DEFAULT_MODELS_USAGE_DATE_FROM },
],
dateTo: [null as string | null, { setDateRange: (_, { dateTo }) => dateTo }],
page: [
1,
{
setPage: (_, { page }) => page,
sortModels: () => 1,
setDateRange: () => 1,
setSelectedEngine: () => 1,
setSelectedSavedQueryIds: () => 1,
},
],
sortBy: [
'read_bytes' as ModelsUsageSortBy,
{ sortModels: (_, { sortBy }) => sortBy, setSelectedEngine: () => 'read_bytes' },
],
sortDirection: [
'desc' as ModelsUsageSortDirection,
{
sortModels: (_, { sortDirection }) => sortDirection,
setSelectedEngine: () => 'desc',
},
],
jobsPage: [
1,
{
setJobsPage: (_, { page }) => page,
setDateRange: () => 1,
setSelectedEngine: () => 1,
setSelectedJobStatus: () => 1,
setSelectedSavedQueryIds: () => 1,
},
],
selectedEngine: [
DataModelingUsageListEngine.All as DataModelingUsageListEngine,
{ setSelectedEngine: (_, { engine }) => engine },
],
selectedJobStatus: [
null as DataModelingJobsListStatus | null,
{ setSelectedJobStatus: (_, { status }) => status },
],
selectedSavedQueryIds: [[] as string[], { setSelectedSavedQueryIds: (_, { savedQueryIds }) => savedQueryIds }],
failed: [false, { loadUsage: () => false, loadUsageFailure: () => true }],
materializationJobsFailed: [
false,
{ loadMaterializationJobs: () => false, loadMaterializationJobsFailure: () => true },
],
}),
loaders(({ values }) => ({
usage: [
null as ModelUsageApi | null,
{
loadUsage: async (_: void, breakpoint): Promise<ModelUsageApi> => {
const dateRange = completeUtcUsageDateRange(values.dateFrom, values.dateTo)
const result = await api.dataModelingUsageList(String(getCurrentTeamId()), {
days: dateRange.days,
date_from: dateRange.date_from,
date_to: dateRange.date_to,
engine: values.managedWarehouseEnabled
? values.selectedEngine
: DataModelingUsageListEngine.Clickhouse,
page: values.page,
page_size: 20,
sort_by: values.sortBy,
sort_direction: values.sortDirection,
saved_query_ids: values.selectedSavedQueryIds.join(',') || undefined,
})
breakpoint()
return result
},
},
],
materializationJobs: [
null as PaginatedDataModelingJobListApi | null,
{
loadMaterializationJobs: async (_: void, breakpoint): Promise<PaginatedDataModelingJobListApi> => {
const result = await dataModelingJobsList(String(getCurrentTeamId()), {
after: materializationJobsDateFrom(values.dateFrom, values.dateTo),
limit: MATERIALIZATION_JOBS_PAGE_SIZE,
offset: (values.jobsPage - 1) * MATERIALIZATION_JOBS_PAGE_SIZE,
saved_query_ids:
values.selectedSavedQueryIds.length > 0 ? values.selectedSavedQueryIds : undefined,
all_engines: values.managedWarehouseEnabled || undefined,
engine:
values.managedWarehouseEnabled && values.selectedEngine !== DataModelingUsageListEngine.All
? JOB_ENGINE_BY_USAGE_ENGINE[values.selectedEngine]
: undefined,
status: values.selectedJobStatus ?? undefined,
})
breakpoint()
return result
},
},
],
})),
selectors({
managedWarehouseEnabled: [
(s) => [s.featureFlags],
(featureFlags: FeatureFlagsSet): boolean => !!featureFlags[FEATURE_FLAGS.MANAGED_TRINO_QUERY],
],
materializedViewOptions: [
(s) => [s.dataWarehouseSavedQueries],
(savedQueries: DataWarehouseSavedQuerySummary[]): MaterializedViewOption[] =>
savedQueries
.filter((view) => view.is_materialized)
.map((view) => ({ id: view.id, name: view.name ?? 'Untitled view' }))
.sort((a, b) => a.name.localeCompare(b.name)),
],
viewFilterOptions: [
(s) => [s.materializedViewOptions, s.selectedSavedQueryIds, s.usage],
(
materializedViewOptions: MaterializedViewOption[],
selectedSavedQueryIds: string[],
usage: ModelUsageApi | null
): MaterializedViewOption[] => {
const optionsById = new Map(materializedViewOptions.map((option) => [option.id, option]))
const selectedIds = new Set(selectedSavedQueryIds)
for (const breakdown of usage?.status_breakdown ?? []) {
for (const model of breakdown.models) {
if (selectedIds.has(model.saved_query_id) && !optionsById.has(model.saved_query_id)) {
optionsById.set(model.saved_query_id, { id: model.saved_query_id, name: model.name })
}
}
}
for (const id of selectedSavedQueryIds) {
if (!optionsById.has(id)) {
optionsById.set(id, { id, name: 'Unavailable model' })
}
}
return [...optionsById.values()]
},
],
selectedMaterializedViewOptions: [
(s) => [s.viewFilterOptions, s.selectedSavedQueryIds],
(viewFilterOptions: MaterializedViewOption[], selectedSavedQueryIds: string[]) => {
const optionsById = new Map(viewFilterOptions.map((option) => [option.id, option]))
return selectedSavedQueryIds.map((id) => optionsById.get(id)!)
},
],
jobsFailed: [
(s) => [s.materializationJobsFailed, s.dataWarehouseSavedQueriesFailed],
(materializationJobsFailed: boolean, dataWarehouseSavedQueriesFailed: boolean): boolean =>
materializationJobsFailed || dataWarehouseSavedQueriesFailed,
],
jobsLoading: [
(s) => [s.materializationJobsLoading, s.dataWarehouseSavedQueriesLoading],
(materializationJobsLoading: boolean, dataWarehouseSavedQueriesLoading: boolean): boolean =>
materializationJobsLoading || dataWarehouseSavedQueriesLoading,
],
chartLabels: [
(s) => [s.usage],
(usage: ModelUsageApi | null): string[] => usage?.trends.map((day) => day.day) ?? [],
],
chartSeries: [
(s) => [s.usage],
(usage: ModelUsageApi | null): Series[] =>
(usage?.trends[0]?.engines ?? []).map(({ engine }) => ({
key: engine,
label: MATERIALIZATION_ENGINE_LABELS[engine] ?? engine,
data:
usage?.trends.map(
(day) => day.engines.find((item) => item.engine === engine)?.read_bytes ?? NaN
) ?? [],
})),
],
statusChartLabels: [
(s) => [s.usage],
(usage: ModelUsageApi | null): string[] =>
MODEL_STATUS_ORDER.filter((status) =>
usage?.status_breakdown.some((item) => item.status === status)
).map(modelStatusLabel),
],
statusChartSeries: [
(s) => [s.usage],
(usage: ModelUsageApi | null): Series[] => {
const engines = [...new Set(usage?.status_breakdown.map((item) => item.engine) ?? [])]
const statuses = MODEL_STATUS_ORDER.filter((status) =>
usage?.status_breakdown.some((item) => item.status === status)
)
return engines.map((engine) => ({
key: engine,
label: MATERIALIZATION_ENGINE_LABELS[engine] ?? engine,
data: statuses.map(
(status) =>
usage?.status_breakdown.find((item) => item.engine === engine && item.status === status)
?.count ?? 0
),
}))
},
],
}),
listeners(({ actions, values }) => ({
setDateRange: () => {
posthog.capture('models usage date range changed', {
date_from: values.dateFrom,
date_to: values.dateTo,
})
actions.loadUsage()
actions.loadMaterializationJobs()
},
setPage: () => actions.loadUsage(),
sortModels: () => {
posthog.capture('models usage sort changed', {
sort_by: values.sortBy,
sort_direction: values.sortDirection,
})
actions.loadUsage()
},
setJobsPage: () => actions.loadMaterializationJobs(),
setSelectedEngine: () => {
actions.loadUsage()
actions.loadMaterializationJobs()
},
setSelectedJobStatus: () => actions.loadMaterializationJobs(),
setSelectedSavedQueryIds: () => {
actions.loadUsage()
actions.loadMaterializationJobs()
},
refreshJobs: () => {
actions.loadDataWarehouseSavedQueries()
actions.loadMaterializationJobs()
},
refresh: () => {
actions.loadUsage()
actions.refreshJobs()
},
loadUsageSuccess: ({ usage }) => {
posthog.capture('models usage loaded', {
date_from: values.dateFrom,
date_to: values.dateTo,
has_runs: usage.overview.runs > 0,
has_measurements: usage.overview.measured_runs > 0,
})
},
loadUsageFailure: () => {
posthog.capture('models usage load failed')
},
openModel: () => {
posthog.capture('models usage model opened')
},
})),
afterMount(({ actions }) => {
actions.loadUsage()
actions.loadMaterializationJobs()
}),
])