:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-modeling): let a view pin its cadence below consumer demand (#128082)
Jovan Sakovic807db8c6caed
diff --git a/frontend/src/scenes/data-warehouse/saved_queries/MaterializationStatusPanel.tsx b/frontend/src/scenes/data-warehouse/saved_queries/MaterializationStatusPanel.tsxindex 438c5f697943..2663abf72e3f 100644--- a/frontend/src/scenes/data-warehouse/saved_queries/MaterializationStatusPanel.tsx+++ b/frontend/src/scenes/data-warehouse/saved_queries/MaterializationStatusPanel.tsx@@ -149,6 +149,9 @@ export function MaterializationStatusPanel({incrementalDraft,syncFrequencyDraft,savingMaterialization,+ cadencePin,+ nodeFrequency,+ nodeFrequencyLoading,} = useValues(jobsLogic)const {loadDataModelingJobs,@@ -159,6 +162,7 @@ export function MaterializationStatusPanel({setInitialSyncFrequency,setIncrementalDraft,setSyncFrequencyDraft,+ setPinDraft,} = useActions(jobsLogic)const { updatingDataWarehouseSavedQuery, materializationActionLoading } = useValues(dataWarehouseViewsLogic)@@ -278,7 +282,10 @@ export function MaterializationStatusPanel({</div>)}<div className="flex flex-col gap-2 items-start mt-2">- {kind !== 'endpoint' && (+ {kind !== 'endpoint' && nodeFrequencyLoading && !nodeFrequency ? (+ // Without the pin state the bar would first render as unpinned, then jump.+ <LemonSkeleton className="h-8 w-80" />+ ) : kind !== 'endpoint' ? (<SyncFrequencySelectbounds={savedQuery.sync_frequency_bounds}disabledReason={@@ -295,9 +302,15 @@ export function MaterializationStatusPanel({'never'}onChange={setSyncFrequencyDraft}- loading={updatingDataWarehouseSavedQuery || materializationActionLoading}+ loading={+ savingMaterialization ||+ updatingDataWarehouseSavedQuery ||+ materializationActionLoading+ }+ pin={cadencePin}+ onUnpin={() => setPinDraft(false)}/>- )}+ ) : null}{showRunActions && (<div className="flex flex-wrap items-center gap-2"><MaterializationRunActions viewId={viewId} kind={kind} />diff --git a/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.test.ts b/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.test.tsindex b23f4ecb103d..22469b2b3ca6 100644--- a/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.test.ts+++ b/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.test.ts@@ -1,3 +1,5 @@+import { DataModelingSyncInterval } from '~/types'+import { SyncFrequencyBoundsApi } from 'products/data_warehouse/frontend/generated/api.schemas'import {@@ -6,7 +8,10 @@ import {defaultCadenceWithin,inheritedCadence,modeDisabledReason,+ needsPin,+ pinNote,SyncFrequencyValue,+ unpinnedCadence,speedUpWarning,unsatisfiableReason,} from './SyncFrequencySelect'@@ -14,6 +19,9 @@ import {describe('SyncFrequencySelect', () => {const blocker = (name: string): { id: string; name: string } => ({ id: `id-${name}`, name })+ type PinOptions = NonNullable<Parameters<typeof buildOptions>[1]>+ const pinning = (options: PinOptions): PinOptions => options+const bounds = (overrides: Partial<SyncFrequencyBoundsApi> = {}): SyncFrequencyBoundsApi =>({frequency_mode: 'tiered',@@ -36,14 +44,131 @@ describe('SyncFrequencySelect', () => {expect(options.every((option) => !option.disabledReason)).toBe(true)})- it('disables a blocked cadence and names what blocks it', () => {- const options = buildOptions(bounds())+ it.each([+ ['without pinning', pinning({}), 'Too slow for downstream view daily_revenue', 'Every 1 day'],+ [+ 'with pinning, a too-slow cadence names the views the pin delays',+ pinning({+ canPin: true,+ unpinnedBy: { '24hour': '6hour' },+ pinDelays: { '24hour': [blocker('daily_revenue'), blocker('mrr')] },+ }),+ undefined,+ 'Pin this view to every 1 day. Downstream views daily_revenue, mrr refresh more often and will read older data from it.',+ ],+ [+ 'with pinning, a too-slow cadence falls back to the view that caps it',+ pinning({ canPin: true, unpinnedBy: { '24hour': '6hour' } }),+ undefined,+ 'Pin this view to every 1 day. Downstream view daily_revenue refreshes more often and will read older data from it.',+ ],+ [+ 'with pinning, a pin that strands a downstream view stays off',+ pinning({ canPin: true, unpinnedBy: { '24hour': '6hour' }, pinBlockedBy: { '24hour': blocker('maps') } }),+ "Can't pin: downstream view maps reads fresh data only through this view",+ 'Every 1 day',+ ],+ [+ 'with pinning, a pin that strands a view the caller cannot read is still refused',+ pinning({+ canPin: true,+ unpinnedBy: { '24hour': '6hour' },+ pinBlockedBy: { '24hour': { id: null, name: null } },+ }),+ "Can't pin: a downstream view reads fresh data only through this view",+ 'Every 1 day',+ ],+ ])(+ 'disables a blocked cadence and names what blocks it %s',+ (_case, pinOptions, consumerReason, consumerTooltip) => {+ const options = buildOptions(bounds(), pinOptions)++ const bySource = options.find((option) => option.value === '15min')+ const byConsumer = options.find((option) => option.value === '24hour')+ expect(bySource?.disabledReason).toBe('More often than upstream source stripe_invoices syncs')+ expect(byConsumer?.disabledReason).toBe(consumerReason)+ expect(byConsumer?.tooltip).toBe(consumerTooltip)+ expect(options.find((option) => option.value === '6hour')?.disabledReason).toBeUndefined()+ }+ )++ it.each([+ [+ 'a pinned view, whose bounds no longer carry a consumer ceiling',+ true,+ "Can't pin: downstream view maps reads fresh data only through this view",+ ],+ ['an unpinned view that can take the cadence without a pin', false, undefined],+ ])('only withholds a slower cadence that strands a downstream view for %s', (_case, pinned, reason) => {+ const pinnedBounds = bounds({+ options: [+ { cadence: '6hour', allowed: true, blocked_by: null, blocker: null },+ { cadence: '24hour', allowed: true, blocked_by: null, blocker: null },+ ],+ } as Partial<SyncFrequencyBoundsApi>)++ const options = buildOptions(pinnedBounds, {+ canPin: true,+ pinned,+ pinBlockedBy: { '24hour': blocker('maps') },+ })- const bySource = options.find((option) => option.value === '15min')- const byConsumer = options.find((option) => option.value === '24hour')- expect(bySource?.disabledReason).toBe('More often than upstream source stripe_invoices syncs')- expect(byConsumer?.disabledReason).toBe('Too slow for downstream view daily_revenue')- expect(options.find((option) => option.value === '6hour')?.disabledReason).toBeUndefined()+ expect(options.find((option) => option.value === '24hour')?.disabledReason).toBe(reason)+ })++ describe('needsPin', () => {+ it.each([+ ['a downstream view asks for fresher data than the cadence', '24hour', { '24hour': '15min' }, true],+ ['nothing downstream asks for fresher data', '24hour', { '24hour': '24hour' }, false],+ ['the source floor holds the view back instead', '15min', { '15min': '1hour' }, false],+ ['the cadence is not in the bounds', '24hour', {}, false],+ ['the view is paused', 'never', { '24hour': '15min' }, false],+ ])('is decided from the node bounds when %s', (_case, cadence, unpinnedBy, expected) => {+ expect(+ needsPin(unpinnedBy as Partial<Record<string, DataModelingSyncInterval>>, cadence as SyncFrequencyValue)+ ).toBe(expected)+ })++ it('asks a pinned view for a pin on a slower cadence, which the saved-query bounds no longer refuse', () => {+ const options = buildOptions(+ bounds({+ options: [+ { cadence: '6hour', allowed: true, blocked_by: null, blocker: null },+ { cadence: '24hour', allowed: true, blocked_by: null, blocker: null },+ ],+ } as Partial<SyncFrequencyBoundsApi>),+ { canPin: true, unpinnedBy: { '24hour': '15min' } }+ )++ expect(options.find((option) => option.value === '24hour')?.tooltip).toContain(+ 'Pin this view to every 1 day.'+ )+ })+ })++ describe('pinNote', () => {+ it.each([+ [+ 'a saved pin nothing reads faster than',+ false,+ [],+ 'Pinned to refresh every 1 day, even when downstream views need fresher data.',+ ],+ [+ 'a saved pin that delays one downstream view',+ false,+ ['daily_revenue'],+ 'Pinned to refresh every 1 day. Downstream view daily_revenue refreshes more often, so it reads data up to 1 day old.',+ ],+ [+ 'an unsaved pin that delays two downstream views',+ true,+ ['daily_revenue', 'mrr'],+ 'Saving pins this view to refresh every 1 day. Downstream views daily_revenue, mrr refresh more often, so they read data up to 1 day old.',+ ],+ ])('describes %s', (_case, pending, names, expected) => {+ expect(pinNote('24hour', names.map(blocker), { pending })).toBe(expected)+ })})it('explains the range only where a per-view cadence is writable', () => {@@ -281,7 +406,7 @@ describe('SyncFrequencySelect', () => {'24hour',{cadence: '6hour',- note: 'Refreshes every 6 hours to keep up with downstream view maps. Your 1 day setting applies again if maps refreshes less often.',+ note: 'Refreshes every 6 hours to keep up with downstream view maps. Your 1 day setting applies again if downstream views refresh less often.',},],[@@ -290,7 +415,7 @@ describe('SyncFrequencySelect', () => {'24hour',{cadence: '6hour',- note: 'Refreshes every 6 hours to keep up with a downstream view. Your 1 day setting applies again if that view refreshes less often.',+ note: 'Refreshes every 6 hours to keep up with a downstream view. Your 1 day setting applies again if downstream views refresh less often.',},],])('when %s', (_case, input, value, expected) => {@@ -298,3 +423,36 @@ describe('SyncFrequencySelect', () => {})})})++describe('unpinnedCadence', () => {+ const delaysBy = {+ '15min': [],+ '30min': [{ name: 'signups' }],+ '1hour': [{ name: 'signups' }],+ '6hour': [{ name: 'signups' }],+ }+ // Server-calculated, so a pin further downstream can stop demand: 1 hour unpinned runs hourly even+ // though a descendant below that pin asks for 15 minutes.+ const unpinnedBy = {+ '15min': '15min',+ '30min': '30min',+ '1hour': '1hour',+ '6hour': '15min',+ } as const++ it.each([+ [+ 'a pin that delays a consumer drops back to the cadence the server reports',+ '6hour',+ {+ cadence: '15min',+ note: 'Once you save, this view refreshes every 15 minutes to keep up with downstream view signups.',+ },+ ],+ ['another pin that stops demand keeps the cadence it reports', '1hour', null],+ ['a pin that delays nobody changes nothing', '15min', null],+ ['a paused view changes nothing', 'never', null],+ ])('%s', (_case, value, expected) => {+ expect(unpinnedCadence(unpinnedBy, delaysBy, value as SyncFrequencyValue)).toEqual(expected)+ })+})diff --git a/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.tsx b/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.tsxindex 463cece1a3c4..9b897c5d48c3 100644--- a/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.tsx+++ b/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.tsx@@ -1,10 +1,11 @@-import { IconInfo, IconRabbit, IconTortoise } from '@posthog/icons'+import { IconInfo, IconPinFilled, IconRabbit, IconTortoise, IconX } from '@posthog/icons'import { LemonLabel, LemonSegmentedButton, LemonSegmentedButtonOption, Spinner } from '@posthog/lemon-ui'import { Tooltip } from 'lib/lemon-ui/Tooltip'import { DataModelingSyncInterval } from '~/types'+import { NodePinBlockerApi, NodePinDelayApi } from 'products/data_modeling/frontend/generated/api.schemas'import { SyncFrequencyBoundsApi } from 'products/data_warehouse/frontend/generated/api.schemas'export type SyncFrequencyValue = DataModelingSyncInterval | 'never'@@ -62,6 +63,21 @@ export function modeDisabledReason(bounds?: SyncFrequencyBoundsApi | null): strireturn (bounds && MODE_DISABLED_REASONS[bounds.frequency_mode]) || null}+export interface CadencePin {+ /** True when the pin is saved or the next save writes it. */+ pinned: boolean+ /** True when the next save changes the pin. */+ pending: boolean+ /** Downstream views that refresh more often than the pinned cadence. */+ delayed: NodePinDelayApi[]+ /** Per cadence, the downstream view a pin that slow would leave unable to meet its own cadence. */+ blockedBy: Partial<Record<string, NodePinBlockerApi>>+ /** Per cadence, the downstream views that would read older data if this view were pinned to it. */+ delaysBy: Partial<Record<string, NodePinDelayApi[]>>+ /** Per cadence, the cadence this view runs at once unpinned, which only the server can work out because other pins stop demand. */+ unpinnedBy: Partial<Record<string, DataModelingSyncInterval>>+}+export interface SyncFrequencySelectProps {/** Bounds from the saved query. Undefined while loading, or on surfaces that don't fetch them. */bounds?: SyncFrequencyBoundsApi | null@@ -73,6 +89,9 @@ export interface SyncFrequencySelectProps {/** Takes precedence over the mode's own reason, so access checks win over scheduling ones. */disabledReason?: string'data-attr'?: string+ /** Set when the view has a DAG node, so a cadence slower than a downstream view can pin instead of being refused. */+ pin?: CadencePin | null+ onUnpin?: () => void}/**@@ -88,16 +107,27 @@ export function SyncFrequencySelect({loading,disabledReason,'data-attr': dataAttr,+ pin,+ onUnpin,}: SyncFrequencySelectProps): JSX.Element {- const inherited = inheritedCadence(bounds, value)- const selectedSpeedUp = value && value !== 'never' ? speedUpWarning(bounds, value) : null- const options = buildOptions(bounds)- const explanation = buildExplanation(bounds, { omitCeiling: !!inherited })+ const unpinning = pin && !pin.pinned && pin.pending ? unpinnedCadence(pin.unpinnedBy, pin.delaysBy, value) : null+ const inherited = pin?.pinned || unpinning ? null : inheritedCadence(bounds, value)+ const selected = unpinning?.cadence ?? inherited?.cadence ?? (value && value !== 'never' ? value : undefined)+ const options = buildOptions(bounds, {+ canPin: !!pin,+ pinned: !!pin?.pinned,+ pinBlockedBy: pin?.blockedBy,+ pinDelays: pin?.delaysBy,+ unpinnedBy: pin?.unpinnedBy,+ })+ const canUnpin = !!pin?.pinned && !pin.pending && !!onUnpinconst modeReason = modeDisabledReason(bounds)+ // A pin can make a cadence selectable that the bounds alone refuse, so the bar is only dead when its own options are.+ const nothingSelectable = options.every((option) => option.disabledReason)const reason =disabledReason ??(loading ? 'Saving the new cadence.' : undefined) ??- unsatisfiableReason(bounds) ??+ (nothingSelectable ? unsatisfiableReason(bounds) : null) ??modeReason ??undefined@@ -123,29 +153,85 @@ export function SyncFrequencySelect({<IconRabbit className={PACE_ICON_CLASS} aria-hidden /><LemonSegmentedButton<DataModelingSyncInterval>size="small"- value={inherited?.cadence ?? (value && value !== 'never' ? value : undefined)}+ value={selected}onChange={(next) => {- if (next !== value) {+ // Compare against what the bar shows: a saved cadence the scheduler overrides shows+ // the inherited one, and clicking the saved one again is how the view gets pinned to it.+ if (next !== selected) {onChange(next)+ } else if (canUnpin) {+ onUnpin?.()}}}- options={options}+ options={pin?.pinned ? withPinnedSegment(options, selected, { canUnpin }) : options}disabledReason={reason}/><IconTortoise className={PACE_ICON_CLASS} aria-hidden /></div>- {inherited && (- <span className="text-xs text-secondary max-w-prose">- <WithUpstreamRule text={inherited.note} />- </span>- )}- {selectedSpeedUp && (- <span className="text-xs text-secondary max-w-prose">- <WithUpstreamRule text={selectedSpeedUp} />+ <CadenceNotes+ notes={+ loading+ ? []+ : cadenceNotes({+ pin,+ selected,+ unpinning,+ inherited,+ speedUp: value && value !== 'never' ? speedUpWarning(bounds, value) : null,+ })+ }+ explanation={+ // The inherited note already names the downstream view, so the ceiling sentence would repeat it.+ // With a pin on offer, a downstream view no longer caps how slow this view can go.+ buildExplanation(bounds, { omitCeiling: !!inherited || !!pin })+ }+ />+ </div>+ )+}++interface CadenceNote {+ key: string+ text: string+ /** True when the note leans on the upstream rule, so it links to the explanation. */+ upstreamRule: boolean+}++/** The notes under the bar, in the order they apply: what the pin does, what an unpin does, what the scheduler overrides, what a pick speeds up. */+function cadenceNotes({+ pin,+ selected,+ unpinning,+ inherited,+ speedUp,+}: {+ pin?: CadencePin | null+ selected?: DataModelingSyncInterval+ unpinning: { note: string } | null+ inherited: { note: string } | null+ speedUp: string | null+}): CadenceNote[] {+ const notes: (CadenceNote | null)[] = [+ pin?.pinned && selected+ ? { key: 'pin', text: pinNote(selected, pin.delayed, { pending: pin.pending }), upstreamRule: false }+ : null,+ unpinning ? { key: 'unpinning', text: unpinning.note, upstreamRule: true } : null,+ inherited ? { key: 'inherited', text: inherited.note, upstreamRule: true } : null,+ speedUp ? { key: 'speed-up', text: speedUp, upstreamRule: true } : null,+ ]+ return notes.filter((note): note is CadenceNote => !!note)+}++function CadenceNotes({ notes, explanation }: { notes: CadenceNote[]; explanation: string | null }): JSX.Element {+ return (+ <>+ {notes.map((note) => (+ <span key={note.key} className="text-xs text-secondary max-w-prose">+ {note.upstreamRule ? <WithUpstreamRule text={note.text} /> : note.text}</span>- )}+ ))}{explanation && <span className="text-xs text-secondary max-w-prose">{explanation}</span>}- </div>+ </>)}@@ -172,6 +258,62 @@ export function speedUpWarning(return `${subject} also ${plural ? 'refresh' : 'refreshes'} every ${CADENCE_LABELS[cadence]} to keep up with this view.`}+export function unpinnedCadence(+ unpinnedBy: Partial<Record<string, DataModelingSyncInterval>>,+ delaysBy: Partial<Record<string, { name: string }[]>>,+ value: SyncFrequencyValue | null+): { cadence: DataModelingSyncInterval; note: string } | null {+ if (!value || value === 'never') {+ return null+ }+ const cadence = unpinnedBy[value]+ if (!cadence || cadence === value) {+ return null+ }+ // The names only describe who reads older data under the pin. The cadence itself comes from the server.+ const delayed = (delaysBy[value] ?? []).map((view) => view.name)+ const who = !delayed.length+ ? 'downstream views'+ : delayed.length > 1+ ? `downstream views ${delayed.join(', ')}`+ : `downstream view ${delayed[0]}`+ return {+ cadence,+ note: `Once you save, this view refreshes every ${CADENCE_LABELS[cadence]} to keep up with ${who}.`,+ }+}++/**+ * True when the view, once unpinned, would run faster than this cadence because a downstream view+ * asks for it, which is when holding the cadence takes a pin. Read from the node bounds, which are+ * calculated without this view's own pin, so a pinned view still tells.+ */+export function needsPin(unpinnedBy: CadencePin['unpinnedBy'] | undefined, cadence: SyncFrequencyValue): boolean {+ if (cadence === 'never') {+ return false+ }+ const unpinned = unpinnedBy?.[cadence]+ return !!unpinned && ORDERED_CADENCES.indexOf(unpinned) < ORDERED_CADENCES.indexOf(cadence)+}++/** What a pin means for the downstream views, so nobody pins without seeing who reads older data. */+export function pinNote(+ cadence: DataModelingSyncInterval,+ delayed: NodePinDelayApi[],+ { pending }: { pending: boolean }+): string {+ const every = CADENCE_LABELS[cadence]+ const lead = pending ? `Saving pins this view to refresh every ${every}` : `Pinned to refresh every ${every}`+ if (!delayed.length) {+ return `${lead}, even when downstream views need fresher data.`+ }+ const plural = delayed.length > 1+ return (+ `${lead}. ${plural ? 'Downstream views' : 'Downstream view'} ${delayed.map((view) => view.name).join(', ')} ` ++ `${plural ? 'refresh' : 'refreshes'} more often, so ${plural ? 'they read' : 'it reads'} data up to ${every} old.`+ )+}+const UPSTREAM_RULE ="A view can't be fresher than the data it reads, so upstream views refresh at least as often as the downstream views built on them."@@ -212,12 +354,25 @@ export function inheritedCadence(cadence,note:`Refreshes every ${CADENCE_LABELS[cadence]} to keep up with ${consumer ? `downstream view ${consumer}` : 'a downstream view'}. ` +- `Your ${CADENCE_LABELS[value]} setting applies again if ${consumer ?? 'that view'} refreshes less often.`,+ `Your ${CADENCE_LABELS[value]} setting applies again if downstream views refresh less often.`,}}export function buildOptions(- bounds?: SyncFrequencyBoundsApi | null+ bounds?: SyncFrequencyBoundsApi | null,+ {+ canPin = false,+ pinned = false,+ pinBlockedBy = {},+ pinDelays = {},+ unpinnedBy,+ }: {+ canPin?: boolean+ pinned?: boolean+ pinBlockedBy?: Partial<Record<string, NodePinBlockerApi>>+ pinDelays?: Partial<Record<string, { name: string }[]>>+ unpinnedBy?: CadencePin['unpinnedBy']+ } = {}): LemonSegmentedButtonOption<DataModelingSyncInterval>[] {if (!bounds?.options.length) {return ORDERED_CADENCES.map((cadence) => segment(cadence))@@ -225,6 +380,28 @@ export function buildOptions(return bounds.options.map((option) => {const cadence = option.cadence as DataModelingSyncInterval+ const stranded = pinBlockedBy[cadence]+ // A pinned view has no consumer ceiling, so the bounds alone never mark a slower cadence as needing a pin.+ if (canPin && stranded && (pinned || needsPin(unpinnedBy, cadence))) {+ return {+ ...segment(cadence),+ disabledReason: `Can't pin: ${stranded.name ? `downstream view ${stranded.name}` : 'a downstream view'} reads fresh data only through this view`,+ }+ }+ if (canPin && needsPin(unpinnedBy, cadence)) {+ const delayed = pinDelays[cadence]?.map((view) => view.name) ?? []+ const names = delayed.length ? delayed : option.blocker ? [option.blocker.name] : []+ const who =+ names.length > 1+ ? `Downstream views ${names.join(', ')} refresh`+ : names.length === 1+ ? `Downstream view ${names[0]} refreshes`+ : 'A downstream view refreshes'+ return {+ ...segment(cadence),+ tooltip: `Pin this view to every ${CADENCE_LABELS[cadence]}. ${who} more often and will read older data from it.`,+ }+ }return {...segment(cadence),disabledReason: option.allowed ? undefined : blockedReason(option, bounds),@@ -232,6 +409,39 @@ export function buildOptions(})}+/**+ * The pinned cadence carries a pin, which turns into an unpin cross while the segment is hovered.+ * Clicking the selected segment unpins, so there is no button nested inside the segment's button.+ */+function pinnedSegment(+ option: LemonSegmentedButtonOption<DataModelingSyncInterval>,+ { canUnpin }: { canUnpin: boolean }+): LemonSegmentedButtonOption<DataModelingSyncInterval> {+ const cadence = option.value+ return {+ ...option,+ label: (+ <span className="inline-flex items-center gap-0.5">+ {CADENCE_SEGMENTS[cadence]}+ <IconPinFilled className={canUnpin ? '[li:hover_&]:hidden' : undefined} aria-hidden />+ {canUnpin && <IconX className="hidden [li:hover_&]:inline" aria-hidden />}+ </span>+ ),+ tooltip: canUnpin+ ? `Pinned to every ${CADENCE_LABELS[cadence]}. Click to unpin.`+ : `Pinned to every ${CADENCE_LABELS[cadence]} once you save.`,+ 'data-attr': canUnpin ? 'sync-frequency-unpin' : option['data-attr'],+ }+}++function withPinnedSegment(+ options: LemonSegmentedButtonOption<DataModelingSyncInterval>[],+ selected: DataModelingSyncInterval | undefined,+ { canUnpin }: { canUnpin: boolean }+): LemonSegmentedButtonOption<DataModelingSyncInterval>[] {+ return options.map((option) => (option.value === selected ? pinnedSegment(option, { canUnpin }) : option))+}+/** Abbreviated on the bar, spelled out on hover, so eight segments stay readable at one glance. */function segment(cadence: DataModelingSyncInterval): LemonSegmentedButtonOption<DataModelingSyncInterval> {return { value: cadence, label: CADENCE_SEGMENTS[cadence], tooltip: `Every ${CADENCE_LABELS[cadence]}` }diff --git a/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.test.ts b/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.test.tsindex 9853ad79f25d..580d344cd2af 100644--- a/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.test.ts+++ b/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.test.ts@@ -4,7 +4,7 @@ import { expectLogic } from 'kea-test-utils'import posthog from 'posthog-js'import { createElement } from 'react'-import api from 'lib/api'+import api, { ApiConfig } from 'lib/api'import { FEATURE_FLAGS } from 'lib/constants'import { featureFlagLogic } from 'lib/logic/featureFlagLogic'@@ -18,6 +18,7 @@ import { MaterializationRunActions } from 'products/data_warehouse/frontend/sharimport { dataWarehouseViewsLogic } from './dataWarehouseViewsLogic'import { materializationJobsLogic } from './materializationJobsLogic'import { MaterializationStatusPanel } from './MaterializationStatusPanel'+import { SyncFrequencyValue } from './SyncFrequencySelect'const ELIGIBLE_CHECK = {eligible: true,@@ -75,6 +76,7 @@ describe('materializationJobsLogic', () => {},],'/api/projects/:team_id/data_modeling_jobs/': { results: [], count: 0 },+ '/api/projects/:team_id/data_modeling_nodes/:id/': [404, { detail: 'Not found' }],},post: {'/api/environments/:team_id/warehouse_saved_queries/check_incremental/': () => {@@ -718,6 +720,184 @@ describe('materializationJobsLogic', () => {expect(logic.values.savingMaterialization).toBe(false)expect(logic.values.hasMaterializationChanges).toBe(status !== 200)})++ describe('cadence pin', () => {+ const orgUniverse = { id: 'up-node', name: 'org_universe' }+ const dailyRevenue = { id: 'down-node', name: 'daily_revenue' }+ const TIERED_BOUNDS = {+ frequency_mode: 'tiered',+ options: [+ { cadence: '6hour', allowed: true, blocked_by: null, blocker: null, speeds_up: [orgUniverse] },+ { cadence: '24hour', allowed: false, blocked_by: 'consumer', blocker: dailyRevenue, speeds_up: [] },+ ],+ floor: null,+ ceiling: { label: '6 hours', blocker: dailyRevenue },+ best_effort_sources: [],+ best_effort_sources_withheld: false,+ }+ let nodePatches: { id: string; body: Record<string, unknown> }[]+ let savedQueryPatches: number++ function pinMocks({+ pinned,+ patchStatus = 200,+ nodeOptions = [{ cadence: '24hour', delays_if_pinned: [dailyRevenue], unpinned_cadence: '6hour' }],+ }: {+ pinned: boolean+ patchStatus?: number+ nodeOptions?: Record<string, unknown>[]+ }): Parameters<typeof useMocks>[0] {+ const mocks = apiMocks({ isMaterialized: true, savedQueryExtras: { sync_frequency_bounds: TIERED_BOUNDS } })+ mocks.get!['/api/projects/:team_id/data_modeling_nodes/:id/'] = (req) =>+ new URL(req.request.url).searchParams.get('lookup') === 'saved_query'+ ? [200, { id: 'node-1' }]+ : [404, { detail: 'Not found' }]+ mocks.get!['/api/projects/:team_id/data_modeling_nodes/:id/frequency_bounds/'] = [+ 200,+ { pinned, options: nodeOptions },+ ]+ mocks.patch = {+ '/api/projects/:team_id/data_modeling_nodes/:id/': async ({ request, params }) => {+ nodePatches.push({ id: String(params.id), body: (await request.json()) as Record<string, unknown> })+ return patchStatus === 200+ ? [200, { id: params.id }]+ : [patchStatus, { detail: 'Pick how often this view refreshes before pinning it.' }]+ },+ '/api/environments/:team_id/warehouse_saved_queries/:id/': () => {+ savedQueryPatches += 1+ return [200, { id: 'view-1' }]+ },+ }+ return mocks+ }++ beforeEach(() => {+ nodePatches = []+ savedQueryPatches = 0+ savedSyncFrequency = '6hour'+ })++ it.each([+ [+ 'pins a cadence slower than a downstream view needs',+ false,+ '24hour',+ null,+ { sync_frequency: '24hour', pinned: true },+ ],+ ['unpins and keeps the saved cadence', true, null, false, { pinned: false }],+ ])('%s in one node write', async (_case, pinned, cadence, pinDraft, expectedBody) => {+ useMocks(pinMocks({ pinned }))+ logic = materializationJobsLogic({ viewId: 'view-1' })+ logic.mount()+ await expectLogic(logic).toDispatchActions(['loadNodeFrequencySuccess'])+ expect(logic.values.nodeFrequency?.bounds.pinned).toBe(pinned)++ if (cadence) {+ logic.actions.setSyncFrequencyDraft(cadence as SyncFrequencyValue)+ }+ if (pinDraft !== null) {+ logic.actions.setPinDraft(pinDraft)+ }+ expect(logic.values.hasMaterializationChanges).toBe(true)++ await expectLogic(logic, () => logic.actions.saveMaterializationChanges()).toDispatchActions([+ 'finishSavingMaterialization',+ ])+ expect(nodePatches).toEqual([{ id: 'node-1', body: expectedBody }])+ expect(savedQueryPatches).toBe(0)+ expect(logic.values.hasMaterializationChanges).toBe(false)+ })++ it('drops a pending pin when the draft moves back to a cadence downstream views allow', async () => {+ useMocks(pinMocks({ pinned: false }))+ logic = materializationJobsLogic({ viewId: 'view-1' })+ logic.mount()+ await expectLogic(logic).toDispatchActions(['loadNodeFrequencySuccess'])++ logic.actions.setSyncFrequencyDraft('24hour')+ await expectLogic(logic).toFinishAllListeners()+ expect(logic.values.pinDraft).toBe(true)+ logic.actions.setSyncFrequencyDraft('6hour')+ await expectLogic(logic).toFinishAllListeners()+ expect(logic.values.pinDraft).toBeNull()+ expect(logic.values.hasMaterializationChanges).toBe(false)+ })++ it('pins again when a slower cadence is picked after unpinning', async () => {+ savedSyncFrequency = '24hour'+ useMocks(+ pinMocks({+ pinned: true,+ nodeOptions: [+ { cadence: '12hour', delays_if_pinned: [dailyRevenue], unpinned_cadence: '6hour' },+ { cadence: '24hour', delays_if_pinned: [dailyRevenue], unpinned_cadence: '6hour' },+ ],+ })+ )+ logic = materializationJobsLogic({ viewId: 'view-1' })+ logic.mount()+ await expectLogic(logic).toDispatchActions(['loadSavedQuerySuccess', 'loadNodeFrequencySuccess'])++ logic.actions.setPinDraft(false)+ expect(logic.values.cadencePin?.pinned).toBe(false)+ logic.actions.setSyncFrequencyDraft('12hour')+ await expectLogic(logic).toFinishAllListeners()++ expect(logic.values.pinDraft).toBe(true)+ expect(logic.values.cadencePin?.pinned).toBe(true)+ })++ it('addresses the node API by the active environment, not the project', async () => {+ ApiConfig.setCurrentProjectId(111)+ ApiConfig.setCurrentTeamId(222)+ const mocks = pinMocks({ pinned: false })+ const requested: string[] = []+ for (const path of [+ '/api/projects/:team_id/data_modeling_nodes/:id/',+ '/api/projects/:team_id/data_modeling_nodes/:id/frequency_bounds/',+ ]) {+ const handler = mocks.get![path]+ mocks.get![path] = (req: any) => {+ requested.push(String(req.params.team_id))+ return typeof handler === 'function' ? handler(req) : handler+ }+ }+ useMocks(mocks)+ logic = materializationJobsLogic({ viewId: 'view-1' })+ logic.mount()+ await expectLogic(logic).toDispatchActions(['loadNodeFrequencySuccess'])+ expect(requested).toEqual(['222', '222'])+ })++ it('keeps a poll that started before the save from overwriting the saved cadence', async () => {+ useMocks(pinMocks({ pinned: false }))+ logic = materializationJobsLogic({ viewId: 'view-1' })+ logic.mount()+ await expectLogic(logic).toDispatchActions(['loadSavedQuerySuccess', 'loadNodeFrequencySuccess'])+ const previous = logic.values.savedQuery!+ let resolveStale!: (value: typeof previous) => void+ const stale = new Promise<typeof previous>((resolve) => {+ resolveStale = resolve+ })+ const fetch = jest+ .spyOn(api.dataWarehouseSavedQueries, 'get')+ .mockImplementationOnce(() => stale)+ .mockResolvedValue({ ...previous, sync_frequency: '24hour' })+ try {+ logic.actions.loadSavedQuery()+ logic.actions.setSyncFrequencyDraft('24hour')+ await expectLogic(logic, () => logic.actions.saveMaterializationChanges()).toDispatchActions([+ 'finishSavingMaterialization',+ ])+ resolveStale(previous)+ await expectLogic(logic).toFinishAllListeners()+ expect(logic.values.savedQuery?.sync_frequency).toBe('24hour')+ } finally {+ fetch.mockRestore()+ }+ })+ })it.each(['Running', 'Completed'])('polls a %s job at the appropriate interval and recovers from failure',async (status) => {diff --git a/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.ts b/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.tsindex f8c6c2b16010..16d08e96a46a 100644--- a/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.ts+++ b/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.ts@@ -23,6 +23,15 @@ import type { DataWarehouseSavedQuerySummary } from 'scenes/data-warehouse/savedimport type { DataWarehouseSavedQuery } from '~/types'import { DataModelingSyncInterval, DataWarehouseSavedQueryIncrementalCheck } from '~/types'+import {+ dataModelingNodesFrequencyBoundsRetrieve,+ dataModelingNodesPartialUpdate,+ dataModelingNodesRetrieve,+} from 'products/data_modeling/frontend/generated/api'+import {+ DataModelingNodesRetrieveLookup,+ type NodeFrequencyBoundsApi,+} from 'products/data_modeling/frontend/generated/api.schemas'import { dataModelingJobsList, warehouseSavedQueriesResumeCreate } from 'products/data_warehouse/frontend/generated/api'import { DataModelingJobsListEngine } from 'products/data_warehouse/frontend/generated/api.schemas'import type {@@ -37,7 +46,7 @@ import {} from '../editor/IncrementalConfigFields'import { DEFAULT_MATERIALIZE_SYNC_FREQUENCY, dataWarehouseViewsLogic } from './dataWarehouseViewsLogic'import { latestSuccessfulSyncAt } from './materializationJobUtils'-import { SyncFrequencyValue } from './SyncFrequencySelect'+import { CadencePin, needsPin, SyncFrequencyValue } from './SyncFrequencySelect'const ACTIVE_REFRESH_INTERVAL_MS = 5000const IDLE_REFRESH_INTERVAL_MS = 60000@@ -52,6 +61,15 @@ interface OlderJobsPageResult {jobs: PaginatedDataModelingJobListApi}+export interface NodeFrequency {+ nodeId: string+ bounds: NodeFrequencyBoundsApi+}++function apiErrorDetail(error: unknown): string {+ return error instanceof ApiError && error.detail ? error.detail : 'Try again.'+}+function runHistoryEngineParams(engine: DataModelingJobsListEngine): {all_engines?: booleanengine?: DataModelingJobsListEngine@@ -71,6 +89,7 @@ export interface MaterializationJobsLogicProps {// Generated by kea-typegen. Update if you're an agent, ignore if you're human.export interface materializationJobsLogicValues {+ cadencePin: CadencePin | nulldataModelingJobs: PaginatedDataModelingJobListApi | nulldataModelingJobsError: booleandataModelingJobsLoading: boolean@@ -88,9 +107,13 @@ export interface materializationJobsLogicValues {latestCompletedJob: PaginatedDataModelingJobListApi | nulllatestCompletedJobLoading: booleanmaterializationRefreshPending: boolean+ nodeFrequency: NodeFrequency | null+ nodeFrequencyLoading: booleanolderJobsPage: OlderJobsPageResult | nullolderJobsPageError: booleanolderJobsPageLoading: boolean+ pinChanged: boolean+ pinDraft: boolean | nullrefreshModeChanged: booleanresumingMaterialization: booleanrunHistoryError: boolean@@ -210,6 +233,21 @@ export interface materializationJobsLogicActions {latestCompletedJob: PaginatedDataModelingJobListApipayload?: void}+ loadNodeFrequency: (_: void) => void+ loadNodeFrequencyFailure: (+ error: string,+ errorObject?: any+ ) => {+ error: string+ errorObject?: any+ }+ loadNodeFrequencySuccess: (+ nodeFrequency: NodeFrequency | null,+ payload?: void+ ) => {+ nodeFrequency: NodeFrequency | null+ payload?: void+ }loadOlderJobsPage: (_: void) => voidloadOlderJobsPageFailure: (error: string,@@ -287,6 +325,9 @@ export interface materializationJobsLogicActions {setJobsPage: (page: number) => {page: number}+ setPinDraft: (pinned: boolean | null) => {+ pinned: boolean | null+ }setResumingMaterialization: (resuming: boolean) => {resuming: boolean}@@ -311,11 +352,20 @@ export interface materializationJobsLogicMeta {incrementalDraft: IncrementalConfigDraft,incrementalDraftTouched: boolean) => boolean+ pinChanged: (pinDraft: boolean | null, nodeFrequency: NodeFrequency | null) => booleanhasMaterializationChanges: (savedQuery: DataWarehouseSavedQuery | null,syncFrequencyDraft: SyncFrequencyValue | null,- refreshModeChanged: boolean+ refreshModeChanged: boolean,+ pinChanged: boolean) => boolean+ cadencePin: (+ nodeFrequency: NodeFrequency | null,+ pinDraft: boolean | null,+ pinChanged: boolean,+ syncFrequencyDraft: SyncFrequencyValue | null,+ savedQuery: DataWarehouseSavedQuery | null+ ) => CadencePin | nulljobsPageResults: (jobsPage: number,selectedRunHistoryEngine: DataModelingJobsListEngine,@@ -383,6 +433,7 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([resumeMaterialization: true,setSyncFrequencyDraft: (frequency: SyncFrequencyValue) => ({ frequency }),clearSyncFrequencyDraft: true,+ setPinDraft: (pinned: boolean | null) => ({ pinned }),discardMaterializationChanges: true,saveMaterializationChanges: true,finishSavingMaterialization: true,@@ -407,6 +458,34 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([},},],+ // The cadence pin lives on the view's DAG node, which the saved query payload does not carry.+ nodeFrequency: [+ null as NodeFrequency | null,+ {+ loadNodeFrequency: async (_: void, breakpoint) => {+ const teamId = String(ApiConfig.getCurrentTeamId())+ let frequency: NodeFrequency+ try {+ const node = await dataModelingNodesRetrieve(teamId, props.viewId, {+ lookup: DataModelingNodesRetrieveLookup.SavedQuery,+ })+ const bounds = await dataModelingNodesFrequencyBoundsRetrieve(teamId, node.id)+ frequency = { nodeId: node.id, bounds }+ } catch (e) {+ // A view with no node has nothing to pin, and the picker then refuses slow cadences as before.+ if (!(e instanceof ApiError && e.status === 404)) {+ posthog.captureException(e)+ // Not an answer about the view, so the next saved query reload gets to try again.+ cache.nodeFrequencyRequested = false+ }+ return null+ }+ // Outside the try: a breakpoint throws to cancel a superseded load, and the catch would turn that into "no node".+ breakpoint()+ return frequency+ },+ },+ ],incrementalCheck: [INCREMENTAL_CHECK_LOADING as DataWarehouseSavedQueryIncrementalCheck | null,{@@ -539,6 +618,13 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([discardMaterializationChanges: () => null,},],+ pinDraft: [+ null as boolean | null,+ {+ setPinDraft: (_, { pinned }) => pinned,+ discardMaterializationChanges: () => null,+ },+ ],savingMaterialization: [false,{@@ -624,15 +710,56 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([)},],+ pinChanged: [+ (s) => [s.pinDraft, s.nodeFrequency],+ (pinDraft: boolean | null, nodeFrequency: NodeFrequency | null): boolean =>+ pinDraft !== null && !!nodeFrequency && pinDraft !== nodeFrequency.bounds.pinned,+ ],hasMaterializationChanges: [- (s) => [s.savedQuery, s.syncFrequencyDraft, s.refreshModeChanged],+ (s) => [s.savedQuery, s.syncFrequencyDraft, s.refreshModeChanged, s.pinChanged],(savedQuery: DataWarehouseSavedQuery | null,frequency: SyncFrequencyValue | null,- modeChanged: boolean+ modeChanged: boolean,+ pinChanged: boolean): boolean =>!!savedQuery?.is_materialized &&- (modeChanged || (frequency !== null && frequency !== (savedQuery.sync_frequency || 'never'))),+ (modeChanged ||+ pinChanged ||+ (frequency !== null && frequency !== (savedQuery.sync_frequency || 'never'))),+ ],+ cadencePin: [+ (s) => [s.nodeFrequency, s.pinDraft, s.pinChanged, s.syncFrequencyDraft, s.savedQuery],+ (+ nodeFrequency: NodeFrequency | null,+ pinDraft: boolean | null,+ pinChanged: boolean,+ frequency: SyncFrequencyValue | null,+ savedQuery: DataWarehouseSavedQuery | null+ ): CadencePin | null => {+ if (!nodeFrequency) {+ return null+ }+ const cadence = frequency ?? savedQuery?.sync_frequency+ return {+ pinned: pinDraft ?? nodeFrequency.bounds.pinned,+ pending: pinChanged,+ delayed:+ nodeFrequency.bounds.options.find((option) => option.cadence === cadence)?.delays_if_pinned ??+ [],+ blockedBy: Object.fromEntries(+ nodeFrequency.bounds.options.flatMap((option) =>+ option.pin_blocked_by ? [[option.cadence, option.pin_blocked_by]] : []+ )+ ),+ delaysBy: Object.fromEntries(+ nodeFrequency.bounds.options.map((option) => [option.cadence, option.delays_if_pinned])+ ),+ unpinnedBy: Object.fromEntries(+ nodeFrequency.bounds.options.map((option) => [option.cadence, option.unpinned_cadence])+ ),+ }+ },],jobsPageResults: [(s) => [s.jobsPage, s.selectedRunHistoryEngine, s.dataModelingJobs, s.runHistoryJobs, s.olderJobsPage],@@ -817,6 +944,12 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([actions.scheduleJobsRefresh(IDLE_REFRESH_INTERVAL_MS)},setSyncFrequencyDraft: ({ frequency }) => {+ // A cadence slower than a downstream view needs is only valid pinned, so the pick carries the pin.+ if (values.cadencePin && needsPin(values.cadencePin.unpinnedBy, frequency)) {+ actions.setPinDraft(true)+ } else if (values.pinDraft) {+ actions.setPinDraft(null)+ }// The cadence bar reports a change on every click, including the segment that is already// selected, so a draft can hold the cadence that is already saved. Such a draft is invisible// because it is not a change, until something else moves the saved cadence. Pausing sets the@@ -839,15 +972,59 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([: { ...EMPTY_INCREMENTAL_DRAFT, incrementalKey: values.incrementalCheck?.key_candidates[0] ?? null })},- saveMaterializationChanges: () => {+ saveMaterializationChanges: async () => {const draft = values.incrementalDraftif (!values.savedQuery || !values.hasMaterializationChanges) {actions.finishSavingMaterialization()return}+ let cadenceWritten = false+ if (values.pinChanged && values.nodeFrequency) {+ // The cadence and the pin go in one node write: a cadence slower than a downstream view+ // needs is refused unless it is pinned in the same request.+ try {+ await dataModelingNodesPartialUpdate(+ String(ApiConfig.getCurrentTeamId()),+ values.nodeFrequency.nodeId,+ {+ ...(values.syncFrequencyDraft !== null+ ? { sync_frequency: values.syncFrequencyDraft }+ : {}),+ pinned: !!values.pinDraft,+ }+ )+ } catch (e) {+ lemonToast.error(`Couldn't save the cadence. ${apiErrorDetail(e)}`)+ actions.finishSavingMaterialization()+ return+ }+ cadenceWritten = true+ if (!values.refreshModeChanged) {+ // Keep the draft on screen until both reloads land, so the picker never shows the old+ // pin state between the write and the fresh bounds.+ // Through the loader, so its breakpoint drops any poll that started before the write.+ const savedQueryReload = actions.loadSavedQuery()+ try {+ const bounds = await dataModelingNodesFrequencyBoundsRetrieve(+ String(ApiConfig.getCurrentTeamId()),+ values.nodeFrequency.nodeId+ )+ actions.loadNodeFrequencySuccess({ nodeId: values.nodeFrequency.nodeId, bounds })+ } catch {+ actions.loadNodeFrequency()+ }+ await savedQueryReload+ actions.discardMaterializationChanges()+ actions.finishSavingMaterialization()+ return+ }+ actions.loadNodeFrequency()+ }actions.updateDataWarehouseSavedQuery({id: props.viewId,- ...(values.syncFrequencyDraft !== null ? { sync_frequency: values.syncFrequencyDraft } : {}),+ ...(values.syncFrequencyDraft !== null && !cadenceWritten+ ? { sync_frequency: values.syncFrequencyDraft }+ : {}),...(values.refreshModeChanged? {incremental:@@ -890,6 +1067,10 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([if (!savedQuery || props.kind === 'endpoint') {return}+ if (savedQuery.is_materialized && !cache.nodeFrequencyRequested) {+ cache.nodeFrequencyRequested = true+ actions.loadNodeFrequency()+ }// Reflect the saved config so a materialized view's editor starts from what is// actually configured, not from the empty draft.if (!values.incrementalDraftTouched && savedQuery.incremental?.enabled) {@@ -913,6 +1094,10 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([// doesn't own our `savedQuery`. Reload it so the displayed frequency reflects the new value// immediately instead of reverting until the next poll.if (payload?.id === props.viewId) {+ // Pausing a view clears its pin on the server, so the pin held here may be out of date.+ if (values.nodeFrequency) {+ actions.loadNodeFrequency()+ }if (values.savingMaterialization && values.savedQuery) {actions.loadSavedQuerySuccess({ ...values.savedQuery, ...payload } as DataWarehouseSavedQuery)actions.discardMaterializationChanges()diff --git a/frontend/src/types.ts b/frontend/src/types.tsindex 2d2ec9af1f61..3d2cc6b81824 100644--- a/frontend/src/types.ts+++ b/frontend/src/types.ts@@ -6266,6 +6266,7 @@ export interface DataModelingNode {last_run_status?: DataModelingJobStatuslast_run_error?: string | nullsync_interval?: DataModelingSyncInterval+ pinned?: booleansuspended?: NodeApiSuspended/** Set on endpoint nodes stamped at materialization enable; older nodes carry only the name */endpoint?: NodeEndpointApi | nulldiff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 6eccf7aabe7e..3b95c2a2a39f 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -104,7 +104,6 @@1 products/dashboards/backend/widgets/list_widget.py1 products/data_modeling/backend/logic/cohort_scheduling.py1 products/data_modeling/backend/logic/gitsync/model_parser.py-1 products/data_modeling/backend/logic/node_frequency.py1 products/data_modeling/backend/logic/schedule_reconcile.py1 products/data_modeling/backend/logic/tier_membership.py1 products/data_modeling/backend/models/datawarehouse_managed_viewset.pydiff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.pyindex 77a13b13e611..be64bab3d047 100644--- a/products/data_modeling/backend/facade/api.py+++ b/products/data_modeling/backend/facade/api.py@@ -11,6 +11,7 @@ _B = "products.data_modeling.backend."_LAZY = {"UnsatisfiableFrequencyError": "logic.freshness",+ "SCHEDULABLE_BUCKETS": "logic.freshness","UnsupportedFrequencyTargetError": "logic.freshness","humanize_cadence": "logic.freshness","get_model_usage": "logic.usage",@@ -92,8 +93,12 @@ _LAZY = {"node_states_by_saved_query": "logic.node_frequency","get_declared_target": "logic.node_frequency","set_declared_target": "logic.node_frequency",+ "is_declared_pin": "logic.node_frequency","saved_query_target_bounds": "logic.node_frequency","SavedQueryFrequencyBounds": "logic.node_frequency",+ "saved_query_pin_preview": "logic.node_frequency",+ "cadence_changes": "logic.cadence_changes",+ "clear_legacy_sync_interval": "logic.cadence_changes","clear_node_suspension": "logic.node_suspension","is_node_suspended": "logic.node_suspension","mark_node_suspended": "logic.node_suspension",diff --git a/products/data_modeling/backend/logic/cadence_changes.py b/products/data_modeling/backend/logic/cadence_changes.pynew file mode 100644index 000000000000..88bf8fb0ecd1--- /dev/null+++ b/products/data_modeling/backend/logic/cadence_changes.py@@ -0,0 +1,42 @@+"""Shared bookkeeping for writes to a saved query's cadence, used by every surface that makes one."""++from datetime import timedelta+from uuid import UUID++from posthog.models.activity_logging.activity_log import Change++from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery+++def clear_legacy_sync_interval(team_id: int, saved_query_id: UUID) -> None:+ """Drop the saved query's own interval, which the node target replaces on tiered teams.++ The node target is the only store of frequency intent. A stale interval left in the column would+ answer for a cadence the user just set or cleared, so every cadence write clears it. A caller+ holding the row must also clear its in-memory copy before serializing it.+ """+ DataWarehouseSavedQuery.objects.filter(team_id=team_id, pk=saved_query_id).update(sync_frequency_interval=None)+++def cadence_changes(+ targets: tuple[timedelta | None, timedelta | None], pins: tuple[bool, bool] | None = None+) -> list[Change]:+ """The activity log entries for a cadence write, one per field that changed.++ The cadence lives on the DAG node, so `changes_between()` on the saved query sees nothing. Every+ cadence writer builds its entries here so the fields read alike in the activity log.+ """++ def render(target: timedelta | None) -> str | None:+ return str(target) if target is not None else None++ fields: list[tuple[str, str | bool | None, str | bool | None]] = [+ ("sync_frequency_interval", render(targets[0]), render(targets[1]))+ ]+ if pins is not None:+ fields.append(("sync_frequency_pinned", pins[0], pins[1]))+ return [+ Change(type="DataWarehouseSavedQuery", action="changed", field=field, before=before, after=after)+ for field, before, after in fields+ if before != after+ ]diff --git a/products/data_modeling/backend/logic/freshness.py b/products/data_modeling/backend/logic/freshness.pyindex de0a1679e21c..e7e71ea52946 100644--- a/products/data_modeling/backend/logic/freshness.py+++ b/products/data_modeling/backend/logic/freshness.py@@ -22,6 +22,10 @@ Vocabulary, one term per concept:staler than a consumer requires).In interval-space a smaller timedelta means fresher/more frequent, so as plaintimedeltas: source_floor <= target <= consumer_ceiling.+- pinned: an opt-in on a node with a declared target. Its effective cadence is its own target,+ so consumers do not speed it up, and its ancestors see the pinned target instead of the demand+ below it. It has no consumer ceiling. Its children inherit the coarser of its source floor and+ its pinned target as their floor contribution, so a child with another, fresher parent stays fast.Worked example, a chain src -> view -> endpoint:src is an imported table syncing every 1hour (its source interval), endpoint@@ -37,7 +41,7 @@ Edges are (upstream_id, downstream_id): data flows upstream -> downstream, so aimport dataclassesfrom collections import defaultdict, deque-from collections.abc import Iterable, Mapping+from collections.abc import Collection, Iterable, Mappingfrom datetime import timedeltafrom typing import Literal@@ -128,18 +132,26 @@ def _adjacency(edges: list[tuple[str, str]]) -> Adjacency:return Adjacency(children=children, parents=parents)+def _pinned_targets(pinned: Collection[str], declared_targets: Mapping[str, timedelta]) -> dict[str, timedelta]:+ """The pinned nodes that have a target to hold. A pin without a target pins nothing."""+ return {node: declared_targets[node] for node in pinned if node in declared_targets}++def compute_effective_cadences(*,nodes: set[str],edges: list[tuple[str, str]],declared_targets: dict[str, timedelta],+ pinned: Collection[str] = frozenset(),) -> dict[str, timedelta | None]:"""Resolve each node's effective cadence.Returns a value for every id in `nodes`. `None` means "unscheduled" — a node withno declared target and no scheduled descendant demanding freshness (the- ride-downstream opt-out). Source nodes are not expected in `nodes`.+ ride-downstream opt-out). Source nodes are not expected in `nodes`. A pinned node+ resolves to its own declared target whatever its consumers demand."""+ pinned_targets = _pinned_targets(pinned, declared_targets)adj = _adjacency(edges)# reverse-topological pass, iterative because recursion overflows on deep chainsout_degree = {node: sum(1 for child in adj.children.get(node, []) if child in nodes) for node in nodes}@@ -148,11 +160,14 @@ def compute_effective_cadences(while queue:node = queue.popleft()candidates: list[timedelta] = []- if node in declared_targets:- candidates.append(declared_targets[node])- for child in adj.children.get(node, []):- if child in nodes and (child_effective := resolved[child]) is not None:- candidates.append(child_effective)+ if node in pinned_targets:+ candidates.append(pinned_targets[node])+ else:+ if node in declared_targets:+ candidates.append(declared_targets[node])+ for child in adj.children.get(node, []):+ if child in nodes and (child_effective := resolved[child]) is not None:+ candidates.append(child_effective)# min = the finest demand wins (smaller timedelta = fresher)resolved[node] = min(candidates) if candidates else Nonefor parent in adj.parents.get(node, []):@@ -228,9 +243,21 @@ def ancestors_of(node_id: str, edges: list[tuple[str, str]]) -> set[str]:return seen-def all_source_floor_bounds(edges: list[tuple[str, str]], source_intervals: dict[str, timedelta]) -> dict[str, Bound]:+def descendants_of(node_id: str, edges: list[tuple[str, str]]) -> set[str]:+ """Every node downstream of `node_id`, however far — the cone that reads what it delivers."""+ return ancestors_of(node_id, [(downstream, upstream) for upstream, downstream in edges])+++def all_source_floor_bounds(+ edges: list[tuple[str, str]],+ source_intervals: dict[str, timedelta],+ pinned_targets: Mapping[str, timedelta] | None = None,+) -> dict[str, Bound]:"""Every node's source floor, with the source that set it, in one forward pass.+ A pinned node (`pinned_targets` maps it to its pinned cadence) passes its children the coarser+ of its own floor and its pin, blamed on the pinned node when the pin is the coarser side.+A source node's floor is its own sync interval. A derived node's floor is the finest (min)among its parents' floors, because its output changes whenever any input changes: a viewjoining events with a weekly import produces new rows continuously, so the weekly side must@@ -244,14 +271,19 @@ def all_source_floor_bounds(edges: list[tuple[str, str]], source_intervals: dictall_ids = set(source_intervals) | {node for edge in edges for node in edge}in_degree = {node: len(adj.parents.get(node, [])) for node in all_ids}queue = deque(node for node in all_ids if in_degree[node] == 0)+ pinned_targets = pinned_targets or {}floor: dict[str, Bound] = {}+ passed_down: dict[str, Bound] = {}while queue:node = queue.popleft()if node in source_intervals:floor[node] = Bound(value=source_intervals[node], blocker=node)else:- inherited = [floor[parent] for parent in adj.parents.get(node, [])]+ inherited = [passed_down[parent] for parent in adj.parents.get(node, [])]floor[node] = _finest(inherited) if inherited else Bound(value=STREAMING, blocker=None)+ passed_down[node] = floor[node]+ if node in pinned_targets:+ passed_down[node] = max(floor[node], Bound(value=pinned_targets[node], blocker=node), key=_bound_sort_key)for child in adj.children.get(node, []):in_degree[child] -= 1if in_degree[child] == 0:@@ -260,13 +292,17 @@ def all_source_floor_bounds(edges: list[tuple[str, str]], source_intervals: dictdef all_consumer_ceiling_bounds(- edges: list[tuple[str, str]], declared_targets: dict[str, timedelta]+ edges: list[tuple[str, str]],+ declared_targets: dict[str, timedelta],+ pinned: Collection[str] = frozenset(),) -> dict[str, Bound | None]:"""Every node's consumer ceiling, with the descendant that declared it, in one reverse pass.The ceiling is the finest declared target among strict descendants; None when no descendant- declares one. Cyclic nodes are omitted.+ declares one. Cyclic nodes are omitted. A pinned node has no ceiling, so its ancestors see+ only its own target and none of the demand below it."""+ pinned_targets = _pinned_targets(pinned, declared_targets)adj = _adjacency(edges)all_ids = set(declared_targets) | {node for edge in edges for node in edge}out_degree = {node: len(adj.children.get(node, [])) for node in all_ids}@@ -280,7 +316,7 @@ def all_consumer_ceiling_bounds(if child in declared_targets]candidates += [bound for child in adj.children.get(node, []) if (bound := ceiling.get(child)) is not None]- ceiling[node] = _finest(candidates) if candidates else None+ ceiling[node] = _finest(candidates) if candidates and node not in pinned_targets else Nonefor parent in adj.parents.get(node, []):out_degree[parent] -= 1if out_degree[parent] == 0:@@ -288,18 +324,26 @@ def all_consumer_ceiling_bounds(return ceiling-def all_source_floors(edges: list[tuple[str, str]], source_intervals: dict[str, timedelta]) -> dict[str, timedelta]:+def all_source_floors(+ edges: list[tuple[str, str]],+ source_intervals: dict[str, timedelta],+ pinned_targets: Mapping[str, timedelta] | None = None,+) -> dict[str, timedelta]:"""Every node's source floor, for callers that need the value and not the blame."""- return {node: bound.value for node, bound in all_source_floor_bounds(edges, source_intervals).items()}+ return {+ node: bound.value for node, bound in all_source_floor_bounds(edges, source_intervals, pinned_targets).items()+ }def all_consumer_ceilings(- edges: list[tuple[str, str]], declared_targets: dict[str, timedelta]+ edges: list[tuple[str, str]],+ declared_targets: dict[str, timedelta],+ pinned: Collection[str] = frozenset(),) -> dict[str, timedelta | None]:"""Every node's consumer ceiling, for callers that need the value and not the blame."""return {node: (bound.value if bound is not None else None)- for node, bound in all_consumer_ceiling_bounds(edges, declared_targets).items()+ for node, bound in all_consumer_ceiling_bounds(edges, declared_targets, pinned).items()}@@ -348,6 +392,7 @@ def clamp_to_source_floor(*,edges: list[tuple[str, str]],source_intervals: dict[str, timedelta],+ pinned: Collection[str] = frozenset(),) -> tuple[dict[str, timedelta | None], list[ClampedCadence]]:"""Coarsen every node scheduled finer than its sources can deliver to the nearest bucket >= itssource floor, returning the adjusted cadences and the list of changes.@@ -356,8 +401,12 @@ def clamp_to_source_floor(consumer, which is intended: the consumer's fast inputs keep it changing at its fine cadence,while the columns derived from the slow lineage update as often as that lineage delivers.Streaming/best-effort sources have a zero floor and are never clamped.++ A pinned node's effective cadence is its pinned target, so its children's floors read it from+ `effective`."""- floors = all_source_floors(edges, source_intervals)+ pinned_targets = {node: cadence for node in pinned if (cadence := effective.get(node)) is not None}+ floors = all_source_floors(edges, source_intervals, pinned_targets)clamped: dict[str, timedelta | None] = {}changes: list[ClampedCadence] = []for node_id, cadence in effective.items():@@ -437,6 +486,7 @@ def compute_target_bounds(edges: list[tuple[str, str]],declared_targets: dict[str, timedelta],source_intervals: dict[str, timedelta],+ pinned: Collection[str] = frozenset(),) -> TargetBounds:"""Resolve one node's selectable cadences, marking each blocked option with what blocks it.@@ -444,8 +494,13 @@ def compute_target_bounds(API payload, and any UI that offers a choice all read this, so a refusal and a disabled optioncan never disagree about the same node."""- floor = all_source_floor_bounds(edges, source_intervals).get(node_id, Bound(value=STREAMING, blocker=None))- ceiling = all_consumer_ceiling_bounds(edges, declared_targets).get(node_id)+ pinned_targets = _pinned_targets(pinned, declared_targets)+ floor = all_source_floor_bounds(edges, source_intervals, pinned_targets).get(+ node_id, Bound(value=STREAMING, blocker=None)+ )+ ceiling = all_consumer_ceiling_bounds(edges, declared_targets, pinned).get(node_id)+ if node_id in pinned:+ ceiling = Nonereturn _bounds_from(floor=floor, ceiling=ceiling)@@ -491,20 +546,38 @@ def _describe(node_id: str | None, names: Mapping[str, str] | None, fallback: streturn name or fallback-def _nothing_satisfiable_message(bounds: TargetBounds, names: Mapping[str, str] | None) -> str:[email protected](frozen=True, kw_only=True, slots=True)+class FloorReason:+ node: str # the node that sets the floor+ why: str # why it holds the floor there+++def _floor_reason(floor: Bound, names: Mapping[str, str] | None, pinned: Collection[str]) -> FloorReason:+ """The node that sets a floor, and why it holds the floor there: a slow source, or a pinned view."""+ if floor.blocker is not None and floor.blocker in pinned:+ view = _describe(floor.blocker, names, "an upstream view")+ return FloorReason(node=view, why=f"{view} is pinned to refresh every {humanize_cadence(floor.value)}")+ source = _describe(floor.blocker, names, "an upstream source")+ return FloorReason(node=source, why=f"{source} only syncs every {humanize_cadence(floor.value)}")+++def _nothing_satisfiable_message(+ bounds: TargetBounds, names: Mapping[str, str] | None, pinned: Collection[str] = frozenset()+) -> str:"""Explain a node with no legal cadence at all: crossed bounds, or a source slower than 30 days."""floor, ceiling = bounds.floor, bounds.ceiling- source = _describe(floor.blocker, names, "an upstream source") if floor else ""+ floor_reason = _floor_reason(floor, names, pinned) if floor else FloorReason(node="", why="")+ source = floor_reason.nodeconsumer = _describe(ceiling.blocker, names, "a view or endpoint built on this one") if ceiling else ""if floor is not None and ceiling is not None:return (- f"No cadence works here: {source} only syncs every {humanize_cadence(floor.value)},"+ f"No cadence works here: {floor_reason.why},"f" but {consumer} refreshes every {humanize_cadence(ceiling.value)}."f" Slow down {consumer} or speed up {source}.")if floor is not None:return (- f"No cadence works here: {source} only syncs every {humanize_cadence(floor.value)},"+ f"No cadence works here: {floor_reason.why},"f" less often than anything this can refresh at. Speed up {source} first.")if ceiling is not None:@@ -523,11 +596,13 @@ def validate_declared_target(declared_targets: dict[str, timedelta],source_intervals: dict[str, timedelta],names: Mapping[str, str] | None = None,+ pinned: Collection[str] = frozenset(),) -> None:"""Raise if `target` is not selectable or falls outside the node's bounds.`names` maps node ids to display names; supply it to have refusals name the source or view- that blocks the target, which is the thing a person has to change to unblock it.+ that blocks the target, which is the thing a person has to change to unblock it. `pinned`+ holds the pin state the write would leave, including `node_id`'s own."""if target not in SCHEDULABLE_BUCKETS:supported = ", ".join(format_cadence(interval) for interval in sorted(SCHEDULABLE_BUCKETS))@@ -536,7 +611,11 @@ def validate_declared_target()bounds = compute_target_bounds(- node_id=node_id, edges=edges, declared_targets=declared_targets, source_intervals=source_intervals+ node_id=node_id,+ edges=edges,+ declared_targets=declared_targets,+ source_intervals=source_intervals,+ pinned=pinned,)option = bounds.option_for(target)if option is None or option.allowed:@@ -544,16 +623,14 @@ def validate_declared_target(# Nothing legal to fall back to, so no message here can end in "pick X instead"if not bounds.satisfiable:- raise UnsatisfiableFrequencyError(_nothing_satisfiable_message(bounds, names))+ raise UnsatisfiableFrequencyError(_nothing_satisfiable_message(bounds, names, pinned))# Suggest a cadence the same call would accept, and name the bound itself rather than a# direction. Bounds are inclusive, so the bound is always a legal answer, while "or slower" /# "or faster" promises room that may not exist: a 15min ceiling has nothing fresher to offer.if option.blocked_by == "source" and bounds.floor is not None:- source = _describe(option.blocker, names, "an upstream source")raise UnsatisfiableFrequencyError(- f"Can't refresh every {humanize_cadence(target)}: {source} only syncs every"- f" {humanize_cadence(bounds.floor.value)}."+ f"Can't refresh every {humanize_cadence(target)}: {_floor_reason(bounds.floor, names, pinned).why}."f" Pick {humanize_cadence(min(bounds.allowed))} instead.")if option.blocked_by == "consumer" and bounds.ceiling is not None:@@ -579,6 +656,7 @@ def find_invalid_targets(edges: list[tuple[str, str]],declared_targets: dict[str, timedelta],source_intervals: dict[str, timedelta],+ pinned: Collection[str] = frozenset(),) -> list[InvalidTarget]:"""Re-validate every declared target against its current bounds.@@ -586,8 +664,8 @@ def find_invalid_targets(graph edits move floors. Runtime freshness stays correct (finest demand wins) — whatbreaks is declared == effective, so run this on any graph mutation and surface the result."""- floors = all_source_floors(edges, source_intervals)- ceilings = all_consumer_ceilings(edges, declared_targets)+ floors = all_source_floors(edges, source_intervals, _pinned_targets(pinned, declared_targets))+ ceilings = all_consumer_ceilings(edges, declared_targets, pinned)invalid: list[InvalidTarget] = []for node_id, declared in declared_targets.items():source_floor = floors.get(node_id, STREAMING)@@ -601,3 +679,106 @@ def find_invalid_targets())return invalid+++def _below_source_floor(+ *,+ edges: list[tuple[str, str]],+ declared_targets: Mapping[str, timedelta],+ source_intervals: dict[str, timedelta],+ pinned: Collection[str],+) -> dict[str, InvalidTarget]:+ return {+ invalid.node_id: invalid+ for invalid in find_invalid_targets(+ edges=edges, declared_targets=dict(declared_targets), source_intervals=source_intervals, pinned=pinned+ )+ if is_finer_than(invalid.declared, invalid.source_floor)+ }+++def nodes_below_source_floor(+ *,+ edges: list[tuple[str, str]],+ declared_targets: Mapping[str, timedelta],+ source_intervals: dict[str, timedelta],+ pinned: Collection[str],+) -> set[str]:+ """The nodes whose declared target is finer than their sources can deliver."""+ return set(+ _below_source_floor(+ edges=edges, declared_targets=declared_targets, source_intervals=source_intervals, pinned=pinned+ )+ )+++def descendants_broken_by_pin(+ *,+ node_id: str,+ target: timedelta,+ edges: list[tuple[str, str]],+ declared_targets: Mapping[str, timedelta],+ source_intervals: dict[str, timedelta],+ pinned: Collection[str],+ already_below_floor: Collection[str] | None = None,+) -> list[InvalidTarget]:+ """The downstream targets that pinning `node_id` to `target` would put out of reach.++ A pin raises its children's source floor. A child with another, fresher parent keeps that+ parent's floor, but a child that reads fresh data only through this node can no longer meet+ its own target. Targets that were already out of reach of their source floor without the pin+ don't count against it. A target that only broke its consumer ceiling still counts, because+ the node can meet it at runtime and the pin's floor would take that away.++ `already_below_floor` is `nodes_below_source_floor` without this pin. It does not depend on+ `target`, so a caller checking many targets computes it once and passes it in.+ """+ targets = {**declared_targets, node_id: target}+ others = set(pinned) - {node_id}+ if already_below_floor is None:+ already_below_floor = nodes_below_source_floor(+ edges=edges, declared_targets=targets, source_intervals=source_intervals, pinned=others+ )+ descendants = descendants_of(node_id, edges)+ broken = _below_source_floor(+ edges=edges, declared_targets=targets, source_intervals=source_intervals, pinned=others | {node_id}+ )+ return sorted(+ (invalid for node, invalid in broken.items() if node in descendants and node not in already_below_floor),+ key=lambda invalid: invalid.node_id,+ )+++def validate_pin(+ *,+ node_id: str,+ target: timedelta,+ edges: list[tuple[str, str]],+ declared_targets: Mapping[str, timedelta],+ source_intervals: dict[str, timedelta],+ pinned: Collection[str],+ names: Mapping[str, str] | None = None,+) -> None:+ """Raise UnsatisfiableFrequencyError when pinning `node_id` to `target` strands a downstream view.++ The message names the first stranded view the caller may see, and generic prose otherwise.+ """+ broken = descendants_broken_by_pin(+ node_id=node_id,+ target=target,+ edges=edges,+ declared_targets=declared_targets,+ source_intervals=source_intervals,+ pinned=pinned,+ )+ if not broken:+ return+ named = [invalid for invalid in broken if (names or {}).get(invalid.node_id)]+ stranded = (named or broken)[0]+ view = _describe(stranded.node_id, names, "")+ subject, fix = (f"downstream view {view}", view) if view else ("a downstream view", "that view")+ raise UnsatisfiableFrequencyError(+ f"Can't pin this view to {humanize_cadence(target)}: {subject} refreshes every "+ f"{humanize_cadence(stranded.declared)} and has no fresher input than this view. "+ f"Slow down {fix} first, or pin a faster cadence."+ )diff --git a/products/data_modeling/backend/logic/node_frequency.py b/products/data_modeling/backend/logic/node_frequency.pyindex 4bcf787c46b3..d7d4e54bb274 100644--- a/products/data_modeling/backend/logic/node_frequency.py+++ b/products/data_modeling/backend/logic/node_frequency.py@@ -9,8 +9,9 @@ typed column.import uuidimport dataclasses-from collections.abc import Iterable+from collections.abc import Collection, Iterable, Mappingfrom datetime import timedelta+from typing import Anyfrom django.db.models import Q, QuerySet@@ -18,14 +19,19 @@ from products.data_modeling.backend.facade.contracts import SavedQueryNodeState,from products.data_modeling.backend.logic.cohort_scheduling import MINUTES_PER_WEEKfrom products.data_modeling.backend.logic.freshness import (STREAMING,+ ClampedCadence,TargetBounds,+ TargetOption,all_source_floors,ancestors_of,clamp_to_source_floor,compute_effective_cadences,compute_target_bounds,+ descendants_broken_by_pin,+ descendants_of,intersect_target_bounds,max_data_age,+ nodes_below_source_floor,normalize_seed_target,)from products.data_modeling.backend.logic.node_suspension import merged_suspension_state@@ -39,6 +45,7 @@ _SYSTEM_KEY = "system"_FREQUENCY_KEY = "frequency"_TARGET_SECONDS_KEY = "target_seconds"_ANCHOR_MINUTES_KEY = "anchor_minutes"+_PINNED_KEY = "pinned"# `resolve_dependency_to_node` stamps this on source (TABLE) nodes at creation._ORIGIN_KEY = "origin"@@ -79,6 +86,16 @@ def get_declared_target(node: Node) -> timedelta | None:return timedelta(seconds=seconds) if seconds is not None else None+def is_declared_pin(properties: Mapping[str, Any] | None) -> bool:+ """Return whether node properties say it holds its declared target instead of following its consumers."""+ return bool((properties or {}).get(_SYSTEM_KEY, {}).get(_FREQUENCY_KEY, {}).get(_PINNED_KEY, False))+++def get_declared_pin(node: Node) -> bool:+ """Return whether the node holds its declared target instead of following its consumers."""+ return is_declared_pin(node.properties)++def declared_targets_by_saved_query(team_id: int, saved_query_ids: Iterable[str | uuid.UUID]) -> dict[str, timedelta]:"""Declared freshness target per saved query id, for those whose node carries one.@@ -135,17 +152,35 @@ def node_states_by_saved_query(def set_declared_target(node: Node, target: timedelta | None) -> None:- """Set (or clear, with None) the node's declared target without touching sibling system state."""+ """Set (or clear, with None) the node's declared target without touching sibling system state.++ Clearing the target also clears the pin, because a pin holds the target and has nothing to hold without one.+ """properties = node.properties or {}frequency = properties.setdefault(_SYSTEM_KEY, {}).setdefault(_FREQUENCY_KEY, {})if target is None:frequency.pop(_TARGET_SECONDS_KEY, None)+ frequency.pop(_PINNED_KEY, None)else:frequency[_TARGET_SECONDS_KEY] = int(target.total_seconds())node.properties = propertiesnode.save(update_fields=["properties"])+def set_declared_pin(node: Node, pinned: bool) -> None:+ """Pin (or unpin) the node's declared target without touching sibling system state."""+ if pinned and get_declared_target(node) is None:+ raise ValueError("a node needs a declared target before it can be pinned")+ properties = node.properties or {}+ frequency = properties.setdefault(_SYSTEM_KEY, {}).setdefault(_FREQUENCY_KEY, {})+ if pinned:+ frequency[_PINNED_KEY] = True+ else:+ frequency.pop(_PINNED_KEY, None)+ node.properties = properties+ node.save(update_fields=["properties"])++def get_declared_anchor(node: Node) -> int | None:"""Return the node's declared schedule anchor (minutes past Monday 00:00 UTC), or None."""return (node.properties or {}).get(_SYSTEM_KEY, {}).get(_FREQUENCY_KEY, {}).get(_ANCHOR_MINUTES_KEY)@@ -243,7 +278,7 @@ def resolve_source_intervals(source_nodes: list[Node]) -> tuple[dict[str, timedereturn source_intervals, best_effort[email protected](frozen=True, kw_only=True, slots=True)class FrequencyGraph:"""The primitives `freshness.py` operates on, extracted from one DAG's Node/Edge rows."""@@ -251,6 +286,7 @@ class FrequencyGraph:edges: list[tuple[str, str]] # (upstream_id, downstream_id), includes source tablesdeclared_targets: dict[str, timedelta] # declared per-node targetsdeclared_anchors: dict[str, int] # declared per-node schedule anchors (minutes past Monday 00:00 UTC)+ pinned: set[str] # nodes that hold their declared target instead of following their consumerssource_intervals: dict[str, timedelta] # per source (TABLE) nodebest_effort_source_ids: set[str] # sources treated as STREAMING but not guaranteednames: dict[str, str] # node id -> display name, so a bound can name what set it@@ -270,10 +306,13 @@ def build_frequency_graph(dag: DAG) -> FrequencyGraph:schedulable = {str(node.id) for node in nodes if node.type in SAVED_QUERY_NODE_TYPES}declared_targets: dict[str, timedelta] = {}declared_anchors: dict[str, int] = {}+ pinned: set[str] = set()for node in nodes:target = get_declared_target(node)if target is not None:declared_targets[str(node.id)] = target+ if get_declared_pin(node):+ pinned.add(str(node.id))anchor = get_declared_anchor(node)if anchor is not None:declared_anchors[str(node.id)] = anchor@@ -286,6 +325,7 @@ def build_frequency_graph(dag: DAG) -> FrequencyGraph:edges=edges,declared_targets=declared_targets,declared_anchors=declared_anchors,+ pinned=pinned,source_intervals=source_intervals,best_effort_source_ids=best_effort,names={str(node.id): node.name for node in nodes},@@ -303,6 +343,7 @@ class SavedQueryFrequencyBounds:best_effort_source_ids: set[str] # upstream sources with no schedule, so the floor is a guessmax_data_age: timedeltaspeeds_up: dict[timedelta, set[str]] # cadence -> upstream nodes that would run more often if this were set to it+ pinned: booldef saved_query_target_bounds(team_id: int, saved_query_id: str | uuid.UUID) -> SavedQueryFrequencyBounds | None:@@ -337,6 +378,7 @@ def saved_query_target_bounds(team_id: int, saved_query_id: str | uuid.UUID) ->edges=graph.edges,declared_targets=graph.declared_targets,source_intervals=graph.source_intervals,+ pinned=graph.pinned,))names.update(graph.names)@@ -356,6 +398,69 @@ def saved_query_target_bounds(team_id: int, saved_query_id: str | uuid.UUID) ->best_effort_source_ids=best_effort,max_data_age=max(data_ages),speeds_up=speeds_up,+ pinned=any(get_declared_pin(node) for node in nodes),+ )++[email protected](frozen=True, kw_only=True, slots=True)+class SavedQueryPinPreview:+ """What pinning a saved query to each cadence would do, for the one endpoint that shows it."""++ # cadence -> downstream nodes that would read data older than their own cadence if this were pinned to it+ delays_if_pinned: dict[timedelta, set[str]]+ # cadence -> downstream nodes whose own target a pin at that cadence would put out of reach+ pin_blocked_by: dict[timedelta, set[str]]+ # cadence -> the cadence this view would run at if it held that target without a pin+ unpinned_cadence: dict[timedelta, timedelta]+++def saved_query_pin_preview(team_id: int, saved_query_id: str | uuid.UUID) -> SavedQueryPinPreview:+ """The pin preview for every cadence the saved query's nodes offer, folded across their DAGs.++ Kept apart from `saved_query_target_bounds` because it costs a full-graph pass per cadence and+ only the node frequency endpoint reads it.+ """+ delays_if_pinned: dict[timedelta, set[str]] = {}+ pin_blocked_by: dict[timedelta, set[str]] = {}+ unpinned_cadence: dict[timedelta, timedelta] = {}+ graphs: dict[str, FrequencyGraph] = {}+ for node in Node.objects.filter(team_id=team_id, saved_query_id=saved_query_id).select_related("dag"):+ dag_key = str(node.dag_id)+ if dag_key not in graphs:+ graphs[dag_key] = build_frequency_graph(node.dag)+ graph = graphs[dag_key]+ node_id = str(node.id)+ options = compute_target_bounds(+ node_id=node_id,+ edges=graph.edges,+ declared_targets=graph.declared_targets,+ source_intervals=graph.source_intervals,+ pinned=graph.pinned,+ ).options+ for cadence, delayed in _delays_if_pinned_by_cadence(node_id, graph, options).items():+ delays_if_pinned.setdefault(cadence, set()).update(delayed)+ below_floor = nodes_below_source_floor(+ edges=graph.edges,+ declared_targets=graph.declared_targets,+ source_intervals=graph.source_intervals,+ pinned=graph.pinned - {node_id},+ )+ for option in options:+ stranded = descendants_broken_by_pin(+ node_id=node_id,+ target=option.value,+ edges=graph.edges,+ declared_targets=graph.declared_targets,+ source_intervals=graph.source_intervals,+ pinned=graph.pinned,+ already_below_floor=below_floor,+ )+ pin_blocked_by.setdefault(option.value, set()).update(invalid.node_id for invalid in stranded)+ for cadence, runs_at in _unpinned_cadence_by_cadence(node_id, graph, options).items():+ # A saved query with nodes in several DAGs runs at the finest cadence any of them asks for+ unpinned_cadence[cadence] = min(runs_at, unpinned_cadence.get(cadence, runs_at))+ return SavedQueryPinPreview(+ delays_if_pinned=delays_if_pinned, pin_blocked_by=pin_blocked_by, unpinned_cadence=unpinned_cadence)@@ -370,8 +475,7 @@ def _speeds_up_by_cadence("""def run(declared_targets: dict[str, timedelta]) -> dict[str, timedelta | None]:- effective = compute_effective_cadences(nodes=graph.nodes, edges=graph.edges, declared_targets=declared_targets)- return clamp_to_source_floor(effective, edges=graph.edges, source_intervals=graph.source_intervals)[0]+ return scheduled_cadences(graph, declared_targets=declared_targets)[0]upstream = ancestors & graph.nodesephemeral = {@@ -397,6 +501,63 @@ def _speeds_up_by_cadence(return result+def _delays_if_pinned_by_cadence(+ node_id: str, graph: FrequencyGraph, options: Iterable[TargetOption]+) -> dict[timedelta, set[str]]:+ """The downstream views that would read data older than their own cadence if `node_id` were pinned to each option.++ A descendant's effective cadence comes from its own target and its own consumers, never from+ `node_id`, so one pass over the current graph answers every option.+ """+ effective = compute_effective_cadences(+ nodes=graph.nodes, edges=graph.edges, declared_targets=graph.declared_targets, pinned=graph.pinned+ )+ descendants = descendants_of(node_id, graph.edges) & graph.nodes+ return {+ option.value: {+ descendant+ for descendant in descendants+ if (cadence := effective.get(descendant)) is not None and cadence < option.value+ }+ for option in options+ }+++def _unpinned_cadence_by_cadence(+ node_id: str, graph: FrequencyGraph, options: Iterable[TargetOption]+) -> dict[timedelta, timedelta]:+ """The cadence `node_id` would run at for each option if it held that target without a pin.++ Computed with only this node's pin removed, so a pin further downstream still stops the demand+ of the views below it, and then clamped to the source floor like the scheduler does.+ """+ others = graph.pinned - {node_id}+ result: dict[timedelta, timedelta] = {}+ for option in options:+ runs_at = scheduled_cadences(+ graph, declared_targets={**graph.declared_targets, node_id: option.value}, pinned=others+ )[0].get(node_id)+ result[option.value] = runs_at or option.value+ return result+++def scheduled_cadences(+ graph: FrequencyGraph,+ *,+ declared_targets: Mapping[str, timedelta] | None = None,+ pinned: Collection[str] | None = None,+) -> tuple[dict[str, timedelta | None], list[ClampedCadence]]:+ """The cadence the scheduler runs each node at: effective cadences, then the source-floor clamp.++ The one place this pair is spelled out, so the scheduler, the tier reports and the pickers can't+ disagree about pins. `declared_targets` and `pinned` override the graph's own for what-if reads.+ """+ declared = dict(graph.declared_targets if declared_targets is None else declared_targets)+ pins = graph.pinned if pinned is None else pinned+ effective = compute_effective_cadences(nodes=graph.nodes, edges=graph.edges, declared_targets=declared, pinned=pins)+ return clamp_to_source_floor(effective, edges=graph.edges, source_intervals=graph.source_intervals, pinned=pins)++def schedulable_nodes(dag: DAG) -> QuerySet[Node]:"""The DAG's schedulable nodes: everything that carries a live saved query (not a sourcetable, not a soft-deleted query). The one definition of "what gets a freshness target"."""diff --git a/products/data_modeling/backend/logic/schedule_reconcile.py b/products/data_modeling/backend/logic/schedule_reconcile.pyindex ce33baf2031f..f8f2bdafc7f2 100644--- a/products/data_modeling/backend/logic/schedule_reconcile.py+++ b/products/data_modeling/backend/logic/schedule_reconcile.py@@ -52,18 +52,21 @@ from products.data_modeling.backend.logic.freshness import (ClampedCadence,InvalidTarget,UnsupportedFrequencyTargetError,- clamp_to_source_floor,- compute_effective_cadences,find_invalid_targets,format_cadence,validate_declared_target,+ validate_pin,)from products.data_modeling.backend.logic.node_frequency import (FrequencyGraph,build_frequency_graph,+ get_declared_pin,+ get_declared_target,persist_seed_targets,+ scheduled_cadences,seed_targets,set_declared_anchor,+ set_declared_pin,set_declared_target,)from products.data_modeling.backend.models.dag import DAG@@ -133,7 +136,10 @@ def _warn_on_invalid_targets(dag: DAG, graph: FrequencyGraph | None = None) -> Nif graph is None:graph = build_frequency_graph(dag)for invalid in find_invalid_targets(- edges=graph.edges, declared_targets=graph.declared_targets, source_intervals=graph.source_intervals+ edges=graph.edges,+ declared_targets=graph.declared_targets,+ source_intervals=graph.source_intervals,+ pinned=graph.pinned,):logger.warning("Declared freshness target outside its legal range",@@ -151,6 +157,8 @@ def apply_saved_query_frequency_target(*,reconcile: bool = True,visible_names: Mapping[str, str] | None = None,+ pinned: bool | None = None,+ keep_target: bool = False,) -> int:"""Write a frequency target through to the DAG node(s) carrying this saved query.@@ -169,6 +177,13 @@ def apply_saved_query_frequency_target(of a node they may not read. Callers holding a per-user map (the saved-query surfaces) pass itand get the better message; everything left out falls back to generic prose.+ `pinned` sets or clears the pin alongside the target; None leaves the stored pin as it is. A pin+ lets the target sit slower than the node's consumers. `keep_target` ignores `target` and leaves+ each node's stored target as it is, read after the row lock so a pin written on its own never+ overwrites a cadence that committed in the meantime. Unpinning without changing the target is+ not validated, because the target was legal when it was pinned and the scheduler only uses it+ as a floor for consumer demand once unpinned.+Atomic because a saved query can still carry nodes in several DAGs while duplicates are beingconsolidated away: without it, a target rejected by the third node stays written on the firsttwo, and their reconciles are already queued. One node per saved query is the end state, which@@ -176,6 +191,12 @@ def apply_saved_query_frequency_target("""written = 0with transaction.atomic():+ # Validating a pin reads other nodes' targets, so two writes on different nodes of one DAG+ # must not validate against each other's stale state: each could pass and the pair strand a+ # consumer. Taking the DAG row first, in id order, makes them take turns.+ dag_ids = Node.objects.filter(team=saved_query.team, saved_query=saved_query).values_list("dag_id", flat=True)+ locked_dags = DAG.objects.select_for_update(no_key=True).filter(team_id=saved_query.team_id, pk__in=dag_ids)+ list(locked_dags.order_by("pk").values_list("pk", flat=True))# target and anchor share one JSON field and each setter rewrites the whole blob, so an# unlocked read-modify-write racing the anchor path would silently drop the other's keyfor node in (@@ -183,19 +204,38 @@ def apply_saved_query_frequency_target(.filter(team=saved_query.team, saved_query=saved_query).select_related("dag", "dag__team")):- if target is None:+ node_target = get_declared_target(node) if keep_target else target+ if node_target is None:+ if keep_target and pinned:+ raise ValueError("a node needs a declared target before it can be pinned")set_declared_target(node, None)else:- graph = build_frequency_graph(node.dag)- validate_declared_target(- node_id=str(node.id),- target=target,- edges=graph.edges,- declared_targets=graph.declared_targets,- source_intervals=graph.source_intervals,- names=visible_names or {},- )- set_declared_target(node, target)+ node_id = str(node.id)+ pin = get_declared_pin(node) if pinned is None else pinned+ if pinned is not False or node_target != get_declared_target(node):+ graph = build_frequency_graph(node.dag)+ validate_declared_target(+ node_id=node_id,+ target=node_target,+ edges=graph.edges,+ declared_targets=graph.declared_targets,+ source_intervals=graph.source_intervals,+ names=visible_names or {},+ pinned=(graph.pinned - {node_id}) | ({node_id} if pin else set()),+ )+ if pin:+ validate_pin(+ node_id=node_id,+ target=node_target,+ edges=graph.edges,+ declared_targets=graph.declared_targets,+ source_intervals=graph.source_intervals,+ pinned=graph.pinned,+ names=visible_names or {},+ )+ set_declared_target(node, node_target)+ if pinned is not None:+ set_declared_pin(node, pinned)written += 1if reconcile:maybe_reconcile_dag(node.dag)@@ -220,6 +260,7 @@ def check_saved_query_frequency_target(declared_targets=graph.declared_targets,source_intervals=graph.source_intervals,names=visible_names or {},+ pinned=graph.pinned,)@@ -259,10 +300,7 @@ def reconcile_dag_schedules(dag: DAG, *, graph: FrequencyGraph | None = None) ->team = dag.teamif graph is None:graph = build_frequency_graph(dag)- effective = compute_effective_cadences(- nodes=graph.nodes, edges=graph.edges, declared_targets=graph.declared_targets- )- effective, _clamped = clamp_to_source_floor(effective, edges=graph.edges, source_intervals=graph.source_intervals)+ effective, _clamped = scheduled_cadences(graph)desired_tiers = bucket_into_cadence_tiers(effective, graph.declared_anchors)_apply_reconciliation(dag_id=str(dag.id),@@ -314,8 +352,7 @@ def preview_dag_schedules(dag: DAG, *, seed: bool = False) -> DagSchedulePreview"""graph = build_frequency_graph(dag)declared = {**seed_targets(dag), **graph.declared_targets} if seed else graph.declared_targets- effective = compute_effective_cadences(nodes=graph.nodes, edges=graph.edges, declared_targets=declared)- effective, clamped = clamp_to_source_floor(effective, edges=graph.edges, source_intervals=graph.source_intervals)+ effective, clamped = scheduled_cadences(graph, declared_targets=declared)desired_tiers = bucket_into_cadence_tiers(effective, graph.declared_anchors)existing_ids = list_existing_schedule_ids(str(dag.id))plan = plan_schedule_reconciliation(str(dag.id), desired_tiers, existing_ids)@@ -326,7 +363,7 @@ def preview_dag_schedules(dag: DAG, *, seed: bool = False) -> DagSchedulePreviewbest_effort_source_ids=graph.best_effort_source_ids,clamped=clamped,invalid_targets=find_invalid_targets(- edges=graph.edges, declared_targets=declared, source_intervals=graph.source_intervals+ edges=graph.edges, declared_targets=declared, source_intervals=graph.source_intervals, pinned=graph.pinned),unsupported_tiers=sorted({tier.interval for tier in desired_tiers if tier.interval not in SCHEDULABLE_BUCKETS}),seeded=seed,diff --git a/products/data_modeling/backend/logic/tier_membership.py b/products/data_modeling/backend/logic/tier_membership.pyindex 1a16c924b21d..f34051f47b59 100644--- a/products/data_modeling/backend/logic/tier_membership.py+++ b/products/data_modeling/backend/logic/tier_membership.py@@ -24,12 +24,8 @@ from products.data_modeling.backend.logic.cohort_scheduling import (bucket_into_cadence_tiers,interval_seconds_from_schedule_id,)-from products.data_modeling.backend.logic.freshness import (- SCHEDULABLE_BUCKETS,- clamp_to_source_floor,- compute_effective_cadences,-)-from products.data_modeling.backend.logic.node_frequency import build_frequency_graph+from products.data_modeling.backend.logic.freshness import SCHEDULABLE_BUCKETS+from products.data_modeling.backend.logic.node_frequency import build_frequency_graph, scheduled_cadencesfrom products.data_modeling.backend.models.dag import DAGfrom products.data_modeling.backend.models.node import NodeTypefrom products.data_modeling.backend.schedule import DATA_MODELING_EXECUTE_DAG_WORKFLOW@@ -124,10 +120,7 @@ async def read_live_tiers(temporal: Client, dag_id: str) -> list[LiveTier]:def _desired_tiers(dag: DAG) -> dict[Tier, set[str]]:"""The tiers reconcile would want for this DAG: effective cadences → clamp to floor → buckets."""graph = build_frequency_graph(dag)- effective = compute_effective_cadences(- nodes=graph.nodes, edges=graph.edges, declared_targets=graph.declared_targets- )- effective, _clamped = clamp_to_source_floor(effective, edges=graph.edges, source_intervals=graph.source_intervals)+ effective, _clamped = scheduled_cadences(graph)return bucket_into_cadence_tiers(effective, graph.declared_anchors)diff --git a/products/data_modeling/backend/logic/tier_run_report.py b/products/data_modeling/backend/logic/tier_run_report.pyindex 47d4ba6e5b41..f40c04d467c5 100644--- a/products/data_modeling/backend/logic/tier_run_report.py+++ b/products/data_modeling/backend/logic/tier_run_report.py@@ -27,8 +27,7 @@ from products.data_modeling.backend.logic.cohort_scheduling import (tier_schedule_id,tier_sort_key,)-from products.data_modeling.backend.logic.freshness import clamp_to_source_floor, compute_effective_cadences-from products.data_modeling.backend.logic.node_frequency import build_frequency_graph+from products.data_modeling.backend.logic.node_frequency import build_frequency_graph, scheduled_cadencesfrom products.data_modeling.backend.models.dag import DAGfrom products.data_modeling.backend.models.data_modeling_job import (DataModelingJob,@@ -150,10 +149,7 @@ def untargeted_nodes(dag: DAG) -> list[Node]:def _desired_tiers(dag: DAG) -> dict[Tier, set[str]]:"""The cohorts the reconciler would build, mirroring its propagation + clamp + bucketing."""graph = build_frequency_graph(dag)- effective = compute_effective_cadences(- nodes=graph.nodes, edges=graph.edges, declared_targets=graph.declared_targets- )- effective, _clamped = clamp_to_source_floor(effective, edges=graph.edges, source_intervals=graph.source_intervals)+ effective, _clamped = scheduled_cadences(graph)return bucket_into_cadence_tiers(effective, graph.declared_anchors)diff --git a/products/data_modeling/backend/management/commands/set_schedule_anchor.py b/products/data_modeling/backend/management/commands/set_schedule_anchor.pyindex 5bc325180fe8..a300441ca303 100644--- a/products/data_modeling/backend/management/commands/set_schedule_anchor.py+++ b/products/data_modeling/backend/management/commands/set_schedule_anchor.py@@ -15,10 +15,10 @@ from products.data_modeling.backend.logic.cohort_scheduling import (format_tier,tier_sort_key,)-from products.data_modeling.backend.logic.freshness import clamp_to_source_floor, compute_effective_cadencesfrom products.data_modeling.backend.logic.node_frequency import (build_frequency_graph,schedulable_nodes,+ scheduled_cadences,set_declared_anchor,)from products.data_modeling.backend.logic.schedule_reconcile import reconcile_dag_schedules@@ -91,12 +91,7 @@ class Command(BaseCommand):plans: list[tuple[DAG, set[str], str]] = []for dag, node_ids in sorted(nodes_by_dag.items(), key=lambda kv: str(kv[0].id)):graph = build_frequency_graph(dag)- effective = compute_effective_cadences(- nodes=graph.nodes, edges=graph.edges, declared_targets=graph.declared_targets- )- effective, _clamped = clamp_to_source_floor(- effective, edges=graph.edges, source_intervals=graph.source_intervals- )+ effective, _clamped = scheduled_cadences(graph)if options["with_upstream"]:node_ids = self._expand_with_upstream(graph.edges, node_ids, graph.nodes)diff --git a/products/data_modeling/backend/presentation/views/metric_visibility.py b/products/data_modeling/backend/presentation/views/metric_visibility.pyindex 7cd00a8d8988..03fd18fb668a 100644--- a/products/data_modeling/backend/presentation/views/metric_visibility.py+++ b/products/data_modeling/backend/presentation/views/metric_visibility.py@@ -1,23 +1,32 @@from typing import TYPE_CHECKING+from uuid import UUIDfrom django.db.models import Q, QuerySetfrom rest_framework.exceptions import PermissionDeniedfrom products.data_modeling.backend.facade.flags import is_managed_viewset_enabled-from products.data_modeling.backend.facade.models import NodeType+from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery, NodeType+from products.warehouse_sources.backend.facade.api import allowed_table_idsif TYPE_CHECKING:from posthog.models.team import Teamfrom products.access_control.backend.facade.user_access_control import AccessControlLevel, UserAccessControl- from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery+ from products.data_modeling.backend.facade.api import SavedQueryFrequencyBoundsDELETE_DENIED = "Deleting data models requires data warehouse write access."EDIT_DENIED = "Editing data models requires data warehouse write access."VIEW_EDIT_DENIED = "You need edit access to this view."+def _as_uuid(value: str) -> UUID | None:+ try:+ return UUID(value)+ except ValueError:+ return None++class MetricNodeVisibilityMixin:user_access_control: "UserAccessControl"team: "Team"@@ -37,6 +46,46 @@ class MetricNodeVisibilityMixin:):raise PermissionDenied(VIEW_EDIT_DENIED)+ def _readable_node_names(self, resolved: "SavedQueryFrequencyBounds") -> dict[str, str]:+ """The names in a lineage this caller may read, keyed by node id.++ Fails closed and withholds the node id along with the name, as the saved-query surfaces do: a+ view under a `none` grant must not be named, or counted, by the cadence of a view beside it. A+ node that resolves to no resource (one that predates the origin stamp) is withheld too.+ """+ team_id = self.team.pk+ visible: dict[str, str] = {}+ node_ids_by_saved_query: dict[str, list[str]] = {}+ node_ids_by_table: dict[str, list[str]] = {}+ for node_id, identity in resolved.identities.items():+ if node_id not in resolved.names:+ continue+ if identity.is_posthog_table:+ visible[node_id] = resolved.names[node_id] # events, persons and friends: readable by anyone+ elif identity.saved_query_id is not None:+ node_ids_by_saved_query.setdefault(identity.saved_query_id, []).append(node_id)+ elif identity.warehouse_table_id is not None:+ node_ids_by_table.setdefault(identity.warehouse_table_id, []).append(node_id)++ def reveal(node_ids: list[str]) -> None:+ visible.update({node_id: resolved.names[node_id] for node_id in node_ids})++ if node_ids_by_saved_query:+ creators = DataWarehouseSavedQuery.objects.filter(+ id__in=node_ids_by_saved_query, team_id=team_id+ ).values_list("id", "created_by_id")+ levels = self.user_access_control.bulk_object_access_levels(+ "warehouse_view", [(str(pk), created_by) for pk, created_by in creators]+ )+ for saved_query_id, level in levels.items():+ if level is not None and level != "none":+ reveal(node_ids_by_saved_query[saved_query_id])++ table_ids = [table_id for raw in node_ids_by_table if (table_id := _as_uuid(raw)) is not None]+ for table_id in allowed_table_ids(team_id, self.user_access_control, ids=table_ids):+ reveal(node_ids_by_table[str(table_id)])+ return visible+def _metric_nodes_hidden(self) -> bool:return not self.user_access_control.check_access_level_for_resource("data_catalog", required_level="viewer")diff --git a/products/data_modeling/backend/presentation/views/node.py b/products/data_modeling/backend/presentation/views/node.pyindex 62b3e95f3cf7..6d9c1c350a62 100644--- a/products/data_modeling/backend/presentation/views/node.py+++ b/products/data_modeling/backend/presentation/views/node.py@@ -1,7 +1,8 @@+from datetime import timedeltafrom typing import Any, castfrom uuid import UUID-from django.db import models+from django.db import models, transactionfrom django.db.models import Exists, OuterRef, Subqueryfrom drf_spectacular.types import OpenApiTypes@@ -12,16 +13,28 @@ from rest_framework.exceptions import NotFound, PermissionDeniedfrom posthog.api.pagination import BoundedPageNumberPaginationfrom posthog.api.routing import TeamAndOrgViewSetMixin+from posthog.helpers.impersonation import is_impersonatedfrom posthog.models import User+from posthog.models.activity_logging.activity_log import Detail, log_activityfrom posthog.rbac.query_access import assert_user_can_read_queryfrom products.data_modeling.backend.facade.api import (+ SCHEDULABLE_BUCKETS,MaterializationForbiddenError,MaterializationRunDirection,+ SavedQueryFrequencyBounds,+ UnsatisfiableFrequencyError,+ UnsupportedFrequencyTargetError,+ apply_saved_query_frequency_target,+ cadence_changes,+ clear_legacy_sync_interval,delete_node,endpoint_link,get_declared_target,+ is_declared_pin,run_related_node_materializations,+ saved_query_pin_preview,+ saved_query_target_bounds,suspension_state,unsuspend_nodes,)@@ -29,6 +42,7 @@ from products.data_modeling.backend.facade.models import (SAVED_QUERY_NODE_TYPES,DataModelingJob,DataModelingJobEngine,+ DataWarehouseSavedQuery,Edge,LineageIssueKind,Node,@@ -40,7 +54,16 @@ from products.data_modeling.backend.presentation.views.metric_visibility importEDIT_DENIED,MetricNodeVisibilityMixin,)-from products.warehouse_sources.backend.facade.models import sync_frequency_interval_to_sync_frequency+from products.warehouse_sources.backend.facade.models import (+ sync_frequency_interval_to_sync_frequency,+ sync_frequency_to_sync_frequency_interval,+)++NEVER = "never"+CADENCE_CHOICES = [NEVER] + [+ sync_frequency_interval_to_sync_frequency(bucket) for bucket in sorted(SCHEDULABLE_BUCKETS)+]+PIN_NEEDS_A_CADENCE = "Pick how often this view refreshes before pinning it."class NodeSuspensionSerializer(serializers.Serializer):@@ -95,7 +118,72 @@ class LineageIssueSerializer(serializers.Serializer):at = serializers.DateTimeField(allow_null=True, help_text="When the issue was recorded.")+class DeclaredPinField(serializers.BooleanField):+ def get_attribute(self, instance: Node) -> bool:+ return is_declared_pin(instance.properties)+++class NodePinDelaySerializer(serializers.Serializer):+ id = serializers.CharField(help_text="Node id of the downstream view or endpoint.")+ name = serializers.CharField(help_text="Name of the downstream view or endpoint.")+++class NodePinBlockerSerializer(serializers.Serializer):+ id = serializers.CharField(+ allow_null=True,+ help_text="Node id of the downstream view that stops the pin, or null when the caller may not read it.",+ )+ name = serializers.CharField(+ allow_null=True,+ help_text="Name of the downstream view that stops the pin, or null when the caller may not read it.",+ )+++class NodeFrequencyOptionSerializer(serializers.Serializer):+ cadence = serializers.ChoiceField(choices=CADENCE_CHOICES[1:], help_text="A cadence the view could refresh at.")+ delays_if_pinned = NodePinDelaySerializer(+ many=True,+ help_text="Downstream views and endpoints that refresh more often than this cadence. If this view "+ "were pinned to it, they would read data older than their own cadence.",+ )+ pin_blocked_by = NodePinBlockerSerializer(+ allow_null=True,+ help_text="The downstream view that stops a pin at this cadence. It reads fresh data only through "+ "this view, so a pin this slow would leave it unable to refresh as often as its own cadence. Null "+ "when this cadence can be pinned.",+ )+ unpinned_cadence = serializers.ChoiceField(+ choices=CADENCE_CHOICES[1:],+ help_text="The cadence the view would run at if it held this cadence without a pin. Finer than this "+ "cadence when a downstream view needs fresher data than the pin allows.",+ )+++class NodeFrequencyBoundsSerializer(serializers.Serializer):+ pinned = serializers.BooleanField(+ help_text="True when the view refreshes at its own cadence instead of as often as its downstream "+ "views and endpoints need."+ )+ options = NodeFrequencyOptionSerializer(+ many=True, help_text="Every cadence a picker may show, finest first. Empty when the node has no view."+ )++class NodeSerializer(serializers.ModelSerializer):+ # nosemgrep: choices-need-a-class -- the values are SCHEDULABLE_BUCKETS, and the enum is already named by ENUM_NAME_OVERRIDES+ sync_frequency = serializers.ChoiceField(+ choices=CADENCE_CHOICES,+ write_only=True,+ required=False,+ help_text="How often this view refreshes. 'never' clears the cadence, and with it the pin. "+ "Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.",+ )+ pinned = DeclaredPinField(+ required=False,+ help_text="Pin this view to its own cadence. A pinned view does not refresh more often when downstream "+ "views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a "+ "cadence. Set false to refresh as often as downstream views need again.",+ )suspended = serializers.SerializerMethodField(read_only=True)endpoint = serializers.SerializerMethodField(read_only=True)upstream_count = serializers.SerializerMethodField(read_only=True)@@ -130,6 +218,8 @@ class NodeSerializer(serializers.ModelSerializer):"last_run_error","user_tag","sync_interval",+ "sync_frequency",+ "pinned","suspended","endpoint",]@@ -252,6 +342,8 @@ class NodeSerializer(serializers.ModelSerializer):raise serializers.ValidationError("Nodes of views PostHog manages cannot be modified.")if self.instance is not None and self.instance.type == NodeType.METRIC:raise serializers.ValidationError("Metric nodes are maintained by the data catalog.")+ if self.writes_cadence(attrs) and (self.instance is None or self.instance.saved_query_id is None):+ raise serializers.ValidationError("Only views and endpoints have a cadence to set.")node_type = attrs.get("type")if node_type is not None and self.instance is not None:# A full PUT round-trips the node's own type, so a type equal to the current one is@@ -260,6 +352,82 @@ class NodeSerializer(serializers.ModelSerializer):raise serializers.ValidationError("A node's type cannot be changed through the API.")return attrs+ @staticmethod+ def writes_cadence(attrs: dict[str, Any]) -> bool:+ return "sync_frequency" in attrs or "pinned" in attrs++ def update(self, instance: Node, validated_data: dict[str, Any]) -> Node:+ if not self.writes_cadence(validated_data):+ return super().update(instance, validated_data)+ visible_names: dict[str, str] = validated_data.pop("visible_names", {})+ frequency = validated_data.pop("sync_frequency", None)+ pinned: bool | None = validated_data.pop("pinned", None)+ with transaction.atomic():+ _write_cadence(self.context["request"], instance, frequency, pinned, visible_names)+ return super().update(instance, validated_data)+++def _write_cadence(+ request: request.Request, node: Node, frequency: str | None, pinned: bool | None, visible_names: dict[str, str]+) -> None:+ """Write the cadence and pin to every node of the node's view, and record it on the view's activity log."""+ saved_query = node.saved_query+ assert saved_query is not None+ previous_target, previous_pinned = get_declared_target(node), is_declared_pin(node.properties)+ keep_target = frequency is None+ if frequency is None:+ # A pin on its own keeps the stored cadence, which the write reads again under the row lock.+ target = previous_target+ else:+ target = None if frequency == NEVER else sync_frequency_to_sync_frequency_interval(frequency)+ if pinned and target is None:+ raise serializers.ValidationError(PIN_NEEDS_A_CADENCE)+ if target is not None:+ # Scheduling a view is the same grant as materializing it directly.+ assert_user_can_read_query(saved_query.query, saved_query.team_id, cast(User, request.user))+ try:+ apply_saved_query_frequency_target(+ saved_query, target, visible_names=visible_names, pinned=pinned, keep_target=keep_target+ )+ except (UnsatisfiableFrequencyError, UnsupportedFrequencyTargetError) as e:+ raise serializers.ValidationError(str(e))+ except ValueError:+ if not keep_target:+ raise+ # The cadence was cleared between the read above and the write, so there is nothing to pin.+ raise serializers.ValidationError(PIN_NEEDS_A_CADENCE)+ if frequency is not None:+ clear_legacy_sync_interval(saved_query.team_id, saved_query.pk)+ saved_query.sync_frequency_interval = None+ node.refresh_from_db(fields=["properties"])+ _log_cadence_change(+ request,+ saved_query,+ (previous_target, get_declared_target(node)),+ (previous_pinned, is_declared_pin(node.properties)),+ )+++def _log_cadence_change(+ request: request.Request,+ saved_query: DataWarehouseSavedQuery,+ targets: tuple[timedelta | None, timedelta | None],+ pins: tuple[bool, bool],+) -> None:+ changes = cadence_changes(targets, pins)+ if not changes:+ return+ log_activity(+ organization_id=saved_query.team.organization_id,+ team_id=saved_query.team_id,+ user=cast(User, request.user),+ was_impersonated=is_impersonated(request),+ item_id=saved_query.id,+ scope="DataWarehouseSavedQuery",+ activity="updated",+ detail=Detail(name=saved_query.name, changes=changes),+ )+class NodePagination(BoundedPageNumberPagination):page_size = 1000@@ -372,8 +540,63 @@ class NodeViewSet(def perform_update(self, serializer: serializers.BaseSerializer) -> None:self._require_warehouse_access(level="editor", message=EDIT_DENIED)- self._require_view_edit_access(cast(Node, serializer.instance).saved_query)- serializer.save()+ node = cast(Node, serializer.instance)+ self._require_view_edit_access(node.saved_query)+ if not NodeSerializer.writes_cadence(serializer.validated_data):+ serializer.save()+ return+ # A refusal names what blocks the cadence, so it names only nodes this caller can list.+ serializer.save(visible_names=self._visible_names_around(node))++ def _visible_names_around(self, node: Node, resolved: SavedQueryFrequencyBounds | None = None) -> dict[str, str]:+ """Names of the nodes around this view's cadence that the caller may list and read."""+ if resolved is None and node.saved_query_id:+ resolved = saved_query_target_bounds(self.team_id, node.saved_query_id)+ if resolved is None:+ return {}+ dag_ids = Node.objects.filter(team_id=self.team_id, saved_query_id=node.saved_query_id).values("dag_id")+ listed = self._exclude_hidden_nodes(Node.objects.filter(team_id=self.team_id, dag_id__in=dag_ids))+ listed_ids = {str(node_id) for node_id in listed.values_list("id", flat=True)}+ return {node_id: name for node_id, name in self._readable_node_names(resolved).items() if node_id in listed_ids}++ @extend_schema(responses=NodeFrequencyBoundsSerializer)+ @action(methods=["GET"], detail=True)+ def frequency_bounds(self, request: request.Request, *args: Any, **kwargs: Any) -> response.Response:+ """The cadences this view could refresh at, and what pinning it to each one would delay downstream."""+ self._require_warehouse_access(level="viewer", message=_READ_DENIED)+ node = self.get_object()+ resolved = saved_query_target_bounds(self.team_id, node.saved_query_id) if node.saved_query_id else None+ if resolved is None:+ return response.Response({"pinned": False, "options": []})+ names = self._visible_names_around(node, resolved)+ assert node.saved_query_id is not None+ preview = saved_query_pin_preview(self.team_id, node.saved_query_id)++ def pin_blocker(cadence: timedelta) -> dict[str, str | None] | None:+ stranded = sorted(preview.pin_blocked_by.get(cadence, ()))+ if not stranded:+ return None+ # Name a view the caller can see when there is one. Otherwise the pin is still refused, but+ # neither the id nor the name of a view the caller may not read goes out with the refusal.+ node_id = next((node_id for node_id in stranded if node_id in names), None)+ return {"id": node_id, "name": names[node_id]} if node_id else {"id": None, "name": None}++ options = [+ {+ "cadence": sync_frequency_interval_to_sync_frequency(option.value),+ "delays_if_pinned": [+ {"id": node_id, "name": names[node_id]}+ for node_id in sorted(preview.delays_if_pinned.get(option.value, ()))+ if node_id in names+ ],+ "pin_blocked_by": pin_blocker(option.value),+ "unpinned_cadence": sync_frequency_interval_to_sync_frequency(+ preview.unpinned_cadence.get(option.value, option.value)+ ),+ }+ for option in resolved.bounds.options+ ]+ return response.Response(NodeFrequencyBoundsSerializer({"pinned": resolved.pinned, "options": options}).data)def perform_destroy(self, instance: Node) -> None:self._require_warehouse_access(level="editor", message=DELETE_DENIED)diff --git a/products/data_modeling/backend/test/test_freshness.py b/products/data_modeling/backend/test/test_freshness.pyindex 621ace74663c..4cd83f049b3e 100644--- a/products/data_modeling/backend/test/test_freshness.py+++ b/products/data_modeling/backend/test/test_freshness.py@@ -15,11 +15,13 @@ from products.data_modeling.backend.logic.freshness import (clamp_to_source_floor,compute_effective_cadences,compute_target_bounds,+ descendants_broken_by_pin,find_invalid_targets,intersect_target_bounds,max_data_age,normalize_seed_target,validate_declared_target,+ validate_pin,)M5 = timedelta(minutes=5)@@ -71,10 +73,30 @@ class TestComputeEffectiveCadences(TestCase):{"a": H1},{"a": H1, "b": None},),+ # a pinned node keeps its own target, and the node above it sees that target, not c's 15min+ (+ "pinned_middle_holds_and_cuts_demand",+ {"a", "b", "c"},+ [("src", "a"), ("a", "b"), ("b", "c")],+ {"b": H1, "c": M15},+ {"a": H1, "b": H1, "c": M15},+ {"b"},+ ),+ # a 15min view over a fresh parent and a parent pinned hourly: only the pinned one stays hourly+ (+ "pinned_parent_of_a_fast_join",+ {"contacts", "signups"},+ [("s3", "signups"), ("crm", "contacts"), ("contacts", "signups")],+ {"contacts": H1, "signups": M15},+ {"contacts": H1, "signups": M15},+ {"contacts"},+ ),])- def test_effective_cadence(self, _name, nodes, edges, targets, expected):- self.assertEqual(compute_effective_cadences(nodes=nodes, edges=edges, declared_targets=targets), expected)+ def test_effective_cadence(self, _name, nodes, edges, targets, expected, pinned=frozenset()):+ self.assertEqual(+ compute_effective_cadences(nodes=nodes, edges=edges, declared_targets=targets, pinned=pinned), expected+ )def test_deep_chain_does_not_overflow(self):# prod DAGs can chain far past Python's recursion limit; propagation must stay iterative@@ -114,12 +136,43 @@ class TestValidateFrequencyTarget(TestCase):{"src": H6},False,),+ # a pinned node may run slower than its consumers+ ("pinned_above_descendant_ceiling_ok", "a", DAY, [("a", "ep")], {"ep": M15}, {}, True, {"a"}),+ # but never faster than its sources+ ("pinned_below_imported_floor", "a", M15, [("src", "a")], {}, {"src": H6}, False, {"a"}),+ # one fresh parent keeps a 15min join legal under a parent pinned hourly+ (+ "fresh_parent_beside_a_pinned_one_ok",+ "signups",+ M15,+ [("s3", "signups"), ("crm", "contacts"), ("contacts", "signups")],+ {"contacts": H1},+ {"s3": STREAMING, "crm": STREAMING},+ True,+ {"contacts"},+ ),+ # every parent pinned hourly: nothing feeds the join more often than that+ (+ "every_parent_pinned_slower",+ "signups",+ M15,+ [("crm", "contacts"), ("crm", "accounts"), ("contacts", "signups"), ("accounts", "signups")],+ {"contacts": H1, "accounts": H6},+ {"crm": STREAMING},+ False,+ {"contacts", "accounts"},+ ),])- def test_validate(self, _name, node_id, target, edges, targets, source_intervals, ok):+ def test_validate(self, _name, node_id, target, edges, targets, source_intervals, ok, pinned=frozenset()):if ok:validate_declared_target(- node_id=node_id, target=target, edges=edges, declared_targets=targets, source_intervals=source_intervals+ node_id=node_id,+ target=target,+ edges=edges,+ declared_targets=targets,+ source_intervals=source_intervals,+ pinned=pinned,)else:with self.assertRaises(UnsatisfiableFrequencyError):@@ -129,8 +182,25 @@ class TestValidateFrequencyTarget(TestCase):edges=edges,declared_targets=targets,source_intervals=source_intervals,+ pinned=pinned,)+ def test_refusal_under_pinned_parents_names_the_pinned_view(self):+ with self.assertRaises(UnsatisfiableFrequencyError) as raised:+ validate_declared_target(+ node_id="signups",+ target=M15,+ edges=[("crm", "contacts"), ("contacts", "signups")],+ declared_targets={"contacts": H1},+ source_intervals={"crm": STREAMING},+ names={"contacts": "org_universe"},+ pinned={"contacts"},+ )+ self.assertEqual(+ str(raised.exception),+ "Can't refresh every 15 minutes: org_universe is pinned to refresh every 1 hour. Pick 1 hour instead.",+ )+@parameterized.expand([# 45min would silently degrade to hourly in the spec builder@@ -314,6 +384,74 @@ class TestIntersectTargetBounds(TestCase):self.assertTrue(all(option.allowed for option in options.values()))+class TestPinBreaksDescendants(TestCase):+ @parameterized.expand(+ [+ ("chain_child_reads_only_the_pinned_view", [("src", "up"), ("up", "maps")], {"maps": H6}, ["maps"]),+ (+ "fan_in_child_keeps_a_fresh_input",+ [("src", "up"), ("up", "maps"), ("src", "maps")],+ {"maps": H6},+ [],+ ),+ # maps is already finer than the slow source it reads through up, so the pin is not to blame+ (+ "already_invalid_child_does_not_block",+ [("slow", "up"), ("up", "maps")],+ {"maps": M15},+ [],+ ),+ # middle already breaks its consumer ceiling (fast wants 15m) but still meets its own 1h+ # target at runtime; the pin's floor is what would take that away+ (+ "existing_ceiling_violation_does_not_hide_a_new_floor_violation",+ [("src", "up"), ("up", "middle"), ("middle", "fast"), ("src", "fast")],+ {"middle": H1, "fast": M15},+ ["middle"],+ ),+ ]+ )+ def test_descendants_broken_by_pin(self, _name, edges, child_targets, expected):+ broken = descendants_broken_by_pin(+ node_id="up",+ target=DAY,+ edges=edges,+ declared_targets={"up": H1, **child_targets},+ source_intervals={"src": STREAMING, "slow": H6},+ pinned=set(),+ )+ self.assertEqual([invalid.node_id for invalid in broken], expected)++ @parameterized.expand(+ [+ (+ "named",+ {"maps": "maps"},+ "Can't pin this view to 1 day: downstream view maps refreshes every 6 hours and has no "+ "fresher input than this view. Slow down maps first, or pin a faster cadence.",+ ),+ (+ "withheld",+ {},+ "Can't pin this view to 1 day: a downstream view refreshes every 6 hours and has no "+ "fresher input than this view. Slow down that view first, or pin a faster cadence.",+ ),+ ]+ )+ def test_refusal_names_the_downstream_view_it_would_strand(self, _name, names, message):+ with self.assertRaises(UnsatisfiableFrequencyError) as raised:+ validate_pin(+ node_id="up",+ target=DAY,+ edges=[("src", "up"), ("up", "maps")],+ declared_targets={"up": DAY, "maps": H6},+ source_intervals={"src": STREAMING},+ pinned=set(),+ names=names,+ )+ self.assertEqual(str(raised.exception), message)++class TestFindInvalidTargets(TestCase):@parameterized.expand([@@ -352,11 +490,32 @@ class TestFindInvalidTargets(TestCase):{"src": STREAMING},[],),+ # pinned slower than its consumer is deliberate, and its ancestor answers to the pin, not the consumer+ (+ "pinned_slower_than_consumer_is_valid",+ [("src", "up"), ("up", "a"), ("a", "ep"), ("src", "ep")],+ {"up": H1, "a": H1, "ep": M15},+ {"src": STREAMING},+ [],+ {"a"},+ ),+ # every parent pinned slower than the child: the child cannot be served at its target+ (+ "every_parent_pinned_slower_flags_the_child",+ [("src", "p1"), ("src", "p2"), ("p1", "c"), ("p2", "c")],+ {"p1": H1, "p2": H6, "c": M15},+ {"src": STREAMING},+ [InvalidTarget(node_id="c", declared=M15, source_floor=H1, consumer_ceiling=None)],+ {"p1", "p2"},+ ),])- def test_find_invalid_targets(self, _name, edges, targets, source_intervals, expected):+ def test_find_invalid_targets(self, _name, edges, targets, source_intervals, expected, pinned=frozenset()):self.assertEqual(- find_invalid_targets(edges=edges, declared_targets=targets, source_intervals=source_intervals), expected+ find_invalid_targets(+ edges=edges, declared_targets=targets, source_intervals=source_intervals, pinned=pinned+ ),+ expected,)@@ -392,6 +551,15 @@ class TestClampToSourceFloor(TestCase):self.assertEqual(changes[0].source_floor, source_intervals["src"])self.assertEqual(changes[0].clamped_to, expected_clamped_to)+ def test_a_child_of_only_pinned_parents_runs_no_faster_than_the_fastest_pin(self):+ clamped, _ = clamp_to_source_floor(+ {"p1": H1, "p2": H6, "c": M15},+ edges=[("src", "p1"), ("src", "p2"), ("p1", "c"), ("p2", "c")],+ source_intervals={"src": STREAMING},+ pinned={"p1", "p2"},+ )+ self.assertEqual(clamped, {"p1": H1, "p2": H6, "c": H1})+class TestNormalizeSeedTarget(TestCase):@parameterized.expand(diff --git a/products/data_modeling/backend/test/test_schedule_reconcile.py b/products/data_modeling/backend/test/test_schedule_reconcile.pyindex 6629873a95d3..394694e8c4f6 100644--- a/products/data_modeling/backend/test/test_schedule_reconcile.py+++ b/products/data_modeling/backend/test/test_schedule_reconcile.py@@ -6,6 +6,7 @@ from unittest import mockfrom django.test import SimpleTestCase+from parameterized import parameterizedfrom temporalio.client import ScheduleAlreadyRunningError, ScheduleListActionStartWorkflowfrom temporalio.service import RPCError, RPCStatusCode@@ -15,8 +16,10 @@ from products.data_modeling.backend.logic.cohort_scheduling import tier_schedulefrom products.data_modeling.backend.logic.freshness import UnsupportedFrequencyTargetErrorfrom products.data_modeling.backend.logic.node_frequency import (get_declared_anchor,+ get_declared_pin,get_declared_target,set_declared_anchor,+ set_declared_pin,set_declared_target,)from products.data_modeling.backend.logic.saved_query_dag_sync import promote_dag_view_nodes_to_matview@@ -24,6 +27,7 @@ from products.data_modeling.backend.logic.schedule_reconcile import (DagScheduleTeardown,TeamScheduleTeardownError,apply_saved_query_frequency_anchor,+ apply_saved_query_frequency_target,bootstrap_dag_to_tiers,convert_dag_to_tiers,delete_dag_schedules,@@ -469,15 +473,28 @@ class TestMaybeReconcileDag(BaseTest):update.assert_called_once()self.assertEqual(update.call_args.kwargs["id"], tier_id)- def test_upstream_slower_than_its_consumer_runs_at_the_consumer_cadence_and_keeps_its_setting(self):+ @parameterized.expand(+ [+ ("unpinned_runs_at_the_consumer_cadence", False, True, {H6: ["upstream", "consumer"]}),+ ("pinned_keeps_its_own_cadence", True, True, {DAY: ["upstream"], H6: ["consumer"]}),+ ("pinned_holds_back_a_consumer_with_no_fresher_parent", True, False, {DAY: ["upstream", "consumer"]}),+ ]+ )+ def test_upstream_slower_than_its_consumer_keeps_its_setting(+ self, _name, pinned, consumer_reads_source, expected_tiers+ ):dag = DAG.get_or_create_default(self.team)source = _warehouse_source_node(self.team, dag, sync_frequency_interval=H1)upstream = _saved_query_node(self.team, dag, "org_universe", NodeType.MAT_VIEW)consumer = _saved_query_node(self.team, dag, "maps", NodeType.MAT_VIEW)Edge.objects.create(team=self.team, dag=dag, source=source, target=upstream)Edge.objects.create(team=self.team, dag=dag, source=upstream, target=consumer)+ if consumer_reads_source:+ Edge.objects.create(team=self.team, dag=dag, source=source, target=consumer)set_declared_target(upstream, DAY)set_declared_target(consumer, H6)+ if pinned:+ set_declared_pin(upstream, True)with (mock.patch(f"{RECONCILE}.async_connect", new=mock.AsyncMock(return_value=self._temporal_listing([]))),@@ -486,13 +503,42 @@ class TestMaybeReconcileDag(BaseTest):with self.captureOnCommitCallbacks(execute=True):maybe_reconcile_dag(dag)- create.assert_called_once()- self.assertEqual(create.call_args.kwargs["id"], tier_schedule_id(str(dag.id), H6))- scheduled = create.call_args.kwargs["schedule"].action.args[0]["node_ids"]- self.assertEqual(sorted(scheduled), sorted([str(upstream.id), str(consumer.id)]))+ ids = {"upstream": str(upstream.id), "consumer": str(consumer.id)}+ created = {+ call.kwargs["id"]: sorted(call.kwargs["schedule"].action.args[0]["node_ids"])+ for call in create.call_args_list+ }+ self.assertEqual(+ created,+ {+ tier_schedule_id(str(dag.id), cadence): sorted(ids[name] for name in names)+ for cadence, names in expected_tiers.items()+ },+ )upstream.refresh_from_db()self.assertEqual(get_declared_target(upstream), DAY)+ def test_a_pin_on_its_own_keeps_the_stored_target(self):+ node = _saved_query_node(self.team, DAG.get_or_create_default(self.team), "mv", NodeType.MAT_VIEW)+ set_declared_target(node, H6)+ assert node.saved_query is not None++ with mock.patch(f"{RECONCILE}.maybe_reconcile_dag"):+ apply_saved_query_frequency_target(node.saved_query, None, pinned=True, keep_target=True)++ node.refresh_from_db()+ self.assertEqual((get_declared_target(node), get_declared_pin(node)), (H6, True))++ def test_a_pin_on_its_own_is_refused_when_the_node_has_no_target(self):+ node = _saved_query_node(self.team, DAG.get_or_create_default(self.team), "mv", NodeType.MAT_VIEW)+ assert node.saved_query is not None++ with mock.patch(f"{RECONCILE}.maybe_reconcile_dag"), self.assertRaises(ValueError):+ apply_saved_query_frequency_target(node.saved_query, None, pinned=True, keep_target=True)++ node.refresh_from_db()+ self.assertFalse(get_declared_pin(node))+def test_reconcile_failure_never_raises_past_commit(self):dag = self._dag_with_target()with (diff --git a/products/data_modeling/backend/tests/api/test_node_api.py b/products/data_modeling/backend/tests/api/test_node_api.pyindex 7251e6e67c14..0cf3a7d9d0ba 100644--- a/products/data_modeling/backend/tests/api/test_node_api.py+++ b/products/data_modeling/backend/tests/api/test_node_api.py@@ -14,8 +14,13 @@ from rest_framework import statusfrom temporalio.common import WorkflowIDConflictPolicy, WorkflowIDReusePolicyfrom posthog.models import Team+from posthog.models.activity_logging.activity_log import ActivityLog-from products.data_modeling.backend.logic.node_frequency import set_declared_target+from products.data_modeling.backend.logic.node_frequency import (+ get_declared_pin,+ get_declared_target,+ set_declared_target,+)from products.data_modeling.backend.logic.node_materialization import start_node_materializationfrom products.data_modeling.backend.logic.node_suspension import (mark_node_suspended,@@ -750,6 +755,135 @@ class TestNodeViewSet(APIBaseTest):self.assertEqual(len(lineage["edges"]), 2 if flag_enabled else 1)self.assertEqual(detail_response.status_code, status.HTTP_200_OK if flag_enabled else status.HTTP_404_NOT_FOUND)+ def _node_url(self, node: Node) -> str:+ return f"/api/environments/{self.team.id}/data_modeling_nodes/{node.id}/"++ def _give_the_view_a_consumer(self, consumer_target: timedelta, *, reads_the_table_too: bool = False) -> Node:+ self.view_node.type = NodeType.MAT_VIEW+ self.view_node.save(update_fields=["type"])+ consumer = Node.objects.create(+ team=self.team,+ dag=self.dag,+ saved_query=DataWarehouseSavedQuery.objects.create(+ name="consumer_view", team=self.team, query={"query": "SELECT 1", "kind": "HogQLQuery"}+ ),+ type=NodeType.MAT_VIEW,+ )+ Edge.objects.create(team=self.team, dag=self.dag, source=self.view_node, target=consumer)+ if reads_the_table_too:+ Edge.objects.create(team=self.team, dag=self.dag, source=self.table_node, target=consumer)+ set_declared_target(consumer, consumer_target)+ return consumer++ def _patch_cadence(self, node: Node, payload: dict):+ with patch("products.data_modeling.backend.logic.schedule_reconcile.maybe_reconcile_dag"):+ return self.client.patch(self._node_url(node), data=payload, format="json")++ def test_a_pinned_view_keeps_a_cadence_slower_than_its_consumer_until_unpinned(self):+ self._give_the_view_a_consumer(timedelta(hours=6), reads_the_table_too=True)++ bounds = self.client.get(f"{self._node_url(self.view_node)}frequency_bounds/").json()+ options = {option["cadence"]: option for option in bounds["options"]}+ self.assertFalse(bounds["pinned"])+ self.assertEqual([view["name"] for view in options["24hour"]["delays_if_pinned"]], ["consumer_view"])+ self.assertEqual(options["6hour"]["delays_if_pinned"], [])+ # Unpinned, the consumer's 6h demand wins, so the preview of unpinning a daily view is 6h+ self.assertEqual((options["24hour"]["unpinned_cadence"], options["6hour"]["unpinned_cadence"]), ("6hour",) * 2)++ # Without the pin the consumer's 6h demand refuses a daily cadence, so both have to land together.+ self.assertEqual(self._patch_cadence(self.view_node, {"sync_frequency": "24hour"}).status_code, 400)+ pinned = self._patch_cadence(self.view_node, {"sync_frequency": "24hour", "pinned": True})+ self.assertEqual(pinned.status_code, 200, pinned.json())+ self.assertEqual((pinned.json()["sync_interval"], pinned.json()["pinned"]), ("24hour", True))+ self.view_node.refresh_from_db()+ self.assertEqual(+ (get_declared_target(self.view_node), get_declared_pin(self.view_node)), (timedelta(days=1), True)+ )+ self.assertTrue(self.client.get(f"{self._node_url(self.view_node)}frequency_bounds/").json()["pinned"])+ log = ActivityLog.objects.get(scope="DataWarehouseSavedQuery", item_id=str(self.saved_query.id))+ assert log.detail is not None+ changes = {change["field"]: (change["before"], change["after"]) for change in log.detail["changes"]}+ self.assertEqual(+ changes, {"sync_frequency_interval": (None, "1 day, 0:00:00"), "sync_frequency_pinned": (False, True)}+ )++ unpinned = self._patch_cadence(self.view_node, {"pinned": False})+ self.assertEqual(unpinned.status_code, 200, unpinned.json())+ self.assertFalse(unpinned.json()["pinned"])+ self.view_node.refresh_from_db()+ self.assertEqual(+ (get_declared_target(self.view_node), get_declared_pin(self.view_node)), (timedelta(days=1), False)+ )++ def test_a_cadence_write_clears_the_legacy_saved_query_interval(self):+ DataWarehouseSavedQuery.objects.filter(pk=self.saved_query.pk).update(+ sync_frequency_interval=timedelta(hours=12)+ )+ set_declared_target(self.view_node, timedelta(hours=6))++ response = self._patch_cadence(self.view_node, {"sync_frequency": "never"})++ self.assertEqual(response.status_code, 200, response.json())+ self.assertIsNone(response.json()["sync_interval"])+ self.saved_query.refresh_from_db()+ self.assertIsNone(self.saved_query.sync_frequency_interval)++ @parameterized.expand(+ [+ ("consumer_reads_only_this_view", False, {"6hour": None, "24hour": "consumer_view"}),+ ("consumer_has_a_fresher_input", True, {"6hour": None, "24hour": None}),+ ]+ )+ def test_a_pin_that_strands_a_downstream_view_is_marked_and_refused(+ self, _name, reads_the_table_too, expected_blockers+ ):+ self._give_the_view_a_consumer(timedelta(hours=6), reads_the_table_too=reads_the_table_too)++ bounds = self.client.get(f"{self._node_url(self.view_node)}frequency_bounds/").json()+ blockers = {+ option["cadence"]: option["pin_blocked_by"] and option["pin_blocked_by"]["name"]+ for option in bounds["options"]+ if option["cadence"] in expected_blockers+ }+ self.assertEqual(blockers, expected_blockers)++ response = self._patch_cadence(self.view_node, {"sync_frequency": "24hour", "pinned": True})+ if expected_blockers["24hour"] is None:+ self.assertEqual(response.status_code, 200, response.json())+ return+ self.assertEqual(response.status_code, 400, response.json())+ self.assertEqual(+ response.json()["detail"],+ "Can't pin this view to 1 day: downstream view consumer_view refreshes every 6 hours and has no "+ "fresher input than this view. Slow down consumer_view first, or pin a faster cadence.",+ )+ self.view_node.refresh_from_db()+ self.assertEqual((get_declared_target(self.view_node), get_declared_pin(self.view_node)), (None, False))++ @parameterized.expand(+ [+ ("no_cadence_yet", {"pinned": True}),+ ("cadence_cleared_in_the_same_write", {"sync_frequency": "never", "pinned": True}),+ ]+ )+ def test_pinning_needs_a_cadence(self, _name, payload):+ response = self._patch_cadence(self.view_node, payload)++ self.assertEqual(response.status_code, 400, response.json())+ self.view_node.refresh_from_db()+ self.assertFalse(get_declared_pin(self.view_node))++ @parameterized.expand(["table", "managed_view"])+ def test_cadence_is_only_writable_on_views_the_team_owns(self, kind):+ node = self.table_node if kind == "table" else self._create_managed_node()++ with patch("posthoganalytics.feature_enabled", return_value=True):+ response = self._patch_cadence(node, {"sync_frequency": "24hour", "pinned": True})++ self.assertEqual(response.status_code, 400, response.json())+ node.refresh_from_db()+ self.assertEqual((get_declared_target(node), get_declared_pin(node)), (None, False))+def test_cannot_patch_node_of_managed_view(self):node = self._create_managed_node()@@ -1020,6 +1154,52 @@ class TestNodeAccessControl(WarehouseAccessControlTestMixin):self.node.refresh_from_db()self.assertEqual(suspension_state(self.node) == {}, expected_status == status.HTTP_200_OK)+ @parameterized.expand(+ [+ ("viewer", status.HTTP_403_FORBIDDEN),+ ("editor", status.HTTP_200_OK),+ ]+ )+ def test_setting_a_cadence_requires_warehouse_write_access(self, access_level, expected_status):+ user = self.viewer_user if access_level == "viewer" else self.editor_user+ self._create_access_control(user, access_level=access_level)+ self.client.force_login(user)++ with patch("products.data_modeling.backend.logic.schedule_reconcile.maybe_reconcile_dag"):+ response = self.client.patch(+ f"{self._list_url()}{self.node.id}/", data={"sync_frequency": "24hour", "pinned": True}, format="json"+ )++ self.assertEqual(response.status_code, expected_status)+ self.node.refresh_from_db()+ self.assertEqual(get_declared_pin(self.node), expected_status == status.HTTP_200_OK)++ def test_pin_bounds_do_not_name_a_downstream_view_the_member_is_denied(self):+ consumer = Node.objects.create(+ team=self.team,+ dag=self.dag,+ saved_query=DataWarehouseSavedQuery.objects.create(+ name="hidden_consumer", team=self.team, query={"query": "SELECT 1", "kind": "HogQLQuery"}+ ),+ type=NodeType.MAT_VIEW,+ )+ Edge.objects.create(team=self.team, dag=self.dag, source=self.node, target=consumer)+ set_declared_target(consumer, timedelta(hours=6))+ self._create_access_control(self.editor_user, access_level="editor")+ self._create_access_control(+ self.editor_user, resource="warehouse_view", resource_id=str(consumer.saved_query_id), access_level="none"+ )+ self.client.force_login(self.editor_user)++ response = self.client.get(f"{self._list_url()}{self.node.id}/frequency_bounds/")++ self.assertEqual(response.status_code, status.HTTP_200_OK)+ options = {option["cadence"]: option for option in response.json()["options"]}+ self.assertEqual(options["24hour"]["pin_blocked_by"], {"id": None, "name": None})+ self.assertEqual(options["24hour"]["delays_if_pinned"], [])+ self.assertNotIn("hidden_consumer", response.content.decode())+ self.assertNotIn(str(consumer.id), response.content.decode())+@parameterized.expand([("node_viewer", "node", "viewer", status.HTTP_403_FORBIDDEN),diff --git a/products/data_modeling/frontend/generated/api.schemas.ts b/products/data_modeling/frontend/generated/api.schemas.tsindex abe815bb3dd5..e1d1064875f3 100644--- a/products/data_modeling/frontend/generated/api.schemas.ts+++ b/products/data_modeling/frontend/generated/api.schemas.ts@@ -102,6 +102,32 @@ export const NodeOriginEnumApi = {Warehouse: 'warehouse',} as const+/**+ * * `never` - never+ * * `15min` - 15min+ * * `30min` - 30min+ * * `1hour` - 1hour+ * * `6hour` - 6hour+ * * `12hour` - 12hour+ * * `24hour` - 24hour+ * * `7day` - 7day+ * * `30day` - 30day+ */+export type SavedQuerySyncFrequencyEnumApi =+ (typeof SavedQuerySyncFrequencyEnumApi)[keyof typeof SavedQuerySyncFrequencyEnumApi]++export const SavedQuerySyncFrequencyEnumApi = {+ Never: 'never',+ '15min': '15min',+ '30min': '30min',+ '1hour': '1hour',+ '6hour': '6hour',+ '12hour': '12hour',+ '24hour': '24hour',+ '7day': '7day',+ '30day': '30day',+} as const+export interface NodeSuspensionApi {/** When the node was suspended. */at: string@@ -167,6 +193,20 @@ export interface NodeApi {readonly user_tag: string | null/** @nullable */readonly sync_interval: string | null+ /** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.+ *+ * * `never` - never+ * * `15min` - 15min+ * * `30min` - 30min+ * * `1hour` - 1hour+ * * `6hour` - 6hour+ * * `12hour` - 12hour+ * * `24hour` - 24hour+ * * `7day` - 7day+ * * `30day` - 30day */+ sync_frequency?: SavedQuerySyncFrequencyEnumApi+ /** Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again. */+ pinned?: boolean/** Engines this node is suspended for after repeated materialization failures. Suspended engines are skipped by scheduled DAG runs until the node is resumed. */readonly suspended: NodeApiSuspended/** The endpoint version this node's materialization backs, or null for nodes that are not endpoints. */@@ -231,12 +271,106 @@ export interface PatchedNodeApi {readonly user_tag?: string | null/** @nullable */readonly sync_interval?: string | null+ /** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.+ *+ * * `never` - never+ * * `15min` - 15min+ * * `30min` - 30min+ * * `1hour` - 1hour+ * * `6hour` - 6hour+ * * `12hour` - 12hour+ * * `24hour` - 24hour+ * * `7day` - 7day+ * * `30day` - 30day */+ sync_frequency?: SavedQuerySyncFrequencyEnumApi+ /** Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again. */+ pinned?: boolean/** Engines this node is suspended for after repeated materialization failures. Suspended engines are skipped by scheduled DAG runs until the node is resumed. */readonly suspended?: PatchedNodeApiSuspended/** The endpoint version this node's materialization backs, or null for nodes that are not endpoints. */readonly endpoint?: NodeEndpointApi | null}+/**+ * * `15min` - 15min+ * * `30min` - 30min+ * * `1hour` - 1hour+ * * `6hour` - 6hour+ * * `12hour` - 12hour+ * * `24hour` - 24hour+ * * `7day` - 7day+ * * `30day` - 30day+ */+export type MaterializeSyncFrequencyEnumApi =+ (typeof MaterializeSyncFrequencyEnumApi)[keyof typeof MaterializeSyncFrequencyEnumApi]++export const MaterializeSyncFrequencyEnumApi = {+ '15min': '15min',+ '30min': '30min',+ '1hour': '1hour',+ '6hour': '6hour',+ '12hour': '12hour',+ '24hour': '24hour',+ '7day': '7day',+ '30day': '30day',+} as const++export interface NodePinDelayApi {+ /** Node id of the downstream view or endpoint. */+ id: string+ /** Name of the downstream view or endpoint. */+ name: string+}++export interface NodePinBlockerApi {+ /**+ * Node id of the downstream view that stops the pin, or null when the caller may not read it.+ * @nullable+ */+ id: string | null+ /**+ * Name of the downstream view that stops the pin, or null when the caller may not read it.+ * @nullable+ */+ name: string | null+}++export interface NodeFrequencyOptionApi {+ /** A cadence the view could refresh at.+ *+ * * `15min` - 15min+ * * `30min` - 30min+ * * `1hour` - 1hour+ * * `6hour` - 6hour+ * * `12hour` - 12hour+ * * `24hour` - 24hour+ * * `7day` - 7day+ * * `30day` - 30day */+ cadence: MaterializeSyncFrequencyEnumApi+ /** Downstream views and endpoints that refresh more often than this cadence. If this view were pinned to it, they would read data older than their own cadence. */+ delays_if_pinned: NodePinDelayApi[]+ /** The downstream view that stops a pin at this cadence. It reads fresh data only through this view, so a pin this slow would leave it unable to refresh as often as its own cadence. Null when this cadence can be pinned. */+ pin_blocked_by: NodePinBlockerApi | null+ /** The cadence the view would run at if it held this cadence without a pin. Finer than this cadence when a downstream view needs fresher data than the pin allows.+ *+ * * `15min` - 15min+ * * `30min` - 30min+ * * `1hour` - 1hour+ * * `6hour` - 6hour+ * * `12hour` - 12hour+ * * `24hour` - 24hour+ * * `7day` - 7day+ * * `30day` - 30day */+ unpinned_cadence: MaterializeSyncFrequencyEnumApi+}++export interface NodeFrequencyBoundsApi {+ /** True when the view refreshes at its own cadence instead of as often as its downstream views and endpoints need. */+ pinned: boolean+ /** Every cadence a picker may show, finest first. Empty when the node has no view. */+ options: NodeFrequencyOptionApi[]+}+export interface NodeResumeApi {/** False when the node was not suspended to begin with. */resumed: booleandiff --git a/products/data_modeling/frontend/generated/api.ts b/products/data_modeling/frontend/generated/api.tsindex 214e3ab418e6..0337794240e7 100644--- a/products/data_modeling/frontend/generated/api.ts+++ b/products/data_modeling/frontend/generated/api.ts@@ -18,6 +18,7 @@ import type {LineageResponseApi,ModelUsageApi,NodeApi,+ NodeFrequencyBoundsApi,NodeResumeApi,NodeRunApi,NodeRunResponseApi,@@ -239,6 +240,24 @@ export const dataModelingNodesDestroy = async (projectId: string, id: string, op})}+export const getDataModelingNodesFrequencyBoundsRetrieveUrl = (projectId: string, id: string) => {+ return `/api/projects/${projectId}/data_modeling_nodes/${id}/frequency_bounds/`+}++/**+ * The cadences this view could refresh at, and what pinning it to each one would delay downstream.+ */+export const dataModelingNodesFrequencyBoundsRetrieve = async (+ projectId: string,+ id: string,+ options?: RequestInit+): Promise<NodeFrequencyBoundsApi> => {+ return apiMutator<NodeFrequencyBoundsApi>(getDataModelingNodesFrequencyBoundsRetrieveUrl(projectId, id), {+ ...options,+ method: 'GET',+ })+}+export const getDataModelingNodesMaterializeCreateUrl = (projectId: string, id: string) => {return `/api/projects/${projectId}/data_modeling_nodes/${id}/materialize/`}diff --git a/products/data_modeling/frontend/generated/api.zod.ts b/products/data_modeling/frontend/generated/api.zod.tsindex 50b7f9e5de7d..5590100d40a9 100644--- a/products/data_modeling/frontend/generated/api.zod.ts+++ b/products/data_modeling/frontend/generated/api.zod.ts@@ -30,6 +30,21 @@ export const DataModelingNodesUpdateBody = /* @__PURE__ */ zod.object({'\* `table` - Table\n\* `view` - View\n\* `matview` - Mat View\n\* `endpoint` - Endpoint\n\* `metric` - Metric'),description: zod.string().max(dataModelingNodesUpdateBodyDescriptionMax).optional(),+ sync_frequency: zod+ .enum(['never', '15min', '30min', '1hour', '6hour', '12hour', '24hour', '7day', '30day'])+ .describe(+ '\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day'+ )+ .optional()+ .describe(+ "How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.\n\n\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day"+ ),+ pinned: zod+ .boolean()+ .optional()+ .describe(+ 'Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again.'+ ),})export const dataModelingNodesPartialUpdateBodyNameMax = 2048@@ -45,6 +60,21 @@ export const DataModelingNodesPartialUpdateBody = /* @__PURE__ */ zod.object({'\* `table` - Table\n\* `view` - View\n\* `matview` - Mat View\n\* `endpoint` - Endpoint\n\* `metric` - Metric'),description: zod.string().max(dataModelingNodesPartialUpdateBodyDescriptionMax).optional(),+ sync_frequency: zod+ .enum(['never', '15min', '30min', '1hour', '6hour', '12hour', '24hour', '7day', '30day'])+ .describe(+ '\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day'+ )+ .optional()+ .describe(+ "How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.\n\n\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day"+ ),+ pinned: zod+ .boolean()+ .optional()+ .describe(+ 'Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again.'+ ),})/**@@ -63,6 +93,21 @@ export const DataModelingNodesMaterializeCreateBody = /* @__PURE__ */ zod.object'\* `table` - Table\n\* `view` - View\n\* `matview` - Mat View\n\* `endpoint` - Endpoint\n\* `metric` - Metric'),description: zod.string().max(dataModelingNodesMaterializeCreateBodyDescriptionMax).optional(),+ sync_frequency: zod+ .enum(['never', '15min', '30min', '1hour', '6hour', '12hour', '24hour', '7day', '30day'])+ .describe(+ '\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day'+ )+ .optional()+ .describe(+ "How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.\n\n\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day"+ ),+ pinned: zod+ .boolean()+ .optional()+ .describe(+ 'Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again.'+ ),})/**diff --git a/products/data_modeling/frontend/nodeDetail/ModelHealthSummary.tsx b/products/data_modeling/frontend/nodeDetail/ModelHealthSummary.tsxindex 5979c0000707..f5008f26ddb6 100644--- a/products/data_modeling/frontend/nodeDetail/ModelHealthSummary.tsx+++ b/products/data_modeling/frontend/nodeDetail/ModelHealthSummary.tsx@@ -118,7 +118,7 @@ export function ModelHealthSummary({</div><div><dt className="text-secondary mb-1">- <Tooltip title="The refresh schedule saved for this model, including pauses and suspension.">+ <Tooltip title="How often this view refreshes, including pauses and suspension. A downstream view can make it refresh more often than its own setting."><span className="border-b border-dashed border-secondary cursor-help">Refresh schedule</span>diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsxindex 3c52d753d67a..1f82bcdac41f 100644--- a/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx+++ b/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx@@ -2,7 +2,12 @@ import { useActions, useValues } from 'kea'import type { ReactNode } from 'react'import { materializationJobsLogic } from 'scenes/data-warehouse/saved_queries/materializationJobsLogic'-import { CADENCE_LABELS, modeDisabledReason } from 'scenes/data-warehouse/saved_queries/SyncFrequencySelect'+import {+ CADENCE_LABELS,+ inheritedCadence,+ modeDisabledReason,+ SyncFrequencyValue,+} from 'scenes/data-warehouse/saved_queries/SyncFrequencySelect'import { urls } from 'scenes/urls'import { DataModelingSyncInterval } from '~/types'@@ -39,6 +44,7 @@ export function NodeDetailOverview({ id, metadata }: { id: string; metadata?: ResavedQueryError: statusRefreshError,savedQueryLoading: statusRefreshing,lastSuccessfulSyncAt,+ nodeFrequency,} = useValues(materializationLogic)const { refreshMaterialization } = useActions(materializationLogic)const savedQuery = polledSavedQuery ?? sceneSavedQuery@@ -75,7 +81,16 @@ export function NodeDetailOverview({ id, metadata }: { id: string; metadata?: Re// run stopped, not that this model stopped refreshing.const suspension = (savedQuery?.suspended ?? node.suspended)?.[SERVING_ENGINE]const suspended = !!suspension- const cadence = savedQuery?.sync_frequency+ // The saved setting is a lower bound: a downstream view can make this one run more often unless it is pinned.+ // The node in the scene store is not reloaded after a pin save, so the refreshed bounds win when they have loaded.+ const inherited =+ (nodeFrequency?.bounds.pinned ?? node.pinned)+ ? null+ : inheritedCadence(+ savedQuery?.sync_frequency_bounds,+ (savedQuery?.sync_frequency as SyncFrequencyValue | undefined) ?? null+ )+ const cadence = inherited?.cadence ?? savedQuery?.sync_frequencyconst schedule = suspended? 'Suspended after repeated failures': node.type === 'endpoint'diff --git a/products/data_warehouse/backend/presentation/views/saved_query/editing.py b/products/data_warehouse/backend/presentation/views/saved_query/editing.pyindex 1f2c5b16fbb9..1b6677877190 100644--- a/products/data_warehouse/backend/presentation/views/saved_query/editing.py+++ b/products/data_warehouse/backend/presentation/views/saved_query/editing.py@@ -628,15 +628,7 @@ class DataWarehouseSavedQuerySerializer(if frequency_changed and previous_target != target:# The cadence lives on the DAG node, so changes_between() sees nothing and# log_activity would discard the whole updated entry as a no-op.- changes.append(- Change(- type="DataWarehouseSavedQuery",- action="changed",- field="sync_frequency_interval",- before=str(previous_target) if previous_target is not None else None,- after=str(target) if target is not None else None,- )- )+ changes.extend(modeling_api.cadence_changes((previous_target, target)))log_activity(organization_id=team.organization_id,team_id=team.id,diff --git a/products/data_warehouse/backend/tests/api/test_saved_query.py b/products/data_warehouse/backend/tests/api/test_saved_query.pyindex 6708dd72b96f..8a73b4061aa3 100644--- a/products/data_warehouse/backend/tests/api/test_saved_query.py+++ b/products/data_warehouse/backend/tests/api/test_saved_query.py@@ -1441,12 +1441,16 @@ class TestSavedQuery(APIBaseTest):("materialized_without_a_target", NodeType.MAT_VIEW, None, ["upstream_view"]),# an ephemeral view never runs, so nothing about it speeds up("ephemeral", NodeType.VIEW, timedelta(days=1), []),+ # a pinned view keeps its own cadence whatever its consumers ask for+ ("pinned", NodeType.MAT_VIEW, timedelta(days=1), [], True),])- def test_an_option_names_the_upstream_views_it_speeds_up(self, _name, upstream_type, upstream_target, expected):+ def test_an_option_names_the_upstream_views_it_speeds_up(+ self, _name, upstream_type, upstream_target, expected, upstream_pinned=False+ ):# A finer cadence downstream drags slower upstream views along with it. The picker has to# say so before the write, or the upstream cadences change behind the user's back.- from products.data_modeling.backend.facade.api import set_declared_target+ from products.data_modeling.backend.facade.api import apply_saved_query_frequency_target, set_declared_targetupstream = self._create_view_with_a_consumer(consumer_target=timedelta(days=1))upstream_node = Node.objects.get(saved_query_id=upstream["id"])@@ -1454,6 +1458,10 @@ class TestSavedQuery(APIBaseTest):upstream_node.save(update_fields=["type"])if upstream_target is not None:set_declared_target(upstream_node, upstream_target)+ if upstream_pinned:+ apply_saved_query_frequency_target(+ DataWarehouseSavedQuery.objects.get(id=upstream["id"]), None, pinned=True, keep_target=True+ )consumer_id = DataWarehouseSavedQuery.objects.get(team=self.team, name="consumer_view").idoptions = {option["cadence"]: option for option in self._read_frequency_bounds(str(consumer_id))["options"]}diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex a538b8094e5a..f1b8651f78a2 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -60636,6 +60636,20 @@ export namespace Schemas {readonly user_tag: string | null;/** @nullable */readonly sync_interval: string | null;+ /** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.+ *+ * * `never` - never+ * * `15min` - 15min+ * * `30min` - 30min+ * * `1hour` - 1hour+ * * `6hour` - 6hour+ * * `12hour` - 12hour+ * * `24hour` - 24hour+ * * `7day` - 7day+ * * `30day` - 30day */+ sync_frequency?: SavedQuerySyncFrequencyEnum;+ /** Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again. */+ pinned?: boolean;/** Engines this node is suspended for after repeated materialization failures. Suspended engines are skipped by scheduled DAG runs until the node is resumed. */readonly suspended: NodeSuspended;/** The endpoint version this node's materialization backs, or null for nodes that are not endpoints. */@@ -66341,6 +66355,62 @@ export namespace Schemas {specificity_rejection?: SpecificityMetadata | null;}+ export interface NodePinDelay {+ /** Node id of the downstream view or endpoint. */+ id: string;+ /** Name of the downstream view or endpoint. */+ name: string;+ }++ export interface NodePinBlocker {+ /**+ * Node id of the downstream view that stops the pin, or null when the caller may not read it.+ * @nullable+ */+ id: string | null;+ /**+ * Name of the downstream view that stops the pin, or null when the caller may not read it.+ * @nullable+ */+ name: string | null;+ }++ export interface NodeFrequencyOption {+ /** A cadence the view could refresh at.+ *+ * * `15min` - 15min+ * * `30min` - 30min+ * * `1hour` - 1hour+ * * `6hour` - 6hour+ * * `12hour` - 12hour+ * * `24hour` - 24hour+ * * `7day` - 7day+ * * `30day` - 30day */+ cadence: MaterializeSyncFrequencyEnum;+ /** Downstream views and endpoints that refresh more often than this cadence. If this view were pinned to it, they would read data older than their own cadence. */+ delays_if_pinned: NodePinDelay[];+ /** The downstream view that stops a pin at this cadence. It reads fresh data only through this view, so a pin this slow would leave it unable to refresh as often as its own cadence. Null when this cadence can be pinned. */+ pin_blocked_by: NodePinBlocker | null;+ /** The cadence the view would run at if it held this cadence without a pin. Finer than this cadence when a downstream view needs fresher data than the pin allows.+ *+ * * `15min` - 15min+ * * `30min` - 30min+ * * `1hour` - 1hour+ * * `6hour` - 6hour+ * * `12hour` - 12hour+ * * `24hour` - 24hour+ * * `7day` - 7day+ * * `30day` - 30day */+ unpinned_cadence: MaterializeSyncFrequencyEnum;+ }++ export interface NodeFrequencyBounds {+ /** True when the view refreshes at its own cadence instead of as often as its downstream views and endpoints need. */+ pinned: boolean;+ /** Every cadence a picker may show, finest first. Empty when the node has no view. */+ options: NodeFrequencyOption[];+ }+export interface NodeResume {/** False when the node was not suspended to begin with. */resumed: boolean;@@ -82957,6 +83027,20 @@ export namespace Schemas {readonly user_tag?: string | null;/** @nullable */readonly sync_interval?: string | null;+ /** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.+ *+ * * `never` - never+ * * `15min` - 15min+ * * `30min` - 30min+ * * `1hour` - 1hour+ * * `6hour` - 6hour+ * * `12hour` - 12hour+ * * `24hour` - 24hour+ * * `7day` - 7day+ * * `30day` - 30day */+ sync_frequency?: SavedQuerySyncFrequencyEnum;+ /** Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again. */+ pinned?: boolean;/** Engines this node is suspended for after repeated materialization failures. Suspended engines are skipped by scheduled DAG runs until the node is resumed. */readonly suspended?: PatchedNodeSuspended;/** The endpoint version this node's materialization backs, or null for nodes that are not endpoints. */