Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-modeling): let a view pin its cadence below consumer demand (#128082)

Jovan Sakovic807db8c6caed
diff --git a/frontend/src/scenes/data-warehouse/saved_queries/MaterializationStatusPanel.tsx b/frontend/src/scenes/data-warehouse/saved_queries/MaterializationStatusPanel.tsx
index 438c5f697943..2663abf72e3f 100644
--- a/frontend/src/scenes/data-warehouse/saved_queries/MaterializationStatusPanel.tsx
+++ b/frontend/src/scenes/data-warehouse/saved_queries/MaterializationStatusPanel.tsx
@@ -149,6 +149,9 @@ export function MaterializationStatusPanel({
incrementalDraft,
syncFrequencyDraft,
savingMaterialization,
+ cadencePin,
+ nodeFrequency,
+ nodeFrequencyLoading,
} = useValues(jobsLogic)
const {
loadDataModelingJobs,
@@ -159,6 +162,7 @@ export function MaterializationStatusPanel({
setInitialSyncFrequency,
setIncrementalDraft,
setSyncFrequencyDraft,
+ setPinDraft,
} = useActions(jobsLogic)
const { updatingDataWarehouseSavedQuery, materializationActionLoading } = useValues(dataWarehouseViewsLogic)
@@ -278,7 +282,10 @@ export function MaterializationStatusPanel({
</div>
)}
<div className="flex flex-col gap-2 items-start mt-2">
- {kind !== 'endpoint' && (
+ {kind !== 'endpoint' && nodeFrequencyLoading && !nodeFrequency ? (
+ // Without the pin state the bar would first render as unpinned, then jump.
+ <LemonSkeleton className="h-8 w-80" />
+ ) : kind !== 'endpoint' ? (
<SyncFrequencySelect
bounds={savedQuery.sync_frequency_bounds}
disabledReason={
@@ -295,9 +302,15 @@ export function MaterializationStatusPanel({
'never'
}
onChange={setSyncFrequencyDraft}
- loading={updatingDataWarehouseSavedQuery || materializationActionLoading}
+ loading={
+ savingMaterialization ||
+ updatingDataWarehouseSavedQuery ||
+ materializationActionLoading
+ }
+ pin={cadencePin}
+ onUnpin={() => setPinDraft(false)}
/>
- )}
+ ) : null}
{showRunActions && (
<div className="flex flex-wrap items-center gap-2">
<MaterializationRunActions viewId={viewId} kind={kind} />
diff --git a/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.test.ts b/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.test.ts
index b23f4ecb103d..22469b2b3ca6 100644
--- a/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.test.ts
+++ b/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.test.ts
@@ -1,3 +1,5 @@
+import { DataModelingSyncInterval } from '~/types'
+
import { SyncFrequencyBoundsApi } from 'products/data_warehouse/frontend/generated/api.schemas'
import {
@@ -6,7 +8,10 @@ import {
defaultCadenceWithin,
inheritedCadence,
modeDisabledReason,
+ needsPin,
+ pinNote,
SyncFrequencyValue,
+ unpinnedCadence,
speedUpWarning,
unsatisfiableReason,
} from './SyncFrequencySelect'
@@ -14,6 +19,9 @@ import {
describe('SyncFrequencySelect', () => {
const blocker = (name: string): { id: string; name: string } => ({ id: `id-${name}`, name })
+ type PinOptions = NonNullable<Parameters<typeof buildOptions>[1]>
+ const pinning = (options: PinOptions): PinOptions => options
+
const bounds = (overrides: Partial<SyncFrequencyBoundsApi> = {}): SyncFrequencyBoundsApi =>
({
frequency_mode: 'tiered',
@@ -36,14 +44,131 @@ describe('SyncFrequencySelect', () => {
expect(options.every((option) => !option.disabledReason)).toBe(true)
})
- it('disables a blocked cadence and names what blocks it', () => {
- const options = buildOptions(bounds())
+ it.each([
+ ['without pinning', pinning({}), 'Too slow for downstream view daily_revenue', 'Every 1 day'],
+ [
+ 'with pinning, a too-slow cadence names the views the pin delays',
+ pinning({
+ canPin: true,
+ unpinnedBy: { '24hour': '6hour' },
+ pinDelays: { '24hour': [blocker('daily_revenue'), blocker('mrr')] },
+ }),
+ undefined,
+ 'Pin this view to every 1 day. Downstream views daily_revenue, mrr refresh more often and will read older data from it.',
+ ],
+ [
+ 'with pinning, a too-slow cadence falls back to the view that caps it',
+ pinning({ canPin: true, unpinnedBy: { '24hour': '6hour' } }),
+ undefined,
+ 'Pin this view to every 1 day. Downstream view daily_revenue refreshes more often and will read older data from it.',
+ ],
+ [
+ 'with pinning, a pin that strands a downstream view stays off',
+ pinning({ canPin: true, unpinnedBy: { '24hour': '6hour' }, pinBlockedBy: { '24hour': blocker('maps') } }),
+ "Can't pin: downstream view maps reads fresh data only through this view",
+ 'Every 1 day',
+ ],
+ [
+ 'with pinning, a pin that strands a view the caller cannot read is still refused',
+ pinning({
+ canPin: true,
+ unpinnedBy: { '24hour': '6hour' },
+ pinBlockedBy: { '24hour': { id: null, name: null } },
+ }),
+ "Can't pin: a downstream view reads fresh data only through this view",
+ 'Every 1 day',
+ ],
+ ])(
+ 'disables a blocked cadence and names what blocks it %s',
+ (_case, pinOptions, consumerReason, consumerTooltip) => {
+ const options = buildOptions(bounds(), pinOptions)
+
+ const bySource = options.find((option) => option.value === '15min')
+ const byConsumer = options.find((option) => option.value === '24hour')
+ expect(bySource?.disabledReason).toBe('More often than upstream source stripe_invoices syncs')
+ expect(byConsumer?.disabledReason).toBe(consumerReason)
+ expect(byConsumer?.tooltip).toBe(consumerTooltip)
+ expect(options.find((option) => option.value === '6hour')?.disabledReason).toBeUndefined()
+ }
+ )
+
+ it.each([
+ [
+ 'a pinned view, whose bounds no longer carry a consumer ceiling',
+ true,
+ "Can't pin: downstream view maps reads fresh data only through this view",
+ ],
+ ['an unpinned view that can take the cadence without a pin', false, undefined],
+ ])('only withholds a slower cadence that strands a downstream view for %s', (_case, pinned, reason) => {
+ const pinnedBounds = bounds({
+ options: [
+ { cadence: '6hour', allowed: true, blocked_by: null, blocker: null },
+ { cadence: '24hour', allowed: true, blocked_by: null, blocker: null },
+ ],
+ } as Partial<SyncFrequencyBoundsApi>)
+
+ const options = buildOptions(pinnedBounds, {
+ canPin: true,
+ pinned,
+ pinBlockedBy: { '24hour': blocker('maps') },
+ })
- const bySource = options.find((option) => option.value === '15min')
- const byConsumer = options.find((option) => option.value === '24hour')
- expect(bySource?.disabledReason).toBe('More often than upstream source stripe_invoices syncs')
- expect(byConsumer?.disabledReason).toBe('Too slow for downstream view daily_revenue')
- expect(options.find((option) => option.value === '6hour')?.disabledReason).toBeUndefined()
+ expect(options.find((option) => option.value === '24hour')?.disabledReason).toBe(reason)
+ })
+
+ describe('needsPin', () => {
+ it.each([
+ ['a downstream view asks for fresher data than the cadence', '24hour', { '24hour': '15min' }, true],
+ ['nothing downstream asks for fresher data', '24hour', { '24hour': '24hour' }, false],
+ ['the source floor holds the view back instead', '15min', { '15min': '1hour' }, false],
+ ['the cadence is not in the bounds', '24hour', {}, false],
+ ['the view is paused', 'never', { '24hour': '15min' }, false],
+ ])('is decided from the node bounds when %s', (_case, cadence, unpinnedBy, expected) => {
+ expect(
+ needsPin(unpinnedBy as Partial<Record<string, DataModelingSyncInterval>>, cadence as SyncFrequencyValue)
+ ).toBe(expected)
+ })
+
+ it('asks a pinned view for a pin on a slower cadence, which the saved-query bounds no longer refuse', () => {
+ const options = buildOptions(
+ bounds({
+ options: [
+ { cadence: '6hour', allowed: true, blocked_by: null, blocker: null },
+ { cadence: '24hour', allowed: true, blocked_by: null, blocker: null },
+ ],
+ } as Partial<SyncFrequencyBoundsApi>),
+ { canPin: true, unpinnedBy: { '24hour': '15min' } }
+ )
+
+ expect(options.find((option) => option.value === '24hour')?.tooltip).toContain(
+ 'Pin this view to every 1 day.'
+ )
+ })
+ })
+
+ describe('pinNote', () => {
+ it.each([
+ [
+ 'a saved pin nothing reads faster than',
+ false,
+ [],
+ 'Pinned to refresh every 1 day, even when downstream views need fresher data.',
+ ],
+ [
+ 'a saved pin that delays one downstream view',
+ false,
+ ['daily_revenue'],
+ 'Pinned to refresh every 1 day. Downstream view daily_revenue refreshes more often, so it reads data up to 1 day old.',
+ ],
+ [
+ 'an unsaved pin that delays two downstream views',
+ true,
+ ['daily_revenue', 'mrr'],
+ 'Saving pins this view to refresh every 1 day. Downstream views daily_revenue, mrr refresh more often, so they read data up to 1 day old.',
+ ],
+ ])('describes %s', (_case, pending, names, expected) => {
+ expect(pinNote('24hour', names.map(blocker), { pending })).toBe(expected)
+ })
})
it('explains the range only where a per-view cadence is writable', () => {
@@ -281,7 +406,7 @@ describe('SyncFrequencySelect', () => {
'24hour',
{
cadence: '6hour',
- note: 'Refreshes every 6 hours to keep up with downstream view maps. Your 1 day setting applies again if maps refreshes less often.',
+ note: 'Refreshes every 6 hours to keep up with downstream view maps. Your 1 day setting applies again if downstream views refresh less often.',
},
],
[
@@ -290,7 +415,7 @@ describe('SyncFrequencySelect', () => {
'24hour',
{
cadence: '6hour',
- note: 'Refreshes every 6 hours to keep up with a downstream view. Your 1 day setting applies again if that view refreshes less often.',
+ note: 'Refreshes every 6 hours to keep up with a downstream view. Your 1 day setting applies again if downstream views refresh less often.',
},
],
])('when %s', (_case, input, value, expected) => {
@@ -298,3 +423,36 @@ describe('SyncFrequencySelect', () => {
})
})
})
+
+describe('unpinnedCadence', () => {
+ const delaysBy = {
+ '15min': [],
+ '30min': [{ name: 'signups' }],
+ '1hour': [{ name: 'signups' }],
+ '6hour': [{ name: 'signups' }],
+ }
+ // Server-calculated, so a pin further downstream can stop demand: 1 hour unpinned runs hourly even
+ // though a descendant below that pin asks for 15 minutes.
+ const unpinnedBy = {
+ '15min': '15min',
+ '30min': '30min',
+ '1hour': '1hour',
+ '6hour': '15min',
+ } as const
+
+ it.each([
+ [
+ 'a pin that delays a consumer drops back to the cadence the server reports',
+ '6hour',
+ {
+ cadence: '15min',
+ note: 'Once you save, this view refreshes every 15 minutes to keep up with downstream view signups.',
+ },
+ ],
+ ['another pin that stops demand keeps the cadence it reports', '1hour', null],
+ ['a pin that delays nobody changes nothing', '15min', null],
+ ['a paused view changes nothing', 'never', null],
+ ])('%s', (_case, value, expected) => {
+ expect(unpinnedCadence(unpinnedBy, delaysBy, value as SyncFrequencyValue)).toEqual(expected)
+ })
+})
diff --git a/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.tsx b/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.tsx
index 463cece1a3c4..9b897c5d48c3 100644
--- a/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.tsx
+++ b/frontend/src/scenes/data-warehouse/saved_queries/SyncFrequencySelect.tsx
@@ -1,10 +1,11 @@
-import { IconInfo, IconRabbit, IconTortoise } from '@posthog/icons'
+import { IconInfo, IconPinFilled, IconRabbit, IconTortoise, IconX } from '@posthog/icons'
import { LemonLabel, LemonSegmentedButton, LemonSegmentedButtonOption, Spinner } from '@posthog/lemon-ui'
import { Tooltip } from 'lib/lemon-ui/Tooltip'
import { DataModelingSyncInterval } from '~/types'
+import { NodePinBlockerApi, NodePinDelayApi } from 'products/data_modeling/frontend/generated/api.schemas'
import { SyncFrequencyBoundsApi } from 'products/data_warehouse/frontend/generated/api.schemas'
export type SyncFrequencyValue = DataModelingSyncInterval | 'never'
@@ -62,6 +63,21 @@ export function modeDisabledReason(bounds?: SyncFrequencyBoundsApi | null): stri
return (bounds && MODE_DISABLED_REASONS[bounds.frequency_mode]) || null
}
+export interface CadencePin {
+ /** True when the pin is saved or the next save writes it. */
+ pinned: boolean
+ /** True when the next save changes the pin. */
+ pending: boolean
+ /** Downstream views that refresh more often than the pinned cadence. */
+ delayed: NodePinDelayApi[]
+ /** Per cadence, the downstream view a pin that slow would leave unable to meet its own cadence. */
+ blockedBy: Partial<Record<string, NodePinBlockerApi>>
+ /** Per cadence, the downstream views that would read older data if this view were pinned to it. */
+ delaysBy: Partial<Record<string, NodePinDelayApi[]>>
+ /** Per cadence, the cadence this view runs at once unpinned, which only the server can work out because other pins stop demand. */
+ unpinnedBy: Partial<Record<string, DataModelingSyncInterval>>
+}
+
export interface SyncFrequencySelectProps {
/** Bounds from the saved query. Undefined while loading, or on surfaces that don't fetch them. */
bounds?: SyncFrequencyBoundsApi | null
@@ -73,6 +89,9 @@ export interface SyncFrequencySelectProps {
/** Takes precedence over the mode's own reason, so access checks win over scheduling ones. */
disabledReason?: string
'data-attr'?: string
+ /** Set when the view has a DAG node, so a cadence slower than a downstream view can pin instead of being refused. */
+ pin?: CadencePin | null
+ onUnpin?: () => void
}
/**
@@ -88,16 +107,27 @@ export function SyncFrequencySelect({
loading,
disabledReason,
'data-attr': dataAttr,
+ pin,
+ onUnpin,
}: SyncFrequencySelectProps): JSX.Element {
- const inherited = inheritedCadence(bounds, value)
- const selectedSpeedUp = value && value !== 'never' ? speedUpWarning(bounds, value) : null
- const options = buildOptions(bounds)
- const explanation = buildExplanation(bounds, { omitCeiling: !!inherited })
+ const unpinning = pin && !pin.pinned && pin.pending ? unpinnedCadence(pin.unpinnedBy, pin.delaysBy, value) : null
+ const inherited = pin?.pinned || unpinning ? null : inheritedCadence(bounds, value)
+ const selected = unpinning?.cadence ?? inherited?.cadence ?? (value && value !== 'never' ? value : undefined)
+ const options = buildOptions(bounds, {
+ canPin: !!pin,
+ pinned: !!pin?.pinned,
+ pinBlockedBy: pin?.blockedBy,
+ pinDelays: pin?.delaysBy,
+ unpinnedBy: pin?.unpinnedBy,
+ })
+ const canUnpin = !!pin?.pinned && !pin.pending && !!onUnpin
const modeReason = modeDisabledReason(bounds)
+ // A pin can make a cadence selectable that the bounds alone refuse, so the bar is only dead when its own options are.
+ const nothingSelectable = options.every((option) => option.disabledReason)
const reason =
disabledReason ??
(loading ? 'Saving the new cadence.' : undefined) ??
- unsatisfiableReason(bounds) ??
+ (nothingSelectable ? unsatisfiableReason(bounds) : null) ??
modeReason ??
undefined
@@ -123,29 +153,85 @@ export function SyncFrequencySelect({
<IconRabbit className={PACE_ICON_CLASS} aria-hidden />
<LemonSegmentedButton<DataModelingSyncInterval>
size="small"
- value={inherited?.cadence ?? (value && value !== 'never' ? value : undefined)}
+ value={selected}
onChange={(next) => {
- if (next !== value) {
+ // Compare against what the bar shows: a saved cadence the scheduler overrides shows
+ // the inherited one, and clicking the saved one again is how the view gets pinned to it.
+ if (next !== selected) {
onChange(next)
+ } else if (canUnpin) {
+ onUnpin?.()
}
}}
- options={options}
+ options={pin?.pinned ? withPinnedSegment(options, selected, { canUnpin }) : options}
disabledReason={reason}
/>
<IconTortoise className={PACE_ICON_CLASS} aria-hidden />
</div>
- {inherited && (
- <span className="text-xs text-secondary max-w-prose">
- <WithUpstreamRule text={inherited.note} />
- </span>
- )}
- {selectedSpeedUp && (
- <span className="text-xs text-secondary max-w-prose">
- <WithUpstreamRule text={selectedSpeedUp} />
+ <CadenceNotes
+ notes={
+ loading
+ ? []
+ : cadenceNotes({
+ pin,
+ selected,
+ unpinning,
+ inherited,
+ speedUp: value && value !== 'never' ? speedUpWarning(bounds, value) : null,
+ })
+ }
+ explanation={
+ // The inherited note already names the downstream view, so the ceiling sentence would repeat it.
+ // With a pin on offer, a downstream view no longer caps how slow this view can go.
+ buildExplanation(bounds, { omitCeiling: !!inherited || !!pin })
+ }
+ />
+ </div>
+ )
+}
+
+interface CadenceNote {
+ key: string
+ text: string
+ /** True when the note leans on the upstream rule, so it links to the explanation. */
+ upstreamRule: boolean
+}
+
+/** The notes under the bar, in the order they apply: what the pin does, what an unpin does, what the scheduler overrides, what a pick speeds up. */
+function cadenceNotes({
+ pin,
+ selected,
+ unpinning,
+ inherited,
+ speedUp,
+}: {
+ pin?: CadencePin | null
+ selected?: DataModelingSyncInterval
+ unpinning: { note: string } | null
+ inherited: { note: string } | null
+ speedUp: string | null
+}): CadenceNote[] {
+ const notes: (CadenceNote | null)[] = [
+ pin?.pinned && selected
+ ? { key: 'pin', text: pinNote(selected, pin.delayed, { pending: pin.pending }), upstreamRule: false }
+ : null,
+ unpinning ? { key: 'unpinning', text: unpinning.note, upstreamRule: true } : null,
+ inherited ? { key: 'inherited', text: inherited.note, upstreamRule: true } : null,
+ speedUp ? { key: 'speed-up', text: speedUp, upstreamRule: true } : null,
+ ]
+ return notes.filter((note): note is CadenceNote => !!note)
+}
+
+function CadenceNotes({ notes, explanation }: { notes: CadenceNote[]; explanation: string | null }): JSX.Element {
+ return (
+ <>
+ {notes.map((note) => (
+ <span key={note.key} className="text-xs text-secondary max-w-prose">
+ {note.upstreamRule ? <WithUpstreamRule text={note.text} /> : note.text}
</span>
- )}
+ ))}
{explanation && <span className="text-xs text-secondary max-w-prose">{explanation}</span>}
- </div>
+ </>
)
}
@@ -172,6 +258,62 @@ export function speedUpWarning(
return `${subject} also ${plural ? 'refresh' : 'refreshes'} every ${CADENCE_LABELS[cadence]} to keep up with this view.`
}
+export function unpinnedCadence(
+ unpinnedBy: Partial<Record<string, DataModelingSyncInterval>>,
+ delaysBy: Partial<Record<string, { name: string }[]>>,
+ value: SyncFrequencyValue | null
+): { cadence: DataModelingSyncInterval; note: string } | null {
+ if (!value || value === 'never') {
+ return null
+ }
+ const cadence = unpinnedBy[value]
+ if (!cadence || cadence === value) {
+ return null
+ }
+ // The names only describe who reads older data under the pin. The cadence itself comes from the server.
+ const delayed = (delaysBy[value] ?? []).map((view) => view.name)
+ const who = !delayed.length
+ ? 'downstream views'
+ : delayed.length > 1
+ ? `downstream views ${delayed.join(', ')}`
+ : `downstream view ${delayed[0]}`
+ return {
+ cadence,
+ note: `Once you save, this view refreshes every ${CADENCE_LABELS[cadence]} to keep up with ${who}.`,
+ }
+}
+
+/**
+ * True when the view, once unpinned, would run faster than this cadence because a downstream view
+ * asks for it, which is when holding the cadence takes a pin. Read from the node bounds, which are
+ * calculated without this view's own pin, so a pinned view still tells.
+ */
+export function needsPin(unpinnedBy: CadencePin['unpinnedBy'] | undefined, cadence: SyncFrequencyValue): boolean {
+ if (cadence === 'never') {
+ return false
+ }
+ const unpinned = unpinnedBy?.[cadence]
+ return !!unpinned && ORDERED_CADENCES.indexOf(unpinned) < ORDERED_CADENCES.indexOf(cadence)
+}
+
+/** What a pin means for the downstream views, so nobody pins without seeing who reads older data. */
+export function pinNote(
+ cadence: DataModelingSyncInterval,
+ delayed: NodePinDelayApi[],
+ { pending }: { pending: boolean }
+): string {
+ const every = CADENCE_LABELS[cadence]
+ const lead = pending ? `Saving pins this view to refresh every ${every}` : `Pinned to refresh every ${every}`
+ if (!delayed.length) {
+ return `${lead}, even when downstream views need fresher data.`
+ }
+ const plural = delayed.length > 1
+ return (
+ `${lead}. ${plural ? 'Downstream views' : 'Downstream view'} ${delayed.map((view) => view.name).join(', ')} ` +
+ `${plural ? 'refresh' : 'refreshes'} more often, so ${plural ? 'they read' : 'it reads'} data up to ${every} old.`
+ )
+}
+
const UPSTREAM_RULE =
"A view can't be fresher than the data it reads, so upstream views refresh at least as often as the downstream views built on them."
@@ -212,12 +354,25 @@ export function inheritedCadence(
cadence,
note:
`Refreshes every ${CADENCE_LABELS[cadence]} to keep up with ${consumer ? `downstream view ${consumer}` : 'a downstream view'}. ` +
- `Your ${CADENCE_LABELS[value]} setting applies again if ${consumer ?? 'that view'} refreshes less often.`,
+ `Your ${CADENCE_LABELS[value]} setting applies again if downstream views refresh less often.`,
}
}
export function buildOptions(
- bounds?: SyncFrequencyBoundsApi | null
+ bounds?: SyncFrequencyBoundsApi | null,
+ {
+ canPin = false,
+ pinned = false,
+ pinBlockedBy = {},
+ pinDelays = {},
+ unpinnedBy,
+ }: {
+ canPin?: boolean
+ pinned?: boolean
+ pinBlockedBy?: Partial<Record<string, NodePinBlockerApi>>
+ pinDelays?: Partial<Record<string, { name: string }[]>>
+ unpinnedBy?: CadencePin['unpinnedBy']
+ } = {}
): LemonSegmentedButtonOption<DataModelingSyncInterval>[] {
if (!bounds?.options.length) {
return ORDERED_CADENCES.map((cadence) => segment(cadence))
@@ -225,6 +380,28 @@ export function buildOptions(
return bounds.options.map((option) => {
const cadence = option.cadence as DataModelingSyncInterval
+ const stranded = pinBlockedBy[cadence]
+ // A pinned view has no consumer ceiling, so the bounds alone never mark a slower cadence as needing a pin.
+ if (canPin && stranded && (pinned || needsPin(unpinnedBy, cadence))) {
+ return {
+ ...segment(cadence),
+ disabledReason: `Can't pin: ${stranded.name ? `downstream view ${stranded.name}` : 'a downstream view'} reads fresh data only through this view`,
+ }
+ }
+ if (canPin && needsPin(unpinnedBy, cadence)) {
+ const delayed = pinDelays[cadence]?.map((view) => view.name) ?? []
+ const names = delayed.length ? delayed : option.blocker ? [option.blocker.name] : []
+ const who =
+ names.length > 1
+ ? `Downstream views ${names.join(', ')} refresh`
+ : names.length === 1
+ ? `Downstream view ${names[0]} refreshes`
+ : 'A downstream view refreshes'
+ return {
+ ...segment(cadence),
+ tooltip: `Pin this view to every ${CADENCE_LABELS[cadence]}. ${who} more often and will read older data from it.`,
+ }
+ }
return {
...segment(cadence),
disabledReason: option.allowed ? undefined : blockedReason(option, bounds),
@@ -232,6 +409,39 @@ export function buildOptions(
})
}
+/**
+ * The pinned cadence carries a pin, which turns into an unpin cross while the segment is hovered.
+ * Clicking the selected segment unpins, so there is no button nested inside the segment's button.
+ */
+function pinnedSegment(
+ option: LemonSegmentedButtonOption<DataModelingSyncInterval>,
+ { canUnpin }: { canUnpin: boolean }
+): LemonSegmentedButtonOption<DataModelingSyncInterval> {
+ const cadence = option.value
+ return {
+ ...option,
+ label: (
+ <span className="inline-flex items-center gap-0.5">
+ {CADENCE_SEGMENTS[cadence]}
+ <IconPinFilled className={canUnpin ? '[li:hover_&]:hidden' : undefined} aria-hidden />
+ {canUnpin && <IconX className="hidden [li:hover_&]:inline" aria-hidden />}
+ </span>
+ ),
+ tooltip: canUnpin
+ ? `Pinned to every ${CADENCE_LABELS[cadence]}. Click to unpin.`
+ : `Pinned to every ${CADENCE_LABELS[cadence]} once you save.`,
+ 'data-attr': canUnpin ? 'sync-frequency-unpin' : option['data-attr'],
+ }
+}
+
+function withPinnedSegment(
+ options: LemonSegmentedButtonOption<DataModelingSyncInterval>[],
+ selected: DataModelingSyncInterval | undefined,
+ { canUnpin }: { canUnpin: boolean }
+): LemonSegmentedButtonOption<DataModelingSyncInterval>[] {
+ return options.map((option) => (option.value === selected ? pinnedSegment(option, { canUnpin }) : option))
+}
+
/** Abbreviated on the bar, spelled out on hover, so eight segments stay readable at one glance. */
function segment(cadence: DataModelingSyncInterval): LemonSegmentedButtonOption<DataModelingSyncInterval> {
return { value: cadence, label: CADENCE_SEGMENTS[cadence], tooltip: `Every ${CADENCE_LABELS[cadence]}` }
diff --git a/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.test.ts b/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.test.ts
index 9853ad79f25d..580d344cd2af 100644
--- a/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.test.ts
+++ b/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.test.ts
@@ -4,7 +4,7 @@ import { expectLogic } from 'kea-test-utils'
import posthog from 'posthog-js'
import { createElement } from 'react'
-import api from 'lib/api'
+import api, { ApiConfig } from 'lib/api'
import { FEATURE_FLAGS } from 'lib/constants'
import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
@@ -18,6 +18,7 @@ import { MaterializationRunActions } from 'products/data_warehouse/frontend/shar
import { dataWarehouseViewsLogic } from './dataWarehouseViewsLogic'
import { materializationJobsLogic } from './materializationJobsLogic'
import { MaterializationStatusPanel } from './MaterializationStatusPanel'
+import { SyncFrequencyValue } from './SyncFrequencySelect'
const ELIGIBLE_CHECK = {
eligible: true,
@@ -75,6 +76,7 @@ describe('materializationJobsLogic', () => {
},
],
'/api/projects/:team_id/data_modeling_jobs/': { results: [], count: 0 },
+ '/api/projects/:team_id/data_modeling_nodes/:id/': [404, { detail: 'Not found' }],
},
post: {
'/api/environments/:team_id/warehouse_saved_queries/check_incremental/': () => {
@@ -718,6 +720,184 @@ describe('materializationJobsLogic', () => {
expect(logic.values.savingMaterialization).toBe(false)
expect(logic.values.hasMaterializationChanges).toBe(status !== 200)
})
+
+ describe('cadence pin', () => {
+ const orgUniverse = { id: 'up-node', name: 'org_universe' }
+ const dailyRevenue = { id: 'down-node', name: 'daily_revenue' }
+ const TIERED_BOUNDS = {
+ frequency_mode: 'tiered',
+ options: [
+ { cadence: '6hour', allowed: true, blocked_by: null, blocker: null, speeds_up: [orgUniverse] },
+ { cadence: '24hour', allowed: false, blocked_by: 'consumer', blocker: dailyRevenue, speeds_up: [] },
+ ],
+ floor: null,
+ ceiling: { label: '6 hours', blocker: dailyRevenue },
+ best_effort_sources: [],
+ best_effort_sources_withheld: false,
+ }
+ let nodePatches: { id: string; body: Record<string, unknown> }[]
+ let savedQueryPatches: number
+
+ function pinMocks({
+ pinned,
+ patchStatus = 200,
+ nodeOptions = [{ cadence: '24hour', delays_if_pinned: [dailyRevenue], unpinned_cadence: '6hour' }],
+ }: {
+ pinned: boolean
+ patchStatus?: number
+ nodeOptions?: Record<string, unknown>[]
+ }): Parameters<typeof useMocks>[0] {
+ const mocks = apiMocks({ isMaterialized: true, savedQueryExtras: { sync_frequency_bounds: TIERED_BOUNDS } })
+ mocks.get!['/api/projects/:team_id/data_modeling_nodes/:id/'] = (req) =>
+ new URL(req.request.url).searchParams.get('lookup') === 'saved_query'
+ ? [200, { id: 'node-1' }]
+ : [404, { detail: 'Not found' }]
+ mocks.get!['/api/projects/:team_id/data_modeling_nodes/:id/frequency_bounds/'] = [
+ 200,
+ { pinned, options: nodeOptions },
+ ]
+ mocks.patch = {
+ '/api/projects/:team_id/data_modeling_nodes/:id/': async ({ request, params }) => {
+ nodePatches.push({ id: String(params.id), body: (await request.json()) as Record<string, unknown> })
+ return patchStatus === 200
+ ? [200, { id: params.id }]
+ : [patchStatus, { detail: 'Pick how often this view refreshes before pinning it.' }]
+ },
+ '/api/environments/:team_id/warehouse_saved_queries/:id/': () => {
+ savedQueryPatches += 1
+ return [200, { id: 'view-1' }]
+ },
+ }
+ return mocks
+ }
+
+ beforeEach(() => {
+ nodePatches = []
+ savedQueryPatches = 0
+ savedSyncFrequency = '6hour'
+ })
+
+ it.each([
+ [
+ 'pins a cadence slower than a downstream view needs',
+ false,
+ '24hour',
+ null,
+ { sync_frequency: '24hour', pinned: true },
+ ],
+ ['unpins and keeps the saved cadence', true, null, false, { pinned: false }],
+ ])('%s in one node write', async (_case, pinned, cadence, pinDraft, expectedBody) => {
+ useMocks(pinMocks({ pinned }))
+ logic = materializationJobsLogic({ viewId: 'view-1' })
+ logic.mount()
+ await expectLogic(logic).toDispatchActions(['loadNodeFrequencySuccess'])
+ expect(logic.values.nodeFrequency?.bounds.pinned).toBe(pinned)
+
+ if (cadence) {
+ logic.actions.setSyncFrequencyDraft(cadence as SyncFrequencyValue)
+ }
+ if (pinDraft !== null) {
+ logic.actions.setPinDraft(pinDraft)
+ }
+ expect(logic.values.hasMaterializationChanges).toBe(true)
+
+ await expectLogic(logic, () => logic.actions.saveMaterializationChanges()).toDispatchActions([
+ 'finishSavingMaterialization',
+ ])
+ expect(nodePatches).toEqual([{ id: 'node-1', body: expectedBody }])
+ expect(savedQueryPatches).toBe(0)
+ expect(logic.values.hasMaterializationChanges).toBe(false)
+ })
+
+ it('drops a pending pin when the draft moves back to a cadence downstream views allow', async () => {
+ useMocks(pinMocks({ pinned: false }))
+ logic = materializationJobsLogic({ viewId: 'view-1' })
+ logic.mount()
+ await expectLogic(logic).toDispatchActions(['loadNodeFrequencySuccess'])
+
+ logic.actions.setSyncFrequencyDraft('24hour')
+ await expectLogic(logic).toFinishAllListeners()
+ expect(logic.values.pinDraft).toBe(true)
+ logic.actions.setSyncFrequencyDraft('6hour')
+ await expectLogic(logic).toFinishAllListeners()
+ expect(logic.values.pinDraft).toBeNull()
+ expect(logic.values.hasMaterializationChanges).toBe(false)
+ })
+
+ it('pins again when a slower cadence is picked after unpinning', async () => {
+ savedSyncFrequency = '24hour'
+ useMocks(
+ pinMocks({
+ pinned: true,
+ nodeOptions: [
+ { cadence: '12hour', delays_if_pinned: [dailyRevenue], unpinned_cadence: '6hour' },
+ { cadence: '24hour', delays_if_pinned: [dailyRevenue], unpinned_cadence: '6hour' },
+ ],
+ })
+ )
+ logic = materializationJobsLogic({ viewId: 'view-1' })
+ logic.mount()
+ await expectLogic(logic).toDispatchActions(['loadSavedQuerySuccess', 'loadNodeFrequencySuccess'])
+
+ logic.actions.setPinDraft(false)
+ expect(logic.values.cadencePin?.pinned).toBe(false)
+ logic.actions.setSyncFrequencyDraft('12hour')
+ await expectLogic(logic).toFinishAllListeners()
+
+ expect(logic.values.pinDraft).toBe(true)
+ expect(logic.values.cadencePin?.pinned).toBe(true)
+ })
+
+ it('addresses the node API by the active environment, not the project', async () => {
+ ApiConfig.setCurrentProjectId(111)
+ ApiConfig.setCurrentTeamId(222)
+ const mocks = pinMocks({ pinned: false })
+ const requested: string[] = []
+ for (const path of [
+ '/api/projects/:team_id/data_modeling_nodes/:id/',
+ '/api/projects/:team_id/data_modeling_nodes/:id/frequency_bounds/',
+ ]) {
+ const handler = mocks.get![path]
+ mocks.get![path] = (req: any) => {
+ requested.push(String(req.params.team_id))
+ return typeof handler === 'function' ? handler(req) : handler
+ }
+ }
+ useMocks(mocks)
+ logic = materializationJobsLogic({ viewId: 'view-1' })
+ logic.mount()
+ await expectLogic(logic).toDispatchActions(['loadNodeFrequencySuccess'])
+ expect(requested).toEqual(['222', '222'])
+ })
+
+ it('keeps a poll that started before the save from overwriting the saved cadence', async () => {
+ useMocks(pinMocks({ pinned: false }))
+ logic = materializationJobsLogic({ viewId: 'view-1' })
+ logic.mount()
+ await expectLogic(logic).toDispatchActions(['loadSavedQuerySuccess', 'loadNodeFrequencySuccess'])
+ const previous = logic.values.savedQuery!
+ let resolveStale!: (value: typeof previous) => void
+ const stale = new Promise<typeof previous>((resolve) => {
+ resolveStale = resolve
+ })
+ const fetch = jest
+ .spyOn(api.dataWarehouseSavedQueries, 'get')
+ .mockImplementationOnce(() => stale)
+ .mockResolvedValue({ ...previous, sync_frequency: '24hour' })
+ try {
+ logic.actions.loadSavedQuery()
+ logic.actions.setSyncFrequencyDraft('24hour')
+ await expectLogic(logic, () => logic.actions.saveMaterializationChanges()).toDispatchActions([
+ 'finishSavingMaterialization',
+ ])
+ resolveStale(previous)
+ await expectLogic(logic).toFinishAllListeners()
+ expect(logic.values.savedQuery?.sync_frequency).toBe('24hour')
+ } finally {
+ fetch.mockRestore()
+ }
+ })
+ })
it.each(['Running', 'Completed'])(
'polls a %s job at the appropriate interval and recovers from failure',
async (status) => {
diff --git a/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.ts b/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.ts
index f8c6c2b16010..16d08e96a46a 100644
--- a/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.ts
+++ b/frontend/src/scenes/data-warehouse/saved_queries/materializationJobsLogic.ts
@@ -23,6 +23,15 @@ import type { DataWarehouseSavedQuerySummary } from 'scenes/data-warehouse/saved
import type { DataWarehouseSavedQuery } from '~/types'
import { DataModelingSyncInterval, DataWarehouseSavedQueryIncrementalCheck } from '~/types'
+import {
+ dataModelingNodesFrequencyBoundsRetrieve,
+ dataModelingNodesPartialUpdate,
+ dataModelingNodesRetrieve,
+} from 'products/data_modeling/frontend/generated/api'
+import {
+ DataModelingNodesRetrieveLookup,
+ type NodeFrequencyBoundsApi,
+} from 'products/data_modeling/frontend/generated/api.schemas'
import { dataModelingJobsList, warehouseSavedQueriesResumeCreate } from 'products/data_warehouse/frontend/generated/api'
import { DataModelingJobsListEngine } from 'products/data_warehouse/frontend/generated/api.schemas'
import type {
@@ -37,7 +46,7 @@ import {
} from '../editor/IncrementalConfigFields'
import { DEFAULT_MATERIALIZE_SYNC_FREQUENCY, dataWarehouseViewsLogic } from './dataWarehouseViewsLogic'
import { latestSuccessfulSyncAt } from './materializationJobUtils'
-import { SyncFrequencyValue } from './SyncFrequencySelect'
+import { CadencePin, needsPin, SyncFrequencyValue } from './SyncFrequencySelect'
const ACTIVE_REFRESH_INTERVAL_MS = 5000
const IDLE_REFRESH_INTERVAL_MS = 60000
@@ -52,6 +61,15 @@ interface OlderJobsPageResult {
jobs: PaginatedDataModelingJobListApi
}
+export interface NodeFrequency {
+ nodeId: string
+ bounds: NodeFrequencyBoundsApi
+}
+
+function apiErrorDetail(error: unknown): string {
+ return error instanceof ApiError && error.detail ? error.detail : 'Try again.'
+}
+
function runHistoryEngineParams(engine: DataModelingJobsListEngine): {
all_engines?: boolean
engine?: DataModelingJobsListEngine
@@ -71,6 +89,7 @@ export interface MaterializationJobsLogicProps {
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface materializationJobsLogicValues {
+ cadencePin: CadencePin | null
dataModelingJobs: PaginatedDataModelingJobListApi | null
dataModelingJobsError: boolean
dataModelingJobsLoading: boolean
@@ -88,9 +107,13 @@ export interface materializationJobsLogicValues {
latestCompletedJob: PaginatedDataModelingJobListApi | null
latestCompletedJobLoading: boolean
materializationRefreshPending: boolean
+ nodeFrequency: NodeFrequency | null
+ nodeFrequencyLoading: boolean
olderJobsPage: OlderJobsPageResult | null
olderJobsPageError: boolean
olderJobsPageLoading: boolean
+ pinChanged: boolean
+ pinDraft: boolean | null
refreshModeChanged: boolean
resumingMaterialization: boolean
runHistoryError: boolean
@@ -210,6 +233,21 @@ export interface materializationJobsLogicActions {
latestCompletedJob: PaginatedDataModelingJobListApi
payload?: void
}
+ loadNodeFrequency: (_: void) => void
+ loadNodeFrequencyFailure: (
+ error: string,
+ errorObject?: any
+ ) => {
+ error: string
+ errorObject?: any
+ }
+ loadNodeFrequencySuccess: (
+ nodeFrequency: NodeFrequency | null,
+ payload?: void
+ ) => {
+ nodeFrequency: NodeFrequency | null
+ payload?: void
+ }
loadOlderJobsPage: (_: void) => void
loadOlderJobsPageFailure: (
error: string,
@@ -287,6 +325,9 @@ export interface materializationJobsLogicActions {
setJobsPage: (page: number) => {
page: number
}
+ setPinDraft: (pinned: boolean | null) => {
+ pinned: boolean | null
+ }
setResumingMaterialization: (resuming: boolean) => {
resuming: boolean
}
@@ -311,11 +352,20 @@ export interface materializationJobsLogicMeta {
incrementalDraft: IncrementalConfigDraft,
incrementalDraftTouched: boolean
) => boolean
+ pinChanged: (pinDraft: boolean | null, nodeFrequency: NodeFrequency | null) => boolean
hasMaterializationChanges: (
savedQuery: DataWarehouseSavedQuery | null,
syncFrequencyDraft: SyncFrequencyValue | null,
- refreshModeChanged: boolean
+ refreshModeChanged: boolean,
+ pinChanged: boolean
) => boolean
+ cadencePin: (
+ nodeFrequency: NodeFrequency | null,
+ pinDraft: boolean | null,
+ pinChanged: boolean,
+ syncFrequencyDraft: SyncFrequencyValue | null,
+ savedQuery: DataWarehouseSavedQuery | null
+ ) => CadencePin | null
jobsPageResults: (
jobsPage: number,
selectedRunHistoryEngine: DataModelingJobsListEngine,
@@ -383,6 +433,7 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([
resumeMaterialization: true,
setSyncFrequencyDraft: (frequency: SyncFrequencyValue) => ({ frequency }),
clearSyncFrequencyDraft: true,
+ setPinDraft: (pinned: boolean | null) => ({ pinned }),
discardMaterializationChanges: true,
saveMaterializationChanges: true,
finishSavingMaterialization: true,
@@ -407,6 +458,34 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([
},
},
],
+ // The cadence pin lives on the view's DAG node, which the saved query payload does not carry.
+ nodeFrequency: [
+ null as NodeFrequency | null,
+ {
+ loadNodeFrequency: async (_: void, breakpoint) => {
+ const teamId = String(ApiConfig.getCurrentTeamId())
+ let frequency: NodeFrequency
+ try {
+ const node = await dataModelingNodesRetrieve(teamId, props.viewId, {
+ lookup: DataModelingNodesRetrieveLookup.SavedQuery,
+ })
+ const bounds = await dataModelingNodesFrequencyBoundsRetrieve(teamId, node.id)
+ frequency = { nodeId: node.id, bounds }
+ } catch (e) {
+ // A view with no node has nothing to pin, and the picker then refuses slow cadences as before.
+ if (!(e instanceof ApiError && e.status === 404)) {
+ posthog.captureException(e)
+ // Not an answer about the view, so the next saved query reload gets to try again.
+ cache.nodeFrequencyRequested = false
+ }
+ return null
+ }
+ // Outside the try: a breakpoint throws to cancel a superseded load, and the catch would turn that into "no node".
+ breakpoint()
+ return frequency
+ },
+ },
+ ],
incrementalCheck: [
INCREMENTAL_CHECK_LOADING as DataWarehouseSavedQueryIncrementalCheck | null,
{
@@ -539,6 +618,13 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([
discardMaterializationChanges: () => null,
},
],
+ pinDraft: [
+ null as boolean | null,
+ {
+ setPinDraft: (_, { pinned }) => pinned,
+ discardMaterializationChanges: () => null,
+ },
+ ],
savingMaterialization: [
false,
{
@@ -624,15 +710,56 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([
)
},
],
+ pinChanged: [
+ (s) => [s.pinDraft, s.nodeFrequency],
+ (pinDraft: boolean | null, nodeFrequency: NodeFrequency | null): boolean =>
+ pinDraft !== null && !!nodeFrequency && pinDraft !== nodeFrequency.bounds.pinned,
+ ],
hasMaterializationChanges: [
- (s) => [s.savedQuery, s.syncFrequencyDraft, s.refreshModeChanged],
+ (s) => [s.savedQuery, s.syncFrequencyDraft, s.refreshModeChanged, s.pinChanged],
(
savedQuery: DataWarehouseSavedQuery | null,
frequency: SyncFrequencyValue | null,
- modeChanged: boolean
+ modeChanged: boolean,
+ pinChanged: boolean
): boolean =>
!!savedQuery?.is_materialized &&
- (modeChanged || (frequency !== null && frequency !== (savedQuery.sync_frequency || 'never'))),
+ (modeChanged ||
+ pinChanged ||
+ (frequency !== null && frequency !== (savedQuery.sync_frequency || 'never'))),
+ ],
+ cadencePin: [
+ (s) => [s.nodeFrequency, s.pinDraft, s.pinChanged, s.syncFrequencyDraft, s.savedQuery],
+ (
+ nodeFrequency: NodeFrequency | null,
+ pinDraft: boolean | null,
+ pinChanged: boolean,
+ frequency: SyncFrequencyValue | null,
+ savedQuery: DataWarehouseSavedQuery | null
+ ): CadencePin | null => {
+ if (!nodeFrequency) {
+ return null
+ }
+ const cadence = frequency ?? savedQuery?.sync_frequency
+ return {
+ pinned: pinDraft ?? nodeFrequency.bounds.pinned,
+ pending: pinChanged,
+ delayed:
+ nodeFrequency.bounds.options.find((option) => option.cadence === cadence)?.delays_if_pinned ??
+ [],
+ blockedBy: Object.fromEntries(
+ nodeFrequency.bounds.options.flatMap((option) =>
+ option.pin_blocked_by ? [[option.cadence, option.pin_blocked_by]] : []
+ )
+ ),
+ delaysBy: Object.fromEntries(
+ nodeFrequency.bounds.options.map((option) => [option.cadence, option.delays_if_pinned])
+ ),
+ unpinnedBy: Object.fromEntries(
+ nodeFrequency.bounds.options.map((option) => [option.cadence, option.unpinned_cadence])
+ ),
+ }
+ },
],
jobsPageResults: [
(s) => [s.jobsPage, s.selectedRunHistoryEngine, s.dataModelingJobs, s.runHistoryJobs, s.olderJobsPage],
@@ -817,6 +944,12 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([
actions.scheduleJobsRefresh(IDLE_REFRESH_INTERVAL_MS)
},
setSyncFrequencyDraft: ({ frequency }) => {
+ // A cadence slower than a downstream view needs is only valid pinned, so the pick carries the pin.
+ if (values.cadencePin && needsPin(values.cadencePin.unpinnedBy, frequency)) {
+ actions.setPinDraft(true)
+ } else if (values.pinDraft) {
+ actions.setPinDraft(null)
+ }
// The cadence bar reports a change on every click, including the segment that is already
// selected, so a draft can hold the cadence that is already saved. Such a draft is invisible
// because it is not a change, until something else moves the saved cadence. Pausing sets the
@@ -839,15 +972,59 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([
: { ...EMPTY_INCREMENTAL_DRAFT, incrementalKey: values.incrementalCheck?.key_candidates[0] ?? null }
)
},
- saveMaterializationChanges: () => {
+ saveMaterializationChanges: async () => {
const draft = values.incrementalDraft
if (!values.savedQuery || !values.hasMaterializationChanges) {
actions.finishSavingMaterialization()
return
}
+ let cadenceWritten = false
+ if (values.pinChanged && values.nodeFrequency) {
+ // The cadence and the pin go in one node write: a cadence slower than a downstream view
+ // needs is refused unless it is pinned in the same request.
+ try {
+ await dataModelingNodesPartialUpdate(
+ String(ApiConfig.getCurrentTeamId()),
+ values.nodeFrequency.nodeId,
+ {
+ ...(values.syncFrequencyDraft !== null
+ ? { sync_frequency: values.syncFrequencyDraft }
+ : {}),
+ pinned: !!values.pinDraft,
+ }
+ )
+ } catch (e) {
+ lemonToast.error(`Couldn't save the cadence. ${apiErrorDetail(e)}`)
+ actions.finishSavingMaterialization()
+ return
+ }
+ cadenceWritten = true
+ if (!values.refreshModeChanged) {
+ // Keep the draft on screen until both reloads land, so the picker never shows the old
+ // pin state between the write and the fresh bounds.
+ // Through the loader, so its breakpoint drops any poll that started before the write.
+ const savedQueryReload = actions.loadSavedQuery()
+ try {
+ const bounds = await dataModelingNodesFrequencyBoundsRetrieve(
+ String(ApiConfig.getCurrentTeamId()),
+ values.nodeFrequency.nodeId
+ )
+ actions.loadNodeFrequencySuccess({ nodeId: values.nodeFrequency.nodeId, bounds })
+ } catch {
+ actions.loadNodeFrequency()
+ }
+ await savedQueryReload
+ actions.discardMaterializationChanges()
+ actions.finishSavingMaterialization()
+ return
+ }
+ actions.loadNodeFrequency()
+ }
actions.updateDataWarehouseSavedQuery({
id: props.viewId,
- ...(values.syncFrequencyDraft !== null ? { sync_frequency: values.syncFrequencyDraft } : {}),
+ ...(values.syncFrequencyDraft !== null && !cadenceWritten
+ ? { sync_frequency: values.syncFrequencyDraft }
+ : {}),
...(values.refreshModeChanged
? {
incremental:
@@ -890,6 +1067,10 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([
if (!savedQuery || props.kind === 'endpoint') {
return
}
+ if (savedQuery.is_materialized && !cache.nodeFrequencyRequested) {
+ cache.nodeFrequencyRequested = true
+ actions.loadNodeFrequency()
+ }
// Reflect the saved config so a materialized view's editor starts from what is
// actually configured, not from the empty draft.
if (!values.incrementalDraftTouched && savedQuery.incremental?.enabled) {
@@ -913,6 +1094,10 @@ export const materializationJobsLogic = kea<materializationJobsLogicType>([
// doesn't own our `savedQuery`. Reload it so the displayed frequency reflects the new value
// immediately instead of reverting until the next poll.
if (payload?.id === props.viewId) {
+ // Pausing a view clears its pin on the server, so the pin held here may be out of date.
+ if (values.nodeFrequency) {
+ actions.loadNodeFrequency()
+ }
if (values.savingMaterialization && values.savedQuery) {
actions.loadSavedQuerySuccess({ ...values.savedQuery, ...payload } as DataWarehouseSavedQuery)
actions.discardMaterializationChanges()
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 2d2ec9af1f61..3d2cc6b81824 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -6266,6 +6266,7 @@ export interface DataModelingNode {
last_run_status?: DataModelingJobStatus
last_run_error?: string | null
sync_interval?: DataModelingSyncInterval
+ pinned?: boolean
suspended?: NodeApiSuspended
/** Set on endpoint nodes stamped at materialization enable; older nodes carry only the name */
endpoint?: NodeEndpointApi | null
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 6eccf7aabe7e..3b95c2a2a39f 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -104,7 +104,6 @@
1 products/dashboards/backend/widgets/list_widget.py
1 products/data_modeling/backend/logic/cohort_scheduling.py
1 products/data_modeling/backend/logic/gitsync/model_parser.py
-1 products/data_modeling/backend/logic/node_frequency.py
1 products/data_modeling/backend/logic/schedule_reconcile.py
1 products/data_modeling/backend/logic/tier_membership.py
1 products/data_modeling/backend/models/datawarehouse_managed_viewset.py
diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.py
index 77a13b13e611..be64bab3d047 100644
--- a/products/data_modeling/backend/facade/api.py
+++ b/products/data_modeling/backend/facade/api.py
@@ -11,6 +11,7 @@ _B = "products.data_modeling.backend."
_LAZY = {
"UnsatisfiableFrequencyError": "logic.freshness",
+ "SCHEDULABLE_BUCKETS": "logic.freshness",
"UnsupportedFrequencyTargetError": "logic.freshness",
"humanize_cadence": "logic.freshness",
"get_model_usage": "logic.usage",
@@ -92,8 +93,12 @@ _LAZY = {
"node_states_by_saved_query": "logic.node_frequency",
"get_declared_target": "logic.node_frequency",
"set_declared_target": "logic.node_frequency",
+ "is_declared_pin": "logic.node_frequency",
"saved_query_target_bounds": "logic.node_frequency",
"SavedQueryFrequencyBounds": "logic.node_frequency",
+ "saved_query_pin_preview": "logic.node_frequency",
+ "cadence_changes": "logic.cadence_changes",
+ "clear_legacy_sync_interval": "logic.cadence_changes",
"clear_node_suspension": "logic.node_suspension",
"is_node_suspended": "logic.node_suspension",
"mark_node_suspended": "logic.node_suspension",
diff --git a/products/data_modeling/backend/logic/cadence_changes.py b/products/data_modeling/backend/logic/cadence_changes.py
new file mode 100644
index 000000000000..88bf8fb0ecd1
--- /dev/null
+++ b/products/data_modeling/backend/logic/cadence_changes.py
@@ -0,0 +1,42 @@
+"""Shared bookkeeping for writes to a saved query's cadence, used by every surface that makes one."""
+
+from datetime import timedelta
+from uuid import UUID
+
+from posthog.models.activity_logging.activity_log import Change
+
+from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
+
+
+def clear_legacy_sync_interval(team_id: int, saved_query_id: UUID) -> None:
+ """Drop the saved query's own interval, which the node target replaces on tiered teams.
+
+ The node target is the only store of frequency intent. A stale interval left in the column would
+ answer for a cadence the user just set or cleared, so every cadence write clears it. A caller
+ holding the row must also clear its in-memory copy before serializing it.
+ """
+ DataWarehouseSavedQuery.objects.filter(team_id=team_id, pk=saved_query_id).update(sync_frequency_interval=None)
+
+
+def cadence_changes(
+ targets: tuple[timedelta | None, timedelta | None], pins: tuple[bool, bool] | None = None
+) -> list[Change]:
+ """The activity log entries for a cadence write, one per field that changed.
+
+ The cadence lives on the DAG node, so `changes_between()` on the saved query sees nothing. Every
+ cadence writer builds its entries here so the fields read alike in the activity log.
+ """
+
+ def render(target: timedelta | None) -> str | None:
+ return str(target) if target is not None else None
+
+ fields: list[tuple[str, str | bool | None, str | bool | None]] = [
+ ("sync_frequency_interval", render(targets[0]), render(targets[1]))
+ ]
+ if pins is not None:
+ fields.append(("sync_frequency_pinned", pins[0], pins[1]))
+ return [
+ Change(type="DataWarehouseSavedQuery", action="changed", field=field, before=before, after=after)
+ for field, before, after in fields
+ if before != after
+ ]
diff --git a/products/data_modeling/backend/logic/freshness.py b/products/data_modeling/backend/logic/freshness.py
index de0a1679e21c..e7e71ea52946 100644
--- a/products/data_modeling/backend/logic/freshness.py
+++ b/products/data_modeling/backend/logic/freshness.py
@@ -22,6 +22,10 @@ Vocabulary, one term per concept:
staler than a consumer requires).
In interval-space a smaller timedelta means fresher/more frequent, so as plain
timedeltas: source_floor <= target <= consumer_ceiling.
+- pinned: an opt-in on a node with a declared target. Its effective cadence is its own target,
+ so consumers do not speed it up, and its ancestors see the pinned target instead of the demand
+ below it. It has no consumer ceiling. Its children inherit the coarser of its source floor and
+ its pinned target as their floor contribution, so a child with another, fresher parent stays fast.
Worked example, a chain src -> view -> endpoint:
src is an imported table syncing every 1hour (its source interval), endpoint
@@ -37,7 +41,7 @@ Edges are (upstream_id, downstream_id): data flows upstream -> downstream, so a
import dataclasses
from collections import defaultdict, deque
-from collections.abc import Iterable, Mapping
+from collections.abc import Collection, Iterable, Mapping
from datetime import timedelta
from typing import Literal
@@ -128,18 +132,26 @@ def _adjacency(edges: list[tuple[str, str]]) -> Adjacency:
return Adjacency(children=children, parents=parents)
+def _pinned_targets(pinned: Collection[str], declared_targets: Mapping[str, timedelta]) -> dict[str, timedelta]:
+ """The pinned nodes that have a target to hold. A pin without a target pins nothing."""
+ return {node: declared_targets[node] for node in pinned if node in declared_targets}
+
+
def compute_effective_cadences(
*,
nodes: set[str],
edges: list[tuple[str, str]],
declared_targets: dict[str, timedelta],
+ pinned: Collection[str] = frozenset(),
) -> dict[str, timedelta | None]:
"""Resolve each node's effective cadence.
Returns a value for every id in `nodes`. `None` means "unscheduled" — a node with
no declared target and no scheduled descendant demanding freshness (the
- ride-downstream opt-out). Source nodes are not expected in `nodes`.
+ ride-downstream opt-out). Source nodes are not expected in `nodes`. A pinned node
+ resolves to its own declared target whatever its consumers demand.
"""
+ pinned_targets = _pinned_targets(pinned, declared_targets)
adj = _adjacency(edges)
# reverse-topological pass, iterative because recursion overflows on deep chains
out_degree = {node: sum(1 for child in adj.children.get(node, []) if child in nodes) for node in nodes}
@@ -148,11 +160,14 @@ def compute_effective_cadences(
while queue:
node = queue.popleft()
candidates: list[timedelta] = []
- if node in declared_targets:
- candidates.append(declared_targets[node])
- for child in adj.children.get(node, []):
- if child in nodes and (child_effective := resolved[child]) is not None:
- candidates.append(child_effective)
+ if node in pinned_targets:
+ candidates.append(pinned_targets[node])
+ else:
+ if node in declared_targets:
+ candidates.append(declared_targets[node])
+ for child in adj.children.get(node, []):
+ if child in nodes and (child_effective := resolved[child]) is not None:
+ candidates.append(child_effective)
# min = the finest demand wins (smaller timedelta = fresher)
resolved[node] = min(candidates) if candidates else None
for parent in adj.parents.get(node, []):
@@ -228,9 +243,21 @@ def ancestors_of(node_id: str, edges: list[tuple[str, str]]) -> set[str]:
return seen
-def all_source_floor_bounds(edges: list[tuple[str, str]], source_intervals: dict[str, timedelta]) -> dict[str, Bound]:
+def descendants_of(node_id: str, edges: list[tuple[str, str]]) -> set[str]:
+ """Every node downstream of `node_id`, however far — the cone that reads what it delivers."""
+ return ancestors_of(node_id, [(downstream, upstream) for upstream, downstream in edges])
+
+
+def all_source_floor_bounds(
+ edges: list[tuple[str, str]],
+ source_intervals: dict[str, timedelta],
+ pinned_targets: Mapping[str, timedelta] | None = None,
+) -> dict[str, Bound]:
"""Every node's source floor, with the source that set it, in one forward pass.
+ A pinned node (`pinned_targets` maps it to its pinned cadence) passes its children the coarser
+ of its own floor and its pin, blamed on the pinned node when the pin is the coarser side.
+
A source node's floor is its own sync interval. A derived node's floor is the finest (min)
among its parents' floors, because its output changes whenever any input changes: a view
joining events with a weekly import produces new rows continuously, so the weekly side must
@@ -244,14 +271,19 @@ def all_source_floor_bounds(edges: list[tuple[str, str]], source_intervals: dict
all_ids = set(source_intervals) | {node for edge in edges for node in edge}
in_degree = {node: len(adj.parents.get(node, [])) for node in all_ids}
queue = deque(node for node in all_ids if in_degree[node] == 0)
+ pinned_targets = pinned_targets or {}
floor: dict[str, Bound] = {}
+ passed_down: dict[str, Bound] = {}
while queue:
node = queue.popleft()
if node in source_intervals:
floor[node] = Bound(value=source_intervals[node], blocker=node)
else:
- inherited = [floor[parent] for parent in adj.parents.get(node, [])]
+ inherited = [passed_down[parent] for parent in adj.parents.get(node, [])]
floor[node] = _finest(inherited) if inherited else Bound(value=STREAMING, blocker=None)
+ passed_down[node] = floor[node]
+ if node in pinned_targets:
+ passed_down[node] = max(floor[node], Bound(value=pinned_targets[node], blocker=node), key=_bound_sort_key)
for child in adj.children.get(node, []):
in_degree[child] -= 1
if in_degree[child] == 0:
@@ -260,13 +292,17 @@ def all_source_floor_bounds(edges: list[tuple[str, str]], source_intervals: dict
def all_consumer_ceiling_bounds(
- edges: list[tuple[str, str]], declared_targets: dict[str, timedelta]
+ edges: list[tuple[str, str]],
+ declared_targets: dict[str, timedelta],
+ pinned: Collection[str] = frozenset(),
) -> dict[str, Bound | None]:
"""Every node's consumer ceiling, with the descendant that declared it, in one reverse pass.
The ceiling is the finest declared target among strict descendants; None when no descendant
- declares one. Cyclic nodes are omitted.
+ declares one. Cyclic nodes are omitted. A pinned node has no ceiling, so its ancestors see
+ only its own target and none of the demand below it.
"""
+ pinned_targets = _pinned_targets(pinned, declared_targets)
adj = _adjacency(edges)
all_ids = set(declared_targets) | {node for edge in edges for node in edge}
out_degree = {node: len(adj.children.get(node, [])) for node in all_ids}
@@ -280,7 +316,7 @@ def all_consumer_ceiling_bounds(
if child in declared_targets
]
candidates += [bound for child in adj.children.get(node, []) if (bound := ceiling.get(child)) is not None]
- ceiling[node] = _finest(candidates) if candidates else None
+ ceiling[node] = _finest(candidates) if candidates and node not in pinned_targets else None
for parent in adj.parents.get(node, []):
out_degree[parent] -= 1
if out_degree[parent] == 0:
@@ -288,18 +324,26 @@ def all_consumer_ceiling_bounds(
return ceiling
-def all_source_floors(edges: list[tuple[str, str]], source_intervals: dict[str, timedelta]) -> dict[str, timedelta]:
+def all_source_floors(
+ edges: list[tuple[str, str]],
+ source_intervals: dict[str, timedelta],
+ pinned_targets: Mapping[str, timedelta] | None = None,
+) -> dict[str, timedelta]:
"""Every node's source floor, for callers that need the value and not the blame."""
- return {node: bound.value for node, bound in all_source_floor_bounds(edges, source_intervals).items()}
+ return {
+ node: bound.value for node, bound in all_source_floor_bounds(edges, source_intervals, pinned_targets).items()
+ }
def all_consumer_ceilings(
- edges: list[tuple[str, str]], declared_targets: dict[str, timedelta]
+ edges: list[tuple[str, str]],
+ declared_targets: dict[str, timedelta],
+ pinned: Collection[str] = frozenset(),
) -> dict[str, timedelta | None]:
"""Every node's consumer ceiling, for callers that need the value and not the blame."""
return {
node: (bound.value if bound is not None else None)
- for node, bound in all_consumer_ceiling_bounds(edges, declared_targets).items()
+ for node, bound in all_consumer_ceiling_bounds(edges, declared_targets, pinned).items()
}
@@ -348,6 +392,7 @@ def clamp_to_source_floor(
*,
edges: list[tuple[str, str]],
source_intervals: dict[str, timedelta],
+ pinned: Collection[str] = frozenset(),
) -> tuple[dict[str, timedelta | None], list[ClampedCadence]]:
"""Coarsen every node scheduled finer than its sources can deliver to the nearest bucket >= its
source floor, returning the adjusted cadences and the list of changes.
@@ -356,8 +401,12 @@ def clamp_to_source_floor(
consumer, which is intended: the consumer's fast inputs keep it changing at its fine cadence,
while the columns derived from the slow lineage update as often as that lineage delivers.
Streaming/best-effort sources have a zero floor and are never clamped.
+
+ A pinned node's effective cadence is its pinned target, so its children's floors read it from
+ `effective`.
"""
- floors = all_source_floors(edges, source_intervals)
+ pinned_targets = {node: cadence for node in pinned if (cadence := effective.get(node)) is not None}
+ floors = all_source_floors(edges, source_intervals, pinned_targets)
clamped: dict[str, timedelta | None] = {}
changes: list[ClampedCadence] = []
for node_id, cadence in effective.items():
@@ -437,6 +486,7 @@ def compute_target_bounds(
edges: list[tuple[str, str]],
declared_targets: dict[str, timedelta],
source_intervals: dict[str, timedelta],
+ pinned: Collection[str] = frozenset(),
) -> TargetBounds:
"""Resolve one node's selectable cadences, marking each blocked option with what blocks it.
@@ -444,8 +494,13 @@ def compute_target_bounds(
API payload, and any UI that offers a choice all read this, so a refusal and a disabled option
can never disagree about the same node.
"""
- floor = all_source_floor_bounds(edges, source_intervals).get(node_id, Bound(value=STREAMING, blocker=None))
- ceiling = all_consumer_ceiling_bounds(edges, declared_targets).get(node_id)
+ pinned_targets = _pinned_targets(pinned, declared_targets)
+ floor = all_source_floor_bounds(edges, source_intervals, pinned_targets).get(
+ node_id, Bound(value=STREAMING, blocker=None)
+ )
+ ceiling = all_consumer_ceiling_bounds(edges, declared_targets, pinned).get(node_id)
+ if node_id in pinned:
+ ceiling = None
return _bounds_from(floor=floor, ceiling=ceiling)
@@ -491,20 +546,38 @@ def _describe(node_id: str | None, names: Mapping[str, str] | None, fallback: st
return name or fallback
-def _nothing_satisfiable_message(bounds: TargetBounds, names: Mapping[str, str] | None) -> str:
[email protected](frozen=True, kw_only=True, slots=True)
+class FloorReason:
+ node: str # the node that sets the floor
+ why: str # why it holds the floor there
+
+
+def _floor_reason(floor: Bound, names: Mapping[str, str] | None, pinned: Collection[str]) -> FloorReason:
+ """The node that sets a floor, and why it holds the floor there: a slow source, or a pinned view."""
+ if floor.blocker is not None and floor.blocker in pinned:
+ view = _describe(floor.blocker, names, "an upstream view")
+ return FloorReason(node=view, why=f"{view} is pinned to refresh every {humanize_cadence(floor.value)}")
+ source = _describe(floor.blocker, names, "an upstream source")
+ return FloorReason(node=source, why=f"{source} only syncs every {humanize_cadence(floor.value)}")
+
+
+def _nothing_satisfiable_message(
+ bounds: TargetBounds, names: Mapping[str, str] | None, pinned: Collection[str] = frozenset()
+) -> str:
"""Explain a node with no legal cadence at all: crossed bounds, or a source slower than 30 days."""
floor, ceiling = bounds.floor, bounds.ceiling
- source = _describe(floor.blocker, names, "an upstream source") if floor else ""
+ floor_reason = _floor_reason(floor, names, pinned) if floor else FloorReason(node="", why="")
+ source = floor_reason.node
consumer = _describe(ceiling.blocker, names, "a view or endpoint built on this one") if ceiling else ""
if floor is not None and ceiling is not None:
return (
- f"No cadence works here: {source} only syncs every {humanize_cadence(floor.value)},"
+ f"No cadence works here: {floor_reason.why},"
f" but {consumer} refreshes every {humanize_cadence(ceiling.value)}."
f" Slow down {consumer} or speed up {source}."
)
if floor is not None:
return (
- f"No cadence works here: {source} only syncs every {humanize_cadence(floor.value)},"
+ f"No cadence works here: {floor_reason.why},"
f" less often than anything this can refresh at. Speed up {source} first."
)
if ceiling is not None:
@@ -523,11 +596,13 @@ def validate_declared_target(
declared_targets: dict[str, timedelta],
source_intervals: dict[str, timedelta],
names: Mapping[str, str] | None = None,
+ pinned: Collection[str] = frozenset(),
) -> None:
"""Raise if `target` is not selectable or falls outside the node's bounds.
`names` maps node ids to display names; supply it to have refusals name the source or view
- that blocks the target, which is the thing a person has to change to unblock it.
+ that blocks the target, which is the thing a person has to change to unblock it. `pinned`
+ holds the pin state the write would leave, including `node_id`'s own.
"""
if target not in SCHEDULABLE_BUCKETS:
supported = ", ".join(format_cadence(interval) for interval in sorted(SCHEDULABLE_BUCKETS))
@@ -536,7 +611,11 @@ def validate_declared_target(
)
bounds = compute_target_bounds(
- node_id=node_id, edges=edges, declared_targets=declared_targets, source_intervals=source_intervals
+ node_id=node_id,
+ edges=edges,
+ declared_targets=declared_targets,
+ source_intervals=source_intervals,
+ pinned=pinned,
)
option = bounds.option_for(target)
if option is None or option.allowed:
@@ -544,16 +623,14 @@ def validate_declared_target(
# Nothing legal to fall back to, so no message here can end in "pick X instead"
if not bounds.satisfiable:
- raise UnsatisfiableFrequencyError(_nothing_satisfiable_message(bounds, names))
+ raise UnsatisfiableFrequencyError(_nothing_satisfiable_message(bounds, names, pinned))
# Suggest a cadence the same call would accept, and name the bound itself rather than a
# direction. Bounds are inclusive, so the bound is always a legal answer, while "or slower" /
# "or faster" promises room that may not exist: a 15min ceiling has nothing fresher to offer.
if option.blocked_by == "source" and bounds.floor is not None:
- source = _describe(option.blocker, names, "an upstream source")
raise UnsatisfiableFrequencyError(
- f"Can't refresh every {humanize_cadence(target)}: {source} only syncs every"
- f" {humanize_cadence(bounds.floor.value)}."
+ f"Can't refresh every {humanize_cadence(target)}: {_floor_reason(bounds.floor, names, pinned).why}."
f" Pick {humanize_cadence(min(bounds.allowed))} instead."
)
if option.blocked_by == "consumer" and bounds.ceiling is not None:
@@ -579,6 +656,7 @@ def find_invalid_targets(
edges: list[tuple[str, str]],
declared_targets: dict[str, timedelta],
source_intervals: dict[str, timedelta],
+ pinned: Collection[str] = frozenset(),
) -> list[InvalidTarget]:
"""Re-validate every declared target against its current bounds.
@@ -586,8 +664,8 @@ def find_invalid_targets(
graph edits move floors. Runtime freshness stays correct (finest demand wins) — what
breaks is declared == effective, so run this on any graph mutation and surface the result.
"""
- floors = all_source_floors(edges, source_intervals)
- ceilings = all_consumer_ceilings(edges, declared_targets)
+ floors = all_source_floors(edges, source_intervals, _pinned_targets(pinned, declared_targets))
+ ceilings = all_consumer_ceilings(edges, declared_targets, pinned)
invalid: list[InvalidTarget] = []
for node_id, declared in declared_targets.items():
source_floor = floors.get(node_id, STREAMING)
@@ -601,3 +679,106 @@ def find_invalid_targets(
)
)
return invalid
+
+
+def _below_source_floor(
+ *,
+ edges: list[tuple[str, str]],
+ declared_targets: Mapping[str, timedelta],
+ source_intervals: dict[str, timedelta],
+ pinned: Collection[str],
+) -> dict[str, InvalidTarget]:
+ return {
+ invalid.node_id: invalid
+ for invalid in find_invalid_targets(
+ edges=edges, declared_targets=dict(declared_targets), source_intervals=source_intervals, pinned=pinned
+ )
+ if is_finer_than(invalid.declared, invalid.source_floor)
+ }
+
+
+def nodes_below_source_floor(
+ *,
+ edges: list[tuple[str, str]],
+ declared_targets: Mapping[str, timedelta],
+ source_intervals: dict[str, timedelta],
+ pinned: Collection[str],
+) -> set[str]:
+ """The nodes whose declared target is finer than their sources can deliver."""
+ return set(
+ _below_source_floor(
+ edges=edges, declared_targets=declared_targets, source_intervals=source_intervals, pinned=pinned
+ )
+ )
+
+
+def descendants_broken_by_pin(
+ *,
+ node_id: str,
+ target: timedelta,
+ edges: list[tuple[str, str]],
+ declared_targets: Mapping[str, timedelta],
+ source_intervals: dict[str, timedelta],
+ pinned: Collection[str],
+ already_below_floor: Collection[str] | None = None,
+) -> list[InvalidTarget]:
+ """The downstream targets that pinning `node_id` to `target` would put out of reach.
+
+ A pin raises its children's source floor. A child with another, fresher parent keeps that
+ parent's floor, but a child that reads fresh data only through this node can no longer meet
+ its own target. Targets that were already out of reach of their source floor without the pin
+ don't count against it. A target that only broke its consumer ceiling still counts, because
+ the node can meet it at runtime and the pin's floor would take that away.
+
+ `already_below_floor` is `nodes_below_source_floor` without this pin. It does not depend on
+ `target`, so a caller checking many targets computes it once and passes it in.
+ """
+ targets = {**declared_targets, node_id: target}
+ others = set(pinned) - {node_id}
+ if already_below_floor is None:
+ already_below_floor = nodes_below_source_floor(
+ edges=edges, declared_targets=targets, source_intervals=source_intervals, pinned=others
+ )
+ descendants = descendants_of(node_id, edges)
+ broken = _below_source_floor(
+ edges=edges, declared_targets=targets, source_intervals=source_intervals, pinned=others | {node_id}
+ )
+ return sorted(
+ (invalid for node, invalid in broken.items() if node in descendants and node not in already_below_floor),
+ key=lambda invalid: invalid.node_id,
+ )
+
+
+def validate_pin(
+ *,
+ node_id: str,
+ target: timedelta,
+ edges: list[tuple[str, str]],
+ declared_targets: Mapping[str, timedelta],
+ source_intervals: dict[str, timedelta],
+ pinned: Collection[str],
+ names: Mapping[str, str] | None = None,
+) -> None:
+ """Raise UnsatisfiableFrequencyError when pinning `node_id` to `target` strands a downstream view.
+
+ The message names the first stranded view the caller may see, and generic prose otherwise.
+ """
+ broken = descendants_broken_by_pin(
+ node_id=node_id,
+ target=target,
+ edges=edges,
+ declared_targets=declared_targets,
+ source_intervals=source_intervals,
+ pinned=pinned,
+ )
+ if not broken:
+ return
+ named = [invalid for invalid in broken if (names or {}).get(invalid.node_id)]
+ stranded = (named or broken)[0]
+ view = _describe(stranded.node_id, names, "")
+ subject, fix = (f"downstream view {view}", view) if view else ("a downstream view", "that view")
+ raise UnsatisfiableFrequencyError(
+ f"Can't pin this view to {humanize_cadence(target)}: {subject} refreshes every "
+ f"{humanize_cadence(stranded.declared)} and has no fresher input than this view. "
+ f"Slow down {fix} first, or pin a faster cadence."
+ )
diff --git a/products/data_modeling/backend/logic/node_frequency.py b/products/data_modeling/backend/logic/node_frequency.py
index 4bcf787c46b3..d7d4e54bb274 100644
--- a/products/data_modeling/backend/logic/node_frequency.py
+++ b/products/data_modeling/backend/logic/node_frequency.py
@@ -9,8 +9,9 @@ typed column.
import uuid
import dataclasses
-from collections.abc import Iterable
+from collections.abc import Collection, Iterable, Mapping
from datetime import timedelta
+from typing import Any
from django.db.models import Q, QuerySet
@@ -18,14 +19,19 @@ from products.data_modeling.backend.facade.contracts import SavedQueryNodeState,
from products.data_modeling.backend.logic.cohort_scheduling import MINUTES_PER_WEEK
from products.data_modeling.backend.logic.freshness import (
STREAMING,
+ ClampedCadence,
TargetBounds,
+ TargetOption,
all_source_floors,
ancestors_of,
clamp_to_source_floor,
compute_effective_cadences,
compute_target_bounds,
+ descendants_broken_by_pin,
+ descendants_of,
intersect_target_bounds,
max_data_age,
+ nodes_below_source_floor,
normalize_seed_target,
)
from products.data_modeling.backend.logic.node_suspension import merged_suspension_state
@@ -39,6 +45,7 @@ _SYSTEM_KEY = "system"
_FREQUENCY_KEY = "frequency"
_TARGET_SECONDS_KEY = "target_seconds"
_ANCHOR_MINUTES_KEY = "anchor_minutes"
+_PINNED_KEY = "pinned"
# `resolve_dependency_to_node` stamps this on source (TABLE) nodes at creation.
_ORIGIN_KEY = "origin"
@@ -79,6 +86,16 @@ def get_declared_target(node: Node) -> timedelta | None:
return timedelta(seconds=seconds) if seconds is not None else None
+def is_declared_pin(properties: Mapping[str, Any] | None) -> bool:
+ """Return whether node properties say it holds its declared target instead of following its consumers."""
+ return bool((properties or {}).get(_SYSTEM_KEY, {}).get(_FREQUENCY_KEY, {}).get(_PINNED_KEY, False))
+
+
+def get_declared_pin(node: Node) -> bool:
+ """Return whether the node holds its declared target instead of following its consumers."""
+ return is_declared_pin(node.properties)
+
+
def declared_targets_by_saved_query(team_id: int, saved_query_ids: Iterable[str | uuid.UUID]) -> dict[str, timedelta]:
"""Declared freshness target per saved query id, for those whose node carries one.
@@ -135,17 +152,35 @@ def node_states_by_saved_query(
def set_declared_target(node: Node, target: timedelta | None) -> None:
- """Set (or clear, with None) the node's declared target without touching sibling system state."""
+ """Set (or clear, with None) the node's declared target without touching sibling system state.
+
+ Clearing the target also clears the pin, because a pin holds the target and has nothing to hold without one.
+ """
properties = node.properties or {}
frequency = properties.setdefault(_SYSTEM_KEY, {}).setdefault(_FREQUENCY_KEY, {})
if target is None:
frequency.pop(_TARGET_SECONDS_KEY, None)
+ frequency.pop(_PINNED_KEY, None)
else:
frequency[_TARGET_SECONDS_KEY] = int(target.total_seconds())
node.properties = properties
node.save(update_fields=["properties"])
+def set_declared_pin(node: Node, pinned: bool) -> None:
+ """Pin (or unpin) the node's declared target without touching sibling system state."""
+ if pinned and get_declared_target(node) is None:
+ raise ValueError("a node needs a declared target before it can be pinned")
+ properties = node.properties or {}
+ frequency = properties.setdefault(_SYSTEM_KEY, {}).setdefault(_FREQUENCY_KEY, {})
+ if pinned:
+ frequency[_PINNED_KEY] = True
+ else:
+ frequency.pop(_PINNED_KEY, None)
+ node.properties = properties
+ node.save(update_fields=["properties"])
+
+
def get_declared_anchor(node: Node) -> int | None:
"""Return the node's declared schedule anchor (minutes past Monday 00:00 UTC), or None."""
return (node.properties or {}).get(_SYSTEM_KEY, {}).get(_FREQUENCY_KEY, {}).get(_ANCHOR_MINUTES_KEY)
@@ -243,7 +278,7 @@ def resolve_source_intervals(source_nodes: list[Node]) -> tuple[dict[str, timede
return source_intervals, best_effort
[email protected](frozen=True, kw_only=True, slots=True)
class FrequencyGraph:
"""The primitives `freshness.py` operates on, extracted from one DAG's Node/Edge rows."""
@@ -251,6 +286,7 @@ class FrequencyGraph:
edges: list[tuple[str, str]] # (upstream_id, downstream_id), includes source tables
declared_targets: dict[str, timedelta] # declared per-node targets
declared_anchors: dict[str, int] # declared per-node schedule anchors (minutes past Monday 00:00 UTC)
+ pinned: set[str] # nodes that hold their declared target instead of following their consumers
source_intervals: dict[str, timedelta] # per source (TABLE) node
best_effort_source_ids: set[str] # sources treated as STREAMING but not guaranteed
names: dict[str, str] # node id -> display name, so a bound can name what set it
@@ -270,10 +306,13 @@ def build_frequency_graph(dag: DAG) -> FrequencyGraph:
schedulable = {str(node.id) for node in nodes if node.type in SAVED_QUERY_NODE_TYPES}
declared_targets: dict[str, timedelta] = {}
declared_anchors: dict[str, int] = {}
+ pinned: set[str] = set()
for node in nodes:
target = get_declared_target(node)
if target is not None:
declared_targets[str(node.id)] = target
+ if get_declared_pin(node):
+ pinned.add(str(node.id))
anchor = get_declared_anchor(node)
if anchor is not None:
declared_anchors[str(node.id)] = anchor
@@ -286,6 +325,7 @@ def build_frequency_graph(dag: DAG) -> FrequencyGraph:
edges=edges,
declared_targets=declared_targets,
declared_anchors=declared_anchors,
+ pinned=pinned,
source_intervals=source_intervals,
best_effort_source_ids=best_effort,
names={str(node.id): node.name for node in nodes},
@@ -303,6 +343,7 @@ class SavedQueryFrequencyBounds:
best_effort_source_ids: set[str] # upstream sources with no schedule, so the floor is a guess
max_data_age: timedelta
speeds_up: dict[timedelta, set[str]] # cadence -> upstream nodes that would run more often if this were set to it
+ pinned: bool
def saved_query_target_bounds(team_id: int, saved_query_id: str | uuid.UUID) -> SavedQueryFrequencyBounds | None:
@@ -337,6 +378,7 @@ def saved_query_target_bounds(team_id: int, saved_query_id: str | uuid.UUID) ->
edges=graph.edges,
declared_targets=graph.declared_targets,
source_intervals=graph.source_intervals,
+ pinned=graph.pinned,
)
)
names.update(graph.names)
@@ -356,6 +398,69 @@ def saved_query_target_bounds(team_id: int, saved_query_id: str | uuid.UUID) ->
best_effort_source_ids=best_effort,
max_data_age=max(data_ages),
speeds_up=speeds_up,
+ pinned=any(get_declared_pin(node) for node in nodes),
+ )
+
+
[email protected](frozen=True, kw_only=True, slots=True)
+class SavedQueryPinPreview:
+ """What pinning a saved query to each cadence would do, for the one endpoint that shows it."""
+
+ # cadence -> downstream nodes that would read data older than their own cadence if this were pinned to it
+ delays_if_pinned: dict[timedelta, set[str]]
+ # cadence -> downstream nodes whose own target a pin at that cadence would put out of reach
+ pin_blocked_by: dict[timedelta, set[str]]
+ # cadence -> the cadence this view would run at if it held that target without a pin
+ unpinned_cadence: dict[timedelta, timedelta]
+
+
+def saved_query_pin_preview(team_id: int, saved_query_id: str | uuid.UUID) -> SavedQueryPinPreview:
+ """The pin preview for every cadence the saved query's nodes offer, folded across their DAGs.
+
+ Kept apart from `saved_query_target_bounds` because it costs a full-graph pass per cadence and
+ only the node frequency endpoint reads it.
+ """
+ delays_if_pinned: dict[timedelta, set[str]] = {}
+ pin_blocked_by: dict[timedelta, set[str]] = {}
+ unpinned_cadence: dict[timedelta, timedelta] = {}
+ graphs: dict[str, FrequencyGraph] = {}
+ for node in Node.objects.filter(team_id=team_id, saved_query_id=saved_query_id).select_related("dag"):
+ dag_key = str(node.dag_id)
+ if dag_key not in graphs:
+ graphs[dag_key] = build_frequency_graph(node.dag)
+ graph = graphs[dag_key]
+ node_id = str(node.id)
+ options = compute_target_bounds(
+ node_id=node_id,
+ edges=graph.edges,
+ declared_targets=graph.declared_targets,
+ source_intervals=graph.source_intervals,
+ pinned=graph.pinned,
+ ).options
+ for cadence, delayed in _delays_if_pinned_by_cadence(node_id, graph, options).items():
+ delays_if_pinned.setdefault(cadence, set()).update(delayed)
+ below_floor = nodes_below_source_floor(
+ edges=graph.edges,
+ declared_targets=graph.declared_targets,
+ source_intervals=graph.source_intervals,
+ pinned=graph.pinned - {node_id},
+ )
+ for option in options:
+ stranded = descendants_broken_by_pin(
+ node_id=node_id,
+ target=option.value,
+ edges=graph.edges,
+ declared_targets=graph.declared_targets,
+ source_intervals=graph.source_intervals,
+ pinned=graph.pinned,
+ already_below_floor=below_floor,
+ )
+ pin_blocked_by.setdefault(option.value, set()).update(invalid.node_id for invalid in stranded)
+ for cadence, runs_at in _unpinned_cadence_by_cadence(node_id, graph, options).items():
+ # A saved query with nodes in several DAGs runs at the finest cadence any of them asks for
+ unpinned_cadence[cadence] = min(runs_at, unpinned_cadence.get(cadence, runs_at))
+ return SavedQueryPinPreview(
+ delays_if_pinned=delays_if_pinned, pin_blocked_by=pin_blocked_by, unpinned_cadence=unpinned_cadence
)
@@ -370,8 +475,7 @@ def _speeds_up_by_cadence(
"""
def run(declared_targets: dict[str, timedelta]) -> dict[str, timedelta | None]:
- effective = compute_effective_cadences(nodes=graph.nodes, edges=graph.edges, declared_targets=declared_targets)
- return clamp_to_source_floor(effective, edges=graph.edges, source_intervals=graph.source_intervals)[0]
+ return scheduled_cadences(graph, declared_targets=declared_targets)[0]
upstream = ancestors & graph.nodes
ephemeral = {
@@ -397,6 +501,63 @@ def _speeds_up_by_cadence(
return result
+def _delays_if_pinned_by_cadence(
+ node_id: str, graph: FrequencyGraph, options: Iterable[TargetOption]
+) -> dict[timedelta, set[str]]:
+ """The downstream views that would read data older than their own cadence if `node_id` were pinned to each option.
+
+ A descendant's effective cadence comes from its own target and its own consumers, never from
+ `node_id`, so one pass over the current graph answers every option.
+ """
+ effective = compute_effective_cadences(
+ nodes=graph.nodes, edges=graph.edges, declared_targets=graph.declared_targets, pinned=graph.pinned
+ )
+ descendants = descendants_of(node_id, graph.edges) & graph.nodes
+ return {
+ option.value: {
+ descendant
+ for descendant in descendants
+ if (cadence := effective.get(descendant)) is not None and cadence < option.value
+ }
+ for option in options
+ }
+
+
+def _unpinned_cadence_by_cadence(
+ node_id: str, graph: FrequencyGraph, options: Iterable[TargetOption]
+) -> dict[timedelta, timedelta]:
+ """The cadence `node_id` would run at for each option if it held that target without a pin.
+
+ Computed with only this node's pin removed, so a pin further downstream still stops the demand
+ of the views below it, and then clamped to the source floor like the scheduler does.
+ """
+ others = graph.pinned - {node_id}
+ result: dict[timedelta, timedelta] = {}
+ for option in options:
+ runs_at = scheduled_cadences(
+ graph, declared_targets={**graph.declared_targets, node_id: option.value}, pinned=others
+ )[0].get(node_id)
+ result[option.value] = runs_at or option.value
+ return result
+
+
+def scheduled_cadences(
+ graph: FrequencyGraph,
+ *,
+ declared_targets: Mapping[str, timedelta] | None = None,
+ pinned: Collection[str] | None = None,
+) -> tuple[dict[str, timedelta | None], list[ClampedCadence]]:
+ """The cadence the scheduler runs each node at: effective cadences, then the source-floor clamp.
+
+ The one place this pair is spelled out, so the scheduler, the tier reports and the pickers can't
+ disagree about pins. `declared_targets` and `pinned` override the graph's own for what-if reads.
+ """
+ declared = dict(graph.declared_targets if declared_targets is None else declared_targets)
+ pins = graph.pinned if pinned is None else pinned
+ effective = compute_effective_cadences(nodes=graph.nodes, edges=graph.edges, declared_targets=declared, pinned=pins)
+ return clamp_to_source_floor(effective, edges=graph.edges, source_intervals=graph.source_intervals, pinned=pins)
+
+
def schedulable_nodes(dag: DAG) -> QuerySet[Node]:
"""The DAG's schedulable nodes: everything that carries a live saved query (not a source
table, not a soft-deleted query). The one definition of "what gets a freshness target"."""
diff --git a/products/data_modeling/backend/logic/schedule_reconcile.py b/products/data_modeling/backend/logic/schedule_reconcile.py
index ce33baf2031f..f8f2bdafc7f2 100644
--- a/products/data_modeling/backend/logic/schedule_reconcile.py
+++ b/products/data_modeling/backend/logic/schedule_reconcile.py
@@ -52,18 +52,21 @@ from products.data_modeling.backend.logic.freshness import (
ClampedCadence,
InvalidTarget,
UnsupportedFrequencyTargetError,
- clamp_to_source_floor,
- compute_effective_cadences,
find_invalid_targets,
format_cadence,
validate_declared_target,
+ validate_pin,
)
from products.data_modeling.backend.logic.node_frequency import (
FrequencyGraph,
build_frequency_graph,
+ get_declared_pin,
+ get_declared_target,
persist_seed_targets,
+ scheduled_cadences,
seed_targets,
set_declared_anchor,
+ set_declared_pin,
set_declared_target,
)
from products.data_modeling.backend.models.dag import DAG
@@ -133,7 +136,10 @@ def _warn_on_invalid_targets(dag: DAG, graph: FrequencyGraph | None = None) -> N
if graph is None:
graph = build_frequency_graph(dag)
for invalid in find_invalid_targets(
- edges=graph.edges, declared_targets=graph.declared_targets, source_intervals=graph.source_intervals
+ edges=graph.edges,
+ declared_targets=graph.declared_targets,
+ source_intervals=graph.source_intervals,
+ pinned=graph.pinned,
):
logger.warning(
"Declared freshness target outside its legal range",
@@ -151,6 +157,8 @@ def apply_saved_query_frequency_target(
*,
reconcile: bool = True,
visible_names: Mapping[str, str] | None = None,
+ pinned: bool | None = None,
+ keep_target: bool = False,
) -> int:
"""Write a frequency target through to the DAG node(s) carrying this saved query.
@@ -169,6 +177,13 @@ def apply_saved_query_frequency_target(
of a node they may not read. Callers holding a per-user map (the saved-query surfaces) pass it
and get the better message; everything left out falls back to generic prose.
+ `pinned` sets or clears the pin alongside the target; None leaves the stored pin as it is. A pin
+ lets the target sit slower than the node's consumers. `keep_target` ignores `target` and leaves
+ each node's stored target as it is, read after the row lock so a pin written on its own never
+ overwrites a cadence that committed in the meantime. Unpinning without changing the target is
+ not validated, because the target was legal when it was pinned and the scheduler only uses it
+ as a floor for consumer demand once unpinned.
+
Atomic because a saved query can still carry nodes in several DAGs while duplicates are being
consolidated away: without it, a target rejected by the third node stays written on the first
two, and their reconciles are already queued. One node per saved query is the end state, which
@@ -176,6 +191,12 @@ def apply_saved_query_frequency_target(
"""
written = 0
with transaction.atomic():
+ # Validating a pin reads other nodes' targets, so two writes on different nodes of one DAG
+ # must not validate against each other's stale state: each could pass and the pair strand a
+ # consumer. Taking the DAG row first, in id order, makes them take turns.
+ dag_ids = Node.objects.filter(team=saved_query.team, saved_query=saved_query).values_list("dag_id", flat=True)
+ locked_dags = DAG.objects.select_for_update(no_key=True).filter(team_id=saved_query.team_id, pk__in=dag_ids)
+ list(locked_dags.order_by("pk").values_list("pk", flat=True))
# target and anchor share one JSON field and each setter rewrites the whole blob, so an
# unlocked read-modify-write racing the anchor path would silently drop the other's key
for node in (
@@ -183,19 +204,38 @@ def apply_saved_query_frequency_target(
.filter(team=saved_query.team, saved_query=saved_query)
.select_related("dag", "dag__team")
):
- if target is None:
+ node_target = get_declared_target(node) if keep_target else target
+ if node_target is None:
+ if keep_target and pinned:
+ raise ValueError("a node needs a declared target before it can be pinned")
set_declared_target(node, None)
else:
- graph = build_frequency_graph(node.dag)
- validate_declared_target(
- node_id=str(node.id),
- target=target,
- edges=graph.edges,
- declared_targets=graph.declared_targets,
- source_intervals=graph.source_intervals,
- names=visible_names or {},
- )
- set_declared_target(node, target)
+ node_id = str(node.id)
+ pin = get_declared_pin(node) if pinned is None else pinned
+ if pinned is not False or node_target != get_declared_target(node):
+ graph = build_frequency_graph(node.dag)
+ validate_declared_target(
+ node_id=node_id,
+ target=node_target,
+ edges=graph.edges,
+ declared_targets=graph.declared_targets,
+ source_intervals=graph.source_intervals,
+ names=visible_names or {},
+ pinned=(graph.pinned - {node_id}) | ({node_id} if pin else set()),
+ )
+ if pin:
+ validate_pin(
+ node_id=node_id,
+ target=node_target,
+ edges=graph.edges,
+ declared_targets=graph.declared_targets,
+ source_intervals=graph.source_intervals,
+ pinned=graph.pinned,
+ names=visible_names or {},
+ )
+ set_declared_target(node, node_target)
+ if pinned is not None:
+ set_declared_pin(node, pinned)
written += 1
if reconcile:
maybe_reconcile_dag(node.dag)
@@ -220,6 +260,7 @@ def check_saved_query_frequency_target(
declared_targets=graph.declared_targets,
source_intervals=graph.source_intervals,
names=visible_names or {},
+ pinned=graph.pinned,
)
@@ -259,10 +300,7 @@ def reconcile_dag_schedules(dag: DAG, *, graph: FrequencyGraph | None = None) ->
team = dag.team
if graph is None:
graph = build_frequency_graph(dag)
- effective = compute_effective_cadences(
- nodes=graph.nodes, edges=graph.edges, declared_targets=graph.declared_targets
- )
- effective, _clamped = clamp_to_source_floor(effective, edges=graph.edges, source_intervals=graph.source_intervals)
+ effective, _clamped = scheduled_cadences(graph)
desired_tiers = bucket_into_cadence_tiers(effective, graph.declared_anchors)
_apply_reconciliation(
dag_id=str(dag.id),
@@ -314,8 +352,7 @@ def preview_dag_schedules(dag: DAG, *, seed: bool = False) -> DagSchedulePreview
"""
graph = build_frequency_graph(dag)
declared = {**seed_targets(dag), **graph.declared_targets} if seed else graph.declared_targets
- effective = compute_effective_cadences(nodes=graph.nodes, edges=graph.edges, declared_targets=declared)
- effective, clamped = clamp_to_source_floor(effective, edges=graph.edges, source_intervals=graph.source_intervals)
+ effective, clamped = scheduled_cadences(graph, declared_targets=declared)
desired_tiers = bucket_into_cadence_tiers(effective, graph.declared_anchors)
existing_ids = list_existing_schedule_ids(str(dag.id))
plan = plan_schedule_reconciliation(str(dag.id), desired_tiers, existing_ids)
@@ -326,7 +363,7 @@ def preview_dag_schedules(dag: DAG, *, seed: bool = False) -> DagSchedulePreview
best_effort_source_ids=graph.best_effort_source_ids,
clamped=clamped,
invalid_targets=find_invalid_targets(
- edges=graph.edges, declared_targets=declared, source_intervals=graph.source_intervals
+ edges=graph.edges, declared_targets=declared, source_intervals=graph.source_intervals, pinned=graph.pinned
),
unsupported_tiers=sorted({tier.interval for tier in desired_tiers if tier.interval not in SCHEDULABLE_BUCKETS}),
seeded=seed,
diff --git a/products/data_modeling/backend/logic/tier_membership.py b/products/data_modeling/backend/logic/tier_membership.py
index 1a16c924b21d..f34051f47b59 100644
--- a/products/data_modeling/backend/logic/tier_membership.py
+++ b/products/data_modeling/backend/logic/tier_membership.py
@@ -24,12 +24,8 @@ from products.data_modeling.backend.logic.cohort_scheduling import (
bucket_into_cadence_tiers,
interval_seconds_from_schedule_id,
)
-from products.data_modeling.backend.logic.freshness import (
- SCHEDULABLE_BUCKETS,
- clamp_to_source_floor,
- compute_effective_cadences,
-)
-from products.data_modeling.backend.logic.node_frequency import build_frequency_graph
+from products.data_modeling.backend.logic.freshness import SCHEDULABLE_BUCKETS
+from products.data_modeling.backend.logic.node_frequency import build_frequency_graph, scheduled_cadences
from products.data_modeling.backend.models.dag import DAG
from products.data_modeling.backend.models.node import NodeType
from products.data_modeling.backend.schedule import DATA_MODELING_EXECUTE_DAG_WORKFLOW
@@ -124,10 +120,7 @@ async def read_live_tiers(temporal: Client, dag_id: str) -> list[LiveTier]:
def _desired_tiers(dag: DAG) -> dict[Tier, set[str]]:
"""The tiers reconcile would want for this DAG: effective cadences → clamp to floor → buckets."""
graph = build_frequency_graph(dag)
- effective = compute_effective_cadences(
- nodes=graph.nodes, edges=graph.edges, declared_targets=graph.declared_targets
- )
- effective, _clamped = clamp_to_source_floor(effective, edges=graph.edges, source_intervals=graph.source_intervals)
+ effective, _clamped = scheduled_cadences(graph)
return bucket_into_cadence_tiers(effective, graph.declared_anchors)
diff --git a/products/data_modeling/backend/logic/tier_run_report.py b/products/data_modeling/backend/logic/tier_run_report.py
index 47d4ba6e5b41..f40c04d467c5 100644
--- a/products/data_modeling/backend/logic/tier_run_report.py
+++ b/products/data_modeling/backend/logic/tier_run_report.py
@@ -27,8 +27,7 @@ from products.data_modeling.backend.logic.cohort_scheduling import (
tier_schedule_id,
tier_sort_key,
)
-from products.data_modeling.backend.logic.freshness import clamp_to_source_floor, compute_effective_cadences
-from products.data_modeling.backend.logic.node_frequency import build_frequency_graph
+from products.data_modeling.backend.logic.node_frequency import build_frequency_graph, scheduled_cadences
from products.data_modeling.backend.models.dag import DAG
from products.data_modeling.backend.models.data_modeling_job import (
DataModelingJob,
@@ -150,10 +149,7 @@ def untargeted_nodes(dag: DAG) -> list[Node]:
def _desired_tiers(dag: DAG) -> dict[Tier, set[str]]:
"""The cohorts the reconciler would build, mirroring its propagation + clamp + bucketing."""
graph = build_frequency_graph(dag)
- effective = compute_effective_cadences(
- nodes=graph.nodes, edges=graph.edges, declared_targets=graph.declared_targets
- )
- effective, _clamped = clamp_to_source_floor(effective, edges=graph.edges, source_intervals=graph.source_intervals)
+ effective, _clamped = scheduled_cadences(graph)
return bucket_into_cadence_tiers(effective, graph.declared_anchors)
diff --git a/products/data_modeling/backend/management/commands/set_schedule_anchor.py b/products/data_modeling/backend/management/commands/set_schedule_anchor.py
index 5bc325180fe8..a300441ca303 100644
--- a/products/data_modeling/backend/management/commands/set_schedule_anchor.py
+++ b/products/data_modeling/backend/management/commands/set_schedule_anchor.py
@@ -15,10 +15,10 @@ from products.data_modeling.backend.logic.cohort_scheduling import (
format_tier,
tier_sort_key,
)
-from products.data_modeling.backend.logic.freshness import clamp_to_source_floor, compute_effective_cadences
from products.data_modeling.backend.logic.node_frequency import (
build_frequency_graph,
schedulable_nodes,
+ scheduled_cadences,
set_declared_anchor,
)
from products.data_modeling.backend.logic.schedule_reconcile import reconcile_dag_schedules
@@ -91,12 +91,7 @@ class Command(BaseCommand):
plans: list[tuple[DAG, set[str], str]] = []
for dag, node_ids in sorted(nodes_by_dag.items(), key=lambda kv: str(kv[0].id)):
graph = build_frequency_graph(dag)
- effective = compute_effective_cadences(
- nodes=graph.nodes, edges=graph.edges, declared_targets=graph.declared_targets
- )
- effective, _clamped = clamp_to_source_floor(
- effective, edges=graph.edges, source_intervals=graph.source_intervals
- )
+ effective, _clamped = scheduled_cadences(graph)
if options["with_upstream"]:
node_ids = self._expand_with_upstream(graph.edges, node_ids, graph.nodes)
diff --git a/products/data_modeling/backend/presentation/views/metric_visibility.py b/products/data_modeling/backend/presentation/views/metric_visibility.py
index 7cd00a8d8988..03fd18fb668a 100644
--- a/products/data_modeling/backend/presentation/views/metric_visibility.py
+++ b/products/data_modeling/backend/presentation/views/metric_visibility.py
@@ -1,23 +1,32 @@
from typing import TYPE_CHECKING
+from uuid import UUID
from django.db.models import Q, QuerySet
from rest_framework.exceptions import PermissionDenied
from products.data_modeling.backend.facade.flags import is_managed_viewset_enabled
-from products.data_modeling.backend.facade.models import NodeType
+from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery, NodeType
+from products.warehouse_sources.backend.facade.api import allowed_table_ids
if TYPE_CHECKING:
from posthog.models.team import Team
from products.access_control.backend.facade.user_access_control import AccessControlLevel, UserAccessControl
- from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
+ from products.data_modeling.backend.facade.api import SavedQueryFrequencyBounds
DELETE_DENIED = "Deleting data models requires data warehouse write access."
EDIT_DENIED = "Editing data models requires data warehouse write access."
VIEW_EDIT_DENIED = "You need edit access to this view."
+def _as_uuid(value: str) -> UUID | None:
+ try:
+ return UUID(value)
+ except ValueError:
+ return None
+
+
class MetricNodeVisibilityMixin:
user_access_control: "UserAccessControl"
team: "Team"
@@ -37,6 +46,46 @@ class MetricNodeVisibilityMixin:
):
raise PermissionDenied(VIEW_EDIT_DENIED)
+ def _readable_node_names(self, resolved: "SavedQueryFrequencyBounds") -> dict[str, str]:
+ """The names in a lineage this caller may read, keyed by node id.
+
+ Fails closed and withholds the node id along with the name, as the saved-query surfaces do: a
+ view under a `none` grant must not be named, or counted, by the cadence of a view beside it. A
+ node that resolves to no resource (one that predates the origin stamp) is withheld too.
+ """
+ team_id = self.team.pk
+ visible: dict[str, str] = {}
+ node_ids_by_saved_query: dict[str, list[str]] = {}
+ node_ids_by_table: dict[str, list[str]] = {}
+ for node_id, identity in resolved.identities.items():
+ if node_id not in resolved.names:
+ continue
+ if identity.is_posthog_table:
+ visible[node_id] = resolved.names[node_id] # events, persons and friends: readable by anyone
+ elif identity.saved_query_id is not None:
+ node_ids_by_saved_query.setdefault(identity.saved_query_id, []).append(node_id)
+ elif identity.warehouse_table_id is not None:
+ node_ids_by_table.setdefault(identity.warehouse_table_id, []).append(node_id)
+
+ def reveal(node_ids: list[str]) -> None:
+ visible.update({node_id: resolved.names[node_id] for node_id in node_ids})
+
+ if node_ids_by_saved_query:
+ creators = DataWarehouseSavedQuery.objects.filter(
+ id__in=node_ids_by_saved_query, team_id=team_id
+ ).values_list("id", "created_by_id")
+ levels = self.user_access_control.bulk_object_access_levels(
+ "warehouse_view", [(str(pk), created_by) for pk, created_by in creators]
+ )
+ for saved_query_id, level in levels.items():
+ if level is not None and level != "none":
+ reveal(node_ids_by_saved_query[saved_query_id])
+
+ table_ids = [table_id for raw in node_ids_by_table if (table_id := _as_uuid(raw)) is not None]
+ for table_id in allowed_table_ids(team_id, self.user_access_control, ids=table_ids):
+ reveal(node_ids_by_table[str(table_id)])
+ return visible
+
def _metric_nodes_hidden(self) -> bool:
return not self.user_access_control.check_access_level_for_resource("data_catalog", required_level="viewer")
diff --git a/products/data_modeling/backend/presentation/views/node.py b/products/data_modeling/backend/presentation/views/node.py
index 62b3e95f3cf7..6d9c1c350a62 100644
--- a/products/data_modeling/backend/presentation/views/node.py
+++ b/products/data_modeling/backend/presentation/views/node.py
@@ -1,7 +1,8 @@
+from datetime import timedelta
from typing import Any, cast
from uuid import UUID
-from django.db import models
+from django.db import models, transaction
from django.db.models import Exists, OuterRef, Subquery
from drf_spectacular.types import OpenApiTypes
@@ -12,16 +13,28 @@ from rest_framework.exceptions import NotFound, PermissionDenied
from posthog.api.pagination import BoundedPageNumberPagination
from posthog.api.routing import TeamAndOrgViewSetMixin
+from posthog.helpers.impersonation import is_impersonated
from posthog.models import User
+from posthog.models.activity_logging.activity_log import Detail, log_activity
from posthog.rbac.query_access import assert_user_can_read_query
from products.data_modeling.backend.facade.api import (
+ SCHEDULABLE_BUCKETS,
MaterializationForbiddenError,
MaterializationRunDirection,
+ SavedQueryFrequencyBounds,
+ UnsatisfiableFrequencyError,
+ UnsupportedFrequencyTargetError,
+ apply_saved_query_frequency_target,
+ cadence_changes,
+ clear_legacy_sync_interval,
delete_node,
endpoint_link,
get_declared_target,
+ is_declared_pin,
run_related_node_materializations,
+ saved_query_pin_preview,
+ saved_query_target_bounds,
suspension_state,
unsuspend_nodes,
)
@@ -29,6 +42,7 @@ from products.data_modeling.backend.facade.models import (
SAVED_QUERY_NODE_TYPES,
DataModelingJob,
DataModelingJobEngine,
+ DataWarehouseSavedQuery,
Edge,
LineageIssueKind,
Node,
@@ -40,7 +54,16 @@ from products.data_modeling.backend.presentation.views.metric_visibility import
EDIT_DENIED,
MetricNodeVisibilityMixin,
)
-from products.warehouse_sources.backend.facade.models import sync_frequency_interval_to_sync_frequency
+from products.warehouse_sources.backend.facade.models import (
+ sync_frequency_interval_to_sync_frequency,
+ sync_frequency_to_sync_frequency_interval,
+)
+
+NEVER = "never"
+CADENCE_CHOICES = [NEVER] + [
+ sync_frequency_interval_to_sync_frequency(bucket) for bucket in sorted(SCHEDULABLE_BUCKETS)
+]
+PIN_NEEDS_A_CADENCE = "Pick how often this view refreshes before pinning it."
class NodeSuspensionSerializer(serializers.Serializer):
@@ -95,7 +118,72 @@ class LineageIssueSerializer(serializers.Serializer):
at = serializers.DateTimeField(allow_null=True, help_text="When the issue was recorded.")
+class DeclaredPinField(serializers.BooleanField):
+ def get_attribute(self, instance: Node) -> bool:
+ return is_declared_pin(instance.properties)
+
+
+class NodePinDelaySerializer(serializers.Serializer):
+ id = serializers.CharField(help_text="Node id of the downstream view or endpoint.")
+ name = serializers.CharField(help_text="Name of the downstream view or endpoint.")
+
+
+class NodePinBlockerSerializer(serializers.Serializer):
+ id = serializers.CharField(
+ allow_null=True,
+ help_text="Node id of the downstream view that stops the pin, or null when the caller may not read it.",
+ )
+ name = serializers.CharField(
+ allow_null=True,
+ help_text="Name of the downstream view that stops the pin, or null when the caller may not read it.",
+ )
+
+
+class NodeFrequencyOptionSerializer(serializers.Serializer):
+ cadence = serializers.ChoiceField(choices=CADENCE_CHOICES[1:], help_text="A cadence the view could refresh at.")
+ delays_if_pinned = NodePinDelaySerializer(
+ many=True,
+ help_text="Downstream views and endpoints that refresh more often than this cadence. If this view "
+ "were pinned to it, they would read data older than their own cadence.",
+ )
+ pin_blocked_by = NodePinBlockerSerializer(
+ allow_null=True,
+ help_text="The downstream view that stops a pin at this cadence. It reads fresh data only through "
+ "this view, so a pin this slow would leave it unable to refresh as often as its own cadence. Null "
+ "when this cadence can be pinned.",
+ )
+ unpinned_cadence = serializers.ChoiceField(
+ choices=CADENCE_CHOICES[1:],
+ help_text="The cadence the view would run at if it held this cadence without a pin. Finer than this "
+ "cadence when a downstream view needs fresher data than the pin allows.",
+ )
+
+
+class NodeFrequencyBoundsSerializer(serializers.Serializer):
+ pinned = serializers.BooleanField(
+ help_text="True when the view refreshes at its own cadence instead of as often as its downstream "
+ "views and endpoints need."
+ )
+ options = NodeFrequencyOptionSerializer(
+ many=True, help_text="Every cadence a picker may show, finest first. Empty when the node has no view."
+ )
+
+
class NodeSerializer(serializers.ModelSerializer):
+ # nosemgrep: choices-need-a-class -- the values are SCHEDULABLE_BUCKETS, and the enum is already named by ENUM_NAME_OVERRIDES
+ sync_frequency = serializers.ChoiceField(
+ choices=CADENCE_CHOICES,
+ write_only=True,
+ required=False,
+ help_text="How often this view refreshes. 'never' clears the cadence, and with it the pin. "
+ "Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.",
+ )
+ pinned = DeclaredPinField(
+ required=False,
+ help_text="Pin this view to its own cadence. A pinned view does not refresh more often when downstream "
+ "views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a "
+ "cadence. Set false to refresh as often as downstream views need again.",
+ )
suspended = serializers.SerializerMethodField(read_only=True)
endpoint = serializers.SerializerMethodField(read_only=True)
upstream_count = serializers.SerializerMethodField(read_only=True)
@@ -130,6 +218,8 @@ class NodeSerializer(serializers.ModelSerializer):
"last_run_error",
"user_tag",
"sync_interval",
+ "sync_frequency",
+ "pinned",
"suspended",
"endpoint",
]
@@ -252,6 +342,8 @@ class NodeSerializer(serializers.ModelSerializer):
raise serializers.ValidationError("Nodes of views PostHog manages cannot be modified.")
if self.instance is not None and self.instance.type == NodeType.METRIC:
raise serializers.ValidationError("Metric nodes are maintained by the data catalog.")
+ if self.writes_cadence(attrs) and (self.instance is None or self.instance.saved_query_id is None):
+ raise serializers.ValidationError("Only views and endpoints have a cadence to set.")
node_type = attrs.get("type")
if node_type is not None and self.instance is not None:
# A full PUT round-trips the node's own type, so a type equal to the current one is
@@ -260,6 +352,82 @@ class NodeSerializer(serializers.ModelSerializer):
raise serializers.ValidationError("A node's type cannot be changed through the API.")
return attrs
+ @staticmethod
+ def writes_cadence(attrs: dict[str, Any]) -> bool:
+ return "sync_frequency" in attrs or "pinned" in attrs
+
+ def update(self, instance: Node, validated_data: dict[str, Any]) -> Node:
+ if not self.writes_cadence(validated_data):
+ return super().update(instance, validated_data)
+ visible_names: dict[str, str] = validated_data.pop("visible_names", {})
+ frequency = validated_data.pop("sync_frequency", None)
+ pinned: bool | None = validated_data.pop("pinned", None)
+ with transaction.atomic():
+ _write_cadence(self.context["request"], instance, frequency, pinned, visible_names)
+ return super().update(instance, validated_data)
+
+
+def _write_cadence(
+ request: request.Request, node: Node, frequency: str | None, pinned: bool | None, visible_names: dict[str, str]
+) -> None:
+ """Write the cadence and pin to every node of the node's view, and record it on the view's activity log."""
+ saved_query = node.saved_query
+ assert saved_query is not None
+ previous_target, previous_pinned = get_declared_target(node), is_declared_pin(node.properties)
+ keep_target = frequency is None
+ if frequency is None:
+ # A pin on its own keeps the stored cadence, which the write reads again under the row lock.
+ target = previous_target
+ else:
+ target = None if frequency == NEVER else sync_frequency_to_sync_frequency_interval(frequency)
+ if pinned and target is None:
+ raise serializers.ValidationError(PIN_NEEDS_A_CADENCE)
+ if target is not None:
+ # Scheduling a view is the same grant as materializing it directly.
+ assert_user_can_read_query(saved_query.query, saved_query.team_id, cast(User, request.user))
+ try:
+ apply_saved_query_frequency_target(
+ saved_query, target, visible_names=visible_names, pinned=pinned, keep_target=keep_target
+ )
+ except (UnsatisfiableFrequencyError, UnsupportedFrequencyTargetError) as e:
+ raise serializers.ValidationError(str(e))
+ except ValueError:
+ if not keep_target:
+ raise
+ # The cadence was cleared between the read above and the write, so there is nothing to pin.
+ raise serializers.ValidationError(PIN_NEEDS_A_CADENCE)
+ if frequency is not None:
+ clear_legacy_sync_interval(saved_query.team_id, saved_query.pk)
+ saved_query.sync_frequency_interval = None
+ node.refresh_from_db(fields=["properties"])
+ _log_cadence_change(
+ request,
+ saved_query,
+ (previous_target, get_declared_target(node)),
+ (previous_pinned, is_declared_pin(node.properties)),
+ )
+
+
+def _log_cadence_change(
+ request: request.Request,
+ saved_query: DataWarehouseSavedQuery,
+ targets: tuple[timedelta | None, timedelta | None],
+ pins: tuple[bool, bool],
+) -> None:
+ changes = cadence_changes(targets, pins)
+ if not changes:
+ return
+ log_activity(
+ organization_id=saved_query.team.organization_id,
+ team_id=saved_query.team_id,
+ user=cast(User, request.user),
+ was_impersonated=is_impersonated(request),
+ item_id=saved_query.id,
+ scope="DataWarehouseSavedQuery",
+ activity="updated",
+ detail=Detail(name=saved_query.name, changes=changes),
+ )
+
class NodePagination(BoundedPageNumberPagination):
page_size = 1000
@@ -372,8 +540,63 @@ class NodeViewSet(
def perform_update(self, serializer: serializers.BaseSerializer) -> None:
self._require_warehouse_access(level="editor", message=EDIT_DENIED)
- self._require_view_edit_access(cast(Node, serializer.instance).saved_query)
- serializer.save()
+ node = cast(Node, serializer.instance)
+ self._require_view_edit_access(node.saved_query)
+ if not NodeSerializer.writes_cadence(serializer.validated_data):
+ serializer.save()
+ return
+ # A refusal names what blocks the cadence, so it names only nodes this caller can list.
+ serializer.save(visible_names=self._visible_names_around(node))
+
+ def _visible_names_around(self, node: Node, resolved: SavedQueryFrequencyBounds | None = None) -> dict[str, str]:
+ """Names of the nodes around this view's cadence that the caller may list and read."""
+ if resolved is None and node.saved_query_id:
+ resolved = saved_query_target_bounds(self.team_id, node.saved_query_id)
+ if resolved is None:
+ return {}
+ dag_ids = Node.objects.filter(team_id=self.team_id, saved_query_id=node.saved_query_id).values("dag_id")
+ listed = self._exclude_hidden_nodes(Node.objects.filter(team_id=self.team_id, dag_id__in=dag_ids))
+ listed_ids = {str(node_id) for node_id in listed.values_list("id", flat=True)}
+ return {node_id: name for node_id, name in self._readable_node_names(resolved).items() if node_id in listed_ids}
+
+ @extend_schema(responses=NodeFrequencyBoundsSerializer)
+ @action(methods=["GET"], detail=True)
+ def frequency_bounds(self, request: request.Request, *args: Any, **kwargs: Any) -> response.Response:
+ """The cadences this view could refresh at, and what pinning it to each one would delay downstream."""
+ self._require_warehouse_access(level="viewer", message=_READ_DENIED)
+ node = self.get_object()
+ resolved = saved_query_target_bounds(self.team_id, node.saved_query_id) if node.saved_query_id else None
+ if resolved is None:
+ return response.Response({"pinned": False, "options": []})
+ names = self._visible_names_around(node, resolved)
+ assert node.saved_query_id is not None
+ preview = saved_query_pin_preview(self.team_id, node.saved_query_id)
+
+ def pin_blocker(cadence: timedelta) -> dict[str, str | None] | None:
+ stranded = sorted(preview.pin_blocked_by.get(cadence, ()))
+ if not stranded:
+ return None
+ # Name a view the caller can see when there is one. Otherwise the pin is still refused, but
+ # neither the id nor the name of a view the caller may not read goes out with the refusal.
+ node_id = next((node_id for node_id in stranded if node_id in names), None)
+ return {"id": node_id, "name": names[node_id]} if node_id else {"id": None, "name": None}
+
+ options = [
+ {
+ "cadence": sync_frequency_interval_to_sync_frequency(option.value),
+ "delays_if_pinned": [
+ {"id": node_id, "name": names[node_id]}
+ for node_id in sorted(preview.delays_if_pinned.get(option.value, ()))
+ if node_id in names
+ ],
+ "pin_blocked_by": pin_blocker(option.value),
+ "unpinned_cadence": sync_frequency_interval_to_sync_frequency(
+ preview.unpinned_cadence.get(option.value, option.value)
+ ),
+ }
+ for option in resolved.bounds.options
+ ]
+ return response.Response(NodeFrequencyBoundsSerializer({"pinned": resolved.pinned, "options": options}).data)
def perform_destroy(self, instance: Node) -> None:
self._require_warehouse_access(level="editor", message=DELETE_DENIED)
diff --git a/products/data_modeling/backend/test/test_freshness.py b/products/data_modeling/backend/test/test_freshness.py
index 621ace74663c..4cd83f049b3e 100644
--- a/products/data_modeling/backend/test/test_freshness.py
+++ b/products/data_modeling/backend/test/test_freshness.py
@@ -15,11 +15,13 @@ from products.data_modeling.backend.logic.freshness import (
clamp_to_source_floor,
compute_effective_cadences,
compute_target_bounds,
+ descendants_broken_by_pin,
find_invalid_targets,
intersect_target_bounds,
max_data_age,
normalize_seed_target,
validate_declared_target,
+ validate_pin,
)
M5 = timedelta(minutes=5)
@@ -71,10 +73,30 @@ class TestComputeEffectiveCadences(TestCase):
{"a": H1},
{"a": H1, "b": None},
),
+ # a pinned node keeps its own target, and the node above it sees that target, not c's 15min
+ (
+ "pinned_middle_holds_and_cuts_demand",
+ {"a", "b", "c"},
+ [("src", "a"), ("a", "b"), ("b", "c")],
+ {"b": H1, "c": M15},
+ {"a": H1, "b": H1, "c": M15},
+ {"b"},
+ ),
+ # a 15min view over a fresh parent and a parent pinned hourly: only the pinned one stays hourly
+ (
+ "pinned_parent_of_a_fast_join",
+ {"contacts", "signups"},
+ [("s3", "signups"), ("crm", "contacts"), ("contacts", "signups")],
+ {"contacts": H1, "signups": M15},
+ {"contacts": H1, "signups": M15},
+ {"contacts"},
+ ),
]
)
- def test_effective_cadence(self, _name, nodes, edges, targets, expected):
- self.assertEqual(compute_effective_cadences(nodes=nodes, edges=edges, declared_targets=targets), expected)
+ def test_effective_cadence(self, _name, nodes, edges, targets, expected, pinned=frozenset()):
+ self.assertEqual(
+ compute_effective_cadences(nodes=nodes, edges=edges, declared_targets=targets, pinned=pinned), expected
+ )
def test_deep_chain_does_not_overflow(self):
# prod DAGs can chain far past Python's recursion limit; propagation must stay iterative
@@ -114,12 +136,43 @@ class TestValidateFrequencyTarget(TestCase):
{"src": H6},
False,
),
+ # a pinned node may run slower than its consumers
+ ("pinned_above_descendant_ceiling_ok", "a", DAY, [("a", "ep")], {"ep": M15}, {}, True, {"a"}),
+ # but never faster than its sources
+ ("pinned_below_imported_floor", "a", M15, [("src", "a")], {}, {"src": H6}, False, {"a"}),
+ # one fresh parent keeps a 15min join legal under a parent pinned hourly
+ (
+ "fresh_parent_beside_a_pinned_one_ok",
+ "signups",
+ M15,
+ [("s3", "signups"), ("crm", "contacts"), ("contacts", "signups")],
+ {"contacts": H1},
+ {"s3": STREAMING, "crm": STREAMING},
+ True,
+ {"contacts"},
+ ),
+ # every parent pinned hourly: nothing feeds the join more often than that
+ (
+ "every_parent_pinned_slower",
+ "signups",
+ M15,
+ [("crm", "contacts"), ("crm", "accounts"), ("contacts", "signups"), ("accounts", "signups")],
+ {"contacts": H1, "accounts": H6},
+ {"crm": STREAMING},
+ False,
+ {"contacts", "accounts"},
+ ),
]
)
- def test_validate(self, _name, node_id, target, edges, targets, source_intervals, ok):
+ def test_validate(self, _name, node_id, target, edges, targets, source_intervals, ok, pinned=frozenset()):
if ok:
validate_declared_target(
- node_id=node_id, target=target, edges=edges, declared_targets=targets, source_intervals=source_intervals
+ node_id=node_id,
+ target=target,
+ edges=edges,
+ declared_targets=targets,
+ source_intervals=source_intervals,
+ pinned=pinned,
)
else:
with self.assertRaises(UnsatisfiableFrequencyError):
@@ -129,8 +182,25 @@ class TestValidateFrequencyTarget(TestCase):
edges=edges,
declared_targets=targets,
source_intervals=source_intervals,
+ pinned=pinned,
)
+ def test_refusal_under_pinned_parents_names_the_pinned_view(self):
+ with self.assertRaises(UnsatisfiableFrequencyError) as raised:
+ validate_declared_target(
+ node_id="signups",
+ target=M15,
+ edges=[("crm", "contacts"), ("contacts", "signups")],
+ declared_targets={"contacts": H1},
+ source_intervals={"crm": STREAMING},
+ names={"contacts": "org_universe"},
+ pinned={"contacts"},
+ )
+ self.assertEqual(
+ str(raised.exception),
+ "Can't refresh every 15 minutes: org_universe is pinned to refresh every 1 hour. Pick 1 hour instead.",
+ )
+
@parameterized.expand(
[
# 45min would silently degrade to hourly in the spec builder
@@ -314,6 +384,74 @@ class TestIntersectTargetBounds(TestCase):
self.assertTrue(all(option.allowed for option in options.values()))
+class TestPinBreaksDescendants(TestCase):
+ @parameterized.expand(
+ [
+ ("chain_child_reads_only_the_pinned_view", [("src", "up"), ("up", "maps")], {"maps": H6}, ["maps"]),
+ (
+ "fan_in_child_keeps_a_fresh_input",
+ [("src", "up"), ("up", "maps"), ("src", "maps")],
+ {"maps": H6},
+ [],
+ ),
+ # maps is already finer than the slow source it reads through up, so the pin is not to blame
+ (
+ "already_invalid_child_does_not_block",
+ [("slow", "up"), ("up", "maps")],
+ {"maps": M15},
+ [],
+ ),
+ # middle already breaks its consumer ceiling (fast wants 15m) but still meets its own 1h
+ # target at runtime; the pin's floor is what would take that away
+ (
+ "existing_ceiling_violation_does_not_hide_a_new_floor_violation",
+ [("src", "up"), ("up", "middle"), ("middle", "fast"), ("src", "fast")],
+ {"middle": H1, "fast": M15},
+ ["middle"],
+ ),
+ ]
+ )
+ def test_descendants_broken_by_pin(self, _name, edges, child_targets, expected):
+ broken = descendants_broken_by_pin(
+ node_id="up",
+ target=DAY,
+ edges=edges,
+ declared_targets={"up": H1, **child_targets},
+ source_intervals={"src": STREAMING, "slow": H6},
+ pinned=set(),
+ )
+ self.assertEqual([invalid.node_id for invalid in broken], expected)
+
+ @parameterized.expand(
+ [
+ (
+ "named",
+ {"maps": "maps"},
+ "Can't pin this view to 1 day: downstream view maps refreshes every 6 hours and has no "
+ "fresher input than this view. Slow down maps first, or pin a faster cadence.",
+ ),
+ (
+ "withheld",
+ {},
+ "Can't pin this view to 1 day: a downstream view refreshes every 6 hours and has no "
+ "fresher input than this view. Slow down that view first, or pin a faster cadence.",
+ ),
+ ]
+ )
+ def test_refusal_names_the_downstream_view_it_would_strand(self, _name, names, message):
+ with self.assertRaises(UnsatisfiableFrequencyError) as raised:
+ validate_pin(
+ node_id="up",
+ target=DAY,
+ edges=[("src", "up"), ("up", "maps")],
+ declared_targets={"up": DAY, "maps": H6},
+ source_intervals={"src": STREAMING},
+ pinned=set(),
+ names=names,
+ )
+ self.assertEqual(str(raised.exception), message)
+
+
class TestFindInvalidTargets(TestCase):
@parameterized.expand(
[
@@ -352,11 +490,32 @@ class TestFindInvalidTargets(TestCase):
{"src": STREAMING},
[],
),
+ # pinned slower than its consumer is deliberate, and its ancestor answers to the pin, not the consumer
+ (
+ "pinned_slower_than_consumer_is_valid",
+ [("src", "up"), ("up", "a"), ("a", "ep"), ("src", "ep")],
+ {"up": H1, "a": H1, "ep": M15},
+ {"src": STREAMING},
+ [],
+ {"a"},
+ ),
+ # every parent pinned slower than the child: the child cannot be served at its target
+ (
+ "every_parent_pinned_slower_flags_the_child",
+ [("src", "p1"), ("src", "p2"), ("p1", "c"), ("p2", "c")],
+ {"p1": H1, "p2": H6, "c": M15},
+ {"src": STREAMING},
+ [InvalidTarget(node_id="c", declared=M15, source_floor=H1, consumer_ceiling=None)],
+ {"p1", "p2"},
+ ),
]
)
- def test_find_invalid_targets(self, _name, edges, targets, source_intervals, expected):
+ def test_find_invalid_targets(self, _name, edges, targets, source_intervals, expected, pinned=frozenset()):
self.assertEqual(
- find_invalid_targets(edges=edges, declared_targets=targets, source_intervals=source_intervals), expected
+ find_invalid_targets(
+ edges=edges, declared_targets=targets, source_intervals=source_intervals, pinned=pinned
+ ),
+ expected,
)
@@ -392,6 +551,15 @@ class TestClampToSourceFloor(TestCase):
self.assertEqual(changes[0].source_floor, source_intervals["src"])
self.assertEqual(changes[0].clamped_to, expected_clamped_to)
+ def test_a_child_of_only_pinned_parents_runs_no_faster_than_the_fastest_pin(self):
+ clamped, _ = clamp_to_source_floor(
+ {"p1": H1, "p2": H6, "c": M15},
+ edges=[("src", "p1"), ("src", "p2"), ("p1", "c"), ("p2", "c")],
+ source_intervals={"src": STREAMING},
+ pinned={"p1", "p2"},
+ )
+ self.assertEqual(clamped, {"p1": H1, "p2": H6, "c": H1})
+
class TestNormalizeSeedTarget(TestCase):
@parameterized.expand(
diff --git a/products/data_modeling/backend/test/test_schedule_reconcile.py b/products/data_modeling/backend/test/test_schedule_reconcile.py
index 6629873a95d3..394694e8c4f6 100644
--- a/products/data_modeling/backend/test/test_schedule_reconcile.py
+++ b/products/data_modeling/backend/test/test_schedule_reconcile.py
@@ -6,6 +6,7 @@ from unittest import mock
from django.test import SimpleTestCase
+from parameterized import parameterized
from temporalio.client import ScheduleAlreadyRunningError, ScheduleListActionStartWorkflow
from temporalio.service import RPCError, RPCStatusCode
@@ -15,8 +16,10 @@ from products.data_modeling.backend.logic.cohort_scheduling import tier_schedule
from products.data_modeling.backend.logic.freshness import UnsupportedFrequencyTargetError
from products.data_modeling.backend.logic.node_frequency import (
get_declared_anchor,
+ get_declared_pin,
get_declared_target,
set_declared_anchor,
+ set_declared_pin,
set_declared_target,
)
from products.data_modeling.backend.logic.saved_query_dag_sync import promote_dag_view_nodes_to_matview
@@ -24,6 +27,7 @@ from products.data_modeling.backend.logic.schedule_reconcile import (
DagScheduleTeardown,
TeamScheduleTeardownError,
apply_saved_query_frequency_anchor,
+ apply_saved_query_frequency_target,
bootstrap_dag_to_tiers,
convert_dag_to_tiers,
delete_dag_schedules,
@@ -469,15 +473,28 @@ class TestMaybeReconcileDag(BaseTest):
update.assert_called_once()
self.assertEqual(update.call_args.kwargs["id"], tier_id)
- def test_upstream_slower_than_its_consumer_runs_at_the_consumer_cadence_and_keeps_its_setting(self):
+ @parameterized.expand(
+ [
+ ("unpinned_runs_at_the_consumer_cadence", False, True, {H6: ["upstream", "consumer"]}),
+ ("pinned_keeps_its_own_cadence", True, True, {DAY: ["upstream"], H6: ["consumer"]}),
+ ("pinned_holds_back_a_consumer_with_no_fresher_parent", True, False, {DAY: ["upstream", "consumer"]}),
+ ]
+ )
+ def test_upstream_slower_than_its_consumer_keeps_its_setting(
+ self, _name, pinned, consumer_reads_source, expected_tiers
+ ):
dag = DAG.get_or_create_default(self.team)
source = _warehouse_source_node(self.team, dag, sync_frequency_interval=H1)
upstream = _saved_query_node(self.team, dag, "org_universe", NodeType.MAT_VIEW)
consumer = _saved_query_node(self.team, dag, "maps", NodeType.MAT_VIEW)
Edge.objects.create(team=self.team, dag=dag, source=source, target=upstream)
Edge.objects.create(team=self.team, dag=dag, source=upstream, target=consumer)
+ if consumer_reads_source:
+ Edge.objects.create(team=self.team, dag=dag, source=source, target=consumer)
set_declared_target(upstream, DAY)
set_declared_target(consumer, H6)
+ if pinned:
+ set_declared_pin(upstream, True)
with (
mock.patch(f"{RECONCILE}.async_connect", new=mock.AsyncMock(return_value=self._temporal_listing([]))),
@@ -486,13 +503,42 @@ class TestMaybeReconcileDag(BaseTest):
with self.captureOnCommitCallbacks(execute=True):
maybe_reconcile_dag(dag)
- create.assert_called_once()
- self.assertEqual(create.call_args.kwargs["id"], tier_schedule_id(str(dag.id), H6))
- scheduled = create.call_args.kwargs["schedule"].action.args[0]["node_ids"]
- self.assertEqual(sorted(scheduled), sorted([str(upstream.id), str(consumer.id)]))
+ ids = {"upstream": str(upstream.id), "consumer": str(consumer.id)}
+ created = {
+ call.kwargs["id"]: sorted(call.kwargs["schedule"].action.args[0]["node_ids"])
+ for call in create.call_args_list
+ }
+ self.assertEqual(
+ created,
+ {
+ tier_schedule_id(str(dag.id), cadence): sorted(ids[name] for name in names)
+ for cadence, names in expected_tiers.items()
+ },
+ )
upstream.refresh_from_db()
self.assertEqual(get_declared_target(upstream), DAY)
+ def test_a_pin_on_its_own_keeps_the_stored_target(self):
+ node = _saved_query_node(self.team, DAG.get_or_create_default(self.team), "mv", NodeType.MAT_VIEW)
+ set_declared_target(node, H6)
+ assert node.saved_query is not None
+
+ with mock.patch(f"{RECONCILE}.maybe_reconcile_dag"):
+ apply_saved_query_frequency_target(node.saved_query, None, pinned=True, keep_target=True)
+
+ node.refresh_from_db()
+ self.assertEqual((get_declared_target(node), get_declared_pin(node)), (H6, True))
+
+ def test_a_pin_on_its_own_is_refused_when_the_node_has_no_target(self):
+ node = _saved_query_node(self.team, DAG.get_or_create_default(self.team), "mv", NodeType.MAT_VIEW)
+ assert node.saved_query is not None
+
+ with mock.patch(f"{RECONCILE}.maybe_reconcile_dag"), self.assertRaises(ValueError):
+ apply_saved_query_frequency_target(node.saved_query, None, pinned=True, keep_target=True)
+
+ node.refresh_from_db()
+ self.assertFalse(get_declared_pin(node))
+
def test_reconcile_failure_never_raises_past_commit(self):
dag = self._dag_with_target()
with (
diff --git a/products/data_modeling/backend/tests/api/test_node_api.py b/products/data_modeling/backend/tests/api/test_node_api.py
index 7251e6e67c14..0cf3a7d9d0ba 100644
--- a/products/data_modeling/backend/tests/api/test_node_api.py
+++ b/products/data_modeling/backend/tests/api/test_node_api.py
@@ -14,8 +14,13 @@ from rest_framework import status
from temporalio.common import WorkflowIDConflictPolicy, WorkflowIDReusePolicy
from posthog.models import Team
+from posthog.models.activity_logging.activity_log import ActivityLog
-from products.data_modeling.backend.logic.node_frequency import set_declared_target
+from products.data_modeling.backend.logic.node_frequency import (
+ get_declared_pin,
+ get_declared_target,
+ set_declared_target,
+)
from products.data_modeling.backend.logic.node_materialization import start_node_materialization
from products.data_modeling.backend.logic.node_suspension import (
mark_node_suspended,
@@ -750,6 +755,135 @@ class TestNodeViewSet(APIBaseTest):
self.assertEqual(len(lineage["edges"]), 2 if flag_enabled else 1)
self.assertEqual(detail_response.status_code, status.HTTP_200_OK if flag_enabled else status.HTTP_404_NOT_FOUND)
+ def _node_url(self, node: Node) -> str:
+ return f"/api/environments/{self.team.id}/data_modeling_nodes/{node.id}/"
+
+ def _give_the_view_a_consumer(self, consumer_target: timedelta, *, reads_the_table_too: bool = False) -> Node:
+ self.view_node.type = NodeType.MAT_VIEW
+ self.view_node.save(update_fields=["type"])
+ consumer = Node.objects.create(
+ team=self.team,
+ dag=self.dag,
+ saved_query=DataWarehouseSavedQuery.objects.create(
+ name="consumer_view", team=self.team, query={"query": "SELECT 1", "kind": "HogQLQuery"}
+ ),
+ type=NodeType.MAT_VIEW,
+ )
+ Edge.objects.create(team=self.team, dag=self.dag, source=self.view_node, target=consumer)
+ if reads_the_table_too:
+ Edge.objects.create(team=self.team, dag=self.dag, source=self.table_node, target=consumer)
+ set_declared_target(consumer, consumer_target)
+ return consumer
+
+ def _patch_cadence(self, node: Node, payload: dict):
+ with patch("products.data_modeling.backend.logic.schedule_reconcile.maybe_reconcile_dag"):
+ return self.client.patch(self._node_url(node), data=payload, format="json")
+
+ def test_a_pinned_view_keeps_a_cadence_slower_than_its_consumer_until_unpinned(self):
+ self._give_the_view_a_consumer(timedelta(hours=6), reads_the_table_too=True)
+
+ bounds = self.client.get(f"{self._node_url(self.view_node)}frequency_bounds/").json()
+ options = {option["cadence"]: option for option in bounds["options"]}
+ self.assertFalse(bounds["pinned"])
+ self.assertEqual([view["name"] for view in options["24hour"]["delays_if_pinned"]], ["consumer_view"])
+ self.assertEqual(options["6hour"]["delays_if_pinned"], [])
+ # Unpinned, the consumer's 6h demand wins, so the preview of unpinning a daily view is 6h
+ self.assertEqual((options["24hour"]["unpinned_cadence"], options["6hour"]["unpinned_cadence"]), ("6hour",) * 2)
+
+ # Without the pin the consumer's 6h demand refuses a daily cadence, so both have to land together.
+ self.assertEqual(self._patch_cadence(self.view_node, {"sync_frequency": "24hour"}).status_code, 400)
+ pinned = self._patch_cadence(self.view_node, {"sync_frequency": "24hour", "pinned": True})
+ self.assertEqual(pinned.status_code, 200, pinned.json())
+ self.assertEqual((pinned.json()["sync_interval"], pinned.json()["pinned"]), ("24hour", True))
+ self.view_node.refresh_from_db()
+ self.assertEqual(
+ (get_declared_target(self.view_node), get_declared_pin(self.view_node)), (timedelta(days=1), True)
+ )
+ self.assertTrue(self.client.get(f"{self._node_url(self.view_node)}frequency_bounds/").json()["pinned"])
+ log = ActivityLog.objects.get(scope="DataWarehouseSavedQuery", item_id=str(self.saved_query.id))
+ assert log.detail is not None
+ changes = {change["field"]: (change["before"], change["after"]) for change in log.detail["changes"]}
+ self.assertEqual(
+ changes, {"sync_frequency_interval": (None, "1 day, 0:00:00"), "sync_frequency_pinned": (False, True)}
+ )
+
+ unpinned = self._patch_cadence(self.view_node, {"pinned": False})
+ self.assertEqual(unpinned.status_code, 200, unpinned.json())
+ self.assertFalse(unpinned.json()["pinned"])
+ self.view_node.refresh_from_db()
+ self.assertEqual(
+ (get_declared_target(self.view_node), get_declared_pin(self.view_node)), (timedelta(days=1), False)
+ )
+
+ def test_a_cadence_write_clears_the_legacy_saved_query_interval(self):
+ DataWarehouseSavedQuery.objects.filter(pk=self.saved_query.pk).update(
+ sync_frequency_interval=timedelta(hours=12)
+ )
+ set_declared_target(self.view_node, timedelta(hours=6))
+
+ response = self._patch_cadence(self.view_node, {"sync_frequency": "never"})
+
+ self.assertEqual(response.status_code, 200, response.json())
+ self.assertIsNone(response.json()["sync_interval"])
+ self.saved_query.refresh_from_db()
+ self.assertIsNone(self.saved_query.sync_frequency_interval)
+
+ @parameterized.expand(
+ [
+ ("consumer_reads_only_this_view", False, {"6hour": None, "24hour": "consumer_view"}),
+ ("consumer_has_a_fresher_input", True, {"6hour": None, "24hour": None}),
+ ]
+ )
+ def test_a_pin_that_strands_a_downstream_view_is_marked_and_refused(
+ self, _name, reads_the_table_too, expected_blockers
+ ):
+ self._give_the_view_a_consumer(timedelta(hours=6), reads_the_table_too=reads_the_table_too)
+
+ bounds = self.client.get(f"{self._node_url(self.view_node)}frequency_bounds/").json()
+ blockers = {
+ option["cadence"]: option["pin_blocked_by"] and option["pin_blocked_by"]["name"]
+ for option in bounds["options"]
+ if option["cadence"] in expected_blockers
+ }
+ self.assertEqual(blockers, expected_blockers)
+
+ response = self._patch_cadence(self.view_node, {"sync_frequency": "24hour", "pinned": True})
+ if expected_blockers["24hour"] is None:
+ self.assertEqual(response.status_code, 200, response.json())
+ return
+ self.assertEqual(response.status_code, 400, response.json())
+ self.assertEqual(
+ response.json()["detail"],
+ "Can't pin this view to 1 day: downstream view consumer_view refreshes every 6 hours and has no "
+ "fresher input than this view. Slow down consumer_view first, or pin a faster cadence.",
+ )
+ self.view_node.refresh_from_db()
+ self.assertEqual((get_declared_target(self.view_node), get_declared_pin(self.view_node)), (None, False))
+
+ @parameterized.expand(
+ [
+ ("no_cadence_yet", {"pinned": True}),
+ ("cadence_cleared_in_the_same_write", {"sync_frequency": "never", "pinned": True}),
+ ]
+ )
+ def test_pinning_needs_a_cadence(self, _name, payload):
+ response = self._patch_cadence(self.view_node, payload)
+
+ self.assertEqual(response.status_code, 400, response.json())
+ self.view_node.refresh_from_db()
+ self.assertFalse(get_declared_pin(self.view_node))
+
+ @parameterized.expand(["table", "managed_view"])
+ def test_cadence_is_only_writable_on_views_the_team_owns(self, kind):
+ node = self.table_node if kind == "table" else self._create_managed_node()
+
+ with patch("posthoganalytics.feature_enabled", return_value=True):
+ response = self._patch_cadence(node, {"sync_frequency": "24hour", "pinned": True})
+
+ self.assertEqual(response.status_code, 400, response.json())
+ node.refresh_from_db()
+ self.assertEqual((get_declared_target(node), get_declared_pin(node)), (None, False))
+
def test_cannot_patch_node_of_managed_view(self):
node = self._create_managed_node()
@@ -1020,6 +1154,52 @@ class TestNodeAccessControl(WarehouseAccessControlTestMixin):
self.node.refresh_from_db()
self.assertEqual(suspension_state(self.node) == {}, expected_status == status.HTTP_200_OK)
+ @parameterized.expand(
+ [
+ ("viewer", status.HTTP_403_FORBIDDEN),
+ ("editor", status.HTTP_200_OK),
+ ]
+ )
+ def test_setting_a_cadence_requires_warehouse_write_access(self, access_level, expected_status):
+ user = self.viewer_user if access_level == "viewer" else self.editor_user
+ self._create_access_control(user, access_level=access_level)
+ self.client.force_login(user)
+
+ with patch("products.data_modeling.backend.logic.schedule_reconcile.maybe_reconcile_dag"):
+ response = self.client.patch(
+ f"{self._list_url()}{self.node.id}/", data={"sync_frequency": "24hour", "pinned": True}, format="json"
+ )
+
+ self.assertEqual(response.status_code, expected_status)
+ self.node.refresh_from_db()
+ self.assertEqual(get_declared_pin(self.node), expected_status == status.HTTP_200_OK)
+
+ def test_pin_bounds_do_not_name_a_downstream_view_the_member_is_denied(self):
+ consumer = Node.objects.create(
+ team=self.team,
+ dag=self.dag,
+ saved_query=DataWarehouseSavedQuery.objects.create(
+ name="hidden_consumer", team=self.team, query={"query": "SELECT 1", "kind": "HogQLQuery"}
+ ),
+ type=NodeType.MAT_VIEW,
+ )
+ Edge.objects.create(team=self.team, dag=self.dag, source=self.node, target=consumer)
+ set_declared_target(consumer, timedelta(hours=6))
+ self._create_access_control(self.editor_user, access_level="editor")
+ self._create_access_control(
+ self.editor_user, resource="warehouse_view", resource_id=str(consumer.saved_query_id), access_level="none"
+ )
+ self.client.force_login(self.editor_user)
+
+ response = self.client.get(f"{self._list_url()}{self.node.id}/frequency_bounds/")
+
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
+ options = {option["cadence"]: option for option in response.json()["options"]}
+ self.assertEqual(options["24hour"]["pin_blocked_by"], {"id": None, "name": None})
+ self.assertEqual(options["24hour"]["delays_if_pinned"], [])
+ self.assertNotIn("hidden_consumer", response.content.decode())
+ self.assertNotIn(str(consumer.id), response.content.decode())
+
@parameterized.expand(
[
("node_viewer", "node", "viewer", status.HTTP_403_FORBIDDEN),
diff --git a/products/data_modeling/frontend/generated/api.schemas.ts b/products/data_modeling/frontend/generated/api.schemas.ts
index abe815bb3dd5..e1d1064875f3 100644
--- a/products/data_modeling/frontend/generated/api.schemas.ts
+++ b/products/data_modeling/frontend/generated/api.schemas.ts
@@ -102,6 +102,32 @@ export const NodeOriginEnumApi = {
Warehouse: 'warehouse',
} as const
+/**
+ * * `never` - never
+ * * `15min` - 15min
+ * * `30min` - 30min
+ * * `1hour` - 1hour
+ * * `6hour` - 6hour
+ * * `12hour` - 12hour
+ * * `24hour` - 24hour
+ * * `7day` - 7day
+ * * `30day` - 30day
+ */
+export type SavedQuerySyncFrequencyEnumApi =
+ (typeof SavedQuerySyncFrequencyEnumApi)[keyof typeof SavedQuerySyncFrequencyEnumApi]
+
+export const SavedQuerySyncFrequencyEnumApi = {
+ Never: 'never',
+ '15min': '15min',
+ '30min': '30min',
+ '1hour': '1hour',
+ '6hour': '6hour',
+ '12hour': '12hour',
+ '24hour': '24hour',
+ '7day': '7day',
+ '30day': '30day',
+} as const
+
export interface NodeSuspensionApi {
/** When the node was suspended. */
at: string
@@ -167,6 +193,20 @@ export interface NodeApi {
readonly user_tag: string | null
/** @nullable */
readonly sync_interval: string | null
+ /** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.
+ *
+ * * `never` - never
+ * * `15min` - 15min
+ * * `30min` - 30min
+ * * `1hour` - 1hour
+ * * `6hour` - 6hour
+ * * `12hour` - 12hour
+ * * `24hour` - 24hour
+ * * `7day` - 7day
+ * * `30day` - 30day */
+ sync_frequency?: SavedQuerySyncFrequencyEnumApi
+ /** Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again. */
+ pinned?: boolean
/** Engines this node is suspended for after repeated materialization failures. Suspended engines are skipped by scheduled DAG runs until the node is resumed. */
readonly suspended: NodeApiSuspended
/** The endpoint version this node's materialization backs, or null for nodes that are not endpoints. */
@@ -231,12 +271,106 @@ export interface PatchedNodeApi {
readonly user_tag?: string | null
/** @nullable */
readonly sync_interval?: string | null
+ /** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.
+ *
+ * * `never` - never
+ * * `15min` - 15min
+ * * `30min` - 30min
+ * * `1hour` - 1hour
+ * * `6hour` - 6hour
+ * * `12hour` - 12hour
+ * * `24hour` - 24hour
+ * * `7day` - 7day
+ * * `30day` - 30day */
+ sync_frequency?: SavedQuerySyncFrequencyEnumApi
+ /** Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again. */
+ pinned?: boolean
/** Engines this node is suspended for after repeated materialization failures. Suspended engines are skipped by scheduled DAG runs until the node is resumed. */
readonly suspended?: PatchedNodeApiSuspended
/** The endpoint version this node's materialization backs, or null for nodes that are not endpoints. */
readonly endpoint?: NodeEndpointApi | null
}
+/**
+ * * `15min` - 15min
+ * * `30min` - 30min
+ * * `1hour` - 1hour
+ * * `6hour` - 6hour
+ * * `12hour` - 12hour
+ * * `24hour` - 24hour
+ * * `7day` - 7day
+ * * `30day` - 30day
+ */
+export type MaterializeSyncFrequencyEnumApi =
+ (typeof MaterializeSyncFrequencyEnumApi)[keyof typeof MaterializeSyncFrequencyEnumApi]
+
+export const MaterializeSyncFrequencyEnumApi = {
+ '15min': '15min',
+ '30min': '30min',
+ '1hour': '1hour',
+ '6hour': '6hour',
+ '12hour': '12hour',
+ '24hour': '24hour',
+ '7day': '7day',
+ '30day': '30day',
+} as const
+
+export interface NodePinDelayApi {
+ /** Node id of the downstream view or endpoint. */
+ id: string
+ /** Name of the downstream view or endpoint. */
+ name: string
+}
+
+export interface NodePinBlockerApi {
+ /**
+ * Node id of the downstream view that stops the pin, or null when the caller may not read it.
+ * @nullable
+ */
+ id: string | null
+ /**
+ * Name of the downstream view that stops the pin, or null when the caller may not read it.
+ * @nullable
+ */
+ name: string | null
+}
+
+export interface NodeFrequencyOptionApi {
+ /** A cadence the view could refresh at.
+ *
+ * * `15min` - 15min
+ * * `30min` - 30min
+ * * `1hour` - 1hour
+ * * `6hour` - 6hour
+ * * `12hour` - 12hour
+ * * `24hour` - 24hour
+ * * `7day` - 7day
+ * * `30day` - 30day */
+ cadence: MaterializeSyncFrequencyEnumApi
+ /** Downstream views and endpoints that refresh more often than this cadence. If this view were pinned to it, they would read data older than their own cadence. */
+ delays_if_pinned: NodePinDelayApi[]
+ /** The downstream view that stops a pin at this cadence. It reads fresh data only through this view, so a pin this slow would leave it unable to refresh as often as its own cadence. Null when this cadence can be pinned. */
+ pin_blocked_by: NodePinBlockerApi | null
+ /** The cadence the view would run at if it held this cadence without a pin. Finer than this cadence when a downstream view needs fresher data than the pin allows.
+ *
+ * * `15min` - 15min
+ * * `30min` - 30min
+ * * `1hour` - 1hour
+ * * `6hour` - 6hour
+ * * `12hour` - 12hour
+ * * `24hour` - 24hour
+ * * `7day` - 7day
+ * * `30day` - 30day */
+ unpinned_cadence: MaterializeSyncFrequencyEnumApi
+}
+
+export interface NodeFrequencyBoundsApi {
+ /** True when the view refreshes at its own cadence instead of as often as its downstream views and endpoints need. */
+ pinned: boolean
+ /** Every cadence a picker may show, finest first. Empty when the node has no view. */
+ options: NodeFrequencyOptionApi[]
+}
+
export interface NodeResumeApi {
/** False when the node was not suspended to begin with. */
resumed: boolean
diff --git a/products/data_modeling/frontend/generated/api.ts b/products/data_modeling/frontend/generated/api.ts
index 214e3ab418e6..0337794240e7 100644
--- a/products/data_modeling/frontend/generated/api.ts
+++ b/products/data_modeling/frontend/generated/api.ts
@@ -18,6 +18,7 @@ import type {
LineageResponseApi,
ModelUsageApi,
NodeApi,
+ NodeFrequencyBoundsApi,
NodeResumeApi,
NodeRunApi,
NodeRunResponseApi,
@@ -239,6 +240,24 @@ export const dataModelingNodesDestroy = async (projectId: string, id: string, op
})
}
+export const getDataModelingNodesFrequencyBoundsRetrieveUrl = (projectId: string, id: string) => {
+ return `/api/projects/${projectId}/data_modeling_nodes/${id}/frequency_bounds/`
+}
+
+/**
+ * The cadences this view could refresh at, and what pinning it to each one would delay downstream.
+ */
+export const dataModelingNodesFrequencyBoundsRetrieve = async (
+ projectId: string,
+ id: string,
+ options?: RequestInit
+): Promise<NodeFrequencyBoundsApi> => {
+ return apiMutator<NodeFrequencyBoundsApi>(getDataModelingNodesFrequencyBoundsRetrieveUrl(projectId, id), {
+ ...options,
+ method: 'GET',
+ })
+}
+
export const getDataModelingNodesMaterializeCreateUrl = (projectId: string, id: string) => {
return `/api/projects/${projectId}/data_modeling_nodes/${id}/materialize/`
}
diff --git a/products/data_modeling/frontend/generated/api.zod.ts b/products/data_modeling/frontend/generated/api.zod.ts
index 50b7f9e5de7d..5590100d40a9 100644
--- a/products/data_modeling/frontend/generated/api.zod.ts
+++ b/products/data_modeling/frontend/generated/api.zod.ts
@@ -30,6 +30,21 @@ export const DataModelingNodesUpdateBody = /* @__PURE__ */ zod.object({
'\* `table` - Table\n\* `view` - View\n\* `matview` - Mat View\n\* `endpoint` - Endpoint\n\* `metric` - Metric'
),
description: zod.string().max(dataModelingNodesUpdateBodyDescriptionMax).optional(),
+ sync_frequency: zod
+ .enum(['never', '15min', '30min', '1hour', '6hour', '12hour', '24hour', '7day', '30day'])
+ .describe(
+ '\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day'
+ )
+ .optional()
+ .describe(
+ "How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.\n\n\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day"
+ ),
+ pinned: zod
+ .boolean()
+ .optional()
+ .describe(
+ 'Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again.'
+ ),
})
export const dataModelingNodesPartialUpdateBodyNameMax = 2048
@@ -45,6 +60,21 @@ export const DataModelingNodesPartialUpdateBody = /* @__PURE__ */ zod.object({
'\* `table` - Table\n\* `view` - View\n\* `matview` - Mat View\n\* `endpoint` - Endpoint\n\* `metric` - Metric'
),
description: zod.string().max(dataModelingNodesPartialUpdateBodyDescriptionMax).optional(),
+ sync_frequency: zod
+ .enum(['never', '15min', '30min', '1hour', '6hour', '12hour', '24hour', '7day', '30day'])
+ .describe(
+ '\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day'
+ )
+ .optional()
+ .describe(
+ "How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.\n\n\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day"
+ ),
+ pinned: zod
+ .boolean()
+ .optional()
+ .describe(
+ 'Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again.'
+ ),
})
/**
@@ -63,6 +93,21 @@ export const DataModelingNodesMaterializeCreateBody = /* @__PURE__ */ zod.object
'\* `table` - Table\n\* `view` - View\n\* `matview` - Mat View\n\* `endpoint` - Endpoint\n\* `metric` - Metric'
),
description: zod.string().max(dataModelingNodesMaterializeCreateBodyDescriptionMax).optional(),
+ sync_frequency: zod
+ .enum(['never', '15min', '30min', '1hour', '6hour', '12hour', '24hour', '7day', '30day'])
+ .describe(
+ '\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day'
+ )
+ .optional()
+ .describe(
+ "How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.\n\n\* `never` - never\n\* `15min` - 15min\n\* `30min` - 30min\n\* `1hour` - 1hour\n\* `6hour` - 6hour\n\* `12hour` - 12hour\n\* `24hour` - 24hour\n\* `7day` - 7day\n\* `30day` - 30day"
+ ),
+ pinned: zod
+ .boolean()
+ .optional()
+ .describe(
+ 'Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again.'
+ ),
})
/**
diff --git a/products/data_modeling/frontend/nodeDetail/ModelHealthSummary.tsx b/products/data_modeling/frontend/nodeDetail/ModelHealthSummary.tsx
index 5979c0000707..f5008f26ddb6 100644
--- a/products/data_modeling/frontend/nodeDetail/ModelHealthSummary.tsx
+++ b/products/data_modeling/frontend/nodeDetail/ModelHealthSummary.tsx
@@ -118,7 +118,7 @@ export function ModelHealthSummary({
</div>
<div>
<dt className="text-secondary mb-1">
- <Tooltip title="The refresh schedule saved for this model, including pauses and suspension.">
+ <Tooltip title="How often this view refreshes, including pauses and suspension. A downstream view can make it refresh more often than its own setting.">
<span className="border-b border-dashed border-secondary cursor-help">
Refresh schedule
</span>
diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx
index 3c52d753d67a..1f82bcdac41f 100644
--- a/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx
+++ b/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx
@@ -2,7 +2,12 @@ import { useActions, useValues } from 'kea'
import type { ReactNode } from 'react'
import { materializationJobsLogic } from 'scenes/data-warehouse/saved_queries/materializationJobsLogic'
-import { CADENCE_LABELS, modeDisabledReason } from 'scenes/data-warehouse/saved_queries/SyncFrequencySelect'
+import {
+ CADENCE_LABELS,
+ inheritedCadence,
+ modeDisabledReason,
+ SyncFrequencyValue,
+} from 'scenes/data-warehouse/saved_queries/SyncFrequencySelect'
import { urls } from 'scenes/urls'
import { DataModelingSyncInterval } from '~/types'
@@ -39,6 +44,7 @@ export function NodeDetailOverview({ id, metadata }: { id: string; metadata?: Re
savedQueryError: statusRefreshError,
savedQueryLoading: statusRefreshing,
lastSuccessfulSyncAt,
+ nodeFrequency,
} = useValues(materializationLogic)
const { refreshMaterialization } = useActions(materializationLogic)
const savedQuery = polledSavedQuery ?? sceneSavedQuery
@@ -75,7 +81,16 @@ export function NodeDetailOverview({ id, metadata }: { id: string; metadata?: Re
// run stopped, not that this model stopped refreshing.
const suspension = (savedQuery?.suspended ?? node.suspended)?.[SERVING_ENGINE]
const suspended = !!suspension
- const cadence = savedQuery?.sync_frequency
+ // The saved setting is a lower bound: a downstream view can make this one run more often unless it is pinned.
+ // The node in the scene store is not reloaded after a pin save, so the refreshed bounds win when they have loaded.
+ const inherited =
+ (nodeFrequency?.bounds.pinned ?? node.pinned)
+ ? null
+ : inheritedCadence(
+ savedQuery?.sync_frequency_bounds,
+ (savedQuery?.sync_frequency as SyncFrequencyValue | undefined) ?? null
+ )
+ const cadence = inherited?.cadence ?? savedQuery?.sync_frequency
const schedule = suspended
? 'Suspended after repeated failures'
: node.type === 'endpoint'
diff --git a/products/data_warehouse/backend/presentation/views/saved_query/editing.py b/products/data_warehouse/backend/presentation/views/saved_query/editing.py
index 1f2c5b16fbb9..1b6677877190 100644
--- a/products/data_warehouse/backend/presentation/views/saved_query/editing.py
+++ b/products/data_warehouse/backend/presentation/views/saved_query/editing.py
@@ -628,15 +628,7 @@ class DataWarehouseSavedQuerySerializer(
if frequency_changed and previous_target != target:
# The cadence lives on the DAG node, so changes_between() sees nothing and
# log_activity would discard the whole updated entry as a no-op.
- changes.append(
- Change(
- type="DataWarehouseSavedQuery",
- action="changed",
- field="sync_frequency_interval",
- before=str(previous_target) if previous_target is not None else None,
- after=str(target) if target is not None else None,
- )
- )
+ changes.extend(modeling_api.cadence_changes((previous_target, target)))
log_activity(
organization_id=team.organization_id,
team_id=team.id,
diff --git a/products/data_warehouse/backend/tests/api/test_saved_query.py b/products/data_warehouse/backend/tests/api/test_saved_query.py
index 6708dd72b96f..8a73b4061aa3 100644
--- a/products/data_warehouse/backend/tests/api/test_saved_query.py
+++ b/products/data_warehouse/backend/tests/api/test_saved_query.py
@@ -1441,12 +1441,16 @@ class TestSavedQuery(APIBaseTest):
("materialized_without_a_target", NodeType.MAT_VIEW, None, ["upstream_view"]),
# an ephemeral view never runs, so nothing about it speeds up
("ephemeral", NodeType.VIEW, timedelta(days=1), []),
+ # a pinned view keeps its own cadence whatever its consumers ask for
+ ("pinned", NodeType.MAT_VIEW, timedelta(days=1), [], True),
]
)
- def test_an_option_names_the_upstream_views_it_speeds_up(self, _name, upstream_type, upstream_target, expected):
+ def test_an_option_names_the_upstream_views_it_speeds_up(
+ self, _name, upstream_type, upstream_target, expected, upstream_pinned=False
+ ):
# A finer cadence downstream drags slower upstream views along with it. The picker has to
# say so before the write, or the upstream cadences change behind the user's back.
- from products.data_modeling.backend.facade.api import set_declared_target
+ from products.data_modeling.backend.facade.api import apply_saved_query_frequency_target, set_declared_target
upstream = self._create_view_with_a_consumer(consumer_target=timedelta(days=1))
upstream_node = Node.objects.get(saved_query_id=upstream["id"])
@@ -1454,6 +1458,10 @@ class TestSavedQuery(APIBaseTest):
upstream_node.save(update_fields=["type"])
if upstream_target is not None:
set_declared_target(upstream_node, upstream_target)
+ if upstream_pinned:
+ apply_saved_query_frequency_target(
+ DataWarehouseSavedQuery.objects.get(id=upstream["id"]), None, pinned=True, keep_target=True
+ )
consumer_id = DataWarehouseSavedQuery.objects.get(team=self.team, name="consumer_view").id
options = {option["cadence"]: option for option in self._read_frequency_bounds(str(consumer_id))["options"]}
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index a538b8094e5a..f1b8651f78a2 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -60636,6 +60636,20 @@ export namespace Schemas {
readonly user_tag: string | null;
/** @nullable */
readonly sync_interval: string | null;
+ /** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.
+ *
+ * * `never` - never
+ * * `15min` - 15min
+ * * `30min` - 30min
+ * * `1hour` - 1hour
+ * * `6hour` - 6hour
+ * * `12hour` - 12hour
+ * * `24hour` - 24hour
+ * * `7day` - 7day
+ * * `30day` - 30day */
+ sync_frequency?: SavedQuerySyncFrequencyEnum;
+ /** Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again. */
+ pinned?: boolean;
/** Engines this node is suspended for after repeated materialization failures. Suspended engines are skipped by scheduled DAG runs until the node is resumed. */
readonly suspended: NodeSuspended;
/** The endpoint version this node's materialization backs, or null for nodes that are not endpoints. */
@@ -66341,6 +66355,62 @@ export namespace Schemas {
specificity_rejection?: SpecificityMetadata | null;
}
+ export interface NodePinDelay {
+ /** Node id of the downstream view or endpoint. */
+ id: string;
+ /** Name of the downstream view or endpoint. */
+ name: string;
+ }
+
+ export interface NodePinBlocker {
+ /**
+ * Node id of the downstream view that stops the pin, or null when the caller may not read it.
+ * @nullable
+ */
+ id: string | null;
+ /**
+ * Name of the downstream view that stops the pin, or null when the caller may not read it.
+ * @nullable
+ */
+ name: string | null;
+ }
+
+ export interface NodeFrequencyOption {
+ /** A cadence the view could refresh at.
+ *
+ * * `15min` - 15min
+ * * `30min` - 30min
+ * * `1hour` - 1hour
+ * * `6hour` - 6hour
+ * * `12hour` - 12hour
+ * * `24hour` - 24hour
+ * * `7day` - 7day
+ * * `30day` - 30day */
+ cadence: MaterializeSyncFrequencyEnum;
+ /** Downstream views and endpoints that refresh more often than this cadence. If this view were pinned to it, they would read data older than their own cadence. */
+ delays_if_pinned: NodePinDelay[];
+ /** The downstream view that stops a pin at this cadence. It reads fresh data only through this view, so a pin this slow would leave it unable to refresh as often as its own cadence. Null when this cadence can be pinned. */
+ pin_blocked_by: NodePinBlocker | null;
+ /** The cadence the view would run at if it held this cadence without a pin. Finer than this cadence when a downstream view needs fresher data than the pin allows.
+ *
+ * * `15min` - 15min
+ * * `30min` - 30min
+ * * `1hour` - 1hour
+ * * `6hour` - 6hour
+ * * `12hour` - 12hour
+ * * `24hour` - 24hour
+ * * `7day` - 7day
+ * * `30day` - 30day */
+ unpinned_cadence: MaterializeSyncFrequencyEnum;
+ }
+
+ export interface NodeFrequencyBounds {
+ /** True when the view refreshes at its own cadence instead of as often as its downstream views and endpoints need. */
+ pinned: boolean;
+ /** Every cadence a picker may show, finest first. Empty when the node has no view. */
+ options: NodeFrequencyOption[];
+ }
+
export interface NodeResume {
/** False when the node was not suspended to begin with. */
resumed: boolean;
@@ -82957,6 +83027,20 @@ export namespace Schemas {
readonly user_tag?: string | null;
/** @nullable */
readonly sync_interval?: string | null;
+ /** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.
+ *
+ * * `never` - never
+ * * `15min` - 15min
+ * * `30min` - 30min
+ * * `1hour` - 1hour
+ * * `6hour` - 6hour
+ * * `12hour` - 12hour
+ * * `24hour` - 24hour
+ * * `7day` - 7day
+ * * `30day` - 30day */
+ sync_frequency?: SavedQuerySyncFrequencyEnum;
+ /** Pin this view to its own cadence. A pinned view does not refresh more often when downstream views or endpoints need fresher data, so they read data up to one pinned interval old. Requires a cadence. Set false to refresh as often as downstream views need again. */
+ pinned?: boolean;
/** Engines this node is suspended for after repeated materialization failures. Suspended engines are skipped by scheduled DAG runs until the node is resumed. */
readonly suspended?: PatchedNodeSuspended;
/** The endpoint version this node's materialization backs, or null for nodes that are not endpoints. */