Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): show the cadence a view runs at on lineage cards (#129143)

Jovan Sakovicd267fb85a646
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 912b0ee02007..b4a5457249c3 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -6269,6 +6269,7 @@ export interface DataModelingNode {
last_run_status?: DataModelingJobStatus
last_run_error?: string | null
sync_interval?: DataModelingSyncInterval
+ effective_sync_interval?: DataModelingSyncInterval | null
pinned?: boolean
suspended?: NodeApiSuspended
/** Set on endpoint nodes stamped at materialization enable; older nodes carry only the name */
diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.py
index 4046087cfa03..be358a0e74f9 100644
--- a/products/data_modeling/backend/facade/api.py
+++ b/products/data_modeling/backend/facade/api.py
@@ -99,6 +99,7 @@ _LAZY = {
"saved_query_target_bounds": "logic.node_frequency",
"SavedQueryFrequencyBounds": "logic.node_frequency",
"saved_query_pin_preview": "logic.node_frequency",
+ "scheduled_cadences_by_node": "logic.node_frequency",
"cadence_changes": "logic.cadence_changes",
"clear_legacy_sync_interval": "logic.cadence_changes",
"clear_node_suspension": "logic.node_suspension",
diff --git a/products/data_modeling/backend/logic/node_frequency.py b/products/data_modeling/backend/logic/node_frequency.py
index c242efde709b..185f390426b8 100644
--- a/products/data_modeling/backend/logic/node_frequency.py
+++ b/products/data_modeling/backend/logic/node_frequency.py
@@ -619,6 +619,14 @@ def _unpinned_cadence_by_cadence(
return result
+def scheduled_cadences_by_node(team_id: int, dag_ids: Iterable[uuid.UUID | str]) -> dict[str, timedelta | None]:
+ """The cadence the scheduler runs each node of these DAGs at, keyed by node id."""
+ result: dict[str, timedelta | None] = {}
+ for dag in DAG.objects.filter(team_id=team_id, id__in=set(dag_ids)):
+ result.update(scheduled_cadences(build_frequency_graph(dag))[0])
+ return result
+
+
def scheduled_cadences(
graph: FrequencyGraph,
*,
diff --git a/products/data_modeling/backend/presentation/views/node.py b/products/data_modeling/backend/presentation/views/node.py
index 9e8ba2c41aa9..05bb88db369d 100644
--- a/products/data_modeling/backend/presentation/views/node.py
+++ b/products/data_modeling/backend/presentation/views/node.py
@@ -36,6 +36,7 @@ from products.data_modeling.backend.facade.api import (
run_related_node_materializations,
saved_query_pin_preview,
saved_query_target_bounds,
+ scheduled_cadences_by_node,
suspension_state,
unsuspend_nodes,
)
@@ -194,6 +195,12 @@ class NodeSerializer(serializers.ModelSerializer):
last_run_error = serializers.SerializerMethodField(read_only=True)
user_tag = serializers.SerializerMethodField(read_only=True)
sync_interval = serializers.SerializerMethodField(read_only=True)
+ effective_sync_interval = serializers.SerializerMethodField(
+ read_only=True,
+ help_text="How often this node actually refreshes. A downstream view can make an unpinned view "
+ "refresh more often than `sync_interval`, and a slow source can make it refresh less often. "
+ "Null when the node is not scheduled.",
+ )
lineage_issue = serializers.SerializerMethodField(read_only=True)
origin = serializers.SerializerMethodField(read_only=True)
warehouse_table_id = serializers.SerializerMethodField(read_only=True)
@@ -219,6 +226,7 @@ class NodeSerializer(serializers.ModelSerializer):
"last_run_error",
"user_tag",
"sync_interval",
+ "effective_sync_interval",
"sync_frequency",
"pinned",
"suspended",
@@ -305,6 +313,14 @@ class NodeSerializer(serializers.ModelSerializer):
return sync_frequency_interval_to_sync_frequency(node.saved_query.sync_frequency_interval)
return None
+ def get_effective_sync_interval(self, node: Node) -> str | None:
+ # Built once per DAG for the whole response, not once per listed node.
+ by_dag = self.context.setdefault("_scheduled_cadences_by_dag", {})
+ if node.dag_id not in by_dag:
+ by_dag[node.dag_id] = scheduled_cadences_by_node(node.team_id, [node.dag_id])
+ cadence = by_dag[node.dag_id].get(str(node.id))
+ return sync_frequency_interval_to_sync_frequency(cadence) if cadence is not None else None
+
@extend_schema_field(LineageIssueSerializer(allow_null=True))
def get_lineage_issue(self, node: Node) -> dict[str, Any] | None:
return node.lineage_issue
diff --git a/products/data_modeling/backend/tests/api/test_node_api.py b/products/data_modeling/backend/tests/api/test_node_api.py
index 2ec4f8720e6b..67d827ebb389 100644
--- a/products/data_modeling/backend/tests/api/test_node_api.py
+++ b/products/data_modeling/backend/tests/api/test_node_api.py
@@ -19,6 +19,7 @@ from posthog.models.activity_logging.activity_log import ActivityLog
from products.data_modeling.backend.logic.node_frequency import (
get_declared_pin,
get_declared_target,
+ set_declared_pin,
set_declared_target,
)
from products.data_modeling.backend.logic.node_materialization import start_node_materialization
@@ -838,6 +839,18 @@ class TestNodeViewSet(APIBaseTest):
(get_declared_target(self.view_node), get_declared_pin(self.view_node)), (timedelta(days=1), False)
)
+ @parameterized.expand([("unpinned", False, "6hour"), ("pinned", True, "24hour")])
+ def test_list_reports_the_cadence_a_view_runs_at(self, _name, pinned, expected):
+ self._give_the_view_a_consumer(timedelta(hours=6), reads_the_table_too=True)
+ set_declared_target(self.view_node, timedelta(days=1))
+ set_declared_pin(self.view_node, pinned)
+
+ response = self.client.get(f"/api/environments/{self.team.id}/data_modeling_nodes/")
+
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
+ listed = next(node for node in response.json()["results"] if node["id"] == str(self.view_node.id))
+ self.assertEqual((listed["sync_interval"], listed["effective_sync_interval"]), ("24hour", expected))
+
def test_a_cadence_write_clears_the_legacy_saved_query_interval(self):
DataWarehouseSavedQuery.objects.filter(pk=self.saved_query.pk).update(
sync_frequency_interval=timedelta(hours=12)
diff --git a/products/data_modeling/frontend/freshness.test.ts b/products/data_modeling/frontend/freshness.test.ts
index 35814c75d064..1b81441f3167 100644
--- a/products/data_modeling/frontend/freshness.test.ts
+++ b/products/data_modeling/frontend/freshness.test.ts
@@ -41,6 +41,16 @@ describe('modelsBehindSchedule', () => {
expect(modelsBehindSchedule([stale], NOW)).toEqual([])
})
+ it('measures a model against the cadence it runs at, not its own setting', () => {
+ const inherited = node('inherited', {
+ sync_interval: '24hour',
+ effective_sync_interval: '1hour',
+ last_run_at: hoursAgo(3),
+ })
+
+ expect(modelsBehindSchedule([inherited], NOW).map((model) => model.intervalSeconds)).toEqual([3600])
+ })
+
it('measures a model that has never finished a run from when it was created', () => {
const never = node('never', { last_run_at: null, created_at: hoursAgo(10) })
diff --git a/products/data_modeling/frontend/freshness.ts b/products/data_modeling/frontend/freshness.ts
index bad97bbdc8b5..388caf13531c 100644
--- a/products/data_modeling/frontend/freshness.ts
+++ b/products/data_modeling/frontend/freshness.ts
@@ -37,10 +37,12 @@ export function modelsBehindSchedule(nodes: DataModelingNode[], now: number): Be
const behind: BehindScheduleModel[] = []
for (const node of nodes) {
- if (!node.sync_interval || !MATERIALIZING_TYPES.has(node.type)) {
+ // A downstream view can make a model run more often than its own setting.
+ const runsAt = node.sync_interval ? (node.effective_sync_interval ?? node.sync_interval) : undefined
+ if (!runsAt || !MATERIALIZING_TYPES.has(node.type)) {
continue
}
- const intervalSeconds = SYNC_INTERVAL_SECONDS[node.sync_interval]
+ const intervalSeconds = SYNC_INTERVAL_SECONDS[runsAt]
if (!intervalSeconds) {
continue
}
diff --git a/products/data_modeling/frontend/generated/api.schemas.ts b/products/data_modeling/frontend/generated/api.schemas.ts
index e1d1064875f3..3d6557bb836f 100644
--- a/products/data_modeling/frontend/generated/api.schemas.ts
+++ b/products/data_modeling/frontend/generated/api.schemas.ts
@@ -193,6 +193,11 @@ export interface NodeApi {
readonly user_tag: string | null
/** @nullable */
readonly sync_interval: string | null
+ /**
+ * How often this node actually refreshes. A downstream view can make an unpinned view refresh more often than `sync_interval`, and a slow source can make it refresh less often. Null when the node is not scheduled.
+ * @nullable
+ */
+ readonly effective_sync_interval: string | null
/** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.
*
* * `never` - never
@@ -271,6 +276,11 @@ export interface PatchedNodeApi {
readonly user_tag?: string | null
/** @nullable */
readonly sync_interval?: string | null
+ /**
+ * How often this node actually refreshes. A downstream view can make an unpinned view refresh more often than `sync_interval`, and a slow source can make it refresh less often. Null when the node is not scheduled.
+ * @nullable
+ */
+ readonly effective_sync_interval?: string | null
/** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.
*
* * `never` - never
diff --git a/products/data_modeling/frontend/lineage/LineageNode.tsx b/products/data_modeling/frontend/lineage/LineageNode.tsx
index 80cacc6bc826..43fffbde05a1 100644
--- a/products/data_modeling/frontend/lineage/LineageNode.tsx
+++ b/products/data_modeling/frontend/lineage/LineageNode.tsx
@@ -9,6 +9,7 @@ import {
IconExternal,
IconPauseFilled,
IconPencil,
+ IconPinFilled,
IconPlay,
IconPlayFilled,
IconTarget,
@@ -32,7 +33,7 @@ import { DataModelingNode } from '~/types'
import { MATERIALIZING_TYPES } from 'products/data_modeling/frontend/freshness'
import { servingSuspension } from 'products/data_modeling/frontend/suspension'
-import { syncIntervalToShorthand } from 'products/data_warehouse/frontend/utils'
+import { SYNC_FREQUENCY_ORDER, syncIntervalToShorthand } from 'products/data_warehouse/frontend/utils'
import { ElkDirection, NodeHandle } from './autolayout'
import { LineageGraphNode } from './lineageHiddenNodes'
@@ -48,6 +49,8 @@ export type LineageNodeShape = Pick<
| 'name'
| 'type'
| 'sync_interval'
+ | 'effective_sync_interval'
+ | 'pinned'
| 'last_run_at'
| 'last_run_status'
| 'upstream_count'
@@ -182,14 +185,40 @@ function RunArrow({
)
}
+function CadenceLabel({ node }: { node: LineageNodeShape }): JSX.Element {
+ const runsAt = node.effective_sync_interval ?? node.sync_interval ?? undefined
+ const differs = !node.pinned && !!runsAt && !!node.sync_interval && runsAt !== node.sync_interval
+ // The scheduler runs a node faster than its own setting to serve a downstream view,
+ // and slower when one of its sources refreshes less often than the node is set to.
+ const faster =
+ differs &&
+ SYNC_FREQUENCY_ORDER.indexOf(runsAt as (typeof SYNC_FREQUENCY_ORDER)[number]) <
+ SYNC_FREQUENCY_ORDER.indexOf(node.sync_interval as (typeof SYNC_FREQUENCY_ORDER)[number])
+ const title = !runsAt
+ ? 'This node is not set to sync on a schedule yet'
+ : node.pinned
+ ? 'Pinned to this cadence. Downstream views may read older data from it.'
+ : differs
+ ? faster
+ ? `Refreshes this often to keep up with a downstream view. Its own setting is ${syncIntervalToShorthand(node.sync_interval)}.`
+ : `Refreshes this often because a source it reads from refreshes less often. Its own setting is ${syncIntervalToShorthand(node.sync_interval)}.`
+ : null
+ return (
+ <Tooltip title={title}>
+ <span className="inline-flex items-center gap-0.5">
+ <span translate="no">{syncIntervalToShorthand(runsAt)}</span>
+ {node.pinned && <IconPinFilled className="text-[10px]" />}
+ </span>
+ </Tooltip>
+ )
+}
+
function MetadataBar({ node }: { node: LineageNodeShape }): JSX.Element {
return (
<div className="flex items-center bg-primary dark:bg-primary/60 rounded-b-lg px-2.5 py-1.5 justify-between">
<div className="flex gap-1 text-[10px] items-center">
<IconClockRewind className="scale-x-[-1]" />
- <Tooltip title={node.sync_interval ? null : 'This node is not set to sync on a schedule yet'}>
- {syncIntervalToShorthand(node.sync_interval)}
- </Tooltip>
+ <CadenceLabel node={node} />
<IconActivity />
{node.last_run_at ? (
<Tooltip title="Last successful run.">
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 6cb6044e1d68..128d54c3b96a 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -61180,6 +61180,11 @@ export namespace Schemas {
readonly user_tag: string | null;
/** @nullable */
readonly sync_interval: string | null;
+ /**
+ * How often this node actually refreshes. A downstream view can make an unpinned view refresh more often than `sync_interval`, and a slow source can make it refresh less often. Null when the node is not scheduled.
+ * @nullable
+ */
+ readonly effective_sync_interval: string | null;
/** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.
*
* * `never` - never
@@ -83700,6 +83705,11 @@ export namespace Schemas {
readonly user_tag?: string | null;
/** @nullable */
readonly sync_interval?: string | null;
+ /**
+ * How often this node actually refreshes. A downstream view can make an unpinned view refresh more often than `sync_interval`, and a slow source can make it refresh less often. Null when the node is not scheduled.
+ * @nullable
+ */
+ readonly effective_sync_interval?: string | null;
/** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.
*
* * `never` - never