:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): show the cadence a view runs at on lineage cards (#129143)
Jovan Sakovicd267fb85a646
diff --git a/frontend/src/types.ts b/frontend/src/types.tsindex 912b0ee02007..b4a5457249c3 100644--- a/frontend/src/types.ts+++ b/frontend/src/types.ts@@ -6269,6 +6269,7 @@ export interface DataModelingNode {last_run_status?: DataModelingJobStatuslast_run_error?: string | nullsync_interval?: DataModelingSyncInterval+ effective_sync_interval?: DataModelingSyncInterval | nullpinned?: booleansuspended?: NodeApiSuspended/** Set on endpoint nodes stamped at materialization enable; older nodes carry only the name */diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.pyindex 4046087cfa03..be358a0e74f9 100644--- a/products/data_modeling/backend/facade/api.py+++ b/products/data_modeling/backend/facade/api.py@@ -99,6 +99,7 @@ _LAZY = {"saved_query_target_bounds": "logic.node_frequency","SavedQueryFrequencyBounds": "logic.node_frequency","saved_query_pin_preview": "logic.node_frequency",+ "scheduled_cadences_by_node": "logic.node_frequency","cadence_changes": "logic.cadence_changes","clear_legacy_sync_interval": "logic.cadence_changes","clear_node_suspension": "logic.node_suspension",diff --git a/products/data_modeling/backend/logic/node_frequency.py b/products/data_modeling/backend/logic/node_frequency.pyindex c242efde709b..185f390426b8 100644--- a/products/data_modeling/backend/logic/node_frequency.py+++ b/products/data_modeling/backend/logic/node_frequency.py@@ -619,6 +619,14 @@ def _unpinned_cadence_by_cadence(return result+def scheduled_cadences_by_node(team_id: int, dag_ids: Iterable[uuid.UUID | str]) -> dict[str, timedelta | None]:+ """The cadence the scheduler runs each node of these DAGs at, keyed by node id."""+ result: dict[str, timedelta | None] = {}+ for dag in DAG.objects.filter(team_id=team_id, id__in=set(dag_ids)):+ result.update(scheduled_cadences(build_frequency_graph(dag))[0])+ return result++def scheduled_cadences(graph: FrequencyGraph,*,diff --git a/products/data_modeling/backend/presentation/views/node.py b/products/data_modeling/backend/presentation/views/node.pyindex 9e8ba2c41aa9..05bb88db369d 100644--- a/products/data_modeling/backend/presentation/views/node.py+++ b/products/data_modeling/backend/presentation/views/node.py@@ -36,6 +36,7 @@ from products.data_modeling.backend.facade.api import (run_related_node_materializations,saved_query_pin_preview,saved_query_target_bounds,+ scheduled_cadences_by_node,suspension_state,unsuspend_nodes,)@@ -194,6 +195,12 @@ class NodeSerializer(serializers.ModelSerializer):last_run_error = serializers.SerializerMethodField(read_only=True)user_tag = serializers.SerializerMethodField(read_only=True)sync_interval = serializers.SerializerMethodField(read_only=True)+ effective_sync_interval = serializers.SerializerMethodField(+ read_only=True,+ help_text="How often this node actually refreshes. A downstream view can make an unpinned view "+ "refresh more often than `sync_interval`, and a slow source can make it refresh less often. "+ "Null when the node is not scheduled.",+ )lineage_issue = serializers.SerializerMethodField(read_only=True)origin = serializers.SerializerMethodField(read_only=True)warehouse_table_id = serializers.SerializerMethodField(read_only=True)@@ -219,6 +226,7 @@ class NodeSerializer(serializers.ModelSerializer):"last_run_error","user_tag","sync_interval",+ "effective_sync_interval","sync_frequency","pinned","suspended",@@ -305,6 +313,14 @@ class NodeSerializer(serializers.ModelSerializer):return sync_frequency_interval_to_sync_frequency(node.saved_query.sync_frequency_interval)return None+ def get_effective_sync_interval(self, node: Node) -> str | None:+ # Built once per DAG for the whole response, not once per listed node.+ by_dag = self.context.setdefault("_scheduled_cadences_by_dag", {})+ if node.dag_id not in by_dag:+ by_dag[node.dag_id] = scheduled_cadences_by_node(node.team_id, [node.dag_id])+ cadence = by_dag[node.dag_id].get(str(node.id))+ return sync_frequency_interval_to_sync_frequency(cadence) if cadence is not None else None+@extend_schema_field(LineageIssueSerializer(allow_null=True))def get_lineage_issue(self, node: Node) -> dict[str, Any] | None:return node.lineage_issuediff --git a/products/data_modeling/backend/tests/api/test_node_api.py b/products/data_modeling/backend/tests/api/test_node_api.pyindex 2ec4f8720e6b..67d827ebb389 100644--- a/products/data_modeling/backend/tests/api/test_node_api.py+++ b/products/data_modeling/backend/tests/api/test_node_api.py@@ -19,6 +19,7 @@ from posthog.models.activity_logging.activity_log import ActivityLogfrom products.data_modeling.backend.logic.node_frequency import (get_declared_pin,get_declared_target,+ set_declared_pin,set_declared_target,)from products.data_modeling.backend.logic.node_materialization import start_node_materialization@@ -838,6 +839,18 @@ class TestNodeViewSet(APIBaseTest):(get_declared_target(self.view_node), get_declared_pin(self.view_node)), (timedelta(days=1), False))+ @parameterized.expand([("unpinned", False, "6hour"), ("pinned", True, "24hour")])+ def test_list_reports_the_cadence_a_view_runs_at(self, _name, pinned, expected):+ self._give_the_view_a_consumer(timedelta(hours=6), reads_the_table_too=True)+ set_declared_target(self.view_node, timedelta(days=1))+ set_declared_pin(self.view_node, pinned)++ response = self.client.get(f"/api/environments/{self.team.id}/data_modeling_nodes/")++ self.assertEqual(response.status_code, status.HTTP_200_OK)+ listed = next(node for node in response.json()["results"] if node["id"] == str(self.view_node.id))+ self.assertEqual((listed["sync_interval"], listed["effective_sync_interval"]), ("24hour", expected))+def test_a_cadence_write_clears_the_legacy_saved_query_interval(self):DataWarehouseSavedQuery.objects.filter(pk=self.saved_query.pk).update(sync_frequency_interval=timedelta(hours=12)diff --git a/products/data_modeling/frontend/freshness.test.ts b/products/data_modeling/frontend/freshness.test.tsindex 35814c75d064..1b81441f3167 100644--- a/products/data_modeling/frontend/freshness.test.ts+++ b/products/data_modeling/frontend/freshness.test.ts@@ -41,6 +41,16 @@ describe('modelsBehindSchedule', () => {expect(modelsBehindSchedule([stale], NOW)).toEqual([])})+ it('measures a model against the cadence it runs at, not its own setting', () => {+ const inherited = node('inherited', {+ sync_interval: '24hour',+ effective_sync_interval: '1hour',+ last_run_at: hoursAgo(3),+ })++ expect(modelsBehindSchedule([inherited], NOW).map((model) => model.intervalSeconds)).toEqual([3600])+ })+it('measures a model that has never finished a run from when it was created', () => {const never = node('never', { last_run_at: null, created_at: hoursAgo(10) })diff --git a/products/data_modeling/frontend/freshness.ts b/products/data_modeling/frontend/freshness.tsindex bad97bbdc8b5..388caf13531c 100644--- a/products/data_modeling/frontend/freshness.ts+++ b/products/data_modeling/frontend/freshness.ts@@ -37,10 +37,12 @@ export function modelsBehindSchedule(nodes: DataModelingNode[], now: number): Beconst behind: BehindScheduleModel[] = []for (const node of nodes) {- if (!node.sync_interval || !MATERIALIZING_TYPES.has(node.type)) {+ // A downstream view can make a model run more often than its own setting.+ const runsAt = node.sync_interval ? (node.effective_sync_interval ?? node.sync_interval) : undefined+ if (!runsAt || !MATERIALIZING_TYPES.has(node.type)) {continue}- const intervalSeconds = SYNC_INTERVAL_SECONDS[node.sync_interval]+ const intervalSeconds = SYNC_INTERVAL_SECONDS[runsAt]if (!intervalSeconds) {continue}diff --git a/products/data_modeling/frontend/generated/api.schemas.ts b/products/data_modeling/frontend/generated/api.schemas.tsindex e1d1064875f3..3d6557bb836f 100644--- a/products/data_modeling/frontend/generated/api.schemas.ts+++ b/products/data_modeling/frontend/generated/api.schemas.ts@@ -193,6 +193,11 @@ export interface NodeApi {readonly user_tag: string | null/** @nullable */readonly sync_interval: string | null+ /**+ * How often this node actually refreshes. A downstream view can make an unpinned view refresh more often than `sync_interval`, and a slow source can make it refresh less often. Null when the node is not scheduled.+ * @nullable+ */+ readonly effective_sync_interval: string | null/** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.** * `never` - never@@ -271,6 +276,11 @@ export interface PatchedNodeApi {readonly user_tag?: string | null/** @nullable */readonly sync_interval?: string | null+ /**+ * How often this node actually refreshes. A downstream view can make an unpinned view refresh more often than `sync_interval`, and a slow source can make it refresh less often. Null when the node is not scheduled.+ * @nullable+ */+ readonly effective_sync_interval?: string | null/** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.** * `never` - neverdiff --git a/products/data_modeling/frontend/lineage/LineageNode.tsx b/products/data_modeling/frontend/lineage/LineageNode.tsxindex 80cacc6bc826..43fffbde05a1 100644--- a/products/data_modeling/frontend/lineage/LineageNode.tsx+++ b/products/data_modeling/frontend/lineage/LineageNode.tsx@@ -9,6 +9,7 @@ import {IconExternal,IconPauseFilled,IconPencil,+ IconPinFilled,IconPlay,IconPlayFilled,IconTarget,@@ -32,7 +33,7 @@ import { DataModelingNode } from '~/types'import { MATERIALIZING_TYPES } from 'products/data_modeling/frontend/freshness'import { servingSuspension } from 'products/data_modeling/frontend/suspension'-import { syncIntervalToShorthand } from 'products/data_warehouse/frontend/utils'+import { SYNC_FREQUENCY_ORDER, syncIntervalToShorthand } from 'products/data_warehouse/frontend/utils'import { ElkDirection, NodeHandle } from './autolayout'import { LineageGraphNode } from './lineageHiddenNodes'@@ -48,6 +49,8 @@ export type LineageNodeShape = Pick<| 'name'| 'type'| 'sync_interval'+ | 'effective_sync_interval'+ | 'pinned'| 'last_run_at'| 'last_run_status'| 'upstream_count'@@ -182,14 +185,40 @@ function RunArrow({)}+function CadenceLabel({ node }: { node: LineageNodeShape }): JSX.Element {+ const runsAt = node.effective_sync_interval ?? node.sync_interval ?? undefined+ const differs = !node.pinned && !!runsAt && !!node.sync_interval && runsAt !== node.sync_interval+ // The scheduler runs a node faster than its own setting to serve a downstream view,+ // and slower when one of its sources refreshes less often than the node is set to.+ const faster =+ differs &&+ SYNC_FREQUENCY_ORDER.indexOf(runsAt as (typeof SYNC_FREQUENCY_ORDER)[number]) <+ SYNC_FREQUENCY_ORDER.indexOf(node.sync_interval as (typeof SYNC_FREQUENCY_ORDER)[number])+ const title = !runsAt+ ? 'This node is not set to sync on a schedule yet'+ : node.pinned+ ? 'Pinned to this cadence. Downstream views may read older data from it.'+ : differs+ ? faster+ ? `Refreshes this often to keep up with a downstream view. Its own setting is ${syncIntervalToShorthand(node.sync_interval)}.`+ : `Refreshes this often because a source it reads from refreshes less often. Its own setting is ${syncIntervalToShorthand(node.sync_interval)}.`+ : null+ return (+ <Tooltip title={title}>+ <span className="inline-flex items-center gap-0.5">+ <span translate="no">{syncIntervalToShorthand(runsAt)}</span>+ {node.pinned && <IconPinFilled className="text-[10px]" />}+ </span>+ </Tooltip>+ )+}+function MetadataBar({ node }: { node: LineageNodeShape }): JSX.Element {return (<div className="flex items-center bg-primary dark:bg-primary/60 rounded-b-lg px-2.5 py-1.5 justify-between"><div className="flex gap-1 text-[10px] items-center"><IconClockRewind className="scale-x-[-1]" />- <Tooltip title={node.sync_interval ? null : 'This node is not set to sync on a schedule yet'}>- {syncIntervalToShorthand(node.sync_interval)}- </Tooltip>+ <CadenceLabel node={node} /><IconActivity />{node.last_run_at ? (<Tooltip title="Last successful run.">diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 6cb6044e1d68..128d54c3b96a 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -61180,6 +61180,11 @@ export namespace Schemas {readonly user_tag: string | null;/** @nullable */readonly sync_interval: string | null;+ /**+ * How often this node actually refreshes. A downstream view can make an unpinned view refresh more often than `sync_interval`, and a slow source can make it refresh less often. Null when the node is not scheduled.+ * @nullable+ */+ readonly effective_sync_interval: string | null;/** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.** * `never` - never@@ -83700,6 +83705,11 @@ export namespace Schemas {readonly user_tag?: string | null;/** @nullable */readonly sync_interval?: string | null;+ /**+ * How often this node actually refreshes. A downstream view can make an unpinned view refresh more often than `sync_interval`, and a slow source can make it refresh less often. Null when the node is not scheduled.+ * @nullable+ */+ readonly effective_sync_interval?: string | null;/** How often this view refreshes. 'never' clears the cadence, and with it the pin. Read the stored cadence back from `sync_interval`. Only views and endpoints the team owns accept it.** * `never` - never