:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-modeling): refresh downstream view columns when a view changes (#124802)
Thiago Salvatorefa62ec7f1907
diff --git a/posthog/hogql/query.py b/posthog/hogql/query.pyindex 100c31972e08..d454e5167f8b 100644--- a/posthog/hogql/query.py+++ b/posthog/hogql/query.py@@ -85,6 +85,7 @@ from posthog.hogql.warehouse_warnings import record_warningsfrom posthog.clickhouse.client import sync_executefrom posthog.clickhouse.client.connection import ClickHouseUser, Workload+from posthog.clickhouse.client.limit import ConcurrencyLimitExceededfrom posthog.clickhouse.query_tagging import get_query_tag_value, get_query_tags, tag_queriesfrom posthog.dataclasses import frozenfrom posthog.direct_query_cancellation import build_direct_query_cancellation_token@@ -1096,12 +1097,14 @@ class HogQLQueryExecutor:def _refreshed_stale_view_columns(self, started_at: datetime) -> bool:view_ids = self.context.referenced_saved_query_ids- if not self.refresh_stale_view_columns or not view_ids:+ if not self.refresh_stale_view_columns or not view_ids or self.user is None:return Falsetry:if data_modeling_api.refresh_view_columns(self.team, view_ids, self.user):return Truereturn data_modeling_api.views_updated_since(self.team, view_ids, started_at)+ except ConcurrencyLimitExceeded:+ return Falseexcept Exception as e:capture_exception(e, {"team_id": self.team.pk})return Falsediff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.pyindex 086cecb60265..c37db9f3f689 100644--- a/products/data_modeling/backend/facade/api.py+++ b/products/data_modeling/backend/facade/api.py@@ -59,6 +59,7 @@ _LAZY = {"upstream_saved_query_ids": "logic.saved_query_reads","refresh_view_columns": "logic.view_column_refresh","views_updated_since": "logic.view_column_refresh",+ "schedule_downstream_view_column_refresh": "tasks.refresh_downstream_view_columns","unbuilt_upstream_saved_query_ids": "logic.saved_query_reads","last_query_change_at": "logic.saved_query_reads","completed_full_refresh_workflow_ids": "logic.saved_query_reads",diff --git a/products/data_modeling/backend/facade/tasks.py b/products/data_modeling/backend/facade/tasks.pyindex 74b11beb9e9c..66d8c7b61225 100644--- a/products/data_modeling/backend/facade/tasks.py+++ b/products/data_modeling/backend/facade/tasks.py@@ -8,5 +8,11 @@ core's scheduler registers.from products.data_modeling.backend.schedule import build_schedule_specfrom products.data_modeling.backend.tasks.cleanup_deleted_saved_query_nodes import cleanup_deleted_saved_query_nodesfrom products.data_modeling.backend.tasks.cleanup_test_saved_queries import cleanup_expired_test_saved_queries+from products.data_modeling.backend.tasks.refresh_downstream_view_columns import refresh_downstream_view_columns-__all__ = ["build_schedule_spec", "cleanup_deleted_saved_query_nodes", "cleanup_expired_test_saved_queries"]+__all__ = [+ "build_schedule_spec",+ "cleanup_deleted_saved_query_nodes",+ "cleanup_expired_test_saved_queries",+ "refresh_downstream_view_columns",+]diff --git a/products/data_modeling/backend/logic/view_column_refresh.py b/products/data_modeling/backend/logic/view_column_refresh.pyindex 17fc27843cb1..245a259dcaf4 100644--- a/products/data_modeling/backend/logic/view_column_refresh.py+++ b/products/data_modeling/backend/logic/view_column_refresh.py@@ -8,8 +8,17 @@ from django.utils import timezoneimport structlog+from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded+from products.access_control.backend.facade.api import get_restricted_properties_with_group_type_index_for_team-from products.data_modeling.backend.logic.saved_query_reads import dependent_saved_query_ids+from products.access_control.backend.facade.user_access_control import UserAccessControl+from products.data_modeling.backend.logic.saved_query_reads import (+ allowed_saved_query_ids,+ dependent_saved_query_ids,+ get_saved_query_ids_for_nodes,+ reachable_node_ids,+ saved_query_node_ids,+)from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQueryif TYPE_CHECKING:@@ -20,7 +29,10 @@ logger = structlog.get_logger(__name__)def refresh_view_columns(team: "Team", saved_query_ids: Collection[UUID | str], user: "User | None") -> set[UUID]:- """Re-infer the stored columns of these views, upstream first, and return the ids whose columns changed."""+ """Re-infer the stored columns of these views, upstream first, and return the ids whose columns changed.++ With no user, inference runs as the system and bypasses warehouse access control.+ """if _infers_masked_columns(team, user):return set()views = {@@ -39,6 +51,21 @@ def views_updated_since(team: "Team", saved_query_ids: Collection[UUID | str], s).exists()+def refresh_downstream_view_columns(team: "Team", saved_query_id: UUID | str, user: "User | None") -> set[UUID]:+ """Re-infer the stored columns of every view that reads from this one, directly or through other views."""+ downstream_node_ids = reachable_node_ids(+ team.pk, saved_query_node_ids(team.pk, saved_query_id), upstream=False, max_depth=None+ )+ downstream_ids = [UUID(view_id) for view_id in get_saved_query_ids_for_nodes(team.pk, downstream_node_ids)]+ return refresh_view_columns(team, _readable_by(team, user, downstream_ids), user)+++def _readable_by(team: "Team", user: "User | None", view_ids: list[UUID]) -> Collection[UUID]:+ if user is None:+ return view_ids+ return allowed_saved_query_ids(team.pk, UserAccessControl(user=user, team=team), ids=view_ids)++def _infers_masked_columns(team: "Team", user: "User | None") -> bool:return bool(get_restricted_properties_with_group_type_index_for_team(user=user, team_id=team.pk))@@ -52,8 +79,14 @@ def _upstream_first(team_id: int, view_ids: set[UUID]) -> list[UUID]:def _refresh_columns(view: DataWarehouseSavedQuery, user: "User | None") -> bool:+ # Inference uses the team's default modifiers, so a materialized upstream view is read from its backing table.+ # A query with useMaterializedViews=False reads that view's SQL instead. When the SQL changed and the view has+ # not materialized again, such a query can still fail on stale columns after this refresh. The refresh that+ # runs when materialization publishes the new backing table fixes it.try:- columns = view.get_columns(user=user)+ columns = view.get_columns(user=user, bypass_warehouse_access_control=user is None)+ except ConcurrencyLimitExceeded:+ raiseexcept Exception:logger.warning("view_column_refresh_failed", team_id=view.team_id, saved_query_id=str(view.id), exc_info=True)return Falsediff --git a/products/data_modeling/backend/models/datawarehouse_saved_query.py b/products/data_modeling/backend/models/datawarehouse_saved_query.pyindex df667add6d3c..4af6661826ed 100644--- a/products/data_modeling/backend/models/datawarehouse_saved_query.py+++ b/products/data_modeling/backend/models/datawarehouse_saved_query.py@@ -516,7 +516,9 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,"""Whether ``set_columns(columns)`` would leave this view's stored columns unchanged."""return columns == self.columns and list(columns.keys()) == self.column_order- def get_columns(self, user: Optional["User"] = None) -> dict[str, dict[str, Any]]:+ def get_columns(+ self, user: Optional["User"] = None, *, bypass_warehouse_access_control: bool = False+ ) -> dict[str, dict[str, Any]]:from posthog.hogql.query import execute_hogql_query # noqa: PLC0415 — keeps the heavy dep off the import pathquery = self.query or {}@@ -540,6 +542,7 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,user=user,query_type="HogQLQuery",refresh_stale_view_columns=False,+ bypass_warehouse_access_control=bypass_warehouse_access_control,)result = response.typesdiff --git a/products/data_modeling/backend/tasks/refresh_downstream_view_columns.py b/products/data_modeling/backend/tasks/refresh_downstream_view_columns.pynew file mode 100644index 000000000000..64b7988a4f54--- /dev/null+++ b/products/data_modeling/backend/tasks/refresh_downstream_view_columns.py@@ -0,0 +1,29 @@+from uuid import UUID++from django.db import transaction++from celery import shared_task++from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded+from posthog.models.scoping import with_team_scope+from posthog.models.team import Team+from posthog.models.user import User++from products.data_modeling.backend.logic import view_column_refresh+++@shared_task(ignore_result=True, autoretry_for=(ConcurrencyLimitExceeded,), retry_backoff=True, max_retries=3)+@with_team_scope()+def refresh_downstream_view_columns(team_id: int, saved_query_id: str, user_id: int | None) -> None:+ team = Team.objects.get(pk=team_id)+ user = User.objects.filter(pk=user_id).first() if user_id is not None else None+ if user_id is not None and user is None:+ return+ view_column_refresh.refresh_downstream_view_columns(team, saved_query_id, user)+++def schedule_downstream_view_column_refresh(team_id: int, saved_query_id: UUID, user_id: int | None) -> None:+ """Refresh the stored columns of the views downstream of this one once the current transaction commits."""+ transaction.on_commit(+ lambda: refresh_downstream_view_columns.delay(team_id, str(saved_query_id), user_id), robust=True+ )diff --git a/products/data_warehouse/backend/logic/data_load/create_table.py b/products/data_warehouse/backend/logic/data_load/create_table.pyindex a9d8135b18b7..3478d3b61eaf 100644--- a/products/data_warehouse/backend/logic/data_load/create_table.py+++ b/products/data_warehouse/backend/logic/data_load/create_table.py@@ -12,6 +12,7 @@ from posthog.exceptions_capture import capture_exceptionfrom posthog.sync import database_sync_to_async, database_sync_to_async_poolfrom posthog.temporal.common.logger import get_logger+from products.data_modeling.backend.facade import api as modeling_apifrom products.data_modeling.backend.facade.models import (DataModelingJob,DataWarehouseSavedQuery,@@ -122,6 +123,7 @@ async def create_table_from_saved_query(# TODO: handle dlt columns schemas. Need to refactor dag pipeline to pass through schema or propagate from upstream tables# set_columns records the DESCRIBE column order (which follows the view's SELECT order for# materialized backing tables) alongside `columns`, since jsonb loses key order.+ published_columns = (table_created.columns, table_created.column_order)table_created.set_columns(await sync_to_async(table_created.get_columns)())table_created.row_count = await database_sync_to_async(table_created.get_count)()@@ -155,6 +157,10 @@ async def create_table_from_saved_query(await logger.adebug(str(e))await asave_datawarehousetable(table_created)+ if (table_created.columns, table_created.column_order) != published_columns:+ await database_sync_to_async(modeling_api.schedule_downstream_view_column_refresh)(+ team_id, saved_query.id, None+ )return CreateTableResult(table=table_created,diff --git a/products/data_warehouse/backend/logic/data_load/test/test_create_table.py b/products/data_warehouse/backend/logic/data_load/test/test_create_table.pyindex 3423c30db91e..8508269b0de0 100644--- a/products/data_warehouse/backend/logic/data_load/test/test_create_table.py+++ b/products/data_warehouse/backend/logic/data_load/test/test_create_table.py@@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, patchfrom posthog.models import Teamfrom products.data_modeling.backend.facade.models import DataModelingJob, DataWarehouseSavedQuery+from products.data_modeling.backend.facade.tasks import refresh_downstream_view_columnsfrom products.data_warehouse.backend.logic.data_load.create_table import create_table_from_saved_queryfrom products.warehouse_sources.backend.facade.models import DataWarehouseTable@@ -116,6 +117,7 @@ async def test_materialization_replaces_table_metadata_in_one_snapshot(team: Teanew_callable=AsyncMock,return_value=20.0,),+ patch.object(refresh_downstream_view_columns, "delay") as refresh_delay,):await create_table_from_saved_query(str(job.id), str(saved_query.id), team.pk, "replacement-folder")@@ -134,3 +136,4 @@ async def test_materialization_replaces_table_metadata_in_one_snapshot(team: Teaassert table.row_count == 20assert table.size_in_s3_mib == 20.0assert table.created_via == DataWarehouseTable.CreatedVia.MATERIALIZED_VIEW+ refresh_delay.assert_called_once_with(team.pk, str(saved_query.id), None)diff --git a/products/data_warehouse/backend/presentation/views/saved_query/editing.py b/products/data_warehouse/backend/presentation/views/saved_query/editing.pyindex 3a96dc3e711e..962d8a5efabd 100644--- a/products/data_warehouse/backend/presentation/views/saved_query/editing.py+++ b/products/data_warehouse/backend/presentation/views/saved_query/editing.py@@ -568,10 +568,15 @@ class DataWarehouseSavedQuerySerializer(self._write_view_description(view, description)if inferred_columns is not None:+ columns_changed = not before_update.stores_columns(inferred_columns)view.set_columns(inferred_columns)view.external_tables = inferred_external_tablesview.status = DataWarehouseSavedQuery.Status.MODIFIEDview.save()+ if columns_changed:+ modeling_api.schedule_downstream_view_column_refresh(+ view.team_id, view.id, self.context["request"].user.id+ )team = Team.objects.get(id=view.team_id)diff --git a/products/data_warehouse/backend/tests/api/test_saved_query.py b/products/data_warehouse/backend/tests/api/test_saved_query.pyindex 45e72540691d..aaeadc5ecd92 100644--- a/products/data_warehouse/backend/tests/api/test_saved_query.py+++ b/products/data_warehouse/backend/tests/api/test_saved_query.py@@ -23,10 +23,13 @@ from temporalio.service import RPCError, RPCStatusCodefrom posthog.hogql.errors import ExposedHogQLError+from posthog.constants import AvailableFeaturefrom posthog.errors import CHQueryErrorS3Error, wrap_clickhouse_query_errorfrom posthog.models import ActivityLog+from posthog.models.organization import OrganizationMembershipfrom posthog.models.scoping import team_scope+from products.access_control.backend.models.access_control import AccessControlfrom products.data_modeling.backend.facade.api import UnsatisfiableFrequencyError, mark_node_suspended, suspension_statefrom products.data_modeling.backend.facade.models import (DAG,@@ -39,6 +42,7 @@ from products.data_modeling.backend.facade.models import (Node,NodeType,)+from products.data_modeling.backend.facade.tasks import refresh_downstream_view_columnsfrom products.data_tools.backend.facade.models import DataWarehouseJoinfrom products.data_tools.backend.models.datawarehouse_saved_query_folder import DataWarehouseSavedQueryFolderfrom products.data_warehouse.backend.presentation.views.saved_query.editing import (@@ -1915,6 +1919,79 @@ class TestSavedQuery(APIBaseTest):# Verify get_columns was calledmock_get_columns.assert_called_once()+ @parameterized.expand(+ [+ ("renamed_column", "select 1 as new_id", ["new_id"], True),+ ("unchanged_columns", "select 2 as old_id", ["old_id"], False),+ ]+ )+ def test_update_refreshes_the_columns_of_downstream_views(+ self, _name: str, upstream_sql: str, expected_downstream_columns: list[str], refreshes: bool+ ) -> None:+ upstream = self._create_view("upstream_view", "select 1 as old_id")+ self._create_view("downstream_view", "select * from upstream_view")++ with (+ patch.object(+ refresh_downstream_view_columns, "delay", wraps=refresh_downstream_view_columns.delay+ ) as delay,+ self.captureOnCommitCallbacks(execute=True),+ ):+ response = self.client.patch(+ f"/api/environments/{self.team.id}/warehouse_saved_queries/{upstream['id']}",+ {+ "query": {"kind": "HogQLQuery", "query": upstream_sql},+ "edited_history_id": upstream["latest_history_id"],+ },+ )++ self.assertEqual(response.status_code, 200, response.content)+ downstream = DataWarehouseSavedQuery.objects.get(team=self.team, name="downstream_view")+ assert downstream.column_order == expected_downstream_columns+ assert delay.called == refreshes++ @parameterized.expand([("editor_exists", False), ("editor_deleted_before_refresh", True)])+ def test_update_skips_downstream_views_the_editor_cannot_read(self, _name: str, delete_editor: bool) -> None:+ self.organization.available_product_features = [+ {"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}+ ]+ self.organization.save(update_fields=["available_product_features"])+ upstream = self._create_view("upstream_view", "select 1 as old_id")+ downstream = self._create_view("downstream_view", "select * from upstream_view")+ editor = self._create_user("[email protected]")+ AccessControl.objects.create(+ team=self.team,+ resource="warehouse_view",+ resource_id=downstream["id"],+ organization_member=OrganizationMembership.objects.get(user=editor, organization=self.organization),+ access_level="none",+ )+ self.client.force_login(editor)++ with self.captureOnCommitCallbacks() as on_commit_callbacks:+ response = self.client.patch(+ f"/api/environments/{self.team.id}/warehouse_saved_queries/{upstream['id']}",+ {+ "query": {"kind": "HogQLQuery", "query": "select 1 as new_id"},+ "edited_history_id": upstream["latest_history_id"],+ },+ )+ if delete_editor:+ editor.delete()+ for callback in on_commit_callbacks:+ callback()++ self.assertEqual(response.status_code, 200, response.content)+ assert DataWarehouseSavedQuery.objects.get(pk=downstream["id"]).column_order == ["old_id"]++ def _create_view(self, name: str, sql: str) -> dict[str, Any]:+ response = self.client.post(+ f"/api/environments/{self.team.id}/warehouse_saved_queries/",+ {"name": name, "query": {"kind": "HogQLQuery", "query": sql}},+ )+ self.assertEqual(response.status_code, 201, response.content)+ return response.json()+@parameterized.expand([("direct", False, False),