Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-modeling): refresh downstream view columns when a view changes (#124802)

Thiago Salvatorefa62ec7f1907
diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py
index 100c31972e08..d454e5167f8b 100644
--- a/posthog/hogql/query.py
+++ b/posthog/hogql/query.py
@@ -85,6 +85,7 @@ from posthog.hogql.warehouse_warnings import record_warnings
from posthog.clickhouse.client import sync_execute
from posthog.clickhouse.client.connection import ClickHouseUser, Workload
+from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
from posthog.clickhouse.query_tagging import get_query_tag_value, get_query_tags, tag_queries
from posthog.dataclasses import frozen
from posthog.direct_query_cancellation import build_direct_query_cancellation_token
@@ -1096,12 +1097,14 @@ class HogQLQueryExecutor:
def _refreshed_stale_view_columns(self, started_at: datetime) -> bool:
view_ids = self.context.referenced_saved_query_ids
- if not self.refresh_stale_view_columns or not view_ids:
+ if not self.refresh_stale_view_columns or not view_ids or self.user is None:
return False
try:
if data_modeling_api.refresh_view_columns(self.team, view_ids, self.user):
return True
return data_modeling_api.views_updated_since(self.team, view_ids, started_at)
+ except ConcurrencyLimitExceeded:
+ return False
except Exception as e:
capture_exception(e, {"team_id": self.team.pk})
return False
diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.py
index 086cecb60265..c37db9f3f689 100644
--- a/products/data_modeling/backend/facade/api.py
+++ b/products/data_modeling/backend/facade/api.py
@@ -59,6 +59,7 @@ _LAZY = {
"upstream_saved_query_ids": "logic.saved_query_reads",
"refresh_view_columns": "logic.view_column_refresh",
"views_updated_since": "logic.view_column_refresh",
+ "schedule_downstream_view_column_refresh": "tasks.refresh_downstream_view_columns",
"unbuilt_upstream_saved_query_ids": "logic.saved_query_reads",
"last_query_change_at": "logic.saved_query_reads",
"completed_full_refresh_workflow_ids": "logic.saved_query_reads",
diff --git a/products/data_modeling/backend/facade/tasks.py b/products/data_modeling/backend/facade/tasks.py
index 74b11beb9e9c..66d8c7b61225 100644
--- a/products/data_modeling/backend/facade/tasks.py
+++ b/products/data_modeling/backend/facade/tasks.py
@@ -8,5 +8,11 @@ core's scheduler registers.
from products.data_modeling.backend.schedule import build_schedule_spec
from products.data_modeling.backend.tasks.cleanup_deleted_saved_query_nodes import cleanup_deleted_saved_query_nodes
from products.data_modeling.backend.tasks.cleanup_test_saved_queries import cleanup_expired_test_saved_queries
+from products.data_modeling.backend.tasks.refresh_downstream_view_columns import refresh_downstream_view_columns
-__all__ = ["build_schedule_spec", "cleanup_deleted_saved_query_nodes", "cleanup_expired_test_saved_queries"]
+__all__ = [
+ "build_schedule_spec",
+ "cleanup_deleted_saved_query_nodes",
+ "cleanup_expired_test_saved_queries",
+ "refresh_downstream_view_columns",
+]
diff --git a/products/data_modeling/backend/logic/view_column_refresh.py b/products/data_modeling/backend/logic/view_column_refresh.py
index 17fc27843cb1..245a259dcaf4 100644
--- a/products/data_modeling/backend/logic/view_column_refresh.py
+++ b/products/data_modeling/backend/logic/view_column_refresh.py
@@ -8,8 +8,17 @@ from django.utils import timezone
import structlog
+from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
+
from products.access_control.backend.facade.api import get_restricted_properties_with_group_type_index_for_team
-from products.data_modeling.backend.logic.saved_query_reads import dependent_saved_query_ids
+from products.access_control.backend.facade.user_access_control import UserAccessControl
+from products.data_modeling.backend.logic.saved_query_reads import (
+ allowed_saved_query_ids,
+ dependent_saved_query_ids,
+ get_saved_query_ids_for_nodes,
+ reachable_node_ids,
+ saved_query_node_ids,
+)
from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
if TYPE_CHECKING:
@@ -20,7 +29,10 @@ logger = structlog.get_logger(__name__)
def refresh_view_columns(team: "Team", saved_query_ids: Collection[UUID | str], user: "User | None") -> set[UUID]:
- """Re-infer the stored columns of these views, upstream first, and return the ids whose columns changed."""
+ """Re-infer the stored columns of these views, upstream first, and return the ids whose columns changed.
+
+ With no user, inference runs as the system and bypasses warehouse access control.
+ """
if _infers_masked_columns(team, user):
return set()
views = {
@@ -39,6 +51,21 @@ def views_updated_since(team: "Team", saved_query_ids: Collection[UUID | str], s
).exists()
+def refresh_downstream_view_columns(team: "Team", saved_query_id: UUID | str, user: "User | None") -> set[UUID]:
+ """Re-infer the stored columns of every view that reads from this one, directly or through other views."""
+ downstream_node_ids = reachable_node_ids(
+ team.pk, saved_query_node_ids(team.pk, saved_query_id), upstream=False, max_depth=None
+ )
+ downstream_ids = [UUID(view_id) for view_id in get_saved_query_ids_for_nodes(team.pk, downstream_node_ids)]
+ return refresh_view_columns(team, _readable_by(team, user, downstream_ids), user)
+
+
+def _readable_by(team: "Team", user: "User | None", view_ids: list[UUID]) -> Collection[UUID]:
+ if user is None:
+ return view_ids
+ return allowed_saved_query_ids(team.pk, UserAccessControl(user=user, team=team), ids=view_ids)
+
+
def _infers_masked_columns(team: "Team", user: "User | None") -> bool:
return bool(get_restricted_properties_with_group_type_index_for_team(user=user, team_id=team.pk))
@@ -52,8 +79,14 @@ def _upstream_first(team_id: int, view_ids: set[UUID]) -> list[UUID]:
def _refresh_columns(view: DataWarehouseSavedQuery, user: "User | None") -> bool:
+ # Inference uses the team's default modifiers, so a materialized upstream view is read from its backing table.
+ # A query with useMaterializedViews=False reads that view's SQL instead. When the SQL changed and the view has
+ # not materialized again, such a query can still fail on stale columns after this refresh. The refresh that
+ # runs when materialization publishes the new backing table fixes it.
try:
- columns = view.get_columns(user=user)
+ columns = view.get_columns(user=user, bypass_warehouse_access_control=user is None)
+ except ConcurrencyLimitExceeded:
+ raise
except Exception:
logger.warning("view_column_refresh_failed", team_id=view.team_id, saved_query_id=str(view.id), exc_info=True)
return False
diff --git a/products/data_modeling/backend/models/datawarehouse_saved_query.py b/products/data_modeling/backend/models/datawarehouse_saved_query.py
index df667add6d3c..4af6661826ed 100644
--- a/products/data_modeling/backend/models/datawarehouse_saved_query.py
+++ b/products/data_modeling/backend/models/datawarehouse_saved_query.py
@@ -516,7 +516,9 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,
"""Whether ``set_columns(columns)`` would leave this view's stored columns unchanged."""
return columns == self.columns and list(columns.keys()) == self.column_order
- def get_columns(self, user: Optional["User"] = None) -> dict[str, dict[str, Any]]:
+ def get_columns(
+ self, user: Optional["User"] = None, *, bypass_warehouse_access_control: bool = False
+ ) -> dict[str, dict[str, Any]]:
from posthog.hogql.query import execute_hogql_query # noqa: PLC0415 — keeps the heavy dep off the import path
query = self.query or {}
@@ -540,6 +542,7 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,
user=user,
query_type="HogQLQuery",
refresh_stale_view_columns=False,
+ bypass_warehouse_access_control=bypass_warehouse_access_control,
)
result = response.types
diff --git a/products/data_modeling/backend/tasks/refresh_downstream_view_columns.py b/products/data_modeling/backend/tasks/refresh_downstream_view_columns.py
new file mode 100644
index 000000000000..64b7988a4f54
--- /dev/null
+++ b/products/data_modeling/backend/tasks/refresh_downstream_view_columns.py
@@ -0,0 +1,29 @@
+from uuid import UUID
+
+from django.db import transaction
+
+from celery import shared_task
+
+from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
+from posthog.models.scoping import with_team_scope
+from posthog.models.team import Team
+from posthog.models.user import User
+
+from products.data_modeling.backend.logic import view_column_refresh
+
+
+@shared_task(ignore_result=True, autoretry_for=(ConcurrencyLimitExceeded,), retry_backoff=True, max_retries=3)
+@with_team_scope()
+def refresh_downstream_view_columns(team_id: int, saved_query_id: str, user_id: int | None) -> None:
+ team = Team.objects.get(pk=team_id)
+ user = User.objects.filter(pk=user_id).first() if user_id is not None else None
+ if user_id is not None and user is None:
+ return
+ view_column_refresh.refresh_downstream_view_columns(team, saved_query_id, user)
+
+
+def schedule_downstream_view_column_refresh(team_id: int, saved_query_id: UUID, user_id: int | None) -> None:
+ """Refresh the stored columns of the views downstream of this one once the current transaction commits."""
+ transaction.on_commit(
+ lambda: refresh_downstream_view_columns.delay(team_id, str(saved_query_id), user_id), robust=True
+ )
diff --git a/products/data_warehouse/backend/logic/data_load/create_table.py b/products/data_warehouse/backend/logic/data_load/create_table.py
index a9d8135b18b7..3478d3b61eaf 100644
--- a/products/data_warehouse/backend/logic/data_load/create_table.py
+++ b/products/data_warehouse/backend/logic/data_load/create_table.py
@@ -12,6 +12,7 @@ from posthog.exceptions_capture import capture_exception
from posthog.sync import database_sync_to_async, database_sync_to_async_pool
from posthog.temporal.common.logger import get_logger
+from products.data_modeling.backend.facade import api as modeling_api
from products.data_modeling.backend.facade.models import (
DataModelingJob,
DataWarehouseSavedQuery,
@@ -122,6 +123,7 @@ async def create_table_from_saved_query(
# TODO: handle dlt columns schemas. Need to refactor dag pipeline to pass through schema or propagate from upstream tables
# set_columns records the DESCRIBE column order (which follows the view's SELECT order for
# materialized backing tables) alongside `columns`, since jsonb loses key order.
+ published_columns = (table_created.columns, table_created.column_order)
table_created.set_columns(await sync_to_async(table_created.get_columns)())
table_created.row_count = await database_sync_to_async(table_created.get_count)()
@@ -155,6 +157,10 @@ async def create_table_from_saved_query(
await logger.adebug(str(e))
await asave_datawarehousetable(table_created)
+ if (table_created.columns, table_created.column_order) != published_columns:
+ await database_sync_to_async(modeling_api.schedule_downstream_view_column_refresh)(
+ team_id, saved_query.id, None
+ )
return CreateTableResult(
table=table_created,
diff --git a/products/data_warehouse/backend/logic/data_load/test/test_create_table.py b/products/data_warehouse/backend/logic/data_load/test/test_create_table.py
index 3423c30db91e..8508269b0de0 100644
--- a/products/data_warehouse/backend/logic/data_load/test/test_create_table.py
+++ b/products/data_warehouse/backend/logic/data_load/test/test_create_table.py
@@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, patch
from posthog.models import Team
from products.data_modeling.backend.facade.models import DataModelingJob, DataWarehouseSavedQuery
+from products.data_modeling.backend.facade.tasks import refresh_downstream_view_columns
from products.data_warehouse.backend.logic.data_load.create_table import create_table_from_saved_query
from products.warehouse_sources.backend.facade.models import DataWarehouseTable
@@ -116,6 +117,7 @@ async def test_materialization_replaces_table_metadata_in_one_snapshot(team: Tea
new_callable=AsyncMock,
return_value=20.0,
),
+ patch.object(refresh_downstream_view_columns, "delay") as refresh_delay,
):
await create_table_from_saved_query(str(job.id), str(saved_query.id), team.pk, "replacement-folder")
@@ -134,3 +136,4 @@ async def test_materialization_replaces_table_metadata_in_one_snapshot(team: Tea
assert table.row_count == 20
assert table.size_in_s3_mib == 20.0
assert table.created_via == DataWarehouseTable.CreatedVia.MATERIALIZED_VIEW
+ refresh_delay.assert_called_once_with(team.pk, str(saved_query.id), None)
diff --git a/products/data_warehouse/backend/presentation/views/saved_query/editing.py b/products/data_warehouse/backend/presentation/views/saved_query/editing.py
index 3a96dc3e711e..962d8a5efabd 100644
--- a/products/data_warehouse/backend/presentation/views/saved_query/editing.py
+++ b/products/data_warehouse/backend/presentation/views/saved_query/editing.py
@@ -568,10 +568,15 @@ class DataWarehouseSavedQuerySerializer(
self._write_view_description(view, description)
if inferred_columns is not None:
+ columns_changed = not before_update.stores_columns(inferred_columns)
view.set_columns(inferred_columns)
view.external_tables = inferred_external_tables
view.status = DataWarehouseSavedQuery.Status.MODIFIED
view.save()
+ if columns_changed:
+ modeling_api.schedule_downstream_view_column_refresh(
+ view.team_id, view.id, self.context["request"].user.id
+ )
team = Team.objects.get(id=view.team_id)
diff --git a/products/data_warehouse/backend/tests/api/test_saved_query.py b/products/data_warehouse/backend/tests/api/test_saved_query.py
index 45e72540691d..aaeadc5ecd92 100644
--- a/products/data_warehouse/backend/tests/api/test_saved_query.py
+++ b/products/data_warehouse/backend/tests/api/test_saved_query.py
@@ -23,10 +23,13 @@ from temporalio.service import RPCError, RPCStatusCode
from posthog.hogql.errors import ExposedHogQLError
+from posthog.constants import AvailableFeature
from posthog.errors import CHQueryErrorS3Error, wrap_clickhouse_query_error
from posthog.models import ActivityLog
+from posthog.models.organization import OrganizationMembership
from posthog.models.scoping import team_scope
+from products.access_control.backend.models.access_control import AccessControl
from products.data_modeling.backend.facade.api import UnsatisfiableFrequencyError, mark_node_suspended, suspension_state
from products.data_modeling.backend.facade.models import (
DAG,
@@ -39,6 +42,7 @@ from products.data_modeling.backend.facade.models import (
Node,
NodeType,
)
+from products.data_modeling.backend.facade.tasks import refresh_downstream_view_columns
from products.data_tools.backend.facade.models import DataWarehouseJoin
from products.data_tools.backend.models.datawarehouse_saved_query_folder import DataWarehouseSavedQueryFolder
from products.data_warehouse.backend.presentation.views.saved_query.editing import (
@@ -1915,6 +1919,79 @@ class TestSavedQuery(APIBaseTest):
# Verify get_columns was called
mock_get_columns.assert_called_once()
+ @parameterized.expand(
+ [
+ ("renamed_column", "select 1 as new_id", ["new_id"], True),
+ ("unchanged_columns", "select 2 as old_id", ["old_id"], False),
+ ]
+ )
+ def test_update_refreshes_the_columns_of_downstream_views(
+ self, _name: str, upstream_sql: str, expected_downstream_columns: list[str], refreshes: bool
+ ) -> None:
+ upstream = self._create_view("upstream_view", "select 1 as old_id")
+ self._create_view("downstream_view", "select * from upstream_view")
+
+ with (
+ patch.object(
+ refresh_downstream_view_columns, "delay", wraps=refresh_downstream_view_columns.delay
+ ) as delay,
+ self.captureOnCommitCallbacks(execute=True),
+ ):
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/warehouse_saved_queries/{upstream['id']}",
+ {
+ "query": {"kind": "HogQLQuery", "query": upstream_sql},
+ "edited_history_id": upstream["latest_history_id"],
+ },
+ )
+
+ self.assertEqual(response.status_code, 200, response.content)
+ downstream = DataWarehouseSavedQuery.objects.get(team=self.team, name="downstream_view")
+ assert downstream.column_order == expected_downstream_columns
+ assert delay.called == refreshes
+
+ @parameterized.expand([("editor_exists", False), ("editor_deleted_before_refresh", True)])
+ def test_update_skips_downstream_views_the_editor_cannot_read(self, _name: str, delete_editor: bool) -> None:
+ self.organization.available_product_features = [
+ {"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}
+ ]
+ self.organization.save(update_fields=["available_product_features"])
+ upstream = self._create_view("upstream_view", "select 1 as old_id")
+ downstream = self._create_view("downstream_view", "select * from upstream_view")
+ editor = self._create_user("[email protected]")
+ AccessControl.objects.create(
+ team=self.team,
+ resource="warehouse_view",
+ resource_id=downstream["id"],
+ organization_member=OrganizationMembership.objects.get(user=editor, organization=self.organization),
+ access_level="none",
+ )
+ self.client.force_login(editor)
+
+ with self.captureOnCommitCallbacks() as on_commit_callbacks:
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/warehouse_saved_queries/{upstream['id']}",
+ {
+ "query": {"kind": "HogQLQuery", "query": "select 1 as new_id"},
+ "edited_history_id": upstream["latest_history_id"],
+ },
+ )
+ if delete_editor:
+ editor.delete()
+ for callback in on_commit_callbacks:
+ callback()
+
+ self.assertEqual(response.status_code, 200, response.content)
+ assert DataWarehouseSavedQuery.objects.get(pk=downstream["id"]).column_order == ["old_id"]
+
+ def _create_view(self, name: str, sql: str) -> dict[str, Any]:
+ response = self.client.post(
+ f"/api/environments/{self.team.id}/warehouse_saved_queries/",
+ {"name": name, "query": {"kind": "HogQLQuery", "query": sql}},
+ )
+ self.assertEqual(response.status_code, 201, response.content)
+ return response.json()
+
@parameterized.expand(
[
("direct", False, False),