:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): refresh stale view columns and retry on unknown identifier (#124800)
Thiago Salvatoreb3dc0b2d1e4d
diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.pyindex 1813d214f56b..8eb82b4f6191 100644--- a/posthog/hogql/ast.py+++ b/posthog/hogql/ast.py@@ -24,7 +24,7 @@ from posthog.hogql.database.models import (UnknownDatabaseField,VirtualTable,)-from posthog.hogql.errors import NotImplementedError, QueryError, ResolutionError+from posthog.hogql.errors import FIELD_NOT_FOUND_MESSAGE, NotImplementedError, QueryError, ResolutionError# :NOTE: when you add new AST fields or nodes, add them to CloningVisitor and TraversingVisitor in visitor.py as well.# :NOTE2: also search for ":TRICKY:" in "resolver.py" when modifying SelectQuery or JoinExpr@@ -212,7 +212,7 @@ class FieldAliasType(Type):raise NotImplementedError("FieldAliasType.resolve_table_type not implemented")-@dataclass(kw_only=True, slots=True)+@dataclass(kw_only=True, slots=True, frozen=False)class BaseTableType(Type):def resolve_database_table(self, context: HogQLContext) -> Table:raise NotImplementedError("BaseTableType.resolve_database_table not overridden")@@ -239,7 +239,7 @@ class BaseTableType(Type):)return FieldType(name=name, table_type=self)- raise QueryError(f"Field not found: {name}")+ raise QueryError(f"{FIELD_NOT_FOUND_MESSAGE}: {name}")TableOrSelectType = Union[BaseTableType, "SelectSetQueryType", "SelectQueryType", "SelectQueryAliasType"]diff --git a/posthog/hogql/errors.py b/posthog/hogql/errors.pyindex f6f61a3c9c12..7fad9fcd9fad 100644--- a/posthog/hogql/errors.py+++ b/posthog/hogql/errors.py@@ -4,6 +4,9 @@ from typing import TYPE_CHECKING, Optionalif TYPE_CHECKING:from .ast import Expr+UNRESOLVED_FIELD_MESSAGE = "Unable to resolve field"+FIELD_NOT_FOUND_MESSAGE = "Field not found"+# Basediff --git a/posthog/hogql/query.py b/posthog/hogql/query.pyindex 22bfd1eac9cf..62e5cd1a0be7 100644--- a/posthog/hogql/query.py+++ b/posthog/hogql/query.py@@ -1,7 +1,10 @@import dataclasses+from datetime import datetimefrom time import sleepfrom typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union, cast+from django.utils import timezone+from opentelemetry import tracefrom posthog.schema import (@@ -53,7 +56,15 @@ from posthog.hogql.direct_sql import (get_adapter_for_source,get_raw_adapter_for_source,)-from posthog.hogql.errors import BaseHogQLError, ExposedHogQLError, InternalHogQLError, QueryError, ResolutionError+from posthog.hogql.errors import (+ FIELD_NOT_FOUND_MESSAGE,+ UNRESOLVED_FIELD_MESSAGE,+ BaseHogQLError,+ ExposedHogQLError,+ InternalHogQLError,+ QueryError,+ ResolutionError,+)from posthog.hogql.feature_extractor import extract_hogql_featuresfrom posthog.hogql.filters import replace_filtersfrom posthog.hogql.functions.prompt_jev import PromptJevFinder@@ -76,13 +87,20 @@ from posthog.clickhouse.client.connection import ClickHouseUser, Workloadfrom posthog.clickhouse.query_tagging import get_query_tag_value, get_query_tags, tag_queriesfrom posthog.dataclasses import frozenfrom posthog.direct_query_cancellation import build_direct_query_cancellation_token-from posthog.errors import CHQueryErrorS3Error, CHQueryErrorS3FileChangedDuringRead, ExposedCHQueryError+from posthog.errors import (+ CHQueryErrorS3Error,+ CHQueryErrorS3FileChangedDuringRead,+ CHQueryErrorUnknownIdentifier,+ ExposedCHQueryError,+)+from posthog.exceptions_capture import capture_exceptionfrom posthog.models.event.new_events_schema import use_new_events_schemafrom posthog.models.team import Teamfrom posthog.models.user import Userfrom posthog.settings import HOGQL_INCREASED_MAX_EXECUTION_TIMEfrom products.access_control.backend.facade.user_access_control import UserAccessControl+from products.data_modeling.backend.facade import api as data_modeling_apifrom products.warehouse_sources.backend.facade.types import ManagedWarehouseSQLModeif TYPE_CHECKING:@@ -144,6 +162,7 @@ class HogQLQueryExecutor:bypass_warehouse_access_control: bool = Falseuser_access_control: Optional[UserAccessControl] = Noneresolver_factory: Optional[ResolverFactory] = None+ refresh_stale_view_columns: bool = True__uninitialized_context: ClassVar[HogQLContext] = HogQLContext()@@ -1063,6 +1082,38 @@ class HogQLQueryExecutor:@tracer.start_as_current_span("HogQLQueryExecutor.execute")def execute(self) -> HogQLQueryResponse:+ query = self.query+ started_at = timezone.now()+ try:+ return self._execute()+ except (CHQueryErrorUnknownIdentifier, QueryError) as error:+ if not _names_a_missing_column(error) or not self._refreshed_stale_view_columns(started_at):+ raise+ return self._with_fresh_database(query).execute()++ def _refreshed_stale_view_columns(self, started_at: datetime) -> bool:+ view_ids = self.context.referenced_saved_query_ids+ if not self.refresh_stale_view_columns or not view_ids:+ return False+ try:+ if data_modeling_api.refresh_view_columns(self.team, view_ids, self.user):+ return True+ return data_modeling_api.views_updated_since(self.team, view_ids, started_at)+ except Exception as e:+ capture_exception(e, {"team_id": self.team.pk})+ return False++ def _with_fresh_database(+ self, query: Union[str, ast.SelectQuery, ast.SelectSetQuery] | None+ ) -> "HogQLQueryExecutor":+ return dataclasses.replace(+ self,+ query=query,+ context=dataclasses.replace(self.context, database=None),+ refresh_stale_view_columns=False,+ )++ def _execute(self) -> HogQLQueryResponse:self._executing = Truetrace.get_current_span().set_attribute("team_id", self.team.pk)try:@@ -1135,5 +1186,12 @@ class HogQLQueryExecutor:]+_MISSING_COLUMN_MESSAGES = (UNRESOLVED_FIELD_MESSAGE, FIELD_NOT_FOUND_MESSAGE)+++def _names_a_missing_column(error: Exception) -> bool:+ return isinstance(error, CHQueryErrorUnknownIdentifier) or str(error).startswith(_MISSING_COLUMN_MESSAGES)++def execute_hogql_query(*args, **kwargs) -> HogQLQueryResponse:return HogQLQueryExecutor(*args, **kwargs).execute()diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.pyindex 96e3620b0268..04c7c9ea07d2 100644--- a/posthog/hogql/resolver.py+++ b/posthog/hogql/resolver.py@@ -30,7 +30,13 @@ from posthog.hogql.database.schema.log_entries import ()from posthog.hogql.database.schema.persons import PersonsTablefrom posthog.hogql.database.trino_unnest_table import resolve_internal_trino_table_function-from posthog.hogql.errors import ImpossibleASTError, NotImplementedError, QueryError, ResolutionError+from posthog.hogql.errors import (+ UNRESOLVED_FIELD_MESSAGE,+ ImpossibleASTError,+ NotImplementedError,+ QueryError,+ ResolutionError,+)from posthog.hogql.escape_sql import safe_identifierfrom posthog.hogql.functions import find_hogql_posthog_functionfrom posthog.hogql.functions.action import matches_action@@ -2543,7 +2549,7 @@ class Resolver(CloningVisitor):## One likely cause is that the database context isn't set up as you# expect it to be.- raise QueryError(f"Unable to resolve field: {name}{suggestion_suffix}", node=node, fix=fix)+ raise QueryError(f"{UNRESOLVED_FIELD_MESSAGE}: {name}{suggestion_suffix}", node=node, fix=fix)else:type = ast.UnresolvedFieldType(name=name)self.context.add_error(diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.pyindex 7438acad3411..76a6e42165e7 100644--- a/products/data_modeling/backend/facade/api.py+++ b/products/data_modeling/backend/facade/api.py@@ -56,6 +56,8 @@ _LAZY = {"saved_query_node_ids": "logic.saved_query_reads","upstream_table_refs": "logic.saved_query_reads","upstream_saved_query_ids": "logic.saved_query_reads",+ "refresh_view_columns": "logic.view_column_refresh",+ "views_updated_since": "logic.view_column_refresh","unbuilt_upstream_saved_query_ids": "logic.saved_query_reads","last_query_change_at": "logic.saved_query_reads","completed_full_refresh_workflow_ids": "logic.saved_query_reads",diff --git a/products/data_modeling/backend/logic/view_column_refresh.py b/products/data_modeling/backend/logic/view_column_refresh.pynew file mode 100644index 000000000000..17fc27843cb1--- /dev/null+++ b/products/data_modeling/backend/logic/view_column_refresh.py@@ -0,0 +1,72 @@+from collections.abc import Collection+from datetime import datetime+from graphlib import TopologicalSorter+from typing import TYPE_CHECKING+from uuid import UUID++from django.utils import timezone++import structlog++from products.access_control.backend.facade.api import get_restricted_properties_with_group_type_index_for_team+from products.data_modeling.backend.logic.saved_query_reads import dependent_saved_query_ids+from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery++if TYPE_CHECKING:+ from posthog.models.team import Team+ from posthog.models.user import User++logger = structlog.get_logger(__name__)+++def refresh_view_columns(team: "Team", saved_query_ids: Collection[UUID | str], user: "User | None") -> set[UUID]:+ """Re-infer the stored columns of these views, upstream first, and return the ids whose columns changed."""+ if _infers_masked_columns(team, user):+ return set()+ views = {+ view.id: view+ for view in DataWarehouseSavedQuery.objects.select_related("team")+ .filter(team_id=team.pk, id__in=list(saved_query_ids), managed_viewset__isnull=True)+ .exclude(deleted=True)+ }+ return {view_id for view_id in _upstream_first(team.pk, set(views)) if _refresh_columns(views[view_id], user)}+++def views_updated_since(team: "Team", saved_query_ids: Collection[UUID | str], since: datetime) -> bool:+ """Whether any of these views was written after ``since``, for example by another request's refresh."""+ return DataWarehouseSavedQuery.objects.filter(+ team_id=team.pk, id__in=list(saved_query_ids), updated_at__gt=since+ ).exists()+++def _infers_masked_columns(team: "Team", user: "User | None") -> bool:+ return bool(get_restricted_properties_with_group_type_index_for_team(user=user, team_id=team.pk))+++def _upstream_first(team_id: int, view_ids: set[UUID]) -> list[UUID]:+ sorter: TopologicalSorter[UUID] = TopologicalSorter({view_id: set() for view_id in view_ids})+ for source_id, target_ids in dependent_saved_query_ids(team_id, view_ids).items():+ for target_id in target_ids & view_ids:+ sorter.add(target_id, source_id)+ return list(sorter.static_order())+++def _refresh_columns(view: DataWarehouseSavedQuery, user: "User | None") -> bool:+ try:+ columns = view.get_columns(user=user)+ except Exception:+ logger.warning("view_column_refresh_failed", team_id=view.team_id, saved_query_id=str(view.id), exc_info=True)+ return False++ if view.stores_columns(columns):+ return False++ view.set_columns(columns)+ return _save_columns_unless_query_changed(view)+++def _save_columns_unless_query_changed(view: DataWarehouseSavedQuery) -> bool:+ saved_rows = DataWarehouseSavedQuery.objects.filter(+ team_id=view.team_id, pk=view.pk, query_revision=view.query_revision+ ).update(columns=view.columns, column_order=view.column_order, updated_at=timezone.now())+ return saved_rows == 1diff --git a/products/data_modeling/backend/models/datawarehouse_saved_query.py b/products/data_modeling/backend/models/datawarehouse_saved_query.pyindex c739df5a93d5..df667add6d3c 100644--- a/products/data_modeling/backend/models/datawarehouse_saved_query.py+++ b/products/data_modeling/backend/models/datawarehouse_saved_query.py@@ -512,6 +512,10 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,self.columns = columnsself.column_order = list(columns.keys())+ def stores_columns(self, columns: dict[str, Any]) -> bool:+ """Whether ``set_columns(columns)`` would leave this view's stored columns unchanged."""+ return columns == self.columns and list(columns.keys()) == self.column_order+def get_columns(self, user: Optional["User"] = None) -> dict[str, dict[str, Any]]:from posthog.hogql.query import execute_hogql_query # noqa: PLC0415 — keeps the heavy dep off the import path@@ -530,7 +534,13 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,tag_contains_user_hogql()# Printing the AST back to HogQL text would drop COLUMNS(...) and table column alias lists.with app_org_concurrency_slot(self.team):- response = execute_hogql_query(select_query, team=self.team, user=user, query_type="HogQLQuery")+ response = execute_hogql_query(+ select_query,+ team=self.team,+ user=user,+ query_type="HogQLQuery",+ refresh_stale_view_columns=False,+ )result = response.typesif result is None:diff --git a/products/data_modeling/backend/test/test_view_column_refresh.py b/products/data_modeling/backend/test/test_view_column_refresh.pynew file mode 100644index 000000000000..80b199d5d278--- /dev/null+++ b/products/data_modeling/backend/test/test_view_column_refresh.py@@ -0,0 +1,106 @@+from uuid import uuid4++import pytest+from posthog.test.base import BaseTest, ClickhouseTestMixin+from unittest.mock import patch++from parameterized import parameterized++from posthog.hogql.errors import QueryError+from posthog.hogql.property_access_types import RestrictedProperty+from posthog.hogql.query import execute_hogql_query++from posthog.models import PropertyDefinition++from products.data_modeling.backend.logic import view_column_refresh+from products.data_modeling.backend.logic.saved_query_dag_sync import sync_saved_query_to_dag+from products.data_modeling.backend.logic.view_column_refresh import refresh_view_columns+from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery+++class TestViewColumnRefresh(ClickhouseTestMixin, BaseTest):+ def _create_view(self, name: str, sql: str) -> DataWarehouseSavedQuery:+ view = DataWarehouseSavedQuery(team=self.team, name=name, query={"kind": "HogQLQuery", "query": sql})+ view.set_columns(view.get_columns(user=self.user))+ view.save()+ sync_saved_query_to_dag(view, reconcile=False)+ return view++ def _edit_view(self, view: DataWarehouseSavedQuery, sql: str) -> None:+ view.query = {"kind": "HogQLQuery", "query": sql}+ view.set_columns(view.get_columns(user=self.user))+ view.save()++ @parameterized.expand(+ [+ ("select_star", "SELECT * FROM downstream_view"),+ ("select_renamed_column", "SELECT new_id FROM downstream_view"),+ ("select_qualified_renamed_column", "SELECT v.new_id FROM downstream_view AS v"),+ ]+ )+ def test_query_over_views_whose_upstream_renamed_a_column_refreshes_their_columns(+ self, _name: str, sql: str+ ) -> None:+ upstream = self._create_view("upstream_view", "SELECT 1 AS old_id")+ middle = self._create_view("middle_view", "SELECT * FROM upstream_view")+ downstream = self._create_view("downstream_view", "SELECT * FROM middle_view")+ self._edit_view(upstream, "SELECT 1 AS new_id")++ response = execute_hogql_query(sql, team=self.team, user=self.user)++ assert response.columns == ["new_id"]+ assert response.results == [(1,)]+ for view in (middle, downstream):+ view.refresh_from_db()+ assert view.column_order == ["new_id"]++ def test_query_retries_when_another_request_already_refreshed_the_view(self) -> None:+ upstream = self._create_view("upstream_view", "SELECT 1 AS old_id")+ downstream = self._create_view("downstream_view", "SELECT * FROM upstream_view")+ self._edit_view(upstream, "SELECT 1 AS new_id")++ def refreshed_by_another_request(team, saved_query_ids, user):+ refresh_view_columns(team, saved_query_ids, user)+ return set()++ with patch.object(view_column_refresh, "refresh_view_columns", side_effect=refreshed_by_another_request):+ response = execute_hogql_query("SELECT * FROM downstream_view", team=self.team, user=self.user)++ assert response.columns == ["new_id"]+ downstream.refresh_from_db()+ assert downstream.column_order == ["new_id"]++ def test_query_by_a_property_restricted_reader_leaves_stale_columns_untouched(self) -> None:+ upstream = self._create_view("upstream_view", "SELECT 1 AS old_id")+ downstream = self._create_view("downstream_view", "SELECT * FROM upstream_view")+ self._edit_view(upstream, "SELECT 1 AS new_id")++ with (+ patch(+ "products.data_modeling.backend.logic.view_column_refresh.get_restricted_properties_with_group_type_index_for_team",+ return_value={RestrictedProperty(name="$email", property_type=PropertyDefinition.Type.EVENT)},+ ),+ pytest.raises(QueryError),+ ):+ execute_hogql_query("SELECT new_id FROM downstream_view", team=self.team, user=self.user)++ downstream.refresh_from_db()+ assert downstream.column_order == ["old_id"]++ def test_refresh_keeps_the_columns_of_a_query_edit_made_during_inference(self) -> None:+ view = self._create_view("edited_view", "SELECT 1 AS old_id")+ edited_columns = {"edited_id": {"hogql": "IntegerDatabaseField", "clickhouse": "UInt8", "valid": True}}+ inferred_columns = {"new_id": {"hogql": "IntegerDatabaseField", "clickhouse": "UInt8", "valid": True}}++ def edit_the_view_during_inference(*args: object, **kwargs: object) -> dict:+ DataWarehouseSavedQuery.objects.filter(pk=view.pk).update(+ query_revision=uuid4(), columns=edited_columns, column_order=["edited_id"]+ )+ return inferred_columns++ with patch.object(DataWarehouseSavedQuery, "get_columns", side_effect=edit_the_view_during_inference):+ changed = refresh_view_columns(self.team, [view.id], self.user)++ assert changed == set()+ view.refresh_from_db()+ assert view.column_order == ["edited_id"]