Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): refresh stale view columns and retry on unknown identifier (#124800)

Thiago Salvatoreb3dc0b2d1e4d
diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py
index 1813d214f56b..8eb82b4f6191 100644
--- a/posthog/hogql/ast.py
+++ b/posthog/hogql/ast.py
@@ -24,7 +24,7 @@ from posthog.hogql.database.models import (
UnknownDatabaseField,
VirtualTable,
)
-from posthog.hogql.errors import NotImplementedError, QueryError, ResolutionError
+from posthog.hogql.errors import FIELD_NOT_FOUND_MESSAGE, NotImplementedError, QueryError, ResolutionError
# :NOTE: when you add new AST fields or nodes, add them to CloningVisitor and TraversingVisitor in visitor.py as well.
# :NOTE2: also search for ":TRICKY:" in "resolver.py" when modifying SelectQuery or JoinExpr
@@ -212,7 +212,7 @@ class FieldAliasType(Type):
raise NotImplementedError("FieldAliasType.resolve_table_type not implemented")
-@dataclass(kw_only=True, slots=True)
+@dataclass(kw_only=True, slots=True, frozen=False)
class BaseTableType(Type):
def resolve_database_table(self, context: HogQLContext) -> Table:
raise NotImplementedError("BaseTableType.resolve_database_table not overridden")
@@ -239,7 +239,7 @@ class BaseTableType(Type):
)
return FieldType(name=name, table_type=self)
- raise QueryError(f"Field not found: {name}")
+ raise QueryError(f"{FIELD_NOT_FOUND_MESSAGE}: {name}")
TableOrSelectType = Union[BaseTableType, "SelectSetQueryType", "SelectQueryType", "SelectQueryAliasType"]
diff --git a/posthog/hogql/errors.py b/posthog/hogql/errors.py
index f6f61a3c9c12..7fad9fcd9fad 100644
--- a/posthog/hogql/errors.py
+++ b/posthog/hogql/errors.py
@@ -4,6 +4,9 @@ from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING:
from .ast import Expr
+UNRESOLVED_FIELD_MESSAGE = "Unable to resolve field"
+FIELD_NOT_FOUND_MESSAGE = "Field not found"
+
# Base
diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py
index 22bfd1eac9cf..62e5cd1a0be7 100644
--- a/posthog/hogql/query.py
+++ b/posthog/hogql/query.py
@@ -1,7 +1,10 @@
import dataclasses
+from datetime import datetime
from time import sleep
from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union, cast
+from django.utils import timezone
+
from opentelemetry import trace
from posthog.schema import (
@@ -53,7 +56,15 @@ from posthog.hogql.direct_sql import (
get_adapter_for_source,
get_raw_adapter_for_source,
)
-from posthog.hogql.errors import BaseHogQLError, ExposedHogQLError, InternalHogQLError, QueryError, ResolutionError
+from posthog.hogql.errors import (
+ FIELD_NOT_FOUND_MESSAGE,
+ UNRESOLVED_FIELD_MESSAGE,
+ BaseHogQLError,
+ ExposedHogQLError,
+ InternalHogQLError,
+ QueryError,
+ ResolutionError,
+)
from posthog.hogql.feature_extractor import extract_hogql_features
from posthog.hogql.filters import replace_filters
from posthog.hogql.functions.prompt_jev import PromptJevFinder
@@ -76,13 +87,20 @@ from posthog.clickhouse.client.connection import ClickHouseUser, Workload
from posthog.clickhouse.query_tagging import get_query_tag_value, get_query_tags, tag_queries
from posthog.dataclasses import frozen
from posthog.direct_query_cancellation import build_direct_query_cancellation_token
-from posthog.errors import CHQueryErrorS3Error, CHQueryErrorS3FileChangedDuringRead, ExposedCHQueryError
+from posthog.errors import (
+ CHQueryErrorS3Error,
+ CHQueryErrorS3FileChangedDuringRead,
+ CHQueryErrorUnknownIdentifier,
+ ExposedCHQueryError,
+)
+from posthog.exceptions_capture import capture_exception
from posthog.models.event.new_events_schema import use_new_events_schema
from posthog.models.team import Team
from posthog.models.user import User
from posthog.settings import HOGQL_INCREASED_MAX_EXECUTION_TIME
from products.access_control.backend.facade.user_access_control import UserAccessControl
+from products.data_modeling.backend.facade import api as data_modeling_api
from products.warehouse_sources.backend.facade.types import ManagedWarehouseSQLMode
if TYPE_CHECKING:
@@ -144,6 +162,7 @@ class HogQLQueryExecutor:
bypass_warehouse_access_control: bool = False
user_access_control: Optional[UserAccessControl] = None
resolver_factory: Optional[ResolverFactory] = None
+ refresh_stale_view_columns: bool = True
__uninitialized_context: ClassVar[HogQLContext] = HogQLContext()
@@ -1063,6 +1082,38 @@ class HogQLQueryExecutor:
@tracer.start_as_current_span("HogQLQueryExecutor.execute")
def execute(self) -> HogQLQueryResponse:
+ query = self.query
+ started_at = timezone.now()
+ try:
+ return self._execute()
+ except (CHQueryErrorUnknownIdentifier, QueryError) as error:
+ if not _names_a_missing_column(error) or not self._refreshed_stale_view_columns(started_at):
+ raise
+ return self._with_fresh_database(query).execute()
+
+ def _refreshed_stale_view_columns(self, started_at: datetime) -> bool:
+ view_ids = self.context.referenced_saved_query_ids
+ if not self.refresh_stale_view_columns or not view_ids:
+ return False
+ try:
+ if data_modeling_api.refresh_view_columns(self.team, view_ids, self.user):
+ return True
+ return data_modeling_api.views_updated_since(self.team, view_ids, started_at)
+ except Exception as e:
+ capture_exception(e, {"team_id": self.team.pk})
+ return False
+
+ def _with_fresh_database(
+ self, query: Union[str, ast.SelectQuery, ast.SelectSetQuery] | None
+ ) -> "HogQLQueryExecutor":
+ return dataclasses.replace(
+ self,
+ query=query,
+ context=dataclasses.replace(self.context, database=None),
+ refresh_stale_view_columns=False,
+ )
+
+ def _execute(self) -> HogQLQueryResponse:
self._executing = True
trace.get_current_span().set_attribute("team_id", self.team.pk)
try:
@@ -1135,5 +1186,12 @@ class HogQLQueryExecutor:
]
+_MISSING_COLUMN_MESSAGES = (UNRESOLVED_FIELD_MESSAGE, FIELD_NOT_FOUND_MESSAGE)
+
+
+def _names_a_missing_column(error: Exception) -> bool:
+ return isinstance(error, CHQueryErrorUnknownIdentifier) or str(error).startswith(_MISSING_COLUMN_MESSAGES)
+
+
def execute_hogql_query(*args, **kwargs) -> HogQLQueryResponse:
return HogQLQueryExecutor(*args, **kwargs).execute()
diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py
index 96e3620b0268..04c7c9ea07d2 100644
--- a/posthog/hogql/resolver.py
+++ b/posthog/hogql/resolver.py
@@ -30,7 +30,13 @@ from posthog.hogql.database.schema.log_entries import (
)
from posthog.hogql.database.schema.persons import PersonsTable
from posthog.hogql.database.trino_unnest_table import resolve_internal_trino_table_function
-from posthog.hogql.errors import ImpossibleASTError, NotImplementedError, QueryError, ResolutionError
+from posthog.hogql.errors import (
+ UNRESOLVED_FIELD_MESSAGE,
+ ImpossibleASTError,
+ NotImplementedError,
+ QueryError,
+ ResolutionError,
+)
from posthog.hogql.escape_sql import safe_identifier
from posthog.hogql.functions import find_hogql_posthog_function
from posthog.hogql.functions.action import matches_action
@@ -2543,7 +2549,7 @@ class Resolver(CloningVisitor):
#
# One likely cause is that the database context isn't set up as you
# expect it to be.
- raise QueryError(f"Unable to resolve field: {name}{suggestion_suffix}", node=node, fix=fix)
+ raise QueryError(f"{UNRESOLVED_FIELD_MESSAGE}: {name}{suggestion_suffix}", node=node, fix=fix)
else:
type = ast.UnresolvedFieldType(name=name)
self.context.add_error(
diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.py
index 7438acad3411..76a6e42165e7 100644
--- a/products/data_modeling/backend/facade/api.py
+++ b/products/data_modeling/backend/facade/api.py
@@ -56,6 +56,8 @@ _LAZY = {
"saved_query_node_ids": "logic.saved_query_reads",
"upstream_table_refs": "logic.saved_query_reads",
"upstream_saved_query_ids": "logic.saved_query_reads",
+ "refresh_view_columns": "logic.view_column_refresh",
+ "views_updated_since": "logic.view_column_refresh",
"unbuilt_upstream_saved_query_ids": "logic.saved_query_reads",
"last_query_change_at": "logic.saved_query_reads",
"completed_full_refresh_workflow_ids": "logic.saved_query_reads",
diff --git a/products/data_modeling/backend/logic/view_column_refresh.py b/products/data_modeling/backend/logic/view_column_refresh.py
new file mode 100644
index 000000000000..17fc27843cb1
--- /dev/null
+++ b/products/data_modeling/backend/logic/view_column_refresh.py
@@ -0,0 +1,72 @@
+from collections.abc import Collection
+from datetime import datetime
+from graphlib import TopologicalSorter
+from typing import TYPE_CHECKING
+from uuid import UUID
+
+from django.utils import timezone
+
+import structlog
+
+from products.access_control.backend.facade.api import get_restricted_properties_with_group_type_index_for_team
+from products.data_modeling.backend.logic.saved_query_reads import dependent_saved_query_ids
+from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
+
+if TYPE_CHECKING:
+ from posthog.models.team import Team
+ from posthog.models.user import User
+
+logger = structlog.get_logger(__name__)
+
+
+def refresh_view_columns(team: "Team", saved_query_ids: Collection[UUID | str], user: "User | None") -> set[UUID]:
+ """Re-infer the stored columns of these views, upstream first, and return the ids whose columns changed."""
+ if _infers_masked_columns(team, user):
+ return set()
+ views = {
+ view.id: view
+ for view in DataWarehouseSavedQuery.objects.select_related("team")
+ .filter(team_id=team.pk, id__in=list(saved_query_ids), managed_viewset__isnull=True)
+ .exclude(deleted=True)
+ }
+ return {view_id for view_id in _upstream_first(team.pk, set(views)) if _refresh_columns(views[view_id], user)}
+
+
+def views_updated_since(team: "Team", saved_query_ids: Collection[UUID | str], since: datetime) -> bool:
+ """Whether any of these views was written after ``since``, for example by another request's refresh."""
+ return DataWarehouseSavedQuery.objects.filter(
+ team_id=team.pk, id__in=list(saved_query_ids), updated_at__gt=since
+ ).exists()
+
+
+def _infers_masked_columns(team: "Team", user: "User | None") -> bool:
+ return bool(get_restricted_properties_with_group_type_index_for_team(user=user, team_id=team.pk))
+
+
+def _upstream_first(team_id: int, view_ids: set[UUID]) -> list[UUID]:
+ sorter: TopologicalSorter[UUID] = TopologicalSorter({view_id: set() for view_id in view_ids})
+ for source_id, target_ids in dependent_saved_query_ids(team_id, view_ids).items():
+ for target_id in target_ids & view_ids:
+ sorter.add(target_id, source_id)
+ return list(sorter.static_order())
+
+
+def _refresh_columns(view: DataWarehouseSavedQuery, user: "User | None") -> bool:
+ try:
+ columns = view.get_columns(user=user)
+ except Exception:
+ logger.warning("view_column_refresh_failed", team_id=view.team_id, saved_query_id=str(view.id), exc_info=True)
+ return False
+
+ if view.stores_columns(columns):
+ return False
+
+ view.set_columns(columns)
+ return _save_columns_unless_query_changed(view)
+
+
+def _save_columns_unless_query_changed(view: DataWarehouseSavedQuery) -> bool:
+ saved_rows = DataWarehouseSavedQuery.objects.filter(
+ team_id=view.team_id, pk=view.pk, query_revision=view.query_revision
+ ).update(columns=view.columns, column_order=view.column_order, updated_at=timezone.now())
+ return saved_rows == 1
diff --git a/products/data_modeling/backend/models/datawarehouse_saved_query.py b/products/data_modeling/backend/models/datawarehouse_saved_query.py
index c739df5a93d5..df667add6d3c 100644
--- a/products/data_modeling/backend/models/datawarehouse_saved_query.py
+++ b/products/data_modeling/backend/models/datawarehouse_saved_query.py
@@ -512,6 +512,10 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,
self.columns = columns
self.column_order = list(columns.keys())
+ def stores_columns(self, columns: dict[str, Any]) -> bool:
+ """Whether ``set_columns(columns)`` would leave this view's stored columns unchanged."""
+ return columns == self.columns and list(columns.keys()) == self.column_order
+
def get_columns(self, user: Optional["User"] = None) -> dict[str, dict[str, Any]]:
from posthog.hogql.query import execute_hogql_query # noqa: PLC0415 — keeps the heavy dep off the import path
@@ -530,7 +534,13 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,
tag_contains_user_hogql()
# Printing the AST back to HogQL text would drop COLUMNS(...) and table column alias lists.
with app_org_concurrency_slot(self.team):
- response = execute_hogql_query(select_query, team=self.team, user=user, query_type="HogQLQuery")
+ response = execute_hogql_query(
+ select_query,
+ team=self.team,
+ user=user,
+ query_type="HogQLQuery",
+ refresh_stale_view_columns=False,
+ )
result = response.types
if result is None:
diff --git a/products/data_modeling/backend/test/test_view_column_refresh.py b/products/data_modeling/backend/test/test_view_column_refresh.py
new file mode 100644
index 000000000000..80b199d5d278
--- /dev/null
+++ b/products/data_modeling/backend/test/test_view_column_refresh.py
@@ -0,0 +1,106 @@
+from uuid import uuid4
+
+import pytest
+from posthog.test.base import BaseTest, ClickhouseTestMixin
+from unittest.mock import patch
+
+from parameterized import parameterized
+
+from posthog.hogql.errors import QueryError
+from posthog.hogql.property_access_types import RestrictedProperty
+from posthog.hogql.query import execute_hogql_query
+
+from posthog.models import PropertyDefinition
+
+from products.data_modeling.backend.logic import view_column_refresh
+from products.data_modeling.backend.logic.saved_query_dag_sync import sync_saved_query_to_dag
+from products.data_modeling.backend.logic.view_column_refresh import refresh_view_columns
+from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
+
+
+class TestViewColumnRefresh(ClickhouseTestMixin, BaseTest):
+ def _create_view(self, name: str, sql: str) -> DataWarehouseSavedQuery:
+ view = DataWarehouseSavedQuery(team=self.team, name=name, query={"kind": "HogQLQuery", "query": sql})
+ view.set_columns(view.get_columns(user=self.user))
+ view.save()
+ sync_saved_query_to_dag(view, reconcile=False)
+ return view
+
+ def _edit_view(self, view: DataWarehouseSavedQuery, sql: str) -> None:
+ view.query = {"kind": "HogQLQuery", "query": sql}
+ view.set_columns(view.get_columns(user=self.user))
+ view.save()
+
+ @parameterized.expand(
+ [
+ ("select_star", "SELECT * FROM downstream_view"),
+ ("select_renamed_column", "SELECT new_id FROM downstream_view"),
+ ("select_qualified_renamed_column", "SELECT v.new_id FROM downstream_view AS v"),
+ ]
+ )
+ def test_query_over_views_whose_upstream_renamed_a_column_refreshes_their_columns(
+ self, _name: str, sql: str
+ ) -> None:
+ upstream = self._create_view("upstream_view", "SELECT 1 AS old_id")
+ middle = self._create_view("middle_view", "SELECT * FROM upstream_view")
+ downstream = self._create_view("downstream_view", "SELECT * FROM middle_view")
+ self._edit_view(upstream, "SELECT 1 AS new_id")
+
+ response = execute_hogql_query(sql, team=self.team, user=self.user)
+
+ assert response.columns == ["new_id"]
+ assert response.results == [(1,)]
+ for view in (middle, downstream):
+ view.refresh_from_db()
+ assert view.column_order == ["new_id"]
+
+ def test_query_retries_when_another_request_already_refreshed_the_view(self) -> None:
+ upstream = self._create_view("upstream_view", "SELECT 1 AS old_id")
+ downstream = self._create_view("downstream_view", "SELECT * FROM upstream_view")
+ self._edit_view(upstream, "SELECT 1 AS new_id")
+
+ def refreshed_by_another_request(team, saved_query_ids, user):
+ refresh_view_columns(team, saved_query_ids, user)
+ return set()
+
+ with patch.object(view_column_refresh, "refresh_view_columns", side_effect=refreshed_by_another_request):
+ response = execute_hogql_query("SELECT * FROM downstream_view", team=self.team, user=self.user)
+
+ assert response.columns == ["new_id"]
+ downstream.refresh_from_db()
+ assert downstream.column_order == ["new_id"]
+
+ def test_query_by_a_property_restricted_reader_leaves_stale_columns_untouched(self) -> None:
+ upstream = self._create_view("upstream_view", "SELECT 1 AS old_id")
+ downstream = self._create_view("downstream_view", "SELECT * FROM upstream_view")
+ self._edit_view(upstream, "SELECT 1 AS new_id")
+
+ with (
+ patch(
+ "products.data_modeling.backend.logic.view_column_refresh.get_restricted_properties_with_group_type_index_for_team",
+ return_value={RestrictedProperty(name="$email", property_type=PropertyDefinition.Type.EVENT)},
+ ),
+ pytest.raises(QueryError),
+ ):
+ execute_hogql_query("SELECT new_id FROM downstream_view", team=self.team, user=self.user)
+
+ downstream.refresh_from_db()
+ assert downstream.column_order == ["old_id"]
+
+ def test_refresh_keeps_the_columns_of_a_query_edit_made_during_inference(self) -> None:
+ view = self._create_view("edited_view", "SELECT 1 AS old_id")
+ edited_columns = {"edited_id": {"hogql": "IntegerDatabaseField", "clickhouse": "UInt8", "valid": True}}
+ inferred_columns = {"new_id": {"hogql": "IntegerDatabaseField", "clickhouse": "UInt8", "valid": True}}
+
+ def edit_the_view_during_inference(*args: object, **kwargs: object) -> dict:
+ DataWarehouseSavedQuery.objects.filter(pk=view.pk).update(
+ query_revision=uuid4(), columns=edited_columns, column_order=["edited_id"]
+ )
+ return inferred_columns
+
+ with patch.object(DataWarehouseSavedQuery, "get_columns", side_effect=edit_the_view_during_inference):
+ changed = refresh_view_columns(self.team, [view.id], self.user)
+
+ assert changed == set()
+ view.refresh_from_db()
+ assert view.column_order == ["edited_id"]