:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-modeling): expose the window start to an incremental view (#118656)
Raúl Negrón-Otero43a8b7727b96
diff --git a/posthog/temporal/data_modeling/activities/materialize_view.py b/posthog/temporal/data_modeling/activities/materialize_view.pyindex e5b65b82e9a3..9d458131c4c9 100644--- a/posthog/temporal/data_modeling/activities/materialize_view.py+++ b/posthog/temporal/data_modeling/activities/materialize_view.py@@ -27,7 +27,6 @@ from posthog.hogql.database.database import Databasefrom posthog.hogql.errors import ParsingErrorfrom posthog.hogql.functions.prompt_jev import PromptJevFinderfrom posthog.hogql.modifiers import create_default_modifiers_for_team-from posthog.hogql.parser import parse_selectfrom posthog.hogql.printer import prepare_ast_for_printing, print_prepared_astfrom posthog.hogql.query import HogQLQueryExecutorfrom posthog.hogql.resolver import ResolverFactory@@ -69,6 +68,7 @@ from products.data_modeling.backend.facade.api import (WritePlan,clear_incremental_state,inject_incremental_filter,+ parse_model_query,record_incremental_history,resolve_write_plan,set_incremental_state,@@ -652,7 +652,7 @@ async def hogql_table(if window is not None:query_node = inject_incremental_filter(query, incremental_key=window.incremental_key, since=window.since)else:- query_node = parse_select(query)+ query_node = parse_model_query(query)if query_node is None:raise ParsingError(f"Failed to parse query node from query, parse_select() returned None: query={query}")diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.pyindex be358a0e74f9..5aa1ecceeb03 100644--- a/products/data_modeling/backend/facade/api.py+++ b/products/data_modeling/backend/facade/api.py@@ -133,6 +133,9 @@ _LAZY = {"IncrementalFilterError": "logic.incremental_filter","inject_incremental_filter": "logic.incremental_filter","apply_incremental_filter": "logic.incremental_filter",+ "bind_window_start": "logic.incremental_filter",+ "parse_model_query": "logic.incremental_filter",+ "WINDOW_START_PLACEHOLDER": "logic.incremental_filter","EligibilityResult": "logic.incremental_eligibility","check_incremental_eligibility": "logic.incremental_eligibility","propose_incremental_config": "logic.incremental_eligibility",diff --git a/products/data_modeling/backend/logic/incremental_eligibility.py b/products/data_modeling/backend/logic/incremental_eligibility.pyindex d1fcaa0d7590..6f6090c15d62 100644--- a/products/data_modeling/backend/logic/incremental_eligibility.py+++ b/products/data_modeling/backend/logic/incremental_eligibility.py@@ -47,7 +47,7 @@ if TYPE_CHECKING:from posthog.models import Userfrom products.data_modeling.backend.logic.incremental import IncrementalConfig-from products.data_modeling.backend.logic.incremental_filter import find_key_expression+from products.data_modeling.backend.logic.incremental_filter import find_key_expression, parse_model_query# Non-deterministic calls make a window irreproducible: two runs over the same window can disagree,# so a re-run silently rewrites rows. Warned rather than blocked, because a `now()` in a filter is@@ -169,7 +169,7 @@ def check_incremental_eligibility(the key candidates are pickable names and a config naming an expanded column passes. Without it(or when the query does not resolve against it), the raw AST is checked as-is."""try:- node = parse_select(query)+ node = parse_model_query(query)except (ExposedHogQLError, ValueError) as err:return EligibilityResult(eligible=False, blockers=[f"This query could not be parsed: {err}"])diff --git a/products/data_modeling/backend/logic/incremental_filter.py b/products/data_modeling/backend/logic/incremental_filter.pyindex 0a3e64572d64..b040f92205fc 100644--- a/products/data_modeling/backend/logic/incremental_filter.py+++ b/products/data_modeling/backend/logic/incremental_filter.py@@ -16,8 +16,9 @@ The user picks an output column; we do the rest. Two filters go in, and they dofrom typing import Any, Optional, Unionfrom posthog.hogql import ast+from posthog.hogql.errors import ExposedHogQLErrorfrom posthog.hogql.parser import parse_select-from posthog.hogql.visitor import clone_expr+from posthog.hogql.visitor import CloningVisitor, TraversingVisitor, clone_exprSelectLike = Union[ast.SelectQuery, ast.SelectSetQuery]@@ -25,6 +26,11 @@ SelectLike = Union[ast.SelectQuery, ast.SelectSetQuery]# anything a user would plausibly name a column.GUARD_ALIAS = "__ph_incremental"+# A query may name the run's window start as ``{incremental_window_start}`` to bound a scan the+# injected filter cannot reach, such as the right side of a join. Everywhere but an incremental+# run it is bound to NULL, so a query guards it with ``ifNull`` and a floor that keeps every row.+WINDOW_START_PLACEHOLDER = "incremental_window_start"+class IncrementalFilterError(ValueError):"""The query cannot carry an incremental window filter.@@ -40,11 +46,75 @@ def inject_incremental_filter(query: str, *, incremental_key: str, since: Any) -def apply_incremental_filter(node: SelectLike, *, incremental_key: str, since: Any) -> SelectLike:- """Filter an already-parsed query, for callers that replace placeholders before filtering."""- filtered = _push_predicate(node, incremental_key=incremental_key, since=since)+ """Filter an already-parsed query to rows at or after ``since``, which also binds the window start."""+ filtered = _push_predicate(bind_window_start(node, since), incremental_key=incremental_key, since=since)return _wrap_with_guard(filtered, incremental_key=incremental_key, since=since)+def parse_model_query(sql: str) -> SelectLike:+ """Parse a saved query's SQL as every reader outside an incremental run sees it: the window+ start bound to NULL. Only SQL that names the placeholder pays for the binding clone."""+ node = parse_select(sql)+ return bind_window_start(node, None) if WINDOW_START_PLACEHOLDER in sql else node+++def bind_window_start(node: SelectLike, since: Any) -> SelectLike:+ """Bind ``{incremental_window_start}`` to ``since``, which is None outside an incremental run.++ An unbound placeholder resolves as a column and fails the query, so every reader binds it:+ ``DataWarehouseSavedQuery.sql`` carries the NULL binding to readers outside a run. The API+ refuses placeholders in a user's view, so only a managed view a product provides carries it,+ inside an expression that accepts both NULL and the key's type.++ Only that exact placeholder is replaced, with a constant. The general placeholder expander+ evaluates placeholder expressions as Hog, so a query could change its shape, and the tables it+ reads, with the bound value, after the save-time access check saw a different shape.+ """+ bound = _BindWindowStart(since).visit(node)+ assert isinstance(bound, ast.SelectQuery | ast.SelectSetQuery)+ return bound+++def bind_window_start_sql(sql: str) -> str:+ """The SQL with the placeholder bound to NULL, for a reader that parses the text itself.++ The placeholder tokens come from a parse, so a string literal that spells the placeholder stays+ as it is. SQL that does not parse comes back unchanged, and the reader reports the parse error.+ """+ if WINDOW_START_PLACEHOLDER not in sql:+ return sql+ try:+ node = parse_select(sql)+ except (ExposedHogQLError, ValueError):+ return sql+ spans = _WindowStartSpans()+ spans.visit(node)+ for start, end in sorted(spans.spans, reverse=True):+ sql = f"{sql[:start]}NULL{sql[end:]}"+ return sql+++class _WindowStartSpans(TraversingVisitor):+ def __init__(self) -> None:+ super().__init__()+ self.spans: list[tuple[int, int]] = []++ def visit_placeholder(self, node: ast.Placeholder) -> None:+ if node.chain == [WINDOW_START_PLACEHOLDER] and node.start is not None and node.end is not None:+ self.spans.append((node.start, node.end))+++class _BindWindowStart(CloningVisitor):+ def __init__(self, since: Any) -> None:+ super().__init__()+ self.since = since++ def visit_placeholder(self, node: ast.Placeholder) -> ast.Expr:+ if node.chain == [WINDOW_START_PLACEHOLDER]:+ return ast.Constant(value=self.since, start=node.start, end=node.end)+ return super().visit_placeholder(node)++def find_key_expression(select: ast.SelectQuery, incremental_key: str) -> Optional[ast.Expr]:"""The expression behind an output column, or None when the query does not produce it.diff --git a/products/data_modeling/backend/models/datawarehouse_saved_query.py b/products/data_modeling/backend/models/datawarehouse_saved_query.pyindex 1053a7fb0e6e..38d9f5442e29 100644--- a/products/data_modeling/backend/models/datawarehouse_saved_query.py+++ b/products/data_modeling/backend/models/datawarehouse_saved_query.py@@ -47,6 +47,7 @@ from posthog.models.utils import CreatedMetaFields, DeletedMetaFields, UpdatedMefrom posthog.schema_enums import DataWarehouseSavedQueryOriginfrom posthog.sync import database_sync_to_async+from products.data_modeling.backend.logic.incremental_filter import bind_window_start_sqlfrom products.warehouse_sources.backend.facade.hogql import (LEGACY_CLICKHOUSE_HOGQL_MAPPING,STR_TO_HOGQL_MAPPING,@@ -530,17 +531,22 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,"""Whether ``set_columns(columns)`` would leave this view's stored columns unchanged."""return columns == self.columns and list(columns.keys()) == self.column_order+ @property+ def sql(self) -> str:+ """The stored SQL as every reader outside a materialization run sees it: the incremental+ window start bound to NULL. A run binds the real window start on the parsed query instead."""+ query = self.query or {}+ if not isinstance(query, dict) or "query" not in query:+ raise Exception("Saved query is missing a query definition")+ return bind_window_start_sql(query["query"])+def get_columns(self, user: Optional["User"] = None, *, bypass_warehouse_access_control: bool = False) -> dict[str, dict[str, Any]]:from posthog.hogql.query import execute_hogql_query # noqa: PLC0415 — keeps the heavy dep off the import path- query = self.query or {}- if not isinstance(query, dict) or "query" not in query:- raise Exception("Saved query is missing a query definition")-# ClickHouse can still evaluate scalar subqueries before it plans the outer LIMIT 0.- select_query = parse_select(query["query"])+ select_query = parse_select(self.sql)_limit_to_zero_rows(select_query)_InSubqueryZeroRows().visit(select_query)@@ -607,11 +613,7 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,database=database or Database.create_for(self.team.pk, bypass_warehouse_access_control=True),)- query = self.query or {}- if not isinstance(query, dict) or "query" not in query:- raise Exception("Saved query is missing a query definition")-- node = parse_select(query["query"])+ node = parse_select(self.sql)resolved_node = resolve_types(node, context, dialect="clickhouse")table_collector = S3TableVisitor()@@ -663,7 +665,7 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,return SavedQuery(id=str(self.id),name=self.name,- query=query["query"],+ query=self.sql,fields=self.hogql_fields(),# Currently only storing metadata related to the managed viewset, but we can expand this in the future# This is basically just a bag of props that can be used by other methods to properly identify this querydiff --git a/products/data_modeling/backend/models/modeling.py b/products/data_modeling/backend/models/modeling.pyindex 661b255efcdf..229b551195b6 100644--- a/products/data_modeling/backend/models/modeling.py+++ b/products/data_modeling/backend/models/modeling.py@@ -11,13 +11,13 @@ from posthog.hogql.context import HogQLContextfrom posthog.hogql.database.database import Databasefrom posthog.hogql.database.models import SavedQueryfrom posthog.hogql.errors import QueryError-from posthog.hogql.parser import parse_selectfrom posthog.hogql.resolver import Resolver, ResolverFactoryfrom posthog.hogql.visitor import TraversingVisitorfrom posthog.models.team import Teamfrom products.data_modeling.backend.facade.system_tables import DATA_MODELING_ALLOWED_SYSTEM_TABLES+from products.data_modeling.backend.logic.incremental_filter import parse_model_queryLabelPath = list[str]@@ -494,7 +494,7 @@ def get_parents_from_model_query(model_query: The HogQL query string to parse.database: An optional prebuilt database to reuse for dependency resolution."""- hogql_query = parse_select(model_query)+ hogql_query = parse_model_query(model_query)context = HogQLContext(team_id=team.pk,team=team,diff --git a/products/data_modeling/backend/test/test_incremental_filter.py b/products/data_modeling/backend/test/test_incremental_filter.pyindex 64101107e6cf..19fab788c243 100644--- a/products/data_modeling/backend/test/test_incremental_filter.py+++ b/products/data_modeling/backend/test/test_incremental_filter.py@@ -11,6 +11,8 @@ from posthog.hogql.visitor import clear_locationsfrom products.data_modeling.backend.logic.incremental_filter import (GUARD_ALIAS,IncrementalFilterError,+ bind_window_start,+ bind_window_start_sql,inject_incremental_filter,)@@ -123,6 +125,37 @@ class TestInjectIncrementalFilter(SimpleTestCase):op=ast.CompareOperationOp.GtEq,)+ @parameterized.expand([("incremental_run", SINCE), ("outside_a_run", None)])+ def test_binds_the_window_start_placeholder(self, _name: str, since: datetime | None) -> None:+ node = bind_window_start(+ parse_select("SELECT day FROM t WHERE day >= ifNull({incremental_window_start}, '')"), since+ )++ assert isinstance(node, ast.SelectQuery)+ assert node.where is not None+ assert clear_locations(node.where) == ast.CompareOperation(+ left=ast.Field(chain=["day"]),+ right=ast.Call(name="ifNull", args=[ast.Constant(value=since), ast.Constant(value="")]),+ op=ast.CompareOperationOp.GtEq,+ )++ def test_binds_the_placeholder_tokens_in_the_sql_text_and_nothing_else(self) -> None:+ sql = "SELECT day FROM t WHERE day >= ifNull({ incremental_window_start }, '') AND tag = '{incremental_window_start}'"++ assert bind_window_start_sql(sql) == (+ "SELECT day FROM t WHERE day >= ifNull(NULL, '') AND tag = '{incremental_window_start}'"+ )++ def test_leaves_every_other_placeholder_as_it_is(self) -> None:+ node = bind_window_start(parse_select("SELECT day FROM t WHERE x = {other} AND y = {sql(1 + 1)}"), SINCE)++ assert isinstance(node, ast.SelectQuery)+ assert isinstance(node.where, ast.And)+ assert all(+ isinstance(term, ast.CompareOperation) and isinstance(term.right, ast.Placeholder)+ for term in node.where.exprs+ )+def test_rejects_a_key_the_query_does_not_produce(self) -> None:with self.assertRaises(IncrementalFilterError):inject_incremental_filter(diff --git a/products/data_modeling/backend/tests/test_managed_viewset_providers.py b/products/data_modeling/backend/tests/test_managed_viewset_providers.pyindex 9dd39ca5f3c1..6d693c164c45 100644--- a/products/data_modeling/backend/tests/test_managed_viewset_providers.py+++ b/products/data_modeling/backend/tests/test_managed_viewset_providers.py@@ -391,6 +391,23 @@ class TestManagedViewSetProviders(BaseTest):view = next(v for v in self._views(viewset) if v.name == provided[0].name)self.assertEqual(view.incremental_state, state)+ @patch(SCHEDULE_MATERIALIZATION)+ def test_a_view_that_names_the_window_start_gets_a_dag_node(self, _):+ fake_view = ProvidedView(+ name="windowed",+ query="SELECT 1 AS id WHERE 1 >= ifNull({incremental_window_start}, 0)",+ fields={"id": IntegerDatabaseField(name="id")},+ incremental_config=INCREMENTAL_CONFIG,+ )+ with patch.dict(_expected_views_providers, clear=True):+ register_expected_views_provider(KIND, lambda team: [fake_view])++ viewset = self._viewset()+ viewset.sync_views()++ view = next(v for v in self._views(viewset) if v.name == fake_view.name)+ self.assertTrue(Node.objects.filter(saved_query=view).exists())+def test_provisioning_a_new_team_puts_every_view_on_tier_schedules(self):# Managed viewsets are how most new teams first materialize anything (a provider makes# several views at once), and their DAG is brand new. Views after the first must follow thediff --git a/products/managed_warehouse/backend/tests/test_trino_materialization.py b/products/managed_warehouse/backend/tests/test_trino_materialization.pyindex 81f3b43474ba..cef188099760 100644--- a/products/managed_warehouse/backend/tests/test_trino_materialization.py+++ b/products/managed_warehouse/backend/tests/test_trino_materialization.py@@ -88,7 +88,7 @@ class TestTrinoShadowMaterialization(BaseTest):compile_query.assert_called_once_with(self.team.pk,- HogQLQuery.model_validate(self.query),+ HogQLQuery.model_validate({**self.query, "values": {"incremental_window_start": None}}),team=self.team,bypass_warehouse_access_control=True,allowed_system_tables=DATA_MODELING_ALLOWED_SYSTEM_TABLES,@@ -468,6 +468,7 @@ class TestTrinoIncrementalMaterialization(BaseTest):except Exception as error:result = errortransforms = [call.kwargs["select_transform"] for call in compile_query.call_args_list]+ self.window_starts = [call.args[1].values["incremental_window_start"] for call in compile_query.call_args_list]return result, [statement for statement, _ in cursor.statements], transformsdef test_merges_staged_rows_and_reports_their_watermark(self) -> None:@@ -479,6 +480,7 @@ class TestTrinoIncrementalMaterialization(BaseTest):assert isinstance(result, DuckLakeTableResult)assert (result.row_count, result.watermark, result.merged) == (3, latest, True)assert len(transforms) == 1 and transforms[0] is not None+ assert self.window_starts == [self.since]table = f'"cat"."posthog_data_modeling_team_{self.team.pk}"."model_{self.saved_query_id.hex}"'stage = table[:-1] + '__ph_incremental_stage"'assert f"CREATE OR REPLACE TABLE {stage} AS SELECT id, ts FROM orders" in statements@@ -597,6 +599,7 @@ class TestTrinoIncrementalMaterialization(BaseTest):assert (result.row_count, result.watermark, result.merged) == (9, latest, False)# The table is rebuilt from the unfiltered query.assert transforms[-1](parse_select("SELECT id, ts FROM orders")).where is None+ assert self.window_starts[-1] is Nonetable = f'"cat"."posthog_data_modeling_team_{self.team.pk}"."model_{self.saved_query_id.hex}"'assert f"CREATE OR REPLACE TABLE {table} AS SELECT id, ts FROM orders" in statementsassert not any("__ph_incremental_stage" in statement for statement in statements)diff --git a/products/managed_warehouse/backend/trino_materialization.py b/products/managed_warehouse/backend/trino_materialization.pyindex 7e461b789c09..c0c95163932c 100644--- a/products/managed_warehouse/backend/trino_materialization.py+++ b/products/managed_warehouse/backend/trino_materialization.py@@ -107,12 +107,18 @@ def execute_trino_shadow_materialization(def _compile(team: Team, query: HogQLQuery, incremental: TrinoIncrementalWrite | None) -> Any:+ from products.data_modeling.backend.facade.api import ( # noqa: PLC0415 -- the facade loads lazily+ WINDOW_START_PLACEHOLDER,+ apply_incremental_filter,+ )++ # The compiler expands placeholders before select_transform runs, so the window start is bound as a value.+ since = incremental.since if incremental is not None else None+ query = query.model_copy(update={"values": {**(query.values or {}), WINDOW_START_PLACEHOLDER: since}})+def select_transform(node: ast.SelectQuery | ast.SelectSetQuery) -> ast.SelectQuery | ast.SelectSetQuery:if incremental is None:return node- from products.data_modeling.backend.facade.api import ( # noqa: PLC0415 -- the facade loads lazily- apply_incremental_filter,- )return apply_incremental_filter(node, incremental_key=incremental.incremental_key, since=incremental.since)