Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-modeling): expose the window start to an incremental view (#118656)

Raúl Negrón-Otero43a8b7727b96
diff --git a/posthog/temporal/data_modeling/activities/materialize_view.py b/posthog/temporal/data_modeling/activities/materialize_view.py
index e5b65b82e9a3..9d458131c4c9 100644
--- a/posthog/temporal/data_modeling/activities/materialize_view.py
+++ b/posthog/temporal/data_modeling/activities/materialize_view.py
@@ -27,7 +27,6 @@ from posthog.hogql.database.database import Database
from posthog.hogql.errors import ParsingError
from posthog.hogql.functions.prompt_jev import PromptJevFinder
from posthog.hogql.modifiers import create_default_modifiers_for_team
-from posthog.hogql.parser import parse_select
from posthog.hogql.printer import prepare_ast_for_printing, print_prepared_ast
from posthog.hogql.query import HogQLQueryExecutor
from posthog.hogql.resolver import ResolverFactory
@@ -69,6 +68,7 @@ from products.data_modeling.backend.facade.api import (
WritePlan,
clear_incremental_state,
inject_incremental_filter,
+ parse_model_query,
record_incremental_history,
resolve_write_plan,
set_incremental_state,
@@ -652,7 +652,7 @@ async def hogql_table(
if window is not None:
query_node = inject_incremental_filter(query, incremental_key=window.incremental_key, since=window.since)
else:
- query_node = parse_select(query)
+ query_node = parse_model_query(query)
if query_node is None:
raise ParsingError(f"Failed to parse query node from query, parse_select() returned None: query={query}")
diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.py
index be358a0e74f9..5aa1ecceeb03 100644
--- a/products/data_modeling/backend/facade/api.py
+++ b/products/data_modeling/backend/facade/api.py
@@ -133,6 +133,9 @@ _LAZY = {
"IncrementalFilterError": "logic.incremental_filter",
"inject_incremental_filter": "logic.incremental_filter",
"apply_incremental_filter": "logic.incremental_filter",
+ "bind_window_start": "logic.incremental_filter",
+ "parse_model_query": "logic.incremental_filter",
+ "WINDOW_START_PLACEHOLDER": "logic.incremental_filter",
"EligibilityResult": "logic.incremental_eligibility",
"check_incremental_eligibility": "logic.incremental_eligibility",
"propose_incremental_config": "logic.incremental_eligibility",
diff --git a/products/data_modeling/backend/logic/incremental_eligibility.py b/products/data_modeling/backend/logic/incremental_eligibility.py
index d1fcaa0d7590..6f6090c15d62 100644
--- a/products/data_modeling/backend/logic/incremental_eligibility.py
+++ b/products/data_modeling/backend/logic/incremental_eligibility.py
@@ -47,7 +47,7 @@ if TYPE_CHECKING:
from posthog.models import User
from products.data_modeling.backend.logic.incremental import IncrementalConfig
-from products.data_modeling.backend.logic.incremental_filter import find_key_expression
+from products.data_modeling.backend.logic.incremental_filter import find_key_expression, parse_model_query
# Non-deterministic calls make a window irreproducible: two runs over the same window can disagree,
# so a re-run silently rewrites rows. Warned rather than blocked, because a `now()` in a filter is
@@ -169,7 +169,7 @@ def check_incremental_eligibility(
the key candidates are pickable names and a config naming an expanded column passes. Without it
(or when the query does not resolve against it), the raw AST is checked as-is."""
try:
- node = parse_select(query)
+ node = parse_model_query(query)
except (ExposedHogQLError, ValueError) as err:
return EligibilityResult(eligible=False, blockers=[f"This query could not be parsed: {err}"])
diff --git a/products/data_modeling/backend/logic/incremental_filter.py b/products/data_modeling/backend/logic/incremental_filter.py
index 0a3e64572d64..b040f92205fc 100644
--- a/products/data_modeling/backend/logic/incremental_filter.py
+++ b/products/data_modeling/backend/logic/incremental_filter.py
@@ -16,8 +16,9 @@ The user picks an output column; we do the rest. Two filters go in, and they do
from typing import Any, Optional, Union
from posthog.hogql import ast
+from posthog.hogql.errors import ExposedHogQLError
from posthog.hogql.parser import parse_select
-from posthog.hogql.visitor import clone_expr
+from posthog.hogql.visitor import CloningVisitor, TraversingVisitor, clone_expr
SelectLike = Union[ast.SelectQuery, ast.SelectSetQuery]
@@ -25,6 +26,11 @@ SelectLike = Union[ast.SelectQuery, ast.SelectSetQuery]
# anything a user would plausibly name a column.
GUARD_ALIAS = "__ph_incremental"
+# A query may name the run's window start as ``{incremental_window_start}`` to bound a scan the
+# injected filter cannot reach, such as the right side of a join. Everywhere but an incremental
+# run it is bound to NULL, so a query guards it with ``ifNull`` and a floor that keeps every row.
+WINDOW_START_PLACEHOLDER = "incremental_window_start"
+
class IncrementalFilterError(ValueError):
"""The query cannot carry an incremental window filter.
@@ -40,11 +46,75 @@ def inject_incremental_filter(query: str, *, incremental_key: str, since: Any) -
def apply_incremental_filter(node: SelectLike, *, incremental_key: str, since: Any) -> SelectLike:
- """Filter an already-parsed query, for callers that replace placeholders before filtering."""
- filtered = _push_predicate(node, incremental_key=incremental_key, since=since)
+ """Filter an already-parsed query to rows at or after ``since``, which also binds the window start."""
+ filtered = _push_predicate(bind_window_start(node, since), incremental_key=incremental_key, since=since)
return _wrap_with_guard(filtered, incremental_key=incremental_key, since=since)
+def parse_model_query(sql: str) -> SelectLike:
+ """Parse a saved query's SQL as every reader outside an incremental run sees it: the window
+ start bound to NULL. Only SQL that names the placeholder pays for the binding clone."""
+ node = parse_select(sql)
+ return bind_window_start(node, None) if WINDOW_START_PLACEHOLDER in sql else node
+
+
+def bind_window_start(node: SelectLike, since: Any) -> SelectLike:
+ """Bind ``{incremental_window_start}`` to ``since``, which is None outside an incremental run.
+
+ An unbound placeholder resolves as a column and fails the query, so every reader binds it:
+ ``DataWarehouseSavedQuery.sql`` carries the NULL binding to readers outside a run. The API
+ refuses placeholders in a user's view, so only a managed view a product provides carries it,
+ inside an expression that accepts both NULL and the key's type.
+
+ Only that exact placeholder is replaced, with a constant. The general placeholder expander
+ evaluates placeholder expressions as Hog, so a query could change its shape, and the tables it
+ reads, with the bound value, after the save-time access check saw a different shape.
+ """
+ bound = _BindWindowStart(since).visit(node)
+ assert isinstance(bound, ast.SelectQuery | ast.SelectSetQuery)
+ return bound
+
+
+def bind_window_start_sql(sql: str) -> str:
+ """The SQL with the placeholder bound to NULL, for a reader that parses the text itself.
+
+ The placeholder tokens come from a parse, so a string literal that spells the placeholder stays
+ as it is. SQL that does not parse comes back unchanged, and the reader reports the parse error.
+ """
+ if WINDOW_START_PLACEHOLDER not in sql:
+ return sql
+ try:
+ node = parse_select(sql)
+ except (ExposedHogQLError, ValueError):
+ return sql
+ spans = _WindowStartSpans()
+ spans.visit(node)
+ for start, end in sorted(spans.spans, reverse=True):
+ sql = f"{sql[:start]}NULL{sql[end:]}"
+ return sql
+
+
+class _WindowStartSpans(TraversingVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.spans: list[tuple[int, int]] = []
+
+ def visit_placeholder(self, node: ast.Placeholder) -> None:
+ if node.chain == [WINDOW_START_PLACEHOLDER] and node.start is not None and node.end is not None:
+ self.spans.append((node.start, node.end))
+
+
+class _BindWindowStart(CloningVisitor):
+ def __init__(self, since: Any) -> None:
+ super().__init__()
+ self.since = since
+
+ def visit_placeholder(self, node: ast.Placeholder) -> ast.Expr:
+ if node.chain == [WINDOW_START_PLACEHOLDER]:
+ return ast.Constant(value=self.since, start=node.start, end=node.end)
+ return super().visit_placeholder(node)
+
+
def find_key_expression(select: ast.SelectQuery, incremental_key: str) -> Optional[ast.Expr]:
"""The expression behind an output column, or None when the query does not produce it.
diff --git a/products/data_modeling/backend/models/datawarehouse_saved_query.py b/products/data_modeling/backend/models/datawarehouse_saved_query.py
index 1053a7fb0e6e..38d9f5442e29 100644
--- a/products/data_modeling/backend/models/datawarehouse_saved_query.py
+++ b/products/data_modeling/backend/models/datawarehouse_saved_query.py
@@ -47,6 +47,7 @@ from posthog.models.utils import CreatedMetaFields, DeletedMetaFields, UpdatedMe
from posthog.schema_enums import DataWarehouseSavedQueryOrigin
from posthog.sync import database_sync_to_async
+from products.data_modeling.backend.logic.incremental_filter import bind_window_start_sql
from products.warehouse_sources.backend.facade.hogql import (
LEGACY_CLICKHOUSE_HOGQL_MAPPING,
STR_TO_HOGQL_MAPPING,
@@ -530,17 +531,22 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,
"""Whether ``set_columns(columns)`` would leave this view's stored columns unchanged."""
return columns == self.columns and list(columns.keys()) == self.column_order
+ @property
+ def sql(self) -> str:
+ """The stored SQL as every reader outside a materialization run sees it: the incremental
+ window start bound to NULL. A run binds the real window start on the parsed query instead."""
+ query = self.query or {}
+ if not isinstance(query, dict) or "query" not in query:
+ raise Exception("Saved query is missing a query definition")
+ return bind_window_start_sql(query["query"])
+
def get_columns(
self, user: Optional["User"] = None, *, bypass_warehouse_access_control: bool = False
) -> dict[str, dict[str, Any]]:
from posthog.hogql.query import execute_hogql_query # noqa: PLC0415 — keeps the heavy dep off the import path
- query = self.query or {}
- if not isinstance(query, dict) or "query" not in query:
- raise Exception("Saved query is missing a query definition")
-
# ClickHouse can still evaluate scalar subqueries before it plans the outer LIMIT 0.
- select_query = parse_select(query["query"])
+ select_query = parse_select(self.sql)
_limit_to_zero_rows(select_query)
_InSubqueryZeroRows().visit(select_query)
@@ -607,11 +613,7 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,
database=database or Database.create_for(self.team.pk, bypass_warehouse_access_control=True),
)
- query = self.query or {}
- if not isinstance(query, dict) or "query" not in query:
- raise Exception("Saved query is missing a query definition")
-
- node = parse_select(query["query"])
+ node = parse_select(self.sql)
resolved_node = resolve_types(node, context, dialect="clickhouse")
table_collector = S3TableVisitor()
@@ -663,7 +665,7 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,
return SavedQuery(
id=str(self.id),
name=self.name,
- query=query["query"],
+ query=self.sql,
fields=self.hogql_fields(),
# Currently only storing metadata related to the managed viewset, but we can expand this in the future
# This is basically just a bag of props that can be used by other methods to properly identify this query
diff --git a/products/data_modeling/backend/models/modeling.py b/products/data_modeling/backend/models/modeling.py
index 661b255efcdf..229b551195b6 100644
--- a/products/data_modeling/backend/models/modeling.py
+++ b/products/data_modeling/backend/models/modeling.py
@@ -11,13 +11,13 @@ from posthog.hogql.context import HogQLContext
from posthog.hogql.database.database import Database
from posthog.hogql.database.models import SavedQuery
from posthog.hogql.errors import QueryError
-from posthog.hogql.parser import parse_select
from posthog.hogql.resolver import Resolver, ResolverFactory
from posthog.hogql.visitor import TraversingVisitor
from posthog.models.team import Team
from products.data_modeling.backend.facade.system_tables import DATA_MODELING_ALLOWED_SYSTEM_TABLES
+from products.data_modeling.backend.logic.incremental_filter import parse_model_query
LabelPath = list[str]
@@ -494,7 +494,7 @@ def get_parents_from_model_query(
model_query: The HogQL query string to parse.
database: An optional prebuilt database to reuse for dependency resolution.
"""
- hogql_query = parse_select(model_query)
+ hogql_query = parse_model_query(model_query)
context = HogQLContext(
team_id=team.pk,
team=team,
diff --git a/products/data_modeling/backend/test/test_incremental_filter.py b/products/data_modeling/backend/test/test_incremental_filter.py
index 64101107e6cf..19fab788c243 100644
--- a/products/data_modeling/backend/test/test_incremental_filter.py
+++ b/products/data_modeling/backend/test/test_incremental_filter.py
@@ -11,6 +11,8 @@ from posthog.hogql.visitor import clear_locations
from products.data_modeling.backend.logic.incremental_filter import (
GUARD_ALIAS,
IncrementalFilterError,
+ bind_window_start,
+ bind_window_start_sql,
inject_incremental_filter,
)
@@ -123,6 +125,37 @@ class TestInjectIncrementalFilter(SimpleTestCase):
op=ast.CompareOperationOp.GtEq,
)
+ @parameterized.expand([("incremental_run", SINCE), ("outside_a_run", None)])
+ def test_binds_the_window_start_placeholder(self, _name: str, since: datetime | None) -> None:
+ node = bind_window_start(
+ parse_select("SELECT day FROM t WHERE day >= ifNull({incremental_window_start}, '')"), since
+ )
+
+ assert isinstance(node, ast.SelectQuery)
+ assert node.where is not None
+ assert clear_locations(node.where) == ast.CompareOperation(
+ left=ast.Field(chain=["day"]),
+ right=ast.Call(name="ifNull", args=[ast.Constant(value=since), ast.Constant(value="")]),
+ op=ast.CompareOperationOp.GtEq,
+ )
+
+ def test_binds_the_placeholder_tokens_in_the_sql_text_and_nothing_else(self) -> None:
+ sql = "SELECT day FROM t WHERE day >= ifNull({ incremental_window_start }, '') AND tag = '{incremental_window_start}'"
+
+ assert bind_window_start_sql(sql) == (
+ "SELECT day FROM t WHERE day >= ifNull(NULL, '') AND tag = '{incremental_window_start}'"
+ )
+
+ def test_leaves_every_other_placeholder_as_it_is(self) -> None:
+ node = bind_window_start(parse_select("SELECT day FROM t WHERE x = {other} AND y = {sql(1 + 1)}"), SINCE)
+
+ assert isinstance(node, ast.SelectQuery)
+ assert isinstance(node.where, ast.And)
+ assert all(
+ isinstance(term, ast.CompareOperation) and isinstance(term.right, ast.Placeholder)
+ for term in node.where.exprs
+ )
+
def test_rejects_a_key_the_query_does_not_produce(self) -> None:
with self.assertRaises(IncrementalFilterError):
inject_incremental_filter(
diff --git a/products/data_modeling/backend/tests/test_managed_viewset_providers.py b/products/data_modeling/backend/tests/test_managed_viewset_providers.py
index 9dd39ca5f3c1..6d693c164c45 100644
--- a/products/data_modeling/backend/tests/test_managed_viewset_providers.py
+++ b/products/data_modeling/backend/tests/test_managed_viewset_providers.py
@@ -391,6 +391,23 @@ class TestManagedViewSetProviders(BaseTest):
view = next(v for v in self._views(viewset) if v.name == provided[0].name)
self.assertEqual(view.incremental_state, state)
+ @patch(SCHEDULE_MATERIALIZATION)
+ def test_a_view_that_names_the_window_start_gets_a_dag_node(self, _):
+ fake_view = ProvidedView(
+ name="windowed",
+ query="SELECT 1 AS id WHERE 1 >= ifNull({incremental_window_start}, 0)",
+ fields={"id": IntegerDatabaseField(name="id")},
+ incremental_config=INCREMENTAL_CONFIG,
+ )
+ with patch.dict(_expected_views_providers, clear=True):
+ register_expected_views_provider(KIND, lambda team: [fake_view])
+
+ viewset = self._viewset()
+ viewset.sync_views()
+
+ view = next(v for v in self._views(viewset) if v.name == fake_view.name)
+ self.assertTrue(Node.objects.filter(saved_query=view).exists())
+
def test_provisioning_a_new_team_puts_every_view_on_tier_schedules(self):
# Managed viewsets are how most new teams first materialize anything (a provider makes
# several views at once), and their DAG is brand new. Views after the first must follow the
diff --git a/products/managed_warehouse/backend/tests/test_trino_materialization.py b/products/managed_warehouse/backend/tests/test_trino_materialization.py
index 81f3b43474ba..cef188099760 100644
--- a/products/managed_warehouse/backend/tests/test_trino_materialization.py
+++ b/products/managed_warehouse/backend/tests/test_trino_materialization.py
@@ -88,7 +88,7 @@ class TestTrinoShadowMaterialization(BaseTest):
compile_query.assert_called_once_with(
self.team.pk,
- HogQLQuery.model_validate(self.query),
+ HogQLQuery.model_validate({**self.query, "values": {"incremental_window_start": None}}),
team=self.team,
bypass_warehouse_access_control=True,
allowed_system_tables=DATA_MODELING_ALLOWED_SYSTEM_TABLES,
@@ -468,6 +468,7 @@ class TestTrinoIncrementalMaterialization(BaseTest):
except Exception as error:
result = error
transforms = [call.kwargs["select_transform"] for call in compile_query.call_args_list]
+ self.window_starts = [call.args[1].values["incremental_window_start"] for call in compile_query.call_args_list]
return result, [statement for statement, _ in cursor.statements], transforms
def test_merges_staged_rows_and_reports_their_watermark(self) -> None:
@@ -479,6 +480,7 @@ class TestTrinoIncrementalMaterialization(BaseTest):
assert isinstance(result, DuckLakeTableResult)
assert (result.row_count, result.watermark, result.merged) == (3, latest, True)
assert len(transforms) == 1 and transforms[0] is not None
+ assert self.window_starts == [self.since]
table = f'"cat"."posthog_data_modeling_team_{self.team.pk}"."model_{self.saved_query_id.hex}"'
stage = table[:-1] + '__ph_incremental_stage"'
assert f"CREATE OR REPLACE TABLE {stage} AS SELECT id, ts FROM orders" in statements
@@ -597,6 +599,7 @@ class TestTrinoIncrementalMaterialization(BaseTest):
assert (result.row_count, result.watermark, result.merged) == (9, latest, False)
# The table is rebuilt from the unfiltered query.
assert transforms[-1](parse_select("SELECT id, ts FROM orders")).where is None
+ assert self.window_starts[-1] is None
table = f'"cat"."posthog_data_modeling_team_{self.team.pk}"."model_{self.saved_query_id.hex}"'
assert f"CREATE OR REPLACE TABLE {table} AS SELECT id, ts FROM orders" in statements
assert not any("__ph_incremental_stage" in statement for statement in statements)
diff --git a/products/managed_warehouse/backend/trino_materialization.py b/products/managed_warehouse/backend/trino_materialization.py
index 7e461b789c09..c0c95163932c 100644
--- a/products/managed_warehouse/backend/trino_materialization.py
+++ b/products/managed_warehouse/backend/trino_materialization.py
@@ -107,12 +107,18 @@ def execute_trino_shadow_materialization(
def _compile(team: Team, query: HogQLQuery, incremental: TrinoIncrementalWrite | None) -> Any:
+ from products.data_modeling.backend.facade.api import ( # noqa: PLC0415 -- the facade loads lazily
+ WINDOW_START_PLACEHOLDER,
+ apply_incremental_filter,
+ )
+
+ # The compiler expands placeholders before select_transform runs, so the window start is bound as a value.
+ since = incremental.since if incremental is not None else None
+ query = query.model_copy(update={"values": {**(query.values or {}), WINDOW_START_PLACEHOLDER: since}})
+
def select_transform(node: ast.SelectQuery | ast.SelectSetQuery) -> ast.SelectQuery | ast.SelectSetQuery:
if incremental is None:
return node
- from products.data_modeling.backend.facade.api import ( # noqa: PLC0415 -- the facade loads lazily
- apply_incremental_filter,
- )
return apply_incremental_filter(node, incremental_key=incremental.incremental_key, since=incremental.since)