Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-modeling): propose and apply an incremental config (#119694)

Thiago Salvatored7d324230ebd
diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.py
index 092dfdfdac7c..714141d6eac9 100644
--- a/products/data_modeling/backend/facade/api.py
+++ b/products/data_modeling/backend/facade/api.py
@@ -55,6 +55,7 @@ _LAZY = {
"MaterializationFailedError": "facade.contracts",
"MaterializationForbiddenError": "facade.contracts",
"MaterializationRefusedError": "facade.contracts",
+ "IncrementalConfigRefusedError": "facade.contracts",
"enable_saved_query_materialization": "logic.saved_query_materialization",
"saved_query_materialized_at": "logic.saved_query_freshness",
"latest_saved_query_materialization_job": "logic.saved_query_freshness",
@@ -106,6 +107,8 @@ _LAZY = {
"apply_incremental_filter": "logic.incremental_filter",
"EligibilityResult": "logic.incremental_eligibility",
"check_incremental_eligibility": "logic.incremental_eligibility",
+ "propose_incremental_config": "logic.incremental_eligibility",
+ "definition_database": "logic.incremental_eligibility",
}
__all__ = sorted(_LAZY)
diff --git a/products/data_modeling/backend/facade/contracts.py b/products/data_modeling/backend/facade/contracts.py
index bdb73d83b0f3..431ac898d8c4 100644
--- a/products/data_modeling/backend/facade/contracts.py
+++ b/products/data_modeling/backend/facade/contracts.py
@@ -101,6 +101,12 @@ class MaterializationRefusedError(Exception):
pass
+class IncrementalConfigRefusedError(MaterializationRefusedError):
+ def __init__(self, message: str, refused_keys: tuple[str, ...] = ()) -> None:
+ super().__init__(message)
+ self.refused_keys = refused_keys
+
+
class MaterializationForbiddenError(Exception):
pass
diff --git a/products/data_modeling/backend/logic/incremental_eligibility.py b/products/data_modeling/backend/logic/incremental_eligibility.py
index 45da651dc4fd..d1fcaa0d7590 100644
--- a/products/data_modeling/backend/logic/incremental_eligibility.py
+++ b/products/data_modeling/backend/logic/incremental_eligibility.py
@@ -13,11 +13,15 @@ count(DISTINCT), float sum) that Snowflake, Redshift, BigQuery and Databricks al
They have to reject them because they combine partial aggregates across refreshes; we never do.
"""
+from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Optional
+from posthog.schema import HogQLQueryModifiers
+
from posthog.hogql import ast
from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.database import Database
from posthog.hogql.database.models import (
BooleanDatabaseField,
DateDatabaseField,
@@ -33,13 +37,14 @@ from posthog.hogql.database.models import (
UUIDDatabaseField,
)
from posthog.hogql.errors import ExposedHogQLError
+from posthog.hogql.functions.clickhouse.datetime import DATE_GENERATOR_FUNCTIONS
from posthog.hogql.parser import parse_select
from posthog.hogql.property import has_aggregation
from posthog.hogql.resolver import resolve_types
from posthog.hogql.visitor import TraversingVisitor, clear_locations, clone_expr
if TYPE_CHECKING:
- from posthog.hogql.database.database import Database
+ from posthog.models import User
from products.data_modeling.backend.logic.incremental import IncrementalConfig
from products.data_modeling.backend.logic.incremental_filter import find_key_expression
@@ -48,10 +53,9 @@ from products.data_modeling.backend.logic.incremental_filter import find_key_exp
# so a re-run silently rewrites rows. Warned rather than blocked, because a `now()` in a filter is
# common and usually harmless, and blocking it would refuse a lot of reasonable models.
NON_DETERMINISTIC_FUNCTIONS = {
- "now",
+ *(name.lower() for name in DATE_GENERATOR_FUNCTIONS),
+ "nowinblock",
"now64",
- "today",
- "yesterday",
"rand",
"rand32",
"rand64",
@@ -62,6 +66,28 @@ NON_DETERMINISTIC_FUNCTIONS = {
SET_OPERATORS_BLOCKING_INCREMENTAL = {"EXCEPT", "INTERSECT"}
+TIME_ORDER_PRESERVING_FUNCTIONS = {
+ "todate",
+ "todatetime",
+ "todatetime64",
+ "totimezone",
+ "tostartofminute",
+ "tostartoffiveminutes",
+ "tostartoftenminutes",
+ "tostartoffifteenminutes",
+ "tostartofhour",
+ "tostartofday",
+ "tostartofweek",
+ "tomonday",
+ "tostartofmonth",
+ "tostartofquarter",
+ "tostartofyear",
+ "tostartofinterval",
+}
+
+INCREMENTAL_KEY_FIELD = "incremental_key"
+UNIQUE_KEY_FIELD = "unique_key"
+
_CONSTANT_TYPE_LABELS: dict[type, str] = {
ast.DateTimeType: "datetime",
ast.DateType: "date",
@@ -115,15 +141,26 @@ class EligibilityResult:
# Coarse type per candidate, for the picker's type tags. Missing entry: type unknown.
key_candidate_types: dict[str, str] = field(default_factory=dict)
blockers: list[str] = field(default_factory=list)
+ blocked_keys: list[str] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
+def definition_database(team_id: int, *, user: Optional["User"] = None) -> Database:
+ """The team's HogQL schema with every saved query read as its definition, never as its materialized table."""
+ return Database.create_for(
+ team_id=team_id,
+ user=user,
+ modifiers=HogQLQueryModifiers(useMaterializedViews=False),
+ bypass_warehouse_access_control=user is None,
+ )
+
+
def check_incremental_eligibility(
query: str,
config: Optional[IncrementalConfig],
*,
column_types: Optional[dict[str, str]] = None,
- database: Optional["Database"] = None,
+ database: Optional[Database] = None,
) -> EligibilityResult:
"""``column_types`` is the saved query's stored ClickHouse types, used only to catch a nullable
unique key. Omit it and that check is skipped; the runtime guard still catches it.
@@ -173,12 +210,21 @@ def check_incremental_eligibility(
warnings=_unique(warnings),
)
+ key_blockers: list[str] = []
+ unique_key_blockers: list[str] = []
for select in selects:
- _check_key(select, config, blockers, warnings)
- _check_unique_key(select, config, blockers)
+ _check_key(select, config, key_blockers, warnings)
+ _check_unique_key(select, config, unique_key_blockers)
+ _check_unique_key_types(config, key_candidate_types, unique_key_blockers)
- _check_nullable_unique_key(config, column_types, blockers)
+ _check_nullable_unique_key(config, column_types, unique_key_blockers)
_check_determinism(node, warnings)
+ blockers.extend([*key_blockers, *unique_key_blockers])
+ blocked_keys = []
+ if key_blockers:
+ blocked_keys.append(INCREMENTAL_KEY_FIELD)
+ if unique_key_blockers:
+ blocked_keys.append(UNIQUE_KEY_FIELD)
return EligibilityResult(
eligible=not blockers,
@@ -186,10 +232,151 @@ def check_incremental_eligibility(
unique_key_candidates=unique_key_candidates,
key_candidate_types=key_candidate_types,
blockers=_unique(blockers),
+ blocked_keys=blocked_keys,
warnings=_unique(warnings),
)
+def propose_incremental_config(
+ query: str, column_types: Mapping[str, str], *, database: Optional[Database] = None
+) -> Optional[IncrementalConfig]:
+ """The incremental config a grouped query admits on its own, or None when choosing one needs a person."""
+ try:
+ node = parse_select(query)
+ except (ExposedHogQLError, ValueError):
+ return None
+ if not isinstance(node, ast.SelectQuery):
+ return None
+ resolved_selects = _resolved_selects(node, database) or []
+ if any(
+ _is_nondeterministic(select) or _reads_a_data_dependent_value(select) for select in (node, *resolved_selects)
+ ):
+ return None
+ grouped_names = _grouped_output_names(node)
+ if grouped_names is None:
+ return None
+ unique_key = sorted(set(grouped_names))
+ time_keys = [name for name in unique_key if _is_time_type(column_types.get(name))]
+ if len(time_keys) != 1 or not _advances_with_time((resolved_selects or [node])[0], time_keys[0]):
+ return None
+ config = IncrementalConfig(incremental_key=time_keys[0], unique_key=tuple(unique_key))
+ result = check_incremental_eligibility(query, config, column_types=dict(column_types), database=database)
+ return config if result.eligible else None
+
+
+def _is_nondeterministic(node: ast.SelectQuery) -> bool:
+ finder = _NonDeterministicFinder()
+ finder.visit(node)
+ return bool(finder.found)
+
+
+def _reads_a_data_dependent_value(select: ast.SelectQuery) -> bool:
+ finder = _DataDependentValueFinder()
+ for expression in _value_expressions(select):
+ finder.visit(expression)
+ return finder.found or any(
+ _reads_a_data_dependent_value(leaf) for source in _source_nodes(select) for leaf in _leaf_selects(source)
+ )
+
+
+def _advances_with_time(
+ select: ast.SelectQuery, column: str, outer_ctes: Optional[Mapping[str, ast.CTE]] = None
+) -> bool:
+ ctes = {**(outer_ctes or {}), **(select.ctes or {})}
+ expression = find_key_expression(select, column)
+ while isinstance(expression, ast.Alias) or (
+ isinstance(expression, ast.Call) and expression.name.lower() in TIME_ORDER_PRESERVING_FUNCTIONS
+ ):
+ if isinstance(expression, ast.Alias):
+ expression = expression.expr
+ else:
+ expression = expression.args[0] if expression.args else None
+ if not isinstance(expression, ast.Field):
+ return False
+ source = _row_source(select, expression.chain, ctes)
+ if source is None:
+ return False
+ if not isinstance(source, (ast.SelectQuery, ast.SelectSetQuery)):
+ return True
+ return all(_advances_with_time(leaf, str(expression.chain[-1]), ctes) for leaf in _leaf_selects(source))
+
+
+def _row_source(select: ast.SelectQuery, chain: list[str | int], ctes: Mapping[str, ast.CTE]) -> Optional[ast.Expr]:
+ joins: list[ast.JoinExpr] = []
+ join = select.select_from
+ while join is not None:
+ joins.append(join)
+ join = join.next_join
+ if len(chain) > 1:
+ qualifier = str(chain[0])
+ found = next((join for join in joins if qualifier in (join.alias, _table_name(join))), None)
+ else:
+ found = joins[0] if len(joins) == 1 else None
+ if found is None:
+ return None
+ table_name = _table_name(found)
+ cte = ctes.get(table_name) if table_name else None
+ if cte is not None and cte.cte_type == "subquery":
+ return cte.expr
+ return found.table
+
+
+def _table_name(join: ast.JoinExpr) -> Optional[str]:
+ if isinstance(join.table, ast.Field) and join.table.chain:
+ return str(join.table.chain[-1])
+ return None
+
+
+def _value_expressions(select: ast.SelectQuery) -> list[ast.Expr]:
+ column_ctes = [cte.expr for cte in (select.ctes or {}).values() if cte.cte_type != "subquery"]
+ join_constraints = []
+ join: Optional[ast.JoinExpr] = select.select_from
+ while join is not None:
+ if join.constraint is not None:
+ join_constraints.append(join.constraint.expr)
+ join = join.next_join
+ filters = [expression for expression in (select.where, select.prewhere, select.having) if expression is not None]
+ return [
+ *select.select,
+ *(select.group_by or []),
+ *(select.array_join_list or []),
+ *filters,
+ *column_ctes,
+ *join_constraints,
+ ]
+
+
+def _grouped_output_names(select: ast.SelectQuery) -> Optional[list[str]]:
+ if not select.group_by:
+ return None
+ aliases = {item.alias for item in select.select if isinstance(item, ast.Alias)}
+ name_by_expression = {
+ repr(clear_locations(item.expr if isinstance(item, ast.Alias) else item)): name
+ for item in select.select
+ if (name := _output_name(item)) is not None
+ }
+ names = []
+ for entry in select.group_by:
+ if isinstance(entry, ast.Field) and len(entry.chain) == 1 and str(entry.chain[0]) in aliases:
+ names.append(str(entry.chain[0]))
+ continue
+ name = name_by_expression.get(repr(clear_locations(entry)))
+ if name is None:
+ return None
+ names.append(name)
+ return names
+
+
+def _is_time_type(clickhouse_type: Optional[str]) -> bool:
+ return clickhouse_type is not None and _without_low_cardinality(clickhouse_type).removeprefix(
+ "nullable("
+ ).startswith("date")
+
+
+def _without_low_cardinality(clickhouse_type: str) -> str:
+ return clickhouse_type.strip().lower().removeprefix("lowcardinality(")
+
+
def _unique(messages: list[str]) -> list[str]:
"""Recursing into every union branch, CTE and subquery can trip the same blocker several times;
the user needs to read it once."""
@@ -197,7 +384,7 @@ def _unique(messages: list[str]) -> list[str]:
def _resolved_selects(
- node: ast.SelectQuery | ast.SelectSetQuery, database: Optional["Database"]
+ node: ast.SelectQuery | ast.SelectSetQuery, database: Optional[Database]
) -> Optional[list[ast.SelectQuery]]:
"""The query's leaf selects with types resolved, which expands ``*`` to the table's columns.
@@ -299,7 +486,7 @@ def _expression_field_label(field_type: ast.ExpressionFieldType, context: HogQLC
return None
-def _key_candidate_types(selects: list[ast.SelectQuery], database: "Database") -> dict[str, str]:
+def _key_candidate_types(selects: list[ast.SelectQuery], database: Database) -> dict[str, str]:
"""Coarse type labels for the picker's type tags, from the resolved AST.
A raw table column resolves to its schema DatabaseField, which is the reliable source; only
@@ -463,6 +650,9 @@ def _check_key(select: ast.SelectQuery, config: IncrementalConfig, blockers: lis
def _check_unique_key(select: ast.SelectQuery, config: IncrementalConfig, blockers: list[str]) -> None:
+ if not config.unique_key:
+ blockers.append("Choose at least one unique key column, so each refresh can update the rows it recomputes.")
+ return
outputs = _output_names(select)
# An unexpanded star (no database to resolve against) passes every source column through, so a
# name that is not spelled out can still be in the output. The runtime upsert fails loudly if
@@ -476,6 +666,16 @@ def _check_unique_key(select: ast.SelectQuery, config: IncrementalConfig, blocke
)
return
+ aggregate_outputs = _aggregate_output_names(select)
+ aggregates = sorted(column for column in config.unique_key if column in aggregate_outputs)
+ if aggregates:
+ blockers.append(
+ f"Unique key {'columns' if len(aggregates) > 1 else 'column'} {', '.join(aggregates)} "
+ f"{'are aggregates' if len(aggregates) > 1 else 'is an aggregate'}. Its value changes as rows arrive, so "
+ "a refresh adds a new row instead of updating the stored one. Use the columns the query groups by."
+ )
+ return
+
if _is_grouped(select):
grouping = _grouping_key_names(select)
uncovered = sorted(grouping - set(config.unique_key))
@@ -486,6 +686,42 @@ def _check_unique_key(select: ast.SelectQuery, config: IncrementalConfig, blocke
)
+def _aggregate_output_names(select: ast.SelectQuery) -> set[str]:
+ aggregates: set[str] = set()
+ found_more = True
+ while found_more:
+ found_more = False
+ for item in select.select:
+ name = _output_name(item)
+ if name is None or name in aggregates:
+ continue
+ expression = item.expr if isinstance(item, ast.Alias) else item
+ if has_aggregation(expression) or _referenced_names(expression) & aggregates:
+ aggregates.add(name)
+ found_more = True
+ return aggregates
+
+
+def _referenced_names(expression: ast.Expr) -> set[str]:
+ collector = _FieldNameCollector()
+ collector.visit(expression)
+ return collector.names
+
+
+def _check_unique_key_types(
+ config: IncrementalConfig, key_candidate_types: dict[str, str], blockers: list[str]
+) -> None:
+ unusable = sorted(
+ column for column in config.unique_key if key_candidate_types.get(column) in _NON_UNIQUE_KEY_TYPE_LABELS
+ )
+ if unusable:
+ blockers.append(
+ f"Unique key {'columns' if len(unusable) > 1 else 'column'} {', '.join(unusable)} "
+ f"{'have' if len(unusable) > 1 else 'has'} a type that cannot identify a row, such as an array or a map. "
+ "Use plain columns for the unique key."
+ )
+
+
def _check_nullable_unique_key(
config: IncrementalConfig, column_types: Optional[dict[str, str]], blockers: list[str]
) -> None:
@@ -495,7 +731,7 @@ def _check_nullable_unique_key(
return
for column in config.unique_key:
clickhouse_type = column_types.get(column)
- if isinstance(clickhouse_type, str) and clickhouse_type.strip().lower().startswith("nullable("):
+ if isinstance(clickhouse_type, str) and _without_low_cardinality(clickhouse_type).startswith("nullable("):
blockers.append(
f'Unique key column "{column}" can be null. A null key adds a duplicate row on '
f"every run instead of updating. Wrap it in coalesce() to give it a fallback value."
@@ -637,3 +873,35 @@ def _check_determinism(node: ast.Expr, warnings: list[str]) -> None:
f"{names} makes each run depend on when it happened, so re-running a window can change "
"rows it already wrote."
)
+
+
+class _DataDependentValueFinder(TraversingVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.found = False
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ self.found = True
+
+ def visit_select_set_query(self, node: ast.SelectSetQuery) -> None:
+ self.found = True
+
+ def visit_compare_operation(self, node: ast.CompareOperation) -> None:
+ if node.op in (ast.CompareOperationOp.InCohort, ast.CompareOperationOp.NotInCohort):
+ self.found = True
+ super().visit_compare_operation(node)
+
+ def visit_call(self, node: ast.Call) -> None:
+ if node.name.lower() == "cohort":
+ self.found = True
+ super().visit_call(node)
+
+
+class _FieldNameCollector(TraversingVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.names: set[str] = set()
+
+ def visit_field(self, node: ast.Field) -> None:
+ if node.chain:
+ self.names.add(str(node.chain[-1]))
diff --git a/products/data_modeling/backend/logic/saved_query_materialization.py b/products/data_modeling/backend/logic/saved_query_materialization.py
index 21fc02bfcc43..ffa60d8c0bbd 100644
--- a/products/data_modeling/backend/logic/saved_query_materialization.py
+++ b/products/data_modeling/backend/logic/saved_query_materialization.py
@@ -1,7 +1,11 @@
from collections.abc import Callable, Mapping
from datetime import timedelta
+from enum import Enum
+from typing import Any, Literal
from uuid import UUID
+from django.db.models import Q, QuerySet
+
import structlog
from posthog.exceptions_capture import capture_exception
@@ -11,14 +15,21 @@ from posthog.rbac.query_access import assert_user_can_read_query
from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.data_modeling.backend.facade.contracts import (
+ IncrementalConfigRefusedError,
MaterializationFailedError,
MaterializationForbiddenError,
MaterializationRefusedError,
)
from products.data_modeling.backend.logic.freshness import UnsatisfiableFrequencyError, UnsupportedFrequencyTargetError
+from products.data_modeling.backend.logic.incremental import IncrementalConfig
+from products.data_modeling.backend.logic.incremental_eligibility import (
+ check_incremental_eligibility,
+ definition_database,
+)
from products.data_modeling.backend.logic.node_frequency import SavedQueryFrequencyBounds, saved_query_target_bounds
from products.data_modeling.backend.logic.node_materialization import SavedQueryNotFoundError
from products.data_modeling.backend.logic.saved_query_dag_sync import update_node_type
+from products.data_modeling.backend.logic.saved_query_reads import get_saved_query_columns
from products.data_modeling.backend.logic.schedule_reconcile import check_saved_query_frequency_target
from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
from products.data_modeling.backend.models.node import NodeType
@@ -33,6 +44,15 @@ LINEAGE_CHANGED_REFUSAL = "This view's lineage changed while we were setting it
MATERIALIZATION_FAILED_MESSAGE = "Materialization failed. Please try again or contact support."
MATERIALIZATION_ENABLED_ACTIVITY = "materialization_enabled"
SAVED_QUERY_ACTIVITY_SCOPE = "DataWarehouseSavedQuery"
+VIEW_CHANGED_REFUSAL = "This view changed while we were setting it up. Try again."
+
+
+class KeepIncrementalConfig(Enum):
+ KEEP = "keep"
+
+
+KEEP_INCREMENTAL_CONFIG = KeepIncrementalConfig.KEEP
+IncrementalConfigUpdate = IncrementalConfig | None | Literal[KeepIncrementalConfig.KEEP]
def enable_saved_query_materialization(
@@ -43,6 +63,7 @@ def enable_saved_query_materialization(
sync_frequency_interval: timedelta | None,
visible_blocker_names: VisibleBlockerNames,
was_impersonated: bool,
+ incremental_config: IncrementalConfigUpdate = KEEP_INCREMENTAL_CONFIG,
) -> None:
"""Materialize a saved query at the given cadence and start its first run."""
saved_query = (
@@ -56,6 +77,7 @@ def enable_saved_query_materialization(
sync_frequency_interval=sync_frequency_interval,
visible_blocker_names=visible_blocker_names,
was_impersonated=was_impersonated,
+ incremental_config=incremental_config,
)
@@ -66,6 +88,7 @@ def _enable_materialization(
sync_frequency_interval: timedelta | None,
visible_blocker_names: VisibleBlockerNames,
was_impersonated: bool,
+ incremental_config: IncrementalConfigUpdate,
) -> None:
if saved_query.managed_viewset_id is not None:
raise MaterializationRefusedError(MANAGED_VIEWSET_REFUSAL)
@@ -81,6 +104,9 @@ def _enable_materialization(
previous_interval = saved_query.sync_frequency_interval
previously_materialized = saved_query.is_materialized
+ previous_incremental_config = saved_query.incremental_config
+ if incremental_config is not KEEP_INCREMENTAL_CONFIG:
+ _store_incremental_config(saved_query, incremental_config, user)
_save_materialization(saved_query, is_materialized=True, sync_frequency_interval=sync_frequency_interval)
# If this fails, it will set is_materialized = False
@@ -96,6 +122,8 @@ def _enable_materialization(
_save_materialization(
saved_query, is_materialized=previously_materialized, sync_frequency_interval=previous_interval
)
+ if incremental_config is not KEEP_INCREMENTAL_CONFIG:
+ _restore_incremental_config(saved_query, previous_incremental_config)
raise MaterializationRefusedError(LINEAGE_CHANGED_REFUSAL) from error
# Refresh from DB to check if schedule_materialization set is_materialized = False on failure
@@ -135,6 +163,50 @@ def _refuse_a_cadence_the_lineage_forbids(
raise MaterializationRefusedError(str(error)) from error
+def _store_incremental_config(
+ saved_query: DataWarehouseSavedQuery, incremental_config: IncrementalConfig | None, user: User
+) -> None:
+ if incremental_config is not None:
+ _refuse_an_incremental_config_the_query_forbids(saved_query, incremental_config, user)
+ stored = None if incremental_config is None else _stored_incremental_config(incremental_config)
+ if not _unchanged_since_read(saved_query).update(incremental_config=stored):
+ raise IncrementalConfigRefusedError(VIEW_CHANGED_REFUSAL)
+ saved_query.incremental_config = stored
+
+
+def _restore_incremental_config(saved_query: DataWarehouseSavedQuery, previous: dict[str, Any] | None) -> None:
+ _unchanged_since_read(saved_query).update(incremental_config=previous)
+ saved_query.incremental_config = previous
+
+
+def _unchanged_since_read(saved_query: DataWarehouseSavedQuery) -> "QuerySet[DataWarehouseSavedQuery]":
+ config = saved_query.incremental_config
+ same_config = Q(incremental_config__isnull=True) if config is None else Q(incremental_config=config)
+ return DataWarehouseSavedQuery.objects.filter(same_config, pk=saved_query.pk, query=saved_query.query)
+
+
+def _refuse_an_incremental_config_the_query_forbids(
+ saved_query: DataWarehouseSavedQuery, incremental_config: IncrementalConfig, user: User
+) -> None:
+ result = check_incremental_eligibility(
+ (saved_query.query or {}).get("query") or "",
+ incremental_config,
+ column_types=get_saved_query_columns(saved_query.team_id, saved_query.pk),
+ database=definition_database(saved_query.team_id, user=user),
+ )
+ if not result.eligible:
+ raise IncrementalConfigRefusedError(" ".join(result.blockers), tuple(result.blocked_keys))
+
+
+def _stored_incremental_config(incremental_config: IncrementalConfig) -> dict[str, Any]:
+ return {
+ "enabled": True,
+ "incremental_key": incremental_config.incremental_key,
+ "unique_key": list(incremental_config.unique_key),
+ "lookback_seconds": incremental_config.lookback_seconds,
+ }
+
+
def _save_materialization(
saved_query: DataWarehouseSavedQuery, *, is_materialized: bool | None, sync_frequency_interval: timedelta | None
) -> None:
diff --git a/products/data_modeling/backend/test/test_incremental_eligibility.py b/products/data_modeling/backend/test/test_incremental_eligibility.py
index 2c86fc94d9dc..7d3068886873 100644
--- a/products/data_modeling/backend/test/test_incremental_eligibility.py
+++ b/products/data_modeling/backend/test/test_incremental_eligibility.py
@@ -5,10 +5,13 @@ from django.test import SimpleTestCase
from parameterized import parameterized
from posthog.hogql.database.database import Database
-from posthog.hogql.database.models import IntegerDatabaseField, SavedQuery, TableNode
+from posthog.hogql.database.models import DateTimeDatabaseField, IntegerDatabaseField, SavedQuery, TableNode
from products.data_modeling.backend.logic.incremental import IncrementalConfig
-from products.data_modeling.backend.logic.incremental_eligibility import check_incremental_eligibility
+from products.data_modeling.backend.logic.incremental_eligibility import (
+ check_incremental_eligibility,
+ propose_incremental_config,
+)
DAY_KEY = IncrementalConfig(incremental_key="day", unique_key=("day",))
DAY_EVENT_KEY = IncrementalConfig(incremental_key="day", unique_key=("day", "event"))
@@ -84,6 +87,24 @@ class TestIncrementalEligibility(SimpleTestCase):
@parameterized.expand(
[
+ (
+ "unique_key_is_empty",
+ "SELECT timestamp FROM events",
+ IncrementalConfig(incremental_key="timestamp", unique_key=()),
+ "at least one unique key column",
+ ),
+ (
+ "unique_key_includes_an_aggregate",
+ GROUPED,
+ IncrementalConfig(incremental_key="day", unique_key=("day", "c")),
+ "is an aggregate",
+ ),
+ (
+ "unique_key_includes_an_alias_of_an_aggregate",
+ "SELECT toStartOfDay(timestamp) AS day, count() AS c, c + 1 AS revision FROM events GROUP BY day",
+ IncrementalConfig(incremental_key="day", unique_key=("day", "revision")),
+ "is an aggregate",
+ ),
(
"unique_key_misses_a_grouping_column",
"SELECT toStartOfDay(timestamp) AS day, event, count() AS c FROM events GROUP BY day, event",
@@ -473,3 +494,253 @@ class TestStarExpansion:
assert not result.eligible
assert any(expected in blocker for blocker in result.blockers), result.blockers
+
+
+DAILY_EVENT_TYPES = {"day": "DateTime64(6, 'UTC')", "event": "String", "c": "UInt64"}
+
+
+class TestProposeIncrementalConfig(SimpleTestCase):
+ @parameterized.expand(
+ [
+ (
+ "grouped_by_one_time_column",
+ "SELECT toStartOfDay(timestamp) AS day, event, count() AS c FROM events GROUP BY day, event",
+ DAILY_EVENT_TYPES,
+ DAY_EVENT_KEY,
+ ),
+ ("not_grouped", "SELECT uuid, timestamp FROM events", {"uuid": "UUID", "timestamp": "DateTime64(6)"}, None),
+ (
+ "two_time_columns",
+ "SELECT toStartOfDay(timestamp) AS day, toStartOfHour(timestamp) AS hour, count() AS c "
+ "FROM events GROUP BY day, hour",
+ {"day": "DateTime", "hour": "DateTime", "c": "UInt64"},
+ None,
+ ),
+ (
+ "nullable_grouped_column",
+ "SELECT toStartOfDay(timestamp) AS day, event, count() AS c FROM events GROUP BY day, event",
+ {**DAILY_EVENT_TYPES, "event": "Nullable(String)"},
+ None,
+ ),
+ (
+ "grouped_by_a_column_the_output_drops",
+ "SELECT toStartOfDay(timestamp) AS day, count() AS c FROM events GROUP BY day, event",
+ DAILY_EVENT_TYPES,
+ None,
+ ),
+ ("column_types_unknown", GROUPED, {}, None),
+ (
+ "grouping_repeats_the_expression",
+ "SELECT toStartOfDay(timestamp) AS day, count() AS c FROM events GROUP BY toStartOfDay(timestamp)",
+ {"day": "DateTime", "c": "UInt64"},
+ DAY_KEY,
+ ),
+ (
+ "key_from_a_nondeterministic_function",
+ "SELECT today() AS day, event, count() AS c FROM events GROUP BY day, event",
+ {**DAILY_EVENT_TYPES, "day": "Date"},
+ None,
+ ),
+ (
+ "moving_window_filter",
+ f"{GROUPED.replace(' GROUP BY', ' WHERE timestamp >= now() - INTERVAL 7 DAY GROUP BY')}",
+ {"day": "DateTime", "c": "UInt64"},
+ None,
+ ),
+ (
+ "grouped_by_a_source_column_the_output_transforms",
+ "SELECT toStartOfDay(timestamp) AS day, lower(e.event) AS event, count() AS c "
+ "FROM events AS e GROUP BY day, e.event",
+ DAILY_EVENT_TYPES,
+ None,
+ ),
+ (
+ "group_by_all_with_a_mixed_aggregate",
+ "SELECT toStartOfDay(timestamp) AS day, concat(event, toString(count())) AS c FROM events GROUP BY ALL",
+ {"day": "DateTime", "c": "String"},
+ None,
+ ),
+ (
+ "moving_window_from_a_clock_alias",
+ GROUPED.replace(" GROUP BY", " WHERE timestamp >= current_date() - INTERVAL 7 DAY GROUP BY"),
+ {"day": "DateTime", "c": "UInt64"},
+ None,
+ ),
+ (
+ "moving_window_from_a_scalar_subquery",
+ GROUPED.replace(
+ " GROUP BY", " WHERE timestamp >= (SELECT max(timestamp) FROM events) - INTERVAL 7 DAY GROUP BY"
+ ),
+ {"day": "DateTime", "c": "UInt64"},
+ None,
+ ),
+ (
+ "moving_window_from_a_column_cte",
+ "WITH (SELECT max(timestamp) FROM events) AS latest "
+ + GROUPED.replace(" GROUP BY", " WHERE timestamp >= latest - INTERVAL 7 DAY GROUP BY"),
+ {"day": "DateTime", "c": "UInt64"},
+ None,
+ ),
+ (
+ "cohort_filter",
+ GROUPED.replace(" GROUP BY", " WHERE person_id IN COHORT 1 GROUP BY"),
+ {"day": "DateTime", "c": "UInt64"},
+ None,
+ ),
+ (
+ "cohort_function_filter",
+ GROUPED.replace(" GROUP BY", " WHERE person_id IN cohort(1) GROUP BY"),
+ {"day": "DateTime", "c": "UInt64"},
+ None,
+ ),
+ (
+ "date_key_that_wraps_at_the_year",
+ "SELECT toDate(subtractYears(timestamp, toYear(timestamp) - 2000)) AS day, count() AS c "
+ "FROM events GROUP BY day",
+ {"day": "Date", "c": "UInt64"},
+ None,
+ ),
+ (
+ "date_key_that_wraps_inside_a_subquery",
+ "SELECT day, count() AS c FROM (SELECT toDate(subtractYears(timestamp, toYear(timestamp) - 2000)) "
+ "AS day FROM events) GROUP BY day",
+ {"day": "Date", "c": "UInt64"},
+ None,
+ ),
+ (
+ "day_key_inside_a_cte_read_from_a_subquery",
+ "WITH good AS (SELECT toStartOfDay(timestamp) AS ts FROM events GROUP BY ts) "
+ "SELECT toStartOfDay(sub.ts) AS day, count() AS c FROM (SELECT ts FROM good) AS sub GROUP BY day",
+ {"day": "DateTime", "c": "UInt64"},
+ DAY_KEY,
+ ),
+ (
+ "date_key_that_wraps_inside_a_cte_read_from_a_subquery",
+ "WITH bad AS (SELECT toDate(subtractYears(timestamp, toYear(timestamp) - 2000)) AS ts FROM events "
+ "GROUP BY ts) SELECT toStartOfDay(sub.ts) AS day, count() AS c FROM (SELECT ts FROM bad) AS sub "
+ "GROUP BY day",
+ {"day": "DateTime", "c": "UInt64"},
+ None,
+ ),
+ (
+ "array_join_over_a_subquery",
+ GROUPED.replace(" GROUP BY", " ARRAY JOIN (SELECT groupArray(event) FROM events) AS labels GROUP BY"),
+ {"day": "DateTime", "c": "UInt64"},
+ None,
+ ),
+ (
+ "union_branches_can_repeat_a_key",
+ f"{GROUPED} UNION ALL {GROUPED}",
+ {"day": "DateTime", "c": "UInt64"},
+ None,
+ ),
+ (
+ "nullable_grouped_column_behind_low_cardinality",
+ "SELECT toStartOfDay(timestamp) AS day, event, count() AS c FROM events GROUP BY day, event",
+ {**DAILY_EVENT_TYPES, "event": "LowCardinality(Nullable(String))"},
+ None,
+ ),
+ (
+ "group_by_all_over_an_unnamed_expression",
+ "SELECT toStartOfDay(timestamp) AS day, lower(event), count() AS c FROM events GROUP BY ALL",
+ DAILY_EVENT_TYPES,
+ None,
+ ),
+ ]
+ )
+ def test_proposes_a_config_only_when_the_keys_are_certain(
+ self, _name: str, query: str, column_types: dict[str, str], expected: Optional[IncrementalConfig]
+ ) -> None:
+ assert propose_incremental_config(query, column_types) == expected
+
+ @parameterized.expand(
+ [
+ (
+ "grouped_by_a_plain_column",
+ "SELECT toStartOfDay(timestamp) AS day, event, count() AS c FROM events GROUP BY day, event",
+ DAILY_EVENT_TYPES,
+ DAY_EVENT_KEY,
+ ),
+ (
+ "grouped_by_an_array_column",
+ "SELECT toStartOfDay(timestamp) AS day, splitByChar(',', event) AS labels, count() AS c "
+ "FROM events GROUP BY day, labels",
+ {"day": "DateTime", "labels": "Array(String)", "c": "UInt64"},
+ None,
+ ),
+ ]
+ )
+ def test_proposals_against_the_resolved_schema(
+ self, _name: str, query: str, column_types: dict[str, str], expected: Optional[IncrementalConfig]
+ ) -> None:
+ assert propose_incremental_config(query, column_types, database=Database()) == expected
+
+ @parameterized.expand(
+ [
+ ("key_from_the_view_that_advances", "a", DAY_KEY),
+ ("key_from_the_view_that_wraps", "b", None),
+ ]
+ )
+ def test_a_joined_time_key_is_checked_in_the_view_it_comes_from(
+ self, _name: str, key_view: str, expected: Optional[IncrementalConfig]
+ ) -> None:
+ database = Database()
+ database._add_views(
+ TableNode(
+ children={
+ name: TableNode(
+ name=name,
+ table=SavedQuery(
+ id=name,
+ name=name,
+ query=f"SELECT {expression} AS ts FROM events GROUP BY ts",
+ fields={"ts": DateTimeDatabaseField(name="ts")},
+ ),
+ )
+ for name, expression in (
+ ("a", "toStartOfDay(timestamp)"),
+ ("b", "toDate(subtractYears(timestamp, toYear(timestamp) - 2000))"),
+ )
+ }
+ )
+ )
+
+ proposed = propose_incremental_config(
+ f"SELECT toStartOfDay({key_view}.ts) AS day, count() AS c FROM a CROSS JOIN b GROUP BY day",
+ {"day": "DateTime", "c": "UInt64"},
+ database=database,
+ )
+
+ assert proposed == expected
+
+ @parameterized.expand(
+ [
+ ("moving_window", "SELECT timestamp FROM events WHERE timestamp >= now() - INTERVAL 7 DAY"),
+ ("cohort_filter", "SELECT timestamp FROM events WHERE person_id IN COHORT 1"),
+ ]
+ )
+ def test_a_data_dependent_filter_inside_a_view_the_query_reads_blocks_the_proposal(
+ self, _name: str, view_query: str
+ ) -> None:
+ database = Database()
+ database._add_views(
+ TableNode(
+ children={
+ "last_week": TableNode(
+ name="last_week",
+ table=SavedQuery(
+ id="last_week",
+ name="last_week",
+ query=view_query,
+ fields={"timestamp": DateTimeDatabaseField(name="timestamp")},
+ ),
+ )
+ }
+ )
+ )
+
+ proposed = propose_incremental_config(
+ GROUPED.replace("FROM events", "FROM last_week"), {"day": "DateTime", "c": "UInt64"}, database=database
+ )
+
+ assert proposed is None
diff --git a/products/data_modeling/backend/test/test_saved_query_materialization.py b/products/data_modeling/backend/test/test_saved_query_materialization.py
index dd1ccce70623..6f31ca0c4d7b 100644
--- a/products/data_modeling/backend/test/test_saved_query_materialization.py
+++ b/products/data_modeling/backend/test/test_saved_query_materialization.py
@@ -3,6 +3,8 @@ from datetime import timedelta
from posthog.test.base import BaseTest
from unittest.mock import AsyncMock, patch
+from parameterized import parameterized
+
from posthog.constants import AvailableFeature
from posthog.models import ActivityLog, User
from posthog.models.organization import OrganizationMembership
@@ -10,13 +12,21 @@ from posthog.models.organization import OrganizationMembership
from products.access_control.backend.models.access_control import AccessControl
from products.data_modeling.backend.facade import api
from products.data_modeling.backend.logic.freshness import UnsatisfiableFrequencyError
+from products.data_modeling.backend.logic.incremental import IncrementalConfig
from products.data_modeling.backend.models.dag import DAG
from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
from products.data_modeling.backend.models.edge import Edge
from products.data_modeling.backend.models.node import Node, NodeType
from products.data_modeling.backend.test.helpers import warehouse_source_node
+from products.warehouse_sources.backend.facade.models import DataWarehouseTable
GET_V2_DAG_IDS = "products.data_modeling.backend.schedule.get_v2_scheduled_dag_ids"
+DAILY_COUNTS = "SELECT toStartOfDay(timestamp) AS day, count() AS c FROM events GROUP BY day"
+READ_ACCESS_CHECK = "products.data_modeling.backend.logic.saved_query_materialization.assert_user_can_read_query"
+DAY_KEY = IncrementalConfig(incremental_key="day", unique_key=("day",))
+TIMESTAMP_COLUMN = {"timestamp": {"hogql": "DateTimeDatabaseField", "clickhouse": "DateTime64(6, 'UTC')"}}
+HOSTILE_KEY = IncrementalConfig(incremental_key='day"; DROP TABLE events; --', unique_key=("day",))
+STORED_DAY_KEY = {"enabled": True, "incremental_key": "day", "unique_key": ["day"], "lookback_seconds": 0}
NODE_MATERIALIZATION_SYNC_CONNECT = "products.data_modeling.backend.logic.node_materialization.sync_connect"
@@ -38,7 +48,12 @@ class TestEnableSavedQueryMaterialization(BaseTest):
team=self.team, dag=self.dag, name="event_view", saved_query=self.saved_query, type=NodeType.VIEW
)
- def _enable(self, sync_frequency_interval: timedelta, user: User | None = None) -> None:
+ def _enable(
+ self,
+ sync_frequency_interval: timedelta,
+ user: User | None = None,
+ incremental_config: IncrementalConfig | None = None,
+ ) -> None:
api.enable_saved_query_materialization(
self.team.id,
self.saved_query.id,
@@ -46,6 +61,7 @@ class TestEnableSavedQueryMaterialization(BaseTest):
sync_frequency_interval=sync_frequency_interval,
visible_blocker_names=_name_no_blockers,
was_impersonated=False,
+ incremental_config=incremental_config,
)
def test_enabling_retypes_the_node_logs_the_change_and_starts_a_run_for_the_user(self) -> None:
@@ -142,3 +158,111 @@ class TestEnableSavedQueryMaterialization(BaseTest):
self.saved_query.refresh_from_db()
schedule_materialization.assert_not_called()
assert self.saved_query.is_materialized is False
+
+ @parameterized.expand(
+ [
+ ("incremental", None, DAY_KEY, STORED_DAY_KEY),
+ ("full_refresh_clears_leftover_keys", STORED_DAY_KEY, None, None),
+ ]
+ )
+ def test_the_refresh_mode_is_stored_before_the_first_run(
+ self,
+ _name: str,
+ stored_before: dict | None,
+ incremental_config: IncrementalConfig | None,
+ expected: dict | None,
+ ) -> None:
+ self.saved_query.query = {"kind": "HogQLQuery", "query": DAILY_COUNTS}
+ self.saved_query.incremental_config = stored_before
+ self.saved_query.save(update_fields=["query", "incremental_config"])
+ configs_seen_by_the_first_run: list[dict | None] = []
+
+ def record_config(saved_query: DataWarehouseSavedQuery, **_kwargs: object) -> None:
+ configs_seen_by_the_first_run.append(
+ DataWarehouseSavedQuery.objects.get(id=saved_query.id).incremental_config
+ )
+
+ with patch.object(
+ DataWarehouseSavedQuery, "schedule_materialization", autospec=True, side_effect=record_config
+ ):
+ self._enable(timedelta(hours=1), incremental_config=incremental_config)
+
+ assert configs_seen_by_the_first_run == [expected]
+
+ @parameterized.expand([("incremental", DAY_KEY), ("full_refresh", None)])
+ def test_a_config_edited_after_the_view_was_read_is_not_overwritten(
+ self, _name: str, incremental_config: IncrementalConfig | None
+ ) -> None:
+ self.saved_query.query = {"kind": "HogQLQuery", "query": DAILY_COUNTS}
+ self.saved_query.save(update_fields=["query"])
+ edited_config = {**STORED_DAY_KEY, "lookback_seconds": 3600}
+
+ def edit_config_concurrently(*_args: object, **_kwargs: object) -> None:
+ DataWarehouseSavedQuery.objects.filter(id=self.saved_query.id).update(incremental_config=edited_config)
+
+ with (
+ patch(READ_ACCESS_CHECK, side_effect=edit_config_concurrently),
+ patch.object(DataWarehouseSavedQuery, "schedule_materialization") as schedule_materialization,
+ self.assertRaises(api.IncrementalConfigRefusedError),
+ ):
+ self._enable(timedelta(hours=1), incremental_config=incremental_config)
+
+ self.saved_query.refresh_from_db()
+ schedule_materialization.assert_not_called()
+ assert self.saved_query.incremental_config == edited_config
+
+ @parameterized.expand(
+ [
+ ("limit_in_its_own_query", None, None, DAY_KEY, False),
+ (
+ "limit_inside_a_view_it_reads",
+ "SELECT timestamp FROM events ORDER BY timestamp DESC LIMIT 100",
+ "SELECT toStartOfDay(timestamp) AS day, count() AS c FROM recent_events GROUP BY day",
+ DAY_KEY,
+ False,
+ ),
+ (
+ "limit_inside_a_materialized_view_it_reads",
+ "SELECT timestamp FROM events ORDER BY timestamp DESC LIMIT 100",
+ "SELECT toStartOfDay(timestamp) AS day, count() AS c FROM recent_events GROUP BY day",
+ DAY_KEY,
+ True,
+ ),
+ ("key_that_is_not_an_output_column", None, DAILY_COUNTS, HOSTILE_KEY, False),
+ ]
+ )
+ def test_an_incremental_config_the_query_forbids_is_refused_before_anything_is_written(
+ self,
+ _name: str,
+ upstream_query: str | None,
+ query: str | None,
+ incremental_config: IncrementalConfig,
+ upstream_materialized: bool,
+ ) -> None:
+ if upstream_query is not None:
+ DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name="recent_events",
+ query={"kind": "HogQLQuery", "query": upstream_query},
+ columns=TIMESTAMP_COLUMN,
+ is_materialized=upstream_materialized,
+ table=DataWarehouseTable.objects.create(
+ team=self.team, name="recent_events_tbl", format="Delta", columns=TIMESTAMP_COLUMN
+ )
+ if upstream_materialized
+ else None,
+ )
+ if query is not None:
+ self.saved_query.query = {"kind": "HogQLQuery", "query": query}
+ self.saved_query.save(update_fields=["query"])
+
+ with (
+ patch.object(DataWarehouseSavedQuery, "schedule_materialization") as schedule_materialization,
+ self.assertRaises(api.IncrementalConfigRefusedError),
+ ):
+ self._enable(timedelta(hours=1), incremental_config=incremental_config)
+
+ self.saved_query.refresh_from_db()
+ schedule_materialization.assert_not_called()
+ assert self.saved_query.is_materialized is False
+ assert self.saved_query.incremental_config is None
diff --git a/products/data_warehouse/backend/presentation/views/saved_query/editing.py b/products/data_warehouse/backend/presentation/views/saved_query/editing.py
index aec2cce5a383..4fb449f7b412 100644
--- a/products/data_warehouse/backend/presentation/views/saved_query/editing.py
+++ b/products/data_warehouse/backend/presentation/views/saved_query/editing.py
@@ -15,7 +15,6 @@ from rest_framework import exceptions, serializers
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.database import (
MODELS_NAMESPACE_QUERY_ERROR,
- Database,
is_reserved_models_name,
models_namespace_chain,
)
@@ -129,6 +128,11 @@ def _authored_model_answering_to(
return next((query for query in candidates if models_namespace_chain(query) == name.split(".")), None)
+INCREMENTAL_CONFIG_CHANGED_REFUSAL = (
+ "This view's incremental settings changed while you were editing it. Reload the view and try again."
+)
+
+
class DataWarehouseSavedQuerySerializer(
view_state.DataWarehouseSavedQuerySerializerMixin, UserAccessControlSerializerMixin, serializers.ModelSerializer
):
@@ -515,6 +519,13 @@ class DataWarehouseSavedQuerySerializer(
if _as_uuid(edited_history_id) != locked_instance.query_revision:
raise serializers.ValidationError("The query was modified by someone else.")
+ if (
+ query_changed
+ and "incremental_config" not in validated_data
+ and locked_instance.incremental_config != instance.incremental_config
+ ):
+ raise serializers.ValidationError(INCREMENTAL_CONFIG_CHANGED_REFUSAL)
+
if query_changed:
validated_data["query_revision"] = uuid.uuid4()
@@ -674,16 +685,16 @@ class DataWarehouseSavedQuerySerializer(
if not isinstance(sql, str):
raise serializers.ValidationError({"incremental": "This view has no query to make incremental."})
- from products.data_modeling.backend.facade.api import IncrementalConfig, check_incremental_eligibility
+ from products.data_modeling.backend.facade.api import (
+ IncrementalConfig,
+ check_incremental_eligibility,
+ definition_database,
+ )
# The stored column types describe the stored query, so they say nothing about a query being
# replaced. The runtime guard still catches a nullable key on the first incremental run.
column_types = None if query_changed or self.instance is None else self.instance.columns
- # The context only carries a database when the request touches the query or name; a
- # config-only PATCH still has to check `SELECT *` against real columns, so build one then.
- database = self.context.get("database") or Database.create_for(
- team_id=self.context["team_id"], user=cast(User, self.context["request"].user)
- )
+ database = definition_database(self.context["team_id"], user=cast(User, self.context["request"].user))
result = check_incremental_eligibility(
sql,
IncrementalConfig(
diff --git a/products/data_warehouse/backend/presentation/views/saved_query/viewset.py b/products/data_warehouse/backend/presentation/views/saved_query/viewset.py
index 736818a69f93..61bf9eb5a7ab 100644
--- a/products/data_warehouse/backend/presentation/views/saved_query/viewset.py
+++ b/products/data_warehouse/backend/presentation/views/saved_query/viewset.py
@@ -315,7 +315,11 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSe
Parses the SQL only, so it is cheap enough to call from the editor as the user types. Lets
the editor explain why the incremental option is unavailable before anything is saved.
"""
- from products.data_modeling.backend.facade.api import IncrementalConfig, check_incremental_eligibility
+ from products.data_modeling.backend.facade.api import (
+ IncrementalConfig,
+ check_incremental_eligibility,
+ definition_database,
+ )
body = incremental_config.CheckIncrementalSerializer(data=request.data)
body.is_valid(raise_exception=True)
@@ -332,7 +336,7 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, AccessControlViewSe
result = check_incremental_eligibility(
data["query"],
config,
- database=Database.create_for(team_id=self.team_id, user=cast(User, request.user)),
+ database=definition_database(self.team_id, user=cast(User, request.user)),
)
return response.Response(
incremental_config.IncrementalEligibilitySerializer(
diff --git a/products/data_warehouse/backend/tests/api/test_saved_query.py b/products/data_warehouse/backend/tests/api/test_saved_query.py
index 6ec00c02dd4b..53957a2373c6 100644
--- a/products/data_warehouse/backend/tests/api/test_saved_query.py
+++ b/products/data_warehouse/backend/tests/api/test_saved_query.py
@@ -34,7 +34,10 @@ from products.data_modeling.backend.facade.models import (
NodeType,
)
from products.data_tools.backend.models.datawarehouse_saved_query_folder import DataWarehouseSavedQueryFolder
-from products.data_warehouse.backend.presentation.views.saved_query.editing import DataWarehouseSavedQuerySerializer
+from products.data_warehouse.backend.presentation.views.saved_query.editing import (
+ INCREMENTAL_CONFIG_CHANGED_REFUSAL,
+ DataWarehouseSavedQuerySerializer,
+)
from products.data_warehouse.backend.presentation.views.saved_query.viewset import (
SavedQueryMaterializeSerializer,
SavedQueryResumeSchedulesRequestSerializer,
@@ -2637,6 +2640,36 @@ class TestSavedQuery(APIBaseTest):
cache.clear()
+ def test_a_query_edit_is_refused_when_the_incremental_settings_changed_after_the_view_was_read(self) -> None:
+ stored_config = {"enabled": True, "incremental_key": "day", "unique_key": ["day"], "lookback_seconds": 0}
+ view = DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name="daily_counts",
+ query={
+ "kind": "HogQLQuery",
+ "query": "SELECT toStartOfDay(timestamp) AS day, count() AS c FROM events GROUP BY day",
+ },
+ created_by=self.user,
+ )
+ new_query = {
+ "kind": "HogQLQuery",
+ "query": "SELECT toStartOfDay(timestamp) AS day, event, count() AS c FROM events GROUP BY day, event",
+ }
+
+ def store_config_concurrently(*_args: object, **_kwargs: object) -> dict:
+ DataWarehouseSavedQuery.objects.filter(id=view.id).update(incremental_config=stored_config)
+ return {"day": {"hogql": "DateTimeDatabaseField", "clickhouse": "DateTime64(6, 'UTC')", "valid": True}}
+
+ with patch.object(DataWarehouseSavedQuery, "get_columns", side_effect=store_config_concurrently):
+ response = self.client.patch(
+ f"/api/projects/{self.team.id}/warehouse_saved_queries/{view.id}",
+ {"query": new_query, "edited_history_id": str(view.query_revision)},
+ )
+
+ view.refresh_from_db()
+ assert (response.status_code, response.json()["detail"]) == (400, INCREMENTAL_CONFIG_CHANGED_REFUSAL)
+ assert view.query != new_query
+
class TestSavedQueryNameValidation(SimpleTestCase):
def test_unchanged_legacy_root_name_is_allowed_by_serializer(self) -> None:
diff --git a/products/data_warehouse/backend/tests/api/test_saved_query_incremental.py b/products/data_warehouse/backend/tests/api/test_saved_query_incremental.py
index 77a02cc2398e..913e620606dd 100644
--- a/products/data_warehouse/backend/tests/api/test_saved_query_incremental.py
+++ b/products/data_warehouse/backend/tests/api/test_saved_query_incremental.py
@@ -1,15 +1,19 @@
from posthog.test.base import APIBaseTest
from unittest.mock import AsyncMock, patch
+from parameterized import parameterized
+
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
from products.data_warehouse.backend.presentation.views.saved_query.incremental_config import (
CHECK_INCREMENTAL_MAX_QUERY_LENGTH,
CheckIncrementalThrottle,
)
from products.data_warehouse.backend.presentation.views.saved_query.viewset import DataWarehouseSavedQueryViewSet
+from products.warehouse_sources.backend.facade.models import DataWarehouseTable
GROUPED = "SELECT toStartOfDay(timestamp) AS day, count() AS c FROM events GROUP BY day"
CONFIG = {"enabled": True, "incremental_key": "day", "unique_key": ["day"]}
+TIMESTAMP_COLUMN = {"timestamp": {"hogql": "DateTimeDatabaseField", "clickhouse": "DateTime64(6, 'UTC')"}}
class TestSavedQueryIncremental(APIBaseTest):
@@ -30,16 +34,38 @@ class TestSavedQueryIncremental(APIBaseTest):
assert saved_query.incremental_config == {**CONFIG, "lookback_seconds": 0}
assert response.json()["incremental"]["incremental_key"] == "day"
- def test_an_ineligible_query_is_rejected_with_the_reason(self):
+ @parameterized.expand(
+ [
+ (
+ "unique_key_misses_a_grouping_column",
+ "SELECT toStartOfDay(timestamp) AS day, event, count() AS c FROM events GROUP BY day, event",
+ "GROUP BY",
+ ),
+ (
+ "limit_inside_a_materialized_view_it_reads",
+ "SELECT toStartOfDay(timestamp) AS day, count() AS c FROM recent_events GROUP BY day",
+ "LIMIT",
+ ),
+ ]
+ )
+ def test_an_ineligible_query_is_rejected_with_the_reason(self, _name: str, query: str, expected: str):
"""The 400 has to name the construct, or the user has no way to fix it."""
- response = self._create(
- query="SELECT toStartOfDay(timestamp) AS day, event, count() AS c FROM events GROUP BY day, event",
- incremental=CONFIG,
+ DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name="recent_events",
+ query={"kind": "HogQLQuery", "query": "SELECT timestamp FROM events ORDER BY timestamp DESC LIMIT 100"},
+ columns=TIMESTAMP_COLUMN,
+ is_materialized=True,
+ table=DataWarehouseTable.objects.create(
+ team=self.team, name="recent_events_tbl", format="Delta", columns=TIMESTAMP_COLUMN
+ ),
)
+ response = self._create(query=query, incremental=CONFIG)
+
assert response.status_code == 400
assert response.json()["attr"] == "incremental"
- assert "GROUP BY" in response.json()["detail"]
+ assert expected in response.json()["detail"]
def test_a_disabled_config_skips_the_eligibility_check(self):
"""Turning it off must never be blocked by why it could not be turned on."""