Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-modeling): let a managed view set its cadence and incremental config (#118655)

Raúl Negrón-Otero0142bae057a8
diff --git a/products/data_modeling/backend/facade/managed_viewset_hooks.py b/products/data_modeling/backend/facade/managed_viewset_hooks.py
index 28ecf2abe479..9070b8ecc7b0 100644
--- a/products/data_modeling/backend/facade/managed_viewset_hooks.py
+++ b/products/data_modeling/backend/facade/managed_viewset_hooks.py
@@ -19,13 +19,18 @@ from __future__ import annotations
import dataclasses
from collections.abc import Callable
-from typing import TYPE_CHECKING
+from datetime import timedelta
+from typing import TYPE_CHECKING, Any
+
+from products.data_modeling.backend.logic.freshness import ensure_schedulable
if TYPE_CHECKING:
from posthog.hogql.database.models import FieldOrTable
from posthog.models.team import Team
+DEFAULT_MANAGED_VIEW_SYNC_FREQUENCY = timedelta(hours=12)
+
@dataclasses.dataclass(frozen=True)
class ProvidedView:
@@ -37,9 +42,18 @@ class ProvidedView:
name: str
query: str
fields: dict[str, FieldOrTable]
- # Materialized views get a 12h sync schedule. A non-materialized view (e.g. engineering
- # analytics) is computed at query time — no schedule, no S3 table.
+ # Non-materialized views are computed at query time, without a schedule or S3 table.
materialized: bool = True
+ # One of the schedulable cadence buckets, stored as declared: readers never change it, and a run
+ # is never scheduled finer than the view's sources deliver.
+ sync_frequency: timedelta = DEFAULT_MANAGED_VIEW_SYNC_FREQUENCY
+ # The saved query's ``incremental_config`` blob: a run then upserts the rows past the last
+ # watermark instead of rebuilding the table. None keeps every run a full refresh. A dict rather
+ # than IncrementalConfig because that class imports Django, which this module must not.
+ incremental_config: dict[str, Any] | None = None
+
+ def __post_init__(self) -> None:
+ ensure_schedulable(self.sync_frequency)
ExpectedViewsProvider = Callable[["Team"], list[ProvidedView]]
diff --git a/products/data_modeling/backend/logic/freshness.py b/products/data_modeling/backend/logic/freshness.py
index e7e71ea52946..f7877c2d2a1d 100644
--- a/products/data_modeling/backend/logic/freshness.py
+++ b/products/data_modeling/backend/logic/freshness.py
@@ -41,7 +41,7 @@ Edges are (upstream_id, downstream_id): data flows upstream -> downstream, so a
import dataclasses
from collections import defaultdict, deque
-from collections.abc import Collection, Iterable, Mapping
+from collections.abc import Collection, Iterable, Mapping, Set
from datetime import timedelta
from typing import Literal
@@ -588,6 +588,15 @@ def _nothing_satisfiable_message(
return "No cadence works here."
+def ensure_schedulable(target: timedelta) -> None:
+ """Raise if `target` is not one of the cadences a person may pick."""
+ if target not in SCHEDULABLE_BUCKETS:
+ supported = ", ".join(format_cadence(interval) for interval in sorted(SCHEDULABLE_BUCKETS))
+ raise UnsupportedFrequencyTargetError(
+ f"Can't refresh every {humanize_cadence(target)}. Pick one of: {supported}."
+ )
+
+
def validate_declared_target(
*,
node_id: str,
@@ -604,11 +613,7 @@ def validate_declared_target(
that blocks the target, which is the thing a person has to change to unblock it. `pinned`
holds the pin state the write would leave, including `node_id`'s own.
"""
- if target not in SCHEDULABLE_BUCKETS:
- supported = ", ".join(format_cadence(interval) for interval in sorted(SCHEDULABLE_BUCKETS))
- raise UnsupportedFrequencyTargetError(
- f"Can't refresh every {humanize_cadence(target)}. Pick one of: {supported}."
- )
+ ensure_schedulable(target)
bounds = compute_target_bounds(
node_id=node_id,
@@ -656,6 +661,7 @@ def find_invalid_targets(
edges: list[tuple[str, str]],
declared_targets: dict[str, timedelta],
source_intervals: dict[str, timedelta],
+ floor_exempt: Set[str] = frozenset(),
pinned: Collection[str] = frozenset(),
) -> list[InvalidTarget]:
"""Re-validate every declared target against its current bounds.
@@ -663,6 +669,9 @@ def find_invalid_targets(
Targets drift: a descendant declaring a finer target lowers ancestors' ceilings, and
graph edits move floors. Runtime freshness stays correct (finest demand wins) — what
breaks is declared == effective, so run this on any graph mutation and surface the result.
+
+ `floor_exempt` nodes may declare a target finer than their sources deliver: the scheduler
+ clamps them at run time, so the declared value is intent rather than drift.
"""
floors = all_source_floors(edges, source_intervals, _pinned_targets(pinned, declared_targets))
ceilings = all_consumer_ceilings(edges, declared_targets, pinned)
@@ -670,7 +679,7 @@ def find_invalid_targets(
for node_id, declared in declared_targets.items():
source_floor = floors.get(node_id, STREAMING)
consumer_ceiling = ceilings.get(node_id)
- if is_finer_than(declared, source_floor) or (
+ if (node_id not in floor_exempt and is_finer_than(declared, source_floor)) or (
consumer_ceiling is not None and is_coarser_than(declared, consumer_ceiling)
):
invalid.append(
diff --git a/products/data_modeling/backend/logic/incremental.py b/products/data_modeling/backend/logic/incremental.py
index 2a6b15981a7b..cb691f34d0b5 100644
--- a/products/data_modeling/backend/logic/incremental.py
+++ b/products/data_modeling/backend/logic/incremental.py
@@ -69,9 +69,12 @@ class IncrementalState:
def get_incremental_config(saved_query) -> Optional[IncrementalConfig]:
+ return parse_incremental_config(saved_query.incremental_config)
+
+
+def parse_incremental_config(raw: Any) -> Optional[IncrementalConfig]:
"""Return the config only when it is enabled and complete. Anything malformed reads as off,
so a half-written config degrades to a full refresh rather than failing a run."""
- raw = saved_query.incremental_config
if not isinstance(raw, dict) or not raw.get("enabled"):
return None
diff --git a/products/data_modeling/backend/logic/node_frequency.py b/products/data_modeling/backend/logic/node_frequency.py
index d7d4e54bb274..c242efde709b 100644
--- a/products/data_modeling/backend/logic/node_frequency.py
+++ b/products/data_modeling/backend/logic/node_frequency.py
@@ -15,13 +15,17 @@ from typing import Any
from django.db.models import Q, QuerySet
+from posthog.dataclasses import frozen
+
from products.data_modeling.backend.facade.contracts import SavedQueryNodeState, SuspensionMarker
from products.data_modeling.backend.logic.cohort_scheduling import MINUTES_PER_WEEK
from products.data_modeling.backend.logic.freshness import (
STREAMING,
ClampedCadence,
+ InvalidTarget,
TargetBounds,
TargetOption,
+ all_source_floor_bounds,
all_source_floors,
ancestors_of,
clamp_to_source_floor,
@@ -29,6 +33,7 @@ from products.data_modeling.backend.logic.freshness import (
compute_target_bounds,
descendants_broken_by_pin,
descendants_of,
+ find_invalid_targets,
intersect_target_bounds,
max_data_age,
nodes_below_source_floor,
@@ -36,7 +41,7 @@ from products.data_modeling.backend.logic.freshness import (
)
from products.data_modeling.backend.logic.node_suspension import merged_suspension_state
from products.data_modeling.backend.models.dag import DAG
-from products.data_modeling.backend.models.edge import NON_SCHEDULING_EDGES, Edge
+from products.data_modeling.backend.models.edge import Edge
from products.data_modeling.backend.models.node import SAVED_QUERY_NODE_TYPES, Node, NodeType
from products.warehouse_sources.backend.facade.models import DataWarehouseTable, ExternalDataSchema
@@ -156,6 +161,12 @@ def set_declared_target(node: Node, target: timedelta | None) -> None:
Clearing the target also clears the pin, because a pin holds the target and has nothing to hold without one.
"""
+ stage_declared_target(node, target)
+ node.save(update_fields=["properties"])
+
+
+def stage_declared_target(node: Node, target: timedelta | None) -> None:
+ """`set_declared_target` without the save, for callers that write many nodes in one update."""
properties = node.properties or {}
frequency = properties.setdefault(_SYSTEM_KEY, {}).setdefault(_FREQUENCY_KEY, {})
if target is None:
@@ -164,7 +175,6 @@ def set_declared_target(node: Node, target: timedelta | None) -> None:
else:
frequency[_TARGET_SECONDS_KEY] = int(target.total_seconds())
node.properties = properties
- node.save(update_fields=["properties"])
def set_declared_pin(node: Node, pinned: bool) -> None:
@@ -278,7 +288,7 @@ def resolve_source_intervals(source_nodes: list[Node]) -> tuple[dict[str, timede
return source_intervals, best_effort
[email protected](frozen=True, kw_only=True, slots=True)
+@frozen
class FrequencyGraph:
"""The primitives `freshness.py` operates on, extracted from one DAG's Node/Edge rows."""
@@ -291,17 +301,61 @@ class FrequencyGraph:
best_effort_source_ids: set[str] # sources treated as STREAMING but not guaranteed
names: dict[str, str] # node id -> display name, so a bound can name what set it
identities: dict[str, NodeIdentity] # node id -> the resource its name belongs to
+ managed_node_ids: set[str] # nodes whose target the managed viewset provider declares
+ stand_ins: dict[str, str] # stand-in source id -> the node that sets its interval
+
+ def target_bounds(self, node_id: str) -> TargetBounds:
+ """Every cadence `node_id` may be set to, with each blocker given as a real node id."""
+ bounds = compute_target_bounds(
+ node_id=node_id,
+ edges=self.edges,
+ declared_targets=self.declared_targets,
+ source_intervals=self.source_intervals,
+ pinned=self.pinned,
+ )
+
+ def real(blocker: str | None) -> str | None:
+ return self.stand_ins.get(blocker, blocker) if blocker is not None else None
+
+ # A stand-in is a source with no parents, so only the floor side of the bounds can name one.
+ return dataclasses.replace(
+ bounds,
+ options=tuple(dataclasses.replace(option, blocker=real(option.blocker)) for option in bounds.options),
+ floor=bounds.floor and dataclasses.replace(bounds.floor, blocker=real(bounds.floor.blocker)),
+ )
+
+ def invalid_targets(self, declared_targets: dict[str, timedelta] | None = None) -> list[InvalidTarget]:
+ """Declared targets outside their bounds. A provider's cadence may sit below its source floor."""
+ return find_invalid_targets(
+ edges=self.edges,
+ declared_targets=self.declared_targets if declared_targets is None else declared_targets,
+ source_intervals=self.source_intervals,
+ floor_exempt=self.managed_node_ids,
+ pinned=self.pinned,
+ )
+
+
+def _managed_view_source_id(managed_node_id: str) -> str:
+ """The stand-in source a managed view's readers see in its place."""
+ return f"{managed_node_id}:managed-view"
def build_frequency_graph(dag: DAG) -> FrequencyGraph:
"""Extract the freshness graph for a DAG: schedulable nodes, edges, targets, source floors."""
nodes = list(Node.objects.filter(dag=dag).exclude(saved_query__deleted=True))
- edges = [
- (str(source_id), str(target_id))
- for source_id, target_id in Edge.objects.filter(dag=dag)
- .exclude(NON_SCHEDULING_EDGES)
- .values_list("source_id", "target_id")
- ]
+ managed_node_ids = {
+ str(node_id)
+ for node_id in Node.objects.filter(dag=dag, saved_query__managed_viewset__isnull=False).values_list(
+ "id", flat=True
+ )
+ }
+ edges: list[tuple[str, str]] = []
+ managed_reader_edges: list[tuple[str, str]] = []
+ for source_id, target_id in Edge.objects.filter(dag=dag).values_list("source_id", "target_id"):
+ edge = (str(source_id), str(target_id))
+ # The same split as NON_SCHEDULING_EDGES, from ids already loaded instead of a second join.
+ is_managed_reader = edge[0] in managed_node_ids and edge[1] not in managed_node_ids
+ (managed_reader_edges if is_managed_reader else edges).append(edge)
schedulable = {str(node.id) for node in nodes if node.type in SAVED_QUERY_NODE_TYPES}
declared_targets: dict[str, timedelta] = {}
@@ -319,6 +373,42 @@ def build_frequency_graph(dag: DAG) -> FrequencyGraph:
source_nodes = [node for node in nodes if node.type == NodeType.TABLE]
source_intervals, best_effort = resolve_source_intervals(source_nodes)
+ names = {str(node.id): node.name for node in nodes}
+ identities = {str(node.id): node_identity(node) for node in nodes}
+
+ # A reader never sets a managed view's cadence, and it cannot be fresher than that view's data.
+ # The view's readers see a stand-in source that changes as often as that data does.
+ stand_ins: dict[str, str] = {}
+ if managed_reader_edges:
+ runs_every, _clamped = clamp_to_source_floor(
+ compute_effective_cadences(
+ nodes=schedulable, edges=edges, declared_targets=declared_targets, pinned=pinned
+ ),
+ edges=edges,
+ source_intervals=source_intervals,
+ pinned=pinned,
+ )
+ # A materialized view's data changes once per run, whatever its inputs do, so it counts as a
+ # source at its run cadence. A query-time view passes its inputs' changes straight through.
+ materialized = {str(node.id) for node in nodes if node.type == NodeType.MAT_VIEW}
+ run_intervals = {
+ node_id: interval
+ for node_id, interval in runs_every.items()
+ if node_id in materialized and interval is not None
+ }
+ refreshes = all_source_floor_bounds(edges, {**source_intervals, **run_intervals})
+ for managed_id, reader_id in managed_reader_edges:
+ if managed_id not in names:
+ continue
+ stand_in = _managed_view_source_id(managed_id)
+ if stand_in not in stand_ins:
+ refresh = refreshes.get(managed_id)
+ blamed = refresh.blocker if refresh is not None and refresh.blocker in names else managed_id
+ stand_ins[stand_in] = blamed
+ source_intervals[stand_in] = refresh.value if refresh is not None else STREAMING
+ names[stand_in] = names[blamed]
+ identities[stand_in] = identities[blamed]
+ edges.append((stand_in, reader_id))
return FrequencyGraph(
nodes=schedulable,
@@ -328,8 +418,10 @@ def build_frequency_graph(dag: DAG) -> FrequencyGraph:
pinned=pinned,
source_intervals=source_intervals,
best_effort_source_ids=best_effort,
- names={str(node.id): node.name for node in nodes},
- identities={str(node.id): node_identity(node) for node in nodes},
+ names=names,
+ identities=identities,
+ managed_node_ids=managed_node_ids,
+ stand_ins=stand_ins,
)
@@ -372,15 +464,7 @@ def saved_query_target_bounds(team_id: int, saved_query_id: str | uuid.UUID) ->
if dag_key not in graphs:
graphs[dag_key] = build_frequency_graph(node.dag)
graph = graphs[dag_key]
- per_dag.append(
- compute_target_bounds(
- node_id=str(node.id),
- edges=graph.edges,
- declared_targets=graph.declared_targets,
- source_intervals=graph.source_intervals,
- pinned=graph.pinned,
- )
- )
+ per_dag.append(graph.target_bounds(str(node.id)))
names.update(graph.names)
identities.update(graph.identities)
# a best-effort source elsewhere in the DAG says nothing about this node's freshness
@@ -430,13 +514,7 @@ def saved_query_pin_preview(team_id: int, saved_query_id: str | uuid.UUID) -> Sa
graphs[dag_key] = build_frequency_graph(node.dag)
graph = graphs[dag_key]
node_id = str(node.id)
- options = compute_target_bounds(
- node_id=node_id,
- edges=graph.edges,
- declared_targets=graph.declared_targets,
- source_intervals=graph.source_intervals,
- pinned=graph.pinned,
- ).options
+ options = graph.target_bounds(node_id).options
for cadence, delayed in _delays_if_pinned_by_cadence(node_id, graph, options).items():
delays_if_pinned.setdefault(cadence, set()).update(delayed)
below_floor = nodes_below_source_floor(
diff --git a/products/data_modeling/backend/logic/saved_query_dag_sync.py b/products/data_modeling/backend/logic/saved_query_dag_sync.py
index 2f348aedaa73..0abd670c98c9 100644
--- a/products/data_modeling/backend/logic/saved_query_dag_sync.py
+++ b/products/data_modeling/backend/logic/saved_query_dag_sync.py
@@ -23,6 +23,7 @@ from products.data_modeling.backend.models.edge import Edge
from products.data_modeling.backend.models.modeling import UnknownParentError, get_parents_from_model_query
from products.data_modeling.backend.models.node import Node, NodeType
from products.warehouse_sources.backend.facade.models import DataWarehouseTable
+from products.warehouse_sources.backend.facade.types import DataWarehouseManagedViewSetKind
if TYPE_CHECKING:
from posthog.models import Team
@@ -41,14 +42,17 @@ def materializes(saved_query: "DataWarehouseSavedQuery") -> bool:
types the node VIEW, which `get_dag_structure` calls ephemeral and a run skips, so the table can
never come back and the query stops updating for good, without an error.
- Managed views are excluded because their flag is not a statement of intent: the Revenue
- Analytics viewsets set `is_materialized` at provisioning, before anything runs, so reading it
- would enroll a large population of views that have never materialized a row.
+ Revenue Analytics sets its flag before the first build. Requiring an existing table keeps
+ those legacy views out of scheduling until they have actually materialized.
"""
from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
if saved_query.origin == DataWarehouseSavedQuery.Origin.MANAGED_VIEWSET:
- return saved_query.table_id is not None
+ # TODO: drop this branch once the Revenue Analytics provider declares `materialized` per view
+ # instead of hardcoding True. Every managed view can then be judged on `is_materialized`.
+ viewset = saved_query.managed_viewset
+ if viewset is None or viewset.kind == DataWarehouseManagedViewSetKind.REVENUE_ANALYTICS:
+ return saved_query.table_id is not None
# The column is nullable, and NULL predates its default: it means the same as False here, which
# is also how `_materializes_q` reads it.
return bool(saved_query.is_materialized)
@@ -604,8 +608,13 @@ def _materializes_q() -> Q:
"""`materializes` as a filter, for the callers that cannot ask row by row."""
from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
- managed = Q(saved_query__origin=DataWarehouseSavedQuery.Origin.MANAGED_VIEWSET)
- return (~managed & Q(saved_query__is_materialized=True)) | (managed & Q(saved_query__table_id__isnull=False))
+ legacy_managed = Q(saved_query__origin=DataWarehouseSavedQuery.Origin.MANAGED_VIEWSET) & (
+ Q(saved_query__managed_viewset__isnull=True)
+ | Q(saved_query__managed_viewset__kind=DataWarehouseManagedViewSetKind.REVENUE_ANALYTICS)
+ )
+ return (~legacy_managed & Q(saved_query__is_materialized=True)) | (
+ legacy_managed & Q(saved_query__table_id__isnull=False)
+ )
def _promote_view_nodes(nodes: QuerySet[Node]) -> int:
diff --git a/products/data_modeling/backend/logic/schedule_reconcile.py b/products/data_modeling/backend/logic/schedule_reconcile.py
index f8f2bdafc7f2..7567cab572ab 100644
--- a/products/data_modeling/backend/logic/schedule_reconcile.py
+++ b/products/data_modeling/backend/logic/schedule_reconcile.py
@@ -52,7 +52,6 @@ from products.data_modeling.backend.logic.freshness import (
ClampedCadence,
InvalidTarget,
UnsupportedFrequencyTargetError,
- find_invalid_targets,
format_cadence,
validate_declared_target,
validate_pin,
@@ -68,6 +67,7 @@ from products.data_modeling.backend.logic.node_frequency import (
set_declared_anchor,
set_declared_pin,
set_declared_target,
+ stage_declared_target,
)
from products.data_modeling.backend.models.dag import DAG
from products.data_modeling.backend.models.node import Node
@@ -135,12 +135,7 @@ def _warn_on_invalid_targets(dag: DAG, graph: FrequencyGraph | None = None) -> N
"""Surface declared targets that drifted outside their bounds; never blocks the mutation."""
if graph is None:
graph = build_frequency_graph(dag)
- for invalid in find_invalid_targets(
- edges=graph.edges,
- declared_targets=graph.declared_targets,
- source_intervals=graph.source_intervals,
- pinned=graph.pinned,
- ):
+ for invalid in graph.invalid_targets():
logger.warning(
"Declared freshness target outside its legal range",
dag_id=str(dag.id),
@@ -242,6 +237,31 @@ def apply_saved_query_frequency_target(
return written
+def apply_managed_view_frequency_targets(team_id: int, targets: Mapping[str, timedelta | None]) -> None:
+ """Store each managed view's provider cadence on its DAG node(s), keyed by saved query id.
+
+ The provider owns these cadences, so no source floor or consumer ceiling is checked: the
+ scheduler already runs a node no finer than its sources deliver (`clamp_to_source_floor`), and
+ the stored value stays equal to what the provider declares. One atomic write, so a batch that
+ moves an upstream and a downstream view together never lands half-applied. Callers reconcile.
+ """
+ with transaction.atomic():
+ # Ordered so that concurrent syncs of the same viewset take the row locks in the same order.
+ nodes = (
+ Node.objects.select_for_update()
+ .only("id", "saved_query_id", "properties")
+ .filter(team_id=team_id, saved_query_id__in=targets)
+ .order_by("id")
+ )
+ changed = []
+ for node in nodes:
+ target = targets[str(node.saved_query_id)]
+ if get_declared_target(node) != target:
+ stage_declared_target(node, target)
+ changed.append(node)
+ Node.objects.bulk_update(changed, ["properties"])
+
+
def check_saved_query_frequency_target(
saved_query: "DataWarehouseSavedQuery", target: timedelta, *, visible_names: Mapping[str, str] | None = None
) -> None:
@@ -362,9 +382,7 @@ def preview_dag_schedules(dag: DAG, *, seed: bool = False) -> DagSchedulePreview
plan=plan,
best_effort_source_ids=graph.best_effort_source_ids,
clamped=clamped,
- invalid_targets=find_invalid_targets(
- edges=graph.edges, declared_targets=declared, source_intervals=graph.source_intervals, pinned=graph.pinned
- ),
+ invalid_targets=graph.invalid_targets(declared),
unsupported_tiers=sorted({tier.interval for tier in desired_tiers if tier.interval not in SCHEDULABLE_BUCKETS}),
seeded=seed,
)
diff --git a/products/data_modeling/backend/models/datawarehouse_managed_viewset.py b/products/data_modeling/backend/models/datawarehouse_managed_viewset.py
index 6d5c826f5807..cd0723303d3d 100644
--- a/products/data_modeling/backend/models/datawarehouse_managed_viewset.py
+++ b/products/data_modeling/backend/models/datawarehouse_managed_viewset.py
@@ -1,6 +1,8 @@
+from collections import defaultdict
from dataclasses import dataclass
from datetime import timedelta
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, cast
+from uuid import UUID
from django.db import connection, models, transaction
@@ -17,6 +19,7 @@ from posthog.hogql.database.models import (
IntegerDatabaseField,
MapStringDatabaseField,
StringDatabaseField,
+ TableNode,
)
from posthog.exceptions_capture import capture_exception
@@ -24,11 +27,19 @@ from posthog.models.utils import CreatedMetaFields, UpdatedMetaFields, UUIDTMode
from products.data_modeling.backend.facade.managed_viewset_hooks import get_expected_views_provider
from products.data_modeling.backend.facade.system_tables import DATA_MODELING_ALLOWED_SYSTEM_TABLES
+from products.data_modeling.backend.logic.incremental import clear_incremental_state, parse_incremental_config
from products.data_modeling.backend.models.dag import DAG
from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
+from products.data_modeling.backend.models.edge import Edge
+from products.data_modeling.backend.models.node import Node
from products.revenue_analytics.backend.views.schemas import SCHEMAS as REVENUE_ANALYTICS_SCHEMAS
from products.warehouse_sources.backend.facade.types import DataWarehouseManagedViewSetKind
+if TYPE_CHECKING:
+ from posthog.hogql.database.database import Database
+
+ from products.data_modeling.backend.facade.contracts import SavedQueryNodeState
+
logger = structlog.get_logger(__name__)
@@ -37,9 +48,11 @@ class ExpectedView:
name: str
query: dict[str, Any]
columns: dict[str, dict[str, Any]]
- # Materialized views get a 12h sync schedule. A non-materialized view (e.g. engineering
+ # Materialized views get a sync schedule. A non-materialized view (e.g. engineering
# analytics) is computed at query time — no schedule, no S3 table.
- materialized: bool = True
+ materialized: bool
+ sync_frequency: timedelta
+ incremental_config: dict[str, Any] | None
class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTModel):
@@ -76,8 +89,12 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTMod
Materializes views by default.
"""
# Deferred: schedule_reconcile pulls in temporalio; a module-level import would drag it onto django.setup()
+ from products.data_modeling.backend.logic.node_frequency import node_states_by_saved_query # noqa: PLC0415
from products.data_modeling.backend.logic.saved_query_dag_sync import sync_saved_query_to_dag # noqa: PLC0415
- from products.data_modeling.backend.logic.schedule_reconcile import maybe_reconcile_dag # noqa: PLC0415
+ from products.data_modeling.backend.logic.schedule_reconcile import ( # noqa: PLC0415 keeps temporalio off django.setup()
+ apply_managed_view_frequency_targets,
+ maybe_reconcile_dag,
+ )
provider = get_expected_views_provider(self.kind)
if provider is None:
@@ -89,6 +106,8 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTMod
query={"kind": "HogQLQuery", "query": view.query},
columns=self._get_columns_from_fields(view.fields),
materialized=view.materialized,
+ sync_frequency=view.sync_frequency,
+ incremental_config=view.incremental_config,
)
for view in provider(self.team)
]
@@ -128,12 +147,23 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTMod
error=str(e),
)
external_tables_by_view[view.name] = []
+ # Eligibility must see updated upstream definitions, even before this batch is saved.
+ database.tables.add_child(
+ TableNode.create_nested_for_chain(view.name.split("."), table=temp_sq.hogql_definition()),
+ table_conflict_mode="override",
+ )
+
+ incremental_config_by_view = {
+ view.name: self._eligible_incremental_config(view, database) for view in expected_views
+ }
views_created = 0
views_updated = 0
- saved_queries_to_schedule: list[DataWarehouseSavedQuery] = []
+ views_unchanged = 0
saved_queries_to_sync: list[DataWarehouseSavedQuery] = []
+ frequency_targets: dict[str, timedelta | None] = {}
orphaned_views_to_revert: list[DataWarehouseSavedQuery] = []
+ redefined_ids: set[UUID] = set()
# Keep this transaction short: persist DB changes only. The post-commit work
# (schedule_materialization → Temporal RPCs,
@@ -150,44 +180,78 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTMod
[f"{self.team_id}_sync_views_{self.kind}"],
)
+ # Read under the lock, so a row a concurrent sync just created is seen as stored.
+ stored_views = {saved_query.name: saved_query for saved_query in self.saved_queries.exclude(deleted=True)}
+ node_states = node_states_by_saved_query(self.team_id, [v.id for v in stored_views.values()])
+
for view in expected_views:
- # Get the one from the DB or create a new one if doesn't exist yet
- saved_query = DataWarehouseSavedQuery.objects.filter(
- name=view.name, team=self.team, managed_viewset=self
- ).first()
- if saved_query:
- created = False
- else:
+ stored_view = stored_views.get(view.name)
+ created = stored_view is None
+ if stored_view is None:
saved_query = DataWarehouseSavedQuery(
name=view.name,
team=self.team,
managed_viewset=self,
origin=DataWarehouseSavedQuery.Origin.MANAGED_VIEWSET,
)
- created = True
+ elif self._stores_view(
+ stored_view,
+ view,
+ external_tables_by_view[view.name],
+ incremental_config_by_view[view.name],
+ node_states.get(str(stored_view.id)),
+ ):
+ # Most syncs follow a load that changed nothing: no write, no DAG or schedule work.
+ views_unchanged += 1
+ continue
+ else:
+ saved_query = stored_view
+ if saved_query.query != view.query:
+ redefined_ids.add(saved_query.id)
# Do NOT use get_columns because it runs the query, and these are possibly heavy
saved_query.query = view.query
saved_query.set_columns(view.columns)
saved_query.external_tables = external_tables_by_view[view.name]
saved_query.is_materialized = view.materialized
- # A non-materialized view is computed at query time — no sync schedule, and it never
- # enters the schedule list below.
- saved_query.sync_frequency_interval = timedelta(hours=12) if view.materialized else None
+ # The cadence goes straight onto the DAG node below, not through this one-shot field.
+ saved_query.sync_frequency_interval = None
+ saved_query.incremental_config = incremental_config_by_view[view.name]
- saved_query.save()
+ if created:
+ saved_query.save()
+ else:
+ # Only the columns this sync sets. A materialization run writes incremental_state
+ # concurrently, and a whole-row save would put the stale copy read above back.
+ saved_query.save(
+ update_fields=[
+ "query",
+ "columns",
+ "column_order",
+ "external_tables",
+ "is_materialized",
+ "sync_frequency_interval",
+ "incremental_config",
+ "updated_at",
+ ]
+ )
saved_queries_to_sync.append(saved_query)
- if view.materialized:
- saved_queries_to_schedule.append(saved_query)
+ # A non-materialized view is computed at query time, so it has no cadence and no schedule.
+ frequency_targets[str(saved_query.id)] = view.sync_frequency if view.materialized else None
if created:
views_created += 1
else:
views_updated += 1
- orphaned_views_to_revert = list(
- self.saved_queries.exclude(name__in=expected_view_names).exclude(deleted=True)
- )
+ orphaned_views_to_revert = [
+ saved_query for name, saved_query in stored_views.items() if name not in expected_view_names
+ ]
+
+ # In the transaction that writes the new queries, so a failure rolls the redefinition back
+ # and the next sync retries it. The edges are the ones from before this sync, which still
+ # reach every view whose stored rows came from the old upstream.
+ self._rebuild_downstream_of(DAG.get_or_create_default(self.team), redefined_ids)
# No cross-call advisory lock here on purpose: a SESSION-level pg_advisory_lock acquired and
# released across separate autocommit statements leaks under transaction-mode PgBouncer — the
@@ -214,9 +278,18 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTMod
view_name=saved_query.name,
error=str(e),
)
+ try:
+ # After every node exists, so the whole batch lands together.
+ apply_managed_view_frequency_targets(self.team_id, frequency_targets)
+ except Exception as e:
+ capture_exception(e, {"managed_viewset_id": self.id})
+ logger.warning("failed_to_apply_managed_view_cadences", team_id=self.team_id, error=str(e))
- for saved_query in saved_queries_to_schedule:
+ for saved_query in saved_queries_to_sync:
+ if not saved_query.is_materialized:
+ continue
try:
+ # Carries no interval, so this only puts a brand-new DAG onto cadence tiers.
saved_query.schedule_materialization(reconcile=False)
except Exception as e:
capture_exception(e, {"managed_viewset_id": self.id, "view_name": saved_query.name})
@@ -255,6 +328,7 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTMod
kind=self.kind,
views_created=views_created,
views_updated=views_updated,
+ views_unchanged=views_unchanged,
views_deleted=views_deleted,
)
@@ -312,6 +386,90 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTMod
),
}
+ @staticmethod
+ def _stores_view(
+ saved_query: DataWarehouseSavedQuery,
+ view: ExpectedView,
+ external_tables: list[str],
+ incremental_config: dict[str, Any] | None,
+ node_state: "SavedQueryNodeState | None",
+ ) -> bool:
+ """Whether this sync's writes for ``view`` would leave ``saved_query`` and its DAG cadence as they are.
+
+ Covers each field the sync writes and the cadence it stores on the DAG node. A view without a
+ node is not stored: its DAG placement failed earlier and the sync repairs it.
+ """
+ return (
+ node_state is not None
+ and node_state.declared_target == (view.sync_frequency if view.materialized else None)
+ and saved_query.query == view.query
+ and saved_query.stores_columns(view.columns)
+ and set(saved_query.external_tables or []) == set(external_tables)
+ and saved_query.is_materialized == view.materialized
+ and saved_query.sync_frequency_interval is None
+ and saved_query.incremental_config == incremental_config
+ )
+
+ def _rebuild_downstream_of(self, dag: DAG, redefined_ids: set[UUID]) -> None:
+ """Clear the watermark of every incremental view downstream of a redefined view.
+
+ A view's own definition fingerprint resets its watermark when its query changes, but not when
+ an upstream query changes. Its stored rows came from the old upstream, so it must rebuild.
+ """
+ if not redefined_ids:
+ return
+ children: dict[UUID, list[UUID]] = defaultdict(list)
+ for source_id, target_id in Edge.objects.filter(team_id=self.team_id, dag=dag).values_list(
+ "source_id", "target_id"
+ ):
+ children[source_id].append(target_id)
+ frontier = list(
+ Node.objects.filter(team_id=self.team_id, dag=dag, saved_query_id__in=redefined_ids).values_list(
+ "id", flat=True
+ )
+ )
+ reached: set[UUID] = set()
+ while frontier:
+ for child in children[frontier.pop()]:
+ if child not in reached:
+ reached.add(child)
+ frontier.append(child)
+ downstream = DataWarehouseSavedQuery.objects.filter(
+ team_id=self.team_id,
+ id__in=Node.objects.filter(id__in=reached).values("saved_query_id"),
+ deleted=False,
+ )
+ for saved_query in downstream:
+ if saved_query.incremental_state:
+ clear_incremental_state(saved_query)
+
+ def _eligible_incremental_config(self, view: ExpectedView, database: "Database") -> dict[str, Any] | None:
+ """The provider's incremental config, or None when the query cannot carry it.
+
+ The API runs this check before it saves a config. Without it a key the query does not
+ output only fails on the second run, and every later run alternates a rebuild and a failure.
+ """
+ from products.data_modeling.backend.logic.incremental_eligibility import ( # noqa: PLC0415 keeps posthog.schema off django.setup()
+ check_incremental_eligibility,
+ )
+
+ if view.incremental_config is None or not view.materialized:
+ return None
+ config = parse_incremental_config(view.incremental_config)
+ if config is None:
+ logger.warning("managed_view_incremental_config_malformed", team_id=self.team_id, view_name=view.name)
+ return None
+ result = check_incremental_eligibility(view.query["query"], config, database=database)
+ if not result.eligible:
+ logger.warning(
+ "managed_view_not_incremental",
+ team_id=self.team_id,
+ view_name=view.name,
+ blockers=result.blockers,
+ )
+ return None
+ return view.incremental_config
+
@staticmethod
def _get_columns_from_fields(fields: dict[str, FieldOrTable]) -> dict[str, dict[str, Any]]:
return {
diff --git a/products/data_modeling/backend/test/test_node_frequency.py b/products/data_modeling/backend/test/test_node_frequency.py
index 00412db21543..8a5bf2e0db1e 100644
--- a/products/data_modeling/backend/test/test_node_frequency.py
+++ b/products/data_modeling/backend/test/test_node_frequency.py
@@ -1,3 +1,4 @@
+import itertools
from datetime import timedelta
import pytest
@@ -8,6 +9,7 @@ from parameterized import parameterized
from products.data_modeling.backend.logic.freshness import (
STREAMING,
UnsatisfiableFrequencyError,
+ all_source_floor_bounds,
all_source_floors,
compute_effective_cadences,
validate_declared_target,
@@ -18,6 +20,7 @@ from products.data_modeling.backend.logic.node_frequency import (
get_declared_target,
persist_seed_targets,
resolve_source_intervals,
+ saved_query_target_bounds,
seed_targets,
set_declared_anchor,
set_declared_target,
@@ -146,8 +149,10 @@ class TestBuildFrequencyGraph(BaseTest):
floor = all_source_floors(graph.edges, graph.source_intervals).get(str(matview.id), STREAMING)
self.assertEqual(floor, STREAMING)
- @parameterized.expand([("user_reader", False, H12), ("managed_reader", True, M15)])
- def test_only_a_managed_reader_pulls_a_managed_view_to_its_cadence(self, _name, reader_is_managed, expected):
+ @parameterized.expand([("user_reader", False, H12, H12), ("managed_reader", True, M15, STREAMING)])
+ def test_only_a_managed_reader_pulls_a_managed_view_to_its_cadence(
+ self, _name, reader_is_managed, expected, reader_floor
+ ):
dag = DAG.get_or_create_default(self.team)
viewset = DataWarehouseManagedViewSet.objects.create(
team=self.team, kind=DataWarehouseManagedViewSetKind.REVENUE_ANALYTICS
@@ -168,6 +173,56 @@ class TestBuildFrequencyGraph(BaseTest):
)
self.assertEqual(effective[str(managed_view.id)], expected)
+ floors = all_source_floor_bounds(graph.edges, graph.source_intervals)
+ self.assertEqual(floors[str(reader.id)].value, reader_floor)
+
+ @parameterized.expand(
+ [
+ # (source interval, None for streaming), [(managed node type, declared target)], floor, blocker index
+ ("declared_finer_than_source", H6, [(NodeType.MAT_VIEW, M15)], H6, 1),
+ ("unscheduled_materialized_view", H6, [(NodeType.MAT_VIEW, None)], H6, 0),
+ ("materialized_view", None, [(NodeType.MAT_VIEW, H12)], H12, 1),
+ (
+ "query_time_view_over_a_materialized_view",
+ None,
+ [(NodeType.MAT_VIEW, H12), (NodeType.VIEW, None)],
+ H12,
+ 1,
+ ),
+ ("query_time_view_with_a_declared_cadence", None, [(NodeType.VIEW, H12)], None, None),
+ ]
+ )
+ def test_a_user_reader_cannot_be_fresher_than_a_managed_views_data(
+ self, _name, source_interval, managed_chain, expected_floor, blocker_index
+ ):
+ dag = DAG.get_or_create_default(self.team)
+ viewset = DataWarehouseManagedViewSet.objects.create(
+ team=self.team, kind=DataWarehouseManagedViewSetKind.REVENUE_ANALYTICS
+ )
+ chain = [
+ _warehouse_source_node(self.team, dag, sync_frequency_interval=source_interval)
+ if source_interval
+ else _table_node(self.team, dag, "events", {"origin": "posthog"})
+ ]
+ for index, (node_type, target) in enumerate(managed_chain):
+ managed_view = _saved_query_node(self.team, dag, f"managed_view_{index}", node_type)
+ assert managed_view.saved_query is not None
+ managed_view.saved_query.managed_viewset = viewset
+ managed_view.saved_query.save(update_fields=["managed_viewset"])
+ set_declared_target(managed_view, target)
+ chain.append(managed_view)
+ reader = _saved_query_node(self.team, dag, "reader", NodeType.MAT_VIEW)
+ chain.append(reader)
+ for upstream, downstream in itertools.pairwise(chain):
+ Edge.objects.create(team=self.team, dag=dag, source=upstream, target=downstream)
+
+ assert reader.saved_query_id is not None
+ resolved = saved_query_target_bounds(self.team.id, reader.saved_query_id)
+
+ assert resolved is not None
+ floor = resolved.bounds.floor
+ expected = None if expected_floor is None else (expected_floor, str(chain[blocker_index].id))
+ self.assertEqual((floor.value, floor.blocker) if floor else None, expected)
def test_imported_source_floor_makes_a_tighter_descendant_unsatisfiable(self):
dag = DAG.get_or_create_default(self.team)
diff --git a/products/data_modeling/backend/tests/test_managed_viewset_providers.py b/products/data_modeling/backend/tests/test_managed_viewset_providers.py
index f93b8627b08a..9dd39ca5f3c1 100644
--- a/products/data_modeling/backend/tests/test_managed_viewset_providers.py
+++ b/products/data_modeling/backend/tests/test_managed_viewset_providers.py
@@ -1,3 +1,7 @@
+from collections.abc import Callable
+from datetime import timedelta
+from typing import Any
+
from posthog.test.base import BaseTest
from unittest.mock import AsyncMock, Mock, patch
@@ -33,6 +37,10 @@ from products.data_modeling.backend.facade.models import (
NodeType,
)
from products.data_modeling.backend.logic.cohort_scheduling import is_tier_schedule_id
+from products.data_modeling.backend.logic.flags import MANAGED_VIEWSETS_FEATURE_FLAG
+from products.data_modeling.backend.logic.freshness import UnsupportedFrequencyTargetError
+from products.data_modeling.backend.logic.node_frequency import get_declared_target, set_declared_target
+from products.data_modeling.backend.logic.saved_query_dag_sync import sync_saved_query_to_dag
from products.warehouse_sources.backend.facade.types import DataWarehouseManagedViewSetKind
SCHEDULE_MATERIALIZATION = (
@@ -63,15 +71,21 @@ def _no_schedules():
return temporal
-def _fake_view(name: str = "fake_provider_view", materialized: bool = True) -> ProvidedView:
+def _fake_view(
+ name: str = "fake_provider_view", materialized: bool = True, query: str = "SELECT 1 AS id", **overrides: Any
+) -> ProvidedView:
return ProvidedView(
name=name,
- query="SELECT 1 AS id",
+ query=query,
fields={"id": IntegerDatabaseField(name="id")},
materialized=materialized,
+ **overrides,
)
+INCREMENTAL_CONFIG = {"enabled": True, "incremental_key": "id", "unique_key": ["id"], "lookback_seconds": 60}
+
+
class TestManagedViewSetProviders(BaseTest):
def _viewset(self) -> DataWarehouseManagedViewSet:
return DataWarehouseManagedViewSet.objects.create(team=self.team, kind=KIND)
@@ -94,6 +108,7 @@ class TestManagedViewSetProviders(BaseTest):
view = next(v for v in views if v.name == fake_view.name)
self.assertFalse(view.is_materialized)
self.assertIsNone(view.sync_frequency_interval)
+ self.assertIsNone(view.incremental_config)
# A non-materialized view is computed at query time — it must never be scheduled for
# materialization.
mock_schedule.assert_not_called()
@@ -159,18 +174,222 @@ class TestManagedViewSetProviders(BaseTest):
self.assertIsInstance(view.hogql_definition().fields["value"], field_class)
@patch(SCHEDULE_MATERIALIZATION)
- def test_sync_views_is_idempotent(self, _):
+ @patch(SYNC_SAVED_QUERY_TO_DAG, wraps=sync_saved_query_to_dag)
+ def test_sync_views_is_idempotent(self, mock_sync_saved_query_to_dag: Mock, mock_schedule: Mock) -> None:
fake_view = _fake_view()
with patch.dict(_expected_views_providers, clear=True):
register_expected_views_provider(KIND, lambda team: [fake_view])
viewset = self._viewset()
viewset.sync_views()
- first_ids = sorted(v.id for v in self._views(viewset))
+ first = {v.id: v.updated_at for v in self._views(viewset)}
+ mock_sync_saved_query_to_dag.reset_mock()
+ mock_schedule.reset_mock()
+ viewset.sync_views()
+ second = {v.id: v.updated_at for v in self._views(viewset)}
+
+ self.assertEqual(first, second)
+ mock_sync_saved_query_to_dag.assert_not_called()
+ mock_schedule.assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("query", _fake_view(query="SELECT 2 AS id"), None),
+ (
+ "fields",
+ ProvidedView(
+ name="fake_provider_view",
+ query="SELECT 1 AS id",
+ fields={"id": IntegerDatabaseField(name="id"), "value": StringDatabaseField(name="value")},
+ ),
+ None,
+ ),
+ ("materialized", _fake_view(materialized=False), None),
+ ("sync_frequency", _fake_view(sync_frequency=timedelta(minutes=15)), None),
+ ("incremental_config", _fake_view(incremental_config=INCREMENTAL_CONFIG), None),
+ ("view_set", _fake_view(name="renamed_provider_view"), None),
+ (
+ "external_tables",
+ _fake_view(),
+ lambda view: DataWarehouseSavedQuery.objects.filter(pk=view.pk).update(external_tables=["drifted"]),
+ ),
+ ("node_missing", _fake_view(), lambda view: Node.objects.filter(saved_query=view).delete()),
+ (
+ "node_cadence",
+ _fake_view(),
+ lambda view: set_declared_target(Node.objects.get(saved_query=view), timedelta(minutes=15)),
+ ),
+ ]
+ )
+ @patch(SCHEDULE_MATERIALIZATION)
+ @patch(SYNC_SAVED_QUERY_TO_DAG, wraps=sync_saved_query_to_dag)
+ def test_resync_writes_when_a_declared_field_or_stored_state_changes(
+ self,
+ _name: str,
+ changed_view: ProvidedView,
+ drift: Callable[[DataWarehouseSavedQuery], Any] | None,
+ mock_sync_saved_query_to_dag: Mock,
+ _: Mock,
+ ) -> None:
+ untouched_view = _fake_view(name="untouched_view")
+ provided = [_fake_view(), untouched_view]
+ with patch.dict(_expected_views_providers, clear=True):
+ register_expected_views_provider(KIND, lambda team: provided)
+ viewset = self._viewset()
+ viewset.sync_views()
+ if drift is not None:
+ drift(DataWarehouseSavedQuery.objects.get(managed_viewset=viewset, name="fake_provider_view"))
+ provided = [changed_view, untouched_view]
+ mock_sync_saved_query_to_dag.reset_mock()
+ viewset.sync_views()
+
+ # Only the changed view syncs: the untouched one is skipped.
+ mock_sync_saved_query_to_dag.assert_called_once()
+
+ @patch(SCHEDULE_MATERIALIZATION)
+ def test_sync_views_stores_the_cadence_and_incremental_config_the_provider_declares(self, mock_schedule):
+ fake_view = _fake_view(sync_frequency=timedelta(minutes=15), incremental_config=INCREMENTAL_CONFIG)
+ with patch.dict(_expected_views_providers, clear=True):
+ register_expected_views_provider(KIND, lambda team: [fake_view])
+
+ viewset = self._viewset()
+ viewset.sync_views()
+
+ view = next(v for v in self._views(viewset) if v.name == fake_view.name)
+ self.assertIsNone(view.sync_frequency_interval)
+ self.assertEqual(view.incremental_config, INCREMENTAL_CONFIG)
+ node = Node.objects.get(saved_query=view)
+ self.assertEqual(node.type, NodeType.MAT_VIEW)
+ self.assertEqual(get_declared_target(node), timedelta(minutes=15))
+ mock_schedule.assert_called_once()
+
+ # Lineage resolves a managed view only when this flag puts managed views in the HogQL database.
+ @patch("posthoganalytics.feature_enabled", side_effect=lambda key, *_, **__: key == MANAGED_VIEWSETS_FEATURE_FLAG)
+ @patch(SCHEDULE_MATERIALIZATION)
+ def test_upstream_and_downstream_cadences_change_together(self, _schedule: Mock, _flag: Mock) -> None:
+ def views(cadence: timedelta) -> list[ProvidedView]:
+ return [
+ _fake_view(name="upstream", sync_frequency=cadence),
+ ProvidedView(
+ name="downstream",
+ query="SELECT id FROM upstream",
+ fields={"id": IntegerDatabaseField(name="id")},
+ sync_frequency=cadence,
+ ),
+ ]
+
+ provided = views(timedelta(minutes=15))
+ with patch.dict(_expected_views_providers, clear=True):
+ register_expected_views_provider(KIND, lambda team: provided)
+ viewset = self._viewset()
+ viewset.sync_views()
+ provided = views(timedelta(hours=12))
+ viewset.sync_views()
+
+ for view in self._views(viewset):
+ self.assertTrue(view.is_materialized)
+ self.assertEqual(get_declared_target(Node.objects.get(saved_query=view)), timedelta(hours=12))
+
+ @patch(SCHEDULE_MATERIALIZATION)
+ def test_resync_revalidates_incremental_config_against_updated_upstream_definitions(self, _: Mock) -> None:
+ views = [
+ _fake_view(name="upstream", materialized=False),
+ ProvidedView(
+ name="downstream",
+ query="SELECT id FROM upstream",
+ fields={"id": IntegerDatabaseField(name="id")},
+ incremental_config=INCREMENTAL_CONFIG,
+ ),
+ ]
+ with patch.dict(_expected_views_providers, clear=True):
+ register_expected_views_provider(KIND, lambda team: views)
+ viewset = self._viewset()
+ viewset.sync_views()
+ downstream = DataWarehouseSavedQuery.objects.get(team=self.team, name="downstream")
+ self.assertEqual(downstream.incremental_config, INCREMENTAL_CONFIG)
+
+ views[0] = ProvidedView(
+ name="upstream",
+ query="SELECT 1 AS id LIMIT 1",
+ fields={"id": IntegerDatabaseField(name="id")},
+ materialized=False,
+ )
+ viewset.sync_views()
+
+ downstream.refresh_from_db()
+ self.assertIsNone(downstream.incremental_config)
+
+ @parameterized.expand(
+ [("upstream_redefined", "SELECT 2 AS id", True), ("upstream_unchanged", "SELECT 1 AS id", False)]
+ )
+ # Lineage resolves a managed view only when this flag puts managed views in the HogQL database.
+ @patch("posthoganalytics.feature_enabled", side_effect=lambda key, *_, **__: key == MANAGED_VIEWSETS_FEATURE_FLAG)
+ @patch(SCHEDULE_MATERIALIZATION)
+ def test_resync_rebuilds_every_view_downstream_of_a_redefined_view(
+ self, _name: str, upstream_query: str, rebuilds: bool, _schedule: Mock, _flag: Mock
+ ) -> None:
+ def incremental(name: str, parent: str) -> ProvidedView:
+ return ProvidedView(
+ name=name,
+ query=f"SELECT id FROM {parent}",
+ fields={"id": IntegerDatabaseField(name="id")},
+ incremental_config=INCREMENTAL_CONFIG,
+ )
+
+ def provided(upstream: ProvidedView) -> list[ProvidedView]:
+ return [upstream, incremental("middle", "upstream"), incremental("downstream", "middle")]
+
+ views = provided(_fake_view(name="upstream", materialized=False))
+ state = {"watermark": 7, "watermark_type": "int"}
+ with patch.dict(_expected_views_providers, clear=True):
+ register_expected_views_provider(KIND, lambda team: views)
+ viewset = self._viewset()
+ viewset.sync_views()
+ DataWarehouseSavedQuery.objects.filter(team=self.team, name__in=["middle", "downstream"]).update(
+ incremental_state=state
+ )
+
+ views[:] = provided(_fake_view(name="upstream", materialized=False, query=upstream_query))
+ viewset.sync_views()
+
+ for name in ("middle", "downstream"):
+ view = DataWarehouseSavedQuery.objects.get(team=self.team, name=name)
+ self.assertEqual(view.incremental_state, {} if rebuilds else state, name)
+
+ @patch(SCHEDULE_MATERIALIZATION)
+ def test_an_incremental_key_the_query_does_not_output_is_not_stored(self, _):
+ fake_view = _fake_view(incremental_config={**INCREMENTAL_CONFIG, "incremental_key": "missing"})
+ with patch.dict(_expected_views_providers, clear=True):
+ register_expected_views_provider(KIND, lambda team: [fake_view])
+
+ viewset = self._viewset()
+ viewset.sync_views()
+
+ view = next(v for v in self._views(viewset) if v.name == fake_view.name)
+ self.assertTrue(view.is_materialized)
+ self.assertIsNone(view.incremental_config)
+
+ @patch(SCHEDULE_MATERIALIZATION)
+ def test_resync_keeps_the_state_a_run_wrote_while_it_ran(self, _):
+ provided = [_fake_view(incremental_config=INCREMENTAL_CONFIG)]
+ state = {"watermark": 7, "watermark_type": "int"}
+ original_set_columns = DataWarehouseSavedQuery.set_columns
+
+ def run_finishes_between_read_and_save(saved_query: DataWarehouseSavedQuery, columns: dict[str, Any]) -> None:
+ original_set_columns(saved_query, columns)
+ DataWarehouseSavedQuery.objects.filter(pk=saved_query.pk).update(incremental_state=state)
+
+ with patch.dict(_expected_views_providers, clear=True):
+ register_expected_views_provider(KIND, lambda team: provided)
+
+ viewset = self._viewset()
viewset.sync_views()
- second_ids = sorted(v.id for v in self._views(viewset))
+ provided = [_fake_view(sync_frequency=timedelta(minutes=15), incremental_config=INCREMENTAL_CONFIG)]
+ with patch.object(DataWarehouseSavedQuery, "set_columns", run_finishes_between_read_and_save):
+ viewset.sync_views()
- self.assertEqual(first_ids, second_ids)
+ view = next(v for v in self._views(viewset) if v.name == provided[0].name)
+ self.assertEqual(view.incremental_state, state)
def test_provisioning_a_new_team_puts_every_view_on_tier_schedules(self):
# Managed viewsets are how most new teams first materialize anything (a provider makes
@@ -202,6 +421,10 @@ class TestManagedViewSetProviders(BaseTest):
self.assertEqual(self._views(viewset), [])
+ def test_a_provider_cannot_declare_an_unschedulable_cadence(self):
+ with self.assertRaisesRegex(UnsupportedFrequencyTargetError, "Can.t refresh every 45 minutes"):
+ _fake_view(sync_frequency=timedelta(minutes=45))
+
def test_sync_views_raises_for_unregistered_kind(self):
with patch.dict(_expected_views_providers, clear=True):
viewset = self._viewset()