:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-modeling): let a managed view set its cadence and incremental config (#118655)
Raúl Negrón-Otero0142bae057a8
diff --git a/products/data_modeling/backend/facade/managed_viewset_hooks.py b/products/data_modeling/backend/facade/managed_viewset_hooks.pyindex 28ecf2abe479..9070b8ecc7b0 100644--- a/products/data_modeling/backend/facade/managed_viewset_hooks.py+++ b/products/data_modeling/backend/facade/managed_viewset_hooks.py@@ -19,13 +19,18 @@ from __future__ import annotationsimport dataclassesfrom collections.abc import Callable-from typing import TYPE_CHECKING+from datetime import timedelta+from typing import TYPE_CHECKING, Any++from products.data_modeling.backend.logic.freshness import ensure_schedulableif TYPE_CHECKING:from posthog.hogql.database.models import FieldOrTablefrom posthog.models.team import Team+DEFAULT_MANAGED_VIEW_SYNC_FREQUENCY = timedelta(hours=12)+@dataclasses.dataclass(frozen=True)class ProvidedView:@@ -37,9 +42,18 @@ class ProvidedView:name: strquery: strfields: dict[str, FieldOrTable]- # Materialized views get a 12h sync schedule. A non-materialized view (e.g. engineering- # analytics) is computed at query time — no schedule, no S3 table.+ # Non-materialized views are computed at query time, without a schedule or S3 table.materialized: bool = True+ # One of the schedulable cadence buckets, stored as declared: readers never change it, and a run+ # is never scheduled finer than the view's sources deliver.+ sync_frequency: timedelta = DEFAULT_MANAGED_VIEW_SYNC_FREQUENCY+ # The saved query's ``incremental_config`` blob: a run then upserts the rows past the last+ # watermark instead of rebuilding the table. None keeps every run a full refresh. A dict rather+ # than IncrementalConfig because that class imports Django, which this module must not.+ incremental_config: dict[str, Any] | None = None++ def __post_init__(self) -> None:+ ensure_schedulable(self.sync_frequency)ExpectedViewsProvider = Callable[["Team"], list[ProvidedView]]diff --git a/products/data_modeling/backend/logic/freshness.py b/products/data_modeling/backend/logic/freshness.pyindex e7e71ea52946..f7877c2d2a1d 100644--- a/products/data_modeling/backend/logic/freshness.py+++ b/products/data_modeling/backend/logic/freshness.py@@ -41,7 +41,7 @@ Edges are (upstream_id, downstream_id): data flows upstream -> downstream, so aimport dataclassesfrom collections import defaultdict, deque-from collections.abc import Collection, Iterable, Mapping+from collections.abc import Collection, Iterable, Mapping, Setfrom datetime import timedeltafrom typing import Literal@@ -588,6 +588,15 @@ def _nothing_satisfiable_message(return "No cadence works here."+def ensure_schedulable(target: timedelta) -> None:+ """Raise if `target` is not one of the cadences a person may pick."""+ if target not in SCHEDULABLE_BUCKETS:+ supported = ", ".join(format_cadence(interval) for interval in sorted(SCHEDULABLE_BUCKETS))+ raise UnsupportedFrequencyTargetError(+ f"Can't refresh every {humanize_cadence(target)}. Pick one of: {supported}."+ )++def validate_declared_target(*,node_id: str,@@ -604,11 +613,7 @@ def validate_declared_target(that blocks the target, which is the thing a person has to change to unblock it. `pinned`holds the pin state the write would leave, including `node_id`'s own."""- if target not in SCHEDULABLE_BUCKETS:- supported = ", ".join(format_cadence(interval) for interval in sorted(SCHEDULABLE_BUCKETS))- raise UnsupportedFrequencyTargetError(- f"Can't refresh every {humanize_cadence(target)}. Pick one of: {supported}."- )+ ensure_schedulable(target)bounds = compute_target_bounds(node_id=node_id,@@ -656,6 +661,7 @@ def find_invalid_targets(edges: list[tuple[str, str]],declared_targets: dict[str, timedelta],source_intervals: dict[str, timedelta],+ floor_exempt: Set[str] = frozenset(),pinned: Collection[str] = frozenset(),) -> list[InvalidTarget]:"""Re-validate every declared target against its current bounds.@@ -663,6 +669,9 @@ def find_invalid_targets(Targets drift: a descendant declaring a finer target lowers ancestors' ceilings, andgraph edits move floors. Runtime freshness stays correct (finest demand wins) — whatbreaks is declared == effective, so run this on any graph mutation and surface the result.++ `floor_exempt` nodes may declare a target finer than their sources deliver: the scheduler+ clamps them at run time, so the declared value is intent rather than drift."""floors = all_source_floors(edges, source_intervals, _pinned_targets(pinned, declared_targets))ceilings = all_consumer_ceilings(edges, declared_targets, pinned)@@ -670,7 +679,7 @@ def find_invalid_targets(for node_id, declared in declared_targets.items():source_floor = floors.get(node_id, STREAMING)consumer_ceiling = ceilings.get(node_id)- if is_finer_than(declared, source_floor) or (+ if (node_id not in floor_exempt and is_finer_than(declared, source_floor)) or (consumer_ceiling is not None and is_coarser_than(declared, consumer_ceiling)):invalid.append(diff --git a/products/data_modeling/backend/logic/incremental.py b/products/data_modeling/backend/logic/incremental.pyindex 2a6b15981a7b..cb691f34d0b5 100644--- a/products/data_modeling/backend/logic/incremental.py+++ b/products/data_modeling/backend/logic/incremental.py@@ -69,9 +69,12 @@ class IncrementalState:def get_incremental_config(saved_query) -> Optional[IncrementalConfig]:+ return parse_incremental_config(saved_query.incremental_config)+++def parse_incremental_config(raw: Any) -> Optional[IncrementalConfig]:"""Return the config only when it is enabled and complete. Anything malformed reads as off,so a half-written config degrades to a full refresh rather than failing a run."""- raw = saved_query.incremental_configif not isinstance(raw, dict) or not raw.get("enabled"):return Nonediff --git a/products/data_modeling/backend/logic/node_frequency.py b/products/data_modeling/backend/logic/node_frequency.pyindex d7d4e54bb274..c242efde709b 100644--- a/products/data_modeling/backend/logic/node_frequency.py+++ b/products/data_modeling/backend/logic/node_frequency.py@@ -15,13 +15,17 @@ from typing import Anyfrom django.db.models import Q, QuerySet+from posthog.dataclasses import frozen+from products.data_modeling.backend.facade.contracts import SavedQueryNodeState, SuspensionMarkerfrom products.data_modeling.backend.logic.cohort_scheduling import MINUTES_PER_WEEKfrom products.data_modeling.backend.logic.freshness import (STREAMING,ClampedCadence,+ InvalidTarget,TargetBounds,TargetOption,+ all_source_floor_bounds,all_source_floors,ancestors_of,clamp_to_source_floor,@@ -29,6 +33,7 @@ from products.data_modeling.backend.logic.freshness import (compute_target_bounds,descendants_broken_by_pin,descendants_of,+ find_invalid_targets,intersect_target_bounds,max_data_age,nodes_below_source_floor,@@ -36,7 +41,7 @@ from products.data_modeling.backend.logic.freshness import ()from products.data_modeling.backend.logic.node_suspension import merged_suspension_statefrom products.data_modeling.backend.models.dag import DAG-from products.data_modeling.backend.models.edge import NON_SCHEDULING_EDGES, Edge+from products.data_modeling.backend.models.edge import Edgefrom products.data_modeling.backend.models.node import SAVED_QUERY_NODE_TYPES, Node, NodeTypefrom products.warehouse_sources.backend.facade.models import DataWarehouseTable, ExternalDataSchema@@ -156,6 +161,12 @@ def set_declared_target(node: Node, target: timedelta | None) -> None:Clearing the target also clears the pin, because a pin holds the target and has nothing to hold without one."""+ stage_declared_target(node, target)+ node.save(update_fields=["properties"])+++def stage_declared_target(node: Node, target: timedelta | None) -> None:+ """`set_declared_target` without the save, for callers that write many nodes in one update."""properties = node.properties or {}frequency = properties.setdefault(_SYSTEM_KEY, {}).setdefault(_FREQUENCY_KEY, {})if target is None:@@ -164,7 +175,6 @@ def set_declared_target(node: Node, target: timedelta | None) -> None:else:frequency[_TARGET_SECONDS_KEY] = int(target.total_seconds())node.properties = properties- node.save(update_fields=["properties"])def set_declared_pin(node: Node, pinned: bool) -> None:@@ -278,7 +288,7 @@ def resolve_source_intervals(source_nodes: list[Node]) -> tuple[dict[str, timedereturn source_intervals, best_effort[email protected](frozen=True, kw_only=True, slots=True)+@frozenclass FrequencyGraph:"""The primitives `freshness.py` operates on, extracted from one DAG's Node/Edge rows."""@@ -291,17 +301,61 @@ class FrequencyGraph:best_effort_source_ids: set[str] # sources treated as STREAMING but not guaranteednames: dict[str, str] # node id -> display name, so a bound can name what set itidentities: dict[str, NodeIdentity] # node id -> the resource its name belongs to+ managed_node_ids: set[str] # nodes whose target the managed viewset provider declares+ stand_ins: dict[str, str] # stand-in source id -> the node that sets its interval++ def target_bounds(self, node_id: str) -> TargetBounds:+ """Every cadence `node_id` may be set to, with each blocker given as a real node id."""+ bounds = compute_target_bounds(+ node_id=node_id,+ edges=self.edges,+ declared_targets=self.declared_targets,+ source_intervals=self.source_intervals,+ pinned=self.pinned,+ )++ def real(blocker: str | None) -> str | None:+ return self.stand_ins.get(blocker, blocker) if blocker is not None else None++ # A stand-in is a source with no parents, so only the floor side of the bounds can name one.+ return dataclasses.replace(+ bounds,+ options=tuple(dataclasses.replace(option, blocker=real(option.blocker)) for option in bounds.options),+ floor=bounds.floor and dataclasses.replace(bounds.floor, blocker=real(bounds.floor.blocker)),+ )++ def invalid_targets(self, declared_targets: dict[str, timedelta] | None = None) -> list[InvalidTarget]:+ """Declared targets outside their bounds. A provider's cadence may sit below its source floor."""+ return find_invalid_targets(+ edges=self.edges,+ declared_targets=self.declared_targets if declared_targets is None else declared_targets,+ source_intervals=self.source_intervals,+ floor_exempt=self.managed_node_ids,+ pinned=self.pinned,+ )+++def _managed_view_source_id(managed_node_id: str) -> str:+ """The stand-in source a managed view's readers see in its place."""+ return f"{managed_node_id}:managed-view"def build_frequency_graph(dag: DAG) -> FrequencyGraph:"""Extract the freshness graph for a DAG: schedulable nodes, edges, targets, source floors."""nodes = list(Node.objects.filter(dag=dag).exclude(saved_query__deleted=True))- edges = [- (str(source_id), str(target_id))- for source_id, target_id in Edge.objects.filter(dag=dag)- .exclude(NON_SCHEDULING_EDGES)- .values_list("source_id", "target_id")- ]+ managed_node_ids = {+ str(node_id)+ for node_id in Node.objects.filter(dag=dag, saved_query__managed_viewset__isnull=False).values_list(+ "id", flat=True+ )+ }+ edges: list[tuple[str, str]] = []+ managed_reader_edges: list[tuple[str, str]] = []+ for source_id, target_id in Edge.objects.filter(dag=dag).values_list("source_id", "target_id"):+ edge = (str(source_id), str(target_id))+ # The same split as NON_SCHEDULING_EDGES, from ids already loaded instead of a second join.+ is_managed_reader = edge[0] in managed_node_ids and edge[1] not in managed_node_ids+ (managed_reader_edges if is_managed_reader else edges).append(edge)schedulable = {str(node.id) for node in nodes if node.type in SAVED_QUERY_NODE_TYPES}declared_targets: dict[str, timedelta] = {}@@ -319,6 +373,42 @@ def build_frequency_graph(dag: DAG) -> FrequencyGraph:source_nodes = [node for node in nodes if node.type == NodeType.TABLE]source_intervals, best_effort = resolve_source_intervals(source_nodes)+ names = {str(node.id): node.name for node in nodes}+ identities = {str(node.id): node_identity(node) for node in nodes}++ # A reader never sets a managed view's cadence, and it cannot be fresher than that view's data.+ # The view's readers see a stand-in source that changes as often as that data does.+ stand_ins: dict[str, str] = {}+ if managed_reader_edges:+ runs_every, _clamped = clamp_to_source_floor(+ compute_effective_cadences(+ nodes=schedulable, edges=edges, declared_targets=declared_targets, pinned=pinned+ ),+ edges=edges,+ source_intervals=source_intervals,+ pinned=pinned,+ )+ # A materialized view's data changes once per run, whatever its inputs do, so it counts as a+ # source at its run cadence. A query-time view passes its inputs' changes straight through.+ materialized = {str(node.id) for node in nodes if node.type == NodeType.MAT_VIEW}+ run_intervals = {+ node_id: interval+ for node_id, interval in runs_every.items()+ if node_id in materialized and interval is not None+ }+ refreshes = all_source_floor_bounds(edges, {**source_intervals, **run_intervals})+ for managed_id, reader_id in managed_reader_edges:+ if managed_id not in names:+ continue+ stand_in = _managed_view_source_id(managed_id)+ if stand_in not in stand_ins:+ refresh = refreshes.get(managed_id)+ blamed = refresh.blocker if refresh is not None and refresh.blocker in names else managed_id+ stand_ins[stand_in] = blamed+ source_intervals[stand_in] = refresh.value if refresh is not None else STREAMING+ names[stand_in] = names[blamed]+ identities[stand_in] = identities[blamed]+ edges.append((stand_in, reader_id))return FrequencyGraph(nodes=schedulable,@@ -328,8 +418,10 @@ def build_frequency_graph(dag: DAG) -> FrequencyGraph:pinned=pinned,source_intervals=source_intervals,best_effort_source_ids=best_effort,- names={str(node.id): node.name for node in nodes},- identities={str(node.id): node_identity(node) for node in nodes},+ names=names,+ identities=identities,+ managed_node_ids=managed_node_ids,+ stand_ins=stand_ins,)@@ -372,15 +464,7 @@ def saved_query_target_bounds(team_id: int, saved_query_id: str | uuid.UUID) ->if dag_key not in graphs:graphs[dag_key] = build_frequency_graph(node.dag)graph = graphs[dag_key]- per_dag.append(- compute_target_bounds(- node_id=str(node.id),- edges=graph.edges,- declared_targets=graph.declared_targets,- source_intervals=graph.source_intervals,- pinned=graph.pinned,- )- )+ per_dag.append(graph.target_bounds(str(node.id)))names.update(graph.names)identities.update(graph.identities)# a best-effort source elsewhere in the DAG says nothing about this node's freshness@@ -430,13 +514,7 @@ def saved_query_pin_preview(team_id: int, saved_query_id: str | uuid.UUID) -> Sagraphs[dag_key] = build_frequency_graph(node.dag)graph = graphs[dag_key]node_id = str(node.id)- options = compute_target_bounds(- node_id=node_id,- edges=graph.edges,- declared_targets=graph.declared_targets,- source_intervals=graph.source_intervals,- pinned=graph.pinned,- ).options+ options = graph.target_bounds(node_id).optionsfor cadence, delayed in _delays_if_pinned_by_cadence(node_id, graph, options).items():delays_if_pinned.setdefault(cadence, set()).update(delayed)below_floor = nodes_below_source_floor(diff --git a/products/data_modeling/backend/logic/saved_query_dag_sync.py b/products/data_modeling/backend/logic/saved_query_dag_sync.pyindex 2f348aedaa73..0abd670c98c9 100644--- a/products/data_modeling/backend/logic/saved_query_dag_sync.py+++ b/products/data_modeling/backend/logic/saved_query_dag_sync.py@@ -23,6 +23,7 @@ from products.data_modeling.backend.models.edge import Edgefrom products.data_modeling.backend.models.modeling import UnknownParentError, get_parents_from_model_queryfrom products.data_modeling.backend.models.node import Node, NodeTypefrom products.warehouse_sources.backend.facade.models import DataWarehouseTable+from products.warehouse_sources.backend.facade.types import DataWarehouseManagedViewSetKindif TYPE_CHECKING:from posthog.models import Team@@ -41,14 +42,17 @@ def materializes(saved_query: "DataWarehouseSavedQuery") -> bool:types the node VIEW, which `get_dag_structure` calls ephemeral and a run skips, so the table cannever come back and the query stops updating for good, without an error.- Managed views are excluded because their flag is not a statement of intent: the Revenue- Analytics viewsets set `is_materialized` at provisioning, before anything runs, so reading it- would enroll a large population of views that have never materialized a row.+ Revenue Analytics sets its flag before the first build. Requiring an existing table keeps+ those legacy views out of scheduling until they have actually materialized."""from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQueryif saved_query.origin == DataWarehouseSavedQuery.Origin.MANAGED_VIEWSET:- return saved_query.table_id is not None+ # TODO: drop this branch once the Revenue Analytics provider declares `materialized` per view+ # instead of hardcoding True. Every managed view can then be judged on `is_materialized`.+ viewset = saved_query.managed_viewset+ if viewset is None or viewset.kind == DataWarehouseManagedViewSetKind.REVENUE_ANALYTICS:+ return saved_query.table_id is not None# The column is nullable, and NULL predates its default: it means the same as False here, which# is also how `_materializes_q` reads it.return bool(saved_query.is_materialized)@@ -604,8 +608,13 @@ def _materializes_q() -> Q:"""`materializes` as a filter, for the callers that cannot ask row by row."""from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery- managed = Q(saved_query__origin=DataWarehouseSavedQuery.Origin.MANAGED_VIEWSET)- return (~managed & Q(saved_query__is_materialized=True)) | (managed & Q(saved_query__table_id__isnull=False))+ legacy_managed = Q(saved_query__origin=DataWarehouseSavedQuery.Origin.MANAGED_VIEWSET) & (+ Q(saved_query__managed_viewset__isnull=True)+ | Q(saved_query__managed_viewset__kind=DataWarehouseManagedViewSetKind.REVENUE_ANALYTICS)+ )+ return (~legacy_managed & Q(saved_query__is_materialized=True)) | (+ legacy_managed & Q(saved_query__table_id__isnull=False)+ )def _promote_view_nodes(nodes: QuerySet[Node]) -> int:diff --git a/products/data_modeling/backend/logic/schedule_reconcile.py b/products/data_modeling/backend/logic/schedule_reconcile.pyindex f8f2bdafc7f2..7567cab572ab 100644--- a/products/data_modeling/backend/logic/schedule_reconcile.py+++ b/products/data_modeling/backend/logic/schedule_reconcile.py@@ -52,7 +52,6 @@ from products.data_modeling.backend.logic.freshness import (ClampedCadence,InvalidTarget,UnsupportedFrequencyTargetError,- find_invalid_targets,format_cadence,validate_declared_target,validate_pin,@@ -68,6 +67,7 @@ from products.data_modeling.backend.logic.node_frequency import (set_declared_anchor,set_declared_pin,set_declared_target,+ stage_declared_target,)from products.data_modeling.backend.models.dag import DAGfrom products.data_modeling.backend.models.node import Node@@ -135,12 +135,7 @@ def _warn_on_invalid_targets(dag: DAG, graph: FrequencyGraph | None = None) -> N"""Surface declared targets that drifted outside their bounds; never blocks the mutation."""if graph is None:graph = build_frequency_graph(dag)- for invalid in find_invalid_targets(- edges=graph.edges,- declared_targets=graph.declared_targets,- source_intervals=graph.source_intervals,- pinned=graph.pinned,- ):+ for invalid in graph.invalid_targets():logger.warning("Declared freshness target outside its legal range",dag_id=str(dag.id),@@ -242,6 +237,31 @@ def apply_saved_query_frequency_target(return written+def apply_managed_view_frequency_targets(team_id: int, targets: Mapping[str, timedelta | None]) -> None:+ """Store each managed view's provider cadence on its DAG node(s), keyed by saved query id.++ The provider owns these cadences, so no source floor or consumer ceiling is checked: the+ scheduler already runs a node no finer than its sources deliver (`clamp_to_source_floor`), and+ the stored value stays equal to what the provider declares. One atomic write, so a batch that+ moves an upstream and a downstream view together never lands half-applied. Callers reconcile.+ """+ with transaction.atomic():+ # Ordered so that concurrent syncs of the same viewset take the row locks in the same order.+ nodes = (+ Node.objects.select_for_update()+ .only("id", "saved_query_id", "properties")+ .filter(team_id=team_id, saved_query_id__in=targets)+ .order_by("id")+ )+ changed = []+ for node in nodes:+ target = targets[str(node.saved_query_id)]+ if get_declared_target(node) != target:+ stage_declared_target(node, target)+ changed.append(node)+ Node.objects.bulk_update(changed, ["properties"])++def check_saved_query_frequency_target(saved_query: "DataWarehouseSavedQuery", target: timedelta, *, visible_names: Mapping[str, str] | None = None) -> None:@@ -362,9 +382,7 @@ def preview_dag_schedules(dag: DAG, *, seed: bool = False) -> DagSchedulePreviewplan=plan,best_effort_source_ids=graph.best_effort_source_ids,clamped=clamped,- invalid_targets=find_invalid_targets(- edges=graph.edges, declared_targets=declared, source_intervals=graph.source_intervals, pinned=graph.pinned- ),+ invalid_targets=graph.invalid_targets(declared),unsupported_tiers=sorted({tier.interval for tier in desired_tiers if tier.interval not in SCHEDULABLE_BUCKETS}),seeded=seed,)diff --git a/products/data_modeling/backend/models/datawarehouse_managed_viewset.py b/products/data_modeling/backend/models/datawarehouse_managed_viewset.pyindex 6d5c826f5807..cd0723303d3d 100644--- a/products/data_modeling/backend/models/datawarehouse_managed_viewset.py+++ b/products/data_modeling/backend/models/datawarehouse_managed_viewset.py@@ -1,6 +1,8 @@+from collections import defaultdictfrom dataclasses import dataclassfrom datetime import timedelta-from typing import Any, cast+from typing import TYPE_CHECKING, Any, cast+from uuid import UUIDfrom django.db import connection, models, transaction@@ -17,6 +19,7 @@ from posthog.hogql.database.models import (IntegerDatabaseField,MapStringDatabaseField,StringDatabaseField,+ TableNode,)from posthog.exceptions_capture import capture_exception@@ -24,11 +27,19 @@ from posthog.models.utils import CreatedMetaFields, UpdatedMetaFields, UUIDTModefrom products.data_modeling.backend.facade.managed_viewset_hooks import get_expected_views_providerfrom products.data_modeling.backend.facade.system_tables import DATA_MODELING_ALLOWED_SYSTEM_TABLES+from products.data_modeling.backend.logic.incremental import clear_incremental_state, parse_incremental_configfrom products.data_modeling.backend.models.dag import DAGfrom products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery+from products.data_modeling.backend.models.edge import Edge+from products.data_modeling.backend.models.node import Nodefrom products.revenue_analytics.backend.views.schemas import SCHEMAS as REVENUE_ANALYTICS_SCHEMASfrom products.warehouse_sources.backend.facade.types import DataWarehouseManagedViewSetKind+if TYPE_CHECKING:+ from posthog.hogql.database.database import Database++ from products.data_modeling.backend.facade.contracts import SavedQueryNodeState+logger = structlog.get_logger(__name__)@@ -37,9 +48,11 @@ class ExpectedView:name: strquery: dict[str, Any]columns: dict[str, dict[str, Any]]- # Materialized views get a 12h sync schedule. A non-materialized view (e.g. engineering+ # Materialized views get a sync schedule. A non-materialized view (e.g. engineering# analytics) is computed at query time — no schedule, no S3 table.- materialized: bool = True+ materialized: bool+ sync_frequency: timedelta+ incremental_config: dict[str, Any] | Noneclass DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTModel):@@ -76,8 +89,12 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTModMaterializes views by default."""# Deferred: schedule_reconcile pulls in temporalio; a module-level import would drag it onto django.setup()+ from products.data_modeling.backend.logic.node_frequency import node_states_by_saved_query # noqa: PLC0415from products.data_modeling.backend.logic.saved_query_dag_sync import sync_saved_query_to_dag # noqa: PLC0415- from products.data_modeling.backend.logic.schedule_reconcile import maybe_reconcile_dag # noqa: PLC0415+ from products.data_modeling.backend.logic.schedule_reconcile import ( # noqa: PLC0415 keeps temporalio off django.setup()+ apply_managed_view_frequency_targets,+ maybe_reconcile_dag,+ )provider = get_expected_views_provider(self.kind)if provider is None:@@ -89,6 +106,8 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTModquery={"kind": "HogQLQuery", "query": view.query},columns=self._get_columns_from_fields(view.fields),materialized=view.materialized,+ sync_frequency=view.sync_frequency,+ incremental_config=view.incremental_config,)for view in provider(self.team)]@@ -128,12 +147,23 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTModerror=str(e),)external_tables_by_view[view.name] = []+ # Eligibility must see updated upstream definitions, even before this batch is saved.+ database.tables.add_child(+ TableNode.create_nested_for_chain(view.name.split("."), table=temp_sq.hogql_definition()),+ table_conflict_mode="override",+ )++ incremental_config_by_view = {+ view.name: self._eligible_incremental_config(view, database) for view in expected_views+ }views_created = 0views_updated = 0- saved_queries_to_schedule: list[DataWarehouseSavedQuery] = []+ views_unchanged = 0saved_queries_to_sync: list[DataWarehouseSavedQuery] = []+ frequency_targets: dict[str, timedelta | None] = {}orphaned_views_to_revert: list[DataWarehouseSavedQuery] = []+ redefined_ids: set[UUID] = set()# Keep this transaction short: persist DB changes only. The post-commit work# (schedule_materialization → Temporal RPCs,@@ -150,44 +180,78 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTMod[f"{self.team_id}_sync_views_{self.kind}"],)+ # Read under the lock, so a row a concurrent sync just created is seen as stored.+ stored_views = {saved_query.name: saved_query for saved_query in self.saved_queries.exclude(deleted=True)}+ node_states = node_states_by_saved_query(self.team_id, [v.id for v in stored_views.values()])+for view in expected_views:- # Get the one from the DB or create a new one if doesn't exist yet- saved_query = DataWarehouseSavedQuery.objects.filter(- name=view.name, team=self.team, managed_viewset=self- ).first()- if saved_query:- created = False- else:+ stored_view = stored_views.get(view.name)+ created = stored_view is None+ if stored_view is None:saved_query = DataWarehouseSavedQuery(name=view.name,team=self.team,managed_viewset=self,origin=DataWarehouseSavedQuery.Origin.MANAGED_VIEWSET,)- created = True+ elif self._stores_view(+ stored_view,+ view,+ external_tables_by_view[view.name],+ incremental_config_by_view[view.name],+ node_states.get(str(stored_view.id)),+ ):+ # Most syncs follow a load that changed nothing: no write, no DAG or schedule work.+ views_unchanged += 1+ continue+ else:+ saved_query = stored_view+ if saved_query.query != view.query:+ redefined_ids.add(saved_query.id)# Do NOT use get_columns because it runs the query, and these are possibly heavysaved_query.query = view.querysaved_query.set_columns(view.columns)saved_query.external_tables = external_tables_by_view[view.name]saved_query.is_materialized = view.materialized- # A non-materialized view is computed at query time — no sync schedule, and it never- # enters the schedule list below.- saved_query.sync_frequency_interval = timedelta(hours=12) if view.materialized else None+ # The cadence goes straight onto the DAG node below, not through this one-shot field.+ saved_query.sync_frequency_interval = None+ saved_query.incremental_config = incremental_config_by_view[view.name]- saved_query.save()+ if created:+ saved_query.save()+ else:+ # Only the columns this sync sets. A materialization run writes incremental_state+ # concurrently, and a whole-row save would put the stale copy read above back.+ saved_query.save(+ update_fields=[+ "query",+ "columns",+ "column_order",+ "external_tables",+ "is_materialized",+ "sync_frequency_interval",+ "incremental_config",+ "updated_at",+ ]+ )saved_queries_to_sync.append(saved_query)- if view.materialized:- saved_queries_to_schedule.append(saved_query)+ # A non-materialized view is computed at query time, so it has no cadence and no schedule.+ frequency_targets[str(saved_query.id)] = view.sync_frequency if view.materialized else Noneif created:views_created += 1else:views_updated += 1- orphaned_views_to_revert = list(- self.saved_queries.exclude(name__in=expected_view_names).exclude(deleted=True)- )+ orphaned_views_to_revert = [+ saved_query for name, saved_query in stored_views.items() if name not in expected_view_names+ ]++ # In the transaction that writes the new queries, so a failure rolls the redefinition back+ # and the next sync retries it. The edges are the ones from before this sync, which still+ # reach every view whose stored rows came from the old upstream.+ self._rebuild_downstream_of(DAG.get_or_create_default(self.team), redefined_ids)# No cross-call advisory lock here on purpose: a SESSION-level pg_advisory_lock acquired and# released across separate autocommit statements leaks under transaction-mode PgBouncer — the@@ -214,9 +278,18 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTModview_name=saved_query.name,error=str(e),)+ try:+ # After every node exists, so the whole batch lands together.+ apply_managed_view_frequency_targets(self.team_id, frequency_targets)+ except Exception as e:+ capture_exception(e, {"managed_viewset_id": self.id})+ logger.warning("failed_to_apply_managed_view_cadences", team_id=self.team_id, error=str(e))- for saved_query in saved_queries_to_schedule:+ for saved_query in saved_queries_to_sync:+ if not saved_query.is_materialized:+ continuetry:+ # Carries no interval, so this only puts a brand-new DAG onto cadence tiers.saved_query.schedule_materialization(reconcile=False)except Exception as e:capture_exception(e, {"managed_viewset_id": self.id, "view_name": saved_query.name})@@ -255,6 +328,7 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTModkind=self.kind,views_created=views_created,views_updated=views_updated,+ views_unchanged=views_unchanged,views_deleted=views_deleted,)@@ -312,6 +386,90 @@ class DataWarehouseManagedViewSet(CreatedMetaFields, UpdatedMetaFields, UUIDTMod),}+ @staticmethod+ def _stores_view(+ saved_query: DataWarehouseSavedQuery,+ view: ExpectedView,+ external_tables: list[str],+ incremental_config: dict[str, Any] | None,+ node_state: "SavedQueryNodeState | None",+ ) -> bool:+ """Whether this sync's writes for ``view`` would leave ``saved_query`` and its DAG cadence as they are.++ Covers each field the sync writes and the cadence it stores on the DAG node. A view without a+ node is not stored: its DAG placement failed earlier and the sync repairs it.+ """+ return (+ node_state is not None+ and node_state.declared_target == (view.sync_frequency if view.materialized else None)+ and saved_query.query == view.query+ and saved_query.stores_columns(view.columns)+ and set(saved_query.external_tables or []) == set(external_tables)+ and saved_query.is_materialized == view.materialized+ and saved_query.sync_frequency_interval is None+ and saved_query.incremental_config == incremental_config+ )++ def _rebuild_downstream_of(self, dag: DAG, redefined_ids: set[UUID]) -> None:+ """Clear the watermark of every incremental view downstream of a redefined view.++ A view's own definition fingerprint resets its watermark when its query changes, but not when+ an upstream query changes. Its stored rows came from the old upstream, so it must rebuild.+ """+ if not redefined_ids:+ return+ children: dict[UUID, list[UUID]] = defaultdict(list)+ for source_id, target_id in Edge.objects.filter(team_id=self.team_id, dag=dag).values_list(+ "source_id", "target_id"+ ):+ children[source_id].append(target_id)+ frontier = list(+ Node.objects.filter(team_id=self.team_id, dag=dag, saved_query_id__in=redefined_ids).values_list(+ "id", flat=True+ )+ )+ reached: set[UUID] = set()+ while frontier:+ for child in children[frontier.pop()]:+ if child not in reached:+ reached.add(child)+ frontier.append(child)+ downstream = DataWarehouseSavedQuery.objects.filter(+ team_id=self.team_id,+ id__in=Node.objects.filter(id__in=reached).values("saved_query_id"),+ deleted=False,+ )+ for saved_query in downstream:+ if saved_query.incremental_state:+ clear_incremental_state(saved_query)++ def _eligible_incremental_config(self, view: ExpectedView, database: "Database") -> dict[str, Any] | None:+ """The provider's incremental config, or None when the query cannot carry it.++ The API runs this check before it saves a config. Without it a key the query does not+ output only fails on the second run, and every later run alternates a rebuild and a failure.+ """+ from products.data_modeling.backend.logic.incremental_eligibility import ( # noqa: PLC0415 keeps posthog.schema off django.setup()+ check_incremental_eligibility,+ )++ if view.incremental_config is None or not view.materialized:+ return None+ config = parse_incremental_config(view.incremental_config)+ if config is None:+ logger.warning("managed_view_incremental_config_malformed", team_id=self.team_id, view_name=view.name)+ return None+ result = check_incremental_eligibility(view.query["query"], config, database=database)+ if not result.eligible:+ logger.warning(+ "managed_view_not_incremental",+ team_id=self.team_id,+ view_name=view.name,+ blockers=result.blockers,+ )+ return None+ return view.incremental_config+@staticmethoddef _get_columns_from_fields(fields: dict[str, FieldOrTable]) -> dict[str, dict[str, Any]]:return {diff --git a/products/data_modeling/backend/test/test_node_frequency.py b/products/data_modeling/backend/test/test_node_frequency.pyindex 00412db21543..8a5bf2e0db1e 100644--- a/products/data_modeling/backend/test/test_node_frequency.py+++ b/products/data_modeling/backend/test/test_node_frequency.py@@ -1,3 +1,4 @@+import itertoolsfrom datetime import timedeltaimport pytest@@ -8,6 +9,7 @@ from parameterized import parameterizedfrom products.data_modeling.backend.logic.freshness import (STREAMING,UnsatisfiableFrequencyError,+ all_source_floor_bounds,all_source_floors,compute_effective_cadences,validate_declared_target,@@ -18,6 +20,7 @@ from products.data_modeling.backend.logic.node_frequency import (get_declared_target,persist_seed_targets,resolve_source_intervals,+ saved_query_target_bounds,seed_targets,set_declared_anchor,set_declared_target,@@ -146,8 +149,10 @@ class TestBuildFrequencyGraph(BaseTest):floor = all_source_floors(graph.edges, graph.source_intervals).get(str(matview.id), STREAMING)self.assertEqual(floor, STREAMING)- @parameterized.expand([("user_reader", False, H12), ("managed_reader", True, M15)])- def test_only_a_managed_reader_pulls_a_managed_view_to_its_cadence(self, _name, reader_is_managed, expected):+ @parameterized.expand([("user_reader", False, H12, H12), ("managed_reader", True, M15, STREAMING)])+ def test_only_a_managed_reader_pulls_a_managed_view_to_its_cadence(+ self, _name, reader_is_managed, expected, reader_floor+ ):dag = DAG.get_or_create_default(self.team)viewset = DataWarehouseManagedViewSet.objects.create(team=self.team, kind=DataWarehouseManagedViewSetKind.REVENUE_ANALYTICS@@ -168,6 +173,56 @@ class TestBuildFrequencyGraph(BaseTest):)self.assertEqual(effective[str(managed_view.id)], expected)+ floors = all_source_floor_bounds(graph.edges, graph.source_intervals)+ self.assertEqual(floors[str(reader.id)].value, reader_floor)++ @parameterized.expand(+ [+ # (source interval, None for streaming), [(managed node type, declared target)], floor, blocker index+ ("declared_finer_than_source", H6, [(NodeType.MAT_VIEW, M15)], H6, 1),+ ("unscheduled_materialized_view", H6, [(NodeType.MAT_VIEW, None)], H6, 0),+ ("materialized_view", None, [(NodeType.MAT_VIEW, H12)], H12, 1),+ (+ "query_time_view_over_a_materialized_view",+ None,+ [(NodeType.MAT_VIEW, H12), (NodeType.VIEW, None)],+ H12,+ 1,+ ),+ ("query_time_view_with_a_declared_cadence", None, [(NodeType.VIEW, H12)], None, None),+ ]+ )+ def test_a_user_reader_cannot_be_fresher_than_a_managed_views_data(+ self, _name, source_interval, managed_chain, expected_floor, blocker_index+ ):+ dag = DAG.get_or_create_default(self.team)+ viewset = DataWarehouseManagedViewSet.objects.create(+ team=self.team, kind=DataWarehouseManagedViewSetKind.REVENUE_ANALYTICS+ )+ chain = [+ _warehouse_source_node(self.team, dag, sync_frequency_interval=source_interval)+ if source_interval+ else _table_node(self.team, dag, "events", {"origin": "posthog"})+ ]+ for index, (node_type, target) in enumerate(managed_chain):+ managed_view = _saved_query_node(self.team, dag, f"managed_view_{index}", node_type)+ assert managed_view.saved_query is not None+ managed_view.saved_query.managed_viewset = viewset+ managed_view.saved_query.save(update_fields=["managed_viewset"])+ set_declared_target(managed_view, target)+ chain.append(managed_view)+ reader = _saved_query_node(self.team, dag, "reader", NodeType.MAT_VIEW)+ chain.append(reader)+ for upstream, downstream in itertools.pairwise(chain):+ Edge.objects.create(team=self.team, dag=dag, source=upstream, target=downstream)++ assert reader.saved_query_id is not None+ resolved = saved_query_target_bounds(self.team.id, reader.saved_query_id)++ assert resolved is not None+ floor = resolved.bounds.floor+ expected = None if expected_floor is None else (expected_floor, str(chain[blocker_index].id))+ self.assertEqual((floor.value, floor.blocker) if floor else None, expected)def test_imported_source_floor_makes_a_tighter_descendant_unsatisfiable(self):dag = DAG.get_or_create_default(self.team)diff --git a/products/data_modeling/backend/tests/test_managed_viewset_providers.py b/products/data_modeling/backend/tests/test_managed_viewset_providers.pyindex f93b8627b08a..9dd39ca5f3c1 100644--- a/products/data_modeling/backend/tests/test_managed_viewset_providers.py+++ b/products/data_modeling/backend/tests/test_managed_viewset_providers.py@@ -1,3 +1,7 @@+from collections.abc import Callable+from datetime import timedelta+from typing import Any+from posthog.test.base import BaseTestfrom unittest.mock import AsyncMock, Mock, patch@@ -33,6 +37,10 @@ from products.data_modeling.backend.facade.models import (NodeType,)from products.data_modeling.backend.logic.cohort_scheduling import is_tier_schedule_id+from products.data_modeling.backend.logic.flags import MANAGED_VIEWSETS_FEATURE_FLAG+from products.data_modeling.backend.logic.freshness import UnsupportedFrequencyTargetError+from products.data_modeling.backend.logic.node_frequency import get_declared_target, set_declared_target+from products.data_modeling.backend.logic.saved_query_dag_sync import sync_saved_query_to_dagfrom products.warehouse_sources.backend.facade.types import DataWarehouseManagedViewSetKindSCHEDULE_MATERIALIZATION = (@@ -63,15 +71,21 @@ def _no_schedules():return temporal-def _fake_view(name: str = "fake_provider_view", materialized: bool = True) -> ProvidedView:+def _fake_view(+ name: str = "fake_provider_view", materialized: bool = True, query: str = "SELECT 1 AS id", **overrides: Any+) -> ProvidedView:return ProvidedView(name=name,- query="SELECT 1 AS id",+ query=query,fields={"id": IntegerDatabaseField(name="id")},materialized=materialized,+ **overrides,)+INCREMENTAL_CONFIG = {"enabled": True, "incremental_key": "id", "unique_key": ["id"], "lookback_seconds": 60}++class TestManagedViewSetProviders(BaseTest):def _viewset(self) -> DataWarehouseManagedViewSet:return DataWarehouseManagedViewSet.objects.create(team=self.team, kind=KIND)@@ -94,6 +108,7 @@ class TestManagedViewSetProviders(BaseTest):view = next(v for v in views if v.name == fake_view.name)self.assertFalse(view.is_materialized)self.assertIsNone(view.sync_frequency_interval)+ self.assertIsNone(view.incremental_config)# A non-materialized view is computed at query time — it must never be scheduled for# materialization.mock_schedule.assert_not_called()@@ -159,18 +174,222 @@ class TestManagedViewSetProviders(BaseTest):self.assertIsInstance(view.hogql_definition().fields["value"], field_class)@patch(SCHEDULE_MATERIALIZATION)- def test_sync_views_is_idempotent(self, _):+ @patch(SYNC_SAVED_QUERY_TO_DAG, wraps=sync_saved_query_to_dag)+ def test_sync_views_is_idempotent(self, mock_sync_saved_query_to_dag: Mock, mock_schedule: Mock) -> None:fake_view = _fake_view()with patch.dict(_expected_views_providers, clear=True):register_expected_views_provider(KIND, lambda team: [fake_view])viewset = self._viewset()viewset.sync_views()- first_ids = sorted(v.id for v in self._views(viewset))+ first = {v.id: v.updated_at for v in self._views(viewset)}+ mock_sync_saved_query_to_dag.reset_mock()+ mock_schedule.reset_mock()+ viewset.sync_views()+ second = {v.id: v.updated_at for v in self._views(viewset)}++ self.assertEqual(first, second)+ mock_sync_saved_query_to_dag.assert_not_called()+ mock_schedule.assert_not_called()++ @parameterized.expand(+ [+ ("query", _fake_view(query="SELECT 2 AS id"), None),+ (+ "fields",+ ProvidedView(+ name="fake_provider_view",+ query="SELECT 1 AS id",+ fields={"id": IntegerDatabaseField(name="id"), "value": StringDatabaseField(name="value")},+ ),+ None,+ ),+ ("materialized", _fake_view(materialized=False), None),+ ("sync_frequency", _fake_view(sync_frequency=timedelta(minutes=15)), None),+ ("incremental_config", _fake_view(incremental_config=INCREMENTAL_CONFIG), None),+ ("view_set", _fake_view(name="renamed_provider_view"), None),+ (+ "external_tables",+ _fake_view(),+ lambda view: DataWarehouseSavedQuery.objects.filter(pk=view.pk).update(external_tables=["drifted"]),+ ),+ ("node_missing", _fake_view(), lambda view: Node.objects.filter(saved_query=view).delete()),+ (+ "node_cadence",+ _fake_view(),+ lambda view: set_declared_target(Node.objects.get(saved_query=view), timedelta(minutes=15)),+ ),+ ]+ )+ @patch(SCHEDULE_MATERIALIZATION)+ @patch(SYNC_SAVED_QUERY_TO_DAG, wraps=sync_saved_query_to_dag)+ def test_resync_writes_when_a_declared_field_or_stored_state_changes(+ self,+ _name: str,+ changed_view: ProvidedView,+ drift: Callable[[DataWarehouseSavedQuery], Any] | None,+ mock_sync_saved_query_to_dag: Mock,+ _: Mock,+ ) -> None:+ untouched_view = _fake_view(name="untouched_view")+ provided = [_fake_view(), untouched_view]+ with patch.dict(_expected_views_providers, clear=True):+ register_expected_views_provider(KIND, lambda team: provided)+ viewset = self._viewset()+ viewset.sync_views()+ if drift is not None:+ drift(DataWarehouseSavedQuery.objects.get(managed_viewset=viewset, name="fake_provider_view"))+ provided = [changed_view, untouched_view]+ mock_sync_saved_query_to_dag.reset_mock()+ viewset.sync_views()++ # Only the changed view syncs: the untouched one is skipped.+ mock_sync_saved_query_to_dag.assert_called_once()++ @patch(SCHEDULE_MATERIALIZATION)+ def test_sync_views_stores_the_cadence_and_incremental_config_the_provider_declares(self, mock_schedule):+ fake_view = _fake_view(sync_frequency=timedelta(minutes=15), incremental_config=INCREMENTAL_CONFIG)+ with patch.dict(_expected_views_providers, clear=True):+ register_expected_views_provider(KIND, lambda team: [fake_view])++ viewset = self._viewset()+ viewset.sync_views()++ view = next(v for v in self._views(viewset) if v.name == fake_view.name)+ self.assertIsNone(view.sync_frequency_interval)+ self.assertEqual(view.incremental_config, INCREMENTAL_CONFIG)+ node = Node.objects.get(saved_query=view)+ self.assertEqual(node.type, NodeType.MAT_VIEW)+ self.assertEqual(get_declared_target(node), timedelta(minutes=15))+ mock_schedule.assert_called_once()++ # Lineage resolves a managed view only when this flag puts managed views in the HogQL database.+ @patch("posthoganalytics.feature_enabled", side_effect=lambda key, *_, **__: key == MANAGED_VIEWSETS_FEATURE_FLAG)+ @patch(SCHEDULE_MATERIALIZATION)+ def test_upstream_and_downstream_cadences_change_together(self, _schedule: Mock, _flag: Mock) -> None:+ def views(cadence: timedelta) -> list[ProvidedView]:+ return [+ _fake_view(name="upstream", sync_frequency=cadence),+ ProvidedView(+ name="downstream",+ query="SELECT id FROM upstream",+ fields={"id": IntegerDatabaseField(name="id")},+ sync_frequency=cadence,+ ),+ ]++ provided = views(timedelta(minutes=15))+ with patch.dict(_expected_views_providers, clear=True):+ register_expected_views_provider(KIND, lambda team: provided)+ viewset = self._viewset()+ viewset.sync_views()+ provided = views(timedelta(hours=12))+ viewset.sync_views()++ for view in self._views(viewset):+ self.assertTrue(view.is_materialized)+ self.assertEqual(get_declared_target(Node.objects.get(saved_query=view)), timedelta(hours=12))++ @patch(SCHEDULE_MATERIALIZATION)+ def test_resync_revalidates_incremental_config_against_updated_upstream_definitions(self, _: Mock) -> None:+ views = [+ _fake_view(name="upstream", materialized=False),+ ProvidedView(+ name="downstream",+ query="SELECT id FROM upstream",+ fields={"id": IntegerDatabaseField(name="id")},+ incremental_config=INCREMENTAL_CONFIG,+ ),+ ]+ with patch.dict(_expected_views_providers, clear=True):+ register_expected_views_provider(KIND, lambda team: views)+ viewset = self._viewset()+ viewset.sync_views()+ downstream = DataWarehouseSavedQuery.objects.get(team=self.team, name="downstream")+ self.assertEqual(downstream.incremental_config, INCREMENTAL_CONFIG)++ views[0] = ProvidedView(+ name="upstream",+ query="SELECT 1 AS id LIMIT 1",+ fields={"id": IntegerDatabaseField(name="id")},+ materialized=False,+ )+ viewset.sync_views()++ downstream.refresh_from_db()+ self.assertIsNone(downstream.incremental_config)++ @parameterized.expand(+ [("upstream_redefined", "SELECT 2 AS id", True), ("upstream_unchanged", "SELECT 1 AS id", False)]+ )+ # Lineage resolves a managed view only when this flag puts managed views in the HogQL database.+ @patch("posthoganalytics.feature_enabled", side_effect=lambda key, *_, **__: key == MANAGED_VIEWSETS_FEATURE_FLAG)+ @patch(SCHEDULE_MATERIALIZATION)+ def test_resync_rebuilds_every_view_downstream_of_a_redefined_view(+ self, _name: str, upstream_query: str, rebuilds: bool, _schedule: Mock, _flag: Mock+ ) -> None:+ def incremental(name: str, parent: str) -> ProvidedView:+ return ProvidedView(+ name=name,+ query=f"SELECT id FROM {parent}",+ fields={"id": IntegerDatabaseField(name="id")},+ incremental_config=INCREMENTAL_CONFIG,+ )++ def provided(upstream: ProvidedView) -> list[ProvidedView]:+ return [upstream, incremental("middle", "upstream"), incremental("downstream", "middle")]++ views = provided(_fake_view(name="upstream", materialized=False))+ state = {"watermark": 7, "watermark_type": "int"}+ with patch.dict(_expected_views_providers, clear=True):+ register_expected_views_provider(KIND, lambda team: views)+ viewset = self._viewset()+ viewset.sync_views()+ DataWarehouseSavedQuery.objects.filter(team=self.team, name__in=["middle", "downstream"]).update(+ incremental_state=state+ )++ views[:] = provided(_fake_view(name="upstream", materialized=False, query=upstream_query))+ viewset.sync_views()++ for name in ("middle", "downstream"):+ view = DataWarehouseSavedQuery.objects.get(team=self.team, name=name)+ self.assertEqual(view.incremental_state, {} if rebuilds else state, name)++ @patch(SCHEDULE_MATERIALIZATION)+ def test_an_incremental_key_the_query_does_not_output_is_not_stored(self, _):+ fake_view = _fake_view(incremental_config={**INCREMENTAL_CONFIG, "incremental_key": "missing"})+ with patch.dict(_expected_views_providers, clear=True):+ register_expected_views_provider(KIND, lambda team: [fake_view])++ viewset = self._viewset()+ viewset.sync_views()++ view = next(v for v in self._views(viewset) if v.name == fake_view.name)+ self.assertTrue(view.is_materialized)+ self.assertIsNone(view.incremental_config)++ @patch(SCHEDULE_MATERIALIZATION)+ def test_resync_keeps_the_state_a_run_wrote_while_it_ran(self, _):+ provided = [_fake_view(incremental_config=INCREMENTAL_CONFIG)]+ state = {"watermark": 7, "watermark_type": "int"}+ original_set_columns = DataWarehouseSavedQuery.set_columns++ def run_finishes_between_read_and_save(saved_query: DataWarehouseSavedQuery, columns: dict[str, Any]) -> None:+ original_set_columns(saved_query, columns)+ DataWarehouseSavedQuery.objects.filter(pk=saved_query.pk).update(incremental_state=state)++ with patch.dict(_expected_views_providers, clear=True):+ register_expected_views_provider(KIND, lambda team: provided)++ viewset = self._viewset()viewset.sync_views()- second_ids = sorted(v.id for v in self._views(viewset))+ provided = [_fake_view(sync_frequency=timedelta(minutes=15), incremental_config=INCREMENTAL_CONFIG)]+ with patch.object(DataWarehouseSavedQuery, "set_columns", run_finishes_between_read_and_save):+ viewset.sync_views()- self.assertEqual(first_ids, second_ids)+ view = next(v for v in self._views(viewset) if v.name == provided[0].name)+ self.assertEqual(view.incremental_state, state)def test_provisioning_a_new_team_puts_every_view_on_tier_schedules(self):# Managed viewsets are how most new teams first materialize anything (a provider makes@@ -202,6 +421,10 @@ class TestManagedViewSetProviders(BaseTest):self.assertEqual(self._views(viewset), [])+ def test_a_provider_cannot_declare_an_unschedulable_cadence(self):+ with self.assertRaisesRegex(UnsupportedFrequencyTargetError, "Can.t refresh every 45 minutes"):+ _fake_view(sync_frequency=timedelta(minutes=45))+def test_sync_views_raises_for_unregistered_kind(self):with patch.dict(_expected_views_providers, clear=True):viewset = self._viewset()