:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): keep a reverted view from getting its table back (#121456)
Thiago Salvatorede4952b3d7e1
diff --git a/posthog/temporal/data_modeling/activities/prepare_queryable_table.py b/posthog/temporal/data_modeling/activities/prepare_queryable_table.pyindex 27d019df2e57..8eab41b94c9b 100644--- a/posthog/temporal/data_modeling/activities/prepare_queryable_table.py+++ b/posthog/temporal/data_modeling/activities/prepare_queryable_table.py@@ -1,6 +1,8 @@import asyncioimport dataclasses+from django.db import transaction+import deltalakefrom structlog import get_loggerfrom temporalio import activity@@ -9,9 +11,14 @@ from posthog.sync import database_sync_to_async_poolfrom posthog.temporal.data_modeling.activities.materialize_view import get_aws_storage_optionsfrom posthog.temporal.data_modeling.activities.utils import bind_data_modeling_log_context-from products.data_modeling.backend.facade.api import promote_view_nodes_to_matview+from products.data_modeling.backend.facade.api import (+ link_table_if_materializing,+ promote_view_nodes_to_matview,+ unlink_table,+)from products.data_modeling.backend.facade.models import DataWarehouseSavedQueryfrom products.data_warehouse.backend.facade.api import create_table_from_saved_query+from products.warehouse_sources.backend.facade.api import discard_backing_table, record_table_row_countfrom products.warehouse_sources.backend.facade.models import DataWarehouseTablefrom products.warehouse_sources.backend.facade.temporal import prepare_s3_files_for_querying@@ -41,19 +48,32 @@ def _get_saved_query_with_table(inputs: PrepareQueryableTableInputs) -> DataWarereturn saved_query+def _link_or_discard_table(saved_query: DataWarehouseSavedQuery, saved_query_table: DataWarehouseTable) -> bool:+ with transaction.atomic():+ if link_table_if_materializing(saved_query.team_id, saved_query.id, saved_query_table.id):+ return True+ unlink_table(saved_query.team_id, saved_query.id, saved_query_table.id)+ discard_backing_table(saved_query.team_id, saved_query_table.id)+ LOGGER.info(+ "discarded_table_published_after_revert",+ team_id=saved_query.team_id,+ saved_query_id=str(saved_query.id),+ table_id=str(saved_query_table.id),+ )+ return False++@database_sync_to_async_pooldef _update_saved_query_with_table(inputs: PrepareQueryableTableInputs, saved_query: DataWarehouseSavedQuery, saved_query_table: DataWarehouseTable):- saved_query.refresh_from_db()- saved_query.table_id = saved_query_table.id- saved_query.save()+ if not _link_or_discard_table(saved_query, saved_query_table):+ returnif not inputs.incremental:# `create_table_from_saved_query` already counted the published files, which is the whole# table. Keep that for an incremental run rather than overwriting it with the window.- saved_query_table.row_count = inputs.row_count- saved_query_table.save()+ record_table_row_count(saved_query.team_id, saved_query_table.id, inputs.row_count)# Every other materialization entry point can leave a stale node type behind, so repair it here.# A query reverted mid-run is left alone: it no longer asks to materialize.diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.pyindex ff1b6089456c..b959b55fdb71 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py@@ -67,6 +67,7 @@ from products.data_modeling.backend.facade.api import (definition_fingerprint,get_incremental_config,get_incremental_state,+ link_table_if_materializing,set_incremental_state,)from products.data_modeling.backend.facade.modeling import ResolutionCycleError, bounded_resolver_factory_for_view@@ -87,6 +88,7 @@ from products.data_quality.backend.facade.enums import (SuiteRunTrigger,)from products.data_quality.backend.facade.models import DataQualityCheckRun, DataQualitySuiteRun+from products.data_tools.backend.facade.models import DataWarehouseJoinfrom products.data_warehouse.backend.facade.api import CreateTableResultfrom products.managed_warehouse.backend.facade.contracts import DuckLakeTableResult, TrinoIncrementalWritefrom products.notifications.backend.facade.api import NotificationType, Priority, TargetType@@ -1452,6 +1454,8 @@ class TestPrepareQueryableTableActivity:await database_sync_to_async(warehouse_table.delete)()async def test_updates_saved_query_with_table_reference(self, activity_environment, ateam, asaved_query, ajob):+ asaved_query.is_materialized = True+ await database_sync_to_async(asaved_query.save)()inputs = PrepareQueryableTableInputs(team_id=ateam.pk,job_id=str(ajob.id),@@ -1485,11 +1489,11 @@ class TestPrepareQueryableTableActivity:await database_sync_to_async(warehouse_table.delete)()@pytest.mark.parametrize(- "is_materialized,expected_type",- [(True, NodeType.MAT_VIEW), (False, NodeType.VIEW)],+ "is_materialized,expected_type,keeps_table",+ [(True, NodeType.MAT_VIEW, True), (False, NodeType.VIEW, False)],)async def test_linking_a_table_retypes_the_node_only_while_the_query_asks_to_materialize(- self, activity_environment, ateam, asaved_query, anode, ajob, is_materialized, expected_type+ self, activity_environment, ateam, asaved_query, anode, ajob, is_materialized, expected_type, keeps_table):# A node left typed VIEW is treated as ephemeral by every scheduled DAG run, which skips# materialization without recording a job. Linking a table is where that gets repaired —@@ -1497,7 +1501,7 @@ class TestPrepareQueryableTableActivity:# the customer just turned off.anode.type = NodeType.VIEWawait database_sync_to_async(anode.save)()- asaved_query.is_materialized = is_materialized+ asaved_query.is_materialized = Trueawait database_sync_to_async(asaved_query.save)()inputs = PrepareQueryableTableInputs(@@ -1510,25 +1514,94 @@ class TestPrepareQueryableTableActivity:)warehouse_table = await database_sync_to_async(DataWarehouseTable.objects.create)(team=ateam,- name="test_warehouse_table",+ name=asaved_query.name,format="Delta",)+ view_join = await database_sync_to_async(DataWarehouseJoin.objects.create)(+ team=ateam,+ source_table_name=asaved_query.name,+ source_table_key="id",+ joining_table_name="events",+ joining_table_key="distinct_id",+ field_name="events",+ )++ async def publish_while_the_query_changes(*args, **kwargs):+ await database_sync_to_async(DataWarehouseSavedQuery.objects.filter(id=asaved_query.id).update)(+ is_materialized=is_materialized, table=warehouse_table+ )+ return CreateTableResult(table=warehouse_table, storage_delta_mib=None, total_storage_mib=None)+with (unittest.mock.patch("posthog.temporal.data_modeling.activities.prepare_queryable_table.prepare_s3_files_for_querying") as mock_prepare,unittest.mock.patch(- "posthog.temporal.data_modeling.activities.prepare_queryable_table.create_table_from_saved_query"- ) as mock_create_table,+ "posthog.temporal.data_modeling.activities.prepare_queryable_table.create_table_from_saved_query",+ side_effect=publish_while_the_query_changes,+ ),):mock_prepare.return_value = "test-bucket/queryable_folder"- mock_create_table.return_value = CreateTableResult(- table=warehouse_table, storage_delta_mib=None, total_storage_mib=None- )await activity_environment.run(prepare_queryable_table_activity, inputs)await database_sync_to_async(anode.refresh_from_db)()+ await database_sync_to_async(asaved_query.refresh_from_db)()+ await database_sync_to_async(warehouse_table.refresh_from_db)()+ await database_sync_to_async(view_join.refresh_from_db)()assert anode.type == expected_type+ assert (asaved_query.table_id == warehouse_table.id) is keeps_table+ assert bool(warehouse_table.deleted) is not keeps_table+ assert not view_join.deleted+ await database_sync_to_async(warehouse_table.delete)()++ async def test_a_revert_that_lands_right_after_the_link_still_wins(+ self, activity_environment, ateam, asaved_query, ajob+ ):+ asaved_query.is_materialized = True+ await database_sync_to_async(asaved_query.save)()+ inputs = PrepareQueryableTableInputs(+ team_id=ateam.pk,+ job_id=str(ajob.id),+ saved_query_id=str(asaved_query.id),+ table_uri="s3://test-bucket/test_table",+ file_uris=["s3://test-bucket/test_file.parquet"],+ row_count=10,+ )+ warehouse_table = await database_sync_to_async(DataWarehouseTable.objects.create)(+ team=ateam,+ name=asaved_query.name,+ format="Delta",+ )++ def link_then_revert(team_id, saved_query_id, table_id):+ loaded_before_the_link = DataWarehouseSavedQuery.objects.get(id=asaved_query.id)+ linked = link_table_if_materializing(team_id, saved_query_id, table_id)+ loaded_before_the_link.revert_materialization()+ return linked++ with (+ unittest.mock.patch(+ "posthog.temporal.data_modeling.activities.prepare_queryable_table.prepare_s3_files_for_querying",+ return_value="test-bucket/queryable_folder",+ ),+ unittest.mock.patch(+ "posthog.temporal.data_modeling.activities.prepare_queryable_table.create_table_from_saved_query",+ return_value=CreateTableResult(table=warehouse_table, storage_delta_mib=None, total_storage_mib=None),+ ),+ unittest.mock.patch(+ "posthog.temporal.data_modeling.activities.prepare_queryable_table.link_table_if_materializing",+ side_effect=link_then_revert,+ ),+ unittest.mock.patch(+ "products.data_modeling.backend.logic.schedule_reconcile.apply_saved_query_frequency_target"+ ),+ ):+ await activity_environment.run(prepare_queryable_table_activity, inputs)++ await database_sync_to_async(asaved_query.refresh_from_db)()+ await database_sync_to_async(warehouse_table.refresh_from_db)()+ assert (asaved_query.is_materialized, asaved_query.table_id) == (False, None)+ assert warehouse_table.deletedawait database_sync_to_async(warehouse_table.delete)()diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.pyindex da31c39339fa..890c60215bc1 100644--- a/products/data_modeling/backend/facade/api.py+++ b/products/data_modeling/backend/facade/api.py@@ -57,6 +57,8 @@ _LAZY = {"MaterializationRefusedError": "facade.contracts","IncrementalConfigRefusedError": "facade.contracts","enable_saved_query_materialization": "logic.saved_query_materialization",+ "link_table_if_materializing": "logic.saved_query_dag_sync",+ "unlink_table": "logic.saved_query_dag_sync","saved_query_materialized_at": "logic.saved_query_freshness","latest_saved_query_materialization_job": "logic.saved_query_freshness","is_materialization_fresh": "logic.saved_query_freshness",diff --git a/products/data_modeling/backend/logic/saved_query_dag_sync.py b/products/data_modeling/backend/logic/saved_query_dag_sync.pyindex 200248ff10c4..a80a08debbcb 100644--- a/products/data_modeling/backend/logic/saved_query_dag_sync.py+++ b/products/data_modeling/backend/logic/saved_query_dag_sync.py@@ -4,6 +4,7 @@ from uuid import UUIDfrom django.db import connection, transactionfrom django.db.models import Q, QuerySet+from django.utils import timezoneimport structlog@@ -621,6 +622,31 @@ def _promote_view_nodes(nodes: QuerySet[Node]) -> int:)+def link_table_if_materializing(team_id: int, saved_query_id: UUID | str, table_id: UUID) -> bool:+ """Link a published table to the saved query, unless the query stopped asking to materialize."""+ from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery++ saved_query = DataWarehouseSavedQuery.objects.filter(team_id=team_id, id=saved_query_id)+ with transaction.atomic():+ intent = saved_query.select_for_update().values_list("is_materialized", "origin").first()+ if intent is None:+ return False+ is_materialized, origin = intent+ if not is_materialized and origin != DataWarehouseSavedQuery.Origin.MANAGED_VIEWSET:+ return False+ saved_query.update(table_id=table_id, updated_at=timezone.now())+ return True+++def unlink_table(team_id: int, saved_query_id: UUID | str, table_id: UUID) -> None:+ """Clear the saved query's table link if it still points at this table."""+ from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery++ DataWarehouseSavedQuery.objects.filter(team_id=team_id, id=saved_query_id, table_id=table_id).update(+ table_id=None, updated_at=timezone.now()+ )++def promote_view_nodes_to_matview(saved_query: "DataWarehouseSavedQuery") -> int:"""Repair one saved query's nodes, for the path that just linked its table."""return _promote_view_nodes(Node.objects.filter(team_id=saved_query.team_id, saved_query=saved_query))diff --git a/products/data_modeling/backend/models/datawarehouse_saved_query.py b/products/data_modeling/backend/models/datawarehouse_saved_query.pyindex 36c8f98985b3..68535b6a438c 100644--- a/products/data_modeling/backend/models/datawarehouse_saved_query.py+++ b/products/data_modeling/backend/models/datawarehouse_saved_query.py@@ -418,7 +418,7 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,saved_query_id=str(self.id),)- def revert_materialization(self):+ def revert_materialization(self, *, delete_joins: bool = True):from products.data_modeling.backend.logic.node_suspension import unsuspend_saved_queryfrom products.data_modeling.backend.logic.schedule_reconcile import apply_saved_query_frequency_target@@ -429,9 +429,15 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,self.is_materialized = Falsewith transaction.atomic():+ self.table_id = (+ DataWarehouseSavedQuery.objects.select_for_update()+ .filter(pk=self.pk)+ .values_list("table_id", flat=True)+ .first()+ )# delete the materialized table referenceif self.table is not None:- self.table.soft_delete()+ self.table.soft_delete(delete_joins=delete_joins)self.table_id = Noneself.save()diff --git a/products/warehouse_sources/backend/facade/api.py b/products/warehouse_sources/backend/facade/api.pyindex da2b0dd9460d..e965ae0a9aec 100644--- a/products/warehouse_sources/backend/facade/api.py+++ b/products/warehouse_sources/backend/facade/api.py@@ -24,6 +24,7 @@ from typing import TYPE_CHECKINGfrom uuid import UUIDfrom django.db.models import OuterRef, Prefetch, Q, QuerySet, Subquery+from django.utils import timezone# Source-agnostic storage contract for user-uploaded files — shared with the upload endpoint.from products.warehouse_sources.backend.file_uploads import (@@ -87,6 +88,8 @@ __all__ = ["get_queryable_table","resolve_object_by_name","direct_access_table_ids",+ "discard_backing_table",+ "record_table_row_count","allowed_table_ids","all_queryable_tables","all_queryable_table_columns",@@ -600,6 +603,20 @@ def soft_delete_tables(team_id: int, names: Collection[str]) -> int:return deleted+def record_table_row_count(team_id: int, table_id: UUID, row_count: int) -> None:+ """Set a table's row count, unless the table was deleted in the meantime."""+ _DataWarehouseTable.objects.filter(team_id=team_id, id=table_id).exclude(deleted=True).update(+ row_count=row_count, updated_at=timezone.now()+ )+++def discard_backing_table(team_id: int, table_id: UUID) -> None:+ """Soft-delete a view's backing table but keep the view's joins, which match the table's name."""+ table = _DataWarehouseTable.objects.filter(team_id=team_id, id=table_id).exclude(deleted=True).first()+ if table is not None:+ table.soft_delete(delete_joins=False)++MAX_JOBS_PER_SOURCE = 100diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.pyindex 9d13b1b74ab5..c491bf417e9f 100644--- a/products/warehouse_sources/backend/models/table.py+++ b/products/warehouse_sources/backend/models/table.py@@ -582,13 +582,14 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Deletself.options = {key: value for key, value in (self.options or {}).items() if key != "csv_allow_double_quotes"}return ["external_data_source", "credential", "options"]- def soft_delete(self):+ def soft_delete(self, *, delete_joins: bool = True):from products.data_tools.backend.models.join import DataWarehouseJoin- for join in DataWarehouseJoin.objects.filter(- Q(team_id=self.team.pk) & (Q(source_table_name=self.name) | Q(joining_table_name=self.name))- ).exclude(deleted=True):- join.soft_delete()+ if delete_joins:+ for join in DataWarehouseJoin.objects.filter(+ Q(team_id=self.team.pk) & (Q(source_table_name=self.name) | Q(joining_table_name=self.name))+ ).exclude(deleted=True):+ join.soft_delete()self.deleted = Trueself.deleted_at = timezone.now()