Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): keep a reverted view from getting its table back (#121456)

Thiago Salvatorede4952b3d7e1
diff --git a/posthog/temporal/data_modeling/activities/prepare_queryable_table.py b/posthog/temporal/data_modeling/activities/prepare_queryable_table.py
index 27d019df2e57..8eab41b94c9b 100644
--- a/posthog/temporal/data_modeling/activities/prepare_queryable_table.py
+++ b/posthog/temporal/data_modeling/activities/prepare_queryable_table.py
@@ -1,6 +1,8 @@
import asyncio
import dataclasses
+from django.db import transaction
+
import deltalake
from structlog import get_logger
from temporalio import activity
@@ -9,9 +11,14 @@ from posthog.sync import database_sync_to_async_pool
from posthog.temporal.data_modeling.activities.materialize_view import get_aws_storage_options
from posthog.temporal.data_modeling.activities.utils import bind_data_modeling_log_context
-from products.data_modeling.backend.facade.api import promote_view_nodes_to_matview
+from products.data_modeling.backend.facade.api import (
+ link_table_if_materializing,
+ promote_view_nodes_to_matview,
+ unlink_table,
+)
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
from products.data_warehouse.backend.facade.api import create_table_from_saved_query
+from products.warehouse_sources.backend.facade.api import discard_backing_table, record_table_row_count
from products.warehouse_sources.backend.facade.models import DataWarehouseTable
from products.warehouse_sources.backend.facade.temporal import prepare_s3_files_for_querying
@@ -41,19 +48,32 @@ def _get_saved_query_with_table(inputs: PrepareQueryableTableInputs) -> DataWare
return saved_query
+def _link_or_discard_table(saved_query: DataWarehouseSavedQuery, saved_query_table: DataWarehouseTable) -> bool:
+ with transaction.atomic():
+ if link_table_if_materializing(saved_query.team_id, saved_query.id, saved_query_table.id):
+ return True
+ unlink_table(saved_query.team_id, saved_query.id, saved_query_table.id)
+ discard_backing_table(saved_query.team_id, saved_query_table.id)
+ LOGGER.info(
+ "discarded_table_published_after_revert",
+ team_id=saved_query.team_id,
+ saved_query_id=str(saved_query.id),
+ table_id=str(saved_query_table.id),
+ )
+ return False
+
+
@database_sync_to_async_pool
def _update_saved_query_with_table(
inputs: PrepareQueryableTableInputs, saved_query: DataWarehouseSavedQuery, saved_query_table: DataWarehouseTable
):
- saved_query.refresh_from_db()
- saved_query.table_id = saved_query_table.id
- saved_query.save()
+ if not _link_or_discard_table(saved_query, saved_query_table):
+ return
if not inputs.incremental:
# `create_table_from_saved_query` already counted the published files, which is the whole
# table. Keep that for an incremental run rather than overwriting it with the window.
- saved_query_table.row_count = inputs.row_count
- saved_query_table.save()
+ record_table_row_count(saved_query.team_id, saved_query_table.id, inputs.row_count)
# Every other materialization entry point can leave a stale node type behind, so repair it here.
# A query reverted mid-run is left alone: it no longer asks to materialize.
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
index ff1b6089456c..b959b55fdb71 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
@@ -67,6 +67,7 @@ from products.data_modeling.backend.facade.api import (
definition_fingerprint,
get_incremental_config,
get_incremental_state,
+ link_table_if_materializing,
set_incremental_state,
)
from products.data_modeling.backend.facade.modeling import ResolutionCycleError, bounded_resolver_factory_for_view
@@ -87,6 +88,7 @@ from products.data_quality.backend.facade.enums import (
SuiteRunTrigger,
)
from products.data_quality.backend.facade.models import DataQualityCheckRun, DataQualitySuiteRun
+from products.data_tools.backend.facade.models import DataWarehouseJoin
from products.data_warehouse.backend.facade.api import CreateTableResult
from products.managed_warehouse.backend.facade.contracts import DuckLakeTableResult, TrinoIncrementalWrite
from products.notifications.backend.facade.api import NotificationType, Priority, TargetType
@@ -1452,6 +1454,8 @@ class TestPrepareQueryableTableActivity:
await database_sync_to_async(warehouse_table.delete)()
async def test_updates_saved_query_with_table_reference(self, activity_environment, ateam, asaved_query, ajob):
+ asaved_query.is_materialized = True
+ await database_sync_to_async(asaved_query.save)()
inputs = PrepareQueryableTableInputs(
team_id=ateam.pk,
job_id=str(ajob.id),
@@ -1485,11 +1489,11 @@ class TestPrepareQueryableTableActivity:
await database_sync_to_async(warehouse_table.delete)()
@pytest.mark.parametrize(
- "is_materialized,expected_type",
- [(True, NodeType.MAT_VIEW), (False, NodeType.VIEW)],
+ "is_materialized,expected_type,keeps_table",
+ [(True, NodeType.MAT_VIEW, True), (False, NodeType.VIEW, False)],
)
async def test_linking_a_table_retypes_the_node_only_while_the_query_asks_to_materialize(
- self, activity_environment, ateam, asaved_query, anode, ajob, is_materialized, expected_type
+ self, activity_environment, ateam, asaved_query, anode, ajob, is_materialized, expected_type, keeps_table
):
# A node left typed VIEW is treated as ephemeral by every scheduled DAG run, which skips
# materialization without recording a job. Linking a table is where that gets repaired —
@@ -1497,7 +1501,7 @@ class TestPrepareQueryableTableActivity:
# the customer just turned off.
anode.type = NodeType.VIEW
await database_sync_to_async(anode.save)()
- asaved_query.is_materialized = is_materialized
+ asaved_query.is_materialized = True
await database_sync_to_async(asaved_query.save)()
inputs = PrepareQueryableTableInputs(
@@ -1510,25 +1514,94 @@ class TestPrepareQueryableTableActivity:
)
warehouse_table = await database_sync_to_async(DataWarehouseTable.objects.create)(
team=ateam,
- name="test_warehouse_table",
+ name=asaved_query.name,
format="Delta",
)
+ view_join = await database_sync_to_async(DataWarehouseJoin.objects.create)(
+ team=ateam,
+ source_table_name=asaved_query.name,
+ source_table_key="id",
+ joining_table_name="events",
+ joining_table_key="distinct_id",
+ field_name="events",
+ )
+
+ async def publish_while_the_query_changes(*args, **kwargs):
+ await database_sync_to_async(DataWarehouseSavedQuery.objects.filter(id=asaved_query.id).update)(
+ is_materialized=is_materialized, table=warehouse_table
+ )
+ return CreateTableResult(table=warehouse_table, storage_delta_mib=None, total_storage_mib=None)
+
with (
unittest.mock.patch(
"posthog.temporal.data_modeling.activities.prepare_queryable_table.prepare_s3_files_for_querying"
) as mock_prepare,
unittest.mock.patch(
- "posthog.temporal.data_modeling.activities.prepare_queryable_table.create_table_from_saved_query"
- ) as mock_create_table,
+ "posthog.temporal.data_modeling.activities.prepare_queryable_table.create_table_from_saved_query",
+ side_effect=publish_while_the_query_changes,
+ ),
):
mock_prepare.return_value = "test-bucket/queryable_folder"
- mock_create_table.return_value = CreateTableResult(
- table=warehouse_table, storage_delta_mib=None, total_storage_mib=None
- )
await activity_environment.run(prepare_queryable_table_activity, inputs)
await database_sync_to_async(anode.refresh_from_db)()
+ await database_sync_to_async(asaved_query.refresh_from_db)()
+ await database_sync_to_async(warehouse_table.refresh_from_db)()
+ await database_sync_to_async(view_join.refresh_from_db)()
assert anode.type == expected_type
+ assert (asaved_query.table_id == warehouse_table.id) is keeps_table
+ assert bool(warehouse_table.deleted) is not keeps_table
+ assert not view_join.deleted
+ await database_sync_to_async(warehouse_table.delete)()
+
+ async def test_a_revert_that_lands_right_after_the_link_still_wins(
+ self, activity_environment, ateam, asaved_query, ajob
+ ):
+ asaved_query.is_materialized = True
+ await database_sync_to_async(asaved_query.save)()
+ inputs = PrepareQueryableTableInputs(
+ team_id=ateam.pk,
+ job_id=str(ajob.id),
+ saved_query_id=str(asaved_query.id),
+ table_uri="s3://test-bucket/test_table",
+ file_uris=["s3://test-bucket/test_file.parquet"],
+ row_count=10,
+ )
+ warehouse_table = await database_sync_to_async(DataWarehouseTable.objects.create)(
+ team=ateam,
+ name=asaved_query.name,
+ format="Delta",
+ )
+
+ def link_then_revert(team_id, saved_query_id, table_id):
+ loaded_before_the_link = DataWarehouseSavedQuery.objects.get(id=asaved_query.id)
+ linked = link_table_if_materializing(team_id, saved_query_id, table_id)
+ loaded_before_the_link.revert_materialization()
+ return linked
+
+ with (
+ unittest.mock.patch(
+ "posthog.temporal.data_modeling.activities.prepare_queryable_table.prepare_s3_files_for_querying",
+ return_value="test-bucket/queryable_folder",
+ ),
+ unittest.mock.patch(
+ "posthog.temporal.data_modeling.activities.prepare_queryable_table.create_table_from_saved_query",
+ return_value=CreateTableResult(table=warehouse_table, storage_delta_mib=None, total_storage_mib=None),
+ ),
+ unittest.mock.patch(
+ "posthog.temporal.data_modeling.activities.prepare_queryable_table.link_table_if_materializing",
+ side_effect=link_then_revert,
+ ),
+ unittest.mock.patch(
+ "products.data_modeling.backend.logic.schedule_reconcile.apply_saved_query_frequency_target"
+ ),
+ ):
+ await activity_environment.run(prepare_queryable_table_activity, inputs)
+
+ await database_sync_to_async(asaved_query.refresh_from_db)()
+ await database_sync_to_async(warehouse_table.refresh_from_db)()
+ assert (asaved_query.is_materialized, asaved_query.table_id) == (False, None)
+ assert warehouse_table.deleted
await database_sync_to_async(warehouse_table.delete)()
diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.py
index da31c39339fa..890c60215bc1 100644
--- a/products/data_modeling/backend/facade/api.py
+++ b/products/data_modeling/backend/facade/api.py
@@ -57,6 +57,8 @@ _LAZY = {
"MaterializationRefusedError": "facade.contracts",
"IncrementalConfigRefusedError": "facade.contracts",
"enable_saved_query_materialization": "logic.saved_query_materialization",
+ "link_table_if_materializing": "logic.saved_query_dag_sync",
+ "unlink_table": "logic.saved_query_dag_sync",
"saved_query_materialized_at": "logic.saved_query_freshness",
"latest_saved_query_materialization_job": "logic.saved_query_freshness",
"is_materialization_fresh": "logic.saved_query_freshness",
diff --git a/products/data_modeling/backend/logic/saved_query_dag_sync.py b/products/data_modeling/backend/logic/saved_query_dag_sync.py
index 200248ff10c4..a80a08debbcb 100644
--- a/products/data_modeling/backend/logic/saved_query_dag_sync.py
+++ b/products/data_modeling/backend/logic/saved_query_dag_sync.py
@@ -4,6 +4,7 @@ from uuid import UUID
from django.db import connection, transaction
from django.db.models import Q, QuerySet
+from django.utils import timezone
import structlog
@@ -621,6 +622,31 @@ def _promote_view_nodes(nodes: QuerySet[Node]) -> int:
)
+def link_table_if_materializing(team_id: int, saved_query_id: UUID | str, table_id: UUID) -> bool:
+ """Link a published table to the saved query, unless the query stopped asking to materialize."""
+ from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
+
+ saved_query = DataWarehouseSavedQuery.objects.filter(team_id=team_id, id=saved_query_id)
+ with transaction.atomic():
+ intent = saved_query.select_for_update().values_list("is_materialized", "origin").first()
+ if intent is None:
+ return False
+ is_materialized, origin = intent
+ if not is_materialized and origin != DataWarehouseSavedQuery.Origin.MANAGED_VIEWSET:
+ return False
+ saved_query.update(table_id=table_id, updated_at=timezone.now())
+ return True
+
+
+def unlink_table(team_id: int, saved_query_id: UUID | str, table_id: UUID) -> None:
+ """Clear the saved query's table link if it still points at this table."""
+ from products.data_modeling.backend.models.datawarehouse_saved_query import DataWarehouseSavedQuery
+
+ DataWarehouseSavedQuery.objects.filter(team_id=team_id, id=saved_query_id, table_id=table_id).update(
+ table_id=None, updated_at=timezone.now()
+ )
+
+
def promote_view_nodes_to_matview(saved_query: "DataWarehouseSavedQuery") -> int:
"""Repair one saved query's nodes, for the path that just linked its table."""
return _promote_view_nodes(Node.objects.filter(team_id=saved_query.team_id, saved_query=saved_query))
diff --git a/products/data_modeling/backend/models/datawarehouse_saved_query.py b/products/data_modeling/backend/models/datawarehouse_saved_query.py
index 36c8f98985b3..68535b6a438c 100644
--- a/products/data_modeling/backend/models/datawarehouse_saved_query.py
+++ b/products/data_modeling/backend/models/datawarehouse_saved_query.py
@@ -418,7 +418,7 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,
saved_query_id=str(self.id),
)
- def revert_materialization(self):
+ def revert_materialization(self, *, delete_joins: bool = True):
from products.data_modeling.backend.logic.node_suspension import unsuspend_saved_query
from products.data_modeling.backend.logic.schedule_reconcile import apply_saved_query_frequency_target
@@ -429,9 +429,15 @@ class DataWarehouseSavedQuery(CreatedMetaFields, UUIDTModel, UpdatedMetaFields,
self.is_materialized = False
with transaction.atomic():
+ self.table_id = (
+ DataWarehouseSavedQuery.objects.select_for_update()
+ .filter(pk=self.pk)
+ .values_list("table_id", flat=True)
+ .first()
+ )
# delete the materialized table reference
if self.table is not None:
- self.table.soft_delete()
+ self.table.soft_delete(delete_joins=delete_joins)
self.table_id = None
self.save()
diff --git a/products/warehouse_sources/backend/facade/api.py b/products/warehouse_sources/backend/facade/api.py
index da2b0dd9460d..e965ae0a9aec 100644
--- a/products/warehouse_sources/backend/facade/api.py
+++ b/products/warehouse_sources/backend/facade/api.py
@@ -24,6 +24,7 @@ from typing import TYPE_CHECKING
from uuid import UUID
from django.db.models import OuterRef, Prefetch, Q, QuerySet, Subquery
+from django.utils import timezone
# Source-agnostic storage contract for user-uploaded files — shared with the upload endpoint.
from products.warehouse_sources.backend.file_uploads import (
@@ -87,6 +88,8 @@ __all__ = [
"get_queryable_table",
"resolve_object_by_name",
"direct_access_table_ids",
+ "discard_backing_table",
+ "record_table_row_count",
"allowed_table_ids",
"all_queryable_tables",
"all_queryable_table_columns",
@@ -600,6 +603,20 @@ def soft_delete_tables(team_id: int, names: Collection[str]) -> int:
return deleted
+def record_table_row_count(team_id: int, table_id: UUID, row_count: int) -> None:
+ """Set a table's row count, unless the table was deleted in the meantime."""
+ _DataWarehouseTable.objects.filter(team_id=team_id, id=table_id).exclude(deleted=True).update(
+ row_count=row_count, updated_at=timezone.now()
+ )
+
+
+def discard_backing_table(team_id: int, table_id: UUID) -> None:
+ """Soft-delete a view's backing table but keep the view's joins, which match the table's name."""
+ table = _DataWarehouseTable.objects.filter(team_id=team_id, id=table_id).exclude(deleted=True).first()
+ if table is not None:
+ table.soft_delete(delete_joins=False)
+
+
MAX_JOBS_PER_SOURCE = 100
diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.py
index 9d13b1b74ab5..c491bf417e9f 100644
--- a/products/warehouse_sources/backend/models/table.py
+++ b/products/warehouse_sources/backend/models/table.py
@@ -582,13 +582,14 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
self.options = {key: value for key, value in (self.options or {}).items() if key != "csv_allow_double_quotes"}
return ["external_data_source", "credential", "options"]
- def soft_delete(self):
+ def soft_delete(self, *, delete_joins: bool = True):
from products.data_tools.backend.models.join import DataWarehouseJoin
- for join in DataWarehouseJoin.objects.filter(
- Q(team_id=self.team.pk) & (Q(source_table_name=self.name) | Q(joining_table_name=self.name))
- ).exclude(deleted=True):
- join.soft_delete()
+ if delete_joins:
+ for join in DataWarehouseJoin.objects.filter(
+ Q(team_id=self.team.pk) & (Q(source_table_name=self.name) | Q(joining_table_name=self.name))
+ ).exclude(deleted=True):
+ join.soft_delete()
self.deleted = True
self.deleted_at = timezone.now()