Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): rebuild when deltalite cannot read the table mid-run (#117323)

posthog[bot]d7a67e689a1d
diff --git a/posthog/temporal/data_modeling/activities/incremental_write.py b/posthog/temporal/data_modeling/activities/incremental_write.py
index bb0aa248cb4a..f66d03826783 100644
--- a/posthog/temporal/data_modeling/activities/incremental_write.py
+++ b/posthog/temporal/data_modeling/activities/incremental_write.py
@@ -45,6 +45,14 @@ class SchemaDriftError(IncrementalWriteError):
"""
+class TableUnreadableError(IncrementalWriteError):
+ """deltalite found no usable log for a table that opened earlier in the same run.
+
+ The cause can be a storage listing glitch or a concurrent rewrite. An upsert cannot repair the
+ table in place, so the caller clears the watermark and the retry rebuilds it.
+ """
+
+
def assert_schema_matches(batch: pa.RecordBatch, target: pa.Schema) -> None:
"""Refuse to upsert unless the batch's columns and types match the table's exactly.
@@ -166,7 +174,12 @@ def upsert_batch(
import deltalite
max_row_group_bytes = settings.DATA_WAREHOUSE_DELTALITE_MAX_ROW_GROUP_BYTES
- lite = deltalite.DeltaLiteTable.open(table_uri, storage_options)
+ try:
+ lite = deltalite.DeltaLiteTable.open(table_uri, storage_options)
+ except deltalite.DeltaLiteTableNotFoundError as err:
+ raise TableUnreadableError(
+ "The stored table could not be read during an incremental update. It will be rebuilt on the next attempt."
+ ) from err
return lite.upsert(
pa.Table.from_batches([batch]),
list(unique_key),
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_incremental.py b/posthog/temporal/tests/data_modeling/test_materialize_view_incremental.py
index b99afce85981..101b3f9e701a 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_incremental.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_incremental.py
@@ -22,13 +22,14 @@ import deltalake
from posthog.sync import database_sync_to_async
from posthog.temporal.data_modeling.activities import MaterializeViewInputs, materialize_view_activity
-from posthog.temporal.data_modeling.activities.incremental_write import IncrementalWriteError
+from posthog.temporal.data_modeling.activities.incremental_write import IncrementalWriteError, TableUnreadableError
from posthog.temporal.data_modeling.activities.materialize_view import (
FAILED_UPDATE_REASON_PREFIX,
get_aws_storage_options,
)
from products.data_modeling.backend.facade.api import get_incremental_state
+from products.data_warehouse.backend.facade.api import get_s3_client
pytestmark = [pytest.mark.asyncio, pytest.mark.django_db]
@@ -302,6 +303,48 @@ class TestIncrementalMaterialization:
await database_sync_to_async(asaved_query.refresh_from_db)()
assert get_incremental_state(asaved_query).watermark is None
+ async def test_a_log_that_vanishes_mid_run_clears_the_watermark_so_the_retry_rebuilds(
+ self, activity_environment, ateam, anode, asaved_query, ajob, bucket_name, adag
+ ):
+ """The table opens at the start of the run, then its log is gone when deltalite opens it to
+ upsert. Without the watermark cleared, every retry repeats the same failed upsert."""
+ await _configure(asaved_query)
+
+ with _settings(bucket_name):
+ first = await _run(activity_environment, ateam, anode, ajob, adag, _batch([DAY1], [10]))
+
+ def remove_log_then_yield(*args, **kwargs):
+ async def generator():
+ await asyncio.to_thread(get_s3_client().delete, f"{first.table_uri}/_delta_log", recursive=True)
+ yield _batch([DAY2], [20]), [("day", "DateTime64(6, 'UTC')"), ("c", "Int64")]
+
+ return generator()
+
+ with (
+ unittest.mock.patch(
+ "posthog.temporal.data_modeling.activities.materialize_view.hogql_table", remove_log_then_yield
+ ),
+ unittest.mock.patch(
+ "products.data_modeling.backend.logic.incremental_plan.incremental_enabled", return_value=True
+ ),
+ pytest.raises(TableUnreadableError),
+ ):
+ await activity_environment.run(
+ materialize_view_activity,
+ MaterializeViewInputs(
+ team_id=ateam.pk, dag_id=str(adag.id), node_id=str(anode.id), job_id=str(ajob.id)
+ ),
+ )
+
+ await database_sync_to_async(asaved_query.refresh_from_db)()
+ assert get_incremental_state(asaved_query).watermark is None
+
+ await _run(activity_environment, ateam, anode, ajob, adag, _batch([DAY1, DAY2], [10, 20]))
+
+ await database_sync_to_async(ajob.refresh_from_db)()
+ assert ajob.run_mode == "full_refresh"
+ assert _rows(first.table_uri) == [(DAY1, 10), (DAY2, 20)]
+
async def test_an_unsigned_column_stays_incremental_across_runs(
self, activity_environment, ateam, anode, asaved_query, ajob, bucket_name, adag
):