Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): keep serving a stripe table while it rebuilds (#124901)

Daniel RC255629bde619
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
index 77bff085a6ee..631e49332494 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
@@ -398,17 +398,29 @@ def _dedupe_before_write(pa_table: pa.Table, primary_keys: list[str]) -> pa.Tabl
return deduped
-def _partial_data_loading_applies(export_signal: ExportSignalMessage, schema: ExternalDataSchema) -> bool:
- """Whether this batch feeds partial data loading (first-ever sync, Stripe only).
+def _partial_data_loading_applies(
+ export_signal: ExportSignalMessage, schema: ExternalDataSchema, job: ExternalDataJob
+) -> bool:
+ """Whether this batch feeds partial data loading (first-ever sync of a new table, Stripe only).
+
+ A reset or full refresh also loads the table from empty, but readers already query that
+ table. It keeps its served copy until the final batch publishes the complete new one, so a
+ reader never sees part of the rows. Only a table that this job created publishes each batch.
Read before the write as well as after: `previous_file_uris` only exists to serve this
feature, and collecting it means listing every file in the table.
"""
- return export_signal.is_first_ever_sync and supports_partial_data_loading(schema)
+ if not export_signal.is_first_ever_sync or not supports_partial_data_loading(schema):
+ return False
+ served_table = schema.table
+ return served_table is None or served_table.created_at >= job.created_at
def _rebuild_compaction_due(
- export_signal: ExportSignalMessage, schema: ExternalDataSchema, members: list[tuple[str, int]]
+ export_signal: ExportSignalMessage,
+ schema: ExternalDataSchema,
+ job: ExternalDataJob,
+ members: list[tuple[str, int]],
) -> bool:
"""Whether this write of a rebuild run checks the table for compaction (see `compact_during_rebuild`).
@@ -421,7 +433,7 @@ def _rebuild_compaction_due(
return (
(export_signal.is_first_ever_sync or export_signal.sync_type == "full_refresh")
and export_signal.cdc_write_mode is None
- and not _partial_data_loading_applies(export_signal, schema)
+ and not _partial_data_loading_applies(export_signal, schema, job)
and all(run_uuid == export_signal.run_uuid for run_uuid, _ in members)
and any(index > 0 and index % REBUILD_COMPACT_CHECK_EVERY == 0 for _, index in members)
and is_schema_flag_enabled(schema, REBUILD_COMPACTION_FLAG)
@@ -437,7 +449,7 @@ async def _handle_partial_data_loading(
internal_schema: HogQLSchema,
) -> None:
"""Make data available for querying during first-ever sync for Stripe sources."""
- if not _partial_data_loading_applies(export_signal, schema):
+ if not _partial_data_loading_applies(export_signal, schema, job):
return
current_file_uris = delta_table.file_uris()
@@ -1537,7 +1549,7 @@ def _process_message_reported(
# table can hold millions of files and every batch would pay to build a list nothing reads.
previous_file_uris = (
existing_delta_table.file_uris()
- if existing_delta_table is not None and _partial_data_loading_applies(export_signal, schema)
+ if existing_delta_table is not None and _partial_data_loading_applies(export_signal, schema, job)
else []
)
@@ -1688,7 +1700,7 @@ def _process_message_reported(
# sampled count needs that read only when the commit's own numbers cannot give it: batch 0
# is a sample, so the read would otherwise cost each run a log listing.
with timer.step("post_write"):
- if (sample_file_count and file_count is None) or _partial_data_loading_applies(export_signal, schema):
+ if (sample_file_count and file_count is None) or _partial_data_loading_applies(export_signal, schema, job):
current_delta_table = async_to_sync(delta_table_ref.get_delta_table)()
if current_delta_table is not None:
delta_table = current_delta_table
@@ -1732,7 +1744,7 @@ def _process_message_reported(
finals = final_run_signals(constituents)
else:
finals = [export_signal] if export_signal.is_final_batch else []
- if not finals and _rebuild_compaction_due(export_signal, schema, members):
+ if not finals and _rebuild_compaction_due(export_signal, schema, job, members):
_compact_during_rebuild(delta_table_ref, verify_ownership, timer)
_promote_loaded_prefixes(constituents if constituents is not None else [export_signal], finals, schema)
for run_signal in finals:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
index 7e41e66eb19f..533b57b0f4a7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
@@ -1,3 +1,4 @@
+import datetime as dt
import tempfile
from collections.abc import Iterator
from contextlib import ExitStack, contextmanager
@@ -45,6 +46,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
_file_count_after_write,
_get_write_type,
_mark_job_completed,
+ _partial_data_loading_applies,
_promote_loaded_prefixes,
_promote_staged_cursor,
_read_existing_rows_by_first_pk,
@@ -719,7 +721,9 @@ class TestPostWriteHandleReads:
helper.latest_known_version.return_value = 7 + commits_behind
mock_writer_cls.return_value.write = AsyncMock(return_value=returned)
mock_writer_cls.return_value.deltalite_file_count_change = deltalite_file_count_change
- mock_job_model.objects.prefetch_related.return_value.get.return_value = MagicMock()
+ job = MagicMock()
+ job.schema.table = None
+ mock_job_model.objects.prefetch_related.return_value.get.return_value = job
process_message(_message(batch_index=batch_index, is_first_ever_sync=is_first_ever_sync))
@@ -2475,11 +2479,44 @@ class TestTableHandleReuse:
assert rows.schema.names == ["id", "name", "added"]
+class TestPartialDataLoading:
+ @parameterized.expand(
+ [
+ ("first_sync_of_a_new_table", True, ExternalDataSourceType.STRIPE, None, True),
+ ("table_created_by_this_job", True, ExternalDataSourceType.STRIPE, 1, True),
+ ("rebuild_of_a_served_table", True, ExternalDataSourceType.STRIPE, -1, False),
+ ("other_source", True, ExternalDataSourceType.POSTGRES, None, False),
+ ("not_a_rebuild", False, ExternalDataSourceType.STRIPE, None, False),
+ ]
+ )
+ def test_only_a_new_table_publishes_each_batch(
+ self,
+ _name: str,
+ is_first_ever_sync: bool,
+ source_type: str,
+ table_created_hours_after_job: int | None,
+ expected: bool,
+ ) -> None:
+ job_created_at = dt.datetime(2026, 10, 1, tzinfo=dt.UTC)
+ job = MagicMock(created_at=job_created_at)
+ schema = MagicMock()
+ schema.source.source_type = source_type
+ schema.table = (
+ None
+ if table_created_hours_after_job is None
+ else MagicMock(created_at=job_created_at + dt.timedelta(hours=table_created_hours_after_job))
+ )
+ signal = ExportSignalMessage.from_dict(_message(is_first_ever_sync=is_first_ever_sync))
+
+ assert _partial_data_loading_applies(signal, schema, job) is expected
+
+
class TestCompactionDuringRebuild:
@staticmethod
def _schema(source_type: str = "Postgres") -> MagicMock:
schema = MagicMock()
schema.source.source_type = source_type
+ schema.table = None
return schema
@parameterized.expand(
@@ -2514,12 +2551,15 @@ class TestCompactionDuringRebuild:
signal = ExportSignalMessage.from_dict(_message(**{"batch_index": 10, **overrides}))
members = members or [(signal.run_uuid, signal.batch_index)]
with patch(f"{_PROCESSOR}.is_schema_flag_enabled", return_value=True):
- assert _rebuild_compaction_due(signal, self._schema(source_type or "Postgres"), members) is expected
+ assert (
+ _rebuild_compaction_due(signal, self._schema(source_type or "Postgres"), MagicMock(), members)
+ is expected
+ )
def test_the_flag_turns_it_off(self) -> None:
signal = ExportSignalMessage.from_dict(_message(batch_index=10, sync_type="full_refresh"))
with patch(f"{_PROCESSOR}.is_schema_flag_enabled", return_value=False):
- assert _rebuild_compaction_due(signal, self._schema(), [(signal.run_uuid, 10)]) is False
+ assert _rebuild_compaction_due(signal, self._schema(), MagicMock(), [(signal.run_uuid, 10)]) is False
def _load(self, table_path: str, sync_type: str, first_index: int, *, enabled: bool) -> None:
write_deltalake(table_path, TestTableHandleReuse._SEED)