:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): keep serving a stripe table while it rebuilds (#124901)
Daniel RC255629bde619
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.pyindex 77bff085a6ee..631e49332494 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py@@ -398,17 +398,29 @@ def _dedupe_before_write(pa_table: pa.Table, primary_keys: list[str]) -> pa.Tablreturn deduped-def _partial_data_loading_applies(export_signal: ExportSignalMessage, schema: ExternalDataSchema) -> bool:- """Whether this batch feeds partial data loading (first-ever sync, Stripe only).+def _partial_data_loading_applies(+ export_signal: ExportSignalMessage, schema: ExternalDataSchema, job: ExternalDataJob+) -> bool:+ """Whether this batch feeds partial data loading (first-ever sync of a new table, Stripe only).++ A reset or full refresh also loads the table from empty, but readers already query that+ table. It keeps its served copy until the final batch publishes the complete new one, so a+ reader never sees part of the rows. Only a table that this job created publishes each batch.Read before the write as well as after: `previous_file_uris` only exists to serve thisfeature, and collecting it means listing every file in the table."""- return export_signal.is_first_ever_sync and supports_partial_data_loading(schema)+ if not export_signal.is_first_ever_sync or not supports_partial_data_loading(schema):+ return False+ served_table = schema.table+ return served_table is None or served_table.created_at >= job.created_atdef _rebuild_compaction_due(- export_signal: ExportSignalMessage, schema: ExternalDataSchema, members: list[tuple[str, int]]+ export_signal: ExportSignalMessage,+ schema: ExternalDataSchema,+ job: ExternalDataJob,+ members: list[tuple[str, int]],) -> bool:"""Whether this write of a rebuild run checks the table for compaction (see `compact_during_rebuild`).@@ -421,7 +433,7 @@ def _rebuild_compaction_due(return ((export_signal.is_first_ever_sync or export_signal.sync_type == "full_refresh")and export_signal.cdc_write_mode is None- and not _partial_data_loading_applies(export_signal, schema)+ and not _partial_data_loading_applies(export_signal, schema, job)and all(run_uuid == export_signal.run_uuid for run_uuid, _ in members)and any(index > 0 and index % REBUILD_COMPACT_CHECK_EVERY == 0 for _, index in members)and is_schema_flag_enabled(schema, REBUILD_COMPACTION_FLAG)@@ -437,7 +449,7 @@ async def _handle_partial_data_loading(internal_schema: HogQLSchema,) -> None:"""Make data available for querying during first-ever sync for Stripe sources."""- if not _partial_data_loading_applies(export_signal, schema):+ if not _partial_data_loading_applies(export_signal, schema, job):returncurrent_file_uris = delta_table.file_uris()@@ -1537,7 +1549,7 @@ def _process_message_reported(# table can hold millions of files and every batch would pay to build a list nothing reads.previous_file_uris = (existing_delta_table.file_uris()- if existing_delta_table is not None and _partial_data_loading_applies(export_signal, schema)+ if existing_delta_table is not None and _partial_data_loading_applies(export_signal, schema, job)else [])@@ -1688,7 +1700,7 @@ def _process_message_reported(# sampled count needs that read only when the commit's own numbers cannot give it: batch 0# is a sample, so the read would otherwise cost each run a log listing.with timer.step("post_write"):- if (sample_file_count and file_count is None) or _partial_data_loading_applies(export_signal, schema):+ if (sample_file_count and file_count is None) or _partial_data_loading_applies(export_signal, schema, job):current_delta_table = async_to_sync(delta_table_ref.get_delta_table)()if current_delta_table is not None:delta_table = current_delta_table@@ -1732,7 +1744,7 @@ def _process_message_reported(finals = final_run_signals(constituents)else:finals = [export_signal] if export_signal.is_final_batch else []- if not finals and _rebuild_compaction_due(export_signal, schema, members):+ if not finals and _rebuild_compaction_due(export_signal, schema, job, members):_compact_during_rebuild(delta_table_ref, verify_ownership, timer)_promote_loaded_prefixes(constituents if constituents is not None else [export_signal], finals, schema)for run_signal in finals:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.pyindex 7e41e66eb19f..533b57b0f4a7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py@@ -1,3 +1,4 @@+import datetime as dtimport tempfilefrom collections.abc import Iteratorfrom contextlib import ExitStack, contextmanager@@ -45,6 +46,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_file_count_after_write,_get_write_type,_mark_job_completed,+ _partial_data_loading_applies,_promote_loaded_prefixes,_promote_staged_cursor,_read_existing_rows_by_first_pk,@@ -719,7 +721,9 @@ class TestPostWriteHandleReads:helper.latest_known_version.return_value = 7 + commits_behindmock_writer_cls.return_value.write = AsyncMock(return_value=returned)mock_writer_cls.return_value.deltalite_file_count_change = deltalite_file_count_change- mock_job_model.objects.prefetch_related.return_value.get.return_value = MagicMock()+ job = MagicMock()+ job.schema.table = None+ mock_job_model.objects.prefetch_related.return_value.get.return_value = jobprocess_message(_message(batch_index=batch_index, is_first_ever_sync=is_first_ever_sync))@@ -2475,11 +2479,44 @@ class TestTableHandleReuse:assert rows.schema.names == ["id", "name", "added"]+class TestPartialDataLoading:+ @parameterized.expand(+ [+ ("first_sync_of_a_new_table", True, ExternalDataSourceType.STRIPE, None, True),+ ("table_created_by_this_job", True, ExternalDataSourceType.STRIPE, 1, True),+ ("rebuild_of_a_served_table", True, ExternalDataSourceType.STRIPE, -1, False),+ ("other_source", True, ExternalDataSourceType.POSTGRES, None, False),+ ("not_a_rebuild", False, ExternalDataSourceType.STRIPE, None, False),+ ]+ )+ def test_only_a_new_table_publishes_each_batch(+ self,+ _name: str,+ is_first_ever_sync: bool,+ source_type: str,+ table_created_hours_after_job: int | None,+ expected: bool,+ ) -> None:+ job_created_at = dt.datetime(2026, 10, 1, tzinfo=dt.UTC)+ job = MagicMock(created_at=job_created_at)+ schema = MagicMock()+ schema.source.source_type = source_type+ schema.table = (+ None+ if table_created_hours_after_job is None+ else MagicMock(created_at=job_created_at + dt.timedelta(hours=table_created_hours_after_job))+ )+ signal = ExportSignalMessage.from_dict(_message(is_first_ever_sync=is_first_ever_sync))++ assert _partial_data_loading_applies(signal, schema, job) is expected++class TestCompactionDuringRebuild:@staticmethoddef _schema(source_type: str = "Postgres") -> MagicMock:schema = MagicMock()schema.source.source_type = source_type+ schema.table = Nonereturn schema@parameterized.expand(@@ -2514,12 +2551,15 @@ class TestCompactionDuringRebuild:signal = ExportSignalMessage.from_dict(_message(**{"batch_index": 10, **overrides}))members = members or [(signal.run_uuid, signal.batch_index)]with patch(f"{_PROCESSOR}.is_schema_flag_enabled", return_value=True):- assert _rebuild_compaction_due(signal, self._schema(source_type or "Postgres"), members) is expected+ assert (+ _rebuild_compaction_due(signal, self._schema(source_type or "Postgres"), MagicMock(), members)+ is expected+ )def test_the_flag_turns_it_off(self) -> None:signal = ExportSignalMessage.from_dict(_message(batch_index=10, sync_type="full_refresh"))with patch(f"{_PROCESSOR}.is_schema_flag_enabled", return_value=False):- assert _rebuild_compaction_due(signal, self._schema(), [(signal.run_uuid, 10)]) is False+ assert _rebuild_compaction_due(signal, self._schema(), MagicMock(), [(signal.run_uuid, 10)]) is Falsedef _load(self, table_path: str, sync_type: str, first_index: int, *, enabled: bool) -> None:write_deltalake(table_path, TestTableHandleReuse._SEED)