:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): drop queued size splits that the merge budget no longer needs (#122401)
Tom Owers3b17453593de
diff --git a/products/warehouse_sources/backend/temporal/data_imports/README.md b/products/warehouse_sources/backend/temporal/data_imports/README.mdindex 564dd77382c5..36509e5ea6a5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/README.md+++ b/products/warehouse_sources/backend/temporal/data_imports/README.md@@ -71,7 +71,7 @@ Tuning and gating:- The split floor and the coarsening trigger are both derived from the budget rather than tuned separately (`COARSEN_TRIGGER_DIVISOR`), so the two directions stay consistent by construction.- The coarsening gate's own tunables are `COARSEN_OOM_FREE_DAYS` (default 14, deliberately longer than the split window) and `DATA_WAREHOUSE_COARSEN_BLOCK_MERGE_PEAK_BYTES` (default 1 MB), the peak above which a merge-phase death blocks regardless of how the classification rules explained it. `DELTA_COARSEN_DECLINE_TOTAL` records which gate declined, labelled by reason.- CDC tables are excluded for now.-- Append tables are never split. The budget bounds the working set of a merge, and an append sync never merges, so a split only rewrites every row. Coarsening still applies to them. A split that detection queued for an append table is dropped by the next run, with the temp table of its rewrite (`reason=append_never_merges`). An operator target still runs.+- Append tables are never split. The budget bounds the working set of a merge, and an append sync never merges, so a split only rewrites every row. Coarsening still applies to them. A split that detection queued for an append table is dropped by the next run, with the temp table of its rewrite (`reason=append_never_merges`). An operator target still runs. The next run also drops a size split that is queued for a keyed-merge table when the recorded largest partition is within the merge budget (`reason=under_merge_split_budget`). Other triggers still run.Observability: `warehouse_repartition_flagged` / `started` / `completed` / `failed` / `skipped` PostHog events (with full team/schema/source/table context, before→after scheme, sizes, durations, and trigger reason) plus `DELTA_REPARTITION_*` Prometheus metrics.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition_controller.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition_controller.pyindex 904b54681379..f1d2e68a3837 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition_controller.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition_controller.py@@ -296,6 +296,17 @@ def is_automatic_split_of_append_table(schema: ExternalDataSchema, trigger_reasoreturn schema.is_append and trigger_reason in AUTOMATIC_SPLIT_TRIGGERS+def is_stale_size_split(schema: ExternalDataSchema, trigger_reason: str) -> bool:+ """Whether a queued target is a size split that the current merge budget no longer needs."""+ # A marker queued before the merge budget rose uses the old, smaller target.+ return (+ trigger_reason == "proactive_threshold"+ and schema.sync_type in KEYED_MERGE_SYNC_TYPES+ and schema.max_partition_bytes is not None+ and schema.max_partition_bytes <= split_budget_bytes(schema)+ )++def repartition_activity_has_work(schema: ExternalDataSchema) -> bool:"""Whether the pre-extraction repartition activity would do more than log and return.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test_repartition_controller.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test_repartition_controller.pyindex ad1f40a1e425..ecb9cce9d987 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test_repartition_controller.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test_repartition_controller.py@@ -1130,6 +1130,34 @@ class CancelledError(Exception):pass+class TestIsStaleSizeSplit:+ @pytest.mark.parametrize(+ "sync_type,trigger_reason,max_bytes_offset,expected",+ [+ pytest.param("incremental", "proactive_threshold", -1, True, id="under_budget"),+ pytest.param("incremental", "proactive_threshold", 0, True, id="at_budget"),+ pytest.param("incremental", "proactive_threshold", 1, False, id="over_budget"),+ pytest.param("incremental", "proactive_threshold", None, False, id="unmeasured"),+ pytest.param("webhook", "proactive_threshold", -1, True, id="webhook"),+ pytest.param("xmin", "proactive_threshold", -1, True, id="xmin"),+ pytest.param("incremental", "oom_history", -1, False, id="oom_history"),+ pytest.param("incremental", "admin", -1, False, id="admin"),+ pytest.param("incremental", "coarsening", -1, False, id="coarsening"),+ pytest.param("incremental", "resume", -1, False, id="resume"),+ pytest.param("full_refresh", "proactive_threshold", -1, False, id="full_refresh"),+ pytest.param("append", "proactive_threshold", -1, False, id="append"),+ ],+ )+ def test_only_a_proactive_split_of_a_keyed_table_under_budget_is_stale(+ self, sync_type, trigger_reason, max_bytes_offset, expected+ ):+ schema = ExternalDataSchema(sync_type=sync_type, sync_type_config={})+ if max_bytes_offset is not None:+ schema.sync_type_config = {"max_partition_bytes": ctrl.split_budget_bytes(schema) + max_bytes_offset}++ assert ctrl.is_stale_size_split(schema, trigger_reason) is expected++class TestRepartitionActivity:def _inputs(self, team, schema: ExternalDataSchema) -> RepartitionActivityInputs:job = _make_job(team, schema)@@ -1720,6 +1748,65 @@ class TestRepartitionActivity:mocked.assert_awaited_once()purge.assert_not_called()+ @pytest.mark.parametrize(+ "sync_type,trigger_reason,max_bytes_offset,swap,dropped",+ [+ pytest.param("incremental", "proactive_threshold", -1, False, True, id="incremental_under_budget"),+ pytest.param("webhook", "proactive_threshold", 0, False, True, id="webhook_at_budget"),+ pytest.param("xmin", "proactive_threshold", -1, False, True, id="xmin_under_budget"),+ pytest.param("incremental", "proactive_threshold", 1, False, False, id="over_budget_still_runs"),+ pytest.param("incremental", "proactive_threshold", None, False, False, id="unmeasured_still_runs"),+ pytest.param("incremental", "oom_history", -1, False, False, id="oom_history_still_runs"),+ pytest.param("incremental", "admin", -1, False, False, id="admin_still_runs"),+ pytest.param("incremental", "coarsening", -1, False, False, id="coarsening_still_runs"),+ pytest.param("incremental", "proactive_threshold", -1, True, False, id="staged_swap_is_kept"),+ ],+ )+ def test_a_queued_size_split_under_the_merge_budget_is_dropped(+ self, team, sync_type, trigger_reason, max_bytes_offset, swap, dropped+ ):+ schema = _make_schema(team, {})+ schema.sync_type = sync_type+ schema.save()+ if max_bytes_offset is not None:+ schema.record_partition_measurement(ctrl.split_budget_bytes(schema) + max_bytes_offset)+ schema.set_repartition_pending(+ {+ "partition_mode": "md5",+ "partition_count": 4,+ "partition_keys": ["id"],+ "trigger_reason": trigger_reason,+ "attempts": 0,+ }+ )+ schema.set_repartition_rewrite({"temp_uri": "s3://bucket/t__repartitioned_x", "rows_written": 10})+ if swap:+ schema.set_repartition_swap(+ {"state": "ready", "temp_uri": "s3://bucket/t__repartitioned_x", "live_uri": "s3://bucket/t"}+ )+ mocked = AsyncMock(return_value={"outcome": "completed", "row_count": 6, "partition_mode_after": "md5"})+ purge = AsyncMock(return_value=True)+ with patch.object(repartition_table, "purge_abandoned_rewrite_temp", new=purge):+ capture = self._run(self._inputs(team, schema), mocked)++ schema.refresh_from_db()+ skipped = [c.args[1] for c in capture.call_args_list if c.args[0] == "warehouse_repartition_skipped"]+ if dropped:+ mocked.assert_not_called()+ assert purge.await_args is not None+ assert purge.await_args.args[2] == "s3://bucket/t__repartitioned_x"+ assert schema.repartition_pending is None+ assert schema.repartition_rewrite is None+ assert [props["reason"] for props in skipped] == ["under_merge_split_budget"]+ assert schema.last_repartition_at is None+ else:+ assert "under_merge_split_budget" not in [props["reason"] for props in skipped]+ purge.assert_not_called()+ if swap:+ assert schema.repartition_swap is not None+ else:+ mocked.assert_awaited_once()+@pytest.mark.parametrize("prior_attempts, gives_up", [(0, False), (MAX_REPARTITION_ATTEMPTS - 1, True)])def test_an_append_split_whose_temp_cannot_be_deleted_is_retried_a_bounded_number_of_times(self, team, prior_attempts, gives_up@@ -1751,7 +1838,7 @@ class TestRepartitionActivity:if gives_up:assert schema.repartition_pending is Noneassert schema.repartition_rewrite is None- assert any("dropped append split" in str(c.args[0]) for c in captured.call_args_list)+ assert any("dropped queued split" in str(c.args[0]) for c in captured.call_args_list)else:assert schema.repartition_pending is not Noneassert schema.repartition_pending["dropping"] is Truediff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.pyindex 7e607f0c09fa..371d031acdb6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.py@@ -69,6 +69,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.repcapture_repartition_event,is_automatic_split_of_append_table,is_repartition_hold_enabled,+ is_stale_size_split,maybe_flag_for_repartition,needs_pre_extraction_detection,pre_extraction_measurement_is_redundant,@@ -474,9 +475,12 @@ def _maybe_repartition_table(inputs: RepartitionActivityInputs, logger: Filterin)trigger_reason = (pending or {}).get("trigger_reason", "resume")- # Before the reset deferral too, because a reset would write this split into the new table.+ # Drop unneeded splits before the reset deferral, or the reset would write them into the new table.if swap is None and is_automatic_split_of_append_table(schema, trigger_reason):- _drop_append_split(inputs, schema, table_ref, trigger_reason, logger)+ _drop_queued_split(inputs, schema, table_ref, trigger_reason, "append_never_merges", logger)+ return+ if swap is None and is_stale_size_split(schema, trigger_reason):+ _drop_queued_split(inputs, schema, table_ref, trigger_reason, "under_merge_split_budget", logger)return# Never while a swap is staged: live may already be deleted, and temp is then the only intact copy.@@ -1213,14 +1217,15 @@ def _give_up(DELTA_REPARTITION_TOTAL.labels(outcome="failed").inc()-def _drop_append_split(+def _drop_queued_split(inputs: RepartitionActivityInputs,schema: ExternalDataSchema,table_ref: DeltaTableRef,trigger_reason: str,+ reason: str,logger: FilteringBoundLogger,) -> None:- """Discard an automatic split queued for an append table, with the temp table of its rewrite.+ """Discard an automatic split queued for a table that does not need it, with its temp table.Same order as `_give_up`: the fresh claim fences out a predecessor that may still run as azombie, and the temp table goes before the checkpoint that records where it is.@@ -1229,43 +1234,44 @@ def _drop_append_split(if not schema.set_repartition_claim({"token": claim_token, "job_id": inputs.job_id, "claimed_at": timezone.now().isoformat()}):- logger.info("repartition: append split drop superseded by a newer claim")+ logger.info("repartition: queued split drop superseded by a newer claim")return# A predecessor can stage a swap before the claim above fences it out. Temp is then the only# intact copy, so the swap must finish, and the next run drops nothing. The claim write reloaded# the config under the row lock, so this reads the current markers.if schema.repartition_swap is not None:- logger.info("repartition: a swap is staged, so the append split stays until it lands")+ logger.info("repartition: a swap is staged, so the queued split stays until it lands")returnrewrite_rows = _rewrite_rows_written(schema)# First, so that a delete that keeps failing cannot hold the import of a rewrite that never resumes.release_repartition_hold_if_claimed(schema, claim_token=claim_token)if not _purge_abandoned_temp(schema, table_ref, claim_token, logger):- # The checkpoint is the only record of the temp table, and an append table rarely runs the fresh- # rebuild that sweeps orphans, so the drop retries on later runs, up to the attempt cap.+ # The checkpoint is the only record of the temp table, so a failed delete must retry on later+ # runs, up to the attempt cap.drop_attempts = mark_repartition_dropping_if_claimed(schema, claim_token=claim_token)if drop_attempts is None:- logger.info("repartition: append split drop superseded by a newer claim")+ logger.info("repartition: queued split drop superseded by a newer claim")returnif drop_attempts < MAX_REPARTITION_ATTEMPTS:- logger.info(f"repartition: append split drop waits for the temp table delete, attempt {drop_attempts}")+ logger.info(f"repartition: queued split drop waits for the temp table delete, attempt {drop_attempts}")returncapture_exception(- Exception(f"repartition: could not delete the temp table of a dropped append split schema_id={schema.id}")+ Exception(f"repartition: could not delete the temp table of a dropped queued split schema_id={schema.id}"))if not schema.abandon_repartition_if_claimed(claim_token, stamp_cooldown=False):- logger.info("repartition: append split drop superseded by a newer claim")+ logger.info("repartition: queued split drop superseded by a newer claim")returnlogger.info(- f"repartition: dropped the queued split of an append table, which never merges "+ f"repartition: dropped a queued split that the table does not need reason={reason} "f"schema_id={schema.id} trigger_reason={trigger_reason} rewrite_rows={rewrite_rows}",schema_id=str(schema.id),trigger_reason=trigger_reason,+ reason=reason,rewrite_rows=rewrite_rows,)DELTA_REPARTITION_TOTAL.labels(outcome="skipped").inc()props = base_event_props(schema, schema.source, inputs.job_id)- props.update({"trigger_reason": trigger_reason, "reason": "append_never_merges", "rewrite_rows": rewrite_rows})+ props.update({"trigger_reason": trigger_reason, "reason": reason, "rewrite_rows": rewrite_rows})capture_repartition_event("warehouse_repartition_skipped", props)diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.pyindex cc0f469112d7..3b3fe5784d5c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.py@@ -95,6 +95,7 @@ def _schema(schema.team_id = TEAM_IDschema.delta_revive_required = Noneschema.is_append = False+ schema.max_partition_bytes = Noneschema.sync_frequency_interval = Noneschema.repartition_swap = swapschema.repartition_pending = pending