Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): drop queued size splits that the merge budget no longer needs (#122401)

Tom Owers3b17453593de
diff --git a/products/warehouse_sources/backend/temporal/data_imports/README.md b/products/warehouse_sources/backend/temporal/data_imports/README.md
index 564dd77382c5..36509e5ea6a5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/README.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/README.md
@@ -71,7 +71,7 @@ Tuning and gating:
- The split floor and the coarsening trigger are both derived from the budget rather than tuned separately (`COARSEN_TRIGGER_DIVISOR`), so the two directions stay consistent by construction.
- The coarsening gate's own tunables are `COARSEN_OOM_FREE_DAYS` (default 14, deliberately longer than the split window) and `DATA_WAREHOUSE_COARSEN_BLOCK_MERGE_PEAK_BYTES` (default 1 MB), the peak above which a merge-phase death blocks regardless of how the classification rules explained it. `DELTA_COARSEN_DECLINE_TOTAL` records which gate declined, labelled by reason.
- CDC tables are excluded for now.
-- Append tables are never split. The budget bounds the working set of a merge, and an append sync never merges, so a split only rewrites every row. Coarsening still applies to them. A split that detection queued for an append table is dropped by the next run, with the temp table of its rewrite (`reason=append_never_merges`). An operator target still runs.
+- Append tables are never split. The budget bounds the working set of a merge, and an append sync never merges, so a split only rewrites every row. Coarsening still applies to them. A split that detection queued for an append table is dropped by the next run, with the temp table of its rewrite (`reason=append_never_merges`). An operator target still runs. The next run also drops a size split that is queued for a keyed-merge table when the recorded largest partition is within the merge budget (`reason=under_merge_split_budget`). Other triggers still run.
Observability: `warehouse_repartition_flagged` / `started` / `completed` / `failed` / `skipped` PostHog events (with full team/schema/source/table context, before→after scheme, sizes, durations, and trigger reason) plus `DELTA_REPARTITION_*` Prometheus metrics.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition_controller.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition_controller.py
index 904b54681379..f1d2e68a3837 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition_controller.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/repartition_controller.py
@@ -296,6 +296,17 @@ def is_automatic_split_of_append_table(schema: ExternalDataSchema, trigger_reaso
return schema.is_append and trigger_reason in AUTOMATIC_SPLIT_TRIGGERS
+def is_stale_size_split(schema: ExternalDataSchema, trigger_reason: str) -> bool:
+ """Whether a queued target is a size split that the current merge budget no longer needs."""
+ # A marker queued before the merge budget rose uses the old, smaller target.
+ return (
+ trigger_reason == "proactive_threshold"
+ and schema.sync_type in KEYED_MERGE_SYNC_TYPES
+ and schema.max_partition_bytes is not None
+ and schema.max_partition_bytes <= split_budget_bytes(schema)
+ )
+
+
def repartition_activity_has_work(schema: ExternalDataSchema) -> bool:
"""Whether the pre-extraction repartition activity would do more than log and return.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test_repartition_controller.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test_repartition_controller.py
index ad1f40a1e425..ecb9cce9d987 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test_repartition_controller.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test_repartition_controller.py
@@ -1130,6 +1130,34 @@ class CancelledError(Exception):
pass
+class TestIsStaleSizeSplit:
+ @pytest.mark.parametrize(
+ "sync_type,trigger_reason,max_bytes_offset,expected",
+ [
+ pytest.param("incremental", "proactive_threshold", -1, True, id="under_budget"),
+ pytest.param("incremental", "proactive_threshold", 0, True, id="at_budget"),
+ pytest.param("incremental", "proactive_threshold", 1, False, id="over_budget"),
+ pytest.param("incremental", "proactive_threshold", None, False, id="unmeasured"),
+ pytest.param("webhook", "proactive_threshold", -1, True, id="webhook"),
+ pytest.param("xmin", "proactive_threshold", -1, True, id="xmin"),
+ pytest.param("incremental", "oom_history", -1, False, id="oom_history"),
+ pytest.param("incremental", "admin", -1, False, id="admin"),
+ pytest.param("incremental", "coarsening", -1, False, id="coarsening"),
+ pytest.param("incremental", "resume", -1, False, id="resume"),
+ pytest.param("full_refresh", "proactive_threshold", -1, False, id="full_refresh"),
+ pytest.param("append", "proactive_threshold", -1, False, id="append"),
+ ],
+ )
+ def test_only_a_proactive_split_of_a_keyed_table_under_budget_is_stale(
+ self, sync_type, trigger_reason, max_bytes_offset, expected
+ ):
+ schema = ExternalDataSchema(sync_type=sync_type, sync_type_config={})
+ if max_bytes_offset is not None:
+ schema.sync_type_config = {"max_partition_bytes": ctrl.split_budget_bytes(schema) + max_bytes_offset}
+
+ assert ctrl.is_stale_size_split(schema, trigger_reason) is expected
+
+
class TestRepartitionActivity:
def _inputs(self, team, schema: ExternalDataSchema) -> RepartitionActivityInputs:
job = _make_job(team, schema)
@@ -1720,6 +1748,65 @@ class TestRepartitionActivity:
mocked.assert_awaited_once()
purge.assert_not_called()
+ @pytest.mark.parametrize(
+ "sync_type,trigger_reason,max_bytes_offset,swap,dropped",
+ [
+ pytest.param("incremental", "proactive_threshold", -1, False, True, id="incremental_under_budget"),
+ pytest.param("webhook", "proactive_threshold", 0, False, True, id="webhook_at_budget"),
+ pytest.param("xmin", "proactive_threshold", -1, False, True, id="xmin_under_budget"),
+ pytest.param("incremental", "proactive_threshold", 1, False, False, id="over_budget_still_runs"),
+ pytest.param("incremental", "proactive_threshold", None, False, False, id="unmeasured_still_runs"),
+ pytest.param("incremental", "oom_history", -1, False, False, id="oom_history_still_runs"),
+ pytest.param("incremental", "admin", -1, False, False, id="admin_still_runs"),
+ pytest.param("incremental", "coarsening", -1, False, False, id="coarsening_still_runs"),
+ pytest.param("incremental", "proactive_threshold", -1, True, False, id="staged_swap_is_kept"),
+ ],
+ )
+ def test_a_queued_size_split_under_the_merge_budget_is_dropped(
+ self, team, sync_type, trigger_reason, max_bytes_offset, swap, dropped
+ ):
+ schema = _make_schema(team, {})
+ schema.sync_type = sync_type
+ schema.save()
+ if max_bytes_offset is not None:
+ schema.record_partition_measurement(ctrl.split_budget_bytes(schema) + max_bytes_offset)
+ schema.set_repartition_pending(
+ {
+ "partition_mode": "md5",
+ "partition_count": 4,
+ "partition_keys": ["id"],
+ "trigger_reason": trigger_reason,
+ "attempts": 0,
+ }
+ )
+ schema.set_repartition_rewrite({"temp_uri": "s3://bucket/t__repartitioned_x", "rows_written": 10})
+ if swap:
+ schema.set_repartition_swap(
+ {"state": "ready", "temp_uri": "s3://bucket/t__repartitioned_x", "live_uri": "s3://bucket/t"}
+ )
+ mocked = AsyncMock(return_value={"outcome": "completed", "row_count": 6, "partition_mode_after": "md5"})
+ purge = AsyncMock(return_value=True)
+ with patch.object(repartition_table, "purge_abandoned_rewrite_temp", new=purge):
+ capture = self._run(self._inputs(team, schema), mocked)
+
+ schema.refresh_from_db()
+ skipped = [c.args[1] for c in capture.call_args_list if c.args[0] == "warehouse_repartition_skipped"]
+ if dropped:
+ mocked.assert_not_called()
+ assert purge.await_args is not None
+ assert purge.await_args.args[2] == "s3://bucket/t__repartitioned_x"
+ assert schema.repartition_pending is None
+ assert schema.repartition_rewrite is None
+ assert [props["reason"] for props in skipped] == ["under_merge_split_budget"]
+ assert schema.last_repartition_at is None
+ else:
+ assert "under_merge_split_budget" not in [props["reason"] for props in skipped]
+ purge.assert_not_called()
+ if swap:
+ assert schema.repartition_swap is not None
+ else:
+ mocked.assert_awaited_once()
+
@pytest.mark.parametrize("prior_attempts, gives_up", [(0, False), (MAX_REPARTITION_ATTEMPTS - 1, True)])
def test_an_append_split_whose_temp_cannot_be_deleted_is_retried_a_bounded_number_of_times(
self, team, prior_attempts, gives_up
@@ -1751,7 +1838,7 @@ class TestRepartitionActivity:
if gives_up:
assert schema.repartition_pending is None
assert schema.repartition_rewrite is None
- assert any("dropped append split" in str(c.args[0]) for c in captured.call_args_list)
+ assert any("dropped queued split" in str(c.args[0]) for c in captured.call_args_list)
else:
assert schema.repartition_pending is not None
assert schema.repartition_pending["dropping"] is True
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.py
index 7e607f0c09fa..371d031acdb6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.py
@@ -69,6 +69,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.rep
capture_repartition_event,
is_automatic_split_of_append_table,
is_repartition_hold_enabled,
+ is_stale_size_split,
maybe_flag_for_repartition,
needs_pre_extraction_detection,
pre_extraction_measurement_is_redundant,
@@ -474,9 +475,12 @@ def _maybe_repartition_table(inputs: RepartitionActivityInputs, logger: Filterin
)
trigger_reason = (pending or {}).get("trigger_reason", "resume")
- # Before the reset deferral too, because a reset would write this split into the new table.
+ # Drop unneeded splits before the reset deferral, or the reset would write them into the new table.
if swap is None and is_automatic_split_of_append_table(schema, trigger_reason):
- _drop_append_split(inputs, schema, table_ref, trigger_reason, logger)
+ _drop_queued_split(inputs, schema, table_ref, trigger_reason, "append_never_merges", logger)
+ return
+ if swap is None and is_stale_size_split(schema, trigger_reason):
+ _drop_queued_split(inputs, schema, table_ref, trigger_reason, "under_merge_split_budget", logger)
return
# Never while a swap is staged: live may already be deleted, and temp is then the only intact copy.
@@ -1213,14 +1217,15 @@ def _give_up(
DELTA_REPARTITION_TOTAL.labels(outcome="failed").inc()
-def _drop_append_split(
+def _drop_queued_split(
inputs: RepartitionActivityInputs,
schema: ExternalDataSchema,
table_ref: DeltaTableRef,
trigger_reason: str,
+ reason: str,
logger: FilteringBoundLogger,
) -> None:
- """Discard an automatic split queued for an append table, with the temp table of its rewrite.
+ """Discard an automatic split queued for a table that does not need it, with its temp table.
Same order as `_give_up`: the fresh claim fences out a predecessor that may still run as a
zombie, and the temp table goes before the checkpoint that records where it is.
@@ -1229,43 +1234,44 @@ def _drop_append_split(
if not schema.set_repartition_claim(
{"token": claim_token, "job_id": inputs.job_id, "claimed_at": timezone.now().isoformat()}
):
- logger.info("repartition: append split drop superseded by a newer claim")
+ logger.info("repartition: queued split drop superseded by a newer claim")
return
# A predecessor can stage a swap before the claim above fences it out. Temp is then the only
# intact copy, so the swap must finish, and the next run drops nothing. The claim write reloaded
# the config under the row lock, so this reads the current markers.
if schema.repartition_swap is not None:
- logger.info("repartition: a swap is staged, so the append split stays until it lands")
+ logger.info("repartition: a swap is staged, so the queued split stays until it lands")
return
rewrite_rows = _rewrite_rows_written(schema)
# First, so that a delete that keeps failing cannot hold the import of a rewrite that never resumes.
release_repartition_hold_if_claimed(schema, claim_token=claim_token)
if not _purge_abandoned_temp(schema, table_ref, claim_token, logger):
- # The checkpoint is the only record of the temp table, and an append table rarely runs the fresh
- # rebuild that sweeps orphans, so the drop retries on later runs, up to the attempt cap.
+ # The checkpoint is the only record of the temp table, so a failed delete must retry on later
+ # runs, up to the attempt cap.
drop_attempts = mark_repartition_dropping_if_claimed(schema, claim_token=claim_token)
if drop_attempts is None:
- logger.info("repartition: append split drop superseded by a newer claim")
+ logger.info("repartition: queued split drop superseded by a newer claim")
return
if drop_attempts < MAX_REPARTITION_ATTEMPTS:
- logger.info(f"repartition: append split drop waits for the temp table delete, attempt {drop_attempts}")
+ logger.info(f"repartition: queued split drop waits for the temp table delete, attempt {drop_attempts}")
return
capture_exception(
- Exception(f"repartition: could not delete the temp table of a dropped append split schema_id={schema.id}")
+ Exception(f"repartition: could not delete the temp table of a dropped queued split schema_id={schema.id}")
)
if not schema.abandon_repartition_if_claimed(claim_token, stamp_cooldown=False):
- logger.info("repartition: append split drop superseded by a newer claim")
+ logger.info("repartition: queued split drop superseded by a newer claim")
return
logger.info(
- f"repartition: dropped the queued split of an append table, which never merges "
+ f"repartition: dropped a queued split that the table does not need reason={reason} "
f"schema_id={schema.id} trigger_reason={trigger_reason} rewrite_rows={rewrite_rows}",
schema_id=str(schema.id),
trigger_reason=trigger_reason,
+ reason=reason,
rewrite_rows=rewrite_rows,
)
DELTA_REPARTITION_TOTAL.labels(outcome="skipped").inc()
props = base_event_props(schema, schema.source, inputs.job_id)
- props.update({"trigger_reason": trigger_reason, "reason": "append_never_merges", "rewrite_rows": rewrite_rows})
+ props.update({"trigger_reason": trigger_reason, "reason": reason, "rewrite_rows": rewrite_rows})
capture_repartition_event("warehouse_repartition_skipped", props)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.py
index cc0f469112d7..3b3fe5784d5c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.py
@@ -95,6 +95,7 @@ def _schema(
schema.team_id = TEAM_ID
schema.delta_revive_required = None
schema.is_append = False
+ schema.max_partition_bytes = None
schema.sync_frequency_interval = None
schema.repartition_swap = swap
schema.repartition_pending = pending