Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): keep a repartition run's start across a shutdown hand-off (#128026)

Tom Owers998d239e7e77
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.py
index 371d031acdb6..80ab7b83b0ce 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.py
@@ -1292,7 +1292,7 @@ def _charge_attempt(
Every attempt also stamps `attempt_rows`, the rewrite checkpoint it is about to run from, charged
or not: an attempt killed outright records nothing itself, so that stamp is the only thing a
retry can judge the attempt it retries against (see `_retrying_a_killed_attempt`). `run_rows` is
- the same reading kept once per charged run, so the give-up check weighs the run the cap actually
+ the same reading kept once per sync run, so the give-up check weighs the run the cap actually
counted rather than whichever retry happened to end it (see `_last_run_advanced_rewrite`). Each
stamp records which rewrite that reading came from too, because the reading alone is not
comparable across a restart (see `_rewrite_advanced_past`).
@@ -1305,12 +1305,11 @@ def _charge_attempt(
rewrite_at_start = _rewrite_checkpoint_id(schema)
marker = {**pending, "attempt_rows": rows_at_start, "attempt_rewrite_id": rewrite_at_start}
if not already_charged:
- marker |= {
- "attempts": prior + 1,
- "charged_job_id": job_id,
- "run_rows": rows_at_start,
- "run_rewrite_id": rewrite_at_start,
- }
+ marker |= {"attempts": prior + 1, "charged_job_id": job_id}
+ # A stand-down mid-run (a worker shutdown hand-off) refunds the charge and the run's next retry
+ # charges again. That is still the same run, so it keeps the reading taken when the run began.
+ if pending.get("run_job_id") != job_id:
+ marker |= {"run_rows": rows_at_start, "run_rewrite_id": rewrite_at_start, "run_job_id": job_id}
try:
schema.set_repartition_pending(marker)
except Exception:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.py
index 3b3fe5784d5c..75151d5a525d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.py
@@ -821,6 +821,66 @@ class TestKilledAttemptRetry:
assert failed[0]["error_type"] == "RepartitionAttemptsExhausted"
schema.abandon_repartition_if_claimed.assert_called_once()
+ @patch(f"{MODULE}.capture_exception")
+ @patch(f"{MODULE}.capture_repartition_event")
+ @patch(f"{MODULE}.HeartbeaterSync")
+ @patch(f"{MODULE}.repartition_table_in_place", new_callable=AsyncMock)
+ @patch(f"{MODULE}.DeltaTableRef")
+ @patch(f"{MODULE}.ExternalDataJob")
+ @patch(f"{MODULE}.ExternalDataSchema")
+ def test_a_run_handed_off_mid_rewrite_is_judged_from_where_it_began(
+ self,
+ mock_schema_model: MagicMock,
+ _mock_job_model: MagicMock,
+ _mock_helper_cls: MagicMock,
+ mock_repartition: AsyncMock,
+ _mock_heartbeater: MagicMock,
+ mock_capture_event: MagicMock,
+ _mock_capture_exception: MagicMock,
+ ) -> None:
+ # A worker shutdown hands the rewrite off at a commit and refunds the charge, and the run's next
+ # retry charges again. If that recharge re-stamps where the run began, a run that advanced the
+ # rewrite for hours and then lost one retry to a kill reads as stalled, and the cap abandons it.
+ schema = _schema(
+ name="public.deals",
+ s3_folder_name="deals",
+ pending={**PENDING_TARGET, "attempts": MAX_REPARTITION_ATTEMPTS - 1},
+ rewrite={"rows_written": 301744, "temp_uri": TEMP_URI},
+ )
+ mock_schema_model.objects.select_related.return_value.get.return_value = schema
+ run = RepartitionActivityInputs(team_id=TEAM_ID, schema_id=SCHEMA_ID, job_id=JOB_ID, source_id=SOURCE_ID)
+
+ async def advance_then_hand_off(*, should_stop: Any, **_kwargs: Any) -> dict[str, Any]:
+ schema.repartition_rewrite = {"rows_written": 488925, "temp_uri": TEMP_URI}
+ async with asyncio.timeout(10):
+ while not should_stop():
+ await asyncio.sleep(0.01)
+ raise RepartitionStoppedError("stopped", rows_written=187181)
+
+ mock_repartition.side_effect = advance_then_hand_off
+ environment = ActivityEnvironment()
+ environment.worker_shutdown()
+ with pytest.raises(WorkerShuttingDownError):
+ environment.run(_maybe_repartition_table, run, MagicMock())
+
+ # The retry charges again and then dies without committing, which leaves the marker as it is.
+ mock_repartition.side_effect = None
+ mock_repartition.return_value = {"outcome": "completed"}
+ _maybe_repartition_table(run, MagicMock())
+ assert schema.repartition_pending["attempts"] == MAX_REPARTITION_ATTEMPTS
+
+ _maybe_repartition_table(
+ RepartitionActivityInputs(
+ team_id=TEAM_ID, schema_id=SCHEMA_ID, job_id=str(uuid.uuid4()), source_id=SOURCE_ID
+ ),
+ MagicMock(),
+ )
+
+ emitted = [c.args[0] for c in mock_capture_event.call_args_list]
+ assert "warehouse_repartition_failed" not in emitted
+ schema.abandon_repartition_if_claimed.assert_not_called()
+ assert mock_repartition.await_count == 3
+
@patch(f"{MODULE}.capture_exception")
@patch(f"{MODULE}.capture_repartition_event")
@patch(f"{MODULE}.HeartbeaterSync")