:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): keep a repartition run's start across a shutdown hand-off (#128026)
Tom Owers998d239e7e77
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.pyindex 371d031acdb6..80ab7b83b0ce 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/repartition_table.py@@ -1292,7 +1292,7 @@ def _charge_attempt(Every attempt also stamps `attempt_rows`, the rewrite checkpoint it is about to run from, chargedor not: an attempt killed outright records nothing itself, so that stamp is the only thing aretry can judge the attempt it retries against (see `_retrying_a_killed_attempt`). `run_rows` is- the same reading kept once per charged run, so the give-up check weighs the run the cap actually+ the same reading kept once per sync run, so the give-up check weighs the run the cap actuallycounted rather than whichever retry happened to end it (see `_last_run_advanced_rewrite`). Eachstamp records which rewrite that reading came from too, because the reading alone is notcomparable across a restart (see `_rewrite_advanced_past`).@@ -1305,12 +1305,11 @@ def _charge_attempt(rewrite_at_start = _rewrite_checkpoint_id(schema)marker = {**pending, "attempt_rows": rows_at_start, "attempt_rewrite_id": rewrite_at_start}if not already_charged:- marker |= {- "attempts": prior + 1,- "charged_job_id": job_id,- "run_rows": rows_at_start,- "run_rewrite_id": rewrite_at_start,- }+ marker |= {"attempts": prior + 1, "charged_job_id": job_id}+ # A stand-down mid-run (a worker shutdown hand-off) refunds the charge and the run's next retry+ # charges again. That is still the same run, so it keeps the reading taken when the run began.+ if pending.get("run_job_id") != job_id:+ marker |= {"run_rows": rows_at_start, "run_rewrite_id": rewrite_at_start, "run_job_id": job_id}try:schema.set_repartition_pending(marker)except Exception:diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.pyindex 3b3fe5784d5c..75151d5a525d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_repartition_table.py@@ -821,6 +821,66 @@ class TestKilledAttemptRetry:assert failed[0]["error_type"] == "RepartitionAttemptsExhausted"schema.abandon_repartition_if_claimed.assert_called_once()+ @patch(f"{MODULE}.capture_exception")+ @patch(f"{MODULE}.capture_repartition_event")+ @patch(f"{MODULE}.HeartbeaterSync")+ @patch(f"{MODULE}.repartition_table_in_place", new_callable=AsyncMock)+ @patch(f"{MODULE}.DeltaTableRef")+ @patch(f"{MODULE}.ExternalDataJob")+ @patch(f"{MODULE}.ExternalDataSchema")+ def test_a_run_handed_off_mid_rewrite_is_judged_from_where_it_began(+ self,+ mock_schema_model: MagicMock,+ _mock_job_model: MagicMock,+ _mock_helper_cls: MagicMock,+ mock_repartition: AsyncMock,+ _mock_heartbeater: MagicMock,+ mock_capture_event: MagicMock,+ _mock_capture_exception: MagicMock,+ ) -> None:+ # A worker shutdown hands the rewrite off at a commit and refunds the charge, and the run's next+ # retry charges again. If that recharge re-stamps where the run began, a run that advanced the+ # rewrite for hours and then lost one retry to a kill reads as stalled, and the cap abandons it.+ schema = _schema(+ name="public.deals",+ s3_folder_name="deals",+ pending={**PENDING_TARGET, "attempts": MAX_REPARTITION_ATTEMPTS - 1},+ rewrite={"rows_written": 301744, "temp_uri": TEMP_URI},+ )+ mock_schema_model.objects.select_related.return_value.get.return_value = schema+ run = RepartitionActivityInputs(team_id=TEAM_ID, schema_id=SCHEMA_ID, job_id=JOB_ID, source_id=SOURCE_ID)++ async def advance_then_hand_off(*, should_stop: Any, **_kwargs: Any) -> dict[str, Any]:+ schema.repartition_rewrite = {"rows_written": 488925, "temp_uri": TEMP_URI}+ async with asyncio.timeout(10):+ while not should_stop():+ await asyncio.sleep(0.01)+ raise RepartitionStoppedError("stopped", rows_written=187181)++ mock_repartition.side_effect = advance_then_hand_off+ environment = ActivityEnvironment()+ environment.worker_shutdown()+ with pytest.raises(WorkerShuttingDownError):+ environment.run(_maybe_repartition_table, run, MagicMock())++ # The retry charges again and then dies without committing, which leaves the marker as it is.+ mock_repartition.side_effect = None+ mock_repartition.return_value = {"outcome": "completed"}+ _maybe_repartition_table(run, MagicMock())+ assert schema.repartition_pending["attempts"] == MAX_REPARTITION_ATTEMPTS++ _maybe_repartition_table(+ RepartitionActivityInputs(+ team_id=TEAM_ID, schema_id=SCHEMA_ID, job_id=str(uuid.uuid4()), source_id=SOURCE_ID+ ),+ MagicMock(),+ )++ emitted = [c.args[0] for c in mock_capture_event.call_args_list]+ assert "warehouse_repartition_failed" not in emitted+ schema.abandon_repartition_if_claimed.assert_not_called()+ assert mock_repartition.await_count == 3+@patch(f"{MODULE}.capture_exception")@patch(f"{MODULE}.capture_repartition_event")@patch(f"{MODULE}.HeartbeaterSync")