:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(managed-migrations): re-send an interrupted chunk on resume (#125952)
Eli Reisman038ae359b564
diff --git a/products/managed_migrations/backend/admin/batch_imports.py b/products/managed_migrations/backend/admin/batch_imports.pyindex 824b2f291b57..745a48227388 100644--- a/products/managed_migrations/backend/admin/batch_imports.py+++ b/products/managed_migrations/backend/admin/batch_imports.py@@ -56,7 +56,8 @@ class BatchImportAdminForm(forms.ModelForm):if self.resumes():# Django admin runs the whole POST in one transaction, so this lock holds until save_model# writes the resume. A refusal here re-renders the form with the error and saves nothing.- self.raise_if_resume_unsafe(BatchImport.objects.select_for_update().get(pk=self.instance.pk))+ self.locked_row = BatchImport.objects.select_for_update().get(pk=self.instance.pk)+ self.raise_if_resume_unsafe(self.locked_row)return cleaned_datadef resumes(self) -> bool:@@ -161,7 +162,10 @@ class BatchImportAdmin(admin.ModelAdmin):"Changing the status to running, or using a resume action, clears the lease, backoff ""and pending commit so a worker can claim the job at once. A pending commit on ""a job no worker holds means a worker stopped mid-commit: the chunk it names "- "may or may not have reached the sink. Choosing a resume action: 'Resume (keep progress)' "+ "may or may not have reached the sink. When the saved offset still matches that chunk, "+ "a resume moves the part back to the start of the chunk, so nothing is lost. A chunk "+ "that had arrived is sent twice. "+ "Choosing a resume action: 'Resume (keep progress)' ""continues from the saved byte offset - right for transient pauses and ""for source data that has not changed. 'Resume + re-import in-flight part' ""resets the current part to offset 0 first - right when the source bytes "@@ -203,7 +207,13 @@ class BatchImportAdmin(admin.ModelAdmin):update_fields.append("import_config")if form.resumes():message = obj.status_message if "status_message" in form.changed_data else "Resumed by admin"- update_fields.extend(obj.reset_for_resume(message))+ # Take the reset fields from the row locked in clean(): obj was loaded before the lock, so its+ # state and pending_commit can miss a worker write.+ locked_row = form.locked_row+ reset_fields = locked_row.rewind_and_reset_for_resume(message)+ for name in reset_fields:+ setattr(obj, name, getattr(locked_row, name))+ update_fields.extend(reset_fields)if update_fields:obj.save(update_fields=list(dict.fromkeys([*update_fields, "updated_at"])))@@ -237,8 +247,8 @@ class BatchImportAdmin(admin.ModelAdmin):permissions=["change"],)def resume(self, request, queryset):- """Resume paused jobs from their saved progress: for transient pauses and- unchanged source data. See the 'Worker state' section on the detail page+ """Resume paused jobs from their saved progress, re-sending a chunk a worker left+ mid-commit: for transient pauses and unchanged source data. See the 'Worker state' section on the detail pagefor how this differs from the reset variant."""for batch_import in queryset:try:diff --git a/products/managed_migrations/backend/api/test/test_batch_imports.py b/products/managed_migrations/backend/api/test/test_batch_imports.pyindex 6d0764be9a47..d325e55526c7 100644--- a/products/managed_migrations/backend/api/test/test_batch_imports.py+++ b/products/managed_migrations/backend/api/test/test_batch_imports.py@@ -84,6 +84,7 @@ class TestBatchImportModel(BaseTest):{"key": "day-2", "current_offset": 0, "total_size": None},{"key": "day-3", "current_offset": 0, "total_size": None},],+ "re-sending the interrupted chunk of part day-2 from offset 8999000",),("inflight_part_with_known_total",@@ -96,6 +97,7 @@ class TestBatchImportModel(BaseTest):{"key": "day-1", "current_offset": 0, "total_size": None},{"key": "day-2", "current_offset": 0, "total_size": None},],+ "kept the saved offset of part day-2, which does not match the interrupted chunk",),("first_part_inflight",@@ -108,12 +110,29 @@ class TestBatchImportModel(BaseTest):{"key": "day-1", "current_offset": 0, "total_size": None},{"key": "day-2", "current_offset": 0, "total_size": None},],+ "kept the saved offset of part day-2, which does not match the interrupted chunk",+ ),+ (+ "interrupted_last_chunk_of_a_finished_part",+ [+ {"key": "day-1", "current_offset": 1000, "total_size": 1000},+ {"key": "day-2", "current_offset": 8999445, "total_size": 8999445},+ {"key": "day-3", "current_offset": 0, "total_size": None},+ ],+ "day-2",+ [+ {"key": "day-1", "current_offset": 1000, "total_size": 1000},+ {"key": "day-2", "current_offset": 0, "total_size": None},+ {"key": "day-3", "current_offset": 0, "total_size": None},+ ],+ "re-sending the interrupted chunk of part day-2 from offset 8999000",),("job_paused_before_any_read",[{"key": "day-1", "current_offset": 0, "total_size": None}],"day-1",[{"key": "day-1", "current_offset": 0, "total_size": None}],+ "kept the saved offset of part day-2, which does not match the interrupted chunk",),(# url_list part keys are credential-bearing URLs: the returned key (and the@@ -135,11 +154,12 @@ class TestBatchImportModel(BaseTest):"total_size": None,}],+ "kept the saved offset of part day-2, which does not match the interrupted chunk",),])def test_resume_with_inflight_part_reset_resets_first_unfinished_part(- self, _name, initial_parts, expected_reset_key, expected_parts+ self, _name, initial_parts, expected_reset_key, expected_parts, expected_rewind_note):batch_import = self._paused_import({"parts": initial_parts})@@ -149,6 +169,10 @@ class TestBatchImportModel(BaseTest):batch_import.refresh_from_db()assert batch_import.state is not Noneself.assertNotIn("X-Amz-Signature", batch_import.status_message or "")+ self.assertEqual(+ batch_import.status_message,+ f"Resumed by admin with part {expected_reset_key} reset to offset 0; {expected_rewind_note}",+ )self.assertEqual(batch_import.state["parts"], expected_parts)self.assertEqual(batch_import.status, BatchImport.Status.RUNNING)self.assertIsNone(batch_import.lease_id)@@ -158,18 +182,71 @@ class TestBatchImportModel(BaseTest):self.assertIsNone(batch_import.display_status_message)self.assertIsNone(batch_import.pending_commit)- def test_resume_after_pause_keeps_progress_and_clears_lease(self):+ @parameterized.expand(+ [+ (+ "marker_matches_part",+ {"key": "day-2", "from_offset": 8999000, "to_offset": 8999445, "events": 40},+ 8999000,+ 960,+ "Resumed by admin; re-sending the interrupted chunk of part day-2 from offset 8999000",+ ),+ (+ "marker_from_a_worker_that_did_not_count_events",+ {"key": "day-2", "from_offset": 8999000, "to_offset": 8999445},+ 8999000,+ 1000,+ "Resumed by admin; re-sending the interrupted chunk of part day-2 from offset 8999000",+ ),+ (+ "part_already_at_chunk_start",+ {"key": "day-2", "from_offset": 8999445, "to_offset": 9000000, "interrupted": 3, "events": 40},+ 8999445,+ 1000,+ "Resumed by admin; re-sending the interrupted chunk of part day-2 from offset 8999445",+ ),+ ("no_marker", None, 8999445, 1000, "Resumed by admin"),+ (+ "marker_offset_mismatch",+ {"key": "day-2", "from_offset": 100, "to_offset": 200, "events": 40},+ 8999445,+ 1000,+ "Resumed by admin; kept the saved offset of part day-2, which does not match the interrupted chunk",+ ),+ (+ "marker_part_missing",+ {"key": "day-9", "from_offset": 8999000, "to_offset": 8999445, "events": 40},+ 8999445,+ 1000,+ "Resumed by admin; kept the saved offset of part day-9, which does not match the interrupted chunk",+ ),+ ]+ )+ def test_resume_after_pause_rewinds_an_interrupted_chunk_and_clears_lease(+ self, _name, pending_commit, expected_day_2_offset, expected_events_emitted, expected_message+ ):batch_import = self._paused_import({"parts": self.MIXED_PROGRESS_PARTS})stale_copy = BatchImport.objects.get(id=batch_import.id)worker_progress = {- "parts": [*self.MIXED_PROGRESS_PARTS[:2], {"key": "day-3", "current_offset": 77, "total_size": None}]+ "parts": [*self.MIXED_PROGRESS_PARTS[:2], {"key": "day-3", "current_offset": 77, "total_size": None}],+ "events_emitted": 1000,}- BatchImport.objects.filter(id=batch_import.id).update(state=worker_progress)+ BatchImport.objects.filter(id=batch_import.id).update(state=worker_progress, pending_commit=pending_commit)stale_copy.resume_after_pause()batch_import.refresh_from_db()- self.assertEqual(batch_import.state, worker_progress)+ assert batch_import.state is not None+ self.assertEqual(+ batch_import.state["parts"],+ [+ self.MIXED_PROGRESS_PARTS[0],+ {"key": "day-2", "current_offset": expected_day_2_offset, "total_size": None},+ {"key": "day-3", "current_offset": 77, "total_size": None},+ ],+ )+ self.assertEqual(batch_import.state["events_emitted"], expected_events_emitted)+ self.assertEqual(batch_import.status_message, expected_message)self.assertEqual(batch_import.status, BatchImport.Status.RUNNING)self.assertIsNone(batch_import.lease_id)self.assertIsNone(batch_import.leased_until)@@ -178,6 +255,23 @@ class TestBatchImportModel(BaseTest):self.assertIsNone(batch_import.display_status_message)self.assertIsNone(batch_import.pending_commit)+ def test_resume_after_pause_redacts_the_rewound_part_key(self):+ url_key = "https://u:[email protected]/d.jsonl?X-Amz-Signature=abc"+ batch_import = self._paused_import({"parts": [{"key": url_key, "current_offset": 50, "total_size": None}]})+ BatchImport.objects.filter(id=batch_import.id).update(+ pending_commit={"key": url_key, "from_offset": 10, "to_offset": 50}+ )++ batch_import.resume_after_pause()++ batch_import.refresh_from_db()+ assert batch_import.state is not None+ self.assertEqual(batch_import.state["parts"][0], {"key": url_key, "current_offset": 10, "total_size": None})+ self.assertEqual(+ batch_import.status_message,+ "Resumed by admin; re-sending the interrupted chunk of part https://example.com/d.jsonl from offset 10",+ )+@parameterized.expand([(method, status)@@ -783,6 +877,27 @@ class TestBatchImportAdminChangeForm(BaseTest):any("FOR UPDATE" in q["sql"] and "posthog_batchimport" in q["sql"] for q in queries.captured_queries))+ def test_save_resume_rewinds_from_the_row_locked_at_save(self):+ batch_import = BatchImport.objects.create(+ team=self.team,+ created_by_id=self.user.id,+ import_config={"sink": {"type": "capture", "send_rate": 5000}},+ status=BatchImport.Status.PAUSED,+ state={"parts": [{"key": "day-1", "current_offset": 100, "total_size": None}]},+ )+ form = self._post_admin_form(batch_import, self._form_values(batch_import), {"status": "running"})+ BatchImport.objects.filter(id=batch_import.id).update(+ state=self.WORKER_STATE, pending_commit={"key": "day-1", "from_offset": 8000, "to_offset": 9000}+ )++ self.assertTrue(form.is_valid(), form.errors)+ self._save(form)++ batch_import.refresh_from_db()+ self.assertEqual(batch_import.state, {"parts": [{"key": "day-1", "current_offset": 8000, "total_size": None}]})+ self.assertIsNone(batch_import.pending_commit)+ self.assertEqual(batch_import.status, BatchImport.Status.RUNNING)+def test_save_rejects_a_page_without_initial_values(self):batch_import = BatchImport.objects.create(team=self.team,@@ -801,13 +916,13 @@ class TestBatchImportAdminChangeForm(BaseTest):@parameterized.expand([- ("rate_change_keeps_worker_status", "running", "paused", {"send_rate": "500"}, "paused", 500),- ("operator_pause", "running", "running", {"status": "paused"}, "paused", 5000),- ("operator_resume", "paused", "paused", {"status": "running"}, "running", 5000),+ ("rate_change_keeps_worker_status", "running", "paused", {"send_rate": "500"}, "paused", 500, 9000),+ ("operator_pause", "running", "running", {"status": "paused"}, "paused", 5000, 9000),+ ("operator_resume_rewinds_the_chunk", "paused", "paused", {"status": "running"}, "running", 5000, 8000),])def test_save_writes_only_operator_edits(- self, _name, shown_status, worker_status, edits, expected_status, expected_send_rate+ self, _name, shown_status, worker_status, edits, expected_status, expected_send_rate, expected_offset):batch_import = BatchImport.objects.create(team=self.team,@@ -833,9 +948,14 @@ class TestBatchImportAdminChangeForm(BaseTest):batch_import.refresh_from_db()self.assertEqual(batch_import.status, expected_status)self.assertEqual(batch_import.import_config["sink"]["send_rate"], expected_send_rate)- self.assertEqual(batch_import.state, self.WORKER_STATE)+ self.assertEqual(+ batch_import.state, {"parts": [{"key": "day-1", "current_offset": expected_offset, "total_size": None}]}+ )if expected_status == "running":- self.assertEqual(batch_import.status_message, "Resumed by admin")+ self.assertEqual(+ batch_import.status_message,+ "Resumed by admin; re-sending the interrupted chunk of part day-1 from offset 8000",+ )self.assertIsNone(batch_import.lease_id)self.assertIsNone(batch_import.leased_until)self.assertEqual(batch_import.backoff_attempt, 0)@@ -1334,19 +1454,28 @@ class TestBatchImportAPI(APIBaseTest):backoff_attempt=5,backoff_until=datetime.now(tz=UTC) + timedelta(hours=1),display_status_message="Job paused after a failed commit",- pending_commit={"key": "part-1", "from_offset": 0, "to_offset": 100},+ pending_commit={"key": "part-1", "from_offset": 0, "to_offset": 100, "events": 30},+ state={"parts": [{"key": "part-1", "current_offset": 100, "total_size": 500}], "events_emitted": 250},)response = self.client.post(f"/api/projects/{self.team.id}/managed_migrations/{batch_import.id}/resume")self.assertEqual(response.status_code, 200)batch_import.refresh_from_db()+ assert batch_import.state is not None+ self.assertEqual(+ batch_import.state,+ {"parts": [{"key": "part-1", "current_offset": 0, "total_size": 500}], "events_emitted": 220},+ )self.assertEqual(batch_import.status, BatchImport.Status.RUNNING)self.assertIsNone(batch_import.lease_id)self.assertIsNone(batch_import.leased_until)self.assertEqual(batch_import.backoff_attempt, 0)self.assertIsNone(batch_import.backoff_until)- self.assertEqual(batch_import.status_message, "Resumed by user")+ self.assertEqual(+ batch_import.status_message,+ "Resumed by user; re-sending the interrupted chunk of part part-1 from offset 0",+ )self.assertIsNone(batch_import.display_status_message)self.assertIsNone(batch_import.pending_commit)logged_fields = [diff --git a/products/managed_migrations/backend/models/batch_imports.py b/products/managed_migrations/backend/models/batch_imports.pyindex 77b95f2d0be6..77c15c2e0678 100644--- a/products/managed_migrations/backend/models/batch_imports.py+++ b/products/managed_migrations/backend/models/batch_imports.py@@ -121,9 +121,10 @@ class BatchImport(ModelActivityMixin, UUIDTModel):Flips the job to running and clears the worker lease and backoff - pausingkeeps the worker's lease, so without clearing it no worker can re-claim the- row for up to 30 minutes. Part offsets are untouched: use this when the- source bytes behind the saved offset are unchanged (transient pauses, or a- data fix that preserved byte offsets). When the bytes changed underneath the+ row for up to 30 minutes. Part offsets stay as saved, except that a chunk a+ worker left mid-commit is sent again: use this when the source bytes behind+ the saved offset are unchanged (transient pauses, or a data fix that preserved+ byte offsets). When the bytes changed underneath theoffset, use `resume_with_inflight_part_reset` instead. Raises ValueError ifthe job is not paused."""@@ -137,7 +138,7 @@ class BatchImport(ModelActivityMixin, UUIDTModel):if row.status != BatchImport.Status.PAUSED:raise ValueError(f"Only paused imports can be resumed (status: {row.status})")row.raise_if_commit_in_flight()- row.save(update_fields=row.reset_for_resume(status_message))+ row.save(update_fields=row.rewind_and_reset_for_resume(status_message))self.refresh_from_db()def resume_with_inflight_part_reset(self) -> str:@@ -152,15 +153,17 @@ class BatchImport(ModelActivityMixin, UUIDTModel):(Mixpanel $insert_id, an Amplitude or captured uuid). Other events, includingthe identify events the worker generates for Amplitude, get a new UUID.- Resets the first unfinished part to offset 0, then resumes as- `resume_after_pause` does. Returns the key of the reset part. Raises- ValueError if the job is not paused or has no unfinished part.+ Rewinds a chunk a worker left mid-commit, so its part counts as unfinished even+ when that chunk was the part's last, then resets the first unfinished part to+ offset 0 and resumes. Returns the key of the reset part. Raises ValueError if+ the job is not paused or has no unfinished part."""with transaction.atomic():row = BatchImport.objects.select_for_update().get(pk=self.pk)if row.status != BatchImport.Status.PAUSED:raise ValueError(f"Only paused imports can be reset and resumed (status: {row.status})")row.raise_if_commit_in_flight()+ rewind_note = row.rewind_interrupted_commit()_done, _total, inflight = row.parts_progress()if inflight is None:@@ -174,12 +177,10 @@ class BatchImport(ModelActivityMixin, UUIDTModel):# Redacted: url_list part keys are full source URLs whose query string /# userinfo can carry credentials, and status_message is broadly visible.redacted_key = str(redact_part_key(inflight.get("key")))- row.save(- update_fields=[- *row.reset_for_resume(f"Resumed by admin with part {redacted_key} reset to offset 0"),- "state",- ]- )+ status_message = f"Resumed by admin with part {redacted_key} reset to offset 0"+ if rewind_note is not None:+ status_message = f"{status_message}; {rewind_note}"+ row.save(update_fields=[*row.reset_for_resume(status_message), "state"])self.refresh_from_db()return redacted_key@@ -194,11 +195,48 @@ class BatchImport(ModelActivityMixin, UUIDTModel):if self.commit_in_flight():raise CommitInFlightError()+ def rewind_interrupted_commit(self) -> str | None:+ """Move the part that `pending_commit` names back to the start of its chunk, so the resumed+ worker sends the chunk again instead of skipping a chunk that may not have reached the sink.++ Returns a note for the status message, or None when there is no marker. The part moves only+ from either end of the chunk, because any other offset means the marker does not describe+ the saved progress. A part that moves back gives up the chunk's events in `events_emitted`,+ because the worker counts them again when it re-sends the chunk."""+ if self.pending_commit is None:+ return None+ key = self.pending_commit.get("key")+ from_offset = self.pending_commit.get("from_offset")+ redacted_key = redact_part_key(key)+ state = self.state or {}+ parts = state.get("parts") or []+ part = next((p for p in parts if p.get("key") == key), None)+ if (+ part is None+ or not isinstance(from_offset, int)+ or part.get("current_offset") not in (from_offset, self.pending_commit.get("to_offset"))+ ):+ return f"kept the saved offset of part {redacted_key}, which does not match the interrupted chunk"+ events = self.pending_commit.get("events")+ emitted = state.get("events_emitted")+ if part["current_offset"] != from_offset and isinstance(events, int) and isinstance(emitted, int):+ state["events_emitted"] = max(0, emitted - events)+ part["current_offset"] = from_offset+ return f"re-sending the interrupted chunk of part {redacted_key} from offset {from_offset}"++ def rewind_and_reset_for_resume(self, status_message: str) -> list[str]:+ """Rewind an interrupted chunk, then reset the row for a resume. Call it on a row locked with+ `select_for_update`, because it saves `state` as read from that row."""+ note = self.rewind_interrupted_commit()+ if note is None:+ return self.reset_for_resume(status_message)+ return [*self.reset_for_resume(f"{status_message}; {note}"), "state"]+def reset_for_resume(self, status_message: str) -> list[str]:"""Set the fields every resume path resets, and return their names for `save(update_fields=...)`.Clears the lease so a worker can claim the row at once, and `pending_commit` because a resume- accepts the saved offsets."""+ starts from the saved offsets; `rewind_and_reset_for_resume` moves them back first."""self.status = BatchImport.Status.RUNNINGself.status_message = status_messageself.display_status_message = None