Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(managed-migrations): re-send an interrupted chunk on resume (#125952)

Eli Reisman038ae359b564
diff --git a/products/managed_migrations/backend/admin/batch_imports.py b/products/managed_migrations/backend/admin/batch_imports.py
index 824b2f291b57..745a48227388 100644
--- a/products/managed_migrations/backend/admin/batch_imports.py
+++ b/products/managed_migrations/backend/admin/batch_imports.py
@@ -56,7 +56,8 @@ class BatchImportAdminForm(forms.ModelForm):
if self.resumes():
# Django admin runs the whole POST in one transaction, so this lock holds until save_model
# writes the resume. A refusal here re-renders the form with the error and saves nothing.
- self.raise_if_resume_unsafe(BatchImport.objects.select_for_update().get(pk=self.instance.pk))
+ self.locked_row = BatchImport.objects.select_for_update().get(pk=self.instance.pk)
+ self.raise_if_resume_unsafe(self.locked_row)
return cleaned_data
def resumes(self) -> bool:
@@ -161,7 +162,10 @@ class BatchImportAdmin(admin.ModelAdmin):
"Changing the status to running, or using a resume action, clears the lease, backoff "
"and pending commit so a worker can claim the job at once. A pending commit on "
"a job no worker holds means a worker stopped mid-commit: the chunk it names "
- "may or may not have reached the sink. Choosing a resume action: 'Resume (keep progress)' "
+ "may or may not have reached the sink. When the saved offset still matches that chunk, "
+ "a resume moves the part back to the start of the chunk, so nothing is lost. A chunk "
+ "that had arrived is sent twice. "
+ "Choosing a resume action: 'Resume (keep progress)' "
"continues from the saved byte offset - right for transient pauses and "
"for source data that has not changed. 'Resume + re-import in-flight part' "
"resets the current part to offset 0 first - right when the source bytes "
@@ -203,7 +207,13 @@ class BatchImportAdmin(admin.ModelAdmin):
update_fields.append("import_config")
if form.resumes():
message = obj.status_message if "status_message" in form.changed_data else "Resumed by admin"
- update_fields.extend(obj.reset_for_resume(message))
+ # Take the reset fields from the row locked in clean(): obj was loaded before the lock, so its
+ # state and pending_commit can miss a worker write.
+ locked_row = form.locked_row
+ reset_fields = locked_row.rewind_and_reset_for_resume(message)
+ for name in reset_fields:
+ setattr(obj, name, getattr(locked_row, name))
+ update_fields.extend(reset_fields)
if update_fields:
obj.save(update_fields=list(dict.fromkeys([*update_fields, "updated_at"])))
@@ -237,8 +247,8 @@ class BatchImportAdmin(admin.ModelAdmin):
permissions=["change"],
)
def resume(self, request, queryset):
- """Resume paused jobs from their saved progress: for transient pauses and
- unchanged source data. See the 'Worker state' section on the detail page
+ """Resume paused jobs from their saved progress, re-sending a chunk a worker left
+ mid-commit: for transient pauses and unchanged source data. See the 'Worker state' section on the detail page
for how this differs from the reset variant."""
for batch_import in queryset:
try:
diff --git a/products/managed_migrations/backend/api/test/test_batch_imports.py b/products/managed_migrations/backend/api/test/test_batch_imports.py
index 6d0764be9a47..d325e55526c7 100644
--- a/products/managed_migrations/backend/api/test/test_batch_imports.py
+++ b/products/managed_migrations/backend/api/test/test_batch_imports.py
@@ -84,6 +84,7 @@ class TestBatchImportModel(BaseTest):
{"key": "day-2", "current_offset": 0, "total_size": None},
{"key": "day-3", "current_offset": 0, "total_size": None},
],
+ "re-sending the interrupted chunk of part day-2 from offset 8999000",
),
(
"inflight_part_with_known_total",
@@ -96,6 +97,7 @@ class TestBatchImportModel(BaseTest):
{"key": "day-1", "current_offset": 0, "total_size": None},
{"key": "day-2", "current_offset": 0, "total_size": None},
],
+ "kept the saved offset of part day-2, which does not match the interrupted chunk",
),
(
"first_part_inflight",
@@ -108,12 +110,29 @@ class TestBatchImportModel(BaseTest):
{"key": "day-1", "current_offset": 0, "total_size": None},
{"key": "day-2", "current_offset": 0, "total_size": None},
],
+ "kept the saved offset of part day-2, which does not match the interrupted chunk",
+ ),
+ (
+ "interrupted_last_chunk_of_a_finished_part",
+ [
+ {"key": "day-1", "current_offset": 1000, "total_size": 1000},
+ {"key": "day-2", "current_offset": 8999445, "total_size": 8999445},
+ {"key": "day-3", "current_offset": 0, "total_size": None},
+ ],
+ "day-2",
+ [
+ {"key": "day-1", "current_offset": 1000, "total_size": 1000},
+ {"key": "day-2", "current_offset": 0, "total_size": None},
+ {"key": "day-3", "current_offset": 0, "total_size": None},
+ ],
+ "re-sending the interrupted chunk of part day-2 from offset 8999000",
),
(
"job_paused_before_any_read",
[{"key": "day-1", "current_offset": 0, "total_size": None}],
"day-1",
[{"key": "day-1", "current_offset": 0, "total_size": None}],
+ "kept the saved offset of part day-2, which does not match the interrupted chunk",
),
(
# url_list part keys are credential-bearing URLs: the returned key (and the
@@ -135,11 +154,12 @@ class TestBatchImportModel(BaseTest):
"total_size": None,
}
],
+ "kept the saved offset of part day-2, which does not match the interrupted chunk",
),
]
)
def test_resume_with_inflight_part_reset_resets_first_unfinished_part(
- self, _name, initial_parts, expected_reset_key, expected_parts
+ self, _name, initial_parts, expected_reset_key, expected_parts, expected_rewind_note
):
batch_import = self._paused_import({"parts": initial_parts})
@@ -149,6 +169,10 @@ class TestBatchImportModel(BaseTest):
batch_import.refresh_from_db()
assert batch_import.state is not None
self.assertNotIn("X-Amz-Signature", batch_import.status_message or "")
+ self.assertEqual(
+ batch_import.status_message,
+ f"Resumed by admin with part {expected_reset_key} reset to offset 0; {expected_rewind_note}",
+ )
self.assertEqual(batch_import.state["parts"], expected_parts)
self.assertEqual(batch_import.status, BatchImport.Status.RUNNING)
self.assertIsNone(batch_import.lease_id)
@@ -158,18 +182,71 @@ class TestBatchImportModel(BaseTest):
self.assertIsNone(batch_import.display_status_message)
self.assertIsNone(batch_import.pending_commit)
- def test_resume_after_pause_keeps_progress_and_clears_lease(self):
+ @parameterized.expand(
+ [
+ (
+ "marker_matches_part",
+ {"key": "day-2", "from_offset": 8999000, "to_offset": 8999445, "events": 40},
+ 8999000,
+ 960,
+ "Resumed by admin; re-sending the interrupted chunk of part day-2 from offset 8999000",
+ ),
+ (
+ "marker_from_a_worker_that_did_not_count_events",
+ {"key": "day-2", "from_offset": 8999000, "to_offset": 8999445},
+ 8999000,
+ 1000,
+ "Resumed by admin; re-sending the interrupted chunk of part day-2 from offset 8999000",
+ ),
+ (
+ "part_already_at_chunk_start",
+ {"key": "day-2", "from_offset": 8999445, "to_offset": 9000000, "interrupted": 3, "events": 40},
+ 8999445,
+ 1000,
+ "Resumed by admin; re-sending the interrupted chunk of part day-2 from offset 8999445",
+ ),
+ ("no_marker", None, 8999445, 1000, "Resumed by admin"),
+ (
+ "marker_offset_mismatch",
+ {"key": "day-2", "from_offset": 100, "to_offset": 200, "events": 40},
+ 8999445,
+ 1000,
+ "Resumed by admin; kept the saved offset of part day-2, which does not match the interrupted chunk",
+ ),
+ (
+ "marker_part_missing",
+ {"key": "day-9", "from_offset": 8999000, "to_offset": 8999445, "events": 40},
+ 8999445,
+ 1000,
+ "Resumed by admin; kept the saved offset of part day-9, which does not match the interrupted chunk",
+ ),
+ ]
+ )
+ def test_resume_after_pause_rewinds_an_interrupted_chunk_and_clears_lease(
+ self, _name, pending_commit, expected_day_2_offset, expected_events_emitted, expected_message
+ ):
batch_import = self._paused_import({"parts": self.MIXED_PROGRESS_PARTS})
stale_copy = BatchImport.objects.get(id=batch_import.id)
worker_progress = {
- "parts": [*self.MIXED_PROGRESS_PARTS[:2], {"key": "day-3", "current_offset": 77, "total_size": None}]
+ "parts": [*self.MIXED_PROGRESS_PARTS[:2], {"key": "day-3", "current_offset": 77, "total_size": None}],
+ "events_emitted": 1000,
}
- BatchImport.objects.filter(id=batch_import.id).update(state=worker_progress)
+ BatchImport.objects.filter(id=batch_import.id).update(state=worker_progress, pending_commit=pending_commit)
stale_copy.resume_after_pause()
batch_import.refresh_from_db()
- self.assertEqual(batch_import.state, worker_progress)
+ assert batch_import.state is not None
+ self.assertEqual(
+ batch_import.state["parts"],
+ [
+ self.MIXED_PROGRESS_PARTS[0],
+ {"key": "day-2", "current_offset": expected_day_2_offset, "total_size": None},
+ {"key": "day-3", "current_offset": 77, "total_size": None},
+ ],
+ )
+ self.assertEqual(batch_import.state["events_emitted"], expected_events_emitted)
+ self.assertEqual(batch_import.status_message, expected_message)
self.assertEqual(batch_import.status, BatchImport.Status.RUNNING)
self.assertIsNone(batch_import.lease_id)
self.assertIsNone(batch_import.leased_until)
@@ -178,6 +255,23 @@ class TestBatchImportModel(BaseTest):
self.assertIsNone(batch_import.display_status_message)
self.assertIsNone(batch_import.pending_commit)
+ def test_resume_after_pause_redacts_the_rewound_part_key(self):
+ url_key = "https://u:[email protected]/d.jsonl?X-Amz-Signature=abc"
+ batch_import = self._paused_import({"parts": [{"key": url_key, "current_offset": 50, "total_size": None}]})
+ BatchImport.objects.filter(id=batch_import.id).update(
+ pending_commit={"key": url_key, "from_offset": 10, "to_offset": 50}
+ )
+
+ batch_import.resume_after_pause()
+
+ batch_import.refresh_from_db()
+ assert batch_import.state is not None
+ self.assertEqual(batch_import.state["parts"][0], {"key": url_key, "current_offset": 10, "total_size": None})
+ self.assertEqual(
+ batch_import.status_message,
+ "Resumed by admin; re-sending the interrupted chunk of part https://example.com/d.jsonl from offset 10",
+ )
+
@parameterized.expand(
[
(method, status)
@@ -783,6 +877,27 @@ class TestBatchImportAdminChangeForm(BaseTest):
any("FOR UPDATE" in q["sql"] and "posthog_batchimport" in q["sql"] for q in queries.captured_queries)
)
+ def test_save_resume_rewinds_from_the_row_locked_at_save(self):
+ batch_import = BatchImport.objects.create(
+ team=self.team,
+ created_by_id=self.user.id,
+ import_config={"sink": {"type": "capture", "send_rate": 5000}},
+ status=BatchImport.Status.PAUSED,
+ state={"parts": [{"key": "day-1", "current_offset": 100, "total_size": None}]},
+ )
+ form = self._post_admin_form(batch_import, self._form_values(batch_import), {"status": "running"})
+ BatchImport.objects.filter(id=batch_import.id).update(
+ state=self.WORKER_STATE, pending_commit={"key": "day-1", "from_offset": 8000, "to_offset": 9000}
+ )
+
+ self.assertTrue(form.is_valid(), form.errors)
+ self._save(form)
+
+ batch_import.refresh_from_db()
+ self.assertEqual(batch_import.state, {"parts": [{"key": "day-1", "current_offset": 8000, "total_size": None}]})
+ self.assertIsNone(batch_import.pending_commit)
+ self.assertEqual(batch_import.status, BatchImport.Status.RUNNING)
+
def test_save_rejects_a_page_without_initial_values(self):
batch_import = BatchImport.objects.create(
team=self.team,
@@ -801,13 +916,13 @@ class TestBatchImportAdminChangeForm(BaseTest):
@parameterized.expand(
[
- ("rate_change_keeps_worker_status", "running", "paused", {"send_rate": "500"}, "paused", 500),
- ("operator_pause", "running", "running", {"status": "paused"}, "paused", 5000),
- ("operator_resume", "paused", "paused", {"status": "running"}, "running", 5000),
+ ("rate_change_keeps_worker_status", "running", "paused", {"send_rate": "500"}, "paused", 500, 9000),
+ ("operator_pause", "running", "running", {"status": "paused"}, "paused", 5000, 9000),
+ ("operator_resume_rewinds_the_chunk", "paused", "paused", {"status": "running"}, "running", 5000, 8000),
]
)
def test_save_writes_only_operator_edits(
- self, _name, shown_status, worker_status, edits, expected_status, expected_send_rate
+ self, _name, shown_status, worker_status, edits, expected_status, expected_send_rate, expected_offset
):
batch_import = BatchImport.objects.create(
team=self.team,
@@ -833,9 +948,14 @@ class TestBatchImportAdminChangeForm(BaseTest):
batch_import.refresh_from_db()
self.assertEqual(batch_import.status, expected_status)
self.assertEqual(batch_import.import_config["sink"]["send_rate"], expected_send_rate)
- self.assertEqual(batch_import.state, self.WORKER_STATE)
+ self.assertEqual(
+ batch_import.state, {"parts": [{"key": "day-1", "current_offset": expected_offset, "total_size": None}]}
+ )
if expected_status == "running":
- self.assertEqual(batch_import.status_message, "Resumed by admin")
+ self.assertEqual(
+ batch_import.status_message,
+ "Resumed by admin; re-sending the interrupted chunk of part day-1 from offset 8000",
+ )
self.assertIsNone(batch_import.lease_id)
self.assertIsNone(batch_import.leased_until)
self.assertEqual(batch_import.backoff_attempt, 0)
@@ -1334,19 +1454,28 @@ class TestBatchImportAPI(APIBaseTest):
backoff_attempt=5,
backoff_until=datetime.now(tz=UTC) + timedelta(hours=1),
display_status_message="Job paused after a failed commit",
- pending_commit={"key": "part-1", "from_offset": 0, "to_offset": 100},
+ pending_commit={"key": "part-1", "from_offset": 0, "to_offset": 100, "events": 30},
+ state={"parts": [{"key": "part-1", "current_offset": 100, "total_size": 500}], "events_emitted": 250},
)
response = self.client.post(f"/api/projects/{self.team.id}/managed_migrations/{batch_import.id}/resume")
self.assertEqual(response.status_code, 200)
batch_import.refresh_from_db()
+ assert batch_import.state is not None
+ self.assertEqual(
+ batch_import.state,
+ {"parts": [{"key": "part-1", "current_offset": 0, "total_size": 500}], "events_emitted": 220},
+ )
self.assertEqual(batch_import.status, BatchImport.Status.RUNNING)
self.assertIsNone(batch_import.lease_id)
self.assertIsNone(batch_import.leased_until)
self.assertEqual(batch_import.backoff_attempt, 0)
self.assertIsNone(batch_import.backoff_until)
- self.assertEqual(batch_import.status_message, "Resumed by user")
+ self.assertEqual(
+ batch_import.status_message,
+ "Resumed by user; re-sending the interrupted chunk of part part-1 from offset 0",
+ )
self.assertIsNone(batch_import.display_status_message)
self.assertIsNone(batch_import.pending_commit)
logged_fields = [
diff --git a/products/managed_migrations/backend/models/batch_imports.py b/products/managed_migrations/backend/models/batch_imports.py
index 77b95f2d0be6..77c15c2e0678 100644
--- a/products/managed_migrations/backend/models/batch_imports.py
+++ b/products/managed_migrations/backend/models/batch_imports.py
@@ -121,9 +121,10 @@ class BatchImport(ModelActivityMixin, UUIDTModel):
Flips the job to running and clears the worker lease and backoff - pausing
keeps the worker's lease, so without clearing it no worker can re-claim the
- row for up to 30 minutes. Part offsets are untouched: use this when the
- source bytes behind the saved offset are unchanged (transient pauses, or a
- data fix that preserved byte offsets). When the bytes changed underneath the
+ row for up to 30 minutes. Part offsets stay as saved, except that a chunk a
+ worker left mid-commit is sent again: use this when the source bytes behind
+ the saved offset are unchanged (transient pauses, or a data fix that preserved
+ byte offsets). When the bytes changed underneath the
offset, use `resume_with_inflight_part_reset` instead. Raises ValueError if
the job is not paused.
"""
@@ -137,7 +138,7 @@ class BatchImport(ModelActivityMixin, UUIDTModel):
if row.status != BatchImport.Status.PAUSED:
raise ValueError(f"Only paused imports can be resumed (status: {row.status})")
row.raise_if_commit_in_flight()
- row.save(update_fields=row.reset_for_resume(status_message))
+ row.save(update_fields=row.rewind_and_reset_for_resume(status_message))
self.refresh_from_db()
def resume_with_inflight_part_reset(self) -> str:
@@ -152,15 +153,17 @@ class BatchImport(ModelActivityMixin, UUIDTModel):
(Mixpanel $insert_id, an Amplitude or captured uuid). Other events, including
the identify events the worker generates for Amplitude, get a new UUID.
- Resets the first unfinished part to offset 0, then resumes as
- `resume_after_pause` does. Returns the key of the reset part. Raises
- ValueError if the job is not paused or has no unfinished part.
+ Rewinds a chunk a worker left mid-commit, so its part counts as unfinished even
+ when that chunk was the part's last, then resets the first unfinished part to
+ offset 0 and resumes. Returns the key of the reset part. Raises ValueError if
+ the job is not paused or has no unfinished part.
"""
with transaction.atomic():
row = BatchImport.objects.select_for_update().get(pk=self.pk)
if row.status != BatchImport.Status.PAUSED:
raise ValueError(f"Only paused imports can be reset and resumed (status: {row.status})")
row.raise_if_commit_in_flight()
+ rewind_note = row.rewind_interrupted_commit()
_done, _total, inflight = row.parts_progress()
if inflight is None:
@@ -174,12 +177,10 @@ class BatchImport(ModelActivityMixin, UUIDTModel):
# Redacted: url_list part keys are full source URLs whose query string /
# userinfo can carry credentials, and status_message is broadly visible.
redacted_key = str(redact_part_key(inflight.get("key")))
- row.save(
- update_fields=[
- *row.reset_for_resume(f"Resumed by admin with part {redacted_key} reset to offset 0"),
- "state",
- ]
- )
+ status_message = f"Resumed by admin with part {redacted_key} reset to offset 0"
+ if rewind_note is not None:
+ status_message = f"{status_message}; {rewind_note}"
+ row.save(update_fields=[*row.reset_for_resume(status_message), "state"])
self.refresh_from_db()
return redacted_key
@@ -194,11 +195,48 @@ class BatchImport(ModelActivityMixin, UUIDTModel):
if self.commit_in_flight():
raise CommitInFlightError()
+ def rewind_interrupted_commit(self) -> str | None:
+ """Move the part that `pending_commit` names back to the start of its chunk, so the resumed
+ worker sends the chunk again instead of skipping a chunk that may not have reached the sink.
+
+ Returns a note for the status message, or None when there is no marker. The part moves only
+ from either end of the chunk, because any other offset means the marker does not describe
+ the saved progress. A part that moves back gives up the chunk's events in `events_emitted`,
+ because the worker counts them again when it re-sends the chunk."""
+ if self.pending_commit is None:
+ return None
+ key = self.pending_commit.get("key")
+ from_offset = self.pending_commit.get("from_offset")
+ redacted_key = redact_part_key(key)
+ state = self.state or {}
+ parts = state.get("parts") or []
+ part = next((p for p in parts if p.get("key") == key), None)
+ if (
+ part is None
+ or not isinstance(from_offset, int)
+ or part.get("current_offset") not in (from_offset, self.pending_commit.get("to_offset"))
+ ):
+ return f"kept the saved offset of part {redacted_key}, which does not match the interrupted chunk"
+ events = self.pending_commit.get("events")
+ emitted = state.get("events_emitted")
+ if part["current_offset"] != from_offset and isinstance(events, int) and isinstance(emitted, int):
+ state["events_emitted"] = max(0, emitted - events)
+ part["current_offset"] = from_offset
+ return f"re-sending the interrupted chunk of part {redacted_key} from offset {from_offset}"
+
+ def rewind_and_reset_for_resume(self, status_message: str) -> list[str]:
+ """Rewind an interrupted chunk, then reset the row for a resume. Call it on a row locked with
+ `select_for_update`, because it saves `state` as read from that row."""
+ note = self.rewind_interrupted_commit()
+ if note is None:
+ return self.reset_for_resume(status_message)
+ return [*self.reset_for_resume(f"{status_message}; {note}"), "state"]
+
def reset_for_resume(self, status_message: str) -> list[str]:
"""Set the fields every resume path resets, and return their names for `save(update_fields=...)`.
Clears the lease so a worker can claim the row at once, and `pending_commit` because a resume
- accepts the saved offsets."""
+ starts from the saved offsets; `rewind_and_reset_for_resume` moves them back first."""
self.status = BatchImport.Status.RUNNING
self.status_message = status_message
self.display_status_message = None