:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(managed-migrations): stop admin saves overwriting worker state (#118356)
Eli Reisman20e721143090
diff --git a/posthog/models/activity_logging/activity_log.py b/posthog/models/activity_logging/activity_log.pyindex 6da852b85108..4f4a556f5677 100644--- a/posthog/models/activity_logging/activity_log.py+++ b/posthog/models/activity_logging/activity_log.py@@ -1007,6 +1007,7 @@ field_exclusions: dict[AuditableScope, list[str]] = {"lease_id","backoff_attempt","backoff_until",+ "pending_commit",],"Integration": ["sensitive_config",diff --git a/products/managed_migrations/backend/admin/batch_imports.py b/products/managed_migrations/backend/admin/batch_imports.pyindex c8081f23a728..630c001db5f4 100644--- a/products/managed_migrations/backend/admin/batch_imports.py+++ b/products/managed_migrations/backend/admin/batch_imports.py@@ -1,18 +1,24 @@+from typing import cast+from django import formsfrom django.contrib import admin, messagesfrom django.utils.html import format_html-from products.managed_migrations.backend.models.batch_imports import DEFAULT_SEND_RATE, BatchImport+from products.managed_migrations.backend.models.batch_imports import DEFAULT_SEND_RATE, BatchImport, CommitInFlightError++STATUS_CHANGED_MESSAGE = "This import changed since the page loaded. Reload it and try again."+SINK_TYPE_CHOICES: list[tuple[str, str]] = [("capture", "Capture"), ("kafka", "Kafka")]class BatchImportAdminForm(forms.ModelForm):sink_type = forms.ChoiceField(- choices=[("capture", "Capture"), ("kafka", "Kafka")],+ choices=SINK_TYPE_CHOICES,required=False,help_text="Capture sends events via HTTP; Kafka writes directly to Kafka.",)send_rate = forms.IntegerField(required=False,+ min_value=1,help_text=f"Events per second to send (e.g., {DEFAULT_SEND_RATE}). Leave empty to keep current value.",widget=forms.NumberInput(attrs={"placeholder": f"e.g., {DEFAULT_SEND_RATE}"}),)@@ -23,30 +29,75 @@ class BatchImportAdminForm(forms.ModelForm):def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)+ # Compare submitted values with what the page showed, not the row at save time, so a+ # worker write after page load is not counted as an operator edit.+ for field in self.fields.values():+ field.show_hidden_initial = Trueif self.instance and self.instance.import_config:sink = self.instance.import_config.get("sink", {})- if sink.get("type"):- self.fields["sink_type"].initial = sink["type"]+ sink_type = sink.get("type")+ if sink_type:+ self.fields["sink_type"].initial = sink_type+ # Without its own choice, a trial or noop sink would submit "capture" and be+ # rewritten on every save.+ if sink_type not in dict(SINK_TYPE_CHOICES):+ cast(forms.ChoiceField, self.fields["sink_type"]).choices = [+ *SINK_TYPE_CHOICES,+ (sink_type, f"{sink_type} (current)"),+ ]if sink.get("send_rate"):self.fields["send_rate"].initial = sink["send_rate"]+ def clean(self):+ cleaned_data = super().clean()+ # A page rendered without hidden initial values would make every field read as changed.+ if self.instance.pk and any(self.add_initial_prefix(name) not in self.data for name in self.fields):+ raise forms.ValidationError("This page is out of date. Reload it and make your change again.")+ if self.resumes():+ # Django admin runs the whole POST in one transaction, so this lock holds until save_model+ # writes the resume. A refusal here re-renders the form with the error and saves nothing.+ self.raise_if_resume_unsafe(BatchImport.objects.select_for_update().get(pk=self.instance.pk))+ return cleaned_data++ def resumes(self) -> bool:+ return (+ bool(self.instance.pk)+ and "status" in self.changed_data+ and self.cleaned_data.get("status") == BatchImport.Status.RUNNING+ )++ def raise_if_resume_unsafe(self, current: BatchImport) -> None:+ # A resume clears the lease, so it must act on the status the operator saw, not one that a+ # worker or a customer set since.+ if current.status != self.data.get(self.add_initial_prefix("status")):+ raise forms.ValidationError(STATUS_CHANGED_MESSAGE)+ if current.commit_in_flight():+ raise forms.ValidationError(str(CommitInFlightError()))+def save(self, commit=True):instance = super().save(commit=False)sink_type = self.cleaned_data.get("sink_type")send_rate = self.cleaned_data.get("send_rate")- if sink_type:+ if sink_type and any(name in self.changed_data for name in ("sink_type", "send_rate")):if not instance.import_config:instance.import_config = {}existing_sink = instance.import_config.get("sink", {})- sink = {- "type": sink_type,- "send_rate": send_rate if send_rate is not None else existing_sink.get("send_rate", DEFAULT_SEND_RATE),- }- if sink_type == "kafka":- sink["topic"] = existing_sink.get("topic", "historical")- sink["transaction_timeout_seconds"] = existing_sink.get("transaction_timeout_seconds", 60)+ if sink_type == existing_sink.get("type"):+ sink = {**existing_sink}+ if send_rate is not None:+ sink["send_rate"] = send_rate+ else:+ sink = {+ "type": sink_type,+ "send_rate": send_rate+ if send_rate is not None+ else existing_sink.get("send_rate", DEFAULT_SEND_RATE),+ }+ if sink_type == "kafka":+ sink["topic"] = existing_sink.get("topic", "historical")+ sink["transaction_timeout_seconds"] = existing_sink.get("transaction_timeout_seconds", 60)instance.import_config["sink"] = sink@@ -55,6 +106,10 @@ class BatchImportAdminForm(forms.ModelForm):return instance+EDITABLE_MODEL_FIELDS = ("team", "created_by_id", "status", "status_message")+SINK_FORM_FIELDS = ("sink_type", "send_rate")++@admin.register(BatchImport)class BatchImportAdmin(admin.ModelAdmin):form = BatchImportAdminForm@@ -83,6 +138,7 @@ class BatchImportAdmin(admin.ModelAdmin):"leased_until","backoff_attempt","backoff_until",+ "pending_commit","get_worker_progress",)autocomplete_fields = ("team",)@@ -97,12 +153,14 @@ class BatchImportAdmin(admin.ModelAdmin):"leased_until","backoff_attempt","backoff_until",+ "pending_commit",),"description": (- "A paused job keeps its worker lease; both resume actions clear it "- "(a bare status change leaves the row unclaimable until the lease expires, "- "up to 30 minutes). A running job with a future leased_until is actively "- "claimed by a worker. Choosing a resume action: 'Resume (keep progress)' "+ "A running job with a future leased_until is actively claimed by a worker. "+ "Changing the status to running, or using a resume action, clears the lease, backoff "+ "and pending commit so a worker can claim the job at once. A pending commit on "+ "a job no worker holds means a worker stopped mid-commit: the chunk it names "+ "may or may not have reached the sink. Choosing a resume action: 'Resume (keep progress)' ""continues from the saved byte offset - right for transient pauses and ""for source data that has not changed. 'Resume + re-import in-flight part' ""resets the current part to offset 0 first - right when the source bytes "@@ -131,6 +189,22 @@ class BatchImportAdmin(admin.ModelAdmin):("Metadata", {"fields": ("id", "created_at", "updated_at", "state")}),)+ def save_model(self, request, obj, form, change):+ if not change:+ super().save_model(request, obj, form, change)+ return++ # A full-row save would overwrite the progress, lease and backoff that a running worker+ # saved after the page loaded.+ update_fields = [name for name in form.changed_data if name in EDITABLE_MODEL_FIELDS]+ if any(name in form.changed_data for name in SINK_FORM_FIELDS):+ update_fields.append("import_config")+ if form.resumes():+ message = obj.status_message if "status_message" in form.changed_data else "Resumed by admin"+ update_fields.extend(obj.reset_for_resume(message))+ if update_fields:+ obj.save(update_fields=list(dict.fromkeys([*update_fields, "updated_at"])))+@admin.display(description="Progress")def get_progress(self, obj):"""Parts done / total, from the worker-owned state JSON."""diff --git a/products/managed_migrations/backend/api/batch_imports.py b/products/managed_migrations/backend/api/batch_imports.pyindex 1ebf532ea572..7ae5407ce56f 100644--- a/products/managed_migrations/backend/api/batch_imports.py+++ b/products/managed_migrations/backend/api/batch_imports.py@@ -32,6 +32,7 @@ from products.managed_migrations.backend.models.batch_imports import (DEFAULT_SEND_RATE,BatchImport,BatchImportConfigBuilder,+ CommitInFlightError,ContentType,DateRangeExportSource,get_aws_external_id,@@ -892,27 +893,13 @@ class BatchImportViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):"""Resume a paused batch import."""batch_import = self.get_object()- if batch_import.status != BatchImport.Status.PAUSED:+ try:+ batch_import.resume_from_pause("Resumed by user")+ except CommitInFlightError as e:+ return Response({"error": str(e)}, status=status.HTTP_409_CONFLICT)+ except ValueError:return Response({"error": "Only paused imports can be resumed"}, status=status.HTTP_400_BAD_REQUEST)- batch_import.status = BatchImport.Status.RUNNING- batch_import.status_message = "Resumed by user"- batch_import.lease_id = None- batch_import.leased_until = None- batch_import.backoff_attempt = 0- batch_import.backoff_until = None- batch_import.save(- update_fields=[- "status",- "status_message",- "lease_id",- "leased_until",- "backoff_attempt",- "backoff_until",- "updated_at",- ]- )-return Response({"status": "resumed"})@extend_schema(diff --git a/products/managed_migrations/backend/api/test/test_batch_imports.py b/products/managed_migrations/backend/api/test/test_batch_imports.pyindex a6359106db0c..ab6536eee363 100644--- a/products/managed_migrations/backend/api/test/test_batch_imports.py+++ b/products/managed_migrations/backend/api/test/test_batch_imports.py@@ -3,15 +3,19 @@ from datetime import UTC, datetime, timedeltafrom posthog.test.base import APIBaseTest, BaseTestfrom unittest.mock import patch+from django.contrib import adminfrom django.contrib.admin.sites import AdminSitefrom django.contrib.messages import get_messagesfrom django.contrib.messages.middleware import MessageMiddlewarefrom django.contrib.sessions.middleware import SessionMiddleware+from django.db import connectionfrom django.http import HttpResponsefrom django.test import RequestFactory, SimpleTestCase, override_settings+from django.test.utils import CaptureQueriesContextfrom parameterized import parameterized+from posthog.models.activity_logging.activity_log import ActivityLogfrom posthog.security.url_validation import UNREACHABLE_HOST_MESSAGEfrom products.managed_migrations.backend import trial_storage@@ -21,6 +25,7 @@ from products.managed_migrations.backend.models.batch_imports import (DEFAULT_SEND_RATE,BatchImport,BatchImportConfigBuilder,+ CommitInFlightError,ContentType,get_aws_external_id,)@@ -55,9 +60,10 @@ class TestBatchImportModel(BaseTest):status_message="parse error",display_status_message="Invalid JSON syntax",lease_id="worker-lease",- leased_until=datetime.now(UTC) + timedelta(minutes=25),+ leased_until=datetime.now(UTC) - timedelta(minutes=1),backoff_attempt=3,backoff_until=datetime.now(UTC) + timedelta(minutes=5),+ pending_commit={"key": "day-2", "from_offset": 8999000, "to_offset": 8999445},state=state,)@@ -150,21 +156,27 @@ class TestBatchImportModel(BaseTest):self.assertEqual(batch_import.backoff_attempt, 0)self.assertIsNone(batch_import.backoff_until)self.assertIsNone(batch_import.display_status_message)+ self.assertIsNone(batch_import.pending_commit)def test_resume_after_pause_keeps_progress_and_clears_lease(self):batch_import = self._paused_import({"parts": self.MIXED_PROGRESS_PARTS})+ stale_copy = BatchImport.objects.get(id=batch_import.id)+ worker_progress = {+ "parts": [*self.MIXED_PROGRESS_PARTS[:2], {"key": "day-3", "current_offset": 77, "total_size": None}]+ }+ BatchImport.objects.filter(id=batch_import.id).update(state=worker_progress)- batch_import.resume_after_pause()+ stale_copy.resume_after_pause()batch_import.refresh_from_db()- assert batch_import.state is not None- self.assertEqual(batch_import.state["parts"], self.MIXED_PROGRESS_PARTS)+ self.assertEqual(batch_import.state, worker_progress)self.assertEqual(batch_import.status, BatchImport.Status.RUNNING)self.assertIsNone(batch_import.lease_id)self.assertIsNone(batch_import.leased_until)self.assertEqual(batch_import.backoff_attempt, 0)self.assertIsNone(batch_import.backoff_until)self.assertIsNone(batch_import.display_status_message)+ self.assertIsNone(batch_import.pending_commit)@parameterized.expand([@@ -187,6 +199,49 @@ class TestBatchImportModel(BaseTest):self.assertEqual(batch_import.state["parts"][1]["current_offset"], 8999445)self.assertEqual(batch_import.lease_id, "worker-lease")+ @parameterized.expand(+ [+ (f"{method}_{case}", method, row, refused)+ for method in ("resume_after_pause", "resume_with_inflight_part_reset")+ for case, row, refused in (+ ("marker_and_live_lease", {"leased_until": timedelta(minutes=5)}, True),+ (+ "legacy_commit_message_and_live_lease",+ {+ "leased_until": timedelta(minutes=5),+ "pending_commit": None,+ "status_message": "Starting commit of part day-2 to offset 9, consumed 9 additional bytes",+ },+ True,+ ),+ ("marker_and_expired_lease", {"leased_until": -timedelta(minutes=1)}, False),+ ("marker_and_no_lease", {"lease_id": None, "leased_until": None}, False),+ )+ ]+ )+ def test_resume_refuses_while_a_worker_sends_a_chunk(self, _name, method, row, refused):+ batch_import = self._paused_import({"parts": self.MIXED_PROGRESS_PARTS})+ if isinstance(row.get("leased_until"), timedelta):+ row = {**row, "leased_until": datetime.now(UTC) + row["leased_until"]}+ BatchImport.objects.filter(id=batch_import.id).update(**row)++ if refused:+ with self.assertRaises(CommitInFlightError):+ getattr(BatchImport.objects.get(id=batch_import.id), method)()+ else:+ getattr(BatchImport.objects.get(id=batch_import.id), method)()++ batch_import.refresh_from_db()+ if refused:+ assert batch_import.state is not None+ self.assertEqual(batch_import.status, BatchImport.Status.PAUSED)+ self.assertIsNotNone(batch_import.lease_id)+ self.assertEqual(batch_import.state["parts"][1]["current_offset"], 8999445)+ else:+ self.assertEqual(batch_import.status, BatchImport.Status.RUNNING)+ self.assertIsNone(batch_import.lease_id)+ self.assertIsNone(batch_import.pending_commit)+@parameterized.expand([("all_parts_done", {"parts": [{"key": "day-1", "current_offset": 1000, "total_size": 1000}]}),@@ -605,6 +660,201 @@ class TestBatchImportAdminActions(BaseTest):self.assertEqual(levels, ["success", "warning"])+class TestBatchImportAdminChangeForm(BaseTest):+ WORKER_STATE = {"parts": [{"key": "day-1", "current_offset": 9000, "total_size": None}]}++ def _form_values(self, batch_import: BatchImport) -> dict[str, str]:+ sink = batch_import.import_config["sink"]+ return {+ "team": str(batch_import.team_id),+ "created_by_id": str(batch_import.created_by_id),+ "status": batch_import.status,+ "status_message": batch_import.status_message or "",+ "sink_type": sink["type"],+ "send_rate": str(sink.get("send_rate", "")),+ }++ def _post_admin_form(+ self, batch_import: BatchImport, shown: dict[str, str], edits: dict[str, str], with_initial=True+ ):+ self.user.is_staff = True+ self.user.save()+ self.model_admin = admin.site._registry[BatchImport]+ self.request = RequestFactory().post("/")+ self.request.user = self.user+ # Django admin loads the row when the POST arrives, so it already holds any worker write.+ loaded_at_post = BatchImport.objects.get(id=batch_import.id)+ initial = {f"initial-{name}": value for name, value in shown.items()} if with_initial else {}+ return self.model_admin.get_form(self.request, loaded_at_post, change=True)(+ data={**initial, **shown, **edits}, instance=loaded_at_post+ )++ def _save(self, form) -> None:+ self.model_admin.save_model(self.request, form.save(commit=False), form, change=True)++ @parameterized.expand(+ [+ ("trial", {"type": "trial_s3", "record_limit": 100}, {}, {"type": "trial_s3", "record_limit": 100}),+ ("noop", {"type": "noop"}, {}, {"type": "noop"}),+ (+ "trial_rate_change_keeps_record_limit",+ {"type": "trial_s3", "record_limit": 100},+ {"send_rate": "600"},+ {"type": "trial_s3", "record_limit": 100, "send_rate": 600},+ ),+ ]+ )+ def test_save_changes_only_the_sink_keys_the_operator_edits(self, _name, sink, edits, expected_sink):+ batch_import = BatchImport.objects.create(+ team=self.team,+ created_by_id=self.user.id,+ import_config={"sink": sink},+ status=BatchImport.Status.PAUSED,+ )++ form = self._post_admin_form(batch_import, self._form_values(batch_import), {"status_message": "note", **edits})+ self.assertTrue(form.is_valid(), form.errors)+ self._save(form)++ batch_import.refresh_from_db()+ self.assertEqual(batch_import.import_config["sink"], expected_sink)+ self.assertEqual(batch_import.status_message, "note")++ def test_save_refuses_a_resume_while_a_worker_sends_a_chunk(self):+ batch_import = BatchImport.objects.create(+ team=self.team,+ created_by_id=self.user.id,+ import_config={"sink": {"type": "capture", "send_rate": 5000}},+ status=BatchImport.Status.PAUSED,+ lease_id="live-worker-lease",+ leased_until=datetime.now(UTC) + timedelta(minutes=5),+ pending_commit={"key": "day-1", "from_offset": 0, "to_offset": 10},+ )++ form = self._post_admin_form(batch_import, self._form_values(batch_import), {"status": "running"})++ self.assertFalse(form.is_valid())+ self.assertIn("A worker is sending a chunk", str(form.non_field_errors()))++ @parameterized.expand(+ [+ ("resumed_and_running", BatchImport.Status.RUNNING),+ ("completed", BatchImport.Status.COMPLETED),+ ("failed", BatchImport.Status.FAILED),+ ]+ )+ def test_save_refuses_a_resume_when_the_status_changed_since_page_load(self, _name, current_status):+ batch_import = BatchImport.objects.create(+ team=self.team,+ created_by_id=self.user.id,+ import_config={"sink": {"type": "capture", "send_rate": 5000}},+ status=BatchImport.Status.PAUSED,+ )+ shown = self._form_values(batch_import)+ BatchImport.objects.filter(id=batch_import.id).update(+ status=current_status,+ lease_id="live-worker-lease",+ leased_until=datetime.now(UTC) + timedelta(minutes=5),+ )++ form = self._post_admin_form(batch_import, shown, {"status": "running"})++ self.assertFalse(form.is_valid())+ self.assertIn("changed since the page loaded", str(form.non_field_errors()))++ @parameterized.expand(+ [+ (+ "send_started",+ {"pending_commit": {"key": "day-1", "from_offset": 0, "to_offset": 10}},+ "A worker is sending a chunk",+ ),+ ("resumed_elsewhere", {"status": BatchImport.Status.RUNNING}, "changed since the page loaded"),+ ]+ )+ def test_save_refuses_a_resume_when_the_row_changes_after_the_admin_loads_it(self, _name, change, expected_error):+ batch_import = BatchImport.objects.create(+ team=self.team,+ created_by_id=self.user.id,+ import_config={"sink": {"type": "capture", "send_rate": 5000}},+ status=BatchImport.Status.PAUSED,+ )+ form = self._post_admin_form(batch_import, self._form_values(batch_import), {"status": "running"})+ BatchImport.objects.filter(id=batch_import.id).update(+ lease_id="live-worker-lease", leased_until=datetime.now(UTC) + timedelta(minutes=5), **change+ )++ with CaptureQueriesContext(connection) as queries:+ self.assertFalse(form.is_valid())++ self.assertIn(expected_error, str(form.non_field_errors()))+ self.assertTrue(+ any("FOR UPDATE" in q["sql"] and "posthog_batchimport" in q["sql"] for q in queries.captured_queries)+ )++ def test_save_rejects_a_page_without_initial_values(self):+ batch_import = BatchImport.objects.create(+ team=self.team,+ created_by_id=self.user.id,+ import_config={"sink": {"type": "capture", "send_rate": 5000}},+ status=BatchImport.Status.RUNNING,+ lease_id="live-worker-lease",+ )++ form = self._post_admin_form(+ batch_import, self._form_values(batch_import), {"send_rate": "500"}, with_initial=False+ )++ self.assertFalse(form.is_valid())+ self.assertIn("This page is out of date", str(form.non_field_errors()))++ @parameterized.expand(+ [+ ("rate_change_keeps_worker_status", "running", "paused", {"send_rate": "500"}, "paused", 500),+ ("operator_pause", "running", "running", {"status": "paused"}, "paused", 5000),+ ("operator_resume", "paused", "paused", {"status": "running"}, "running", 5000),+ ]+ )+ def test_save_writes_only_operator_edits(+ self, _name, shown_status, worker_status, edits, expected_status, expected_send_rate+ ):+ batch_import = BatchImport.objects.create(+ team=self.team,+ created_by_id=self.user.id,+ import_config={"sink": {"type": "capture", "send_rate": 5000}},+ secrets={"secret": "value"},+ status=shown_status,+ lease_id="lease-at-page-load",+ leased_until=datetime.now(UTC) - timedelta(minutes=1),+ backoff_attempt=2,+ pending_commit={"key": "day-1", "from_offset": 8000, "to_offset": 9000},+ state={"parts": [{"key": "day-1", "current_offset": 100, "total_size": None}]},+ )+ shown = self._form_values(batch_import)+ BatchImport.objects.filter(id=batch_import.id).update(+ status=worker_status, state=self.WORKER_STATE, lease_id="lease-after-worker-commit"+ )++ form = self._post_admin_form(batch_import, shown, edits)+ self.assertTrue(form.is_valid(), form.errors)+ self._save(form)++ batch_import.refresh_from_db()+ self.assertEqual(batch_import.status, expected_status)+ self.assertEqual(batch_import.import_config["sink"]["send_rate"], expected_send_rate)+ self.assertEqual(batch_import.state, self.WORKER_STATE)+ if expected_status == "running":+ self.assertEqual(batch_import.status_message, "Resumed by admin")+ self.assertIsNone(batch_import.lease_id)+ self.assertIsNone(batch_import.leased_until)+ self.assertEqual(batch_import.backoff_attempt, 0)+ self.assertIsNone(batch_import.pending_commit)+ else:+ self.assertEqual(batch_import.lease_id, "lease-after-worker-commit")+ self.assertEqual(batch_import.backoff_attempt, 2)+ self.assertIsNotNone(batch_import.pending_commit)++class TestBatchImportAPI(APIBaseTest):def test_model_creation_only(self):batch_import = BatchImport.objects.create(@@ -1062,6 +1312,25 @@ class TestBatchImportAPI(APIBaseTest):self.assertEqual(results[0]["id"], str(batch_import.id))self.assertEqual(results[0]["display_status"], expected_display_status)+ def test_resume_refuses_while_a_worker_sends_a_chunk(self):+ batch_import = BatchImport.objects.create(+ team=self.team,+ created_by_id=self.user.id,+ import_config={"source": {"type": "s3"}},+ status=BatchImport.Status.PAUSED,+ lease_id="live-worker-lease",+ leased_until=datetime.now(tz=UTC) + timedelta(minutes=5),+ pending_commit={"key": "part-1", "from_offset": 0, "to_offset": 100},+ )++ response = self.client.post(f"/api/projects/{self.team.id}/managed_migrations/{batch_import.id}/resume")++ self.assertEqual(response.status_code, 409)+ batch_import.refresh_from_db()+ self.assertEqual(batch_import.status, BatchImport.Status.PAUSED)+ self.assertEqual(batch_import.lease_id, "live-worker-lease")+ self.assertIsNotNone(batch_import.pending_commit)+def test_resume_clears_lease_and_backoff(self):batch_import = BatchImport.objects.create(team=self.team,@@ -1070,9 +1339,11 @@ class TestBatchImportAPI(APIBaseTest):secrets={"access_key": "test"},status=BatchImport.Status.PAUSED,lease_id="old-lease-uuid",- leased_until=datetime.now(tz=UTC) + timedelta(hours=1),+ leased_until=datetime.now(tz=UTC) - timedelta(minutes=1),backoff_attempt=5,backoff_until=datetime.now(tz=UTC) + timedelta(hours=1),+ display_status_message="Job paused after a failed commit",+ pending_commit={"key": "part-1", "from_offset": 0, "to_offset": 100},)response = self.client.post(f"/api/projects/{self.team.id}/managed_migrations/{batch_import.id}/resume")@@ -1085,6 +1356,15 @@ class TestBatchImportAPI(APIBaseTest):self.assertEqual(batch_import.backoff_attempt, 0)self.assertIsNone(batch_import.backoff_until)self.assertEqual(batch_import.status_message, "Resumed by user")+ self.assertIsNone(batch_import.display_status_message)+ self.assertIsNone(batch_import.pending_commit)+ logged_fields = [+ change["field"]+ for log in ActivityLog.objects.filter(scope="BatchImport", item_id=str(batch_import.id))+ for change in (log.detail or {}).get("changes") or []+ ]+ self.assertIn("status", logged_fields)+ self.assertNotIn("pending_commit", logged_fields)@parameterized.expand([diff --git a/products/managed_migrations/backend/migrations/0004_batchimport_pending_commit.py b/products/managed_migrations/backend/migrations/0004_batchimport_pending_commit.pynew file mode 100644index 000000000000..90e1111a5a95--- /dev/null+++ b/products/managed_migrations/backend/migrations/0004_batchimport_pending_commit.py@@ -0,0 +1,17 @@+# Generated by Django 5.2.18 on 2026-10-16 07:20++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("managed_migrations", "0003_alter_batchimport_team"),+ ]++ operations = [+ migrations.AddField(+ model_name="batchimport",+ name="pending_commit",+ field=models.JSONField(blank=True, null=True),+ ),+ ]diff --git a/products/managed_migrations/backend/migrations/max_migration.txt b/products/managed_migrations/backend/migrations/max_migration.txtindex 0e41e10acac8..26518050ee84 100644--- a/products/managed_migrations/backend/migrations/max_migration.txt+++ b/products/managed_migrations/backend/migrations/max_migration.txt@@ -1 +1 @@-0003_alter_batchimport_team+0004_batchimport_pending_commitdiff --git a/products/managed_migrations/backend/models/batch_imports.py b/products/managed_migrations/backend/models/batch_imports.pyindex 8acc298df552..11db6f563624 100644--- a/products/managed_migrations/backend/models/batch_imports.py+++ b/products/managed_migrations/backend/models/batch_imports.py@@ -2,7 +2,8 @@ from enum import Enumfrom typing import TYPE_CHECKING, Selffrom django.conf import settings-from django.db import models+from django.db import models, transaction+from django.utils import timezonefrom posthog.helpers.encrypted_fields import EncryptedJSONStringFieldfrom posthog.models.activity_logging.model_activity import ModelActivityMixin@@ -16,6 +17,16 @@ if TYPE_CHECKING:DEFAULT_SEND_RATE = 5_000+# The worker before pending_commit wrote this status message while it sent a chunk.+LEGACY_COMMIT_IN_FLIGHT_MESSAGE_PREFIX = "Starting commit of part "+++class CommitInFlightError(ValueError):+ """A worker is sending a chunk of this import, so a resume now could skip that chunk."""++ def __init__(self) -> None:+ super().__init__("A worker is sending a chunk of this import. Try again in a few minutes.")+def get_aws_external_id(team: "Team") -> str:region = (settings.CLOUD_DEPLOYMENT or "dev").lower()@@ -61,6 +72,9 @@ class BatchImport(ModelActivityMixin, UUIDTModel):# Exponential backoff state (used by rust worker). Mirrors columns used by the worker.backoff_attempt = models.IntegerField(default=0)backoff_until = models.DateTimeField(null=True, blank=True)+ # Worker-owned marker for the chunk being sent. A value on a row no worker holds means a worker+ # stopped mid-commit, so the chunk may not have reached the sink.+ pending_commit = models.JSONField(null=True, blank=True)class Meta:db_table = "posthog_batchimport"@@ -116,9 +130,18 @@ class BatchImport(ModelActivityMixin, UUIDTModel):offset, use `resume_with_inflight_part_reset` instead. Raises ValueError ifthe job is not paused."""- if self.status != BatchImport.Status.PAUSED:- raise ValueError(f"Only paused imports can be resumed (status: {self.status})")- self._flip_to_running("Resumed by admin")+ self.resume_from_pause("Resumed by admin")++ def resume_from_pause(self, status_message: str) -> None:+ """Resume this paused import under a row lock. Raises ValueError if it is not paused, or+ CommitInFlightError while a worker still sends a chunk."""+ with transaction.atomic():+ row = BatchImport.objects.select_for_update().get(pk=self.pk)+ if row.status != BatchImport.Status.PAUSED:+ raise ValueError(f"Only paused imports can be resumed (status: {row.status})")+ row.raise_if_commit_in_flight()+ row.save(update_fields=row.reset_for_resume(status_message))+ self.refresh_from_db()def resume_with_inflight_part_reset(self) -> str:"""Resume a paused import whose in-flight part has a poisoned byte offset.@@ -135,25 +158,51 @@ class BatchImport(ModelActivityMixin, UUIDTModel):`resume_after_pause` does. Returns the key of the reset part. RaisesValueError if the job is not paused or has no unfinished part."""- if self.status != BatchImport.Status.PAUSED:- raise ValueError(f"Only paused imports can be reset and resumed (status: {self.status})")-- _done, _total, inflight = self.parts_progress()- if inflight is None:- raise ValueError("No unfinished part to reset - the job has no resumable work")-- inflight["current_offset"] = 0- # A stale total from a previous download would be wrong too: each download of- # a nondeterministic export can have a different decompressed size.- inflight["total_size"] = None-- # Redacted: url_list part keys are full source URLs whose query string /- # userinfo can carry credentials, and status_message is broadly visible.- redacted_key = str(redact_part_key(inflight.get("key")))- self._flip_to_running(f"Resumed by admin with part {redacted_key} reset to offset 0")+ with transaction.atomic():+ row = BatchImport.objects.select_for_update().get(pk=self.pk)+ if row.status != BatchImport.Status.PAUSED:+ raise ValueError(f"Only paused imports can be reset and resumed (status: {row.status})")+ row.raise_if_commit_in_flight()++ _done, _total, inflight = row.parts_progress()+ if inflight is None:+ raise ValueError("No unfinished part to reset - the job has no resumable work")++ inflight["current_offset"] = 0+ # A stale total from a previous download would be wrong too: each download of+ # a nondeterministic export can have a different decompressed size.+ inflight["total_size"] = None++ # Redacted: url_list part keys are full source URLs whose query string /+ # userinfo can carry credentials, and status_message is broadly visible.+ redacted_key = str(redact_part_key(inflight.get("key")))+ row.save(+ update_fields=[+ *row.reset_for_resume(f"Resumed by admin with part {redacted_key} reset to offset 0"),+ "state",+ ]+ )+ self.refresh_from_db()return redacted_key- def _flip_to_running(self, status_message: str) -> None:+ def commit_in_flight(self) -> bool:+ """True while a worker holds a live lease and is between saving a chunk's offset and the+ sink accepting it, the window in which clearing its lease can lose that chunk."""+ if self.leased_until is None or self.leased_until <= timezone.now():+ return False+ return self.pending_commit is not None or (self.status_message or "").startswith(+ LEGACY_COMMIT_IN_FLIGHT_MESSAGE_PREFIX+ )++ def raise_if_commit_in_flight(self) -> None:+ if self.commit_in_flight():+ raise CommitInFlightError()++ def reset_for_resume(self, status_message: str) -> list[str]:+ """Set the fields every resume path resets, and return their names for `save(update_fields=...)`.++ Clears the lease so a worker can claim the row at once, and `pending_commit` because a resume+ accepts the saved offsets."""self.status = BatchImport.Status.RUNNINGself.status_message = status_messageself.display_status_message = None@@ -161,19 +210,18 @@ class BatchImport(ModelActivityMixin, UUIDTModel):self.leased_until = Noneself.backoff_attempt = 0self.backoff_until = None- self.save(- update_fields=[- "state",- "status",- "status_message",- "display_status_message",- "lease_id",- "leased_until",- "backoff_attempt",- "backoff_until",- "updated_at",- ]- )+ self.pending_commit = None+ return [+ "status",+ "status_message",+ "display_status_message",+ "lease_id",+ "leased_until",+ "backoff_attempt",+ "backoff_until",+ "pending_commit",+ "updated_at",+ ]# Mostly used for manual job creation